Share content id's between identical patterns.

libhtp/INSTALL

@@ -2,7 +2,7 @@ Installation Instructions
 *************************
 
 Copyright (C) 1994, 1995, 1996, 1999, 2000, 2001, 2002, 2004, 2005,
-2006, 2007, 2008 Free Software Foundation, Inc.
+2006, 2007, 2008, 2009 Free Software Foundation, Inc.
 
    This file is free documentation; the Free Software Foundation gives
 unlimited permission to copy, distribute and modify it.
@@ -159,7 +159,7 @@ Particular systems
 CC is not installed, it is recommended to use the following options in
 order to use an ANSI C compiler:
 
-     ./configure CC="cc -Ae"
+     ./configure CC="cc -Ae -D_XOPEN_SOURCE=500"
 
 and if that doesn't work, install pre-built binaries of GCC for HP-UX.
 
@@ -174,6 +174,16 @@ and if that doesn't work, try
 
      ./configure CC="cc -nodtk"
 
+   On Solaris, don't put `/usr/ucb' early in your `PATH'.  This
+directory contains several dysfunctional programs; working variants of
+these programs are available in `/usr/bin'.  So, if you need `/usr/ucb'
+in your `PATH', put it _after_ `/usr/bin'.
+
+   On Haiku, software installed for all users goes in `/boot/common',
+not `/usr/local'.  It is recommended to use the following options:
+
+     ./configure --prefix=/boot/common
+
 Specifying the System Type
 ==========================
 
@@ -189,7 +199,8 @@ type, such as `sun4', or a canonical name which has the form:
 
 where SYSTEM can have one of these forms:
 
-     OS KERNEL-OS
+     OS
+     KERNEL-OS
 
    See the file `config.sub' for the possible values of each field.  If
 `config.sub' isn't included in this package, then this package doesn't

src/detect-content.c

@@ -383,11 +383,11 @@ DoDetectContent(ThreadVars *t, DetectEngineThreadCtx *det_ctx, Packet *p, Signat
     uint16_t pkt_off = det_ctx->pkt_off;
     MpmMatch *temp_m = NULL;
 
+    SCLogDebug("det_ctx->mtc.match[%"PRIu32"].len %"PRIu32"", co->id, det_ctx->mtc.match[co->id].len);
+
     /* Get the top match, we already know we have one. */
     MpmMatch *m = det_ctx->mtc.match[co->id].top;
 
-    SCLogDebug("det_ctx->mtc.match[co->id].len %"PRIu32"", det_ctx->mtc.match[co->id].len);
-
     /* reset de_checking_distancewithin */
     if (!(co->flags & DETECT_CONTENT_WITHIN) &&
         !(co->flags & DETECT_CONTENT_DISTANCE))
@@ -1469,8 +1469,9 @@ int DetectContentSetup (DetectEngineCtx *de_ctx, Signature *s, SigMatch *m, char
             SigMatchAppend(s,m,sm);
             m = sm;
 
-            aux->id = de_ctx->content_max_id;
+            aux->id = DetectContentGetId(de_ctx, aux);
-            de_ctx->content_max_id++;
+            //aux->id = de_ctx->content_max_id;
+            //de_ctx->content_max_id++;
 
             /** We need to setup the modifiers for the chunks respect
               * the last chunk installed inmediatelly before
@@ -1509,8 +1510,9 @@ int DetectContentSetup (DetectEngineCtx *de_ctx, Signature *s, SigMatch *m, char
             cd->chunk_group_id = s->nchunk_groups;
         }
 
-        cd->id = de_ctx->content_max_id;
+        cd->id = DetectContentGetId(de_ctx, cd);
-        de_ctx->content_max_id++;
+        //cd->id = de_ctx->content_max_id;
+        //de_ctx->content_max_id++;
 
         DetectContentPrint(cd);
     }
@@ -1540,6 +1542,117 @@ void DetectContentFree(void *ptr) {
     SCFree(cd);
 }
 
+/* content hash
+ * A per detection engine hash to make sure each pattern has a unique global id
+ * but pattern that are the same share id's.
+ */
+
+typedef struct DetectContentTableElmt_ {
+    uint8_t *pattern;       /**< ptr to the pattern */
+    uint16_t pattern_len;   /**< pattern len */
+    uint32_t id;            /**< pattern id */
+} DetectContentTableElmt;
+
+static char DetectContentTableCompare(void *p1, uint16_t len1, void *p2, uint16_t len2) {
+    SCEnter();
+    BUG_ON(len1 < sizeof(DetectContentTableElmt));
+    BUG_ON(len2 < sizeof(DetectContentTableElmt));
+
+    DetectContentTableElmt *e1 = (DetectContentTableElmt *)p1;
+    DetectContentTableElmt *e2 = (DetectContentTableElmt *)p2;
+
+    if (e1->pattern_len != e2->pattern_len) {
+        SCReturnInt(0);
+    }
+
+    if (memcmp(e1->pattern, e2->pattern, e1->pattern_len) != 0) {
+        SCReturnInt(0);
+    }
+
+    SCReturnInt(1);
+}
+
+static uint32_t DetectContentTableHash(HashTable *ht, void *p, uint16_t len) {
+    SCEnter();
+    BUG_ON(len < sizeof(DetectContentTableElmt));
+
+    DetectContentTableElmt *e = (DetectContentTableElmt *)p;
+    uint32_t hash = e->pattern_len;
+    uint16_t u = 0;
+ 
+    for (u = 0; u < e->pattern_len; u++) {
+        hash += e->pattern[u];
+    }
+
+    SCReturnUInt(hash % ht->array_size);
+}
+
+static void DetectContentTableElmtFree(void *e) {
+    free(e);
+}
+
+int DetectContentTableInitHash(DetectEngineCtx *de_ctx) {
+    SCEnter();
+
+    BUG_ON(de_ctx == NULL);
+
+    de_ctx->content_hash = HashTableInit(65536, DetectContentTableHash, DetectContentTableCompare, DetectContentTableElmtFree);
+
+    BUG_ON(de_ctx->content_hash == NULL);
+
+    SCReturnInt(0);
+}
+
+void DetectContentTableFreeHash(DetectEngineCtx *de_ctx) {
+   SCEnter();
+
+    if (de_ctx == NULL || de_ctx->content_hash == NULL) {
+        SCReturn;
+    }
+
+    HashTableFree(de_ctx->content_hash);
+    SCReturn;
+}
+
+uint32_t DetectContentGetId(DetectEngineCtx *de_ctx, DetectContentData *co) {
+    SCEnter();
+
+    BUG_ON(de_ctx == NULL || de_ctx->content_hash == NULL);
+
+    DetectContentTableElmt *e = NULL;
+    DetectContentTableElmt *r = NULL;
+    uint32_t id = 0;
+
+    e = malloc(sizeof(DetectContentTableElmt));
+    BUG_ON(e == NULL);
+    e->pattern = co->content;
+    e->pattern_len = co->content_len;
+    e->id = 0;
+
+    r = HashTableLookup(de_ctx->content_hash, (void *)e, sizeof(DetectContentTableElmt));
+    if (r == NULL) {
+        e->id = de_ctx->content_max_id;
+        de_ctx->content_max_id++;
+        id = e->id;
+
+        int ret = HashTableAdd(de_ctx->content_hash, e, sizeof(DetectContentTableElmt));
+        BUG_ON(ret != 0);
+
+        e = NULL;
+
+        de_ctx->content_hash_unique++;
+    } else {
+        id = r->id;
+
+        de_ctx->content_hash_shared++;
+    }
+
+    if (e != NULL)
+        free(e);
+
+    SCReturnUInt(id);
+}
+
 #ifdef UNITTESTS /* UNITTESTS */
 
 /**
@@ -2846,7 +2959,7 @@ static int SigTest42TestNegatedContent(void)
  *       anyways, and we don't do a check on the offset
  */
 static int SigTest43TestNegatedContent(void)
-{
+{                                                                                                                                                         // 12345123451234512345123
     return SigTestNegativeTestContent("alert tcp any any -> any any (msg:\"HTTP URI cap\"; content:!twentythree; depth:15; offset:22; sid:1;)",  (uint8_t *)"one four nine fourteen twentythree thirtyfive fourtysix fiftysix");
 }
 

src/detect-content.h

@@ -15,6 +15,15 @@
 /** Set if the pattern is split into multiple chunks */
 #define DETECT_CONTENT_IS_CHUNK          0x0100
 
+#define DETECT_CONTENT_IS_SINGLE(c) (!((c)->flags & DETECT_CONTENT_DISTANCE || \
+                                       (c)->flags & DETECT_CONTENT_WITHIN || \
+                                       (c)->flags & DETECT_CONTENT_DISTANCE_NEXT || \
+                                       (c)->flags & DETECT_CONTENT_WITHIN_NEXT || \
+                                       (c)->flags & DETECT_CONTENT_ISDATAAT_RELATIVE || \
+                                       (c)->flags & DETECT_CONTENT_IS_CHUNK || \
+                                       (c)->depth > 0 || \
+                                       (c)->within > 0))
+
 /** Used for modifier propagations, to know if they are
  * yet updated or not */
 #define CHUNK_UPDATED_DEPTH         0x01
@@ -84,4 +93,8 @@ int DetectContentPropagateModifiers(SigMatch *);
 
 void DetectContentFree(void *);
 
+int DetectContentTableInitHash(DetectEngineCtx *);
+void DetectContentTableFreeHash(DetectEngineCtx *);
+uint32_t DetectContentGetId(DetectEngineCtx *, DetectContentData *);
+
 #endif /* __DETECT_CONTENT_H__ */

src/detect-engine.c

@@ -57,6 +57,9 @@ DetectEngineCtx *DetectEngineCtxInit(void) {
     DetectPortDpHashInit(de_ctx);
     ThresholdHashInit(de_ctx);
     VariableNameInitHash(de_ctx);
+
+    DetectContentTableInitHash(de_ctx);
+
     return de_ctx;
 error:
     return NULL;
@@ -67,9 +70,11 @@ void DetectEngineCtxFree(DetectEngineCtx *de_ctx) {
     if (de_ctx == NULL)
         return;
 
+
     /* Normally the hashes are freed elsewhere, but
      * to be sure look at them again here.
      */
+    DetectContentTableFreeHash(de_ctx); /* normally cleaned up in SigGroupBuild */
     SigGroupHeadHashFree(de_ctx);
     SigGroupHeadMpmHashFree(de_ctx);
     SigGroupHeadMpmUriHashFree(de_ctx);

src/detect.h

@@ -323,6 +323,10 @@ typedef struct DetectEngineCtx_ {
     uint16_t max_uniq_small_toserver_dst_groups;
     uint16_t max_uniq_small_toserver_sp_groups;
     uint16_t max_uniq_small_toserver_dp_groups;
+
+    HashTable *content_hash;
+    uint32_t content_hash_unique;
+    uint32_t content_hash_shared;
 } DetectEngineCtx;
 
 /* Engine groups profiles (low, medium, high, custom) */