detect/engine: set max sig ID per SGH

Present scenario
----------------
Currently, as a part of setting signature count per SGH, a max_idx is
passed which could be as high as the highest signature number (internal
ID).

Issue
-----
Not every SGH needs to evaluate all the signatures while setting
the signature count or while creating the match_array.
In a nonideal scenario, when say, there are 2 SGHs and one SGH has 2
signatures and the other one has 60k, given the current scheme of
evaluating max_idx, the max_idx will be set to 60k, and this shall
later be passed on to SigGroupHeadSetSigCnt or
SigGroupHeadBuildMatchArra which shall traverse over all the 60k sigs
for either SGHs.

Other info
----------
This is a very fast operation as the internal arithmetic is done
bitwise.

Patch
-----
The functions SigGroupHeadSetSigCnt and SigGroupHeadBuildMatchArray can
be optimized by storing the max signature id (internal) per SGH (which
also seemed to be the initial intention as per fn comments).
As a result of this, the sig_array is only walked up until the max sig
id of that respective SGH.

src/detect-engine-build.c

@@ -1009,8 +1009,8 @@ static int RulesGroupByProto(DetectEngineCtx *de_ctx)
         if (lookup_sgh == NULL) {
             SCLogDebug("proto group %d sgh %p is the original", p, sgh_ts[p]);
 
-            SigGroupHeadSetSigCnt(sgh_ts[p], max_idx);
+            SigGroupHeadSetSigCnt(sgh_ts[p], 0);
-            SigGroupHeadBuildMatchArray(de_ctx, sgh_ts[p], max_idx);
+            SigGroupHeadBuildMatchArray(de_ctx, sgh_ts[p], 0);
 
             SigGroupHeadHashAdd(de_ctx, sgh_ts[p]);
             SigGroupHeadStore(de_ctx, sgh_ts[p]);
@@ -1041,8 +1041,8 @@ static int RulesGroupByProto(DetectEngineCtx *de_ctx)
         if (lookup_sgh == NULL) {
             SCLogDebug("proto group %d sgh %p is the original", p, sgh_tc[p]);
 
-            SigGroupHeadSetSigCnt(sgh_tc[p], max_idx);
+            SigGroupHeadSetSigCnt(sgh_tc[p], 0);
-            SigGroupHeadBuildMatchArray(de_ctx, sgh_tc[p], max_idx);
+            SigGroupHeadBuildMatchArray(de_ctx, sgh_tc[p], 0);
 
             SigGroupHeadHashAdd(de_ctx, sgh_tc[p]);
             SigGroupHeadStore(de_ctx, sgh_tc[p]);
@@ -1129,7 +1129,8 @@ static int RuleSetWhitelist(Signature *s)
     return wl;
 }
 
-int CreateGroupedPortList(DetectEngineCtx *de_ctx, DetectPort *port_list, DetectPort **newhead, uint32_t unique_groups, int (*CompareFunc)(DetectPort *, DetectPort *), uint32_t max_idx);
+int CreateGroupedPortList(DetectEngineCtx *de_ctx, DetectPort *port_list, DetectPort **newhead,
+        uint32_t unique_groups, int (*CompareFunc)(DetectPort *, DetectPort *));
 int CreateGroupedPortListCmpCnt(DetectPort *a, DetectPort *b);
 
 static DetectPort *RulesGroupByPorts(DetectEngineCtx *de_ctx, uint8_t ipproto, uint32_t direction)
@@ -1223,7 +1224,7 @@ static DetectPort *RulesGroupByPorts(DetectEngineCtx *de_ctx, uint8_t ipproto, u
     DetectPort *newlist = NULL;
     uint16_t groupmax = (direction == SIG_FLAG_TOCLIENT) ? de_ctx->max_uniq_toclient_groups :
                                                            de_ctx->max_uniq_toserver_groups;
-    CreateGroupedPortList(de_ctx, list, &newlist, groupmax, CreateGroupedPortListCmpCnt, max_idx);
+    CreateGroupedPortList(de_ctx, list, &newlist, groupmax, CreateGroupedPortListCmpCnt);
     list = newlist;
 
     /* step 4: deduplicate the SGH's */
@@ -1243,8 +1244,8 @@ static DetectPort *RulesGroupByPorts(DetectEngineCtx *de_ctx, uint8_t ipproto, u
         if (lookup_sgh == NULL) {
             SCLogDebug("port group %p sgh %p is the original", iter, iter->sh);
 
-            SigGroupHeadSetSigCnt(iter->sh, max_idx);
+            SigGroupHeadSetSigCnt(iter->sh, 0);
-            SigGroupHeadBuildMatchArray(de_ctx, iter->sh, max_idx);
+            SigGroupHeadBuildMatchArray(de_ctx, iter->sh, 0);
             SigGroupHeadSetProtoAndDirection(iter->sh, ipproto, direction);
             SigGroupHeadHashAdd(de_ctx, iter->sh);
             SigGroupHeadStore(de_ctx, iter->sh);
@@ -1541,7 +1542,8 @@ int CreateGroupedPortListCmpCnt(DetectPort *a, DetectPort *b)
  *  The joingr is meant to be a catch all.
  *
  */
-int CreateGroupedPortList(DetectEngineCtx *de_ctx, DetectPort *port_list, DetectPort **newhead, uint32_t unique_groups, int (*CompareFunc)(DetectPort *, DetectPort *), uint32_t max_idx)
+int CreateGroupedPortList(DetectEngineCtx *de_ctx, DetectPort *port_list, DetectPort **newhead,
+        uint32_t unique_groups, int (*CompareFunc)(DetectPort *, DetectPort *))
 {
     DetectPort *tmplist = NULL, *joingr = NULL;
     char insert = 0;
@@ -1560,8 +1562,7 @@ int CreateGroupedPortList(DetectEngineCtx *de_ctx, DetectPort *port_list, Detect
         list->next = NULL;
 
         groups++;
-
+        SigGroupHeadSetSigCnt(list->sh, 0);
-        SigGroupHeadSetSigCnt(list->sh, max_idx);
 
         /* insert it */
         DetectPort *tmpgr = tmplist, *prevtmpgr = NULL;

src/detect-engine-siggroup.c

@@ -347,7 +347,7 @@ int SigGroupHeadAppendSig(const DetectEngineCtx *de_ctx, SigGroupHead **sgh,
 
     /* enable the sig in the bitarray */
     (*sgh)->init->sig_array[s->num / 8] |= 1 << (s->num % 8);
-
+    (*sgh)->init->max_sig_id = MAX(s->num, (*sgh)->init->max_sig_id);
     return 0;
 
 error:
@@ -405,6 +405,8 @@ int SigGroupHeadCopySigs(DetectEngineCtx *de_ctx, SigGroupHead *src, SigGroupHea
     if (src->init->score)
         (*dst)->init->score = MAX((*dst)->init->score, src->init->score);
 
+    if (src->init->max_sig_id)
+        (*dst)->init->max_sig_id = MAX((*dst)->init->max_sig_id, src->init->max_sig_id);
     return 0;
 
 error:
@@ -422,9 +424,9 @@ error:
 void SigGroupHeadSetSigCnt(SigGroupHead *sgh, uint32_t max_idx)
 {
     uint32_t sig;
-
+    sgh->init->max_sig_id = MAX(max_idx, sgh->init->max_sig_id);
     sgh->init->sig_cnt = 0;
-    for (sig = 0; sig < max_idx + 1; sig++) {
+    for (sig = 0; sig < sgh->init->max_sig_id + 1; sig++) {
         if (sgh->init->sig_array[sig / 8] & (1 << (sig % 8)))
             sgh->init->sig_cnt++;
     }
@@ -492,12 +494,13 @@ int SigGroupHeadBuildMatchArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh,
         return 0;
 
     BUG_ON(sgh->init->match_array != NULL);
+    sgh->init->max_sig_id = MAX(sgh->init->max_sig_id, max_idx);
 
     sgh->init->match_array = SCCalloc(sgh->init->sig_cnt, sizeof(Signature *));
     if (sgh->init->match_array == NULL)
         return -1;
 
-    for (sig = 0; sig < max_idx + 1; sig++) {
+    for (sig = 0; sig < sgh->init->max_sig_id + 1; sig++) {
         if (!(sgh->init->sig_array[(sig / 8)] & (1 << (sig % 8))) )
             continue;
 

src/detect.h

@@ -1419,6 +1419,7 @@ typedef struct SigGroupHeadInitData_ {
     uint8_t protos[256];    /**< proto(s) this sgh is for */
     uint32_t direction;     /**< set to SIG_FLAG_TOSERVER, SIG_FLAG_TOCLIENT or both */
     int score;              /**< try to make this group a unique one */
+    uint32_t max_sig_id;    /**< max signature idx for this sgh */
 
     MpmCtx **app_mpms;
     MpmCtx **pkt_mpms;