detect/iponly: Reduce the size of the SigNumArray bitsets

Instead of tracking ip only rules by the internal signum, track them by
a separate counter that starts at zero.  This results in dense
SigNumArrays instead of sparse ones and a much smaller max_idx.

Issue: 4578

src/detect-engine-iponly.c

@@ -875,14 +875,24 @@ error:
  */
 void IPOnlyInit(DetectEngineCtx *de_ctx, DetectEngineIPOnlyCtx *io_ctx)
 {
-    io_ctx->tree_ipv4src = SCRadixCreateRadixTree(SigNumArrayFree,
-                                                  SigNumArrayPrint);
-    io_ctx->tree_ipv4dst = SCRadixCreateRadixTree(SigNumArrayFree,
-                                                  SigNumArrayPrint);
-    io_ctx->tree_ipv6src = SCRadixCreateRadixTree(SigNumArrayFree,
-                                                  SigNumArrayPrint);
-    io_ctx->tree_ipv6dst = SCRadixCreateRadixTree(SigNumArrayFree,
-                                                  SigNumArrayPrint);
+    io_ctx->tree_ipv4src = SCRadixCreateRadixTree(SigNumArrayFree, SigNumArrayPrint);
+    io_ctx->tree_ipv4dst = SCRadixCreateRadixTree(SigNumArrayFree, SigNumArrayPrint);
+    io_ctx->tree_ipv6src = SCRadixCreateRadixTree(SigNumArrayFree, SigNumArrayPrint);
+    io_ctx->tree_ipv6dst = SCRadixCreateRadixTree(SigNumArrayFree, SigNumArrayPrint);
+
+    io_ctx->sig_mapping = SCCalloc(1, de_ctx->sig_array_len * sizeof(uint32_t));
+    if (io_ctx->sig_mapping == NULL) {
+        FatalError("Unable to allocate iponly signature tracking area");
+    }
+    io_ctx->sig_mapping_size = 0;
+}
+
+SigIntId IPOnlyTrackSigNum(DetectEngineIPOnlyCtx *io_ctx, SigIntId signum)
+{
+    SigIntId loc = io_ctx->sig_mapping_size;
+    io_ctx->sig_mapping[loc] = signum;
+    io_ctx->sig_mapping_size++;
+    return loc;
 }
 
 /**
@@ -942,6 +952,10 @@ void IPOnlyDeinit(DetectEngineCtx *de_ctx, DetectEngineIPOnlyCtx *io_ctx)
     if (io_ctx->tree_ipv6dst != NULL)
         SCRadixReleaseRadixTree(io_ctx->tree_ipv6dst);
     io_ctx->tree_ipv6dst = NULL;
+
+    if (io_ctx->sig_mapping != NULL)
+        SCFree(io_ctx->sig_mapping);
+    io_ctx->sig_mapping = NULL;
 }
 
 /**
@@ -1026,20 +1040,18 @@ void IPOnlyMatchPacket(ThreadVars *tv,
     for (u = 0; u < src->size; u++) {
         SCLogDebug("And %"PRIu8" & %"PRIu8, src->array[u], dst->array[u]);
 
-        /* The final results will be at io_tctx */
-        io_tctx->sig_match_array[u] = dst->array[u] & src->array[u];
+        uint8_t bitarray = dst->array[u] & src->array[u];
 
         /* We have to move the logic of the signature checking
          * to the main detect loop, in order to apply the
          * priority of actions (pass, drop, reject, alert) */
-        if (io_tctx->sig_match_array[u] != 0) {
+        if (bitarray) {
             /* We have a match :) Let's see from which signum's */
-            uint8_t bitarray = io_tctx->sig_match_array[u];
             uint8_t i = 0;
 
             for (; i < 8; i++, bitarray = bitarray >> 1) {
                 if (bitarray & 0x01) {
-                    Signature *s = de_ctx->sig_array[u * 8 + i];
+                    Signature *s = de_ctx->sig_array[io_ctx->sig_mapping[u * 8 + i]];
 
                     if ((s->proto.flags & DETECT_PROTO_IPV4) && !PKT_IS_IPV4(p)) {
                         SCLogDebug("ip version didn't match");
@@ -1548,10 +1560,13 @@ void IPOnlyAddSignature(DetectEngineCtx *de_ctx, DetectEngineIPOnlyCtx *io_ctx,
     if (!(s->flags & SIG_FLAG_IPONLY))
         return;
 
+    SigIntId mapped_signum = IPOnlyTrackSigNum(io_ctx, s->num);
+    SCLogDebug("Adding IPs from rule: %" PRIu32 " (%s) as %" PRIu32 " mapped to %" PRIu32 "\n",
+            s->id, s->msg, s->num, mapped_signum);
     /* Set the internal signum to the list before merging */
-    IPOnlyCIDRListSetSigNum(s->cidr_src, s->num);
+    IPOnlyCIDRListSetSigNum(s->cidr_src, mapped_signum);
 
-    IPOnlyCIDRListSetSigNum(s->cidr_dst, s->num);
+    IPOnlyCIDRListSetSigNum(s->cidr_dst, mapped_signum);
 
     /**
      * ipv4 and ipv6 are mixed, but later we will separate them into
@@ -1560,8 +1575,8 @@ void IPOnlyAddSignature(DetectEngineCtx *de_ctx, DetectEngineIPOnlyCtx *io_ctx,
     io_ctx->ip_src = IPOnlyCIDRItemInsert(io_ctx->ip_src, s->cidr_src);
     io_ctx->ip_dst = IPOnlyCIDRItemInsert(io_ctx->ip_dst, s->cidr_dst);
 
-    if (s->num > io_ctx->max_idx)
-        io_ctx->max_idx = s->num;
+    if (mapped_signum > io_ctx->max_idx)
+        io_ctx->max_idx = mapped_signum;
 
     /** no longer ref to this, it's in the table now */
     s->cidr_src = NULL;

src/detect-engine-iponly.h

@@ -35,6 +35,7 @@ void IPOnlyDeinit(DetectEngineCtx *, DetectEngineIPOnlyCtx *);
 void IPOnlyPrepare(DetectEngineCtx *);
 void DetectEngineIPOnlyThreadInit(DetectEngineCtx *, DetectEngineIPOnlyThreadCtx *);
 void DetectEngineIPOnlyThreadDeinit(DetectEngineIPOnlyThreadCtx *);
+SigIntId IPOnlyTrackSigNum(DetectEngineIPOnlyCtx *, SigIntId);
 void IPOnlyAddSignature(DetectEngineCtx *, DetectEngineIPOnlyCtx *, Signature *);
 void IPOnlyRegisterTests(void);
 

src/detect.h

@@ -718,6 +718,11 @@ typedef struct DetectEngineIPOnlyCtx_ {
     /* Used to build the radix trees */
     IPOnlyCIDRItem *ip_src, *ip_dst;
     uint32_t max_idx;
+
+    /* Used to map large signums to smaller values to compact the bitsets
+     * stored in the radix trees */
+    uint32_t *sig_mapping;
+    uint32_t sig_mapping_size;
 } DetectEngineIPOnlyCtx;
 
 typedef struct DetectEngineLookupFlow_ {