datajson: limit impact of feature for non user

The det_ctx structure was inflated by the additoin of the array to
handle JSON context. This commit updates the code to use a growing
buffer and limit the impact.

src/decode.c

@@ -154,6 +154,7 @@ void PacketAlertRecycle(PacketAlert *pa_array, uint16_t cnt)
         struct PacketContextData *current_json = pa_array[i].json_info;
         while (current_json) {
             struct PacketContextData *next_json = current_json->next;
+            SCFree(current_json->json_string);
             SCFree(current_json);
             current_json = next_json;
         }
@@ -169,6 +170,7 @@ void PacketAlertFree(PacketAlert *pa_array)
         struct PacketContextData *allocated_json = pa_array[i].json_info;
         while (allocated_json) {
             struct PacketContextData *next_json = allocated_json->next;
+            SCFree(allocated_json->json_string);
             SCFree(allocated_json);
             allocated_json = next_json;
         }

src/detect-dataset.c

@@ -81,12 +81,17 @@ static int DetectDatajsonBufferMatch(DetectEngineThreadCtx *det_ctx, const Detec
             if (r.json.len > 0) {
                 /* we need to add 3 on length check for the added quotes and colon when
                 building the json string */
-                if ((det_ctx->json_content_len < SIG_JSON_CONTENT_ARRAY_LEN) &&
+                if (r.json.len + strlen(sd->json_key) + 3 < SIG_JSON_CONTENT_ITEM_LEN) {
-                        (r.json.len + strlen(sd->json_key) + 3 < SIG_JSON_CONTENT_ITEM_LEN)) {
+                    if (DetectEngineThreadCtxGetJsonContext(det_ctx) < 0) {
+                        DatajsonUnlockElt(&r);
+                        return 0;
+                    }
                     snprintf(det_ctx->json_content[det_ctx->json_content_len].json_content,
                             SIG_JSON_CONTENT_ITEM_LEN, "\"%s\":%s", sd->json_key, r.json.value);
                     det_ctx->json_content[det_ctx->json_content_len].id = sd->id;
                     det_ctx->json_content_len++;
+                    SCLogDebug("Added json content %u (alloc length %u)", det_ctx->json_content_len,
+                            det_ctx->json_content_capacity);
                 }
             }
             DatajsonUnlockElt(&r);

src/detect-engine-alert.c

@@ -291,15 +291,24 @@ static inline int PacketAlertSetContext(
     if (det_ctx->json_content_len) {
         /* We have some JSON attached in the current detection so let's try
            to see if some need to be used for current signature. */
-        struct PacketContextData *current_json = SCCalloc(1, sizeof(struct PacketContextData));
+        struct PacketContextData *current_json = NULL;
-        if (current_json == NULL) {
+        for (uint8_t i = 0; i < det_ctx->json_content_len; i++) {
-            /* Allocation error, let's return now */
-            return -1;
-        }
-        pa->json_info = current_json;
-        for (size_t i = 0; i < det_ctx->json_content_len; i++) {
             if (s == det_ctx->json_content[i].id) {
-                if (current_json->json_string != NULL) {
+                SCLogDebug("signature %p, content index %u", s, i);
+                if (current_json == NULL) {
+                    /* Allocate the first one */
+                    current_json = SCCalloc(1, sizeof(struct PacketContextData));
+                    if (current_json == NULL) {
+                        /* Allocation error, let's return now */
+                        return -1;
+                    }
+                    if (pa->json_info == NULL) {
+                        /* If this is the first one, set it */
+                        pa->json_info = current_json;
+                    }
+                    current_json->next = NULL;
+                } else {
+                    /* Allocate the next one */
                     struct PacketContextData *next_json =
                             SCCalloc(1, sizeof(struct PacketContextData));
                     if (next_json) {
@@ -311,7 +320,9 @@ static inline int PacketAlertSetContext(
                         return -1;
                     }
                 }
-                current_json->json_string = det_ctx->json_content[i].json_content;
+                current_json->json_string = SCStrdup(det_ctx->json_content[i].json_content);
+                SCLogDebug("json content %u, value '%s' (%p)", (unsigned int)i,
+                        current_json->json_string, s);
             }
         }
     }

src/detect-engine.c

@@ -3416,6 +3416,10 @@ TmEcode DetectEngineThreadCtxInit(ThreadVars *tv, void *initdata, void **data)
     /* Register a counter for Lua memory limit errors. */
     det_ctx->lua_memory_limit_errors = StatsRegisterCounter("detect.lua.memory_limit_errors", tv);
 
+    det_ctx->json_content = NULL;
+    det_ctx->json_content_capacity = 0;
+    det_ctx->json_content_len = 0;
+
 #ifdef PROFILING
     det_ctx->counter_mpm_list = StatsRegisterAvgCounter("detect.mpm_list", tv);
     det_ctx->counter_nonmpm_list = StatsRegisterAvgCounter("detect.nonmpm_list", tv);
@@ -3585,6 +3589,12 @@ static void DetectEngineThreadCtxFree(DetectEngineThreadCtx *det_ctx)
 #endif
     }
 
+    if (det_ctx->json_content) {
+        SCFree(det_ctx->json_content);
+        det_ctx->json_content = NULL;
+        det_ctx->json_content_capacity = 0;
+    }
+
     AppLayerDecoderEventsFreeEvents(&det_ctx->decoder_events);
     PrefilterPktNonPFStatsDump();
     SCFree(det_ctx);
@@ -5051,6 +5061,30 @@ void SCDetectEngineRegisterRateFilterCallback(SCDetectRateFilterFunc fn, void *a
     DetectEngineDeReference(&de_ctx);
 }
 
+int DetectEngineThreadCtxGetJsonContext(DetectEngineThreadCtx *det_ctx)
+{
+    if (det_ctx->json_content_len > SIG_JSON_CONTENT_ARRAY_LEN - 1) {
+        SCLogDebug("json content length %u exceeds maximum %u", det_ctx->json_content_len,
+                SIG_JSON_CONTENT_ARRAY_LEN);
+        return -1;
+    }
+    if (det_ctx->json_content_len >= det_ctx->json_content_capacity) {
+        if (det_ctx->json_content_capacity == 0) {
+            det_ctx->json_content_capacity = 1;
+        } else {
+            det_ctx->json_content_capacity *= 2;
+        }
+        void *tmp = SCRealloc(
+                det_ctx->json_content, det_ctx->json_content_capacity * sizeof(SigJsonContent));
+        if (unlikely(tmp == NULL)) {
+            return -1;
+        }
+        SCLogDebug("reallocated json content array to %u items", det_ctx->json_content_capacity);
+        det_ctx->json_content = tmp;
+    }
+    return 0;
+}
+
 /*************************************Unittest*********************************/
 
 #ifdef UNITTESTS

src/detect-engine.h

@@ -79,6 +79,7 @@ DetectEngineCtx *DetectEngineCtxInit(void);
 DetectEngineCtx *DetectEngineCtxInitStubForDD(void);
 DetectEngineCtx *DetectEngineCtxInitStubForMT(void);
 void DetectEngineCtxFree(DetectEngineCtx *);
+int DetectEngineThreadCtxGetJsonContext(DetectEngineThreadCtx *det_ctx);
 
 int DetectRegisterThreadCtxGlobalFuncs(const char *name,
         void *(*InitFunc)(void *), void *data, void (*FreeFunc)(void *));

src/detect-pcre.c

@@ -168,8 +168,11 @@ static void DetectAlertStoreMatch(DetectEngineThreadCtx *det_ctx, const Signatur
 
     SCLogDebug("json key: %s", json_key);
     /* Setup the data*/
-    if ((det_ctx->json_content_len < SIG_JSON_CONTENT_ARRAY_LEN) &&
+    if (capture_len + strlen(json_key) + 5 < SIG_JSON_CONTENT_ITEM_LEN) {
-            (capture_len + strlen(json_key) + 5 < SIG_JSON_CONTENT_ITEM_LEN)) {
+        if (DetectEngineThreadCtxGetJsonContext(det_ctx) < 0) {
+            SCFree(str_ptr);
+            return;
+        }
         SCJsonBuilder *js = SCJbNewObject();
         if (unlikely(js == NULL)) {
             SCFree(str_ptr);

src/detect.h

@@ -1278,7 +1278,9 @@ typedef struct DetectEngineThreadCtx_ {
     /* byte_* values */
     uint64_t *byte_values;
 
-    size_t json_content_len;
+    SigJsonContent *json_content;
+    uint8_t json_content_capacity;
+    uint8_t json_content_len;
 
     /* counter for the filestore array below -- up here for cache reasons. */
     uint16_t filestore_cnt;
@@ -1382,7 +1384,6 @@ typedef struct DetectEngineThreadCtx_ {
     /** stat of lua memory limit errors. */
     uint16_t lua_memory_limit_errors;
 
-    SigJsonContent json_content[SIG_JSON_CONTENT_ARRAY_LEN];
 #ifdef DEBUG
     uint64_t pkt_stream_add_cnt;
     uint64_t payload_mpm_cnt;

src/output-json-alert.c

@@ -257,6 +257,7 @@ void AlertJsonHeader(const Packet *p, const PacketAlert *pa, SCJsonBuilder *js,
         SCJbOpenObject(js, "context");
         const struct PacketContextData *json_info = pa->json_info;
         while (json_info) {
+            SCLogDebug("JSON string '{%s}'", json_info->json_string);
             /* The string is valid json as it is validated by JANSSON
                during parsing and included later via a format string */
             SCJbSetFormatted(js, json_info->json_string);