Thread local ctx for detection keywords

Some detection keywords need thread local ctx storage. Example is the
filemagic keyword that has a ctx that is modified with each call. That
is not thread safe. This functionality allows registration of thread
local ctxs so that each detect thread works on it's own copy.

src/detect-engine.c

@@ -407,6 +407,16 @@ error:
     return NULL;
 }
 
+static void DetectEngineCtxFreeThreadKeywordData(DetectEngineCtx *de_ctx) {
+    DetectEngineThreadKeywordCtxItem *item = de_ctx->keyword_list;
+    while (item) {
+        DetectEngineThreadKeywordCtxItem *next = item->next;
+        SCFree(item);
+        item = next;
+    }
+    de_ctx->keyword_list = NULL;
+}
+
 void DetectEngineCtxFree(DetectEngineCtx *de_ctx) {
 
     if (de_ctx == NULL)
@@ -443,6 +453,7 @@ void DetectEngineCtxFree(DetectEngineCtx *de_ctx) {
         MpmFactoryDeRegisterAllMpmCtxProfiles(de_ctx);
     }
 
+    DetectEngineCtxFreeThreadKeywordData(de_ctx);
     SCFree(de_ctx);
     //DetectAddressGroupPrintMemory();
     //DetectSigGroupPrintMemory();
@@ -667,6 +678,45 @@ void DetectEngineResetMaxSigId(DetectEngineCtx *de_ctx) {
     de_ctx->signum = 0;
 }
 
+static int DetectEngineThreadCtxInitKeywords(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx) {
+    if (de_ctx->keyword_id > 0) {
+        det_ctx->keyword_ctxs_array = SCMalloc(de_ctx->keyword_id * sizeof(void *));
+        if (det_ctx->keyword_ctxs_array == NULL) {
+            SCLogError(SC_ERR_DETECT_PREPARE, "setting up thread local detect ctx");
+            return TM_ECODE_FAILED;
+        }
+
+        memset(det_ctx->keyword_ctxs_array, 0x00, de_ctx->keyword_id * sizeof(void *));
+
+        det_ctx->keyword_ctxs_size = de_ctx->keyword_id;
+
+        DetectEngineThreadKeywordCtxItem *item = de_ctx->keyword_list;
+        while (item) {
+            det_ctx->keyword_ctxs_array[item->id] = item->InitFunc(item->data);
+            if (det_ctx->keyword_ctxs_array[item->id] == NULL) {
+                SCLogError(SC_ERR_DETECT_PREPARE, "setting up thread local detect ctx "
+                        "for keyword \"%s\" failed", item->name);
+                return TM_ECODE_FAILED;
+            }
+            item = item->next;
+        }
+    }
+    return TM_ECODE_OK;
+}
+
+static void DetectEngineThreadCtxDeinitKeywords(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx) {
+    if (de_ctx->keyword_id > 0) {
+        DetectEngineThreadKeywordCtxItem *item = de_ctx->keyword_list;
+        while (item) {
+            item->FreeFunc(det_ctx->keyword_ctxs_array[item->id]);
+            item = item->next;
+        }
+        det_ctx->keyword_ctxs_size = 0;
+        SCFree(det_ctx->keyword_ctxs_array);
+        det_ctx->keyword_ctxs_array = NULL;
+    }
+}
+
 TmEcode DetectEngineThreadCtxInit(ThreadVars *tv, void *initdata, void **data) {
     DetectEngineCtx *de_ctx = (DetectEngineCtx *)initdata;
     if (de_ctx == NULL)
@@ -734,6 +784,8 @@ TmEcode DetectEngineThreadCtxInit(ThreadVars *tv, void *initdata, void **data) {
         return TM_ECODE_FAILED;
     }
 
+    DetectEngineThreadCtxInitKeywords(de_ctx, det_ctx);
+
     SC_ATOMIC_INIT(det_ctx->so_far_used_by_detect);
 
     *data = (void *)det_ctx;
@@ -814,6 +866,8 @@ static TmEcode DetectEngineThreadCtxInitForLiveRuleSwap(ThreadVars *tv, void *in
         return TM_ECODE_FAILED;
     }
 
+    DetectEngineThreadCtxInitKeywords(de_ctx, det_ctx);
+
     SC_ATOMIC_INIT(det_ctx->so_far_used_by_detect);
 
     *data = (void *)det_ctx;
@@ -868,6 +922,7 @@ TmEcode DetectEngineThreadCtxDeinit(ThreadVars *tv, void *data) {
         SCFree(det_ctx->hcbd);
     }
 
+    DetectEngineThreadCtxDeinitKeywords(det_ctx->de_ctx, det_ctx);
     SCFree(det_ctx);
 
     return TM_ECODE_OK;
@@ -879,6 +934,56 @@ void DetectEngineThreadCtxInfo(ThreadVars *t, DetectEngineThreadCtx *det_ctx) {
     PatternMatchThreadPrint(&det_ctx->mtcu, det_ctx->de_ctx->mpm_matcher);
 }
 
+/** \brief Register Thread keyword context Funcs
+ *
+ *  \param de_ctx detection engine to register in
+ *  \param name keyword name for error printing
+ *  \param InitFunc function ptr
+ *  \param data keyword init data to pass to Func
+ *  \param FreeFunc function ptr
+ *
+ *  \retval id for retrieval of ctx at runtime
+ *  \retval -1 on error
+ *
+ *  \note make sure "data" remains valid and it free'd elsewhere. It's
+ *        recommended to store it in the keywords global ctx so that
+ *        it's freed when the de_ctx is freed.
+ */
+int DetectRegisterThreadCtxFuncs(DetectEngineCtx *de_ctx, const char *name, void *(*InitFunc)(void *), void *data, void (*FreeFunc)(void *)) {
+    BUG_ON(de_ctx == NULL || InitFunc == NULL || FreeFunc == NULL || data == NULL);
+
+    DetectEngineThreadKeywordCtxItem *item = SCMalloc(sizeof(DetectEngineThreadKeywordCtxItem));
+    if (item == NULL)
+        return -1;
+    memset(item, 0x00, sizeof(DetectEngineThreadKeywordCtxItem));
+
+    item->InitFunc = InitFunc;
+    item->FreeFunc = FreeFunc;
+    item->data = data;
+
+    item->next = de_ctx->keyword_list;
+    de_ctx->keyword_list = item;
+    item->id = de_ctx->keyword_id++;
+
+    return item->id;
+}
+
+/** \brief Retrieve thread local keyword ctx by id
+ *
+ *  \param det_ctx detection engine thread ctx to retrieve the ctx from
+ *  \param id id of the ctx returned by DetectRegisterThreadCtxInitFunc at
+ *            keyword init.
+ *
+ *  \retval ctx or NULL on error
+ */
+void *DetectThreadCtxGetKeywordThreadCtx(DetectEngineThreadCtx *det_ctx, int id) {
+    if (id < 0 || id > det_ctx->keyword_ctxs_size || det_ctx->keyword_ctxs_array == NULL)
+        return NULL;
+
+    return det_ctx->keyword_ctxs_array[id];
+}
+
+
 /*************************************Unittest*********************************/
 
 #ifdef UNITTESTS

src/detect.h

@@ -529,6 +529,15 @@ typedef struct ThresholdCtx_    {
     uint32_t th_size;
 } ThresholdCtx;
 
+typedef struct DetectEngineThreadKeywordCtxItem_ {
+    void *(*InitFunc)(void *);
+    void (*FreeFunc)(void *);
+    void *data;
+    struct DetectEngineThreadKeywordCtxItem_ *next;
+    int id;
+    const char *name; /* keyword name, for error printing */
+} DetectEngineThreadKeywordCtxItem;
+
 /** \brief main detection engine ctx */
 typedef struct DetectEngineCtx_ {
     uint8_t flags;
@@ -676,6 +685,10 @@ typedef struct DetectEngineCtx_ {
 
     /** Is detect engine using a delayed init */
     int delayed_detect;
+
+    /** list of keywords that need thread local ctxs */
+    DetectEngineThreadKeywordCtxItem *keyword_list;
+    int keyword_id;
 } DetectEngineCtx;
 
 /* Engine groups profiles (low, medium, high, custom) */
@@ -795,6 +808,11 @@ typedef struct DetectionEngineThreadCtx_ {
      * thread can dump the packets once it has processed them */
     Tmq *cuda_mpm_rc_disp_outq;
 #endif
+
+    /** store for keyword contexts that need a per thread storage because of
+     *  thread safety issues */
+    void **keyword_ctxs_array;
+    int keyword_ctxs_size;
 } DetectEngineThreadCtx;
 
 /** \brief element in sigmatch type table.
@@ -1092,5 +1110,8 @@ int SignatureIsFilemagicInspecting(Signature *);
 int SignatureIsFileMd5Inspecting(Signature *);
 int SignatureIsFilesizeInspecting(Signature *);
 
+int DetectRegisterThreadCtxFuncs(DetectEngineCtx *, const char *name, void *(*InitFunc)(void *), void *data, void (*FreeFunc)(void *));
+void *DetectThreadCtxGetKeywordThreadCtx(DetectEngineThreadCtx *, int);
+
 #endif /* __DETECT_H__ */