prefilter: engine for tcp flags keyword

If there are many rules for TCP flags these rules would be inspected
against each TCP packet. Even though the flags check is not expensive,
the combined cost of inspecting multiple rules against each and every
packet is high.

This patch implements a prefilter engine for flags. If a rule group
has rules looking for specific flags and engine for that flag or
flags combination is set up. This way those rules are only inspected
if the flag is actually present in the packet.

src/detect-flags.c

@@ -29,6 +29,8 @@
 
 #include "detect.h"
 #include "detect-parse.h"
+#include "detect-engine-prefilter.h"
+#include "detect-engine-prefilter-common.h"
 
 #include "flow-var.h"
 #include "decode-events.h"
@@ -60,6 +62,9 @@ static int DetectFlagsMatch (ThreadVars *, DetectEngineThreadCtx *, Packet *, Si
 static int DetectFlagsSetup (DetectEngineCtx *, Signature *, char *);
 static void DetectFlagsFree(void *);
 
+static _Bool PrefilterTcpFlagsIsPrefilterable(const Signature *s);
+static int PrefilterSetupTcpFlags(SigGroupHead *sgh);
+
 /**
  * \brief Registration function for flags: keyword
  */
@@ -72,67 +77,47 @@ void DetectFlagsRegister (void)
     sigmatch_table[DETECT_FLAGS].Free  = DetectFlagsFree;
     sigmatch_table[DETECT_FLAGS].RegisterTests = FlagsRegisterTests;
 
+    sigmatch_table[DETECT_FLAGS].SupportsPrefilter = PrefilterTcpFlagsIsPrefilterable;
+    sigmatch_table[DETECT_FLAGS].SetupPrefilter = PrefilterSetupTcpFlags;
+
     DetectSetupParseRegexes(PARSE_REGEX, &parse_regex, &parse_regex_study);
 }
 
-/**
- * \internal
- * \brief This function is used to match flags on a packet with those passed via flags:
- *
- * \param t pointer to thread vars
- * \param det_ctx pointer to the pattern matcher thread
- * \param p pointer to the current packet
- * \param s pointer to the Signature
- * \param m pointer to the sigmatch
- *
- * \retval 0 no match
- * \retval 1 match
- */
-static int DetectFlagsMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, Packet *p, Signature *s, const SigMatchCtx *ctx)
+static inline int FlagsMatch(const uint8_t pflags, const uint8_t modifier,
+                             const uint8_t dflags, const uint8_t iflags)
 {
-    SCEnter();
-
-    uint8_t flags = 0;
-    const DetectFlagsData *de = (const DetectFlagsData *)ctx;
-
-    if (!(PKT_IS_TCP(p)) || PKT_IS_PSEUDOPKT(p)) {
-        SCReturnInt(0);
-    }
-
-    flags = p->tcph->th_flags;
-
-    if (!de->flags && flags) {
-        if(de->modifier == MODIFIER_NOT) {
+    if (!dflags && pflags) {
+        if(modifier == MODIFIER_NOT) {
             SCReturnInt(1);
         }
 
         SCReturnInt(0);
     }
 
-    flags &= de->ignored_flags;
+    const uint8_t flags = pflags & iflags;
 
-    switch (de->modifier) {
+    switch (modifier) {
         case MODIFIER_ANY:
-            if ((flags & de->flags) > 0) {
+            if ((flags & dflags) > 0) {
                 SCReturnInt(1);
             }
             SCReturnInt(0);
 
         case MODIFIER_PLUS:
-            if (((flags & de->flags) == de->flags)) {
+            if (((flags & dflags) == dflags)) {
                 SCReturnInt(1);
             }
             SCReturnInt(0);
 
         case MODIFIER_NOT:
-            if ((flags & de->flags) != de->flags) {
+            if ((flags & dflags) != dflags) {
                 SCReturnInt(1);
             }
             SCReturnInt(0);
 
         default:
-            SCLogDebug("flags %"PRIu8" and de->flags %"PRIu8"",flags,de->flags);
-            if (flags == de->flags) {
+            SCLogDebug("flags %"PRIu8" and de->flags %"PRIu8"", flags, dflags);
+            if (flags == dflags) {
                 SCReturnInt(1);
             }
     }
@@ -140,6 +125,33 @@ static int DetectFlagsMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, Pack
     SCReturnInt(0);
 }
 
+/**
+ * \internal
+ * \brief This function is used to match flags on a packet with those passed via flags:
+ *
+ * \param t pointer to thread vars
+ * \param det_ctx pointer to the pattern matcher thread
+ * \param p pointer to the current packet
+ * \param s pointer to the Signature
+ * \param m pointer to the sigmatch
+ *
+ * \retval 0 no match
+ * \retval 1 match
+ */
+static int DetectFlagsMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, Packet *p, Signature *s, const SigMatchCtx *ctx)
+{
+    SCEnter();
+
+    if (!(PKT_IS_TCP(p)) || PKT_IS_PSEUDOPKT(p)) {
+        SCReturnInt(0);
+    }
+
+    const DetectFlagsData *de = (const DetectFlagsData *)ctx;
+    const uint8_t flags = p->tcph->th_flags;
+
+    return FlagsMatch(flags, de->modifier, de->flags, de->ignored_flags);
+}
+
 /**
  * \internal
  * \brief This function is used to parse flags options passed via flags: keyword
@@ -540,6 +552,65 @@ int DetectFlagsSignatureNeedsSynOnlyPackets(const Signature *s)
     return 0;
 }
 
+static void
+PrefilterPacketFlagsMatch(DetectEngineThreadCtx *det_ctx, Packet *p, const void *pectx)
+{
+    if (!(PKT_IS_TCP(p)) || PKT_IS_PSEUDOPKT(p)) {
+        SCReturn;
+    }
+
+    const PrefilterPacketHeaderCtx *ctx = pectx;
+    const uint8_t flags = p->tcph->th_flags;
+
+    if (FlagsMatch(flags, ctx->v1.u8[0], ctx->v1.u8[1], ctx->v1.u8[2]))
+    {
+        SCLogDebug("packet matches TCP flags %02x", ctx->v1.u8[1]);
+        PrefilterAddSids(&det_ctx->pmq, ctx->sigs_array, ctx->sigs_cnt);
+    }
+}
+
+static void
+PrefilterPacketFlagsSet(PrefilterPacketHeaderValue *v, void *smctx)
+{
+    const DetectFlagsData *a = smctx;
+    v->u8[0] = a->modifier;
+    v->u8[1] = a->flags;
+    v->u8[2] = a->ignored_flags;
+    SCLogDebug("v->u8[0] = %02x", v->u8[0]);
+}
+
+static _Bool
+PrefilterPacketFlagsCompare(PrefilterPacketHeaderValue v, void *smctx)
+{
+    const DetectFlagsData *a = smctx;
+    if (v.u8[0] == a->modifier &&
+        v.u8[1] == a->flags &&
+        v.u8[2] == a->ignored_flags)
+        return TRUE;
+    return FALSE;
+}
+
+static int PrefilterSetupTcpFlags(SigGroupHead *sgh)
+{
+    return PrefilterSetupPacketHeader(sgh, DETECT_FLAGS,
+            PrefilterPacketFlagsSet,
+            PrefilterPacketFlagsCompare,
+            PrefilterPacketFlagsMatch);
+
+}
+
+static _Bool PrefilterTcpFlagsIsPrefilterable(const Signature *s)
+{
+    const SigMatch *sm;
+    for (sm = s->sm_lists[DETECT_SM_LIST_MATCH] ; sm != NULL; sm = sm->next) {
+        switch (sm->type) {
+            case DETECT_FLAGS:
+                return TRUE;
+        }
+    }
+    return FALSE;
+}
+
 /*
  * ONLY TESTS BELOW THIS COMMENT
  */

src/detect.h

@@ -457,6 +457,8 @@ typedef struct Signature_ {
      * to warn the user about any possible problem */
     char *sig_str;
 
+    int prefilter_list;
+
     /** ptr to the next sig in the list */
     struct Signature_ *next;
 } Signature;