mpm: improve negated mpm

The idea is: if mpm is negated, it's both on mpm and nonmpm sid lists
and we can kick it out in that case during the merge sort.

It only works for patterns that are 'independent'. This means that the
rule doesn't need to only match if the negated mpm pattern is limited
to the first 10 bytes for example.

Or more generally, an negated mpm pattern that has depth, offset,
distance or within settings can't be handled this way. These patterns
are not added to the mpm at all, but just to to non-mpm list. This
makes sense as they will *always* need manual inspection.

Similarly, a pattern that is 'chopped' always needs validation. This
is because in this case we only inspect a part of the final pattern.

src/detect-content.h

@@ -61,6 +61,18 @@
                                         ((c)->flags & DETECT_CONTENT_DEPTH) || \
                                         ((c)->flags & DETECT_CONTENT_OFFSET) ))
 
+/* if a pattern has no depth/offset limits, no relative specifiers and isn't
+ * chopped for the mpm, we can take the mpm and consider this pattern a match
+ * w/o futher inspection. Warning: this may still mean other patterns depend
+ * on this pattern that force match validation anyway. */
+#define DETECT_CONTENT_MPM_IS_CONCLUSIVE(c) \
+                                    !( ((c)->flags & DETECT_CONTENT_DISTANCE) || \
+                                       ((c)->flags & DETECT_CONTENT_WITHIN)   || \
+                                       ((c)->flags & DETECT_CONTENT_DEPTH)    || \
+                                       ((c)->flags & DETECT_CONTENT_OFFSET)   || \
+                                       ((c)->flags & DETECT_CONTENT_FAST_PATTERN_CHOP))
+
+
 #include "util-spm-bm.h"
 
 typedef struct DetectContentData_ {

src/detect-engine-mpm.c

@@ -1376,8 +1376,23 @@ void MpmStoreSetup(const DetectEngineCtx *de_ctx, MpmStore *ms)
 
             const DetectContentData *cd = (DetectContentData *)s->mpm_sm->ctx;
 
-            PopulateMpmHelperAddPatternToPktCtx(ms->mpm_ctx,
-                    cd, s, 0, (cd->flags & DETECT_CONTENT_FAST_PATTERN_CHOP));
+            int skip = 0;
+            /* negated logic: if mpm match can't be used to be sure about this
+             * pattern, we have to inspect the rule fully regardless of mpm
+             * match. So in this case there is no point of adding it at all.
+             * The non-mpm list entry for the sig will make sure the sig is
+             * inspected. */
+            if ((cd->flags & DETECT_CONTENT_NEGATED) &&
+                !(DETECT_CONTENT_MPM_IS_CONCLUSIVE(cd)))
+            {
+                skip = 1;
+                SCLogDebug("not adding negated mpm as it's not 'single'");
+            }
+
+            if (!skip) {
+                PopulateMpmHelperAddPatternToPktCtx(ms->mpm_ctx,
+                        cd, s, 0, (cd->flags & DETECT_CONTENT_FAST_PATTERN_CHOP));
+            }
         }
     }
 

src/detect.c

@@ -765,7 +765,7 @@ static inline void DetectPrefilterMergeSort(DetectEngineCtx *de_ctx,
         goto final;
     }
     while (1) {
-        if (mpm <= nonmpm) {
+        if (mpm < nonmpm) {
             /* Take from mpm list */
             id = mpm;
 
@@ -778,12 +778,12 @@ static inline void DetectPrefilterMergeSort(DetectEngineCtx *de_ctx,
             if (unlikely(--m_cnt == 0)) {
                 /* mpm list is now empty */
                 final_ptr = nonmpm_ptr;
-                 final_cnt = n_cnt;
-                 goto final;
+                final_cnt = n_cnt;
+                goto final;
              }
              mpm_ptr++;
              mpm = *mpm_ptr;
-         } else {
+         } else if (mpm > nonmpm) {
              id = nonmpm;
 
              s = sig_array[id];
@@ -799,6 +799,38 @@ static inline void DetectPrefilterMergeSort(DetectEngineCtx *de_ctx,
              }
              nonmpm_ptr++;
              nonmpm = *nonmpm_ptr;
+
+        } else { /* implied mpm == nonmpm */
+            /* special case: if on both lists, it's a negated mpm pattern */
+
+            /* mpm list may have dups, so skip past them here */
+            while (--m_cnt != 0) {
+                mpm_ptr++;
+                mpm = *mpm_ptr;
+                if (mpm != nonmpm)
+                    break;
+            }
+            /* if mpm is done, update nonmpm_ptrs and jump to final */
+            if (unlikely(m_cnt == 0)) {
+                n_cnt--;
+
+                /* mpm list is now empty */
+                final_ptr = ++nonmpm_ptr;
+                final_cnt = n_cnt;
+                goto final;
+            }
+            /* otherwise, if nonmpm is done jump to final for mpm
+             * mpm ptrs alrady updated */
+            if (unlikely(--n_cnt == 0)) {
+                final_ptr = mpm_ptr;
+                final_cnt = m_cnt;
+                goto final;
+            }
+
+            /* not at end of the lists, update nonmpm. Mpm already
+             * updated in while loop above. */
+            nonmpm_ptr++;
+            nonmpm = *nonmpm_ptr;
         }
     }