detect: absent keyword to test absence of sticky buffer

Ticket: 2224

It takes an argument to match only if the buffer is absent,
or it can still match if the buffer is present, but we test
the absence of some content.

For multi buffers, absent matches if there are 0 buffers.

For file keywords, absent matches if there is no file.

doc/userguide/rules/payload-keywords.rst

@@ -274,6 +274,27 @@ You can also use the negation (!) before isdataat.
 
 .. image:: payload-keywords/isdataat1.png
 
+absent
+------
+
+The keyword ``absent`` checks that a sticky buffer does not exist.
+It can be used without any argument to match only on absent buffer :
+
+Example of ``absent`` in a rule:
+
+.. container:: example-rule
+
+   alert http any any -> any any (msg:"HTTP request without referer";  :example-rule-emphasis:`http.referer; absent;` sid:1; rev:1;)
+
+
+It can take an argument "or_else" to match on absent buffer or on what comes next such as negated content, for instance :
+
+.. container:: example-rule
+
+   alert http any any -> any any (msg:"HTTP request without referer";  :example-rule-emphasis:`http.referer; absent: or_else;` content: !"abc"; sid:1; rev:1;)
+
+For files (ie ``file.data``), absent means there are no files in the transaction.
+
 bsize
 -----
 

src/detect-engine-analyzer.c

@@ -39,6 +39,7 @@
 #include "detect-pcre.h"
 #include "detect-bytejump.h"
 #include "detect-bytetest.h"
+#include "detect-isdataat.h"
 #include "detect-flow.h"
 #include "detect-tcp-flags.h"
 #include "detect-tcp-ack.h"
@@ -855,6 +856,14 @@ static void DumpMatches(RuleAnalyzer *ctx, JsonBuilder *js, const SigMatchData *
                 jb_close(js);
                 break;
             }
+            case DETECT_ABSENT: {
+                const DetectAbsentData *dad = (const DetectAbsentData *)smd->ctx;
+                jb_open_object(js, "absent");
+                jb_set_bool(js, "or_else", dad->or_else);
+                jb_close(js);
+                break;
+            }
+
             case DETECT_IPOPTS: {
                 const DetectIpOptsData *cd = (const DetectIpOptsData *)smd->ctx;
 

src/detect-engine-content-inspection.c

@@ -382,6 +382,13 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx,
                     prev_offset);
         } while(1);
 
+    } else if (smd->type == DETECT_ABSENT) {
+        const DetectAbsentData *id = (DetectAbsentData *)smd->ctx;
+        if (!id->or_else) {
+            // we match only on absent buffer
+            goto no_match;
+        }
+        goto match;
     } else if (smd->type == DETECT_ISDATAAT) {
         SCLogDebug("inspecting isdataat");
 
@@ -646,8 +653,7 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx,
             goto match;
         }
         goto no_match_discontinue;
-    }
-    else if (smd->type == DETECT_LUA) {
+    } else if (smd->type == DETECT_LUA) {
         SCLogDebug("lua starting");
 
         if (DetectLuaMatchBuffer(det_ctx, s, smd, buffer, buffer_len,
@@ -758,6 +764,24 @@ bool DetectEngineContentInspectionBuffer(DetectEngineCtx *de_ctx, DetectEngineTh
         return false;
 }
 
+bool DetectContentInspectionMatchOnAbsentBuffer(const SigMatchData *smd)
+{
+    // we will match on NULL buffers there is one absent
+    bool absent_data = false;
+    while (1) {
+        if (smd->type == DETECT_ABSENT) {
+            absent_data = true;
+            break;
+        }
+        if (smd->is_last) {
+            break;
+        }
+        // smd does not get reused after this loop
+        smd++;
+    }
+    return absent_data;
+}
+
 #ifdef UNITTESTS
 #include "tests/detect-engine-content-inspection.c"
 #endif

src/detect-engine-content-inspection.h

@@ -68,6 +68,12 @@ bool DetectEngineContentInspectionBuffer(DetectEngineCtx *de_ctx, DetectEngineTh
         const Signature *s, const SigMatchData *smd, Packet *p, Flow *f, const InspectionBuffer *b,
         const enum DetectContentInspectionType inspection_mode);
 
+/** \brief tells if we should match on absent buffer, because
+ *  there is an absent keyword being used
+ *  \param smd array of content inspection matches
+ *  \retval bool true to match on absent buffer, false otherwise */
+bool DetectContentInspectionMatchOnAbsentBuffer(const SigMatchData *smd);
+
 void DetectEngineContentInspectionRegisterTests(void);
 
 #endif /* SURICATA_DETECT_ENGINE_CONTENT_INSPECTION_H */

src/detect-engine-mpm.c

@@ -1125,6 +1125,9 @@ void RetrieveFPForSig(const DetectEngineCtx *de_ctx, Signature *s)
         }
 
         for (SigMatch *sm = s->init_data->buffers[x].head; sm != NULL; sm = sm->next) {
+            // a buffer with absent keyword cannot be used as fast_pattern
+            if (sm->type == DETECT_ABSENT)
+                break;
             if (sm->type != DETECT_CONTENT)
                 continue;
 

src/detect-engine-register.h

@@ -93,6 +93,7 @@ enum DetectKeywordId {
     DETECT_LUA,
     DETECT_ISDATAAT,
     DETECT_AL_URILEN,
+    DETECT_ABSENT,
     /* end of content inspection */
 
     DETECT_METADATA,

src/detect-engine.c

@@ -666,6 +666,7 @@ static void AppendAppInspectEngine(DetectEngineCtx *de_ctx,
     new_engine->sm_list = t->sm_list;
     new_engine->sm_list_base = t->sm_list_base;
     new_engine->smd = smd;
+    new_engine->match_on_null = DetectContentInspectionMatchOnAbsentBuffer(smd);
     new_engine->progress = t->progress;
     new_engine->v2 = t->v2;
     SCLogDebug("sm_list %d new_engine->v2 %p/%p/%p", new_engine->sm_list, new_engine->v2.Callback,
@@ -2172,6 +2173,9 @@ uint8_t DetectEngineInspectBufferGeneric(DetectEngineCtx *de_ctx, DetectEngineTh
     const InspectionBuffer *buffer = engine->v2.GetData(det_ctx, transforms,
             f, flags, txv, list_id);
     if (unlikely(buffer == NULL)) {
+        if (eof && engine->match_on_null) {
+            return DETECT_ENGINE_INSPECT_SIG_MATCH;
+        }
         return eof ? DETECT_ENGINE_INSPECT_SIG_CANT_MATCH :
                      DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
     }
@@ -2233,6 +2237,14 @@ uint8_t DetectEngineInspectMultiBufferGeneric(DetectEngineCtx *de_ctx,
         }
         local_id++;
     } while (1);
+    if (local_id == 0) {
+        // That means we did not get even one buffer value from the multi-buffer
+        const bool eof = (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) >
+                          engine->progress);
+        if (eof && engine->match_on_null) {
+            return DETECT_ENGINE_INSPECT_SIG_MATCH;
+        }
+    }
     return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
 }
 

src/detect-file-data.c

@@ -403,6 +403,14 @@ uint8_t DetectEngineInspectFiledata(DetectEngineCtx *de_ctx, DetectEngineThreadC
     if (ffc == NULL) {
         return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES;
     }
+    if (ffc->head == NULL) {
+        const bool eof = (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) >
+                          engine->progress);
+        if (eof && engine->match_on_null) {
+            return DETECT_ENGINE_INSPECT_SIG_MATCH;
+        }
+        return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+    }
 
     int local_file_id = 0;
     File *file = ffc->head;

src/detect-filemagic.c

@@ -307,7 +307,15 @@ static uint8_t DetectEngineInspectFilemagic(DetectEngineCtx *de_ctx, DetectEngin
 
     AppLayerGetFileState files = AppLayerParserGetTxFiles(f, txv, flags);
     FileContainer *ffc = files.fc;
-    if (ffc == NULL) {
+    if (ffc == NULL || ffc->head == NULL) {
+        const bool eof = (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) >
+                          engine->progress);
+        if (eof && engine->match_on_null) {
+            return DETECT_ENGINE_INSPECT_SIG_MATCH;
+        }
+        if (ffc != NULL) {
+            return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+        }
         return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES;
     }
 

src/detect-filename.c

@@ -244,7 +244,15 @@ static uint8_t DetectEngineInspectFilename(DetectEngineCtx *de_ctx, DetectEngine
 
     AppLayerGetFileState files = AppLayerParserGetTxFiles(f, txv, flags);
     FileContainer *ffc = files.fc;
-    if (ffc == NULL) {
+    if (ffc == NULL || ffc->head == NULL) {
+        const bool eof = (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) >
+                          engine->progress);
+        if (eof && engine->match_on_null) {
+            return DETECT_ENGINE_INSPECT_SIG_MATCH;
+        }
+        if (ffc != NULL) {
+            return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+        }
         return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES;
     }
 

src/detect-http-client-body.c

@@ -312,6 +312,9 @@ static uint8_t DetectEngineInspectBufferHttpBody(DetectEngineCtx *de_ctx,
     const InspectionBuffer *buffer = HttpRequestBodyGetDataCallback(
             det_ctx, engine->v2.transforms, f, flags, txv, engine->sm_list, engine->sm_list_base);
     if (buffer == NULL || buffer->inspect == NULL) {
+        if (eof && engine->match_on_null) {
+            return DETECT_ENGINE_INSPECT_SIG_MATCH;
+        }
         return eof ? DETECT_ENGINE_INSPECT_SIG_CANT_MATCH : DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
     }
 

src/detect-http-header.c

@@ -191,6 +191,9 @@ static uint8_t DetectEngineInspectBufferHttpHeader(DetectEngineCtx *de_ctx,
         uint8_t *rawdata = GetBufferForTX(txv, det_ctx, f, flags, &rawdata_len);
         if (rawdata_len == 0) {
             SCLogDebug("no data");
+            if (engine->match_on_null && eof) {
+                return DETECT_ENGINE_INSPECT_SIG_MATCH;
+            }
             goto end;
         }
         /* setup buffer and apply transforms */

src/detect-isdataat.c

@@ -35,6 +35,7 @@
 
 #include "detect-isdataat.h"
 #include "detect-content.h"
+#include "detect-bytetest.h"
 #include "detect-uricontent.h"
 #include "detect-engine-build.h"
 
@@ -56,11 +57,99 @@ static DetectParseRegex parse_regex;
 int DetectIsdataatSetup (DetectEngineCtx *, Signature *, const char *);
 #ifdef UNITTESTS
 static void DetectIsdataatRegisterTests(void);
+static void DetectAbsentRegisterTests(void);
 #endif
 void DetectIsdataatFree(DetectEngineCtx *, void *);
 
 static int DetectEndsWithSetup (DetectEngineCtx *de_ctx, Signature *s, const char *nullstr);
 
+static void DetectAbsentFree(DetectEngineCtx *de_ctx, void *ptr)
+{
+    SCFree(ptr);
+}
+
+static int DetectAbsentSetup(DetectEngineCtx *de_ctx, Signature *s, const char *optstr)
+{
+    if (s->init_data->list == DETECT_SM_LIST_NOTSET) {
+        SCLogError("no buffer for absent keyword");
+        return -1;
+    }
+
+    if (DetectBufferGetActiveList(de_ctx, s) == -1)
+        return -1;
+
+    bool or_else;
+    if (optstr == NULL) {
+        or_else = false;
+    } else if (strcmp(optstr, "or_else") == 0) {
+        or_else = true;
+    } else {
+        SCLogError("unhandled value for absent keyword: %s", optstr);
+        return -1;
+    }
+    if (s->init_data->curbuf == NULL || s->init_data->list != (int)s->init_data->curbuf->id) {
+        SCLogError("unspected buffer for absent keyword");
+        return -1;
+    }
+    const DetectBufferType *b = DetectEngineBufferTypeGetById(de_ctx, s->init_data->list);
+    if (!b || b->frame) {
+        SCLogError("absent does not work with frames");
+        return -1;
+    }
+    if (s->init_data->curbuf->tail != NULL) {
+        SCLogError("absent must come first right after buffer");
+        return -1;
+    }
+    DetectAbsentData *dad = SCMalloc(sizeof(DetectAbsentData));
+    if (unlikely(dad == NULL))
+        return -1;
+
+    dad->or_else = or_else;
+
+    if (SigMatchAppendSMToList(de_ctx, s, DETECT_ABSENT, (SigMatchCtx *)dad, s->init_data->list) ==
+            NULL) {
+        DetectAbsentFree(de_ctx, dad);
+        return -1;
+    }
+    return 0;
+}
+
+bool DetectAbsentValidateContentCallback(Signature *s, const SignatureInitDataBuffer *b)
+{
+    bool has_other = false;
+    bool only_absent = false;
+    bool has_absent = false;
+    for (const SigMatch *sm = b->head; sm != NULL; sm = sm->next) {
+        if (sm->type == DETECT_ABSENT) {
+            has_absent = true;
+            const DetectAbsentData *dad = (const DetectAbsentData *)sm->ctx;
+            if (!dad->or_else) {
+                only_absent = true;
+            }
+        } else {
+            has_other = true;
+            if (sm->type == DETECT_CONTENT) {
+                const DetectContentData *cd = (DetectContentData *)sm->ctx;
+                if (has_absent && (cd->flags & DETECT_CONTENT_FAST_PATTERN)) {
+                    SCLogError("signature can't have absent and fast_pattern on the same buffer");
+                    return false;
+                }
+            }
+        }
+    }
+
+    if (only_absent && has_other) {
+        SCLogError("signature can't have a buffer tested absent and tested with other keywords "
+                   "such as content");
+        return false;
+    } else if (has_absent && !only_absent && !has_other) {
+        SCLogError(
+                "signature with absent: or_else expects other keywords to test on such as content");
+        return false;
+    }
+    return true;
+}
+
 /**
  * \brief Registration function for isdataat: keyword
  */
@@ -82,6 +171,16 @@ void DetectIsdataatRegister(void)
     sigmatch_table[DETECT_ENDS_WITH].Setup = DetectEndsWithSetup;
     sigmatch_table[DETECT_ENDS_WITH].flags = SIGMATCH_NOOPT;
 
+    sigmatch_table[DETECT_ABSENT].name = "absent";
+    sigmatch_table[DETECT_ABSENT].desc = "test if the buffer is absent";
+    sigmatch_table[DETECT_ABSENT].url = "/rules/payload-keywords.html#absent";
+    sigmatch_table[DETECT_ABSENT].Setup = DetectAbsentSetup;
+    sigmatch_table[DETECT_ABSENT].Free = DetectAbsentFree;
+    sigmatch_table[DETECT_ABSENT].flags = SIGMATCH_OPTIONAL_OPT;
+#ifdef UNITTESTS
+    sigmatch_table[DETECT_ABSENT].RegisterTests = DetectAbsentRegisterTests;
+#endif
+
     DetectSetupParseRegexes(PARSE_REGEX, &parse_regex);
 }
 
@@ -584,4 +683,41 @@ void DetectIsdataatRegisterTests(void)
     UtRegisterTest("DetectIsdataatTestPacket02", DetectIsdataatTestPacket02);
     UtRegisterTest("DetectIsdataatTestPacket03", DetectIsdataatTestPacket03);
 }
+
+static int DetectAbsentTestParse01(void)
+{
+    DetectEngineCtx *de_ctx = DetectEngineCtxInit();
+    FAIL_IF(de_ctx == NULL);
+    de_ctx->flags |= DE_QUIET;
+
+    Signature *s = DetectEngineAppendSig(de_ctx,
+            "alert http any any -> any any "
+            "(msg:\"invalid absent only with negated content\"; http.user_agent; "
+            "absent; content:!\"one\"; sid:2;)");
+    FAIL_IF(s != NULL);
+    s = DetectEngineAppendSig(de_ctx, "alert http any any -> any any "
+                                      "(msg:\"invalid absent\"; http.user_agent; "
+                                      "content:!\"one\"; absent; sid:2;)");
+    FAIL_IF(s != NULL);
+    s = DetectEngineAppendSig(de_ctx, "alert http any any -> any any "
+                                      "(msg:\"invalid absent\"; http.user_agent; "
+                                      "content:\"one\"; absent: or_else; sid:2;)");
+    FAIL_IF(s != NULL);
+    s = DetectEngineAppendSig(de_ctx, "alert http any any -> any any "
+                                      "(msg:\"absent without sticky buffer\"; "
+                                      "content:!\"one\"; absent: or_else; sid:2;)");
+    FAIL_IF(s != NULL);
+    s = DetectEngineAppendSig(de_ctx,
+            "alert websocket any any -> any any "
+            "(msg:\"absent with frame\"; "
+            "frame: websocket.pdu; absent: or_else; content:!\"one\"; sid:2;)");
+    FAIL_IF(s != NULL);
+    DetectEngineCtxFree(de_ctx);
+    PASS;
+}
+
+void DetectAbsentRegisterTests(void)
+{
+    UtRegisterTest("DetectAbsentTestParse01", DetectAbsentTestParse01);
+}
 #endif

src/detect-isdataat.h

@@ -34,7 +34,14 @@ typedef struct DetectIsdataatData_ {
     uint8_t flags; /* isdataat options*/
 } DetectIsdataatData;
 
+typedef struct DetectAbsentData_ {
+    /** absent or try to match with other keywords (false means only absent) */
+    bool or_else;
+} DetectAbsentData;
+
 /* prototypes */
 void DetectIsdataatRegister (void);
 
+bool DetectAbsentValidateContentCallback(Signature *s, const SignatureInitDataBuffer *);
+
 #endif /* SURICATA_DETECT_ISDATAAT_H */

src/detect-parse.c

@@ -35,6 +35,7 @@
 
 #include "detect-content.h"
 #include "detect-bsize.h"
+#include "detect-isdataat.h"
 #include "detect-pcre.h"
 #include "detect-uricontent.h"
 #include "detect-reference.h"
@@ -1971,6 +1972,9 @@ static int SigValidate(DetectEngineCtx *de_ctx, Signature *s)
         if (!DetectBsizeValidateContentCallback(s, b)) {
             SCReturnInt(0);
         }
+        if (!DetectAbsentValidateContentCallback(s, b)) {
+            SCReturnInt(0);
+        }
     }
 
     int ts_excl = 0;

src/detect.h

@@ -432,6 +432,8 @@ typedef struct DetectEngineAppInspectionEngine_ {
     uint8_t id;     /**< per sig id used in state keeping */
     bool mpm;
     bool stream;
+    /** will match on a NULL buffer (so an absent buffer) */
+    bool match_on_null;
     uint16_t sm_list;
     uint16_t sm_list_base; /**< base buffer being transformed */
     int16_t progress;