detect: check if signature uses too many buffers

Ticket: #6104

The approach in master branch is to change the prototype of
SigMatchAppendSMToList so that it allocates itself the new SigMatch
This approach requires to change all the 100-ish calls to
SigMatchAppendSMToList and is thus quite a big change.

For branch 7, we still wanted to avoid the buffer overflow, but
did not want such an intrusive change, and still wanted to make
the signature invalid. Instead of changing the prototype of the
function, we make it return early, and set a flag in the signature
which can be later checked by SigValidate

src/detect-parse.c

@@ -473,7 +473,8 @@ void SigMatchAppendSMToList(Signature *s, SigMatch *new, const int list)
                 s->init_data->curbuf == NULL) {
             if (SignatureInitDataBufferCheckExpand(s) < 0) {
                 SCLogError("failed to expand rule buffer array");
-                // return -1; TODO error handle
+                s->init_data->init_flags |= SIG_FLAG_INIT_OVERFLOW;
+                return;
             }
 
             /* initialize new buffer */
@@ -1904,6 +1905,11 @@ static int SigValidate(DetectEngineCtx *de_ctx, Signature *s)
         SCReturnInt(0);
     }
 
+    if (s->init_data->init_flags & SIG_FLAG_INIT_OVERFLOW) {
+        SCLogError("rule %u tries to use too many buffers", s->id);
+        SCReturnInt(0);
+    }
+
     bool has_frame = false;
     bool has_app = false;
     bool has_pkt = false;

src/detect.h

@@ -283,6 +283,7 @@ typedef struct DetectPort_ {
     BIT_U32(8) /**< priority is explicitly set by the priority keyword */
 #define SIG_FLAG_INIT_FILEDATA              BIT_U32(9)  /**< signature has filedata keyword */
 #define SIG_FLAG_INIT_JA3                   BIT_U32(10) /**< signature has ja3 keyword */
+#define SIG_FLAG_INIT_OVERFLOW              BIT_U32(11) /**< signature has overflown buffers */
 
 /* signature mask flags */
 /** \note: additions should be added to the rule analyzer as well */