detect/http: flush bodies when inspecting stream

The HTTP bodies (http_client_body and http_server_body/file_data) use
settings to control how much data we have before doing first inspection:

    request-body-minimal-inspect-size
    response-body-minimal-inspect-size

These settings default to 32k as quite some existing rules need this.

At the same time, the 'raw stream' inspection uses its own limits. By
default it inspects the data in blocks of about 2.5k. This could lead
to a situation where rules would not match.

For example, with 2 rules like this:

    content:"abc"; content:"data="; http_client_body; depth:5; sid:1;
    content:"xyz"; sid:2;

Sid 1 would only be inspected when the POST body reached the 32k limit
or when it was complete. Observed case shows the POST body to be 18k.
Sid 2 is inspected as soon as the 2.5k limit is reached, and then again
for each 2.5k increment. This moves the raw stream tracker forward.

So by the time sid 1 is inspected, some 18/19k into the stream, the
raw stream tracker is actually already moved forward for approximately
17.5k, this leads to the stream match of sid 1 possibly not matching.
Since the body match is at the start of the buffer, it makes sense
that the body and stream are inspected together.

The body inspection uses a tracker 'body_inspected', that keeps track
of how far into the body both MPM and per signature inspection has
moved.

This patch updates the logic in 2 ways:

1. it triggers earlier HTTP body inspection, which is matched to the
   stream inspection. When the detection engine finds it has stream
   data available for inspection, it passes the new 'STREAM_FLUSH'
   flag to the HTTP body inspection code. Which will then do an
   early inspection, even if still before the min inspect size.

2. to still somewhat adhere to the min inspect size, the body
   tracker is not updated until the min inspect size is reached.
   This will lead to some re-evaluation of the same body data.

If raw stream reassembly is disabled, this 'STREAM_FLUSH' flag is
never set, and the old behavior is used.

Bug #2522.

src/detect-engine-hcbd.c

@@ -107,7 +107,7 @@ static const uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_i
                                                DetectEngineCtx *de_ctx,
                                                DetectEngineThreadCtx *det_ctx,
                                                Flow *f, HtpState *htp_state,
-                                               uint8_t flags,
+                                               const uint8_t flags,
                                                uint32_t *buffer_len,
                                                uint32_t *stream_start_offset)
 {
@@ -163,7 +163,7 @@ static const uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_i
         goto end;
     }
 
-    if (!htp_state->cfg->http_body_inline) {
+    if (!htp_state->cfg->http_body_inline && !(flags & STREAM_FLUSH)) {
         /* inspect the body if the transfer is complete or we have hit
         * our body size limit */
         if ((htp_state->cfg->request.body_limit == 0 ||
@@ -205,10 +205,13 @@ static const uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_i
             offset);
     det_ctx->hcbd[index].offset = offset;
 
-    /* move inspected tracker to end of the data. HtpBodyPrune will consider
+    if (htud->request_body.content_len_so_far < htp_state->cfg->request.inspect_min_size) {
-     * the window sizes when freeing data */
+        SCLogDebug("not updating tracker as we're still below inspect_min_size");
-    htud->request_body.body_inspected = htud->request_body.content_len_so_far;
+    } else {
-
+        /* move inspected tracker to end of the data. HtpBodyPrune will consider
+         * the window sizes when freeing data */
+        htud->request_body.body_inspected = htud->request_body.content_len_so_far;
+    }
     buffer = det_ctx->hcbd[index].buffer;
     *buffer_len = det_ctx->hcbd[index].buffer_len;
     *stream_start_offset = det_ctx->hcbd[index].offset;

src/detect-engine-hsbd.c

@@ -118,7 +118,7 @@ InspectionBuffer *HttpServerBodyGetDataCallback(DetectEngineThreadCtx *det_ctx,
               flags & STREAM_EOF ? "true" : "false",
                (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP, tx, flags) > HTP_RESPONSE_BODY) ? "true" : "false");
 
-    if (!htp_state->cfg->http_body_inline) {
+    if (!htp_state->cfg->http_body_inline && !(flags & STREAM_FLUSH)) {
         /* inspect the body if the transfer is complete or we have hit
         * our body size limit */
         if ((htp_state->cfg->response.body_limit == 0 ||
@@ -178,10 +178,14 @@ InspectionBuffer *HttpServerBodyGetDataCallback(DetectEngineThreadCtx *det_ctx,
         }
     }
 
-    /* move inspected tracker to end of the data. HtpBodyPrune will consider
+    if (body->content_len_so_far < htp_state->cfg->response.inspect_min_size) {
-     * the window sizes when freeing data */
+        SCLogDebug("not updating tracker as we're still below inspect_min_size");
-    body->body_inspected = body->content_len_so_far;
+    } else {
-    SCLogDebug("body->body_inspected now: %"PRIu64, body->body_inspected);
+        /* move inspected tracker to end of the data. HtpBodyPrune will consider
+         * the window sizes when freeing data */
+        body->body_inspected = body->content_len_so_far;
+        SCLogDebug("body->body_inspected now: %"PRIu64, body->body_inspected);
+    }
 
     SCReturnPtr(buffer, "InspectionBuffer");
 }

src/detect.c

@@ -984,6 +984,7 @@ static DetectRunScratchpad DetectRunSetup(
             if (p->proto == IPPROTO_TCP && pflow->protoctx &&
                     StreamReassembleRawHasDataReady(pflow->protoctx, p)) {
                 p->flags |= PKT_DETECT_HAS_STREAMDATA;
+                flow_flags |= STREAM_FLUSH;
             }
             SCLogDebug("alproto %u", alproto);
         } else {

src/stream.h

@@ -33,6 +33,7 @@
 #define STREAM_GAP              0x10    /**< data gap encountered */
 #define STREAM_DEPTH            0x20    /**< depth reached */
 #define STREAM_MIDSTREAM        0x40
+#define STREAM_FLUSH            0x80
 
 typedef int (*StreamSegmentCallback)(const Packet *, void *, const uint8_t *, uint32_t);
 int StreamSegmentForEach(const Packet *p, uint8_t flag,