http: implement min size stream logic

Update HTTP parser to set the min inspect depth per transaction. This
allows for signatures to have their fast_pattern in the HTTP body,
while still being able to inspect the raw stream reliably with it.

The inspect depth is set per transaction as it:
- depends on the per personality config for min inspect size
- is set to the size of the actual body if it is smaller

After the initial inspection is done, it is set to 0 which disables
the feature for the rest of the transaction.

This removes the rescanning flush logic in commit
7e004f52c6 and provides an alternative
fix for bug #2522. The old approach caused too much rescanning of
HTTP body data leading to a performance degradation.

Bug #2522

src/app-layer-htp.c

@@ -1773,6 +1773,32 @@ static int HTPCallbackRequestBodyData(htp_tx_data_t *d)
     }
 
 end:
+    if (hstate->conn != NULL) {
+        SCLogDebug("checking body size %"PRIu64" against inspect limit %u (cur %"PRIu64", last %"PRIu64")",
+                tx_ud->request_body.content_len_so_far,
+                hstate->cfg->request.inspect_min_size,
+                (uint64_t)hstate->conn->in_data_counter, hstate->last_request_data_stamp);
+
+        /* if we reach the inspect_min_size we'll trigger inspection,
+         * so make sure that raw stream is also inspected. Set the
+         * data to be used to the ammount of raw bytes we've seen to
+         * get here. */
+        if (tx_ud->request_body.body_inspected == 0 &&
+            tx_ud->request_body.content_len_so_far >= hstate->cfg->request.inspect_min_size) {
+            if ((uint64_t)hstate->conn->in_data_counter > hstate->last_request_data_stamp &&
+                (uint64_t)hstate->conn->in_data_counter - hstate->last_request_data_stamp < (uint64_t)UINT_MAX)
+            {
+                uint32_t x = (uint32_t)((uint64_t)hstate->conn->in_data_counter - hstate->last_request_data_stamp);
+
+                /* body still in progress, but due to min inspect size we need to inspect now */
+                StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER, x);
+                AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOSERVER);
+            }
+        /* after the start of the body, disable the depth logic */
+        } else if (tx_ud->request_body.body_inspected > 0) {
+            StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER, 0);
+        }
+    }
     SCReturnInt(HTP_OK);
 }
 
@@ -1844,6 +1870,31 @@ static int HTPCallbackResponseBodyData(htp_tx_data_t *d)
         }
     }
 
+    if (hstate->conn != NULL) {
+        SCLogDebug("checking body size %"PRIu64" against inspect limit %u (cur %"PRIu64", last %"PRIu64")",
+                tx_ud->response_body.content_len_so_far,
+                hstate->cfg->response.inspect_min_size,
+                (uint64_t)hstate->conn->in_data_counter, hstate->last_response_data_stamp);
+        /* if we reach the inspect_min_size we'll trigger inspection,
+         * so make sure that raw stream is also inspected. Set the
+         * data to be used to the ammount of raw bytes we've seen to
+         * get here. */
+        if (tx_ud->response_body.body_inspected == 0 &&
+            tx_ud->response_body.content_len_so_far >= hstate->cfg->response.inspect_min_size) {
+            if ((uint64_t)hstate->conn->out_data_counter > hstate->last_response_data_stamp &&
+                (uint64_t)hstate->conn->out_data_counter - hstate->last_response_data_stamp < (uint64_t)UINT_MAX)
+            {
+                uint32_t x = (uint32_t)((uint64_t)hstate->conn->out_data_counter - hstate->last_response_data_stamp);
+
+                /* body still in progress, but due to min inspect size we need to inspect now */
+                StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOCLIENT, x);
+                AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOCLIENT);
+            }
+        /* after the start of the body, disable the depth logic */
+        } else if (tx_ud->response_body.body_inspected > 0) {
+            StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOCLIENT, 0);
+        }
+    }
     SCReturnInt(HTP_OK);
 }
 
@@ -1906,6 +1957,41 @@ static int HTPCallbackResponseHasTrailer(htp_tx_t *tx)
     return HTP_OK;
 }
 
+/**\internal
+ * \brief called at start of request
+ * Set min inspect size.
+ */
+static int HTPCallbackRequestStart(htp_tx_t *tx)
+{
+    HtpState *hstate = htp_connp_get_user_data(tx->connp);
+    if (hstate == NULL) {
+        SCReturnInt(HTP_ERROR);
+    }
+
+    if (hstate->cfg)
+        StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER,
+                hstate->cfg->request.inspect_min_size);
+    SCReturnInt(HTP_OK);
+}
+
+/**\internal
+ * \brief called at start of response
+ * Set min inspect size.
+ */
+static int HTPCallbackResponseStart(htp_tx_t *tx)
+{
+    HtpState *hstate = htp_connp_get_user_data(tx->connp);
+    if (hstate == NULL) {
+        SCReturnInt(HTP_ERROR);
+    }
+
+    if (hstate->cfg)
+        StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOCLIENT,
+                hstate->cfg->response.inspect_min_size);
+    SCReturnInt(HTP_OK);
+}
+
+
 /**
  *  \brief  callback for request to store the recent incoming request
             in to the recent_in_tx for the given htp state
@@ -1938,9 +2024,12 @@ static int HTPCallbackRequest(htp_tx_t *tx)
             SCLogDebug("closing file that was being stored");
             (void)HTPFileClose(hstate, NULL, 0, 0, STREAM_TOSERVER);
             htud->tsflags &= ~HTP_FILENAME_SET;
+            StreamTcpReassemblySetMinInspectDepth(hstate->f->protoctx, STREAM_TOSERVER,
+                    (uint32_t)hstate->conn->in_data_counter);
         }
     }
 
+    hstate->last_request_data_stamp = (uint64_t)hstate->conn->in_data_counter;
     /* request done, do raw reassembly now to inspect state and stream
      * at the same time. */
     AppLayerParserTriggerRawStreamReassembly(hstate->f, STREAM_TOSERVER);
@@ -1996,6 +2085,7 @@ static int HTPCallbackResponse(htp_tx_t *tx)
         }
     }
 
+    hstate->last_response_data_stamp = (uint64_t)hstate->conn->out_data_counter;
     SCReturnInt(HTP_OK);
 }
 
@@ -2153,7 +2243,10 @@ static void HTPConfigSetDefaultsPhase1(HTPCfgRec *cfg_prec)
     htp_config_register_request_body_data(cfg_prec->cfg, HTPCallbackRequestBodyData);
     htp_config_register_response_body_data(cfg_prec->cfg, HTPCallbackResponseBodyData);
 
+    htp_config_register_request_start(cfg_prec->cfg, HTPCallbackRequestStart);
     htp_config_register_request_complete(cfg_prec->cfg, HTPCallbackRequest);
+
+    htp_config_register_response_start(cfg_prec->cfg, HTPCallbackResponseStart);
     htp_config_register_response_complete(cfg_prec->cfg, HTPCallbackResponse);
 
     htp_config_set_parse_request_cookies(cfg_prec->cfg, 0);

src/app-layer-htp.h

@@ -234,6 +234,8 @@ typedef struct HtpState_ {
     uint16_t flags;
     uint16_t events;
     uint16_t htp_messages_offset; /**< offset into conn->messages list */
+    uint64_t last_request_data_stamp;
+    uint64_t last_response_data_stamp;
 } HtpState;
 
 /** part of the engine needs the request body (e.g. http_client_body keyword) */

src/detect-engine-hcbd.c

@@ -163,7 +163,7 @@ static const uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_i
         goto end;
     }
 
-    if (!htp_state->cfg->http_body_inline && !(flags & STREAM_FLUSH)) {
+    if (!htp_state->cfg->http_body_inline) {
         /* inspect the body if the transfer is complete or we have hit
         * our body size limit */
         if ((htp_state->cfg->request.body_limit == 0 ||
@@ -205,13 +205,10 @@ static const uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_i
             offset);
     det_ctx->hcbd[index].offset = offset;
 
-    if (htud->request_body.content_len_so_far < htp_state->cfg->request.inspect_min_size) {
-        SCLogDebug("not updating tracker as we're still below inspect_min_size");
-    } else {
-        /* move inspected tracker to end of the data. HtpBodyPrune will consider
-         * the window sizes when freeing data */
-        htud->request_body.body_inspected = htud->request_body.content_len_so_far;
-    }
+    /* move inspected tracker to end of the data. HtpBodyPrune will consider
+     * the window sizes when freeing data */
+    htud->request_body.body_inspected = htud->request_body.content_len_so_far;
+
     buffer = det_ctx->hcbd[index].buffer;
     *buffer_len = det_ctx->hcbd[index].buffer_len;
     *stream_start_offset = det_ctx->hcbd[index].offset;

src/detect-engine-hsbd.c

@@ -118,7 +118,7 @@ InspectionBuffer *HttpServerBodyGetDataCallback(DetectEngineThreadCtx *det_ctx,
               flags & STREAM_EOF ? "true" : "false",
                (AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP, tx, flags) > HTP_RESPONSE_BODY) ? "true" : "false");
 
-    if (!htp_state->cfg->http_body_inline && !(flags & STREAM_FLUSH)) {
+    if (!htp_state->cfg->http_body_inline) {
         /* inspect the body if the transfer is complete or we have hit
         * our body size limit */
         if ((htp_state->cfg->response.body_limit == 0 ||
@@ -178,14 +178,10 @@ InspectionBuffer *HttpServerBodyGetDataCallback(DetectEngineThreadCtx *det_ctx,
         }
     }
 
-    if (body->content_len_so_far < htp_state->cfg->response.inspect_min_size) {
-        SCLogDebug("not updating tracker as we're still below inspect_min_size");
-    } else {
-        /* move inspected tracker to end of the data. HtpBodyPrune will consider
-         * the window sizes when freeing data */
-        body->body_inspected = body->content_len_so_far;
-        SCLogDebug("body->body_inspected now: %"PRIu64, body->body_inspected);
-    }
+    /* move inspected tracker to end of the data. HtpBodyPrune will consider
+     * the window sizes when freeing data */
+    body->body_inspected = body->content_len_so_far;
+    SCLogDebug("body->body_inspected now: %"PRIu64, body->body_inspected);
 
     SCReturnPtr(buffer, "InspectionBuffer");
 }