From 7627756360f6153eccaa0b2d3ee2ceefcb148281 Mon Sep 17 00:00:00 2001 From: Ofer Dagan Date: Tue, 16 Dec 2025 09:49:19 +0100 Subject: [PATCH] detect/detection_filter: add unique_on option Add optional unique_on {src_port|dst_port} to detection_filter for exact distinct port counting within the seconds window. Features: - Runtime uses a single 64k-bit (8192 bytes) union bitmap per threshold entry with O(1) updates. - Follows detection_filter semantics: alerting starts after the threshold (> count), not at it. - On window expiry, the window is reset and the current packet's port is recorded as the first distinct of the new window. Validation: - unique_on requires a ported transport protocol; reject rules that are not tcp/udp/sctp or that use ip (protocol any). Memory management: - Bitmap memory is bounded by detect.thresholds.memcap. - New counters: bitmap_memuse and bitmap_alloc_fail. Tests: - C unit tests for parsing, distinct counting, window reset, and allocation failure fallback. - suricata-verify tests for distinct src/dst port counting. Task #7928 --- doc/userguide/rules/thresholding.rst | 34 +- src/detect-detection-filter.c | 478 ++++++++++++++++++++++++++- src/detect-engine-threshold.c | 208 +++++++++++- src/detect-engine-threshold.h | 1 + src/detect-threshold.h | 9 + src/suricata.c | 1 + 6 files changed, 701 insertions(+), 30 deletions(-) diff --git a/doc/userguide/rules/thresholding.rst b/doc/userguide/rules/thresholding.rst index a35c61d068..f18438ba05 100644 --- a/doc/userguide/rules/thresholding.rst +++ b/doc/userguide/rules/thresholding.rst @@ -179,9 +179,39 @@ internal counter and alert each time the threshold has been reached. Syntax:: - detection_filter: track , count , seconds + detection_filter: track , count , seconds [, unique_on ] -Example: +``unique_on`` (optional) enables distinct counting on a field within the time window: + +- ``unique_on dst_port``: count distinct destination ports +- ``unique_on src_port``: count distinct source ports + +.. note:: + + ``unique_on`` requires a transport protocol with ports. Use it only with + TCP, UDP or SCTP rules. For non-ported protocols (or ``ip`` rules), Suricata + rejects ``unique_on`` during rule parsing. + +When ``unique_on`` is specified, alerts start when the number of distinct values +exceeds ``count`` during the ``seconds`` window, scoped by ``track``. + +Examples: + +.. container:: example-rule + + alert tcp any any -> $HOME_NET any (msg:"Vertical scan: >=10 distinct dst ports in 60s"; + flags:S; ack:0; flow:stateless; + :example-rule-emphasis:`detection_filter: track by_dst, count 10, seconds 60, unique_on dst_port;` + classtype:attempted-recon; sid:100001; rev:1;) + +.. container:: example-rule + + alert tcp any any -> $HOME_NET 22 (msg:"Horizontal scan: >=20 distinct src ports to SSH in 30s"; + flow:stateless; + :example-rule-emphasis:`detection_filter: track by_dst, count 20, seconds 30, unique_on src_port;` + classtype:attempted-recon; sid:100002; rev:1;) + +Without ``unique_on``, the classic behavior applies: .. container:: example-rule diff --git a/src/detect-detection-filter.c b/src/detect-detection-filter.c index 40e495c4fd..764f2af92c 100644 --- a/src/detect-detection-filter.c +++ b/src/detect-detection-filter.c @@ -39,6 +39,7 @@ #include "util-unittest-helper.h" #include "util-debug.h" #include "detect-engine-build.h" +#include "detect-engine-proto.h" #define TRACK_DST 1 #define TRACK_SRC 2 @@ -49,7 +50,11 @@ #define PARSE_REGEX \ "^\\s*(track|count|seconds)\\s+(by_src|by_dst|by_flow|\\d+)\\s*,\\s*(track|count|seconds)\\s+" \ "(by_src|" \ - "by_dst|by_flow|\\d+)\\s*,\\s*(track|count|seconds)\\s+(by_src|by_dst|by_flow|\\d+)\\s*$" + "by_dst|by_flow|\\d+)\\s*,\\s*(track|count|seconds)\\s+(by_src|by_dst|by_flow|\\d+)" \ + "(?:\\s*,\\s*unique_on\\s+(src_port|dst_port))?\\s*$" + +/* minimum number of PCRE submatches expected for detection_filter parse */ +#define DF_PARSE_MIN_SUBMATCHES 5 static DetectParseRegex parse_regex; @@ -104,12 +109,14 @@ static DetectThresholdData *DetectDetectionFilterParse(const char *rawstr) int res = 0; size_t pcre2_len; const char *str_ptr = NULL; - char *args[6] = { NULL, NULL, NULL, NULL, NULL, NULL }; + char *args[8] = { NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL }; char *copy_str = NULL, *df_opt = NULL; int seconds_found = 0, count_found = 0, track_found = 0; int seconds_pos = 0, count_pos = 0; size_t pos = 0; int i = 0; + int parsed_count = 0; + int ret = 0; char *saveptr = NULL; pcre2_match_data *match = NULL; @@ -134,8 +141,8 @@ static DetectThresholdData *DetectDetectionFilterParse(const char *rawstr) if (count_found != 1 || seconds_found != 1 || track_found != 1) goto error; - int ret = DetectParsePcreExec(&parse_regex, &match, rawstr, 0, 0); - if (ret < 5) { + ret = DetectParsePcreExec(&parse_regex, &match, rawstr, 0, 0); + if (ret < DF_PARSE_MIN_SUBMATCHES) { SCLogError("pcre_exec parse error, ret %" PRId32 ", string %s", ret, rawstr); goto error; } @@ -154,6 +161,7 @@ static DetectThresholdData *DetectDetectionFilterParse(const char *rawstr) } args[i] = (char *)str_ptr; + parsed_count++; if (strncasecmp(args[i], "by_dst", strlen("by_dst")) == 0) df->track = TRACK_DST; @@ -165,6 +173,10 @@ static DetectThresholdData *DetectDetectionFilterParse(const char *rawstr) count_pos = i + 1; if (strncasecmp(args[i], "seconds", strlen("seconds")) == 0) seconds_pos = i + 1; + if (strcasecmp(args[i], "src_port") == 0) + df->unique_on = DF_UNIQUE_SRC_PORT; + if (strcasecmp(args[i], "dst_port") == 0) + df->unique_on = DF_UNIQUE_DST_PORT; } if (args[count_pos] == NULL || args[seconds_pos] == NULL) { @@ -184,7 +196,7 @@ static DetectThresholdData *DetectDetectionFilterParse(const char *rawstr) goto error; } - for (i = 0; i < 6; i++) { + for (i = 0; i < parsed_count; i++) { if (args[i] != NULL) pcre2_substring_free((PCRE2_UCHAR *)args[i]); } @@ -193,7 +205,7 @@ static DetectThresholdData *DetectDetectionFilterParse(const char *rawstr) return df; error: - for (i = 0; i < 6; i++) { + for (i = 0; i < parsed_count; i++) { if (args[i] != NULL) pcre2_substring_free((PCRE2_UCHAR *)args[i]); } @@ -240,6 +252,17 @@ static int DetectDetectionFilterSetup(DetectEngineCtx *de_ctx, Signature *s, con if (df == NULL) goto error; + /* unique_on requires a ported L4 protocol: tcp/udp/sctp */ + if (df->unique_on != DF_UNIQUE_NONE) { + const int has_tcp = DetectProtoContainsProto(&s->proto, IPPROTO_TCP); + const int has_udp = DetectProtoContainsProto(&s->proto, IPPROTO_UDP); + const int has_sctp = DetectProtoContainsProto(&s->proto, IPPROTO_SCTP); + if (!(has_tcp || has_udp || has_sctp) || (s->proto.flags & DETECT_PROTO_ANY)) { + SCLogError("detection_filter unique_on requires protocol tcp/udp/sctp"); + goto error; + } + } + if (SCSigMatchAppendSMToList(de_ctx, s, DETECT_DETECTION_FILTER, (SigMatchCtx *)df, DETECT_SM_LIST_THRESHOLD) == NULL) { goto error; @@ -279,6 +302,11 @@ static void DetectDetectionFilterFree(DetectEngineCtx *de_ctx, void *df_ptr) #include "action-globals.h" #include "packet.h" +/* test seams from detect-engine-threshold.c */ +void ThresholdForceAllocFail(int); +uint64_t ThresholdGetBitmapMemuse(void); +uint64_t ThresholdGetBitmapAllocFail(void); + /** * \test DetectDetectionFilterTestParse01 is a test for a valid detection_filter options * @@ -366,6 +394,213 @@ static int DetectDetectionFilterTestParse06(void) PASS; } +/** + * \test unique_on requires tcp/udp/sctp protocol; alert ip should fail + */ +static int DetectDetectionFilterUniqueOnProtoValidationFail(void) +{ + ThresholdInit(); + + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF_NULL(de_ctx); + de_ctx->flags |= DE_QUIET; + + Signature *s = DetectEngineAppendSig(de_ctx, + "alert ip any any -> any any (msg:\"DF proto validation\"; " + "detection_filter: track by_dst, count 2, seconds 60, unique_on dst_port; sid:29;)"); + /* setup should fail, append returns NULL */ + FAIL_IF_NOT_NULL(s); + + DetectEngineCtxFree(de_ctx); + ThresholdDestroy(); + PASS; +} + +/** + * \test DetectDetectionFilterTestParseUnique01 tests parsing unique_on dst_port + */ +static int DetectDetectionFilterTestParseUnique01(void) +{ + DetectThresholdData *df = + DetectDetectionFilterParse("track by_dst, count 10, seconds 60, unique_on dst_port"); + FAIL_IF_NULL(df); + FAIL_IF_NOT(df->track == TRACK_DST); + FAIL_IF_NOT(df->count == 10); + FAIL_IF_NOT(df->seconds == 60); + FAIL_IF_NOT(df->unique_on == DF_UNIQUE_DST_PORT); + DetectDetectionFilterFree(NULL, df); + PASS; +} + +/** + * \test Distinct boundary: exactly 'count' distinct should not alert + */ +static int DetectDetectionFilterDistinctBoundaryNoAlert(void) +{ + ThreadVars th_v; + DetectEngineThreadCtx *det_ctx; + + ThresholdInit(); + memset(&th_v, 0, sizeof(th_v)); + StatsThreadInit(&th_v.stats); + + Packet *p1 = UTHBuildPacketReal(NULL, 0, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); + Packet *p2 = UTHBuildPacketReal(NULL, 0, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 81); + + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF_NULL(de_ctx); + de_ctx->flags |= DE_QUIET; + + Signature *s = DetectEngineAppendSig(de_ctx, + "alert tcp any any -> any any (msg:\"DF distinct boundary no alert\"; " + "detection_filter: track by_dst, count 2, seconds 60, unique_on dst_port; sid:24;)"); + FAIL_IF_NULL(s); + + SigGroupBuild(de_ctx); + DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx); + + SigMatchSignatures(&th_v, de_ctx, det_ctx, p1); + FAIL_IF(PacketAlertCheck(p1, 24)); + SigMatchSignatures(&th_v, de_ctx, det_ctx, p2); + FAIL_IF(PacketAlertCheck(p2, 24)); + + DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); + DetectEngineCtxFree(de_ctx); + UTHFreePackets(&p1, 1); + UTHFreePackets(&p2, 1); + ThresholdDestroy(); + StatsThreadCleanup(&th_v.stats); + PASS; +} + +/** + * \test Distinct window reset: expire and re-trigger after seconds + */ +static int DetectDetectionFilterDistinctWindowReset(void) +{ + ThreadVars th_v; + DetectEngineThreadCtx *det_ctx; + + ThresholdInit(); + memset(&th_v, 0, sizeof(th_v)); + StatsThreadInit(&th_v.stats); + + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF_NULL(de_ctx); + de_ctx->flags |= DE_QUIET; + + Signature *s = DetectEngineAppendSig(de_ctx, + "alert tcp any any -> any any (msg:\"DF distinct window reset\"; " + "detection_filter: track by_dst, count 2, seconds 2, unique_on dst_port; sid:25;)"); + FAIL_IF_NULL(s); + + SigGroupBuild(de_ctx); + DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx); + + Packet *p1 = UTHBuildPacketReal(NULL, 0, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); + p1->ts = TimeGet(); + SigMatchSignatures(&th_v, de_ctx, det_ctx, p1); + FAIL_IF(PacketAlertCheck(p1, 25)); + + Packet *p2 = UTHBuildPacketReal(NULL, 0, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 81); + p2->ts = TimeGet(); + SigMatchSignatures(&th_v, de_ctx, det_ctx, p2); + FAIL_IF(PacketAlertCheck(p2, 25)); + + Packet *p3 = UTHBuildPacketReal(NULL, 0, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 82); + p3->ts = TimeGet(); + SigMatchSignatures(&th_v, de_ctx, det_ctx, p3); + FAIL_IF_NOT(PacketAlertCheck(p3, 25)); + + /* advance time beyond window to force expiration */ + TimeSetIncrementTime(3); + + Packet *p4 = UTHBuildPacketReal(NULL, 0, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); + p4->ts = TimeGet(); + SigMatchSignatures(&th_v, de_ctx, det_ctx, p4); + FAIL_IF(PacketAlertCheck(p4, 25)); + + Packet *p5 = UTHBuildPacketReal(NULL, 0, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 81); + p5->ts = TimeGet(); + SigMatchSignatures(&th_v, de_ctx, det_ctx, p5); + FAIL_IF(PacketAlertCheck(p5, 25)); + + Packet *p6 = UTHBuildPacketReal(NULL, 0, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 82); + p6->ts = TimeGet(); + SigMatchSignatures(&th_v, de_ctx, det_ctx, p6); + FAIL_IF_NOT(PacketAlertCheck(p6, 25)); + + DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); + DetectEngineCtxFree(de_ctx); + UTHFreePackets(&p1, 1); + UTHFreePackets(&p2, 1); + UTHFreePackets(&p3, 1); + UTHFreePackets(&p4, 1); + UTHFreePackets(&p5, 1); + UTHFreePackets(&p6, 1); + ThresholdDestroy(); + StatsThreadCleanup(&th_v.stats); + PASS; +} + +/** + * \test When bitmap alloc fails, unique_on falls back to classic counting (> count) + */ +static int DetectDetectionFilterDistinctAllocFailFallback(void) +{ + ThreadVars th_v; + DetectEngineThreadCtx *det_ctx; + + ThresholdInit(); + memset(&th_v, 0, sizeof(th_v)); + StatsThreadInit(&th_v.stats); + + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF_NULL(de_ctx); + de_ctx->flags |= DE_QUIET; + + /* Force allocation failure for distinct bitmap */ + ThresholdForceAllocFail(1); + + Signature *s = DetectEngineAppendSig(de_ctx, + "alert tcp any any -> any any (msg:\"DF alloc fail fallback\"; " + "detection_filter: track by_dst, count 2, seconds 60, unique_on dst_port; sid:27;)"); + FAIL_IF_NULL(s); + + SigGroupBuild(de_ctx); + DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx); + + Packet *p1 = UTHBuildPacketReal(NULL, 0, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); + Packet *p2 = UTHBuildPacketReal(NULL, 0, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); + Packet *p3 = UTHBuildPacketReal(NULL, 0, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); + + int result = 0; + + /* Classic detection_filter alerts when current_count > count (i.e., 3rd packet) */ + SigMatchSignatures(&th_v, de_ctx, det_ctx, p1); + if (PacketAlertCheck(p1, 27)) + goto end; + SigMatchSignatures(&th_v, de_ctx, det_ctx, p2); + if (PacketAlertCheck(p2, 27)) + goto end; + SigMatchSignatures(&th_v, de_ctx, det_ctx, p3); + if (!PacketAlertCheck(p3, 27)) + goto end; + + result = 1; + +end: + /* cleanup and restore hook */ + ThresholdForceAllocFail(0); + DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); + DetectEngineCtxFree(de_ctx); + UTHFreePackets(&p1, 1); + UTHFreePackets(&p2, 1); + UTHFreePackets(&p3, 1); + ThresholdDestroy(); + StatsThreadCleanup(&th_v.stats); + return result; +} /** * \test DetectDetectionFilterTestSig1 is a test for checking the working of detection_filter @@ -550,11 +785,6 @@ static int DetectDetectionFilterTestSig3(void) FAIL_IF_NOT(PacketTestAction(p, ACTION_DROP)); p->action = 0; - SigMatchSignatures(&th_v, de_ctx, det_ctx, p); - FAIL_IF_NOT(PacketAlertCheck(p, 10)); - FAIL_IF_NOT(PacketTestAction(p, ACTION_DROP)); - p->action = 0; - DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); @@ -564,6 +794,214 @@ static int DetectDetectionFilterTestSig3(void) PASS; } +/** + * \test Verify bitmap memory is tracked in bitmap_memuse counter + */ +static int DetectDetectionFilterDistinctBitmapMemuseTracking(void) +{ + ThreadVars th_v; + DetectEngineThreadCtx *det_ctx; + + ThresholdInit(); + memset(&th_v, 0, sizeof(th_v)); + StatsThreadInit(&th_v.stats); + + /* Record baseline memuse */ + uint64_t baseline_memuse = ThresholdGetBitmapMemuse(); + + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF_NULL(de_ctx); + de_ctx->flags |= DE_QUIET; + + Signature *s = DetectEngineAppendSig(de_ctx, + "alert tcp any any -> any any (msg:\"DF memuse tracking\"; " + "detection_filter: track by_dst, count 2, seconds 60, unique_on dst_port; sid:30;)"); + FAIL_IF_NULL(s); + + SigGroupBuild(de_ctx); + DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx); + + /* Send a packet to trigger threshold entry creation with bitmap */ + Packet *p1 = UTHBuildPacketReal(NULL, 0, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); + SigMatchSignatures(&th_v, de_ctx, det_ctx, p1); + + /* Verify bitmap_memuse increased by 8192 bytes (65536/8) */ + uint64_t after_memuse = ThresholdGetBitmapMemuse(); + FAIL_IF_NOT(after_memuse == baseline_memuse + 8192); + + DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); + DetectEngineCtxFree(de_ctx); + UTHFreePackets(&p1, 1); + ThresholdDestroy(); + + /* After destroy, bitmap_memuse should return to baseline */ + uint64_t final_memuse = ThresholdGetBitmapMemuse(); + FAIL_IF_NOT(final_memuse == baseline_memuse); + + StatsThreadCleanup(&th_v.stats); + PASS; +} + +/** + * \test Verify bitmap_alloc_fail counter increments on forced failure + */ +static int DetectDetectionFilterDistinctAllocFailCounter(void) +{ + ThreadVars th_v; + DetectEngineThreadCtx *det_ctx; + + ThresholdInit(); + memset(&th_v, 0, sizeof(th_v)); + StatsThreadInit(&th_v.stats); + + /* Record baseline alloc fail count */ + uint64_t baseline_fail = ThresholdGetBitmapAllocFail(); + + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF_NULL(de_ctx); + de_ctx->flags |= DE_QUIET; + + /* Force allocation failure */ + ThresholdForceAllocFail(1); + + Signature *s = DetectEngineAppendSig(de_ctx, + "alert tcp any any -> any any (msg:\"DF alloc fail counter\"; " + "detection_filter: track by_dst, count 2, seconds 60, unique_on dst_port; sid:31;)"); + FAIL_IF_NULL(s); + + SigGroupBuild(de_ctx); + DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx); + + /* Send packet to trigger threshold entry creation (bitmap alloc will fail) */ + Packet *p1 = UTHBuildPacketReal(NULL, 0, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); + SigMatchSignatures(&th_v, de_ctx, det_ctx, p1); + + /* Verify alloc_fail counter increased */ + uint64_t after_fail = ThresholdGetBitmapAllocFail(); + FAIL_IF_NOT(after_fail == baseline_fail + 1); + + /* bitmap_memuse should NOT have increased since alloc failed */ + uint64_t memuse = ThresholdGetBitmapMemuse(); + FAIL_IF_NOT(memuse == 0); + + /* cleanup */ + ThresholdForceAllocFail(0); + DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); + DetectEngineCtxFree(de_ctx); + UTHFreePackets(&p1, 1); + ThresholdDestroy(); + StatsThreadCleanup(&th_v.stats); + PASS; +} + +/** + * \test Multiple distinct trackers should accumulate bitmap memory + */ +static int DetectDetectionFilterDistinctMultipleTrackers(void) +{ + ThreadVars th_v; + DetectEngineThreadCtx *det_ctx; + + ThresholdInit(); + memset(&th_v, 0, sizeof(th_v)); + StatsThreadInit(&th_v.stats); + + uint64_t baseline_memuse = ThresholdGetBitmapMemuse(); + + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF_NULL(de_ctx); + de_ctx->flags |= DE_QUIET; + + Signature *s = DetectEngineAppendSig(de_ctx, + "alert tcp any any -> any any (msg:\"DF multi tracker\"; " + "detection_filter: track by_dst, count 2, seconds 60, unique_on dst_port; sid:32;)"); + FAIL_IF_NULL(s); + + SigGroupBuild(de_ctx); + DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx); + + /* Create packets to different destinations - each will create a new threshold entry */ + Packet *p1 = UTHBuildPacketReal(NULL, 0, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); + Packet *p2 = UTHBuildPacketReal(NULL, 0, IPPROTO_TCP, "1.1.1.1", "3.3.3.3", 1024, 80); + Packet *p3 = UTHBuildPacketReal(NULL, 0, IPPROTO_TCP, "1.1.1.1", "4.4.4.4", 1024, 80); + + SigMatchSignatures(&th_v, de_ctx, det_ctx, p1); + SigMatchSignatures(&th_v, de_ctx, det_ctx, p2); + SigMatchSignatures(&th_v, de_ctx, det_ctx, p3); + + /* Verify 3 bitmaps allocated = 3 * 8192 = 24576 bytes */ + uint64_t after_memuse = ThresholdGetBitmapMemuse(); + FAIL_IF_NOT(after_memuse == baseline_memuse + (3 * 8192)); + + DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); + DetectEngineCtxFree(de_ctx); + UTHFreePackets(&p1, 1); + UTHFreePackets(&p2, 1); + UTHFreePackets(&p3, 1); + ThresholdDestroy(); + + /* After destroy, should return to baseline */ + uint64_t final_memuse = ThresholdGetBitmapMemuse(); + FAIL_IF_NOT(final_memuse == baseline_memuse); + + StatsThreadCleanup(&th_v.stats); + PASS; +} + +/** + * \test Bitmap memory is freed when threshold entry expires + */ +static int DetectDetectionFilterDistinctBitmapExpiry(void) +{ + ThreadVars th_v; + DetectEngineThreadCtx *det_ctx; + + ThresholdInit(); + memset(&th_v, 0, sizeof(th_v)); + StatsThreadInit(&th_v.stats); + + uint64_t baseline_memuse = ThresholdGetBitmapMemuse(); + + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF_NULL(de_ctx); + de_ctx->flags |= DE_QUIET; + + /* Use short timeout (2 seconds) */ + Signature *s = DetectEngineAppendSig(de_ctx, + "alert tcp any any -> any any (msg:\"DF bitmap expiry\"; " + "detection_filter: track by_dst, count 2, seconds 2, unique_on dst_port; sid:33;)"); + FAIL_IF_NULL(s); + + SigGroupBuild(de_ctx); + DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx); + + Packet *p1 = UTHBuildPacketReal(NULL, 0, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); + p1->ts = TimeGet(); + SigMatchSignatures(&th_v, de_ctx, det_ctx, p1); + + /* Verify bitmap allocated */ + uint64_t after_alloc = ThresholdGetBitmapMemuse(); + FAIL_IF_NOT(after_alloc == baseline_memuse + 8192); + + /* Advance time beyond the timeout to expire the entry */ + TimeSetIncrementTime(5); + + /* Trigger expiration by calling ThresholdsExpire */ + SCTime_t now = TimeGet(); + ThresholdsExpire(now); + + /* After expiry, bitmap memory should be freed */ + uint64_t after_expiry = ThresholdGetBitmapMemuse(); + FAIL_IF_NOT(after_expiry == baseline_memuse); + + DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); + DetectEngineCtxFree(de_ctx); + UTHFreePackets(&p1, 1); + ThresholdDestroy(); + StatsThreadCleanup(&th_v.stats); + PASS; +} + static void DetectDetectionFilterRegisterTests(void) { UtRegisterTest("DetectDetectionFilterTestParse01", DetectDetectionFilterTestParse01); @@ -572,8 +1010,26 @@ static void DetectDetectionFilterRegisterTests(void) UtRegisterTest("DetectDetectionFilterTestParse04", DetectDetectionFilterTestParse04); UtRegisterTest("DetectDetectionFilterTestParse05", DetectDetectionFilterTestParse05); UtRegisterTest("DetectDetectionFilterTestParse06", DetectDetectionFilterTestParse06); + UtRegisterTest( + "DetectDetectionFilterTestParseUnique01", DetectDetectionFilterTestParseUnique01); UtRegisterTest("DetectDetectionFilterTestSig1", DetectDetectionFilterTestSig1); UtRegisterTest("DetectDetectionFilterTestSig2", DetectDetectionFilterTestSig2); UtRegisterTest("DetectDetectionFilterTestSig3", DetectDetectionFilterTestSig3); + UtRegisterTest("DetectDetectionFilterDistinctBoundaryNoAlert", + DetectDetectionFilterDistinctBoundaryNoAlert); + UtRegisterTest( + "DetectDetectionFilterDistinctWindowReset", DetectDetectionFilterDistinctWindowReset); + UtRegisterTest("DetectDetectionFilterDistinctAllocFailFallback", + DetectDetectionFilterDistinctAllocFailFallback); + UtRegisterTest("DetectDetectionFilterUniqueOnProtoValidationFail", + DetectDetectionFilterUniqueOnProtoValidationFail); + UtRegisterTest("DetectDetectionFilterDistinctBitmapMemuseTracking", + DetectDetectionFilterDistinctBitmapMemuseTracking); + UtRegisterTest("DetectDetectionFilterDistinctAllocFailCounter", + DetectDetectionFilterDistinctAllocFailCounter); + UtRegisterTest("DetectDetectionFilterDistinctMultipleTrackers", + DetectDetectionFilterDistinctMultipleTrackers); + UtRegisterTest( + "DetectDetectionFilterDistinctBitmapExpiry", DetectDetectionFilterDistinctBitmapExpiry); } #endif /* UNITTESTS */ diff --git a/src/detect-engine-threshold.c b/src/detect-engine-threshold.c index 8bdbda8f5c..80501fd6b1 100644 --- a/src/detect-engine-threshold.c +++ b/src/detect-engine-threshold.c @@ -55,6 +55,39 @@ #include "util-hash.h" #include "util-thash.h" #include "util-hash-lookup3.h" +#include "counters.h" + +static SC_ATOMIC_DECLARE(uint64_t, threshold_bitmap_alloc_fail); +static SC_ATOMIC_DECLARE(uint64_t, threshold_bitmap_memuse); + +/* UNITTESTS-only test seam to force allocation failure and query counters */ +#ifdef UNITTESTS +void ThresholdForceAllocFail(int v); +uint64_t ThresholdGetBitmapMemuse(void); +uint64_t ThresholdGetBitmapAllocFail(void); + +static int g_threshold_force_alloc_fail = 0; + +void ThresholdForceAllocFail(int v) +{ + g_threshold_force_alloc_fail = v; +} + +uint64_t ThresholdGetBitmapMemuse(void) +{ + return SC_ATOMIC_GET(threshold_bitmap_memuse); +} + +uint64_t ThresholdGetBitmapAllocFail(void) +{ + return SC_ATOMIC_GET(threshold_bitmap_alloc_fail); +} +#endif + +/* bitmap settings for exact distinct counting of 16-bit ports */ +#define DF_PORT_BITMAP_SIZE (65536u / 8u) +#define DF_PORT_BYTE_IDX(p) ((uint32_t)((p) >> 3)) +#define DF_PORT_BIT_MASK(p) ((uint8_t)(1u << ((p)&7u))) struct Thresholds { THashTableContext *thash; @@ -63,11 +96,46 @@ struct Thresholds { static int ThresholdsInit(struct Thresholds *t); static void ThresholdsDestroy(struct Thresholds *t); +static uint64_t ThresholdBitmapAllocFailCounter(void) +{ + return SC_ATOMIC_GET(threshold_bitmap_alloc_fail); +} + +static uint64_t ThresholdBitmapMemuseCounter(void) +{ + return SC_ATOMIC_GET(threshold_bitmap_memuse); +} + +static uint64_t ThresholdMemuseCounter(void) +{ + if (ctx.thash == NULL) + return 0; + return SC_ATOMIC_GET(ctx.thash->memuse); +} + +static uint64_t ThresholdMemcapCounter(void) +{ + if (ctx.thash == NULL) + return 0; + return SC_ATOMIC_GET(ctx.thash->config.memcap); +} + void ThresholdInit(void) { + SC_ATOMIC_INIT(threshold_bitmap_alloc_fail); + SC_ATOMIC_INIT(threshold_bitmap_memuse); ThresholdsInit(&ctx); } +void ThresholdRegisterGlobalCounters(void) +{ + StatsRegisterGlobalCounter("detect.thresholds.memuse", ThresholdMemuseCounter); + StatsRegisterGlobalCounter("detect.thresholds.memcap", ThresholdMemcapCounter); + StatsRegisterGlobalCounter("detect.thresholds.bitmap_memuse", ThresholdBitmapMemuseCounter); + StatsRegisterGlobalCounter( + "detect.thresholds.bitmap_alloc_fail", ThresholdBitmapAllocFailCounter); +} + void ThresholdDestroy(void) { ThresholdsDestroy(&ctx); @@ -95,6 +163,8 @@ typedef struct ThresholdEntry_ { SCTime_t tv1; /**< Var for time control */ Address addr; /* used for src/dst/either tracking */ Address addr2; /* used for both tracking */ + /* distinct counting state (for detection_filter unique_on ports) */ + uint8_t *distinct_bitmap_union; /* 8192 bytes (65536 bits) */ }; }; @@ -109,9 +179,83 @@ static int ThresholdEntrySet(void *dst, void *src) return 0; } +static void ThresholdDistinctInit(ThresholdEntry *te, const DetectThresholdData *td) +{ + if (td->type != TYPE_DETECTION || td->unique_on == DF_UNIQUE_NONE) { + return; + } + DEBUG_VALIDATE_BUG_ON(td->seconds == 0); + + const uint32_t bitmap_size = DF_PORT_BITMAP_SIZE; + te->current_count = 0; +#ifdef UNITTESTS + if (g_threshold_force_alloc_fail) { + SC_ATOMIC_ADD(threshold_bitmap_alloc_fail, 1); + te->distinct_bitmap_union = NULL; + return; + } +#endif + /* Check memcap before allocating bitmap. + * Bitmap memory is bounded by detect.thresholds.memcap via thash. + * Note: if ctx.thash is NULL (e.g. init failed or unittests), we bypass + * the memcap check but still attempt allocation unless forced to fail. */ + if (ctx.thash != NULL && !THASH_CHECK_MEMCAP(ctx.thash, bitmap_size)) { + SC_ATOMIC_ADD(threshold_bitmap_alloc_fail, 1); + te->distinct_bitmap_union = NULL; + return; + } + + te->distinct_bitmap_union = SCCalloc(1, bitmap_size); + if (te->distinct_bitmap_union == NULL) { + SC_ATOMIC_ADD(threshold_bitmap_alloc_fail, 1); + } else { + /* Track bitmap memory in thash memuse for proper accounting */ + if (ctx.thash != NULL) { + (void)SC_ATOMIC_ADD(ctx.thash->memuse, bitmap_size); + } + SC_ATOMIC_ADD(threshold_bitmap_memuse, bitmap_size); + } +} + +static void ThresholdDistinctReset(ThresholdEntry *te) +{ + const uint32_t bitmap_size = DF_PORT_BITMAP_SIZE; + if (te->distinct_bitmap_union) { + memset(te->distinct_bitmap_union, 0x00, bitmap_size); + } + te->current_count = 0; +} + +static inline void ThresholdDistinctAddPort(ThresholdEntry *te, uint16_t port) +{ + const uint32_t byte_index = DF_PORT_BYTE_IDX(port); + const uint8_t bit_mask = DF_PORT_BIT_MASK(port); + if (te->distinct_bitmap_union) { + bool already = (te->distinct_bitmap_union[byte_index] & bit_mask); + if (!already) { + te->distinct_bitmap_union[byte_index] = + (uint8_t)(te->distinct_bitmap_union[byte_index] | bit_mask); + te->current_count++; + } + } +} + static void ThresholdEntryFree(void *ptr) { - // nothing to free, base data is part of hash + if (ptr == NULL) + return; + + ThresholdEntry *e = ptr; + if (e->distinct_bitmap_union) { + const uint32_t bitmap_size = DF_PORT_BITMAP_SIZE; + /* Decrement bitmap memory from thash memuse */ + if (ctx.thash != NULL) { + (void)SC_ATOMIC_SUB(ctx.thash->memuse, bitmap_size); + } + SC_ATOMIC_SUB(threshold_bitmap_memuse, bitmap_size); + SCFree(e->distinct_bitmap_union); + e->distinct_bitmap_union = NULL; + } } static inline uint32_t HashAddress(const Address *a) @@ -585,8 +729,8 @@ static int AddEntryToFlow(Flow *f, FlowThresholdEntryList *e, SCTime_t packet_ti return 0; } -static int ThresholdHandlePacketSuppress(Packet *p, - const DetectThresholdData *td, uint32_t sid, uint32_t gid) +static int ThresholdHandlePacketSuppress( + Packet *p, const DetectThresholdData *td, uint32_t sid, uint32_t gid) { int ret = 0; DetectAddress *m = NULL; @@ -664,15 +808,14 @@ static uint32_t BackoffCalcNextValue(const uint32_t cur, const uint32_t m) * \retval 1 normal match * \retval 0 no match */ -static int ThresholdSetup(const DetectThresholdData *td, ThresholdEntry *te, - const SCTime_t packet_time, const uint32_t sid, const uint32_t gid, const uint32_t rev, - const uint32_t tenant_id) +static int ThresholdSetup(const DetectThresholdData *td, ThresholdEntry *te, const Packet *p, + const uint32_t sid, const uint32_t gid, const uint32_t rev) { te->key[SID] = sid; te->key[GID] = gid; te->key[REV] = rev; te->key[TRACK] = td->track; - te->key[TENANT] = tenant_id; + te->key[TENANT] = p->tenant_id; te->seconds = td->seconds; te->current_count = 1; @@ -682,8 +825,22 @@ static int ThresholdSetup(const DetectThresholdData *td, ThresholdEntry *te, te->backoff.next_value = td->count; break; default: - te->tv1 = packet_time; + te->tv1 = p->ts; te->tv_timeout = SCTIME_INITIALIZER; + ThresholdDistinctInit(te, td); + /* If unique_on is enabled, we must add the current packet's port to the bitmap. + * ThresholdDistinctInit resets current_count to 0, so we must add the port + * or restore the count if allocation failed. */ + if (td->type == TYPE_DETECTION && td->unique_on != DF_UNIQUE_NONE) { + if (te->distinct_bitmap_union) { + uint16_t port = (td->unique_on == DF_UNIQUE_SRC_PORT) ? p->sp : p->dp; + ThresholdDistinctAddPort(te, port); + } else { + /* Allocation failed (or test mode), fallback to classic counting. + * We must set current_count to 1 for this first packet. */ + te->current_count = 1; + } + } break; } @@ -779,21 +936,38 @@ static int ThresholdCheckUpdate(const DetectEngineCtx *de_ctx, const DetectThres } } break; - case TYPE_DETECTION: + case TYPE_DETECTION: { SCLogDebug("detection_filter"); if (SCTIME_CMP_LTE(p->ts, entry)) { /* within timeout */ - te->current_count++; - if (te->current_count > td->count) { - ret = 1; + if (td->unique_on != DF_UNIQUE_NONE && te->distinct_bitmap_union) { + uint16_t port = (td->unique_on == DF_UNIQUE_SRC_PORT) ? p->sp : p->dp; + ThresholdDistinctAddPort(te, port); + if (te->current_count > td->count) { + ret = 1; + } + } else { + te->current_count++; + if (te->current_count > td->count) { + ret = 1; + } } } else { - /* expired, reset */ + /* expired, reset to new window starting now */ te->tv1 = p->ts; - te->current_count = 1; + ThresholdDistinctReset(te); + + /* record current packet's distinct port as the first in the new window */ + if (td->unique_on != DF_UNIQUE_NONE && te->distinct_bitmap_union) { + uint16_t port = (td->unique_on == DF_UNIQUE_SRC_PORT) ? p->sp : p->dp; + ThresholdDistinctAddPort(te, port); + } else { + te->current_count = 1; + } } break; + } case TYPE_RATE: { SCLogDebug("rate_filter"); const uint8_t original_action = pa->action; @@ -902,7 +1076,7 @@ static int ThresholdGetFromHash(const DetectEngineCtx *de_ctx, struct Thresholds ThresholdEntry *te = res.data->data; if (res.is_new) { // new threshold, set up - r = ThresholdSetup(td, te, p->ts, s->id, s->gid, s->rev, p->tenant_id); + r = ThresholdSetup(td, te, p, s->id, s->gid, s->rev); } else { // existing, check/update r = ThresholdCheckUpdate(de_ctx, td, te, p, s->id, s->gid, s->rev, pa); @@ -933,7 +1107,7 @@ static int ThresholdHandlePacketFlow(const DetectEngineCtx *de_ctx, Flow *f, Pac return 0; // new threshold, set up - ret = ThresholdSetup(td, &new->threshold, p->ts, sid, gid, rev, p->tenant_id); + ret = ThresholdSetup(td, &new->threshold, p, sid, gid, rev); if (AddEntryToFlow(f, new, p->ts) == -1) { SCFree(new); @@ -969,7 +1143,7 @@ int PacketAlertThreshold(const DetectEngineCtx *de_ctx, DetectEngineThreadCtx *d } if (td->type == TYPE_SUPPRESS) { - ret = ThresholdHandlePacketSuppress(p,td,s->id,s->gid); + ret = ThresholdHandlePacketSuppress(p, td, s->id, s->gid); } else if (td->track == TRACK_SRC) { if (PacketIsIPv4(p) && (td->type == TYPE_LIMIT || td->type == TYPE_BOTH)) { int cache_ret = CheckCache(p, td->track, s->id, s->gid, s->rev); diff --git a/src/detect-engine-threshold.h b/src/detect-engine-threshold.h index cb1b09fbbb..b18257eade 100644 --- a/src/detect-engine-threshold.h +++ b/src/detect-engine-threshold.h @@ -29,6 +29,7 @@ #include "detect-threshold.h" void ThresholdInit(void); +void ThresholdRegisterGlobalCounters(void); void ThresholdDestroy(void); uint32_t ThresholdsExpire(const SCTime_t ts); diff --git a/src/detect-threshold.h b/src/detect-threshold.h index 7218a28f57..b82380908b 100644 --- a/src/detect-threshold.h +++ b/src/detect-threshold.h @@ -47,6 +47,13 @@ #define TH_ACTION_SDROP 0x10 #define TH_ACTION_REJECT 0x20 +/* distinct counting support (for detection_filter) */ +enum DetectThresholdUniqueOn { + DF_UNIQUE_NONE = 0, + DF_UNIQUE_SRC_PORT, + DF_UNIQUE_DST_PORT, +}; + /** * \typedef DetectThresholdData * A typedef for DetectThresholdData_ @@ -61,6 +68,8 @@ typedef struct DetectThresholdData_ { uint32_t timeout; /**< timeout */ uint32_t flags; /**< flags used to set option */ uint32_t multiplier; /**< backoff multiplier */ + enum DetectThresholdUniqueOn + unique_on; /**< distinct counting on specific field (DF_UNIQUE_*) */ DetectAddressHead addrs; } DetectThresholdData; diff --git a/src/suricata.c b/src/suricata.c index ec415d4267..903d8abd05 100644 --- a/src/suricata.c +++ b/src/suricata.c @@ -2331,6 +2331,7 @@ void PreRunInit(const int runmode) AppLayerParserPostStreamSetup(); AppLayerRegisterGlobalCounters(); OutputFilestoreRegisterGlobalCounters(); + ThresholdRegisterGlobalCounters(); HttpRangeContainersInit(); }