From cbd4c298ed44473a4a3b211afd05c3bd3b0affd3 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Sat, 25 Sep 2010 08:22:18 +0200 Subject: [PATCH] Initial version of a new bitmask based signature pre-filtering method. --- src/detect-engine-siggroup.c | 19 +--- src/detect.c | 177 ++++++++++++++++++++++++++++++++--- src/detect.h | 53 ++++++----- 3 files changed, 200 insertions(+), 49 deletions(-) diff --git a/src/detect-engine-siggroup.c b/src/detect-engine-siggroup.c index 22437acae3..db6e61408b 100644 --- a/src/detect-engine-siggroup.c +++ b/src/detect-engine-siggroup.c @@ -1594,25 +1594,10 @@ int SigGroupHeadBuildHeadArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh) if (s == NULL) continue; -// sgh->head_array[idx].flags = s->flags; -// sgh->head_array[idx].alproto = s->alproto; -// sgh->head_array[idx].num = s->num; - sgh->head_array[idx].hdr_copy = s->hdr_copy; - sgh->head_array[idx].mpm_pattern_copy = s->mpm_pattern_copy; -// sgh->head_array[idx].mpm_pattern_id_div_8 = s->mpm_pattern_id_div_8; -// sgh->head_array[idx].mpm_pattern_id_mod_8 = s->mpm_pattern_id_mod_8; -// sgh->head_array[idx].mpm_stream_pattern_copy = s->mpm_stream_pattern_copy; -// sgh->head_array[idx].mpm_stream_pattern_id_div_8 = s->mpm_stream_pattern_id_div_8; -// sgh->head_array[idx].mpm_stream_pattern_id_mod_8 = s->mpm_stream_pattern_id_mod_8; + sgh->head_array[idx].hdr_copy1 = s->hdr_copy1; + sgh->head_array[idx].hdr_copy2 = s->hdr_copy2; sgh->head_array[idx].full_sig = s; -// BUG_ON(s->flags != sgh->head_array[idx].flags); -// BUG_ON(s->alproto != sgh->head_array[idx].alproto); -// BUG_ON(s->mpm_pattern_id != sgh->head_array[idx].mpm_pattern_id); -// BUG_ON(s->mpm_stream_pattern_id != sgh->head_array[idx].mpm_stream_pattern_id); -// BUG_ON(s->num != sgh->head_array[idx].num); -// BUG_ON(s != sgh->head_array[idx].full_sig); - idx++; } diff --git a/src/detect.c b/src/detect.c index b3e20e6cf9..42a296c579 100644 --- a/src/detect.c +++ b/src/detect.c @@ -156,6 +156,7 @@ void DetectExitPrintStats(ThreadVars *tv, void *data); void DbgPrintSigs(DetectEngineCtx *, SigGroupHead *); void DbgPrintSigs2(DetectEngineCtx *, SigGroupHead *); +static void PacketCreateMask(Packet *p, SignatureMask *mask, uint16_t alproto, void *alstate, StreamMsg *smsg); /* tm module api functions */ TmEcode Detect(ThreadVars *, Packet *, void *, PacketQueue *, PacketQueue *); @@ -429,17 +430,22 @@ int SigLoadSignatures (DetectEngineCtx *de_ctx, char *sig_file) * \param de_ctx detection engine ctx * \param det_ctx detection engine thread ctx -- array is stored here * \param p packet + * \param mask Packets mask * \param alproto application layer protocol * * Order of SignatureHeader access: - * 1. flags - * 2. alproto - * 3. mpm_pattern_id - * 4. mpm_stream_pattern_id - * 5. num + * 1. mask + * 2. flags + * 3. alproto + * 4. mpm_pattern_id_div8 + * 4. mpm_pattern_id_mod8 + * 5. mpm_stream_pattern_id_div8 + * 5. mpm_stream_pattern_id_mod8 + * 6. num */ static void SigMatchSignaturesBuildMatchArray(DetectEngineCtx *de_ctx, - DetectEngineThreadCtx *det_ctx, Packet *p, uint16_t alproto) + DetectEngineThreadCtx *det_ctx, Packet *p, SignatureMask mask, + uint16_t alproto) { uint32_t i; @@ -449,6 +455,13 @@ static void SigMatchSignaturesBuildMatchArray(DetectEngineCtx *de_ctx, for (i = 0; i < det_ctx->sgh->sig_cnt; i++) { SignatureHeader *s = &det_ctx->sgh->head_array[i]; + if ((mask & s->mask) != s->mask) { + SCLogDebug("Mask mismatch. mask %02X, s->mask %02x, after AND %02x", mask, s->mask, mask & s->mask); + continue; + } + SCLogDebug("Mask match. mask %02X, s->mask %02x, after AND %02x", mask, s->mask, mask & s->mask); + +#if 0 if (!(p->flags & PKT_HAS_FLOW) && s->flags & SIG_FLAG_FLOW) { SCLogDebug("flow in sig but not in packet"); continue; @@ -460,7 +473,7 @@ static void SigMatchSignaturesBuildMatchArray(DetectEngineCtx *de_ctx, SCLogDebug("no payload inspection enabled and sig has payload portion."); continue; } - +#endif /* if the sig has alproto and the session as well they should match */ if (s->flags & SIG_FLAG_APPLAYER && s->alproto != ALPROTO_UNKNOWN && s->alproto != alproto) { if (s->alproto == ALPROTO_DCERPC) { @@ -480,7 +493,7 @@ static void SigMatchSignaturesBuildMatchArray(DetectEngineCtx *de_ctx, * have no matches */ if (!(det_ctx->pmq.pattern_id_bitarray[(s->mpm_pattern_id_div_8)] & s->mpm_pattern_id_mod_8)) { //if (!(det_ctx->pmq.pattern_id_bitarray[(s->mpm_pattern_id / 8)] & (1<<(s->mpm_pattern_id % 8)))) { - SCLogDebug("mpm sig without matches (pat id %"PRIu32" check in content).", s->mpm_pattern_id); + //SCLogDebug("mpm sig without matches (pat id %"PRIu32" check in content).", s->mpm_pattern_id); if (!(s->flags & SIG_FLAG_MPM_NEGCONTENT)) { /* pattern didn't match. There is one case where we will inspect @@ -501,7 +514,7 @@ static void SigMatchSignaturesBuildMatchArray(DetectEngineCtx *de_ctx, /* filter out sigs that want pattern matches, but * have no matches */ if (!(det_ctx->pmq.pattern_id_bitarray[(s->mpm_stream_pattern_id_div_8)] & s->mpm_stream_pattern_id_mod_8)) { - SCLogDebug("mpm stream sig without matches (pat id %"PRIu32" check in content).", s->mpm_stream_pattern_id); + //SCLogDebug("mpm stream sig without matches (pat id %"PRIu32" check in content).", s->mpm_stream_pattern_id); if (!(s->flags & SIG_FLAG_MPM_NEGCONTENT)) { /* pattern didn't match. There is one case where we will inspect @@ -813,6 +826,10 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh det_ctx->sgh = SigMatchSignaturesGetSgh(de_ctx, det_ctx, p); } + /* create our prefilter mask */ + SignatureMask mask = 0; + PacketCreateMask(p, &mask, alproto, alstate, smsg); + /* if we didn't get a sig group head, we * have nothing to do.... */ if (det_ctx->sgh == NULL) { @@ -865,7 +882,7 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh } /* build the match array */ - SigMatchSignaturesBuildMatchArray(de_ctx, det_ctx, p, alproto); + SigMatchSignaturesBuildMatchArray(de_ctx, det_ctx, p, mask, alproto); /* inspect the sigs against the packet */ for (idx = 0; idx < det_ctx->match_array_cnt; idx++) { @@ -873,7 +890,13 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh s = det_ctx->match_array[idx]; SCLogDebug("inspecting signature id %"PRIu32"", s->id); - +#if 0 + if ((mask & s->mask) != s->mask) { + SCLogDebug("Mask mismatch. mask %02X, s->mask %02x, after AND %02x", mask, s->mask, mask & s->mask); + goto next; + } + SCLogDebug("Mask match. mask %02X, s->mask %02x, after AND %02x", mask, s->mask, mask & s->mask); +#endif if (DetectProtoContainsProto(&s->proto, IP_GET_IPPROTO(p)) == 0) { SCLogDebug("proto didn't match"); goto next; @@ -1377,6 +1400,130 @@ deonly: return 1; } +/* Create mask for this packet + it's flow if it has one + * + * Sets SIG_MASK_REQUIRE_PAYLOAD, SIG_MASK_REQUIRE_FLOW, + * SIG_MASK_REQUIRE_HTTP_STATE, SIG_MASK_REQUIRE_DCE_STATE, + * SIG_MASK_REQUIRE_FLOWBIT + */ +static void +PacketCreateMask(Packet *p, SignatureMask *mask, uint16_t alproto, void *alstate, StreamMsg *smsg) { + if (!(p->flags & PKT_NOPAYLOAD_INSPECTION) && (p->payload_len > 0 || smsg != NULL)) { + SCLogDebug("packet has payload"); + (*mask) |= SIG_MASK_REQUIRE_PAYLOAD; + } + + if (p->flags & PKT_HAS_FLOW) { + SCLogDebug("packet has flow"); + (*mask) |= SIG_MASK_REQUIRE_FLOW; + + if (alstate != NULL) { + switch(alproto) { + case ALPROTO_HTTP: + SCLogDebug("packet/flow has http state"); + (*mask) |= SIG_MASK_REQUIRE_HTTP_STATE; + break; + case ALPROTO_SMB: + case ALPROTO_SMB2: + case ALPROTO_DCERPC: + SCLogDebug("packet/flow has dce state"); + (*mask) |= SIG_MASK_REQUIRE_DCE_STATE; + break; + } + } + + SCMutexLock(&p->flow->m); + GenericVar *gv = p->flow->flowvar; + for ( ; gv != NULL; gv = gv->next) { + if (gv->type == DETECT_FLOWBITS) { + SCLogDebug("packet/flow has flowbit(s)"); + (*mask) |= SIG_MASK_REQUIRE_FLOWBIT; + break; + } + } + SCMutexUnlock(&p->flow->m); + } +} + +static int SignatureCreateMask(Signature *s) { + SCEnter(); + + if (s->pmatch != NULL) { + s->mask |= SIG_MASK_REQUIRE_PAYLOAD; + SCLogDebug("sig requires payload"); + } + + if (s->dmatch != NULL) { + s->mask |= SIG_MASK_REQUIRE_DCE_STATE; + SCLogDebug("sig requires dce state"); + } + + if (s->umatch != NULL) { + s->mask |= SIG_MASK_REQUIRE_HTTP_STATE; + SCLogDebug("sig requires dce http state"); + } + + SigMatch *sm; + for (sm = s->amatch ; sm != NULL; sm = sm->next) { + switch(sm->type) { + case DETECT_AL_HTTP_COOKIE: + case DETECT_AL_HTTP_METHOD: + case DETECT_AL_URILEN: + case DETECT_AL_HTTP_CLIENT_BODY: + case DETECT_AL_HTTP_HEADER: + case DETECT_AL_HTTP_URI: + case DETECT_PCRE_HTTPBODY: + case DETECT_PCRE_HTTPCOOKIE: + case DETECT_PCRE_HTTPHEADER: + case DETECT_PCRE_HTTPMETHOD: + s->mask |= SIG_MASK_REQUIRE_HTTP_STATE; + SCLogDebug("sig requires dce http state"); + break; + } + } + + for (sm = s->match ; sm != NULL; sm = sm->next) { + switch(sm->type) { + case DETECT_FLOWBITS: + { + /* figure out what flowbit action */ + DetectFlowbitsData *fb = (DetectFlowbitsData *)sm->ctx; + if (fb->cmd == DETECT_FLOWBITS_CMD_ISSET) { + s->mask |= SIG_MASK_REQUIRE_FLOWBIT; + SCLogDebug("sig requires flowbit(s)"); + } + } + break; + } + } + + if (s->mask & SIG_MASK_REQUIRE_DCE_STATE || + s->mask & SIG_MASK_REQUIRE_HTTP_STATE || + s->mask & SIG_MASK_REQUIRE_FLOWBIT) + { + s->mask |= SIG_MASK_REQUIRE_FLOW; + SCLogDebug("sig requires flow"); + } + + if (s->flags & SIG_FLAG_FLOW) { + s->mask |= SIG_MASK_REQUIRE_FLOW; + SCLogDebug("sig requires flow"); + } + + if (s->amatch != NULL) { + s->mask |= SIG_MASK_REQUIRE_FLOW; + SCLogDebug("sig requires flow"); + } + + if (s->flags & SIG_FLAG_APPLAYER) { + s->mask |= SIG_MASK_REQUIRE_FLOW; + SCLogDebug("sig requires flow"); + } + + SCLogDebug("mask %02X", s->mask); + SCReturnInt(0); +} + /** * \brief Add all signatures to their own source address group * @@ -1490,6 +1637,8 @@ int SigAddressPrepareStage1(DetectEngineCtx *de_ctx) { cnt++; } + SignatureCreateMask(tmp_s); + de_ctx->sig_cnt++; } @@ -4887,8 +5036,10 @@ static int SigTest21Real (int mpm_type) { p1 = UTHBuildPacket((uint8_t *)buf1, buf1len, IPPROTO_TCP); p1->flow = &f; + p1->flags |= PKT_HAS_FLOW; p2 = UTHBuildPacket((uint8_t *)buf2, buf2len, IPPROTO_TCP); p2->flow = &f; + p2->flags |= PKT_HAS_FLOW; DetectEngineCtx *de_ctx = DetectEngineCtxInit(); if (de_ctx == NULL) { @@ -4961,6 +5112,7 @@ static int SigTest22Real (int mpm_type) { p1 = UTHBuildPacket((uint8_t *)buf1, buf1len, IPPROTO_TCP); p1->flow = &f; + p1->flags |= PKT_HAS_FLOW; /* packet 2 */ uint8_t *buf2 = (uint8_t *)"GET /two/ HTTP/1.0\r\n" @@ -4970,6 +5122,7 @@ static int SigTest22Real (int mpm_type) { p2 = UTHBuildPacket((uint8_t *)buf2, buf2len, IPPROTO_TCP); p2->flow = &f; + p2->flags |= PKT_HAS_FLOW; DetectEngineCtx *de_ctx = DetectEngineCtxInit(); if (de_ctx == NULL) { @@ -5045,6 +5198,7 @@ static int SigTest23Real (int mpm_type) { p1 = UTHBuildPacket((uint8_t *)buf1, buf1len, IPPROTO_TCP); p1->flow = &f; + p1->flags |= PKT_HAS_FLOW; /* packet 2 */ uint8_t *buf2 = (uint8_t *)"GET /two/ HTTP/1.0\r\n" @@ -5054,6 +5208,7 @@ static int SigTest23Real (int mpm_type) { p2 = UTHBuildPacket((uint8_t *)buf2, buf2len, IPPROTO_TCP); p2->flow = &f; + p2->flags |= PKT_HAS_FLOW; DetectEngineCtx *de_ctx = DetectEngineCtxInit(); if (de_ctx == NULL) { diff --git a/src/detect.h b/src/detect.h index 6f9401e4e2..66eba04171 100644 --- a/src/detect.h +++ b/src/detect.h @@ -218,6 +218,22 @@ typedef struct DetectPort_ { #define SIG_FLAG_MPM_PACKET 0x00200000 #define SIG_FLAG_MPM_STREAM 0x00400000 +/* signature mask flags */ +#define SIG_MASK_REQUIRE_PAYLOAD 0x01 +#define SIG_MASK_REQUIRE_FLOW 0x02 +//#define SIG_MASK_REQUIRE_PKTVAR 0x04 + +#define SIG_MASK_REQUIRE_FLOWBIT 0x08 +//#define SIG_MASK_REQUIRE_FLOWVAR 0x10 +//#define SIG_MASK_REQUIRE_FLOWINT 0x20 + +#define SIG_MASK_REQUIRE_HTTP_STATE 0x40 +#define SIG_MASK_REQUIRE_DCE_STATE 0x80 + +/* for now a uint8_t is enough */ +#define SignatureMask uint8_t + + /* Detection Engine flags */ #define DE_QUIET 0x01 /**< DE is quiet (esp for unittests) */ @@ -243,26 +259,22 @@ typedef struct SignatureHeader_ { union { struct { uint32_t flags; - /* app layer signature stuff */ - uint16_t alproto; - uint16_t mpm_pattern_id_div_8; + uint8_t mpm_pattern_id_mod_8; + SignatureMask mask; }; - uint64_t hdr_copy; + uint64_t hdr_copy1; }; - - /** pattern in the mpm matcher */ union { struct { - uint8_t mpm_pattern_id_mod_8; - uint8_t pad0; + uint16_t alproto; uint16_t mpm_stream_pattern_id_div_8; uint8_t mpm_stream_pattern_id_mod_8; - uint8_t pad1; SigIntId num; /**< signature number, internal id */ }; - uint64_t mpm_pattern_copy; + uint64_t hdr_copy2; }; + //PatIntId mpm_pattern_id; //PatIntId mpm_stream_pattern_id; @@ -275,27 +287,22 @@ typedef struct Signature_ { union { struct { uint32_t flags; - - /* app layer signature stuff */ - uint16_t alproto; - uint16_t mpm_pattern_id_div_8; + uint8_t mpm_pattern_id_mod_8; + SignatureMask mask; }; - uint64_t hdr_copy; + uint64_t hdr_copy1; }; - - /** pattern in the mpm matcher */ union { struct { - uint8_t mpm_pattern_id_mod_8; - uint8_t pad0; + uint16_t alproto; uint16_t mpm_stream_pattern_id_div_8; uint8_t mpm_stream_pattern_id_mod_8; - uint8_t pad1; SigIntId num; /**< signature number, internal id */ }; - uint64_t mpm_pattern_copy; + uint64_t hdr_copy2; }; + //PatIntId mpm_pattern_id; //PatIntId mpm_stream_pattern_id; @@ -358,9 +365,13 @@ typedef struct Signature_ { uint8_t action; uint8_t rev; + /** classification id **/ uint8_t class; + /* signature match mask */ + //SignatureMask mask; + int prio; uint32_t gid; /**< generator id */