diff --git a/src/Makefile.am b/src/Makefile.am index 974f3da2c7..76cdae49f2 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -19,6 +19,7 @@ flow-var.c flow-var.h \ host.c host.h \ detect.c detect.h \ detect-engine.c detect-engine.h \ +detect-engine-proto.c detect-engine-proto.h \ detect-siggroup.c detect-siggroup.h \ detect-parse.c detect-parse.h \ detect-mpm.c detect-mpm.h \ diff --git a/src/detect-address.c b/src/detect-address.c index 9d903b094b..49781ed5cb 100644 --- a/src/detect-address.c +++ b/src/detect-address.c @@ -38,6 +38,14 @@ static u_int32_t detect_address_group_memory = 0; static u_int32_t detect_address_group_init_cnt = 0; static u_int32_t detect_address_group_free_cnt = 0; +static u_int32_t detect_address_group_head_memory = 0; +static u_int32_t detect_address_group_head_init_cnt = 0; +static u_int32_t detect_address_group_head_free_cnt = 0; + +static u_int32_t detect_address_memory = 0; +static u_int32_t detect_address_init_cnt = 0; +static u_int32_t detect_address_free_cnt = 0; + DetectAddressGroup *DetectAddressGroupInit(void) { DetectAddressGroup *ag = malloc(sizeof(DetectAddressGroup)); if (ag == NULL) { @@ -76,12 +84,25 @@ void DetectAddressGroupFree(DetectAddressGroup *ag) { } void DetectAddressGroupPrintMemory(void) { - printf(" * Address group memory stats:\n"); + printf(" * Address group memory stats (DetectAddressGroup %u):\n", sizeof(DetectAddressGroup)); printf(" - detect_address_group_memory %u\n", detect_address_group_memory); printf(" - detect_address_group_init_cnt %u\n", detect_address_group_init_cnt); printf(" - detect_address_group_free_cnt %u\n", detect_address_group_free_cnt); printf(" - outstanding groups %u\n", detect_address_group_init_cnt - detect_address_group_free_cnt); printf(" * Address group memory stats done\n"); + printf(" * Address group head memory stats (DetectAddressGroupsHead %u):\n", sizeof(DetectAddressGroupsHead)); + printf(" - detect_address_group_head_memory %u\n", detect_address_group_head_memory); + printf(" - detect_address_group_head_init_cnt %u\n", detect_address_group_head_init_cnt); + printf(" - detect_address_group_head_free_cnt %u\n", detect_address_group_head_free_cnt); + printf(" - outstanding groups %u\n", detect_address_group_head_init_cnt - detect_address_group_head_free_cnt); + printf(" * Address group head memory stats done\n"); + printf(" * Address memory stats (DetectAddressData %u):\n", sizeof(DetectAddressData)); + printf(" - detect_address_memory %u\n", detect_address_memory); + printf(" - detect_address_init_cnt %u\n", detect_address_init_cnt); + printf(" - detect_address_free_cnt %u\n", detect_address_free_cnt); + printf(" - outstanding addresses %u\n", detect_address_init_cnt - detect_address_free_cnt); + printf(" * Address memory stats done\n"); + printf(" X Total %u\n", detect_address_group_memory + detect_address_group_head_memory + detect_address_memory); } /* used to see if the exact same address group exists in the list @@ -736,6 +757,9 @@ DetectAddressGroupsHead *DetectAddressGroupsHeadInit(void) { } memset(gh,0,sizeof(DetectAddressGroupsHead)); + detect_address_group_head_init_cnt++; + detect_address_group_head_memory += sizeof(DetectAddressGroupsHead); + return gh; } @@ -752,6 +776,9 @@ void DetectAddressGroupsHeadFree(DetectAddressGroupsHead *gh) { if (gh != NULL) { DetectAddressGroupsHeadCleanup(gh); free(gh); + + detect_address_group_head_free_cnt++; + detect_address_group_head_memory -= sizeof(DetectAddressGroupsHead); } } @@ -1018,6 +1045,9 @@ DetectAddressData *DetectAddressDataInit(void) { } memset(dd,0,sizeof(DetectAddressData)); + detect_address_init_cnt++; + detect_address_memory += sizeof(DetectAddressData); + return dd; error: @@ -1058,6 +1088,9 @@ int DetectAddressSetup (Signature *s, SigMatch *m, char *addressstr) void DetectAddressDataFree(DetectAddressData *dd) { if (dd != NULL) { free(dd); + + detect_address_free_cnt++; + detect_address_memory -= sizeof(DetectAddressData); } } diff --git a/src/detect-engine-proto.c b/src/detect-engine-proto.c new file mode 100644 index 0000000000..7fa94d88bd --- /dev/null +++ b/src/detect-engine-proto.c @@ -0,0 +1,94 @@ +/* Proto part of the detection engine. + * + * Copyright (c) 2008 Victor Julien + * + * TODO move this out of the detection plugin structure */ + +#include "decode.h" +#include "detect.h" +#include "flow-var.h" + +#include "util-cidr.h" +#include "util-unittest.h" + +#include "detect-siggroup.h" + +int DetectProtoSetup (Signature *s, SigMatch *m, char *sidstr); +void DetectProtoTests (void); + +void DetectProtoRegister (void) { + sigmatch_table[DETECT_PROTO].name = "__proto__"; + sigmatch_table[DETECT_PROTO].Match = NULL; + sigmatch_table[DETECT_PROTO].Setup = DetectProtoSetup; + sigmatch_table[DETECT_PROTO].Free = NULL; + sigmatch_table[DETECT_PROTO].RegisterTests = DetectProtoTests; +} + +DetectProto *DetectProtoInit(void) { + DetectProto *dp = malloc(sizeof(DetectProto)); + if (dp == NULL) { + return NULL; + } + memset(dp,0,sizeof(DetectProto)); + + return dp; +} + +/* free a DetectAddressGroup object */ +void DetectProtoFree(DetectProto *dp) { + if (dp == NULL) + return; + + free(dp); +} + +int DetectProtoParse(DetectProto *dp, char *str) { + int proto; + + if (strcasecmp(str,"tcp") == 0) { + proto = IPPROTO_TCP; + dp->proto[(proto/8)] |= 1<<(proto%8); + } else if (strcasecmp(str,"udp") == 0) { + proto = IPPROTO_UDP; + dp->proto[(proto/8)] |= 1<<(proto%8); + } else if (strcasecmp(str,"icmp") == 0) { + proto = IPPROTO_ICMP; + dp->proto[(proto/8)] |= 1<<(proto%8); + } else if (strcasecmp(str,"ip") == 0) { + /* proto is set to 256, a special pseudo proto */ + dp->flags |= DETECT_PROTO_ANY; + memset(&dp->proto,0xFF,sizeof(dp->proto)); + } else { + proto = atoi(str); + dp->proto[(proto/8)] |= 1<<(proto%8); + } + + return 0; +} + +/* XXX remove */ +int DetectProtoSetup (Signature *s, SigMatch *m, char *str) +{ + return 0; +} + +/* TESTS */ + +int ProtoTestParse01 (void) { +/* + DetectProto dp; + + dp = DetectProtoParse("6"); + if (dp) { + DetectProtoFree(dp); + return 1; + } +*/ + return 0; +} + + +void DetectProtoTests(void) { + UtRegisterTest("ProtoTestParse01", ProtoTestParse01, 1); +} + diff --git a/src/detect-engine-proto.h b/src/detect-engine-proto.h new file mode 100644 index 0000000000..53538dd7e6 --- /dev/null +++ b/src/detect-engine-proto.h @@ -0,0 +1,15 @@ +#ifndef __DETECT_PROTO_H__ +#define __DETECT_PROTO_H__ + +#define DETECT_PROTO_ANY 0x1 + +typedef struct DetectProto_ { + u_int8_t proto[32]; /* bitarray 256/8 */ + u_int8_t flags; +} DetectProto; + +/* prototypes */ +void DetectProtoRegister (void); + +#endif /* __DETECT_PROTO_H__ */ + diff --git a/src/detect-parse.c b/src/detect-parse.c index b7dd676350..7d77063914 100644 --- a/src/detect-parse.c +++ b/src/detect-parse.c @@ -258,16 +258,8 @@ error: * */ int SigParseProto(Signature *s, const char *protostr) { - if (strcasecmp(protostr,"tcp") == 0) { - s->ip_proto = 6; - } else if (strcasecmp(protostr,"udp") == 0) { - s->ip_proto = 17; - } else if (strcasecmp(protostr,"icmp") == 0) { - s->ip_proto = 1; - } else if (strcasecmp(protostr,"ip") == 0) { - s->ip_proto = 4; /* XXX VJ does this make sense? */ - } else { - printf("Error: protostr \"%s\" not supported\n", protostr); + int r = DetectProtoParse(&s->proto,protostr); + if (r < 0) { return -1; } diff --git a/src/detect-siggroup.c b/src/detect-siggroup.c index ccb3e29a7f..d6b3bfc641 100644 --- a/src/detect-siggroup.c +++ b/src/detect-siggroup.c @@ -16,6 +16,9 @@ static u_int32_t detect_siggroup_memory = 0; static u_int32_t detect_siggroup_append_cnt = 0; static u_int32_t detect_siggroup_free_cnt = 0; +static u_int32_t detect_siggroup_head_memory = 0; +static u_int32_t detect_siggroup_head_init_cnt = 0; +static u_int32_t detect_siggroup_head_free_cnt = 0; /* XXX eeewww global! move to DetectionEngineCtx once we have that! */ static SigGroupHead *sgh_list = NULL; @@ -134,6 +137,9 @@ int SigGroupAppend(DetectAddressGroup *ag, Signature *s) { goto error; } memset(ag->sh, 0, sizeof(SigGroupHead)); + + detect_siggroup_head_init_cnt++; + detect_siggroup_head_memory += sizeof(SigGroupHead); } if (ag->sh->head == NULL) { @@ -163,15 +169,15 @@ int SigGroupListClean(SigGroupHead *sh) { detect_siggroup_free_cnt++; detect_siggroup_memory -= sizeof(SigGroupContainer); - next_sg = sg->next; + next_sg = sg->next; - sg->s->rulegroup_refcnt--; - sg->s = NULL; - free(sg); + sg->s->rulegroup_refcnt--; + sg->s = NULL; + free(sg); sh->sig_cnt--; - sg = next_sg; + sg = next_sg; } sh->head = NULL; sh->tail = NULL; @@ -192,6 +198,9 @@ int SigGroupListCopyPrepend(DetectAddressGroup *src, DetectAddressGroup *dst) { goto error; } memset(dst->sh, 0, sizeof(SigGroupHead)); + + detect_siggroup_head_init_cnt++; + detect_siggroup_head_memory += sizeof(SigGroupHead); } /* save the head & tail */ @@ -224,6 +233,9 @@ int SigGroupListCopyAppend(DetectAddressGroup *src, DetectAddressGroup *dst) { goto error; } memset(dst->sh, 0, sizeof(SigGroupHead)); + + detect_siggroup_head_init_cnt++; + detect_siggroup_head_memory += sizeof(SigGroupHead); } for (sg = src->sh->head; sg != NULL; sg = sg->next) { @@ -260,15 +272,25 @@ void SigGroupHeadFree(SigGroupHead *sh) { SigGroupListClean(sh); free(sh); + + detect_siggroup_head_free_cnt++; + detect_siggroup_head_memory -= sizeof(SigGroupHead); } void DetectSigGroupPrintMemory(void) { - printf(" * Sig group memory stats:\n"); + printf(" * Sig group memory stats (SigGroupContainer %u):\n", sizeof(SigGroupContainer)); printf(" - detect_siggroup_memory %u\n", detect_siggroup_memory); printf(" - detect_siggroup_append_cnt %u\n", detect_siggroup_append_cnt); printf(" - detect_siggroup_free_cnt %u\n", detect_siggroup_free_cnt); printf(" - outstanding sig containers %u\n", detect_siggroup_append_cnt - detect_siggroup_free_cnt); printf(" * Sig group memory stats done\n"); + printf(" * Sig group head memory stats (SigGroupHead %u):\n", sizeof(SigGroupHead)); + printf(" - detect_siggroup_head_memory %u\n", detect_siggroup_head_memory); + printf(" - detect_siggroup_head_init_cnt %u\n", detect_siggroup_head_init_cnt); + printf(" - detect_siggroup_head_free_cnt %u\n", detect_siggroup_head_free_cnt); + printf(" - outstanding sig containers %u\n", detect_siggroup_head_init_cnt - detect_siggroup_head_free_cnt); + printf(" * Sig group head memory stats done\n"); + printf(" X Total %u\n", detect_siggroup_memory + detect_siggroup_head_memory); } @@ -295,6 +317,8 @@ int SigGroupContentLoad(SigGroupHead *sgh) { SigGroupContainer *sgc = sgh->head; Signature *s; SigMatch *sm; + u_int16_t min_depth = 65535; + u_int16_t min_offset = 65535; if (DetectContentMaxId() == 0) return 0; @@ -320,9 +344,13 @@ int SigGroupContentLoad(SigGroupHead *sgh) { DetectContentData *co = (DetectContentData *)sm->ctx; sgh->content_array[(co->id/8)] |= 1<<(co->id%8); + + if (co->depth < min_depth) min_depth = co->depth; + if (co->offset < min_offset) min_offset = co->offset; } } } + //printf(" * min_depth %u, min_offset %u\n", min_depth, min_offset); return 0; } diff --git a/src/detect.c b/src/detect.c index 791d513630..5359366b10 100644 --- a/src/detect.c +++ b/src/detect.c @@ -12,6 +12,8 @@ #include "detect-siggroup.h" #include "detect-address.h" +#include "detect-engine-proto.h" + #include "detect-content.h" #include "detect-uricontent.h" #include "detect-pcre.h" @@ -42,6 +44,7 @@ #include "util-unittest.h" static DetectEngineCtx *g_de_ctx = NULL; +static u_int32_t mpm_memory_size = 0; SigMatch *SigMatchAlloc(void); void SigMatchFree(SigMatch *sm); @@ -90,19 +93,25 @@ void SigLoadSignatures (void) prevsig->next = sig; prevsig = sig; - sig = SigInit("alert udp any any -> any any (msg:\"ViCtOr nocase test\"; sid:4; rev:13; content:ViCtOr; nocase; depth:150;)"); + sig = SigInit("alert udp any any -> any any (msg:\"ViCtOr nocase test\"; sid:4; rev:13; content:\"ViCtOr\"; nocase; depth:150;)"); if (sig == NULL) return; prevsig->next = sig; prevsig = sig; -/* - sig = SigInit("alert ip any any -> 1.2.3.4 any (msg:\"ViCtOr case test\"; sid:2001; content:ViCtOr; depth:150;)"); + sig = SigInit("alert ip any any -> 1.2.3.4 any (msg:\"ViCtOr case test\"; sid:2001; content:\"ViCtOr\"; depth:150;)"); + if (sig == NULL) + return; + prevsig->next = sig; + prevsig = sig; + + sig = SigInit("alert ip any any -> 1.2.3.4 any (msg:\"IP ONLY\"; sid:2002;)"); if (sig == NULL) return; prevsig->next = sig; prevsig = sig; +/* sig = SigInit("alert ip ANY any -> 192.168.0.0/16 any (msg:\"offset, depth, within test\"; flow:to_client; sid:2002; content:HTTP; depth:4; content:Server:; offset:15; within:100; depth:200;)"); if (sig == NULL) return; @@ -177,13 +186,14 @@ void SigLoadSignatures (void) //#if 0 int good = 0, bad = 0; - FILE *fp = fopen("/etc/vips/rules/bleeding-all.rules", "r"); + //FILE *fp = fopen("/etc/vips/rules/bleeding-all.rules", "r"); //FILE *fp = fopen("/home/victor/rules/vips-http.sigs", "r"); //FILE *fp = fopen("/home/victor/rules/emerging-dshield.rules", "r"); //FILE *fp = fopen("/home/victor/rules/emerging-web-small.rules", "r"); //FILE *fp = fopen("/home/victor/rules/web-misc.rules", "r"); //FILE *fp = fopen("/home/victor/rules/vips-all.sigs", "r"); - //FILE *fp = fopen("/home/victor/rules/all.rules", "r"); + FILE *fp = fopen("/home/victor/rules/all.rules", "r"); + //FILE *fp = fopen("/home/victor/rules/all_noip.rules", "r"); //FILE *fp = fopen("/home/victor/rules/all_iplists.rules", "r"); //FILE *fp = fopen("/home/victor/rules/funky.rules", "r"); //FILE *fp = fopen("/etc/vips/rules/zango.rules", "r"); @@ -248,6 +258,40 @@ int PacketAlertAppend(Packet *p, u_int8_t gid, u_int32_t sid, u_int8_t rev, u_in return 0; } +int SigMatchIPOnlySignatures(ThreadVars *th_v, PatternMatcherThread *pmt, Packet *p) +{ + int fmatch = 0; + Signature *s = NULL; + SigGroupContainer *sg = NULL; + + /* find the right mpm instance */ + DetectAddressGroup *g = DetectAddressLookupGroup(g_de_ctx->io_src_gh,&p->src); + if (g != NULL) { + /* source group found, lets try a dst group */ + g = DetectAddressLookupGroup(g->dst_gh,&p->dst); + } + //printf("SigMatchIPOnlySignatures: g %p\n", g); + + /* no matches, so return */ + if (g == NULL) + return 0; + + /* inspect the sigs against the packet */ + for (sg = g->sh->head; sg != NULL; sg = sg->next) { + s = sg->s; + + fmatch = 1; + + if (!(s->flags & SIG_FLAG_NOALERT)) { + PacketAlertAppend(p, 1, s->id, s->rev, s->prio, s->msg); + + /* set verdict on packet */ + p->action = s->action; + } + } + return fmatch; +} + int SigMatchSignatures(ThreadVars *th_v, PatternMatcherThread *pmt, Packet *p) { int match = 0, fmatch = 0; @@ -255,6 +299,8 @@ int SigMatchSignatures(ThreadVars *th_v, PatternMatcherThread *pmt, Packet *p) SigMatch *sm = NULL; SigGroupContainer *sg = NULL; + SigMatchIPOnlySignatures(th_v,pmt,p); + /* we assume we don't have an uri when we start inspection */ pmt->de_have_httpuri = 0; pmt->de_scanned_httpuri = 0; @@ -387,18 +433,54 @@ void SigCleanSignatures() } } +/* return codes: + * 1: sig is ip only + * 0: sig is not ip only + * + */ +static int SignatureIsIPOnly(Signature *s) { + SigMatch *sm; + + sm = s->match; + if (sm == NULL) + return 1; + + for ( ; sm != NULL; sm = sm->next) { + if (sm->type == DETECT_CONTENT) { + return 0; + } else if (sm->type == DETECT_URICONTENT) { + return 0; + } else if (sm->type == DETECT_PCRE) { + return 0; + } else if (sm->type == DETECT_FLOW) { + return 0; + } else if (sm->type == DETECT_FLOWVAR) { + return 0; + } else if (sm->type == DETECT_DSIZE) { + return 0; + } + } + + return 1; +} /* add all signatures to their own source address group * XXX not currently used */ int SigAddressPrepareStage1(DetectEngineCtx *de_ctx) { Signature *tmp_s = NULL; DetectAddressGroup *gr = NULL; - u_int32_t cnt = 0; + u_int32_t cnt = 0, cnt_iponly = 0; printf("* Building signature grouping structure, stage 1: adding signatures to signature source addresses...\n"); /* now for every rule add the source group */ for (tmp_s = de_ctx->sig_list; tmp_s != NULL; tmp_s = tmp_s->next) { + /* see if the sig is ip only */ + if (SignatureIsIPOnly(tmp_s) == 1) { + tmp_s->flags |= SIG_FLAG_IPONLY; + cnt_iponly++; + } + for (gr = tmp_s->src.ipv4_head; gr != NULL; gr = gr->next) { if (SigGroupAppend(gr,tmp_s) < 0) { goto error; @@ -420,7 +502,7 @@ int SigAddressPrepareStage1(DetectEngineCtx *de_ctx) { de_ctx->sig_cnt++; } //DetectSigGroupPrintMemory(); - printf(" * %u signatures appended %u times to the source addresses\n", de_ctx->sig_cnt, cnt); + printf(" * %u signatures processed. %u are IP-only rules.\n", de_ctx->sig_cnt, cnt_iponly); printf("* Building signature grouping structure, stage 1: adding signatures to signature source addresses... done\n"); return 0; @@ -429,75 +511,76 @@ error: return -1; } -/* fill the global src group head, with the sigs included */ -int SigAddressPrepareStage2(DetectEngineCtx *de_ctx) { - Signature *tmp_s = NULL; - DetectAddressGroup *gr = NULL, *lookup_gr = NULL; - u_int32_t cnt = 0, sigs = 0, insert = 0; +static int BuildSourceAddressList(DetectEngineCtx *de_ctx, Signature *s, int family) { + DetectAddressGroup *gr = NULL, *lookup_gr = NULL, *head = NULL; + int proto; - printf("* Building signature grouping structure, stage 2: building source address list\n"); - - int i; - for (i = 0; i < 256; i++) { - de_ctx->src_gh[i] = DetectAddressGroupsHeadInit(); - if (de_ctx->src_gh[i] == NULL) { - goto error; - } - de_ctx->tmp_gh[i] = DetectAddressGroupsHeadInit(); - if (de_ctx->tmp_gh[i] == NULL) { - goto error; - } + if (family == AF_INET) { + head = s->src.ipv4_head; + } else if (family == AF_INET6) { + head = s->src.ipv6_head; + } else { + head = s->src.any_head; } - /* now for every rule add the source group to our temp list */ - for (tmp_s = de_ctx->sig_list; tmp_s != NULL; tmp_s = tmp_s->next) { - for (gr = tmp_s->src.ipv4_head; gr != NULL; gr = gr->next) { - if ((lookup_gr = DetectAddressGroupLookup(de_ctx->tmp_gh[tmp_s->ip_proto]->ipv4_head,gr->ad)) == NULL) { - DetectAddressGroup *grtmp = DetectAddressGroupInit(); - if (grtmp == NULL) { - goto error; - } - DetectAddressData *adtmp = DetectAddressDataCopy(gr->ad); - if (adtmp == NULL) { - goto error; - } - grtmp->ad = adtmp; - DetectAddressGroupAdd(&de_ctx->tmp_gh[tmp_s->ip_proto]->ipv4_head,grtmp); + /* Normal sigs are added per protocol. For performance reasons we deal with + * ip address only sigs in a different way. */ + if (!(s->flags & SIG_FLAG_IPONLY) || !(s->proto.flags & DETECT_PROTO_ANY)) { + /* for each source address group in the signature... */ + for (gr = head; gr != NULL; gr = gr->next) { + /* ...and each protocol the signature matches on... */ + for (proto = 0; proto < 256; proto++) { + if (s->proto.proto[(proto/8)] & (1<<(proto%8))) { + /* ...see if the group is in the tmp list, and if not add it. */ + if (family == AF_INET) { + lookup_gr = DetectAddressGroupLookup(de_ctx->tmp_gh[proto]->ipv4_head,gr->ad); + } else if (family == AF_INET6) { + lookup_gr = DetectAddressGroupLookup(de_ctx->tmp_gh[proto]->ipv6_head,gr->ad); + } else { + lookup_gr = DetectAddressGroupLookup(de_ctx->tmp_gh[proto]->any_head,gr->ad); + } - SigGroupAppend(grtmp,tmp_s); - cnt++; - insert++; - } else { - /* our group will only have one sig, this one. So add that. */ - SigGroupAppend(lookup_gr,tmp_s); - cnt++; + if (lookup_gr == NULL) { + DetectAddressGroup *grtmp = DetectAddressGroupInit(); + if (grtmp == NULL) { + goto error; + } + DetectAddressData *adtmp = DetectAddressDataCopy(gr->ad); + if (adtmp == NULL) { + goto error; + } + grtmp->ad = adtmp; + + + if (family == AF_INET) { + DetectAddressGroupAdd(&de_ctx->tmp_gh[proto]->ipv4_head, grtmp); + } else if (family == AF_INET6) { + DetectAddressGroupAdd(&de_ctx->tmp_gh[proto]->ipv6_head, grtmp); + } else { + DetectAddressGroupAdd(&de_ctx->tmp_gh[proto]->any_head, grtmp); + } + + SigGroupAppend(grtmp, s); + } else { + /* our group will only have one sig, this one. So add that. */ + SigGroupAppend(lookup_gr, s); + } + } } } - for (gr = tmp_s->src.ipv6_head; gr != NULL; gr = gr->next) { - if ((lookup_gr = DetectAddressGroupLookup(de_ctx->tmp_gh[tmp_s->ip_proto]->ipv6_head,gr->ad)) == NULL) { - DetectAddressGroup *grtmp = DetectAddressGroupInit(); - if (grtmp == NULL) { - goto error; - } - DetectAddressData *adtmp = DetectAddressDataCopy(gr->ad); - if (adtmp == NULL) { - goto error; - } - grtmp->ad = adtmp; - DetectAddressGroupAdd(&de_ctx->tmp_gh[tmp_s->ip_proto]->ipv6_head,grtmp); - - SigGroupAppend(grtmp,tmp_s); - cnt++; - insert++; + } else { + /* for each source address group in the signature... */ + for (gr = head; gr != NULL; gr = gr->next) { + /* ...see if the group is in the tmp list, and if not add it. */ + if (family == AF_INET) { + lookup_gr = DetectAddressGroupLookup(de_ctx->io_tmp_gh->ipv4_head,gr->ad); + } else if (family == AF_INET6) { + lookup_gr = DetectAddressGroupLookup(de_ctx->io_tmp_gh->ipv6_head,gr->ad); } else { - /* our group will only have one sig, this one. So add that. */ - SigGroupAppend(lookup_gr,tmp_s); - cnt++; + lookup_gr = DetectAddressGroupLookup(de_ctx->io_tmp_gh->any_head,gr->ad); } - } - /* XXX review 'any' usage here */ - for (gr = tmp_s->src.any_head; gr != NULL; gr = gr->next) { - if ((lookup_gr = DetectAddressGroupLookup(de_ctx->tmp_gh[tmp_s->ip_proto]->any_head,gr->ad)) == NULL) { + + if (lookup_gr == NULL) { DetectAddressGroup *grtmp = DetectAddressGroupInit(); if (grtmp == NULL) { goto error; @@ -507,17 +590,62 @@ int SigAddressPrepareStage2(DetectEngineCtx *de_ctx) { goto error; } grtmp->ad = adtmp; - DetectAddressGroupAdd(&de_ctx->tmp_gh[tmp_s->ip_proto]->any_head,grtmp); - SigGroupAppend(grtmp,tmp_s); - cnt++; - insert++; + if (family == AF_INET) { + DetectAddressGroupAdd(&de_ctx->io_tmp_gh->ipv4_head, grtmp); + } else if (family == AF_INET6) { + DetectAddressGroupAdd(&de_ctx->io_tmp_gh->ipv6_head, grtmp); + } else { + DetectAddressGroupAdd(&de_ctx->io_tmp_gh->any_head, grtmp); + } + + SigGroupAppend(grtmp, s); } else { /* our group will only have one sig, this one. So add that. */ - SigGroupAppend(lookup_gr,tmp_s); - cnt++; + SigGroupAppend(lookup_gr, s); } } + } + + return 0; +error: + return -1; +} + +/* fill the global src group head, with the sigs included */ +int SigAddressPrepareStage2(DetectEngineCtx *de_ctx) { + Signature *tmp_s = NULL; + DetectAddressGroup *gr = NULL; + u_int32_t cnt = 0, sigs = 0, insert = 0; + + printf("* Building signature grouping structure, stage 2: building source address list...\n"); + + int i; + for (i = 0; i < 256; i++) { + de_ctx->src_gh[i] = DetectAddressGroupsHeadInit(); + if (de_ctx->src_gh[i] == NULL) { + goto error; + } + de_ctx->tmp_gh[i] = DetectAddressGroupsHeadInit(); + if (de_ctx->tmp_gh[i] == NULL) { + goto error; + } + } + /* IP ONLY heads */ + de_ctx->io_src_gh = DetectAddressGroupsHeadInit(); + if (de_ctx->io_src_gh == NULL) { + goto error; + } + de_ctx->io_tmp_gh = DetectAddressGroupsHeadInit(); + if (de_ctx->io_tmp_gh == NULL) { + goto error; + } + + /* now for every rule add the source group to our temp list */ + for (tmp_s = de_ctx->sig_list; tmp_s != NULL; tmp_s = tmp_s->next) { + BuildSourceAddressList(de_ctx,tmp_s,AF_INET); + BuildSourceAddressList(de_ctx,tmp_s,AF_INET6); + BuildSourceAddressList(de_ctx,tmp_s,AF_UNSPEC); sigs++; } @@ -530,7 +658,7 @@ int SigAddressPrepareStage2(DetectEngineCtx *de_ctx) { for (i = 0; i < 256; i++) { for (gr = de_ctx->tmp_gh[i]->ipv4_head; gr != NULL; ) { - //printf("Inserting2'ing: "); DetectAddressDataPrint(gr->ad); + //printf("Inserting2'ing (proto %3d): ",i); DetectAddressDataPrint(gr->ad); printf("\n"); DetectAddressGroup *grnext = gr->next; gr->next = NULL; @@ -540,7 +668,7 @@ int SigAddressPrepareStage2(DetectEngineCtx *de_ctx) { gr = grnext; } for (gr = de_ctx->tmp_gh[i]->ipv6_head; gr != NULL; ) { - //printf("Inserting2'ing: "); DetectAddressDataPrint(gr->ad); + //printf("Inserting2'ing (proto %3d): ",i); DetectAddressDataPrint(gr->ad); printf("\n"); DetectAddressGroup *grnext = gr->next; gr->next = NULL; @@ -552,7 +680,7 @@ int SigAddressPrepareStage2(DetectEngineCtx *de_ctx) { /* XXX whats the point of the any temp list if any is always just * one object.... ??? */ for (gr = de_ctx->tmp_gh[i]->any_head; gr != NULL; ) { - //printf("Inserting2'ing: "); DetectAddressDataPrint(gr->ad); + //printf("Inserting2'ing (proto %3d): ",i); DetectAddressDataPrint(gr->ad); printf("\n"); DetectAddressGroup *grnext = gr->next; gr->next = NULL; @@ -569,6 +697,19 @@ int SigAddressPrepareStage2(DetectEngineCtx *de_ctx) { //printf("g_src_gh[%d] end\n", i); } + /* IP ONLY */ + for (gr = de_ctx->io_tmp_gh->ipv4_head; gr != NULL; ) { + //printf("Inserting2'ing: "); DetectAddressDataPrint(gr->ad); printf("\n"); + DetectAddressGroup *grnext = gr->next; + + gr->next = NULL; + if (DetectAddressGroupInsert(de_ctx->io_src_gh,gr) < 0) + goto error; + + gr = grnext; + } + + //DetectAddressGroupPrintMemory(); @@ -591,9 +732,7 @@ int SigAddressPrepareStage2(DetectEngineCtx *de_ctx) { for (gr = de_ctx->src_gh[6]->ipv6_head; gr != NULL; gr = gr->next) { cnt_ipv6++; } - printf(" * TCP Source any: %u address blocks.\n", cnt_any); - printf(" * TCP Source ipv4: %u address blocks.\n", cnt_ipv4); - printf(" * TCP Source ipv6: %u address blocks.\n", cnt_ipv6); + printf(" * TCP Source address blocks: any: %4u, ipv4: %4u, ipv6: %4u.\n", cnt_any, cnt_ipv4, cnt_ipv6); cnt_any = 0, cnt_ipv4 = 0, cnt_ipv6 = 0; for (gr = de_ctx->src_gh[17]->any_head; gr != NULL; gr = gr->next) { @@ -605,9 +744,7 @@ int SigAddressPrepareStage2(DetectEngineCtx *de_ctx) { for (gr = de_ctx->src_gh[17]->ipv6_head; gr != NULL; gr = gr->next) { cnt_ipv6++; } - printf(" * UDP Source any: %u address blocks.\n", cnt_any); - printf(" * UDP Source ipv4: %u address blocks.\n", cnt_ipv4); - printf(" * UDP Source ipv6: %u address blocks.\n", cnt_ipv6); + printf(" * UDP Source address blocks: any: %4u, ipv4: %4u, ipv6: %4u.\n", cnt_any, cnt_ipv4, cnt_ipv6); cnt_any = 0, cnt_ipv4 = 0, cnt_ipv6 = 0; for (gr = de_ctx->src_gh[1]->any_head; gr != NULL; gr = gr->next) { @@ -619,25 +756,21 @@ int SigAddressPrepareStage2(DetectEngineCtx *de_ctx) { for (gr = de_ctx->src_gh[1]->ipv6_head; gr != NULL; gr = gr->next) { cnt_ipv6++; } - printf(" * ICMP Source any: %u address blocks.\n", cnt_any); - printf(" * ICMP Source ipv4: %u address blocks.\n", cnt_ipv4); - printf(" * ICMP Source ipv6: %u address blocks.\n", cnt_ipv6); + printf(" * ICMP Source address blocks: any: %4u, ipv4: %4u, ipv6: %4u.\n", cnt_any, cnt_ipv4, cnt_ipv6); cnt_any = 0, cnt_ipv4 = 0, cnt_ipv6 = 0; - for (gr = de_ctx->src_gh[4]->any_head; gr != NULL; gr = gr->next) { + for (gr = de_ctx->io_src_gh->any_head; gr != NULL; gr = gr->next) { cnt_any++; } - for (gr = de_ctx->src_gh[4]->ipv4_head; gr != NULL; gr = gr->next) { + for (gr = de_ctx->io_src_gh->ipv4_head; gr != NULL; gr = gr->next) { cnt_ipv4++; } - for (gr = de_ctx->src_gh[4]->ipv6_head; gr != NULL; gr = gr->next) { + for (gr = de_ctx->io_src_gh->ipv6_head; gr != NULL; gr = gr->next) { cnt_ipv6++; } - printf(" * IP Source any: %u address blocks.\n", cnt_any); - printf(" * IP Source ipv4: %u address blocks.\n", cnt_ipv4); - printf(" * IP Source ipv6: %u address blocks.\n", cnt_ipv6); - printf("* Building signature grouping structure, stage 2: building source address list... done\n"); + printf(" * IP-only Source address blocks: any: %4u, ipv4: %4u, ipv6: %4u.\n", cnt_any, cnt_ipv4, cnt_ipv6); + printf("* Building signature grouping structure, stage 2: building source address list... done\n"); return 0; error: printf("SigAddressPrepareStage2 error\n"); @@ -745,6 +878,8 @@ static int BuildDestinationAddressHeads(DetectEngineCtx *de_ctx, DetectAddressGr * a pointer to the earlier one. This saves _a lot_ of memory. */ for (sgr = grdsthead; sgr != NULL; sgr = sgr->next) { + //printf(" * Destination group: "); DetectAddressDataPrint(sgr->ad); printf("\n"); + /* Because a pattern matcher context uses quite some * memory, we first check if we can reuse it from * another group head. */ @@ -795,6 +930,13 @@ static int BuildDestinationAddressHeads(DetectEngineCtx *de_ctx, DetectAddressGr printf("PatternMatchPrepareGroup failed\n"); goto error; } + /* dbg */ + if (!(sgr->sh->flags & SIG_GROUP_HEAD_MPM_COPY) && sgr->sh->mpm_ctx) { + mpm_memory_size += sgr->sh->mpm_ctx->memory_size; + } + if (!(sgr->sh->flags & SIG_GROUP_HEAD_MPM_URI_COPY) && sgr->sh->mpm_uri_ctx) { + mpm_memory_size += sgr->sh->mpm_uri_ctx->memory_size; + } SigGroupHeadAppend(sgr->sh); de_ctx->gh_unique++; @@ -817,7 +959,7 @@ static int BuildDestinationAddressHeads(DetectEngineCtx *de_ctx, DetectAddressGr cnt++; } - printf(" * Source group: "); DetectAddressDataPrint(gr->ad); printf(": %d destination groups.\n", cnt); + //printf(" * Source group: "); DetectAddressDataPrint(gr->ad); printf(": %d destination groups.\n", cnt); cnt = 0; } @@ -849,13 +991,32 @@ int SigAddressPrepareStage3(DetectEngineCtx *de_ctx) { printf ("BuildDestinationAddressHeads(src_gh[%d],AF_UNSPEC) failed\n", i); goto error; } + //printf("Protocol %d, gh: u %u r %u mpm: u %u r %u\n",i, de_ctx->gh_unique, de_ctx->gh_reuse, de_ctx->mpm_unique, de_ctx->mpm_reuse); + } + + /* IP ONLY */ + r = BuildDestinationAddressHeads(de_ctx, de_ctx->io_src_gh,AF_INET); + if (r < 0) { + printf ("BuildDestinationAddressHeads(src_gh[%d],AF_INET) failed\n", i); + goto error; + } + r = BuildDestinationAddressHeads(de_ctx, de_ctx->io_src_gh,AF_INET6); + if (r < 0) { + printf ("BuildDestinationAddressHeads(src_gh[%d],AF_INET6) failed\n", i); + goto error; + } + r = BuildDestinationAddressHeads(de_ctx, de_ctx->io_src_gh,AF_UNSPEC); /* for any */ + if (r < 0) { + printf ("BuildDestinationAddressHeads(src_gh[%d],AF_UNSPEC) failed\n", i); + goto error; } /* cleanup group head (uri)content_array's */ SigGroupHeadFreeMpmArrays(); - //DetectAddressGroupPrintMemory(); - //DetectSigGroupPrintMemory(); + DetectAddressGroupPrintMemory(); + DetectSigGroupPrintMemory(); + //printf("* MPM memory %u\n", mpm_memory_size + ((de_ctx->mpm_unique + de_ctx->mpm_uri_unique) * sizeof(MpmCtx))); printf("* Signature group heads: unique %u, copies %u.\n", de_ctx->gh_unique, de_ctx->gh_reuse); printf("* MPM instances: %u unique, copies %u (none %u).\n", @@ -1164,6 +1325,9 @@ int SigGroupGetSrcAddress(DetectAddressGroupsHead *src) { void SigTableSetup(void) { memset(sigmatch_table, 0, sizeof(sigmatch_table)); + DetectAddressRegister(); + DetectProtoRegister(); + DetectSidRegister(); DetectPriorityRegister(); DetectRevRegister(); @@ -1185,7 +1349,6 @@ void SigTableSetup(void) { DetectFlowRegister(); DetectDsizeRegister(); DetectFlowvarRegister(); - DetectAddressRegister(); DetectNoalertRegister(); u_int8_t i = 0; diff --git a/src/detect.h b/src/detect.h index 88b0906f61..80656fb7b3 100644 --- a/src/detect.h +++ b/src/detect.h @@ -1,6 +1,7 @@ #ifndef __DETECT_H__ #define __DETECT_H__ +#include "detect-engine-proto.h" #include "detect-address.h" #include "detect-content.h" #include "detect-uricontent.h" @@ -9,6 +10,7 @@ #define SIG_FLAG_SP_ANY 0x02 #define SIG_FLAG_DP_ANY 0x04 #define SIG_FLAG_NOALERT 0x08 +#define SIG_FLAG_IPONLY 0x10 /* ip only signature */ typedef struct _PatternMatcherThread { /* detection engine variables */ @@ -48,7 +50,7 @@ typedef struct _Signature { u_int8_t action; DetectAddressGroupsHead src, dst; SigPort sp, dp; - u_int8_t ip_proto; + DetectProto proto; u_int32_t rulegroup_refcnt; struct _SigMatch *match; struct _Signature *next; @@ -75,6 +77,13 @@ typedef struct DetectEngineCtx_ { Signature *sig_list; u_int32_t sig_cnt; + /* ip only sigs: we only add 'alert ip' without + * an ip_proto setting here, so no need to look + * at the proto */ + DetectAddressGroupsHead *io_src_gh; + DetectAddressGroupsHead *io_tmp_gh; + + /* main sigs */ DetectAddressGroupsHead *src_gh[256]; DetectAddressGroupsHead *tmp_gh[256]; @@ -182,6 +191,7 @@ enum { DETECT_DSIZE, DETECT_FLOWVAR, DETECT_ADDRESS, + DETECT_PROTO, DETECT_NOALERT, /* make sure this stays last */ diff --git a/src/util-mpm-wumanber.c b/src/util-mpm-wumanber.c index 54d8392c52..33ef218bb0 100644 --- a/src/util-mpm-wumanber.c +++ b/src/util-mpm-wumanber.c @@ -612,6 +612,9 @@ void WmInitCtx (MpmCtx *mpm_ctx) { return; memset(mpm_ctx->ctx, 0, sizeof(WmCtx)); + + mpm_ctx->memory_cnt++; + mpm_ctx->memory_size += sizeof(WmCtx); } void WmDestroyCtx(MpmCtx *mpm_ctx) {