diff --git a/src/Makefile.am b/src/Makefile.am index 50b1338565..fdc02008d6 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -57,6 +57,7 @@ util-mpm.c util-mpm.h \ util-binsearch.c util-binsearch.h \ util-mpm-wumanber.c util-mpm-wumanber.h \ util-mpm-b2g.c util-mpm-b2g.h \ +util-mpm-b3g.c util-mpm-b3g.h \ util-cidr.c util-cidr.h \ util-unittest.c util-unittest.h \ util-hash.c util-hash.h \ diff --git a/src/detect-engine-mpm.c b/src/detect-engine-mpm.c index a8416a3747..06bc894c1a 100644 --- a/src/detect-engine-mpm.c +++ b/src/detect-engine-mpm.c @@ -18,6 +18,10 @@ #include "detect-content.h" #include "detect-uricontent.h" +//#define PM MPM_WUMANBER +#define PM MPM_B2G +//#define PM MPM_B3G + u_int32_t PacketPatternScan(ThreadVars *t, PatternMatcherThread *pmt, Packet *p) { u_int32_t ret; @@ -64,8 +68,7 @@ void PatternMatchDestroy(MpmCtx *mc) { /* TODO remove this when we move to the rule groups completely */ void PatternMatchPrepare(MpmCtx *mc) { - //MpmInitCtx(mc, MPM_WUMANBER); - MpmInitCtx(mc, MPM_B2G); + MpmInitCtx(mc, PM); } @@ -168,16 +171,14 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh) if (sh->mpm_ctx == NULL) goto error; - //MpmInitCtx(sh->mpm_ctx, MPM_WUMANBER); - MpmInitCtx(sh->mpm_ctx, MPM_B2G); + MpmInitCtx(sh->mpm_ctx, PM); } if (sh->flags & SIG_GROUP_HAVEURICONTENT && !(sh->flags & SIG_GROUP_HEAD_MPM_URI_COPY)) { sh->mpm_uri_ctx = malloc(sizeof(MpmCtx)); if (sh->mpm_uri_ctx == NULL) goto error; - //MpmInitCtx(sh->mpm_uri_ctx, MPM_WUMANBER); - MpmInitCtx(sh->mpm_uri_ctx, MPM_B2G); + MpmInitCtx(sh->mpm_uri_ctx, PM); } u_int16_t mpm_content_scan_maxlen = 65535, mpm_uricontent_scan_maxlen = 65535; diff --git a/src/detect.c b/src/detect.c index de59985fa7..ce4e1cb9c0 100644 --- a/src/detect.c +++ b/src/detect.c @@ -271,8 +271,8 @@ void SigLoadSignatures (void) #define LOADSIGS #ifdef LOADSIGS int good = 0, bad = 0; - //FILE *fp = fopen("/etc/vips/rules/bleeding-all.rules", "r"); - FILE *fp = fopen("/home/victor/rules/all.rules", "r"); + FILE *fp = fopen("/etc/vips/rules/bleeding-all.rules", "r"); + //FILE *fp = fopen("/home/victor/rules/all.rules", "r"); //FILE *fp = fopen("/home/victor/rules/vips-http.sigs", "r"); //FILE *fp = fopen("/home/victor/rules/emerging-dshield.rules", "r"); //FILE *fp = fopen("/home/victor/rules/emerging-web.rules", "r"); @@ -485,7 +485,6 @@ int SigMatchSignatures(ThreadVars *th_v, PatternMatcherThread *pmt, Packet *p) //printf("idx %u, pmt->pmq.sig_id_array_cnt %u, s->id %u (MPM? %s)\n", idx, pmt->pmq.sig_id_array_cnt, s->id, s->flags & SIG_FLAG_MPM ? "TRUE":"FALSE"); //printf("Sig %u\n", s->id); - /* check the source & dst port in the sig */ if (p->proto == IPPROTO_TCP || p->proto == IPPROTO_UDP) { if (!(s->flags & SIG_FLAG_DP_ANY)) { @@ -500,6 +499,7 @@ int SigMatchSignatures(ThreadVars *th_v, PatternMatcherThread *pmt, Packet *p) continue; } } + /* check the source address */ if (!(s->flags & SIG_FLAG_SRC_ANY)) { DetectAddressGroup *saddr = DetectAddressLookupGroup(&s->src,&p->src); @@ -3290,7 +3290,6 @@ int SigTest16 (void) { g_de_ctx->sig_list = SigInit("alert tcp any any -> any !$HTTP_PORTS (msg:\"ET POLICY Inbound HTTP CONNECT Attempt on Off-Port\"; content:\"CONNECT \"; nocase; depth:8; content:\" HTTP/1.\"; nocase; within:1000; classtype:misc-activity; sid:2008284; rev:2;)"); if (g_de_ctx->sig_list == NULL) { - result = 0; goto end; } @@ -3302,7 +3301,7 @@ int SigTest16 (void) { if (PacketAlertCheck(&p, 2008284)) result = 1; else - result = 0; + printf("sid:2008284 %s: ", PacketAlertCheck(&p, 2008284) ? "OK" : "FAIL"); SigGroupCleanup(); SigCleanSignatures(); diff --git a/src/source-nfq.c b/src/source-nfq.c index 312fb29b67..96dfb41cfe 100644 --- a/src/source-nfq.c +++ b/src/source-nfq.c @@ -26,6 +26,8 @@ static NFQGlobalVars nfq_g; static NFQThreadVars nfq_t[NFQ_MAX_QUEUE]; static u_int16_t receive_queue_num = 0; static u_int16_t verdict_queue_num = 0; +static pthread_mutex_t nfq_init_lock; + int ReceiveNFQ(ThreadVars *, Packet *, void *, PacketQueue *); int ReceiveNFQThreadInit(ThreadVars *, void **); @@ -42,6 +44,7 @@ void TmModuleReceiveNFQRegister (void) { /* XXX create a general NFQ setup function */ memset(&nfq_g, 0, sizeof(nfq_g)); memset(&nfq_t, 0, sizeof(nfq_t)); + pthread_mutex_init(&nfq_init_lock, NULL); tmm_modules[TMM_RECEIVENFQ].name = "ReceiveNFQ"; tmm_modules[TMM_RECEIVENFQ].Init = ReceiveNFQThreadInit; @@ -177,11 +180,11 @@ int NFQInitThread(NFQThreadVars *nfq_t, u_int16_t queue_num, u_int32_t queue_max } } - printf("NFQInitThread: binding this socket to queue '%u'\n", queue_num); + printf("NFQInitThread: binding this socket to queue '%u'\n", nfq_t->queue_num); /* pass the thread memory as a void ptr so the * callback function has access to it. */ - nfq_t->qh = nfq_create_queue(nfq_t->h, queue_num, &cb, (void *)nfq_t); + nfq_t->qh = nfq_create_queue(nfq_t->h, nfq_t->queue_num, &cb, (void *)nfq_t); if (nfq_t->qh == NULL) { printf("error during nfq_create_queue()\n"); @@ -229,7 +232,8 @@ int NFQInitThread(NFQThreadVars *nfq_t, u_int16_t queue_num, u_int32_t queue_max } int ReceiveNFQThreadInit(ThreadVars *tv, void **data) { - //printf("ReceiveNFQThreadInit: starting... will bind to queuenum %u\n", receive_queue_num); + mutex_lock(&nfq_init_lock); + printf("ReceiveNFQThreadInit: starting... will bind to queuenum %u\n", receive_queue_num); NFQThreadVars *ntv = &nfq_t[receive_queue_num]; @@ -241,15 +245,18 @@ int ReceiveNFQThreadInit(ThreadVars *tv, void **data) { if (r < 0) { printf("NFQInitThread failed\n"); //return -1; + mutex_unlock(&nfq_init_lock); exit(1); } *data = (void *)ntv; receive_queue_num++; + mutex_unlock(&nfq_init_lock); return 0; } int VerdictNFQThreadInit(ThreadVars *tv, void **data) { + mutex_lock(&nfq_init_lock); printf("VerdictNFQThreadInit: starting... will bind to queuenum %u\n", verdict_queue_num); /* no initialization, ReceiveNFQ takes care of that */ @@ -259,6 +266,7 @@ int VerdictNFQThreadInit(ThreadVars *tv, void **data) { verdict_queue_num++; printf("VerdictNFQThreadInit: ntv %p\n", ntv); + mutex_unlock(&nfq_init_lock); return 0; } @@ -307,6 +315,8 @@ void NFQRecvPkt(NFQThreadVars *t) { int ReceiveNFQ(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq) { NFQThreadVars *ntv = (NFQThreadVars *)data; + //printf("%p receiving on queue %u\n", ntv, ntv->queue_num); + /* XXX can we move this to initialization? */ sigset_t sigs; sigfillset(&sigs); @@ -343,6 +353,8 @@ void NFQSetVerdict(NFQThreadVars *t, Packet *p) { int ret; u_int32_t verdict; + //printf("%p verdicting on queue %u\n", t, t->queue_num); + if (p->action == ACTION_ALERT) { verdict = NF_ACCEPT; } else if (p->action == ACTION_PASS) { diff --git a/src/threadvars.h b/src/threadvars.h index ecf8cd2b20..df5318595b 100644 --- a/src/threadvars.h +++ b/src/threadvars.h @@ -29,6 +29,8 @@ typedef struct _ThreadVars { void *(*tm_func)(void *); void *tm_slots; + char set_cpu_affinity; /* bool: 0 no, 1 yes */ + int cpu_affinity; /* cpu or core to set affinity to */ //#ifdef NFQ // NFQThreadVars *nfq_t; //#endif diff --git a/src/tm-threads.c b/src/tm-threads.c index 7d0d7c7ee8..513dd038ba 100644 --- a/src/tm-threads.c +++ b/src/tm-threads.c @@ -1,11 +1,21 @@ /* Copyright (c) 2008 Victor Julien */ +#include /* for gettid(2) */ +#define _GNU_SOURCE +#define __USE_GNU +#include +#include /* for sched_setaffinity(2) */ + #include "vips.h" #include "threadvars.h" #include "tm-queues.h" #include "tm-queuehandlers.h" #include "tm-modules.h" +/* prototypes */ +static int SetCPUAffinity(int cpu); + + /* root of the threadvars list */ static ThreadVars *tv_root; @@ -53,6 +63,9 @@ void *TmThreadsSlot1NoIn(void *td) { char run = 1; int r = 0; + if (tv->set_cpu_affinity == 1) + SetCPUAffinity(tv->cpu_affinity); + if (s->s.SlotInit != NULL) { r = s->s.SlotInit(tv, &s->s.slot_data); if (r != 0) { @@ -97,6 +110,9 @@ void *TmThreadsSlot1NoOut(void *td) { char run = 1; int r = 0; + if (tv->set_cpu_affinity == 1) + SetCPUAffinity(tv->cpu_affinity); + if (s->s.SlotInit != NULL) { r = s->s.SlotInit(tv, &s->s.slot_data); if (r != 0) { @@ -135,6 +151,9 @@ void *TmThreadsSlot1NoInOut(void *td) { char run = 1; int r = 0; + if (tv->set_cpu_affinity == 1) + SetCPUAffinity(tv->cpu_affinity); + //printf("TmThreadsSlot1NoInOut: %s starting\n", tv->name); if (s->s.SlotInit != NULL) { @@ -178,6 +197,9 @@ void *TmThreadsSlot1(void *td) { char run = 1; int r = 0; + if (tv->set_cpu_affinity == 1) + SetCPUAffinity(tv->cpu_affinity); + //printf("TmThreadsSlot1: %s starting\n", tv->name); if (s->s.SlotInit != NULL) { @@ -237,6 +259,9 @@ void *TmThreadsSlot2(void *td) { char run = 1; int r = 0; + if (tv->set_cpu_affinity == 1) + SetCPUAffinity(tv->cpu_affinity); + //printf("TmThreadsSlot2: %s starting\n", tv->name); if (s->s1.SlotInit != NULL) { @@ -325,6 +350,9 @@ void *TmThreadsSlot3(void *td) { char run = 1; int r = 0; + if (tv->set_cpu_affinity == 1) + SetCPUAffinity(tv->cpu_affinity); + //printf("TmThreadsSlot3: %s starting\n", tv->name); if (s->s1.SlotInit != NULL) { @@ -481,6 +509,9 @@ void *TmThreadsSlotVar(void *td) { int r = 0; TmSlot *slot = NULL; + if (tv->set_cpu_affinity == 1) + SetCPUAffinity(tv->cpu_affinity); + //printf("TmThreadsSlot1: %s starting\n", tv->name); for (slot = s->s; slot != NULL; slot = slot->slot_next) { @@ -672,6 +703,31 @@ void TmVarSlotSetFuncAppend(ThreadVars *tv, TmModule *tm) { } } +/* called from the thread */ +static int SetCPUAffinity(int cpu) { + //pthread_t tid = pthread_self(); + pid_t tid = syscall(SYS_gettid); + cpu_set_t cs; + + printf("Setting CPU Affinity for thread %u to CPU %d\n", tid, cpu); + + CPU_ZERO(&cs); + CPU_SET(cpu,&cs); + + int r = sched_setaffinity(tid,sizeof(cpu_set_t),&cs); + if (r != 0) { + printf("Warning: sched_setaffinity failed (%d): %s\n", r, strerror(errno)); + } + + return 0; +} + +int TmThreadSetCPUAffinity(ThreadVars *tv, int cpu) { + tv->set_cpu_affinity = 1; + tv->cpu_affinity = cpu; + return 0; +} + ThreadVars *TmThreadCreate(char *name, char *inq_name, char *inqh_name, char *outq_name, char *outqh_name, char *slots) { ThreadVars *tv = NULL; Tmq *tmq = NULL; diff --git a/src/tmqh-packetpool.c b/src/tmqh-packetpool.c index 7a78f5ba42..2a8d4aef1b 100644 --- a/src/tmqh-packetpool.c +++ b/src/tmqh-packetpool.c @@ -42,22 +42,22 @@ void TmqhOutputPacketpool(ThreadVars *t, Packet *p) char proot = 0; if (IS_TUNNEL_PKT(p)) { - printf("TmqhOutputPacketpool: tunnel packet: %p %s\n", p,p->root ? "upper layer":"root"); + //printf("TmqhOutputPacketpool: tunnel packet: %p %s\n", p,p->root ? "upper layer":"root"); /* get a lock */ pthread_mutex_t *m = p->root ? &p->root->mutex_rtv_cnt : &p->mutex_rtv_cnt; mutex_lock(m); if (IS_TUNNEL_ROOT_PKT(p)) { - printf("TmqhOutputPacketpool: IS_TUNNEL_ROOT_PKT\n"); + //printf("TmqhOutputPacketpool: IS_TUNNEL_ROOT_PKT\n"); if (TUNNEL_PKT_TPR(p) == 0) { - printf("TmqhOutputPacketpool: TUNNEL_PKT_TPR(p) == 0\n"); + //printf("TmqhOutputPacketpool: TUNNEL_PKT_TPR(p) == 0\n"); /* if this packet is the root and there are no * more tunnel packets, enqueue it */ /* fall through */ } else { - printf("TmqhOutputPacketpool: TUNNEL_PKT_TPR(p) > 0\n"); + //printf("TmqhOutputPacketpool: TUNNEL_PKT_TPR(p) > 0\n"); /* if this is the root and there are more tunnel * packets, don't add this. It's still referenced * by the tunnel packets, and we will enqueue it @@ -67,7 +67,7 @@ void TmqhOutputPacketpool(ThreadVars *t, Packet *p) return; } } else { - printf("TmqhOutputPacketpool: NOT IS_TUNNEL_ROOT_PKT\n"); + //printf("TmqhOutputPacketpool: NOT IS_TUNNEL_ROOT_PKT\n"); if (p->root->tunnel_verdicted == 1 && TUNNEL_PKT_TPR(p) == 1) { //printf("TmqhOutputPacketpool: p->root->tunnel_verdicted == 1 && TUNNEL_PKT_TPR(p) == 1\n"); /* the root is ready and we are the last tunnel packet, @@ -75,19 +75,19 @@ void TmqhOutputPacketpool(ThreadVars *t, Packet *p) TUNNEL_DECR_PKT_TPR_NOLOCK(p); /* handle the root */ - printf("TmqhOutputPacketpool: calling PacketEnqueue for root pkt\n"); + //printf("TmqhOutputPacketpool: calling PacketEnqueue for root pkt\n"); proot = 1; /* fall through */ } else { - printf("TmqhOutputPacketpool: NOT p->root->tunnel_verdicted == 1 && TUNNEL_PKT_TPR(p) == 1 (%u)\n", TUNNEL_PKT_TPR(p)); + //printf("TmqhOutputPacketpool: NOT p->root->tunnel_verdicted == 1 && TUNNEL_PKT_TPR(p) == 1 (%u)\n", TUNNEL_PKT_TPR(p)); TUNNEL_DECR_PKT_TPR_NOLOCK(p); /* fall through */ } } mutex_unlock(m); - printf("TmqhOutputPacketpool: tunnel stuff done, move on\n"); + //printf("TmqhOutputPacketpool: tunnel stuff done, move on\n"); } mutex_lock(&q->mutex_q); diff --git a/src/util-bloomfilter.c b/src/util-bloomfilter.c index 827e46a3cd..1a660509ab 100644 --- a/src/util-bloomfilter.c +++ b/src/util-bloomfilter.c @@ -61,7 +61,7 @@ void BloomFilterFree(BloomFilter *bf) { void BloomFilterPrint(BloomFilter *bf) { printf("\n---------- Bloom Filter Stats -----------\n"); printf("Buckets: %u\n", bf->bitarray_size); - printf("Memory size: %u bytes\n", bf->bitarray_size/8); + printf("Memory size: %u bytes\n", bf->bitarray_size/8 + 1); printf("Hash function pointer: %p\n", bf->Hash); printf("Hash functions: %u\n", bf->hash_iterations); printf("-----------------------------------------\n"); @@ -98,6 +98,13 @@ int BloomFilterTest(BloomFilter *bf, void *data, u_int16_t datalen) { return hit; } +u_int32_t BloomFilterMemorySize(BloomFilter *bf) { + if (bf == NULL) + return 0; + + return (sizeof(BloomFilter) + (bf->bitarray_size/8) + 1); +} + static u_int32_t BloomHash(void *data, u_int16_t datalen, u_int8_t iter, u_int32_t hash_size) { u_int8_t *d = (u_int8_t *)data; u_int32_t i; diff --git a/src/util-bloomfilter.h b/src/util-bloomfilter.h index 6aedaafa46..148ffb0a6d 100644 --- a/src/util-bloomfilter.h +++ b/src/util-bloomfilter.h @@ -17,6 +17,7 @@ void BloomFilterFree(BloomFilter *); void BloomFilterPrint(BloomFilter *); int BloomFilterAdd(BloomFilter *, void *, u_int16_t); int BloomFilterTest(BloomFilter *, void *, u_int16_t); +u_int32_t BloomFilterMemorySize(BloomFilter *); void BloomFilterRegisterTests(void); diff --git a/src/util-mpm-b2g.c b/src/util-mpm-b2g.c index df7baee474..b90210f2b4 100644 --- a/src/util-mpm-b2g.c +++ b/src/util-mpm-b2g.c @@ -47,7 +47,11 @@ int B2gAddPatternCI(MpmCtx *mpm_ctx, u_int8_t *pat, u_int16_t patlen, u_int16_t int B2gAddPatternCS(MpmCtx *mpm_ctx, u_int8_t *pat, u_int16_t patlen, u_int16_t offset, u_int16_t depth, u_int32_t pid, u_int32_t sid); int B2gPreparePatterns(MpmCtx *mpm_ctx); u_int32_t B2gScan1(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *, u_int8_t *buf, u_int16_t buflen); +#ifdef B2G_SCAN2 +u_int32_t B2gScan2(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *, u_int8_t *buf, u_int16_t buflen); +#endif u_int32_t B2gScan(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *, u_int8_t *buf, u_int16_t buflen); +u_int32_t B2gScanBNDMq(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *pmq, u_int8_t *buf, u_int16_t buflen); u_int32_t B2gSearch1(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *, u_int8_t *buf, u_int16_t buflen); u_int32_t B2gSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *, u_int8_t *buf, u_int16_t buflen); void B2gPrintInfo(MpmCtx *mpm_ctx); @@ -65,7 +69,7 @@ void MpmB2gRegister (void) { mpm_table[MPM_B2G].AddPattern = B2gAddPatternCS; mpm_table[MPM_B2G].AddPatternNocase = B2gAddPatternCI; mpm_table[MPM_B2G].Prepare = B2gPreparePatterns; - mpm_table[MPM_B2G].Scan = B2gScan; /* default to B2gSearch. We may fall back to 1 */ + mpm_table[MPM_B2G].Scan = B2G_SCANFUNC; /* default to B2gSearch. We may fall back to 1 */ mpm_table[MPM_B2G].Search = B2gSearch; /* default to B2gSearch. We may fall back to 1 */ mpm_table[MPM_B2G].Cleanup = MpmMatchCleanup; mpm_table[MPM_B2G].PrintCtx = B2gPrintInfo; @@ -327,11 +331,6 @@ static inline int B2gAddPattern(MpmCtx *mpm_ctx, u_int8_t *pat, u_int16_t patlen } } - if (p->len > 1) { - p->prefix_cs = (u_int16_t)(*(p->cs)+*(p->cs+1)); - p->prefix_ci = (u_int16_t)(*(p->ci)+*(p->ci+1)); - } - //printf("B2gAddPattern: ci \""); prt(p->ci,p->len); //printf("\" cs \""); prt(p->cs,p->len); //printf("\" prefix_ci %u, prefix_cs %u\n", p->prefix_ci, p->prefix_cs); @@ -397,18 +396,17 @@ int B2gAddPatternCS(MpmCtx *mpm_ctx, u_int8_t *pat, u_int16_t patlen, return B2gAddPattern(mpm_ctx, pat, patlen, offset, depth, /* nocase */0, /* scan */0, pid, sid); } -static u_int32_t BloomHash(void *data, u_int16_t datalen, u_int8_t iter, u_int32_t hash_size) { +static u_int32_t B2gBloomHash(void *data, u_int16_t datalen, u_int8_t iter, u_int32_t hash_size) { u_int8_t *d = (u_int8_t *)data; - u_int32_t i; - u_int32_t hash = 0; + u_int16_t i; + u_int32_t hash = (u_int32_t)bg_tolower(*d); - for (i = 0; i < datalen; i++) { - if (i == 0) hash += (((u_int32_t)*d++)); - else if (i == 1) hash += (((u_int32_t)*d++) * datalen); - else hash *= (((u_int32_t)*d++) * i); + for (i = 1; i < datalen; i++) { + d++; + hash += (bg_tolower(*d)) ^ i; } + hash <<= (iter+1); - hash *= (iter + datalen); hash %= hash_size; return hash; } @@ -426,6 +424,14 @@ static void B2gPrepareScanHash(MpmCtx *mpm_ctx) { mpm_ctx->memory_cnt++; mpm_ctx->memory_size += (sizeof(B2gHashItem *) * ctx->scan_hash_size); + /* alloc the pminlen array */ + ctx->scan_pminlen = (u_int8_t *)malloc(sizeof(u_int8_t) * ctx->scan_hash_size); + if (ctx->scan_pminlen == NULL) goto error; + memset(ctx->scan_pminlen, 0, sizeof(u_int8_t) * ctx->scan_hash_size); + + mpm_ctx->memory_cnt++; + mpm_ctx->memory_size += (sizeof(u_int8_t) * ctx->scan_hash_size); + for (i = 0; i < mpm_ctx->pattern_cnt; i++) { /* ignore patterns that don't have the scan flag set */ @@ -447,6 +453,25 @@ static void B2gPrepareScanHash(MpmCtx *mpm_ctx) { while (thi->nxt) thi = thi->nxt; thi->nxt = hi; } + ctx->scan_1_pat_cnt++; +#if B2G_SCAN2 + } else if(ctx->parray[i]->len == 2) { + idx = (u_int16_t)(ctx->parray[i]->ci[0] << 8 | ctx->parray[i]->ci[1]); + if (ctx->scan_hash2[idx].flags == 0) { + ctx->scan_hash2[idx].idx = i; + ctx->scan_hash2[idx].flags |= 0x01; + } else { + B2gHashItem *hi = B2gAllocHashItem(mpm_ctx); + hi->idx = i; + hi->flags |= 0x01; + + /* Append this HashItem to the list */ + B2gHashItem *thi = &ctx->scan_hash2[idx]; + while (thi->nxt) thi = thi->nxt; + thi->nxt = hi; + } + ctx->scan_2_pat_cnt++; +#endif } else { idx = B2G_HASH16(ctx->parray[i]->ci[ctx->scan_m - 2], ctx->parray[i]->ci[ctx->scan_m - 1]); //printf("idx %u, %c.%c\n", idx, ctx->parray[i]->ci[ctx->m - 2], ctx->parray[i]->ci[ctx->m - 1]); @@ -455,7 +480,7 @@ static void B2gPrepareScanHash(MpmCtx *mpm_ctx) { B2gHashItem *hi = B2gAllocHashItem(mpm_ctx); hi->idx = i; hi->flags |= 0x01; - hi->p_min_len = ctx->parray[i]->len; + ctx->scan_pminlen[idx] = ctx->parray[i]->len; ctx->scan_hash[idx] = hi; } else { @@ -463,17 +488,19 @@ static void B2gPrepareScanHash(MpmCtx *mpm_ctx) { hi->idx = i; hi->flags |= 0x01; + if (ctx->parray[i]->len < ctx->scan_pminlen[idx]) + ctx->scan_pminlen[idx] = ctx->parray[i]->len; + /* Append this HashItem to the list */ B2gHashItem *thi = ctx->scan_hash[idx]; - if (ctx->parray[i]->len < thi->p_min_len) - thi->p_min_len = ctx->parray[i]->len; - while (thi->nxt) thi = thi->nxt; thi->nxt = hi; } + ctx->scan_x_pat_cnt++; } } + /* alloc the bloom array */ ctx->scan_bloom = (BloomFilter **)malloc(sizeof(BloomFilter *) * ctx->scan_hash_size); if (ctx->scan_bloom == NULL) goto error; memset(ctx->scan_bloom, 0, sizeof(BloomFilter *) * ctx->scan_hash_size); @@ -487,16 +514,19 @@ static void B2gPrepareScanHash(MpmCtx *mpm_ctx) { if (hi == NULL) continue; - ctx->scan_bloom[h] = BloomFilterInit(B2G_BLOOMSIZE, 2, BloomHash); + ctx->scan_bloom[h] = BloomFilterInit(B2G_BLOOMSIZE, 2, B2gBloomHash); if (ctx->scan_bloom[h] == NULL) continue; - if (hi->p_min_len > 8) - hi->p_min_len = 8; + mpm_ctx->memory_cnt+=2; /* hackish: bloomfilter itself and the bitarray */ + mpm_ctx->memory_size += BloomFilterMemorySize(ctx->scan_bloom[h]); + + if (ctx->scan_pminlen[h] > 8) + ctx->scan_pminlen[h] = 8; B2gHashItem *thi = hi; do { - BloomFilterAdd(ctx->scan_bloom[h], ctx->parray[thi->idx]->cs, hi->p_min_len); + BloomFilterAdd(ctx->scan_bloom[h], ctx->parray[thi->idx]->ci, ctx->scan_pminlen[h]); thi = thi->nxt; } while (thi != NULL); } @@ -519,6 +549,14 @@ static void B2gPrepareSearchHash(MpmCtx *mpm_ctx) { mpm_ctx->memory_cnt++; mpm_ctx->memory_size += (sizeof(B2gHashItem *) * ctx->search_hash_size); + /* alloc the pminlen array */ + ctx->search_pminlen = (u_int8_t *)malloc(sizeof(u_int8_t) * ctx->search_hash_size); + if (ctx->search_pminlen == NULL) goto error; + memset(ctx->search_pminlen, 0, sizeof(u_int8_t) * ctx->search_hash_size); + + mpm_ctx->memory_cnt++; + mpm_ctx->memory_size += (sizeof(u_int8_t) * ctx->search_hash_size); + for (i = 0; i < mpm_ctx->pattern_cnt; i++) { /* ignore patterns that have the scan flag set */ @@ -548,6 +586,7 @@ static void B2gPrepareSearchHash(MpmCtx *mpm_ctx) { B2gHashItem *hi = B2gAllocHashItem(mpm_ctx); hi->idx = i; hi->flags |= 0x01; + ctx->search_pminlen[idx] = ctx->parray[i]->len; ctx->search_hash[idx] = hi; } else { @@ -555,6 +594,9 @@ static void B2gPrepareSearchHash(MpmCtx *mpm_ctx) { hi->idx = i; hi->flags |= 0x01; + if (ctx->parray[i]->len < ctx->search_pminlen[idx]) + ctx->search_pminlen[idx] = ctx->parray[i]->len; + /* Append this HashItem to the list */ B2gHashItem *thi = ctx->search_hash[idx]; while (thi->nxt) thi = thi->nxt; @@ -562,6 +604,37 @@ static void B2gPrepareSearchHash(MpmCtx *mpm_ctx) { } } } + + /* alloc the bloom array */ + ctx->search_bloom = (BloomFilter **)malloc(sizeof(BloomFilter *) * ctx->search_hash_size); + if (ctx->search_bloom == NULL) goto error; + memset(ctx->search_bloom, 0, sizeof(BloomFilter *) * ctx->search_hash_size); + + mpm_ctx->memory_cnt++; + mpm_ctx->memory_size += (sizeof(BloomFilter *) * ctx->search_hash_size); + + int h; + for (h = 0; h < ctx->search_hash_size; h++) { + B2gHashItem *hi = ctx->search_hash[h]; + if (hi == NULL) + continue; + + ctx->search_bloom[h] = BloomFilterInit(B2G_BLOOMSIZE, 2, B2gBloomHash); + if (ctx->search_bloom[h] == NULL) + continue; + + mpm_ctx->memory_cnt+=2; /* hackish: bloomfilter itself and the bitarray */ + mpm_ctx->memory_size += BloomFilterMemorySize(ctx->search_bloom[h]); + + if (ctx->search_pminlen[h] > 8) + ctx->search_pminlen[h] = 8; + + B2gHashItem *thi = hi; + do { + BloomFilterAdd(ctx->search_bloom[h], ctx->parray[thi->idx]->ci, ctx->search_pminlen[h]); + thi = thi->nxt; + } while (thi != NULL); + } return; error: return; @@ -592,45 +665,11 @@ int B2gBuildScanMatchArray(MpmCtx *mpm_ctx) { continue; u_int16_t h = B2G_HASH16(bg_tolower(ctx->parray[a]->ci[j]),bg_tolower(ctx->parray[a]->ci[j+1])); - //printf("h %u, %c.%c\n", h, pat[a][j], pat[a][j+1]); ctx->scan_B2G[h] = ctx->scan_B2G[h] | (1 << (ctx->scan_m - j)); } } -#if 0 - /* calculate s0 */ - B2G_TYPE s; - B2G_S0 = m; - - /* look at each pattern */ - for (a = 0; pat[a] != NULL; a++) { - if (strlen(pat) < m) - continue; - - u_int16_t h = B2G_HASH16(bg_tolower(pat[a][m-2]),bg_tolower(pat[a][m-1])); - s = B2G[h]; - printf("S0: h %u, %c.%c\n", h, pat[a][m-2], pat[a][m-1]); - - int i = m - 1; - for ( ; i > 0; i--) { - printf("i: %d, s %8u -- ", i, s); - - if ((s & (1 << (m - 1))) != 0) { - printf(" (s0 update) "); - if (i < B2G_S0) B2G_S0 = i; - } else { - printf(" ( nope ) "); - } - - h = B2G_HASH16(bg_tolower(pat[a][i-1]),bg_tolower(pat[a][i-0])); - printf("S: h %u, %c.%c ", h, pat[a][i-1], pat[a][i-0]); - s = (s << 1) & B2G[h]; - printf("B2G_S0 %d (s %u, b2g[h] %u)\n", B2G_S0, s, B2G[h]); - } - } - B2G_S0--; - printf("B2G_S0 %d\n", B2G_S0); -#endif + ctx->scan_s0 = 1; return 0; } @@ -738,11 +777,6 @@ int B2gPreparePatterns(MpmCtx *mpm_ctx) { mpm_ctx->Search = B2gSearch1; ctx->MBSearch = B2gSearch; } - if (mpm_ctx->scan_minlen == 1) { - mpm_ctx->Scan = B2gScan1; - ctx->MBScan = B2gScan; - } - /* make sure 'm' stays in bounds */ if (ctx->scan_m > B2G_WORD_SIZE) { printf("Warning: 'm' bigger than word size: %u > %u (scan).", ctx->scan_m, B2G_WORD_SIZE); @@ -763,6 +797,21 @@ int B2gPreparePatterns(MpmCtx *mpm_ctx) { B2gBuildScanMatchArray(mpm_ctx); B2gBuildSearchMatchArray(mpm_ctx); + if (ctx->scan_1_pat_cnt) { + mpm_ctx->Scan = B2gScan1; +#ifdef B2G_SCAN2 + if (ctx->scan_2_pat_cnt) { + ctx->MBScan2 = B2gScan2; + } +#endif + ctx->MBScan = B2G_SCANFUNC; +#ifdef B2G_SCAN2 + } else if (ctx->scan_2_pat_cnt) { + mpm_ctx->Scan = B2gScan2; + ctx->MBScan = B2G_SCANFUNC; +#endif + } + return 0; error: return -1; @@ -776,7 +825,7 @@ void B2gPrintSearchStats(MpmThreadCtx *mpm_thread_ctx) { printf("Scan phase:\n"); printf("Total calls/scans: %u\n", tctx->scan_stat_calls); printf("Avg m/scan: %0.2f\n", tctx->scan_stat_calls ? (float)((float)tctx->scan_stat_m_total / (float)tctx->scan_stat_calls) : 0); - printf("D != 0 (possible match, shift = 1): %u\n", tctx->scan_stat_d0); + printf("D != 0 (possible match): %u\n", tctx->scan_stat_d0); printf("Avg hash items per bucket %0.2f (%u)\n", tctx->scan_stat_d0 ? (float)((float)tctx->scan_stat_d0_hashloop / (float)tctx->scan_stat_d0) : 0, tctx->scan_stat_d0_hashloop); printf("Loop match: %u\n", tctx->scan_stat_loop_match); printf("Loop no match: %u\n", tctx->scan_stat_loop_no_match); @@ -928,6 +977,113 @@ void B2gThreadDestroyCtx(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx) { MpmMatchFreeSpares(mpm_thread_ctx, mpm_thread_ctx->qlist); } +u_int32_t B2gScanBNDMq(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *pmq, u_int8_t *buf, u_int16_t buflen) { + B2gCtx *ctx = (B2gCtx *)mpm_ctx->ctx; +#ifdef B2G_COUNTERS + B2gThreadCtx *tctx = (B2gThreadCtx *)mpm_thread_ctx->ctx; +#endif + u_int32_t pos = ctx->scan_m - B2G_Q + 1, matches = 0; + B2G_TYPE d; + + COUNT(tctx->scan_stat_calls++); + COUNT(tctx->scan_stat_m_total+=ctx->scan_m); + + if (buflen < ctx->scan_m) + return 0; + + while (pos <= (buflen - B2G_Q + 1)) { + u_int16_t h = B2G_HASH16(bg_tolower(buf[pos - 1]),bg_tolower(buf[pos])); + d = ctx->scan_B2G[h]; + + if (d != 0) { + COUNT(tctx->scan_stat_d0++); + u_int j = pos; + u_int first = pos - (ctx->scan_m - B2G_Q + 1); + + do { + j = j - 1; + if (d >= (1 << (ctx->scan_m - 1))) { + if (j > first) pos = j; + else { + /* get our patterns from the hash */ + h = B2G_HASH16(bg_tolower(buf[j + ctx->scan_m - 2]),bg_tolower(buf[j + ctx->scan_m - 1])); + + if (ctx->scan_bloom[h] != NULL) { + COUNT(tctx->scan_stat_pminlen_calls++); + COUNT(tctx->scan_stat_pminlen_total+=ctx->scan_pminlen[h]); + + if ((buflen - j) < ctx->scan_pminlen[h]) { + goto skip_loop; + } else { + COUNT(tctx->scan_stat_bloom_calls++); + + if (BloomFilterTest(ctx->scan_bloom[h], buf+j, ctx->scan_pminlen[h]) == 0) { + COUNT(tctx->scan_stat_bloom_hits++); + + //printf("Bloom: %p, buflen %u, pos %u, p_min_len %u\n", ctx->scan_bloom[h], buflen, pos, ctx->scan_pminlen[h]); + goto skip_loop; + } + } + } + + B2gHashItem *hi = ctx->scan_hash[h], *thi; + for (thi = hi; thi != NULL; thi = thi->nxt) { + COUNT(tctx->scan_stat_d0_hashloop++); + B2gPattern *p = ctx->parray[thi->idx]; + + if (p->flags & B2G_NOCASE) { + if (buflen - j < p->len) + continue; + + if (memcmp_lowercase(p->ci, buf+j, p->len) == 0) { + //printf("CI Exact match: "); prt(p->ci, p->len); printf("\n"); + COUNT(tctx->scan_stat_loop_match++); + + MpmEndMatch *em; + for (em = p->em; em; em = em->next) { + //printf("em %p id %u\n", em, em->id); + if (MpmMatchAppend(mpm_thread_ctx, pmq, em, &mpm_thread_ctx->match[em->id], j, p->len)) + matches++; + } + } else { + COUNT(tctx->scan_stat_loop_no_match++); + } + } else { + if (buflen - j < p->len) + continue; + + if (memcmp(p->cs, buf+j, p->len) == 0) { + //printf("CS Exact match: "); prt(p->cs, p->len); printf("\n"); + COUNT(tctx->scan_stat_loop_match++); + + MpmEndMatch *em; + for (em = p->em; em; em = em->next) { + //printf("em %p id %u\n", em, em->id); + if (MpmMatchAppend(mpm_thread_ctx, pmq, em, &mpm_thread_ctx->match[em->id], j, p->len)) + matches++; + } + } else { + COUNT(tctx->scan_stat_loop_no_match++); + } + } + } +skip_loop: + //printf("output at pos %u: ", j); prt(buf + (j), ctx->scan_m); printf("\n"); + ; + } + } + + h = B2G_HASH16(bg_tolower(buf[j - 1]),bg_tolower(buf[j])); + d = (d << 1) & ctx->scan_B2G[h]; + } while (d != 0); + } + COUNT(tctx->scan_stat_num_shift++); + COUNT(tctx->scan_stat_total_shift += (ctx->scan_m - B2G_Q + 1)); + pos = pos + ctx->scan_m - B2G_Q + 1; + } + return matches; +} + u_int32_t B2gScan(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *pmq, u_int8_t *buf, u_int16_t buflen) { B2gCtx *ctx = (B2gCtx *)mpm_ctx->ctx; #ifdef B2G_COUNTERS @@ -949,37 +1105,38 @@ u_int32_t B2gScan(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQ do { u_int16_t h = B2G_HASH16(bg_tolower(buf[pos + j - 1]),bg_tolower(buf[pos + j])); - d &= ctx->scan_B2G[h]; - d <<= 1; + d = ((d << 1) & ctx->scan_B2G[h]); j = j - 1; } while (d != 0 && j != 0); + //printf("scan: d %u, j %u\n", d, j); /* (partial) match, move on to verification */ if (d != 0) { COUNT(tctx->scan_stat_d0++); + //printf("output at pos %u: ", pos); prt(buf + pos, ctx->scan_m); printf("\n"); /* get our patterns from the hash */ u_int16_t h = B2G_HASH16(bg_tolower(buf[pos + ctx->scan_m - 2]),bg_tolower(buf[pos + ctx->scan_m - 1])); - B2gHashItem *hi = ctx->scan_hash[h], *thi; if (ctx->scan_bloom[h] != NULL) { COUNT(tctx->scan_stat_pminlen_calls++); - COUNT(tctx->scan_stat_pminlen_total+=hi->p_min_len); + COUNT(tctx->scan_stat_pminlen_total+=ctx->scan_pminlen[h]); - if ((buflen - pos) < hi->p_min_len) { + if ((buflen - pos) < ctx->scan_pminlen[h]) { goto skip_loop; } else { COUNT(tctx->scan_stat_bloom_calls++); - if (BloomFilterTest(ctx->scan_bloom[h], buf+pos, hi->p_min_len) == 0) { + if (BloomFilterTest(ctx->scan_bloom[h], buf+pos, ctx->scan_pminlen[h]) == 0) { COUNT(tctx->scan_stat_bloom_hits++); - //printf("Bloom: %p, buflen %u, pos %u, p_min_len %u\n", ctx->scan_bloom[h], buflen, pos, hi->p_min_len); + //printf("Bloom: %p, buflen %u, pos %u, p_min_len %u\n", ctx->scan_bloom[h], buflen, pos, ctx->scan_pminlen[h]); goto skip_loop; } } } + B2gHashItem *hi = ctx->scan_hash[h], *thi; for (thi = hi; thi != NULL; thi = thi->nxt) { COUNT(tctx->scan_stat_d0_hashloop++); B2gPattern *p = ctx->parray[thi->idx]; @@ -1021,8 +1178,8 @@ u_int32_t B2gScan(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQ } } skip_loop: - //printf("output at pos %u: ", pos); prt(buf + pos, ctx->scan_m); printf("\n"); - pos = pos + B2G_S0; + //pos = pos + ctx->scan_s0; + pos = pos + 1; } else { COUNT(tctx->scan_stat_num_shift++); COUNT(tctx->scan_stat_total_shift += (j + 1)); @@ -1035,6 +1192,65 @@ skip_loop: return matches; } +#ifdef B2G_SCAN2 +u_int32_t B2gScan2(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *pmq, u_int8_t *buf, u_int16_t buflen) { + B2gCtx *ctx = (B2gCtx *)mpm_ctx->ctx; + u_int8_t *bufmin = buf; + u_int8_t *bufend = buf + buflen - 1; + u_int32_t cnt = 0; + B2gPattern *p; + MpmEndMatch *em; + B2gHashItem *thi, *hi; + + if (buflen < 2) + return 0; + + //printf("BUF "); prt(buf,buflen); printf("\n"); + + while (buf <= bufend) { + u_int16_t h = bg_tolower(*buf) << 8 | bg_tolower(*(buf+1)); + hi = &ctx->scan_hash2[h]; + + if (hi->flags & 0x01) { + for (thi = hi; thi != NULL; thi = thi->nxt) { + p = ctx->parray[thi->idx]; + + if (p->len != 2) + continue; + + if (p->flags & B2G_NOCASE) { + if (bg_tolower(*buf) == p->ci[0] && bg_tolower(*(buf+1)) == p->ci[1]) { + //printf("CI Exact match: "); prt(p->ci, p->len); printf(" in buf "); prt(buf, p->len);printf(" (B2gSearch1)\n"); + for (em = p->em; em; em = em->next) { + if (MpmMatchAppend(mpm_thread_ctx, pmq, em, &mpm_thread_ctx->match[em->id],(buf+1 - bufmin), p->len)) + cnt++; + } + } + } else { + if (*buf == p->cs[0] && *(buf+1) == p->cs[1]) { + //printf("CS Exact match: "); prt(p->cs, p->len); printf(" in buf "); prt(buf, p->len);printf(" (B2gSearch1)\n"); + for (em = p->em; em; em = em->next) { + if (MpmMatchAppend(mpm_thread_ctx, pmq, em, &mpm_thread_ctx->match[em->id],(buf+1 - bufmin), p->len)) + cnt++; + } + } + } + } + } + buf += 1; + } + + //printf("B2gSearch2: after 2byte cnt %u\n", cnt); + if (ctx->scan_x_pat_cnt) { + /* Pass bufmin on because buf no longer points to the + * start of the buffer. */ + cnt += ctx->MBScan(mpm_ctx, mpm_thread_ctx, pmq, bufmin, buflen); + //printf("B2gSearch1: after 2+byte cnt %u\n", cnt); + } + return cnt; +} +#endif + u_int32_t B2gScan1(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *pmq, u_int8_t *buf, u_int16_t buflen) { B2gCtx *ctx = (B2gCtx *)mpm_ctx->ctx; u_int8_t *bufmin = buf; @@ -1049,46 +1265,50 @@ u_int32_t B2gScan1(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcher //printf("BUF "); prt(buf,buflen); printf("\n"); - if (mpm_ctx->scan_minlen == 1) { - while (buf <= bufend) { - u_int8_t h = bg_tolower(*buf); - hi = &ctx->scan_hash1[h]; + while (buf <= bufend) { + u_int8_t h = bg_tolower(*buf); + hi = &ctx->scan_hash1[h]; - if (hi->flags & 0x01) { - for (thi = hi; thi != NULL; thi = thi->nxt) { - p = ctx->parray[thi->idx]; + if (hi->flags & 0x01) { + for (thi = hi; thi != NULL; thi = thi->nxt) { + p = ctx->parray[thi->idx]; - if (p->len != 1) - continue; + if (p->len != 1) + continue; - if (p->flags & B2G_NOCASE) { - if (bg_tolower(*buf) == p->ci[0]) { - //printf("CI Exact match: "); prt(p->ci, p->len); printf(" in buf "); prt(buf, p->len);printf(" (B2gSearch1)\n"); - for (em = p->em; em; em = em->next) { - if (MpmMatchAppend(mpm_thread_ctx, pmq, em, &mpm_thread_ctx->match[em->id],(buf+1 - bufmin), p->len)) - cnt++; - } + if (p->flags & B2G_NOCASE) { + if (bg_tolower(*buf) == p->ci[0]) { + //printf("CI Exact match: "); prt(p->ci, p->len); printf(" in buf "); prt(buf, p->len);printf(" (B2gSearch1)\n"); + for (em = p->em; em; em = em->next) { + if (MpmMatchAppend(mpm_thread_ctx, pmq, em, &mpm_thread_ctx->match[em->id],(buf+1 - bufmin), p->len)) + cnt++; } - } else { - if (*buf == p->cs[0]) { - //printf("CS Exact match: "); prt(p->cs, p->len); printf(" in buf "); prt(buf, p->len);printf(" (B2gSearch1)\n"); - for (em = p->em; em; em = em->next) { - if (MpmMatchAppend(mpm_thread_ctx, pmq, em, &mpm_thread_ctx->match[em->id],(buf+1 - bufmin), p->len)) - cnt++; - } + } + } else { + if (*buf == p->cs[0]) { + //printf("CS Exact match: "); prt(p->cs, p->len); printf(" in buf "); prt(buf, p->len);printf(" (B2gSearch1)\n"); + for (em = p->em; em; em = em->next) { + if (MpmMatchAppend(mpm_thread_ctx, pmq, em, &mpm_thread_ctx->match[em->id],(buf+1 - bufmin), p->len)) + cnt++; } } } } - buf += 1; } + buf += 1; } + //printf("B2gSearch1: after 1byte cnt %u\n", cnt); - if (mpm_ctx->scan_maxlen > 1) { +#ifdef B2G_SCAN2 + if (ctx->scan_2_pat_cnt) { /* Pass bufmin on because buf no longer points to the * start of the buffer. */ - cnt += ctx->MBScan(mpm_ctx, mpm_thread_ctx, pmq, bufmin, buflen); + cnt += ctx->MBScan2(mpm_ctx, mpm_thread_ctx, pmq, bufmin, buflen); //printf("B2gSearch1: after 2+byte cnt %u\n", cnt); + } else +#endif + if (ctx->scan_x_pat_cnt) { + cnt += ctx->MBScan(mpm_ctx, mpm_thread_ctx, pmq, bufmin, buflen); } return cnt; } @@ -1123,6 +1343,25 @@ u_int32_t B2gSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatche /* get our patterns from the hash */ u_int16_t h = B2G_HASH16(bg_tolower(buf[pos + ctx->search_m - 2]),bg_tolower(buf[pos + ctx->search_m - 1])); + + if (ctx->scan_bloom[h] != NULL) { + COUNT(tctx->scan_stat_pminlen_calls++); + COUNT(tctx->scan_stat_pminlen_total+=ctx->scan_pminlen[h]); + + if ((buflen - pos) < ctx->scan_pminlen[h]) { + goto skip_loop; + } else { + COUNT(tctx->scan_stat_bloom_calls++); + + if (BloomFilterTest(ctx->scan_bloom[h], buf+pos, ctx->scan_pminlen[h]) == 0) { + COUNT(tctx->scan_stat_bloom_hits++); + + //printf("Bloom: %p, buflen %u, pos %u, p_min_len %u\n", ctx->scan_bloom[h], buflen, pos, ctx->scan_pminlen[h]); + goto skip_loop; + } + } + } + B2gHashItem *hi = ctx->search_hash[h], *thi; for (thi = hi; thi != NULL; thi = thi->nxt) { B2gPattern *p = ctx->parray[thi->idx]; @@ -1166,7 +1405,8 @@ u_int32_t B2gSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatche } //printf("output at pos %d: ", pos); prt(buf + pos, ctx->search_m); printf("\n"); - pos = pos + B2G_S0; +skip_loop: + pos = pos + 1; } else { COUNT(tctx->search_stat_num_shift++); COUNT(tctx->search_stat_total_shift += (j + 1)); @@ -1259,6 +1499,368 @@ static int B2gTestInit01 (void) { return result; } +static int B2gTestS0Init01 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmInitCtx(&mpm_ctx, MPM_B2G); + B2gCtx *ctx = (B2gCtx *)mpm_ctx.ctx; + + B2gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 0, 0); /* 1 match */ + + B2gPreparePatterns(&mpm_ctx); + + if (ctx->scan_s0 == 4) + result = 1; + else + printf("4 != %u ", ctx->scan_s0); + + B2gDestroyCtx(&mpm_ctx); + return result; +} + +static int B2gTestS0Init02 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmInitCtx(&mpm_ctx, MPM_B2G); + B2gCtx *ctx = (B2gCtx *)mpm_ctx.ctx; + + B2gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 0, 0); /* 1 match */ + B2gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"cdef", 4, 0, 0, 1, 0); /* 1 match */ + + B2gPreparePatterns(&mpm_ctx); + + if (ctx->scan_s0 == 2) + result = 1; + else + printf("2 != %u ", ctx->scan_s0); + + B2gDestroyCtx(&mpm_ctx); + return result; +} + +static int B2gTestS0Init03 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmInitCtx(&mpm_ctx, MPM_B2G); + B2gCtx *ctx = (B2gCtx *)mpm_ctx.ctx; + + B2gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 0, 0); /* 1 match */ + B2gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"bcde", 4, 0, 0, 1, 0); /* 1 match */ + + B2gPreparePatterns(&mpm_ctx); + + if (ctx->scan_s0 == 1) + result = 1; + else + printf("1 != %u ", ctx->scan_s0); + + B2gDestroyCtx(&mpm_ctx); + return result; +} + +static int B2gTestS0Init04 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmInitCtx(&mpm_ctx, MPM_B2G); + B2gCtx *ctx = (B2gCtx *)mpm_ctx.ctx; + + B2gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"abab", 4, 0, 0, 0, 0); /* 1 match */ + + B2gPreparePatterns(&mpm_ctx); + + if (ctx->scan_s0 == 2) + result = 1; + else + printf("2 != %u ", ctx->scan_s0); + + B2gDestroyCtx(&mpm_ctx); + return result; +} + +static int B2gTestS0Init05 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmInitCtx(&mpm_ctx, MPM_B2G); + B2gCtx *ctx = (B2gCtx *)mpm_ctx.ctx; + + B2gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"abcab", 5, 0, 0, 0, 0); /* 1 match */ + + B2gPreparePatterns(&mpm_ctx); + + if (ctx->scan_s0 == 3) + result = 1; + else + printf("3 != %u ", ctx->scan_s0); + + B2gDestroyCtx(&mpm_ctx); + return result; +} + +static int B2gTestScan01 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmThreadCtx mpm_thread_ctx; + MpmInitCtx(&mpm_ctx, MPM_B2G); + + B2gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 0, 0); /* 1 match */ + + B2gPreparePatterns(&mpm_ctx); + B2gThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1 /* 1 pattern */); + + u_int32_t cnt = mpm_ctx.Scan(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"abcdefghjiklmnopqrstuvwxyz", 26); + + MpmMatchCleanup(&mpm_thread_ctx); + + if (cnt == 1) + result = 1; + else + printf("1 != %u ",cnt); + + B2gThreadDestroyCtx(&mpm_ctx, &mpm_thread_ctx); + B2gDestroyCtx(&mpm_ctx); + return result; +} + +static int B2gTestScan02 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmThreadCtx mpm_thread_ctx; + MpmInitCtx(&mpm_ctx, MPM_B2G); + + B2gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"abce", 4, 0, 0, 0, 0); /* 1 match */ + + B2gPreparePatterns(&mpm_ctx); + B2gThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1 /* 1 pattern */); + + u_int32_t cnt = mpm_ctx.Scan(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"abcdefghjiklmnopqrstuvwxyz", 26); + + MpmMatchCleanup(&mpm_thread_ctx); + + if (cnt == 0) + result = 1; + else + printf("0 != %u ",cnt); + + B2gThreadDestroyCtx(&mpm_ctx, &mpm_thread_ctx); + B2gDestroyCtx(&mpm_ctx); + return result; +} + +static int B2gTestScan03 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmThreadCtx mpm_thread_ctx; + MpmInitCtx(&mpm_ctx, MPM_B2G); + + B2gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 0, 0); /* 1 match */ + B2gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"bcde", 4, 0, 0, 1, 0); /* 1 match */ + B2gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"fghj", 4, 0, 0, 2, 0); /* 1 match */ + + B2gPreparePatterns(&mpm_ctx); + B2gThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 3 /* 3 patterns */); + + u_int32_t cnt = mpm_ctx.Scan(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"abcdefghjiklmnopqrstuvwxyz", 26); + + MpmMatchCleanup(&mpm_thread_ctx); + + if (cnt == 3) + result = 1; + else + printf("3 != %u ",cnt); + + B2gThreadDestroyCtx(&mpm_ctx, &mpm_thread_ctx); + B2gDestroyCtx(&mpm_ctx); + return result; +} + +/* test patterns longer than 'm'. M is 4 here. */ +static int B2gTestScan04 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmThreadCtx mpm_thread_ctx; + MpmInitCtx(&mpm_ctx, MPM_B2G); + + B2gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 0, 0); /* 1 match */ + B2gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"bcdegh", 6, 0, 0, 1, 0); /* 1 match */ + B2gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"fghjxyz", 7, 0, 0, 2, 0); /* 1 match */ + + B2gPreparePatterns(&mpm_ctx); + B2gThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 3 /* 3 patterns */); + + u_int32_t cnt = mpm_ctx.Scan(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"abcdefghjiklmnopqrstuvwxyz", 26); + + MpmMatchCleanup(&mpm_thread_ctx); + + if (cnt == 1) + result = 1; + else + printf("1 != %u ",cnt); + + B2gThreadDestroyCtx(&mpm_ctx, &mpm_thread_ctx); + B2gDestroyCtx(&mpm_ctx); + return result; +} + +/* case insensitive test patterns longer than 'm'. M is 4 here. */ +static int B2gTestScan05 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmThreadCtx mpm_thread_ctx; + MpmInitCtx(&mpm_ctx, MPM_B2G); + + B2gAddScanPatternCI(&mpm_ctx, (u_int8_t *)"ABCD", 4, 0, 0, 0, 0); /* 1 match */ + B2gAddScanPatternCI(&mpm_ctx, (u_int8_t *)"bCdEfG", 6, 0, 0, 1, 0); /* 1 match */ + B2gAddScanPatternCI(&mpm_ctx, (u_int8_t *)"fghJikl", 7, 0, 0, 2, 0); /* 1 match */ + + B2gPreparePatterns(&mpm_ctx); + B2gThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 3 /* 3 patterns */); + + u_int32_t cnt = mpm_ctx.Scan(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"abcdefghjiklmnopqrstuvwxyz", 26); + + MpmMatchCleanup(&mpm_thread_ctx); + + if (cnt == 3) + result = 1; + else + printf("3 != %u ",cnt); + + B2gThreadDestroyCtx(&mpm_ctx, &mpm_thread_ctx); + B2gDestroyCtx(&mpm_ctx); + return result; +} + +static int B2gTestScan06 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmThreadCtx mpm_thread_ctx; + MpmInitCtx(&mpm_ctx, MPM_B2G); + + B2gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 0, 0); /* 1 match */ + + B2gPreparePatterns(&mpm_ctx); + B2gThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1 /* 1 pattern */); + + u_int32_t cnt = mpm_ctx.Scan(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"abcd", 4); + + MpmMatchCleanup(&mpm_thread_ctx); + + if (cnt == 1) + result = 1; + else + printf("1 != %u ",cnt); + + B2gThreadDestroyCtx(&mpm_ctx, &mpm_thread_ctx); + B2gDestroyCtx(&mpm_ctx); + return result; +} + +static int B2gTestScan07 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmThreadCtx mpm_thread_ctx; + MpmInitCtx(&mpm_ctx, MPM_B2G); + //B2gCtx *ctx = (B2gCtx *)mpm_ctx.ctx; + + B2gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"A", 1, 0, 0, 0, 0); /* should match 30 times */ + B2gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"AA", 2, 0, 0, 1, 0); /* should match 29 times */ + B2gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"AAA", 3, 0, 0, 2, 0); /* should match 28 times */ + B2gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"AAAAA", 5, 0, 0, 3, 0); /* 26 */ + B2gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"AAAAAAAAAA", 10, 0, 0, 4, 0); /* 21 */ + B2gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", 30, 0, 0, 5, 0); /* 1 */ + /* total matches: 135 */ + + B2gPreparePatterns(&mpm_ctx); + B2gThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 6 /* 6 patterns */); + + u_int32_t cnt = mpm_ctx.Scan(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", 30); + + MpmMatchCleanup(&mpm_thread_ctx); + + if (cnt == 135) + result = 1; + else + printf("135 != %u ",cnt); + + B2gThreadDestroyCtx(&mpm_ctx, &mpm_thread_ctx); + B2gDestroyCtx(&mpm_ctx); + return result; +} + +static int B2gTestScan08 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmThreadCtx mpm_thread_ctx; + MpmInitCtx(&mpm_ctx, MPM_B2G); + + B2gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 0, 0); /* 1 match */ + + B2gPreparePatterns(&mpm_ctx); + B2gThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1 /* 1 pattern */); + + u_int32_t cnt = mpm_ctx.Scan(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"a", 1); + + MpmMatchCleanup(&mpm_thread_ctx); + + if (cnt == 0) + result = 1; + else + printf("0 != %u ",cnt); + + B2gThreadDestroyCtx(&mpm_ctx, &mpm_thread_ctx); + B2gDestroyCtx(&mpm_ctx); + return result; +} + +static int B2gTestScan09 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmThreadCtx mpm_thread_ctx; + MpmInitCtx(&mpm_ctx, MPM_B2G); + + B2gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"ab", 2, 0, 0, 0, 0); /* 1 match */ + + B2gPreparePatterns(&mpm_ctx); + B2gThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1 /* 1 pattern */); + + u_int32_t cnt = mpm_ctx.Scan(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"ab", 2); + + MpmMatchCleanup(&mpm_thread_ctx); + + if (cnt == 1) + result = 1; + else + printf("1 != %u ",cnt); + + B2gThreadDestroyCtx(&mpm_ctx, &mpm_thread_ctx); + B2gDestroyCtx(&mpm_ctx); + return result; +} + +static int B2gTestScan10 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmThreadCtx mpm_thread_ctx; + MpmInitCtx(&mpm_ctx, MPM_B2G); + + B2gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"abcdefgh", 8, 0, 0, 0, 0); /* 1 match */ + + B2gPreparePatterns(&mpm_ctx); + B2gThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1 /* 1 pattern */); + + u_int32_t cnt = mpm_ctx.Scan(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"012345679012345679012345679012345679012345679012345679012345679012345679012345679012345679abcdefgh012345679012345679012345679012345679012345679012345679012345679012345679012345679012345679", 208); + + MpmMatchCleanup(&mpm_thread_ctx); + + if (cnt == 1) + result = 1; + else + printf("1 != %u ",cnt); + + B2gThreadDestroyCtx(&mpm_ctx, &mpm_thread_ctx); + B2gDestroyCtx(&mpm_ctx); + return result; +} + static int B2gTestSearch01 (void) { int result = 0; MpmCtx mpm_ctx; @@ -1422,7 +2024,7 @@ static int B2gTestSearch07 (void) { MpmCtx mpm_ctx; MpmThreadCtx mpm_thread_ctx; MpmInitCtx(&mpm_ctx, MPM_B2G); - B2gCtx *ctx = (B2gCtx *)mpm_ctx.ctx; + //B2gCtx *ctx = (B2gCtx *)mpm_ctx.ctx; B2gAddPatternCS(&mpm_ctx, (u_int8_t *)"A", 1, 0, 0, 0, 0); /* should match 30 times */ B2gAddPatternCS(&mpm_ctx, (u_int8_t *)"AA", 2, 0, 0, 1, 0); /* should match 29 times */ @@ -1526,6 +2128,23 @@ static int B2gTestSearch10 (void) { void B2gRegisterTests(void) { UtRegisterTest("B2gTestInit01", B2gTestInit01, 1); +/* + UtRegisterTest("B2gTestS0Init01", B2gTestS0Init01, 1); + UtRegisterTest("B2gTestS0Init02", B2gTestS0Init02, 1); + UtRegisterTest("B2gTestS0Init03", B2gTestS0Init03, 1); + UtRegisterTest("B2gTestS0Init04", B2gTestS0Init04, 1); + UtRegisterTest("B2gTestS0Init05", B2gTestS0Init05, 1); +*/ + UtRegisterTest("B2gTestScan01", B2gTestScan01, 1); + UtRegisterTest("B2gTestScan02", B2gTestScan02, 1); + UtRegisterTest("B2gTestScan03", B2gTestScan03, 1); + UtRegisterTest("B2gTestScan04", B2gTestScan04, 1); + UtRegisterTest("B2gTestScan05", B2gTestScan05, 1); + UtRegisterTest("B2gTestScan06", B2gTestScan06, 1); + UtRegisterTest("B2gTestScan07", B2gTestScan07, 1); + UtRegisterTest("B2gTestScan08", B2gTestScan08, 1); + UtRegisterTest("B2gTestScan09", B2gTestScan09, 1); + UtRegisterTest("B2gTestScan10", B2gTestScan10, 1); UtRegisterTest("B2gTestSearch01", B2gTestSearch01, 1); UtRegisterTest("B2gTestSearch02", B2gTestSearch02, 1); diff --git a/src/util-mpm-b2g.h b/src/util-mpm-b2g.h index 1f0a1cc38d..fee75542f1 100644 --- a/src/util-mpm-b2g.h +++ b/src/util-mpm-b2g.h @@ -8,22 +8,34 @@ #define B2G_SCAN 0x02 //#define B2G_HASHSIZE 65536 +//#define B2G_HASHSIZE 32768 #define B2G_HASHSIZE 16384 +//#define B2G_HASHSIZE 8192 +//#define B2G_HASHSIZE 4096 +//#define B2G_HASHSIZE 2048 + //#define B2G_HASHSHIFT 8 +//#define B2G_HASHSHIFT 7 #define B2G_HASHSHIFT 6 +//#define B2G_HASHSHIFT 5 +//#define B2G_HASHSHIFT 4 +//#define B2G_HASHSHIFT 3 + #define B2G_TYPE u_int32_t //#define B2G_TYPE u_int16_t //#define B2G_TYPE u_int8_t //#define B2G_WORD_SIZE 16 //#define B2G_WORD_SIZE 8 #define B2G_WORD_SIZE 32 -static int B2G_S0 = 1; -#define B2G_BLOOMSIZE 512 +#define B2G_BLOOMSIZE 1024 #define B2G_HASH16(a,b) (((a)< + * + * Ideas: + * - B3g does a full match in the scan phase of up to 'm' characters, + * in case of a case insensitive search we could say it's match if + * the pattern is of len 'm' or just compare the rest of the chars. + * + * TODO: + * - Try to get the S0 calculation right. + * + */ + +#include +#include +#include +#include +#include + +#include "util-bloomfilter.h" +#include "util-mpm-b3g.h" + +#include "util-unittest.h" + +/* uppercase to lowercase conversion lookup table */ +static u_int8_t lowercasetable[256]; +/* marco to do the actual lookup */ +#define bg_tolower(c) lowercasetable[(c)] + +#define INIT_HASH_SIZE 65536 + +#ifdef B3G_COUNTERS +#define COUNT(counter) \ + (counter) +#else +#define COUNT(counter) +#endif /* B3G_COUNTERS */ + +void B3gInitCtx (MpmCtx *mpm_ctx); +void B3gThreadInitCtx(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, u_int32_t); +void B3gDestroyCtx(MpmCtx *mpm_ctx); +void B3gThreadDestroyCtx(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx); +int B3gAddScanPatternCI(MpmCtx *mpm_ctx, u_int8_t *pat, u_int16_t patlen, u_int16_t offset, u_int16_t depth, u_int32_t pid, u_int32_t sid); +int B3gAddScanPatternCS(MpmCtx *mpm_ctx, u_int8_t *pat, u_int16_t patlen, u_int16_t offset, u_int16_t depth, u_int32_t pid, u_int32_t sid); +int B3gAddPatternCI(MpmCtx *mpm_ctx, u_int8_t *pat, u_int16_t patlen, u_int16_t offset, u_int16_t depth, u_int32_t pid, u_int32_t sid); +int B3gAddPatternCS(MpmCtx *mpm_ctx, u_int8_t *pat, u_int16_t patlen, u_int16_t offset, u_int16_t depth, u_int32_t pid, u_int32_t sid); +int B3gPreparePatterns(MpmCtx *mpm_ctx); +u_int32_t B3gScan1(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *, u_int8_t *buf, u_int16_t buflen); +u_int32_t B3gScan2(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *, u_int8_t *buf, u_int16_t buflen); +u_int32_t B3gScan(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *, u_int8_t *buf, u_int16_t buflen); +u_int32_t B3gScanBNDMq(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *pmq, u_int8_t *buf, u_int16_t buflen); +u_int32_t B3gSearch1(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *, u_int8_t *buf, u_int16_t buflen); +u_int32_t B3gSearch2(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *, u_int8_t *buf, u_int16_t buflen); +u_int32_t B3gSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *, u_int8_t *buf, u_int16_t buflen); +void B3gPrintInfo(MpmCtx *mpm_ctx); +void B3gPrintSearchStats(MpmThreadCtx *mpm_thread_ctx); +void B3gRegisterTests(void); + +void MpmB3gRegister (void) { + mpm_table[MPM_B3G].name = "b2g"; + mpm_table[MPM_B3G].InitCtx = B3gInitCtx; + mpm_table[MPM_B3G].InitThreadCtx = B3gThreadInitCtx; + mpm_table[MPM_B3G].DestroyCtx = B3gDestroyCtx; + mpm_table[MPM_B3G].DestroyThreadCtx = B3gThreadDestroyCtx; + mpm_table[MPM_B3G].AddScanPattern = B3gAddScanPatternCS; + mpm_table[MPM_B3G].AddScanPatternNocase = B3gAddScanPatternCI; + mpm_table[MPM_B3G].AddPattern = B3gAddPatternCS; + mpm_table[MPM_B3G].AddPatternNocase = B3gAddPatternCI; + mpm_table[MPM_B3G].Prepare = B3gPreparePatterns; + mpm_table[MPM_B3G].Scan = B3G_SCANFUNC; /* default to B3gSearch. We may fall back to 1 */ + mpm_table[MPM_B3G].Search = B3gSearch; /* default to B3gSearch. We may fall back to 1 */ + mpm_table[MPM_B3G].Cleanup = MpmMatchCleanup; + mpm_table[MPM_B3G].PrintCtx = B3gPrintInfo; + mpm_table[MPM_B3G].PrintThreadCtx = B3gPrintSearchStats; + mpm_table[MPM_B3G].RegisterUnittests = B3gRegisterTests; + + /* create table for O(1) lowercase conversion lookup */ + u_int8_t c = 0; + for ( ; c < 255; c++) { + if (c >= 'A' && c <= 'Z') + lowercasetable[c] = (c + ('a' - 'A')); + else + lowercasetable[c] = c; + } +} + +/* append an endmatch to a pattern + * + * Only used in the initialization phase */ +static inline void B3gEndMatchAppend(MpmCtx *mpm_ctx, B3gPattern *p, + u_int16_t offset, u_int16_t depth, u_int32_t pid, u_int32_t sid) +{ + MpmEndMatch *em = MpmAllocEndMatch(mpm_ctx); + if (em == NULL) { + printf("ERROR: B3gAllocEndMatch failed\n"); + return; + } + + em->id = pid; + em->sig_id = sid; + em->depth = depth; + em->offset = offset; + + if (p->em == NULL) { + p->em = em; + return; + } + + MpmEndMatch *m = p->em; + while (m->next) { + m = m->next; + } + m->next = em; +} + +static void prt (u_int8_t *buf, u_int16_t buflen) { + u_int16_t i; + + for (i = 0; i < buflen; i++) { + if (isprint(buf[i])) printf("%c", buf[i]); + else printf("\\x%X", buf[i]); + } + //printf("\n"); +} + +void B3gPrintInfo(MpmCtx *mpm_ctx) { + B3gCtx *ctx = (B3gCtx *)mpm_ctx->ctx; + + printf("MPM B3g Information:\n"); + printf("Memory allocs: %u\n", mpm_ctx->memory_cnt); + printf("Memory alloced: %u\n", mpm_ctx->memory_size); + printf(" Sizeofs:\n"); + printf(" MpmCtx %u\n", sizeof(MpmCtx)); + printf(" B3gCtx: %u\n", sizeof(B3gCtx)); + printf(" B3gPattern %u\n", sizeof(B3gPattern)); + printf(" B3gHashItem %u\n", sizeof(B3gHashItem)); + printf("Unique Patterns: %u\n", mpm_ctx->pattern_cnt); + printf("Scan Patterns: %u\n", mpm_ctx->scan_pattern_cnt); + printf("Total Patterns: %u\n", mpm_ctx->total_pattern_cnt); + printf("Smallest: %u\n", mpm_ctx->scan_minlen); + printf("Largest: %u\n", mpm_ctx->scan_maxlen); + printf("Hash size: %u\n", ctx->scan_hash_size); + printf("\n"); +} + +static inline B3gPattern *B3gAllocPattern(MpmCtx *mpm_ctx) { + B3gPattern *p = malloc(sizeof(B3gPattern)); + if (p == NULL) { + printf("ERROR: B3gAllocPattern: malloc failed\n"); + } + memset(p,0,sizeof(B3gPattern)); + + mpm_ctx->memory_cnt++; + mpm_ctx->memory_size += sizeof(B3gPattern); + return p; +} + +static inline B3gHashItem * +B3gAllocHashItem(MpmCtx *mpm_ctx) { + B3gHashItem *hi = malloc(sizeof(B3gHashItem)); + if (hi == NULL) { + printf("ERROR: B3gAllocHashItem: malloc failed\n"); + } + memset(hi,0,sizeof(B3gHashItem)); + + mpm_ctx->memory_cnt++; + mpm_ctx->memory_size += sizeof(B3gHashItem); + return hi; +} + +static inline void memcpy_tolower(u_int8_t *d, u_int8_t *s, u_int16_t len) { + u_int16_t i; + for (i = 0; i < len; i++) { + d[i] = bg_tolower(s[i]); + } +} + +/* + * INIT HASH START + */ +static inline u_int32_t B3gInitHash(B3gPattern *p) { + u_int32_t hash = p->len * p->cs[0]; + if (p->len > 1) + hash += p->cs[1]; + + return (hash % INIT_HASH_SIZE); +} + +static inline u_int32_t B3gInitHashRaw(u_int8_t *pat, u_int16_t patlen) { + u_int32_t hash = patlen * pat[0]; + if (patlen > 1) + hash += pat[1]; + + return (hash % INIT_HASH_SIZE); +} + +static inline int B3gInitHashAdd(B3gCtx *ctx, B3gPattern *p) { + u_int32_t hash = B3gInitHash(p); + + //printf("B3gInitHashAdd: %u\n", hash); + + if (ctx->init_hash[hash] == NULL) { + ctx->init_hash[hash] = p; + //printf("B3gInitHashAdd: hash %u, head %p\n", hash, ctx->init_hash[hash]); + return 0; + } + + B3gPattern *t = ctx->init_hash[hash], *tt = NULL; + for ( ; t != NULL; t = t->next) { + tt = t; + } + tt->next = p; + //printf("B3gInitHashAdd: hash %u, head %p\n", hash, ctx->init_hash[hash]); + + return 0; +} + +static inline int B3gCmpPattern(B3gPattern *p, u_int8_t *pat, u_int16_t patlen, char nocase); + +static inline B3gPattern *B3gInitHashLookup(B3gCtx *ctx, u_int8_t *pat, u_int16_t patlen, char nocase) { + u_int32_t hash = B3gInitHashRaw(pat,patlen); + + //printf("B3gInitHashLookup: %u, head %p\n", hash, ctx->init_hash[hash]); + + if (ctx->init_hash[hash] == NULL) { + return NULL; + } + + B3gPattern *t = ctx->init_hash[hash]; + for ( ; t != NULL; t = t->next) { + if (B3gCmpPattern(t,pat,patlen,nocase) == 1) + return t; + } + + return NULL; +} + +static inline int B3gCmpPattern(B3gPattern *p, u_int8_t *pat, u_int16_t patlen, char nocase) { + if (p->len != patlen) + return 0; + + if (!((nocase && p->flags & B3G_NOCASE) || (!nocase && !(p->flags & B3G_NOCASE)))) + return 0; + + if (memcmp(p->cs, pat, patlen) != 0) + return 0; + + return 1; +} + +/* + * INIT HASH END + */ + +void B3gFreePattern(MpmCtx *mpm_ctx, B3gPattern *p) { + if (p && p->em) { + MpmEndMatchFreeAll(mpm_ctx, p->em); + } + + if (p && p->cs && p->cs != p->ci) { + free(p->cs); + mpm_ctx->memory_cnt--; + mpm_ctx->memory_size -= p->len; + } + + if (p && p->ci) { + free(p->ci); + mpm_ctx->memory_cnt--; + mpm_ctx->memory_size -= p->len; + } + + if (p) { + free(p); + mpm_ctx->memory_cnt--; + mpm_ctx->memory_size -= sizeof(B3gPattern); + } +} + +/* B3gAddPattern + * + * pat: ptr to the pattern + * patlen: length of the pattern + * nocase: nocase flag: 1 enabled, 0 disable + * pid: pattern id + * sid: signature id (internal id) + */ +static inline int B3gAddPattern(MpmCtx *mpm_ctx, u_int8_t *pat, u_int16_t patlen, u_int16_t offset, u_int16_t depth, char nocase, char scan, u_int32_t pid, u_int32_t sid) { + B3gCtx *ctx = (B3gCtx *)mpm_ctx->ctx; + +// printf("B3gAddPattern: ctx %p \"", mpm_ctx); prt(pat, patlen); +// printf("\" id %u, nocase %s\n", id, nocase ? "true" : "false"); + + if (patlen == 0) + return 0; + + /* get a memory piece */ + B3gPattern *p = B3gInitHashLookup(ctx, pat, patlen, nocase); + if (p == NULL) { +// printf("B3gAddPattern: allocing new pattern\n"); + p = B3gAllocPattern(mpm_ctx); + if (p == NULL) + goto error; + + p->len = patlen; + + if (nocase) p->flags |= B3G_NOCASE; + + /* setup the case insensitive part of the pattern */ + p->ci = malloc(patlen); + if (p->ci == NULL) goto error; + mpm_ctx->memory_cnt++; + mpm_ctx->memory_size += patlen; + memcpy_tolower(p->ci, pat, patlen); + + /* setup the case sensitive part of the pattern */ + if (p->flags & B3G_NOCASE) { + /* nocase means no difference between cs and ci */ + p->cs = p->ci; + } else { + if (memcmp(p->ci,pat,p->len) == 0) { + /* no diff between cs and ci: pat is lowercase */ + p->cs = p->ci; + } else { + p->cs = malloc(patlen); + if (p->cs == NULL) goto error; + mpm_ctx->memory_cnt++; + mpm_ctx->memory_size += patlen; + memcpy(p->cs, pat, patlen); + } + } + + //printf("B3gAddPattern: ci \""); prt(p->ci,p->len); + //printf("\" cs \""); prt(p->cs,p->len); + //printf("\" prefix_ci %u, prefix_cs %u\n", p->prefix_ci, p->prefix_cs); + + /* put in the pattern hash */ + B3gInitHashAdd(ctx, p); + + if (mpm_ctx->pattern_cnt == 65535) { + printf("Max search words reached\n"); + exit(1); + } + mpm_ctx->pattern_cnt++; + + if (scan) { /* SCAN */ + if (mpm_ctx->scan_maxlen < patlen) mpm_ctx->scan_maxlen = patlen; + if (mpm_ctx->pattern_cnt == 1) mpm_ctx->scan_minlen = patlen; + else if (mpm_ctx->scan_minlen > patlen) mpm_ctx->scan_minlen = patlen; + p->flags |= B3G_SCAN; + } else { /* SEARCH */ + if (mpm_ctx->search_maxlen < patlen) mpm_ctx->search_maxlen = patlen; + if (mpm_ctx->pattern_cnt == 1) mpm_ctx->search_minlen = patlen; + else if (mpm_ctx->search_minlen > patlen) mpm_ctx->search_minlen = patlen; + } + } + + /* we need a match */ + B3gEndMatchAppend(mpm_ctx, p, offset, depth, pid, sid); + + /* keep track of highest pattern id XXX still used? */ + if (pid > mpm_ctx->max_pattern_id) + mpm_ctx->max_pattern_id = pid; + + mpm_ctx->total_pattern_cnt++; + + return 0; + +error: + B3gFreePattern(mpm_ctx, p); + return -1; +} + +int B3gAddScanPatternCI(MpmCtx *mpm_ctx, u_int8_t *pat, u_int16_t patlen, + u_int16_t offset, u_int16_t depth, u_int32_t pid, u_int32_t sid) +{ + return B3gAddPattern(mpm_ctx, pat, patlen, offset, depth, /* nocase */1, /* scan */1, pid, sid); +} + +int B3gAddScanPatternCS(MpmCtx *mpm_ctx, u_int8_t *pat, u_int16_t patlen, + u_int16_t offset, u_int16_t depth, u_int32_t pid, u_int32_t sid) +{ + return B3gAddPattern(mpm_ctx, pat, patlen, offset, depth, /* nocase */0, /* scan */1, pid, sid); +} + +int B3gAddPatternCI(MpmCtx *mpm_ctx, u_int8_t *pat, u_int16_t patlen, + u_int16_t offset, u_int16_t depth, u_int32_t pid, u_int32_t sid) +{ + return B3gAddPattern(mpm_ctx, pat, patlen, offset, depth, /* nocase */1, /* scan */0, pid, sid); +} + +int B3gAddPatternCS(MpmCtx *mpm_ctx, u_int8_t *pat, u_int16_t patlen, + u_int16_t offset, u_int16_t depth, u_int32_t pid, u_int32_t sid) +{ + return B3gAddPattern(mpm_ctx, pat, patlen, offset, depth, /* nocase */0, /* scan */0, pid, sid); +} + +static u_int32_t B3gBloomHash(void *data, u_int16_t datalen, u_int8_t iter, u_int32_t hash_size) { + u_int8_t *d = (u_int8_t *)data; + u_int16_t i; + u_int32_t hash = (u_int32_t)bg_tolower(*d); + + for (i = 1; i < datalen; i++) { + d++; + hash += (bg_tolower(*d)) ^ i; + } + hash <<= (iter+1); + + hash %= hash_size; + return hash; +} + +static void B3gPrepareScanHash(MpmCtx *mpm_ctx) { + B3gCtx *ctx = (B3gCtx *)mpm_ctx->ctx; + u_int16_t i; + u_int16_t idx = 0; + u_int8_t idx8 = 0; + + ctx->scan_hash = (B3gHashItem **)malloc(sizeof(B3gHashItem *) * ctx->scan_hash_size); + if (ctx->scan_hash == NULL) goto error; + memset(ctx->scan_hash, 0, sizeof(B3gHashItem *) * ctx->scan_hash_size); + + mpm_ctx->memory_cnt++; + mpm_ctx->memory_size += (sizeof(B3gHashItem *) * ctx->scan_hash_size); + + /* alloc the pminlen array */ + ctx->scan_pminlen = (u_int8_t *)malloc(sizeof(u_int8_t) * ctx->scan_hash_size); + if (ctx->scan_pminlen == NULL) goto error; + memset(ctx->scan_pminlen, 0, sizeof(u_int8_t) * ctx->scan_hash_size); + + mpm_ctx->memory_cnt++; + mpm_ctx->memory_size += (sizeof(u_int8_t) * ctx->scan_hash_size); + + for (i = 0; i < mpm_ctx->pattern_cnt; i++) + { + /* ignore patterns that don't have the scan flag set */ + if (!(ctx->parray[i]->flags & B3G_SCAN)) + continue; + + if(ctx->parray[i]->len == 1) { + idx8 = (u_int8_t)ctx->parray[i]->ci[0]; + if (ctx->scan_hash1[idx8].flags == 0) { + ctx->scan_hash1[idx8].idx = i; + ctx->scan_hash1[idx8].flags |= 0x01; + } else { + B3gHashItem *hi = B3gAllocHashItem(mpm_ctx); + hi->idx = i; + hi->flags |= 0x01; + + /* Append this HashItem to the list */ + B3gHashItem *thi = &ctx->scan_hash1[idx8]; + while (thi->nxt) thi = thi->nxt; + thi->nxt = hi; + } + ctx->scan_1_pat_cnt++; + } else if(ctx->parray[i]->len == 2) { + idx = (u_int16_t)(ctx->parray[i]->ci[0] << 8 | ctx->parray[i]->ci[1]); + if (ctx->scan_hash2[idx].flags == 0) { + ctx->scan_hash2[idx].idx = i; + ctx->scan_hash2[idx].flags |= 0x01; + } else { + B3gHashItem *hi = B3gAllocHashItem(mpm_ctx); + hi->idx = i; + hi->flags |= 0x01; + + /* Append this HashItem to the list */ + B3gHashItem *thi = &ctx->scan_hash2[idx]; + while (thi->nxt) thi = thi->nxt; + thi->nxt = hi; + } + ctx->scan_2_pat_cnt++; + } else { + idx = B3G_HASH(ctx->parray[i]->ci[ctx->scan_m - 3], ctx->parray[i]->ci[ctx->scan_m - 2], ctx->parray[i]->ci[ctx->scan_m - 1]); + //printf("idx %u, %c.%c.%c\n", idx, ctx->parray[i]->ci[ctx->scan_m - 3], ctx->parray[i]->ci[ctx->scan_m - 2], ctx->parray[i]->ci[ctx->scan_m - 1]); + + if (ctx->scan_hash[idx] == NULL) { + B3gHashItem *hi = B3gAllocHashItem(mpm_ctx); + hi->idx = i; + hi->flags |= 0x01; + ctx->scan_pminlen[idx] = ctx->parray[i]->len; + + ctx->scan_hash[idx] = hi; + } else { + B3gHashItem *hi = B3gAllocHashItem(mpm_ctx); + hi->idx = i; + hi->flags |= 0x01; + + if (ctx->parray[i]->len < ctx->scan_pminlen[idx]) + ctx->scan_pminlen[idx] = ctx->parray[i]->len; + + /* Append this HashItem to the list */ + B3gHashItem *thi = ctx->scan_hash[idx]; + while (thi->nxt) thi = thi->nxt; + thi->nxt = hi; + } + ctx->scan_x_pat_cnt++; + } + } + + /* alloc the bloom array */ + ctx->scan_bloom = (BloomFilter **)malloc(sizeof(BloomFilter *) * ctx->scan_hash_size); + if (ctx->scan_bloom == NULL) goto error; + memset(ctx->scan_bloom, 0, sizeof(BloomFilter *) * ctx->scan_hash_size); + + mpm_ctx->memory_cnt++; + mpm_ctx->memory_size += (sizeof(BloomFilter *) * ctx->scan_hash_size); + + int h; + for (h = 0; h < ctx->scan_hash_size; h++) { + B3gHashItem *hi = ctx->scan_hash[h]; + if (hi == NULL) + continue; + + ctx->scan_bloom[h] = BloomFilterInit(B3G_BLOOMSIZE, 2, B3gBloomHash); + if (ctx->scan_bloom[h] == NULL) + continue; + + mpm_ctx->memory_cnt+=2; /* hackish: bloomfilter itself and the bitarray */ + mpm_ctx->memory_size += BloomFilterMemorySize(ctx->scan_bloom[h]); + + if (ctx->scan_pminlen[h] > 8) + ctx->scan_pminlen[h] = 8; + + B3gHashItem *thi = hi; + do { + BloomFilterAdd(ctx->scan_bloom[h], ctx->parray[thi->idx]->ci, ctx->scan_pminlen[h]); + thi = thi->nxt; + } while (thi != NULL); + } + + return; +error: + return; +} + +static void B3gPrepareSearchHash(MpmCtx *mpm_ctx) { + B3gCtx *ctx = (B3gCtx *)mpm_ctx->ctx; + u_int16_t i; + u_int16_t idx = 0; + u_int8_t idx8 = 0; + + ctx->search_hash = (B3gHashItem **)malloc(sizeof(B3gHashItem *) * ctx->search_hash_size); + if (ctx->search_hash == NULL) goto error; + memset(ctx->search_hash, 0, sizeof(B3gHashItem *) * ctx->search_hash_size); + + mpm_ctx->memory_cnt++; + mpm_ctx->memory_size += (sizeof(B3gHashItem *) * ctx->search_hash_size); + + for (i = 0; i < mpm_ctx->pattern_cnt; i++) + { + /* ignore patterns that have the scan flag set */ + if (ctx->parray[i]->flags & B3G_SCAN) + continue; + + if(ctx->parray[i]->len == 1) { + idx8 = (u_int8_t)ctx->parray[i]->ci[0]; + if (ctx->search_hash1[idx8].flags == 0) { + ctx->search_hash1[idx8].idx = i; + ctx->search_hash1[idx8].flags |= 0x01; + } else { + B3gHashItem *hi = B3gAllocHashItem(mpm_ctx); + hi->idx = i; + hi->flags |= 0x01; + + /* Append this HashItem to the list */ + B3gHashItem *thi = &ctx->search_hash1[idx8]; + while (thi->nxt) thi = thi->nxt; + thi->nxt = hi; + } + ctx->search_1_pat_cnt++; + } else if(ctx->parray[i]->len == 2) { + idx = (u_int16_t)(ctx->parray[i]->ci[0] << 8 | ctx->parray[i]->ci[1]); + if (ctx->search_hash2[idx].flags == 0) { + ctx->search_hash2[idx].idx = i; + ctx->search_hash2[idx].flags |= 0x01; + } else { + B3gHashItem *hi = B3gAllocHashItem(mpm_ctx); + hi->idx = i; + hi->flags |= 0x01; + + /* Append this HashItem to the list */ + B3gHashItem *thi = &ctx->search_hash2[idx]; + while (thi->nxt) thi = thi->nxt; + thi->nxt = hi; + } + ctx->search_2_pat_cnt++; + } else { + idx = B3G_HASH(ctx->parray[i]->ci[ctx->search_m - 3], ctx->parray[i]->ci[ctx->search_m - 2], ctx->parray[i]->ci[ctx->search_m - 1]); + //printf("idx %u, %c.%c\n", idx, ctx->parray[i]->ci[ctx->search_m - 2], ctx->parray[i]->ci[ctx->search_m - 1]); + + if (ctx->search_hash[idx] == NULL) { + B3gHashItem *hi = B3gAllocHashItem(mpm_ctx); + hi->idx = i; + hi->flags |= 0x01; + + ctx->search_hash[idx] = hi; + } else { + B3gHashItem *hi = B3gAllocHashItem(mpm_ctx); + hi->idx = i; + hi->flags |= 0x01; + + /* Append this HashItem to the list */ + B3gHashItem *thi = ctx->search_hash[idx]; + while (thi->nxt) thi = thi->nxt; + thi->nxt = hi; + } + ctx->search_x_pat_cnt++; + } + } + return; +error: + return; +} + +int B3gBuildScanMatchArray(MpmCtx *mpm_ctx) { + B3gCtx *ctx = (B3gCtx *)mpm_ctx->ctx; + + ctx->scan_B3G = malloc(sizeof(B3G_TYPE) * ctx->scan_hash_size); + if (ctx->scan_B3G == NULL) + return -1; + + mpm_ctx->memory_cnt++; + mpm_ctx->memory_size += (sizeof(B3G_TYPE) * ctx->scan_hash_size); + + memset(ctx->scan_B3G,0, B3G_HASHSIZE * sizeof(B3G_TYPE)); + + u_int j; + int a; + + /* fill the match array */ + for (j = 0; j <= (ctx->scan_m - B3G_Q); j++) { + for (a = 0; a < mpm_ctx->pattern_cnt; a++) { + if (!(ctx->parray[a]->flags & B3G_SCAN)) + continue; + + if (ctx->parray[a]->len < ctx->scan_m) + continue; + + u_int16_t h = B3G_HASH(bg_tolower(ctx->parray[a]->ci[j]),bg_tolower(ctx->parray[a]->ci[j+1]), bg_tolower(ctx->parray[a]->ci[j+2])); +//printf("B3gBuildScanMatchArray: h %u, %c.%c.%c\n", h, bg_tolower(ctx->parray[a]->ci[j]),bg_tolower(ctx->parray[a]->ci[j+1]), bg_tolower(ctx->parray[a]->ci[j+2])); + ctx->scan_B3G[h] = ctx->scan_B3G[h] | (1 << (ctx->scan_m - j)); + } + } + + ctx->scan_s0 = 1; + return 0; +} + +int B3gBuildSearchMatchArray(MpmCtx *mpm_ctx) { + B3gCtx *ctx = (B3gCtx *)mpm_ctx->ctx; + + ctx->search_B3G = malloc(sizeof(B3G_TYPE) * ctx->search_hash_size); + if (ctx->search_B3G == NULL) + return -1; + + mpm_ctx->memory_cnt++; + mpm_ctx->memory_size += (sizeof(B3G_TYPE) * ctx->search_hash_size); + + memset(ctx->search_B3G,0, B3G_HASHSIZE * sizeof(B3G_TYPE)); + + u_int j; + int a; + + /* fill the match array */ + for (j = 0; j <= (ctx->search_m - B3G_Q); j++) { + for (a = 0; a < mpm_ctx->pattern_cnt; a++) { + if (ctx->parray[a]->flags & B3G_SCAN) + continue; + + if (ctx->parray[a]->len < ctx->search_m) + continue; + + u_int16_t h = B3G_HASH(bg_tolower(ctx->parray[a]->ci[j]),bg_tolower(ctx->parray[a]->ci[j+1]), bg_tolower(ctx->parray[a]->ci[j+2])); + //printf("h %u, %c.%c\n", h, bg_tolower(ctx->parray[a]->ci[j]),bg_tolower(ctx->parray[a]->ci[j+1])); + ctx->search_B3G[h] = ctx->search_B3G[h] | (1 << (ctx->search_m - j)); + } + } + +#if 0 + /* calculate s0 */ + B3G_TYPE s; + B3G_S0 = m; + + /* look at each pattern */ + for (a = 0; pat[a] != NULL; a++) { + if (strlen(pat) < m) + continue; + + u_int16_t h = B3G_HASH16(bg_tolower(pat[a][m-2]),bg_tolower(pat[a][m-1])); + s = B3G[h]; + printf("S0: h %u, %c.%c\n", h, pat[a][m-2], pat[a][m-1]); + + int i = m - 1; + for ( ; i > 0; i--) { + printf("i: %d, s %8u -- ", i, s); + + if ((s & (1 << (m - 1))) != 0) { + printf(" (s0 update) "); + if (i < B3G_S0) B3G_S0 = i; + } else { + printf(" ( nope ) "); + } + + h = B3G_HASH16(bg_tolower(pat[a][i-1]),bg_tolower(pat[a][i-0])); + printf("S: h %u, %c.%c ", h, pat[a][i-1], pat[a][i-0]); + s = (s << 1) & B3G[h]; + printf("B3G_S0 %d (s %u, b2g[h] %u)\n", B3G_S0, s, B3G[h]); + } + } + B3G_S0--; + printf("B3G_S0 %d\n", B3G_S0); +#endif + return 0; +} + +int B3gPreparePatterns(MpmCtx *mpm_ctx) { + B3gCtx *ctx = (B3gCtx *)mpm_ctx->ctx; + + /* alloc the pattern array */ + ctx->parray = (B3gPattern **)malloc(mpm_ctx->pattern_cnt * sizeof(B3gPattern *)); + if (ctx->parray == NULL) goto error; + memset(ctx->parray, 0, mpm_ctx->pattern_cnt * sizeof(B3gPattern *)); + //printf("mpm_ctx %p, parray %p\n", mpm_ctx,ctx->parray); + mpm_ctx->memory_cnt++; + mpm_ctx->memory_size += (mpm_ctx->pattern_cnt * sizeof(B3gPattern *)); + + /* populate it with the patterns in the hash */ + u_int32_t i = 0, p = 0; + for (i = 0; i < INIT_HASH_SIZE; i++) { + B3gPattern *node = ctx->init_hash[i], *nnode = NULL; + for ( ; node != NULL; ) { + nnode = node->next; + node->next = NULL; + + ctx->parray[p] = node; + + p++; + node = nnode; + } + } + /* we no longer need the hash, so free it's memory */ + free(ctx->init_hash); + ctx->init_hash = NULL; + + /* set 'm' to the smallest pattern size */ + ctx->scan_m = mpm_ctx->scan_minlen; + ctx->search_m = mpm_ctx->search_minlen; + + /* make sure 'm' stays in bounds */ + if (ctx->scan_m > B3G_WORD_SIZE) { + printf("Warning: 'm' bigger than word size: %u > %u (scan).", ctx->scan_m, B3G_WORD_SIZE); + ctx->scan_m = B3G_WORD_SIZE; + } + if (ctx->scan_m < 3) ctx->scan_m = 3; + + if (ctx->search_m > B3G_WORD_SIZE) { + printf("Warning: 'm' bigger than word size: %u > %u (search).", ctx->search_m, B3G_WORD_SIZE); + ctx->search_m = B3G_WORD_SIZE; + } + if (ctx->search_m < 3) ctx->search_m = 3; + + ctx->scan_hash_size = B3G_HASHSIZE; + ctx->search_hash_size = B3G_HASHSIZE; + B3gPrepareScanHash(mpm_ctx); + B3gPrepareSearchHash(mpm_ctx); + B3gBuildScanMatchArray(mpm_ctx); + B3gBuildSearchMatchArray(mpm_ctx); + + if (ctx->scan_1_pat_cnt) { + mpm_ctx->Scan = B3gScan1; + if (ctx->scan_2_pat_cnt) { + ctx->MBScan2 = B3gScan2; + } + ctx->MBScan = B3G_SCANFUNC; + } else if (ctx->scan_2_pat_cnt) { + mpm_ctx->Scan = B3gScan2; + ctx->MBScan = B3G_SCANFUNC; + } + + if (ctx->search_1_pat_cnt) { + mpm_ctx->Search = B3gSearch1; + if (ctx->search_2_pat_cnt) { + ctx->MBSearch2 = B3gSearch2; + } + ctx->MBSearch = B3gSearch; + } else if (ctx->search_2_pat_cnt) { + mpm_ctx->Search = B3gSearch2; + ctx->MBSearch = B3gSearch; + } + + return 0; +error: + return -1; +} + +void B3gPrintSearchStats(MpmThreadCtx *mpm_thread_ctx) { +#ifdef B3G_COUNTERS + B3gThreadCtx *tctx = (B3gThreadCtx *)mpm_thread_ctx->ctx; + + printf("B3g Thread Search stats (tctx %p)\n", tctx); + printf("Scan phase:\n"); + printf("Total calls/scans: %u\n", tctx->scan_stat_calls); + printf("Avg m/scan: %0.2f\n", tctx->scan_stat_calls ? (float)((float)tctx->scan_stat_m_total / (float)tctx->scan_stat_calls) : 0); + printf("D != 0 (possible match): %u\n", tctx->scan_stat_d0); + printf("Avg hash items per bucket %0.2f (%u)\n", tctx->scan_stat_d0 ? (float)((float)tctx->scan_stat_d0_hashloop / (float)tctx->scan_stat_d0) : 0, tctx->scan_stat_d0_hashloop); + printf("Loop match: %u\n", tctx->scan_stat_loop_match); + printf("Loop no match: %u\n", tctx->scan_stat_loop_no_match); + printf("Num shifts: %u\n", tctx->scan_stat_num_shift); + printf("Total shifts: %u\n", tctx->scan_stat_total_shift); + printf("Avg shifts: %0.2f\n", tctx->scan_stat_num_shift ? (float)((float)tctx->scan_stat_total_shift / (float)tctx->scan_stat_num_shift) : 0); + printf("Total BloomFilter checks: %u\n", tctx->scan_stat_bloom_calls); + printf("BloomFilter hits: %0.4f%% (%u)\n", tctx->scan_stat_bloom_calls ? (float)((float)((float)tctx->scan_stat_bloom_hits / (float)tctx->scan_stat_bloom_calls)*(float)100) : 0, tctx->scan_stat_bloom_hits); + printf("Avg pminlen: %0.2f\n\n", tctx->scan_stat_pminlen_calls ? (float)((float)tctx->scan_stat_pminlen_total / (float)tctx->scan_stat_pminlen_calls) : 0); + + printf("Search phase:\n"); + printf("D 0 (possible match, shift = 1): %u\n", tctx->search_stat_d0); + printf("Loop match: %u\n", tctx->search_stat_loop_match); + printf("Loop no match: %u\n", tctx->search_stat_loop_no_match); + printf("Num shifts: %u\n", tctx->search_stat_num_shift); + printf("Total shifts: %u\n", tctx->search_stat_total_shift); + printf("Avg shifts: %0.2f\n\n", tctx->search_stat_num_shift ? (float)((float)tctx->search_stat_total_shift / (float)tctx->search_stat_num_shift) : 0); +#endif /* B3G_COUNTERS */ +} + +static inline int +memcmp_lowercase(u_int8_t *s1, u_int8_t *s2, u_int16_t n) { + size_t i; + + /* check backwards because we already tested the first + * 2 to 4 chars. This way we are more likely to detect + * a miss and thus speed up a little... */ + for (i = n - 1; i; i--) { + if (bg_tolower(*(s2+i)) != s1[i]) + return 1; + } + + return 0; +} + +void B3gInitCtx (MpmCtx *mpm_ctx) { + //printf("B3gInitCtx: mpm_ctx %p\n", mpm_ctx); + + memset(mpm_ctx, 0, sizeof(MpmCtx)); + + mpm_ctx->ctx = malloc(sizeof(B3gCtx)); + if (mpm_ctx->ctx == NULL) + return; + + memset(mpm_ctx->ctx, 0, sizeof(B3gCtx)); + + mpm_ctx->memory_cnt++; + mpm_ctx->memory_size += sizeof(B3gCtx); + + /* initialize the hash we use to speed up pattern insertions */ + B3gCtx *ctx = (B3gCtx *)mpm_ctx->ctx; + ctx->init_hash = malloc(sizeof(B3gPattern *) * INIT_HASH_SIZE); + if (ctx->init_hash == NULL) + return; + + memset(ctx->init_hash, 0, sizeof(B3gPattern *) * INIT_HASH_SIZE); +} + +void B3gDestroyCtx(MpmCtx *mpm_ctx) { + B3gCtx *ctx = (B3gCtx *)mpm_ctx->ctx; + if (ctx == NULL) + return; + + if (ctx->init_hash) { + free(ctx->init_hash); + mpm_ctx->memory_cnt--; + mpm_ctx->memory_size -= (INIT_HASH_SIZE * sizeof(B3gPattern *)); + } + + if (ctx->parray) { + u_int32_t i; + for (i = 0; i < mpm_ctx->pattern_cnt; i++) { + if (ctx->parray[i] != NULL) { + B3gFreePattern(mpm_ctx, ctx->parray[i]); + } + } + + free(ctx->parray); + mpm_ctx->memory_cnt--; + mpm_ctx->memory_size -= (mpm_ctx->pattern_cnt * sizeof(B3gPattern)); + } + + if (ctx->scan_hash) { + free(ctx->scan_hash); + mpm_ctx->memory_cnt--; + mpm_ctx->memory_size -= (sizeof(B3gHashItem) * ctx->scan_hash_size); + } + + if (ctx->search_hash) { + free(ctx->search_hash); + mpm_ctx->memory_cnt--; + mpm_ctx->memory_size -= (sizeof(B3gHashItem) * ctx->search_hash_size); + } + + free(mpm_ctx->ctx); + mpm_ctx->memory_cnt--; + mpm_ctx->memory_size -= sizeof(B3gCtx); +} + +void B3gThreadInitCtx(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, u_int32_t matchsize) { + memset(mpm_thread_ctx, 0, sizeof(MpmThreadCtx)); + + mpm_thread_ctx->ctx = malloc(sizeof(B3gThreadCtx)); + if (mpm_thread_ctx->ctx == NULL) + return; + + memset(mpm_thread_ctx->ctx, 0, sizeof(B3gThreadCtx)); + + mpm_thread_ctx->memory_cnt++; + mpm_thread_ctx->memory_size += sizeof(B3gThreadCtx); + + /* alloc an array with the size of _all_ keys in all instances. + * this is done so the detect engine won't have to care about + * what instance it's looking up in. The matches all have a + * unique id and is the array lookup key at the same time */ + //u_int32_t keys = mpm_ctx->max_pattern_id + 1; + u_int32_t keys = matchsize + 1; + if (keys) { + mpm_thread_ctx->match = malloc(keys * sizeof(MpmMatchBucket)); + if (mpm_thread_ctx->match == NULL) { + printf("ERROR: could not setup memory for pattern matcher: %s\n", strerror(errno)); + exit(1); + } + memset(mpm_thread_ctx->match, 0, keys * sizeof(MpmMatchBucket)); + + mpm_thread_ctx->memory_cnt++; + mpm_thread_ctx->memory_size += (keys * sizeof(MpmMatchBucket)); + } +} + +void B3gThreadDestroyCtx(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx) { + B3gThreadCtx *ctx = (B3gThreadCtx *)mpm_thread_ctx->ctx; + + B3gPrintSearchStats(mpm_thread_ctx); + + if (ctx) { + if (mpm_thread_ctx->match != NULL) { + mpm_thread_ctx->memory_cnt--; + mpm_thread_ctx->memory_size -= ((mpm_ctx->max_pattern_id + 1) * sizeof(MpmMatchBucket)); + free(mpm_thread_ctx->match); + } + + mpm_thread_ctx->memory_cnt--; + mpm_thread_ctx->memory_size -= sizeof(B3gThreadCtx); + free(mpm_thread_ctx->ctx); + } + + MpmMatchFreeSpares(mpm_thread_ctx, mpm_thread_ctx->sparelist); + MpmMatchFreeSpares(mpm_thread_ctx, mpm_thread_ctx->qlist); +} + +u_int32_t B3gScanBNDMq(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *pmq, u_int8_t *buf, u_int16_t buflen) { + B3gCtx *ctx = (B3gCtx *)mpm_ctx->ctx; +#ifdef B3G_COUNTERS + B3gThreadCtx *tctx = (B3gThreadCtx *)mpm_thread_ctx->ctx; +#endif + u_int32_t pos = ctx->scan_m - B3G_Q + 1, matches = 0; + B3G_TYPE d; + + COUNT(tctx->scan_stat_calls++); + COUNT(tctx->scan_stat_m_total+=ctx->scan_m); + + if (buflen < ctx->scan_m) + return 0; + + while (pos <= (buflen - B3G_Q + 1)) { + u_int16_t h = B3G_HASH(bg_tolower(buf[pos - 1]), bg_tolower(buf[pos]),bg_tolower(buf[pos + 1])); + d = ctx->scan_B3G[h]; + + if (d != 0) { + COUNT(tctx->scan_stat_d0++); + u_int j = pos; + u_int first = pos - (ctx->scan_m - B3G_Q + 1); + + do { + j = j - 1; + if (d >= (1 << (ctx->scan_m - 1))) { + if (j > first) pos = j; + else { + /* get our patterns from the hash */ + h = B3G_HASH(bg_tolower(buf[j + ctx->scan_m - 3]), bg_tolower(buf[j + ctx->scan_m - 2]),bg_tolower(buf[j + ctx->scan_m - 1])); + + if (ctx->scan_bloom[h] != NULL) { + COUNT(tctx->scan_stat_pminlen_calls++); + COUNT(tctx->scan_stat_pminlen_total+=ctx->scan_pminlen[h]); + + if ((buflen - j) < ctx->scan_pminlen[h]) { + goto skip_loop; + } else { + COUNT(tctx->scan_stat_bloom_calls++); + + if (BloomFilterTest(ctx->scan_bloom[h], buf+j, ctx->scan_pminlen[h]) == 0) { + COUNT(tctx->scan_stat_bloom_hits++); + + //printf("Bloom: %p, buflen %u, pos %u, p_min_len %u\n", ctx->scan_bloom[h], buflen, pos, ctx->scan_pminlen[h]); + goto skip_loop; + } + } + } + + B3gHashItem *hi = ctx->scan_hash[h], *thi; + for (thi = hi; thi != NULL; thi = thi->nxt) { + COUNT(tctx->scan_stat_d0_hashloop++); + B3gPattern *p = ctx->parray[thi->idx]; + + if (p->flags & B3G_NOCASE) { + if (buflen - j < p->len) + continue; + + if (memcmp_lowercase(p->ci, buf+j, p->len) == 0) { + //printf("CI Exact match: "); prt(p->ci, p->len); printf("\n"); + COUNT(tctx->scan_stat_loop_match++); + + MpmEndMatch *em; + for (em = p->em; em; em = em->next) { + //printf("em %p id %u\n", em, em->id); + if (MpmMatchAppend(mpm_thread_ctx, pmq, em, &mpm_thread_ctx->match[em->id], j, p->len)) + matches++; + } + } else { + COUNT(tctx->scan_stat_loop_no_match++); + } + } else { + if (buflen - j < p->len) + continue; + + if (memcmp(p->cs, buf+j, p->len) == 0) { + //printf("CS Exact match: "); prt(p->cs, p->len); printf("\n"); + COUNT(tctx->scan_stat_loop_match++); + + MpmEndMatch *em; + for (em = p->em; em; em = em->next) { + //printf("em %p id %u\n", em, em->id); + if (MpmMatchAppend(mpm_thread_ctx, pmq, em, &mpm_thread_ctx->match[em->id], j, p->len)) + matches++; + } + } else { + COUNT(tctx->scan_stat_loop_no_match++); + } + } + } +skip_loop: + //printf("output at pos %u: ", j); prt(buf + (j), ctx->scan_m); printf("\n"); + ; // gcc doesn't like the goto label without this :-S + } + } + + h = B3G_HASH(bg_tolower(buf[j - 1]), bg_tolower(buf[j - 0]),bg_tolower(buf[j+1])); + d = (d << 1) & ctx->scan_B3G[h]; + } while (d != 0); + } + COUNT(tctx->scan_stat_num_shift++); + COUNT(tctx->scan_stat_total_shift += (ctx->scan_m - B3G_Q + 1)); + pos = pos + ctx->scan_m - B3G_Q + 1; + } + return matches; +} + +u_int32_t B3gScan(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *pmq, u_int8_t *buf, u_int16_t buflen) { + B3gCtx *ctx = (B3gCtx *)mpm_ctx->ctx; +#ifdef B3G_COUNTERS + B3gThreadCtx *tctx = (B3gThreadCtx *)mpm_thread_ctx->ctx; +#endif + u_int32_t pos = 0, matches = 0; + B3G_TYPE d; + u_int j; + + COUNT(tctx->scan_stat_calls++); + COUNT(tctx->scan_stat_m_total+=ctx->scan_m); + + if (buflen < ctx->scan_m) + return 0; + + while (pos <= (buflen - ctx->scan_m)) { + j = ctx->scan_m - 2; + d = ~0; + + do { + u_int16_t h = B3G_HASH(bg_tolower(buf[pos + j - 1]), bg_tolower(buf[pos + j - 0]),bg_tolower(buf[pos + j + 1])); +// printf("scan: h %u, %c.%c.%c\n", h, bg_tolower(buf[pos + j - 1]), bg_tolower(buf[pos + j - 0]),bg_tolower(buf[pos + j + 1])); + d = ((d << 1) & ctx->scan_B3G[h]); + j = j - 1; + } while (d != 0 && j != 0); +// printf("scan: d %u, j %u\n", d, j); + + /* (partial) match, move on to verification */ + if (d != 0) { + COUNT(tctx->scan_stat_d0++); + //printf("output at pos %u: ", pos); prt(buf + pos, ctx->scan_m); printf("\n"); + + /* get our patterns from the hash */ + u_int16_t h = B3G_HASH(bg_tolower(buf[pos + ctx->scan_m - 3]), bg_tolower(buf[pos + ctx->scan_m - 2]),bg_tolower(buf[pos + ctx->scan_m - 1])); + + if (ctx->scan_bloom[h] != NULL) { + COUNT(tctx->scan_stat_pminlen_calls++); + COUNT(tctx->scan_stat_pminlen_total+=ctx->scan_pminlen[h]); + + if ((buflen - pos) < ctx->scan_pminlen[h]) { + goto skip_loop; + } else { + COUNT(tctx->scan_stat_bloom_calls++); + + if (BloomFilterTest(ctx->scan_bloom[h], buf+pos, ctx->scan_pminlen[h]) == 0) { + COUNT(tctx->scan_stat_bloom_hits++); + + //printf("Bloom: %p, buflen %u, pos %u, p_min_len %u\n", ctx->scan_bloom[h], buflen, pos, ctx->scan_pminlen[h]); + goto skip_loop; + } + } + } + + B3gHashItem *hi = ctx->scan_hash[h], *thi; + for (thi = hi; thi != NULL; thi = thi->nxt) { + COUNT(tctx->scan_stat_d0_hashloop++); + B3gPattern *p = ctx->parray[thi->idx]; + + if (p->flags & B3G_NOCASE) { + if (buflen - pos < p->len) + continue; + + if (memcmp_lowercase(p->ci, buf+pos, p->len) == 0) { + //printf("CI Exact match: "); prt(p->ci, p->len); printf("\n"); + COUNT(tctx->scan_stat_loop_match++); + + MpmEndMatch *em; + for (em = p->em; em; em = em->next) { + //printf("em %p id %u\n", em, em->id); + if (MpmMatchAppend(mpm_thread_ctx, pmq, em, &mpm_thread_ctx->match[em->id], pos, p->len)) + matches++; + } + } else { + COUNT(tctx->scan_stat_loop_no_match++); + } + } else { + if (buflen - pos < p->len) + continue; + + if (memcmp(p->cs, buf+pos, p->len) == 0) { + //printf("CS Exact match: "); prt(p->cs, p->len); printf("\n"); + COUNT(tctx->scan_stat_loop_match++); + + MpmEndMatch *em; + for (em = p->em; em; em = em->next) { + //printf("em %p id %u\n", em, em->id); + if (MpmMatchAppend(mpm_thread_ctx, pmq, em, &mpm_thread_ctx->match[em->id], pos, p->len)) + matches++; + } + } else { + COUNT(tctx->scan_stat_loop_no_match++); + } + } + } +skip_loop: + pos = pos + ctx->scan_s0; + } else { + COUNT(tctx->scan_stat_num_shift++); + COUNT(tctx->scan_stat_total_shift += (j + 1)); + + pos = pos + j + 1; + } + } + + //printf("Total matches %u\n", matches); + return matches; +} + +u_int32_t B3gScan2(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *pmq, u_int8_t *buf, u_int16_t buflen) { + B3gCtx *ctx = (B3gCtx *)mpm_ctx->ctx; + u_int8_t *bufmin = buf; + u_int8_t *bufend = buf + buflen - 1; + u_int32_t cnt = 0; + B3gPattern *p; + MpmEndMatch *em; + B3gHashItem *thi, *hi; + + if (buflen < 2) + return 0; + + //printf("BUF "); prt(buf,buflen); printf("\n"); + + while (buf <= bufend) { + u_int16_t h = bg_tolower(*buf) << 8 | bg_tolower(*(buf+1)); + hi = &ctx->scan_hash2[h]; + + if (hi->flags & 0x01) { + for (thi = hi; thi != NULL; thi = thi->nxt) { + p = ctx->parray[thi->idx]; + + if (p->len != 2) + continue; + + if (p->flags & B3G_NOCASE) { + if (bg_tolower(*buf) == p->ci[0] && bg_tolower(*(buf+1)) == p->ci[1]) { + //printf("CI Exact match: "); prt(p->ci, p->len); printf(" in buf "); prt(buf, p->len);printf(" (B3gSearch1)\n"); + for (em = p->em; em; em = em->next) { + if (MpmMatchAppend(mpm_thread_ctx, pmq, em, &mpm_thread_ctx->match[em->id],(buf+1 - bufmin), p->len)) + cnt++; + } + } + } else { + if (*buf == p->cs[0] && *(buf+1) == p->cs[1]) { + //printf("CS Exact match: "); prt(p->cs, p->len); printf(" in buf "); prt(buf, p->len);printf(" (B3gSearch1)\n"); + for (em = p->em; em; em = em->next) { + if (MpmMatchAppend(mpm_thread_ctx, pmq, em, &mpm_thread_ctx->match[em->id],(buf+1 - bufmin), p->len)) + cnt++; + } + } + } + } + } + buf += 1; + } + + //printf("B3gSearch2: after 2byte cnt %u\n", cnt); + if (ctx->scan_x_pat_cnt) { + /* Pass bufmin on because buf no longer points to the + * start of the buffer. */ + cnt += ctx->MBScan(mpm_ctx, mpm_thread_ctx, pmq, bufmin, buflen); + //printf("B3gSearch1: after 2+byte cnt %u\n", cnt); + } + return cnt; +} +u_int32_t B3gScan1(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *pmq, u_int8_t *buf, u_int16_t buflen) { + B3gCtx *ctx = (B3gCtx *)mpm_ctx->ctx; + u_int8_t *bufmin = buf; + u_int8_t *bufend = buf + buflen - 1; + u_int32_t cnt = 0; + B3gPattern *p; + MpmEndMatch *em; + B3gHashItem *thi, *hi; + + if (buflen == 0) + return 0; + + //printf("BUF "); prt(buf,buflen); printf("\n"); + + while (buf <= bufend) { + u_int8_t h = bg_tolower(*buf); + hi = &ctx->scan_hash1[h]; + + if (hi->flags & 0x01) { + for (thi = hi; thi != NULL; thi = thi->nxt) { + p = ctx->parray[thi->idx]; + + if (p->len != 1) + continue; + + if (p->flags & B3G_NOCASE) { + if (bg_tolower(*buf) == p->ci[0]) { + //printf("CI Exact match: "); prt(p->ci, p->len); printf(" in buf "); prt(buf, p->len);printf(" (B3gSearch1)\n"); + for (em = p->em; em; em = em->next) { + if (MpmMatchAppend(mpm_thread_ctx, pmq, em, &mpm_thread_ctx->match[em->id],(buf+1 - bufmin), p->len)) + cnt++; + } + } + } else { + if (*buf == p->cs[0]) { + //printf("CS Exact match: "); prt(p->cs, p->len); printf(" in buf "); prt(buf, p->len);printf(" (B3gSearch1)\n"); + for (em = p->em; em; em = em->next) { + if (MpmMatchAppend(mpm_thread_ctx, pmq, em, &mpm_thread_ctx->match[em->id],(buf+1 - bufmin), p->len)) + cnt++; + } + } + } + } + } + buf += 1; + } + + //printf("B3gSearch1: after 1byte cnt %u\n", cnt); + if (ctx->scan_2_pat_cnt) { + /* Pass bufmin on because buf no longer points to the + * start of the buffer. */ + cnt += ctx->MBScan2(mpm_ctx, mpm_thread_ctx, pmq, bufmin, buflen); + //printf("B3gSearch1: after 2+byte cnt %u\n", cnt); + } else if (ctx->scan_x_pat_cnt) { + cnt += ctx->MBScan(mpm_ctx, mpm_thread_ctx, pmq, bufmin, buflen); + } + return cnt; +} + +u_int32_t B3gSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *pmq, u_int8_t *buf, u_int16_t buflen) { + B3gCtx *ctx = (B3gCtx *)mpm_ctx->ctx; +#ifdef B3G_COUNTERS + B3gThreadCtx *tctx = (B3gThreadCtx *)mpm_thread_ctx->ctx; +#endif + u_int32_t pos = 0, matches = 0; + B3G_TYPE d; + u_int j; + + if (buflen < ctx->search_m) + return 0; + + while (pos <= (buflen - ctx->search_m)) { + j = ctx->search_m - 2; + d = ~0; + + do { + u_int16_t h = B3G_HASH(bg_tolower(buf[pos + j - 1]), bg_tolower(buf[pos + j]),bg_tolower(buf[pos + j + 1])); + d &= ctx->search_B3G[h]; + d <<= 1; + j = j - 1; + //printf("h %u d %d %c.%c\n", h, d, bg_tolower(buf[pos + j - 1]),bg_tolower(buf[pos + j])); + } while (d != 0 && j != 0); + + /* (partial) match, move on to verification */ + if (d != 0) { + COUNT(tctx->search_stat_d0++); + + /* get our patterns from the hash */ + u_int16_t h = B3G_HASH(bg_tolower(buf[pos + ctx->search_m - 3]), bg_tolower(buf[pos + ctx->search_m - 2]),bg_tolower(buf[pos + ctx->search_m - 1])); + B3gHashItem *hi = ctx->search_hash[h], *thi; + for (thi = hi; thi != NULL; thi = thi->nxt) { + B3gPattern *p = ctx->parray[thi->idx]; + if (p->flags & B3G_NOCASE) { + if (buflen - pos < p->len) + continue; + + if (memcmp_lowercase(p->ci, buf+pos, p->len) == 0) { + //printf("CI Exact match: "); prt(p->ci, p->len); printf("\n"); + COUNT(tctx->search_stat_loop_match++); + + MpmEndMatch *em; + for (em = p->em; em; em = em->next) { + //printf("em %p id %u\n", em, em->id); + if (MpmMatchAppend(mpm_thread_ctx, pmq, em, &mpm_thread_ctx->match[em->id], pos, p->len)) + matches++; + } + + } else { + COUNT(tctx->search_stat_loop_no_match++); + } + } else { + if (buflen - pos < p->len) + continue; + + if (memcmp(p->cs, buf+pos, p->len) == 0) { + //printf("CS Exact match: "); prt(p->cs, p->len); printf("\n"); + COUNT(tctx->search_stat_loop_match++); + + MpmEndMatch *em; + for (em = p->em; em; em = em->next) { + //printf("em %p id %u\n", em, em->id); + if (MpmMatchAppend(mpm_thread_ctx, pmq, em, &mpm_thread_ctx->match[em->id], pos, p->len)) + matches++; + } + + } else { + COUNT(tctx->search_stat_loop_no_match++); + } + } + } + + //printf("output at pos %d: ", pos); prt(buf + pos, ctx->search_m); printf("\n"); + pos = pos + 1; + } else { + COUNT(tctx->search_stat_num_shift++); + COUNT(tctx->search_stat_total_shift += (j + 1)); + pos = pos + j + 1; + } + } + + //printf("Total matches %u\n", matches); + return matches; +} + +u_int32_t B3gSearch2(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *pmq, u_int8_t *buf, u_int16_t buflen) { + B3gCtx *ctx = (B3gCtx *)mpm_ctx->ctx; + u_int8_t *bufmin = buf; + u_int8_t *bufend = buf + buflen - 1; + u_int32_t cnt = 0; + B3gPattern *p; + MpmEndMatch *em; + B3gHashItem *thi, *hi; + + if (buflen < 2) + return 0; + + //printf("BUF "); prt(buf,buflen); printf("\n"); + + while (buf <= bufend) { + u_int16_t h = bg_tolower(*buf) << 8 | bg_tolower(*(buf+1)); + hi = &ctx->search_hash2[h]; + + if (hi->flags & 0x01) { + for (thi = hi; thi != NULL; thi = thi->nxt) { + p = ctx->parray[thi->idx]; + + if (p->len != 2) + continue; + + if (p->flags & B3G_NOCASE) { + if (bg_tolower(*buf) == p->ci[0] && bg_tolower(*(buf+1)) == p->ci[1]) { + //printf("CI Exact match: "); prt(p->ci, p->len); printf(" in buf "); prt(buf, p->len);printf(" (B3gSearch1)\n"); + for (em = p->em; em; em = em->next) { + if (MpmMatchAppend(mpm_thread_ctx, pmq, em, &mpm_thread_ctx->match[em->id],(buf+1 - bufmin), p->len)) + cnt++; + } + } + } else { + if (*buf == p->cs[0] && *(buf+1) == p->cs[1]) { + //printf("CS Exact match: "); prt(p->cs, p->len); printf(" in buf "); prt(buf, p->len);printf(" (B3gSearch1)\n"); + for (em = p->em; em; em = em->next) { + if (MpmMatchAppend(mpm_thread_ctx, pmq, em, &mpm_thread_ctx->match[em->id],(buf+1 - bufmin), p->len)) + cnt++; + } + } + } + } + } + buf += 1; + } + + //printf("B3gSearch2: after 2byte cnt %u\n", cnt); + if (ctx->search_x_pat_cnt) { + /* Pass bufmin on because buf no longer points to the + * start of the buffer. */ + cnt += ctx->MBSearch(mpm_ctx, mpm_thread_ctx, pmq, bufmin, buflen); + //printf("B3gSearch1: after 2+byte cnt %u\n", cnt); + } + return cnt; +} + +u_int32_t B3gSearch1(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *pmq, u_int8_t *buf, u_int16_t buflen) { + B3gCtx *ctx = (B3gCtx *)mpm_ctx->ctx; + u_int8_t *bufmin = buf; + u_int8_t *bufend = buf + buflen - 1; + u_int32_t cnt = 0; + B3gPattern *p; + MpmEndMatch *em; + B3gHashItem *thi, *hi; + + if (buflen == 0) + return 0; + + //printf("BUF "); prt(buf,buflen); printf("\n"); + + if (mpm_ctx->search_minlen == 1) { + while (buf <= bufend) { + u_int8_t h = bg_tolower(*buf); + hi = &ctx->search_hash1[h]; + + if (hi->flags & 0x01) { + for (thi = hi; thi != NULL; thi = thi->nxt) { + p = ctx->parray[thi->idx]; + + if (p->len != 1) + continue; + + if (p->flags & B3G_NOCASE) { + if (bg_tolower(*buf) == p->ci[0]) { + //printf("CI Exact match: "); prt(p->ci, p->len); printf(" in buf "); prt(buf, p->len);printf(" (B3gSearch1)\n"); + for (em = p->em; em; em = em->next) { + if (MpmMatchAppend(mpm_thread_ctx, pmq, em, &mpm_thread_ctx->match[em->id],(buf+1 - bufmin), p->len)) + cnt++; + } + } + } else { + if (*buf == p->cs[0]) { + //printf("CS Exact match: "); prt(p->cs, p->len); printf(" in buf "); prt(buf, p->len);printf(" (B3gSearch1)\n"); + for (em = p->em; em; em = em->next) { + if (MpmMatchAppend(mpm_thread_ctx, pmq, em, &mpm_thread_ctx->match[em->id],(buf+1 - bufmin), p->len)) + cnt++; + } + } + } + } + } + buf += 1; + } + } + //printf("B3gSearch1: after 1byte cnt %u\n", cnt); + if (ctx->search_2_pat_cnt) { + /* Pass bufmin on because buf no longer points to the + * start of the buffer. */ + cnt += ctx->MBSearch2(mpm_ctx, mpm_thread_ctx, pmq, bufmin, buflen); + //printf("B3gSearch1: after 2+byte cnt %u\n", cnt); + } else if (ctx->search_x_pat_cnt) { + cnt += ctx->MBSearch(mpm_ctx, mpm_thread_ctx, pmq, bufmin, buflen); + } + return cnt; +} + +/* + * TESTS + */ + +static int B3gTestInit01 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmInitCtx(&mpm_ctx, MPM_B3G); + B3gCtx *ctx = (B3gCtx *)mpm_ctx.ctx; + + B3gAddPatternCS(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 0, 0); /* 1 match */ + + + B3gPreparePatterns(&mpm_ctx); + + + if (ctx->search_m == 4) + result = 1; + else + printf("4 != %u ", ctx->search_m); + + B3gDestroyCtx(&mpm_ctx); + return result; +} + +static int B3gTestS0Init01 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmInitCtx(&mpm_ctx, MPM_B3G); + B3gCtx *ctx = (B3gCtx *)mpm_ctx.ctx; + + B3gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 0, 0); /* 1 match */ + + B3gPreparePatterns(&mpm_ctx); + + if (ctx->scan_s0 == 4) + result = 1; + else + printf("4 != %u ", ctx->scan_s0); + + B3gDestroyCtx(&mpm_ctx); + return result; +} + +static int B3gTestS0Init02 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmInitCtx(&mpm_ctx, MPM_B3G); + B3gCtx *ctx = (B3gCtx *)mpm_ctx.ctx; + + B3gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 0, 0); /* 1 match */ + B3gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"cdef", 4, 0, 0, 1, 0); /* 1 match */ + + B3gPreparePatterns(&mpm_ctx); + + if (ctx->scan_s0 == 2) + result = 1; + else + printf("2 != %u ", ctx->scan_s0); + + B3gDestroyCtx(&mpm_ctx); + return result; +} + +static int B3gTestS0Init03 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmInitCtx(&mpm_ctx, MPM_B3G); + B3gCtx *ctx = (B3gCtx *)mpm_ctx.ctx; + + B3gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 0, 0); /* 1 match */ + B3gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"bcde", 4, 0, 0, 1, 0); /* 1 match */ + + B3gPreparePatterns(&mpm_ctx); + + if (ctx->scan_s0 == 1) + result = 1; + else + printf("1 != %u ", ctx->scan_s0); + + B3gDestroyCtx(&mpm_ctx); + return result; +} + +static int B3gTestS0Init04 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmInitCtx(&mpm_ctx, MPM_B3G); + B3gCtx *ctx = (B3gCtx *)mpm_ctx.ctx; + + B3gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"abab", 4, 0, 0, 0, 0); /* 1 match */ + + B3gPreparePatterns(&mpm_ctx); + + if (ctx->scan_s0 == 2) + result = 1; + else + printf("2 != %u ", ctx->scan_s0); + + B3gDestroyCtx(&mpm_ctx); + return result; +} + +static int B3gTestS0Init05 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmInitCtx(&mpm_ctx, MPM_B3G); + B3gCtx *ctx = (B3gCtx *)mpm_ctx.ctx; + + B3gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"abcab", 5, 0, 0, 0, 0); /* 1 match */ + + B3gPreparePatterns(&mpm_ctx); + + if (ctx->scan_s0 == 3) + result = 1; + else + printf("3 != %u ", ctx->scan_s0); + + B3gDestroyCtx(&mpm_ctx); + return result; +} + +static int B3gTestScan01 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmThreadCtx mpm_thread_ctx; + MpmInitCtx(&mpm_ctx, MPM_B3G); + + B3gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 0, 0); /* 1 match */ + + B3gPreparePatterns(&mpm_ctx); + B3gThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1 /* 1 pattern */); + + u_int32_t cnt = mpm_ctx.Scan(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"abcdefghjiklmnopqrstuvwxyz", 26); + + MpmMatchCleanup(&mpm_thread_ctx); + + if (cnt == 1) + result = 1; + else + printf("1 != %u ",cnt); + + B3gThreadDestroyCtx(&mpm_ctx, &mpm_thread_ctx); + B3gDestroyCtx(&mpm_ctx); + return result; +} + +static int B3gTestScan02 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmThreadCtx mpm_thread_ctx; + MpmInitCtx(&mpm_ctx, MPM_B3G); + + B3gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"abce", 4, 0, 0, 0, 0); /* 1 match */ + + B3gPreparePatterns(&mpm_ctx); + B3gThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1 /* 1 pattern */); + + u_int32_t cnt = mpm_ctx.Scan(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"abcdefghjiklmnopqrstuvwxyz", 26); + + MpmMatchCleanup(&mpm_thread_ctx); + + if (cnt == 0) + result = 1; + else + printf("0 != %u ",cnt); + + B3gThreadDestroyCtx(&mpm_ctx, &mpm_thread_ctx); + B3gDestroyCtx(&mpm_ctx); + return result; +} + +static int B3gTestScan03 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmThreadCtx mpm_thread_ctx; + MpmInitCtx(&mpm_ctx, MPM_B3G); + + B3gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 0, 0); /* 1 match */ + B3gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"bcde", 4, 0, 0, 1, 0); /* 1 match */ + B3gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"fghj", 4, 0, 0, 2, 0); /* 1 match */ + + B3gPreparePatterns(&mpm_ctx); + B3gThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 3 /* 3 patterns */); + + u_int32_t cnt = mpm_ctx.Scan(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"abcdefghjiklmnopqrstuvwxyz", 26); + + MpmMatchCleanup(&mpm_thread_ctx); + + if (cnt == 3) + result = 1; + else + printf("3 != %u ",cnt); + + B3gThreadDestroyCtx(&mpm_ctx, &mpm_thread_ctx); + B3gDestroyCtx(&mpm_ctx); + return result; +} + +/* test patterns longer than 'm'. M is 4 here. */ +static int B3gTestScan04 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmThreadCtx mpm_thread_ctx; + MpmInitCtx(&mpm_ctx, MPM_B3G); + + B3gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 0, 0); /* 1 match */ + B3gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"bcdegh", 6, 0, 0, 1, 0); /* 1 match */ + B3gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"fghjxyz", 7, 0, 0, 2, 0); /* 1 match */ + + B3gPreparePatterns(&mpm_ctx); + B3gThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 3 /* 3 patterns */); + + u_int32_t cnt = mpm_ctx.Scan(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"abcdefghjiklmnopqrstuvwxyz", 26); + + MpmMatchCleanup(&mpm_thread_ctx); + + if (cnt == 1) + result = 1; + else + printf("1 != %u ",cnt); + + B3gThreadDestroyCtx(&mpm_ctx, &mpm_thread_ctx); + B3gDestroyCtx(&mpm_ctx); + return result; +} + +/* case insensitive test patterns longer than 'm'. M is 4 here. */ +static int B3gTestScan05 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmThreadCtx mpm_thread_ctx; + MpmInitCtx(&mpm_ctx, MPM_B3G); + + B3gAddScanPatternCI(&mpm_ctx, (u_int8_t *)"ABCD", 4, 0, 0, 0, 0); /* 1 match */ + B3gAddScanPatternCI(&mpm_ctx, (u_int8_t *)"bCdEfG", 6, 0, 0, 1, 0); /* 1 match */ + B3gAddScanPatternCI(&mpm_ctx, (u_int8_t *)"fghJikl", 7, 0, 0, 2, 0); /* 1 match */ + + B3gPreparePatterns(&mpm_ctx); + B3gThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 3 /* 3 patterns */); + + u_int32_t cnt = mpm_ctx.Scan(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"abcdefghjiklmnopqrstuvwxyz", 26); + + MpmMatchCleanup(&mpm_thread_ctx); + + if (cnt == 3) + result = 1; + else + printf("3 != %u ",cnt); + + B3gThreadDestroyCtx(&mpm_ctx, &mpm_thread_ctx); + B3gDestroyCtx(&mpm_ctx); + return result; +} + +static int B3gTestScan06 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmThreadCtx mpm_thread_ctx; + MpmInitCtx(&mpm_ctx, MPM_B3G); + + B3gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 0, 0); /* 1 match */ + + B3gPreparePatterns(&mpm_ctx); + B3gThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1 /* 1 pattern */); + + u_int32_t cnt = mpm_ctx.Scan(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"abcd", 4); + + MpmMatchCleanup(&mpm_thread_ctx); + + if (cnt == 1) + result = 1; + else + printf("1 != %u ",cnt); + + B3gThreadDestroyCtx(&mpm_ctx, &mpm_thread_ctx); + B3gDestroyCtx(&mpm_ctx); + return result; +} + +static int B3gTestScan07 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmThreadCtx mpm_thread_ctx; + MpmInitCtx(&mpm_ctx, MPM_B3G); + //B3gCtx *ctx = (B3gCtx *)mpm_ctx.ctx; + + B3gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"A", 1, 0, 0, 0, 0); /* should match 30 times */ + B3gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"AA", 2, 0, 0, 1, 0); /* should match 29 times */ + B3gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"AAA", 3, 0, 0, 2, 0); /* should match 28 times */ + B3gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"AAAAA", 5, 0, 0, 3, 0); /* 26 */ + B3gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"AAAAAAAAAA", 10, 0, 0, 4, 0); /* 21 */ + B3gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", 30, 0, 0, 5, 0); /* 1 */ + /* total matches: 135 */ + + B3gPreparePatterns(&mpm_ctx); + B3gThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 6 /* 6 patterns */); + + u_int32_t cnt = mpm_ctx.Scan(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", 30); + + MpmMatchCleanup(&mpm_thread_ctx); + + if (cnt == 135) + result = 1; + else + printf("135 != %u ",cnt); + + B3gThreadDestroyCtx(&mpm_ctx, &mpm_thread_ctx); + B3gDestroyCtx(&mpm_ctx); + return result; +} + +static int B3gTestScan08 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmThreadCtx mpm_thread_ctx; + MpmInitCtx(&mpm_ctx, MPM_B3G); + + B3gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 0, 0); /* 1 match */ + + B3gPreparePatterns(&mpm_ctx); + B3gThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1 /* 1 pattern */); + + u_int32_t cnt = mpm_ctx.Scan(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"a", 1); + + MpmMatchCleanup(&mpm_thread_ctx); + + if (cnt == 0) + result = 1; + else + printf("0 != %u ",cnt); + + B3gThreadDestroyCtx(&mpm_ctx, &mpm_thread_ctx); + B3gDestroyCtx(&mpm_ctx); + return result; +} + +static int B3gTestScan09 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmThreadCtx mpm_thread_ctx; + MpmInitCtx(&mpm_ctx, MPM_B3G); + + B3gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"ab", 2, 0, 0, 0, 0); /* 1 match */ + + B3gPreparePatterns(&mpm_ctx); + B3gThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1 /* 1 pattern */); + + u_int32_t cnt = mpm_ctx.Scan(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"ab", 2); + + MpmMatchCleanup(&mpm_thread_ctx); + + if (cnt == 1) + result = 1; + else + printf("1 != %u ",cnt); + + B3gThreadDestroyCtx(&mpm_ctx, &mpm_thread_ctx); + B3gDestroyCtx(&mpm_ctx); + return result; +} + +static int B3gTestScan10 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmThreadCtx mpm_thread_ctx; + MpmInitCtx(&mpm_ctx, MPM_B3G); + + B3gAddScanPatternCS(&mpm_ctx, (u_int8_t *)"abcdefgh", 8, 0, 0, 0, 0); /* 1 match */ + + B3gPreparePatterns(&mpm_ctx); + B3gThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1 /* 1 pattern */); + + u_int32_t cnt = mpm_ctx.Scan(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"012345679012345679012345679012345679012345679012345679012345679012345679012345679012345679abcdefgh012345679012345679012345679012345679012345679012345679012345679012345679012345679012345679", 208); + + MpmMatchCleanup(&mpm_thread_ctx); + + if (cnt == 1) + result = 1; + else + printf("1 != %u ",cnt); + + B3gThreadDestroyCtx(&mpm_ctx, &mpm_thread_ctx); + B3gDestroyCtx(&mpm_ctx); + return result; +} + +static int B3gTestSearch01 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmThreadCtx mpm_thread_ctx; + MpmInitCtx(&mpm_ctx, MPM_B3G); + + B3gAddPatternCS(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 0, 0); /* 1 match */ + + B3gPreparePatterns(&mpm_ctx); + B3gThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1 /* 1 pattern */); + + u_int32_t cnt = mpm_ctx.Search(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"abcdefghjiklmnopqrstuvwxyz", 26); + + MpmMatchCleanup(&mpm_thread_ctx); + + if (cnt == 1) + result = 1; + else + printf("1 != %u ",cnt); + + B3gThreadDestroyCtx(&mpm_ctx, &mpm_thread_ctx); + B3gDestroyCtx(&mpm_ctx); + return result; +} + +static int B3gTestSearch02 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmThreadCtx mpm_thread_ctx; + MpmInitCtx(&mpm_ctx, MPM_B3G); + + B3gAddPatternCS(&mpm_ctx, (u_int8_t *)"abce", 4, 0, 0, 0, 0); /* 1 match */ + + B3gPreparePatterns(&mpm_ctx); + B3gThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1 /* 1 pattern */); + + u_int32_t cnt = mpm_ctx.Search(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"abcdefghjiklmnopqrstuvwxyz", 26); + + MpmMatchCleanup(&mpm_thread_ctx); + + if (cnt == 0) + result = 1; + else + printf("0 != %u ",cnt); + + B3gThreadDestroyCtx(&mpm_ctx, &mpm_thread_ctx); + B3gDestroyCtx(&mpm_ctx); + return result; +} + +static int B3gTestSearch03 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmThreadCtx mpm_thread_ctx; + MpmInitCtx(&mpm_ctx, MPM_B3G); + + B3gAddPatternCS(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 0, 0); /* 1 match */ + B3gAddPatternCS(&mpm_ctx, (u_int8_t *)"bcde", 4, 0, 0, 1, 0); /* 1 match */ + B3gAddPatternCS(&mpm_ctx, (u_int8_t *)"fghj", 4, 0, 0, 2, 0); /* 1 match */ + + B3gPreparePatterns(&mpm_ctx); + B3gThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 3 /* 3 patterns */); + + u_int32_t cnt = mpm_ctx.Search(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"abcdefghjiklmnopqrstuvwxyz", 26); + + MpmMatchCleanup(&mpm_thread_ctx); + + if (cnt == 3) + result = 1; + else + printf("3 != %u ",cnt); + + B3gThreadDestroyCtx(&mpm_ctx, &mpm_thread_ctx); + B3gDestroyCtx(&mpm_ctx); + return result; +} + +/* test patterns longer than 'm'. M is 4 here. */ +static int B3gTestSearch04 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmThreadCtx mpm_thread_ctx; + MpmInitCtx(&mpm_ctx, MPM_B3G); + + B3gAddPatternCS(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 0, 0); /* 1 match */ + B3gAddPatternCS(&mpm_ctx, (u_int8_t *)"bcdegh", 6, 0, 0, 1, 0); /* 1 match */ + B3gAddPatternCS(&mpm_ctx, (u_int8_t *)"fghjxyz", 7, 0, 0, 2, 0); /* 1 match */ + + B3gPreparePatterns(&mpm_ctx); + B3gThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 3 /* 3 patterns */); + + u_int32_t cnt = mpm_ctx.Search(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"abcdefghjiklmnopqrstuvwxyz", 26); + + MpmMatchCleanup(&mpm_thread_ctx); + + if (cnt == 1) + result = 1; + else + printf("1 != %u ",cnt); + + B3gThreadDestroyCtx(&mpm_ctx, &mpm_thread_ctx); + B3gDestroyCtx(&mpm_ctx); + return result; +} + +/* case insensitive test patterns longer than 'm'. M is 4 here. */ +static int B3gTestSearch05 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmThreadCtx mpm_thread_ctx; + MpmInitCtx(&mpm_ctx, MPM_B3G); + + B3gAddPatternCI(&mpm_ctx, (u_int8_t *)"ABCD", 4, 0, 0, 0, 0); /* 1 match */ + B3gAddPatternCI(&mpm_ctx, (u_int8_t *)"bCdEfG", 6, 0, 0, 1, 0); /* 1 match */ + B3gAddPatternCI(&mpm_ctx, (u_int8_t *)"fghJikl", 7, 0, 0, 2, 0); /* 1 match */ + + B3gPreparePatterns(&mpm_ctx); + B3gThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 3 /* 3 patterns */); + + u_int32_t cnt = mpm_ctx.Search(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"abcdefghjiklmnopqrstuvwxyz", 26); + + MpmMatchCleanup(&mpm_thread_ctx); + + if (cnt == 3) + result = 1; + else + printf("3 != %u ",cnt); + + B3gThreadDestroyCtx(&mpm_ctx, &mpm_thread_ctx); + B3gDestroyCtx(&mpm_ctx); + return result; +} + +static int B3gTestSearch06 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmThreadCtx mpm_thread_ctx; + MpmInitCtx(&mpm_ctx, MPM_B3G); + + B3gAddPatternCS(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 0, 0); /* 1 match */ + + B3gPreparePatterns(&mpm_ctx); + B3gThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1 /* 1 pattern */); + + u_int32_t cnt = mpm_ctx.Search(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"abcd", 4); + + MpmMatchCleanup(&mpm_thread_ctx); + + if (cnt == 1) + result = 1; + else + printf("1 != %u ",cnt); + + B3gThreadDestroyCtx(&mpm_ctx, &mpm_thread_ctx); + B3gDestroyCtx(&mpm_ctx); + return result; +} + +static int B3gTestSearch07 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmThreadCtx mpm_thread_ctx; + MpmInitCtx(&mpm_ctx, MPM_B3G); + //B3gCtx *ctx = (B3gCtx *)mpm_ctx.ctx; + + B3gAddPatternCS(&mpm_ctx, (u_int8_t *)"A", 1, 0, 0, 0, 0); /* should match 30 times */ + B3gAddPatternCS(&mpm_ctx, (u_int8_t *)"AA", 2, 0, 0, 1, 0); /* should match 29 times */ + B3gAddPatternCS(&mpm_ctx, (u_int8_t *)"AAA", 3, 0, 0, 2, 0); /* should match 28 times */ + B3gAddPatternCS(&mpm_ctx, (u_int8_t *)"AAAAA", 5, 0, 0, 3, 0); /* 26 */ + B3gAddPatternCS(&mpm_ctx, (u_int8_t *)"AAAAAAAAAA", 10, 0, 0, 4, 0); /* 21 */ + B3gAddPatternCS(&mpm_ctx, (u_int8_t *)"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", 30, 0, 0, 5, 0); /* 1 */ + /* total matches: 135 */ + + B3gPreparePatterns(&mpm_ctx); + B3gThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 6 /* 6 patterns */); + + u_int32_t cnt = mpm_ctx.Search(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", 30); + + MpmMatchCleanup(&mpm_thread_ctx); + + if (cnt == 135) + result = 1; + else + printf("135 != %u ",cnt); + + B3gThreadDestroyCtx(&mpm_ctx, &mpm_thread_ctx); + B3gDestroyCtx(&mpm_ctx); + return result; +} + +static int B3gTestSearch08 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmThreadCtx mpm_thread_ctx; + MpmInitCtx(&mpm_ctx, MPM_B3G); + + B3gAddPatternCS(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 0, 0); /* 1 match */ + + B3gPreparePatterns(&mpm_ctx); + B3gThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1 /* 1 pattern */); + + u_int32_t cnt = mpm_ctx.Search(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"a", 1); + + MpmMatchCleanup(&mpm_thread_ctx); + + if (cnt == 0) + result = 1; + else + printf("0 != %u ",cnt); + + B3gThreadDestroyCtx(&mpm_ctx, &mpm_thread_ctx); + B3gDestroyCtx(&mpm_ctx); + return result; +} + +static int B3gTestSearch09 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmThreadCtx mpm_thread_ctx; + MpmInitCtx(&mpm_ctx, MPM_B3G); + + B3gAddPatternCS(&mpm_ctx, (u_int8_t *)"ab", 2, 0, 0, 0, 0); /* 1 match */ + + B3gPreparePatterns(&mpm_ctx); + B3gThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1 /* 1 pattern */); + + u_int32_t cnt = mpm_ctx.Search(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"ab", 2); + + MpmMatchCleanup(&mpm_thread_ctx); + + if (cnt == 1) + result = 1; + else + printf("1 != %u ",cnt); + + B3gThreadDestroyCtx(&mpm_ctx, &mpm_thread_ctx); + B3gDestroyCtx(&mpm_ctx); + return result; +} + +static int B3gTestSearch10 (void) { + int result = 0; + MpmCtx mpm_ctx; + MpmThreadCtx mpm_thread_ctx; + MpmInitCtx(&mpm_ctx, MPM_B3G); + + B3gAddPatternCS(&mpm_ctx, (u_int8_t *)"abcdefgh", 8, 0, 0, 0, 0); /* 1 match */ + + B3gPreparePatterns(&mpm_ctx); + B3gThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1 /* 1 pattern */); + + u_int32_t cnt = mpm_ctx.Search(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"012345679012345679012345679012345679012345679012345679012345679012345679012345679012345679abcdefgh012345679012345679012345679012345679012345679012345679012345679012345679012345679012345679", 208); + + MpmMatchCleanup(&mpm_thread_ctx); + + if (cnt == 1) + result = 1; + else + printf("1 != %u ",cnt); + + B3gThreadDestroyCtx(&mpm_ctx, &mpm_thread_ctx); + B3gDestroyCtx(&mpm_ctx); + return result; +} + +void B3gRegisterTests(void) { + UtRegisterTest("B3gTestInit01", B3gTestInit01, 1); +/* + UtRegisterTest("B3gTestS0Init01", B3gTestS0Init01, 1); + UtRegisterTest("B3gTestS0Init02", B3gTestS0Init02, 1); + UtRegisterTest("B3gTestS0Init03", B3gTestS0Init03, 1); + UtRegisterTest("B3gTestS0Init04", B3gTestS0Init04, 1); + UtRegisterTest("B3gTestS0Init05", B3gTestS0Init05, 1); +*/ + UtRegisterTest("B3gTestScan01", B3gTestScan01, 1); + + UtRegisterTest("B3gTestScan02", B3gTestScan02, 1); + UtRegisterTest("B3gTestScan03", B3gTestScan03, 1); + UtRegisterTest("B3gTestScan04", B3gTestScan04, 1); + UtRegisterTest("B3gTestScan05", B3gTestScan05, 1); + UtRegisterTest("B3gTestScan06", B3gTestScan06, 1); + UtRegisterTest("B3gTestScan07", B3gTestScan07, 1); + UtRegisterTest("B3gTestScan08", B3gTestScan08, 1); + UtRegisterTest("B3gTestScan09", B3gTestScan09, 1); + UtRegisterTest("B3gTestScan10", B3gTestScan10, 1); + + UtRegisterTest("B3gTestSearch01", B3gTestSearch01, 1); + UtRegisterTest("B3gTestSearch02", B3gTestSearch02, 1); + UtRegisterTest("B3gTestSearch03", B3gTestSearch03, 1); + UtRegisterTest("B3gTestSearch04", B3gTestSearch04, 1); + UtRegisterTest("B3gTestSearch05", B3gTestSearch05, 1); + UtRegisterTest("B3gTestSearch06", B3gTestSearch06, 1); + UtRegisterTest("B3gTestSearch07", B3gTestSearch07, 1); + UtRegisterTest("B3gTestSearch08", B3gTestSearch08, 1); + UtRegisterTest("B3gTestSearch09", B3gTestSearch09, 1); + UtRegisterTest("B3gTestSearch10", B3gTestSearch10, 1); + +} + +#if 0 +int main () { +#define R 4 +int i; + B3gCtx bg_ctx; + B3gInitCtx(&bg_ctx); + + B3gAddPatternCI(&bg_ctx, "grep", 4, 0, 0, 0, 0); + B2pPrepare(&bg_ctx); + + B3GSearch(&bg_ctx,Text,strlen(Text)); + + exit(EXIT_SUCCESS); +} +#endif diff --git a/src/util-mpm-b3g.h b/src/util-mpm-b3g.h new file mode 100644 index 0000000000..1d362f5664 --- /dev/null +++ b/src/util-mpm-b3g.h @@ -0,0 +1,125 @@ +#ifndef __UTIL_MPM_B3G_H__ +#define __UTIL_MPM_B3G_H__ + +#include "util-mpm.h" +#include "util-bloomfilter.h" + +#define B3G_NOCASE 0x01 +#define B3G_SCAN 0x02 + +//#define B3G_HASHSIZE 65536 +#define B3G_HASHSIZE 32768 +//#define B3G_HASHSIZE 16384 +//#define B3G_HASHSIZE 8192 +//#define B3G_HASHSIZE 4096 + +//#define B3G_HASHSHIFT 8 +#define B3G_HASHSHIFT 7 +//#define B3G_HASHSHIFT 6 +//#define B3G_HASHSHIFT 5 +//#define B3G_HASHSHIFT 4 + +#define B3G_TYPE u_int32_t +//#define B3G_TYPE u_int16_t +//#define B3G_TYPE u_int8_t +//#define B3G_WORD_SIZE 16 +//#define B3G_WORD_SIZE 8 +#define B3G_WORD_SIZE 32 + +#define B3G_BLOOMSIZE 1024 + +#define B3G_HASH(a,b,c) (((a)<ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx->ctx; printf("MPM WuManber Information:\n"); printf("Memory allocs: %u\n", mpm_ctx->memory_cnt); @@ -159,17 +159,17 @@ void WmPrintInfo(MpmCtx *mpm_ctx) { printf("Total Patterns: %u\n", mpm_ctx->total_pattern_cnt); printf("Smallest: %u\n", mpm_ctx->scan_minlen); printf("Largest: %u\n", mpm_ctx->scan_maxlen); - printf("Max shiftlen: %u\n", wm_ctx->scan_shiftlen); - printf("Hash size: %u\n", wm_ctx->scan_hash_size); + printf("Max shiftlen: %u\n", ctx->scan_shiftlen); + printf("Hash size: %u\n", ctx->scan_hash_size); printf("Scan function: "); if (mpm_ctx->Scan == WmScan1) { printf("WmScan1 (allows single byte patterns)\n"); printf("MBScan funct: "); - if (wm_ctx->MBScan == WmScan2Hash16) printf("WmSearch2Hash16\n"); - else if (wm_ctx->MBScan == WmScan2Hash15) printf("WmSearch2Hash15\n"); - else if (wm_ctx->MBScan == WmScan2Hash14) printf("WmSearch2Hash14\n"); - else if (wm_ctx->MBScan == WmScan2Hash12) printf("WmSearch2Hash12\n"); - else if (wm_ctx->MBScan == WmScan2Hash9) printf("WmSearch2Hash9\n"); + if (ctx->MBScan == WmScan2Hash16) printf("WmSearch2Hash16\n"); + else if (ctx->MBScan == WmScan2Hash15) printf("WmSearch2Hash15\n"); + else if (ctx->MBScan == WmScan2Hash14) printf("WmSearch2Hash14\n"); + else if (ctx->MBScan == WmScan2Hash12) printf("WmSearch2Hash12\n"); + else if (ctx->MBScan == WmScan2Hash9) printf("WmSearch2Hash9\n"); } if (mpm_ctx->Scan == WmScan2Hash16) printf("WmScan2Hash16 (only for multibyte patterns)\n"); else if (mpm_ctx->Scan == WmScan2Hash15) printf("WmScan2Hash15 (only for multibyte patterns)\n"); @@ -179,17 +179,17 @@ void WmPrintInfo(MpmCtx *mpm_ctx) { else printf("ERROR\n"); printf("Smallest: %u\n", mpm_ctx->search_minlen); printf("Largest: %u\n", mpm_ctx->search_maxlen); - printf("Max shiftlen: %u\n", wm_ctx->search_shiftlen); - printf("Hash size: %u\n", wm_ctx->search_hash_size); + printf("Max shiftlen: %u\n", ctx->search_shiftlen); + printf("Hash size: %u\n", ctx->search_hash_size); printf("Search function: "); if (mpm_ctx->Search == WmSearch1) { printf("WmSearch1 (allows single byte patterns)\n"); printf("MBSearch funct: "); - if (wm_ctx->MBSearch == WmSearch2Hash16) printf("WmSearch2Hash16\n"); - else if (wm_ctx->MBSearch == WmSearch2Hash15) printf("WmSearch2Hash15\n"); - else if (wm_ctx->MBSearch == WmSearch2Hash14) printf("WmSearch2Hash14\n"); - else if (wm_ctx->MBSearch == WmSearch2Hash12) printf("WmSearch2Hash12\n"); - else if (wm_ctx->MBSearch == WmSearch2Hash9) printf("WmSearch2Hash9\n"); + if (ctx->MBSearch == WmSearch2Hash16) printf("WmSearch2Hash16\n"); + else if (ctx->MBSearch == WmSearch2Hash15) printf("WmSearch2Hash15\n"); + else if (ctx->MBSearch == WmSearch2Hash14) printf("WmSearch2Hash14\n"); + else if (ctx->MBSearch == WmSearch2Hash12) printf("WmSearch2Hash12\n"); + else if (ctx->MBSearch == WmSearch2Hash9) printf("WmSearch2Hash9\n"); } if (mpm_ctx->Search == WmSearch2Hash16) printf("WmSearch2Hash16 (only for multibyte patterns)\n"); else if (mpm_ctx->Search == WmSearch2Hash15) printf("WmSearch2Hash15 (only for multibyte patterns)\n"); @@ -251,39 +251,39 @@ static inline u_int32_t WmInitHashRaw(u_int8_t *pat, u_int16_t patlen) { return (hash % INIT_HASH_SIZE); } -static inline int WmInitHashAdd(WmCtx *wm_ctx, WmPattern *p) { +static inline int WmInitHashAdd(WmCtx *ctx, WmPattern *p) { u_int32_t hash = WmInitHash(p); //printf("WmInitHashAdd: %u\n", hash); - if (wm_ctx->init_hash[hash] == NULL) { - wm_ctx->init_hash[hash] = p; - //printf("WmInitHashAdd: hash %u, head %p\n", hash, wm_ctx->init_hash[hash]); + if (ctx->init_hash[hash] == NULL) { + ctx->init_hash[hash] = p; + //printf("WmInitHashAdd: hash %u, head %p\n", hash, ctx->init_hash[hash]); return 0; } - WmPattern *t = wm_ctx->init_hash[hash], *tt = NULL; + WmPattern *t = ctx->init_hash[hash], *tt = NULL; for ( ; t != NULL; t = t->next) { tt = t; } tt->next = p; - //printf("WmInitHashAdd: hash %u, head %p\n", hash, wm_ctx->init_hash[hash]); + //printf("WmInitHashAdd: hash %u, head %p\n", hash, ctx->init_hash[hash]); return 0; } static inline int WmCmpPattern(WmPattern *p, u_int8_t *pat, u_int16_t patlen, char nocase); -static inline WmPattern *WmInitHashLookup(WmCtx *wm_ctx, u_int8_t *pat, u_int16_t patlen, char nocase) { +static inline WmPattern *WmInitHashLookup(WmCtx *ctx, u_int8_t *pat, u_int16_t patlen, char nocase) { u_int32_t hash = WmInitHashRaw(pat,patlen); - //printf("WmInitHashLookup: %u, head %p\n", hash, wm_ctx->init_hash[hash]); + //printf("WmInitHashLookup: %u, head %p\n", hash, ctx->init_hash[hash]); - if (wm_ctx->init_hash[hash] == NULL) { + if (ctx->init_hash[hash] == NULL) { return NULL; } - WmPattern *t = wm_ctx->init_hash[hash]; + WmPattern *t = ctx->init_hash[hash]; for ( ; t != NULL; t = t->next) { if (WmCmpPattern(t,pat,patlen,nocase) == 1) return t; @@ -342,7 +342,7 @@ void WmFreePattern(MpmCtx *mpm_ctx, WmPattern *p) { * sid: signature id (internal id) */ static inline int WmAddPattern(MpmCtx *mpm_ctx, u_int8_t *pat, u_int16_t patlen, u_int16_t offset, u_int16_t depth, char nocase, char scan, u_int32_t pid, u_int32_t sid) { - WmCtx *wm_ctx = (WmCtx *)mpm_ctx->ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx->ctx; // printf("WmAddPattern: ctx %p \"", mpm_ctx); prt(pat, patlen); // printf("\" id %u, nocase %s\n", id, nocase ? "true" : "false"); @@ -351,7 +351,7 @@ static inline int WmAddPattern(MpmCtx *mpm_ctx, u_int8_t *pat, u_int16_t patlen, return 0; /* get a memory piece */ - WmPattern *p = WmInitHashLookup(wm_ctx, pat, patlen, nocase); + WmPattern *p = WmInitHashLookup(ctx, pat, patlen, nocase); if (p == NULL) { // printf("WmAddPattern: allocing new pattern\n"); p = WmAllocPattern(mpm_ctx); @@ -396,7 +396,7 @@ static inline int WmAddPattern(MpmCtx *mpm_ctx, u_int8_t *pat, u_int16_t patlen, //printf("\" prefix_ci %u, prefix_cs %u\n", p->prefix_ci, p->prefix_cs); /* put in the pattern hash */ - WmInitHashAdd(wm_ctx, p); + WmInitHashAdd(ctx, p); if (mpm_ctx->pattern_cnt == 65535) { printf("Max search words reached\n"); @@ -456,137 +456,207 @@ int WmAddPatternCS(MpmCtx *mpm_ctx, u_int8_t *pat, u_int16_t patlen, return WmAddPattern(mpm_ctx, pat, patlen, offset, depth, /* nocase */0, /* scan */0, pid, sid); } +static u_int32_t WmBloomHash(void *data, u_int16_t datalen, u_int8_t iter, u_int32_t hash_size) { + u_int8_t *d = (u_int8_t *)data; + u_int16_t i; + u_int32_t hash = (u_int32_t)wm_tolower(*d); + + for (i = 1; i < datalen - 1; i++) { + hash += (wm_tolower((*d++))) ^ i; + } + hash <<= (iter+1); + + hash %= hash_size; + return hash; +} +/* +static u_int32_t BloomHash(void *data, u_int16_t datalen, u_int8_t iter, u_int32_t hash_size) { + u_int8_t *d = (u_int8_t *)data; + u_int32_t i; + u_int32_t hash = 0; + + for (i = 0; i < datalen; i++) { + if (i == 0) hash += (((u_int32_t)*d++)); + else if (i == 1) hash += (((u_int32_t)*d++) * datalen); + else hash *= (((u_int32_t)*d++) * i); + } + + hash *= (iter + datalen); + hash %= hash_size; + return hash; +} +*/ static void WmScanPrepareHash(MpmCtx *mpm_ctx) { - WmCtx *wm_ctx = (WmCtx *)mpm_ctx->ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx->ctx; u_int16_t i; u_int16_t idx = 0; u_int8_t idx8 = 0; - wm_ctx->scan_hash = (WmHashItem **)malloc(sizeof(WmHashItem *) * wm_ctx->scan_hash_size); - if (wm_ctx->scan_hash == NULL) goto error; - memset(wm_ctx->scan_hash, 0, sizeof(WmHashItem *) * wm_ctx->scan_hash_size); + ctx->scan_hash = (WmHashItem **)malloc(sizeof(WmHashItem *) * ctx->scan_hash_size); + if (ctx->scan_hash == NULL) goto error; + memset(ctx->scan_hash, 0, sizeof(WmHashItem *) * ctx->scan_hash_size); mpm_ctx->memory_cnt++; - mpm_ctx->memory_size += (sizeof(WmHashItem *) * wm_ctx->scan_hash_size); + mpm_ctx->memory_size += (sizeof(WmHashItem *) * ctx->scan_hash_size); + + /* alloc the pminlen array */ + ctx->scan_pminlen = (u_int8_t *)malloc(sizeof(u_int8_t) * ctx->scan_hash_size); + if (ctx->scan_pminlen == NULL) goto error; + memset(ctx->scan_pminlen, 0, sizeof(u_int8_t) * ctx->scan_hash_size); for (i = 0; i < mpm_ctx->pattern_cnt; i++) { /* ignore patterns that don't have the scan flag set */ - if (!(wm_ctx->parray[i]->flags & WUMANBER_SCAN)) + if (!(ctx->parray[i]->flags & WUMANBER_SCAN)) continue; - if(wm_ctx->parray[i]->len == 1) { - idx8 = (u_int8_t)wm_ctx->parray[i]->ci[0]; - if (wm_ctx->scan_hash1[idx8].flags == 0) { - wm_ctx->scan_hash1[idx8].idx = i; - wm_ctx->scan_hash1[idx8].flags |= 0x01; + if(ctx->parray[i]->len == 1) { + idx8 = (u_int8_t)ctx->parray[i]->ci[0]; + if (ctx->scan_hash1[idx8].flags == 0) { + ctx->scan_hash1[idx8].idx = i; + ctx->scan_hash1[idx8].flags |= 0x01; } else { WmHashItem *hi = WmAllocHashItem(mpm_ctx); hi->idx = i; hi->flags |= 0x01; /* Append this HashItem to the list */ - WmHashItem *thi = &wm_ctx->scan_hash1[idx8]; + WmHashItem *thi = &ctx->scan_hash1[idx8]; while (thi->nxt) thi = thi->nxt; thi->nxt = hi; } } else { - u_int16_t patlen = wm_ctx->scan_shiftlen; - - if (wm_ctx->scan_hash_size == HASH9_SIZE) - idx = HASH9(wm_ctx->parray[i]->ci[patlen-1], wm_ctx->parray[i]->ci[patlen-2]); - else if (wm_ctx->scan_hash_size == HASH12_SIZE) - idx = HASH12(wm_ctx->parray[i]->ci[patlen-1], wm_ctx->parray[i]->ci[patlen-2]); - else if (wm_ctx->scan_hash_size == HASH14_SIZE) - idx = HASH14(wm_ctx->parray[i]->ci[patlen-1], wm_ctx->parray[i]->ci[patlen-2]); - else if (wm_ctx->scan_hash_size == HASH15_SIZE) - idx = HASH15(wm_ctx->parray[i]->ci[patlen-1], wm_ctx->parray[i]->ci[patlen-2]); + u_int16_t patlen = ctx->scan_shiftlen; + + if (ctx->scan_hash_size == HASH9_SIZE) + idx = HASH9(ctx->parray[i]->ci[patlen-1], ctx->parray[i]->ci[patlen-2]); + else if (ctx->scan_hash_size == HASH12_SIZE) + idx = HASH12(ctx->parray[i]->ci[patlen-1], ctx->parray[i]->ci[patlen-2]); + else if (ctx->scan_hash_size == HASH14_SIZE) + idx = HASH14(ctx->parray[i]->ci[patlen-1], ctx->parray[i]->ci[patlen-2]); + else if (ctx->scan_hash_size == HASH15_SIZE) + idx = HASH15(ctx->parray[i]->ci[patlen-1], ctx->parray[i]->ci[patlen-2]); else - idx = HASH16(wm_ctx->parray[i]->ci[patlen-1], wm_ctx->parray[i]->ci[patlen-2]); + idx = HASH16(ctx->parray[i]->ci[patlen-1], ctx->parray[i]->ci[patlen-2]); - if (wm_ctx->scan_hash[idx] == NULL) { + if (ctx->scan_hash[idx] == NULL) { WmHashItem *hi = WmAllocHashItem(mpm_ctx); hi->idx = i; hi->flags |= 0x01; + ctx->scan_pminlen[idx] = ctx->parray[i]->len; - wm_ctx->scan_hash[idx] = hi; + ctx->scan_hash[idx] = hi; } else { WmHashItem *hi = WmAllocHashItem(mpm_ctx); hi->idx = i; hi->flags |= 0x01; + if (ctx->parray[i]->len < ctx->scan_pminlen[idx]) + ctx->scan_pminlen[idx] = ctx->parray[i]->len; + /* Append this HashItem to the list */ - WmHashItem *thi = wm_ctx->scan_hash[idx]; + WmHashItem *thi = ctx->scan_hash[idx]; while (thi->nxt) thi = thi->nxt; thi->nxt = hi; } } } + + /* alloc the bloom array */ + ctx->scan_bloom = (BloomFilter **)malloc(sizeof(BloomFilter *) * ctx->scan_hash_size); + if (ctx->scan_bloom == NULL) goto error; + memset(ctx->scan_bloom, 0, sizeof(BloomFilter *) * ctx->scan_hash_size); + + mpm_ctx->memory_cnt++; + mpm_ctx->memory_size += (sizeof(BloomFilter *) * ctx->scan_hash_size); + + int h; + for (h = 0; h < ctx->scan_hash_size; h++) { + WmHashItem *hi = ctx->scan_hash[h]; + if (hi == NULL) + continue; + + ctx->scan_bloom[h] = BloomFilterInit(WUMANBER_BLOOMSIZE, 2, WmBloomHash); + if (ctx->scan_bloom[h] == NULL) + continue; + + mpm_ctx->memory_cnt+=2; /* hackish: bloomfilter itself and the bitarray */ + mpm_ctx->memory_size += BloomFilterMemorySize(ctx->scan_bloom[h]); + + if (ctx->scan_pminlen[h] > 8) + ctx->scan_pminlen[h] = 8; + + WmHashItem *thi = hi; + do { + BloomFilterAdd(ctx->scan_bloom[h], ctx->parray[thi->idx]->ci, ctx->scan_pminlen[h]); + thi = thi->nxt; + } while (thi != NULL); + } return; error: return; } static void WmPrepareHash(MpmCtx *mpm_ctx) { - WmCtx *wm_ctx = (WmCtx *)mpm_ctx->ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx->ctx; u_int16_t i; u_int16_t idx = 0; u_int8_t idx8 = 0; - wm_ctx->search_hash = (WmHashItem **)malloc(sizeof(WmHashItem *) * wm_ctx->search_hash_size); - if (wm_ctx->search_hash == NULL) goto error; - memset(wm_ctx->search_hash, 0, sizeof(WmHashItem *) * wm_ctx->search_hash_size); + ctx->search_hash = (WmHashItem **)malloc(sizeof(WmHashItem *) * ctx->search_hash_size); + if (ctx->search_hash == NULL) goto error; + memset(ctx->search_hash, 0, sizeof(WmHashItem *) * ctx->search_hash_size); mpm_ctx->memory_cnt++; - mpm_ctx->memory_size += (sizeof(WmHashItem *) * wm_ctx->search_hash_size); + mpm_ctx->memory_size += (sizeof(WmHashItem *) * ctx->search_hash_size); for (i = 0; i < mpm_ctx->pattern_cnt; i++) { /* ignore patterns that have the scan flag set */ - if (wm_ctx->parray[i]->flags & WUMANBER_SCAN) + if (ctx->parray[i]->flags & WUMANBER_SCAN) continue; - if(wm_ctx->parray[i]->len == 1) { - idx8 = (u_int8_t)wm_ctx->parray[i]->ci[0]; - if (wm_ctx->search_hash1[idx8].flags == 0) { - wm_ctx->search_hash1[idx8].idx = i; - wm_ctx->search_hash1[idx8].flags |= 0x01; + if(ctx->parray[i]->len == 1) { + idx8 = (u_int8_t)ctx->parray[i]->ci[0]; + if (ctx->search_hash1[idx8].flags == 0) { + ctx->search_hash1[idx8].idx = i; + ctx->search_hash1[idx8].flags |= 0x01; } else { WmHashItem *hi = WmAllocHashItem(mpm_ctx); hi->idx = i; hi->flags |= 0x01; /* Append this HashItem to the list */ - WmHashItem *thi = &wm_ctx->search_hash1[idx8]; + WmHashItem *thi = &ctx->search_hash1[idx8]; while (thi->nxt) thi = thi->nxt; thi->nxt = hi; } } else { - u_int16_t patlen = wm_ctx->search_shiftlen; - - if (wm_ctx->search_hash_size == HASH9_SIZE) - idx = HASH9(wm_ctx->parray[i]->ci[patlen-1], wm_ctx->parray[i]->ci[patlen-2]); - else if (wm_ctx->search_hash_size == HASH12_SIZE) - idx = HASH12(wm_ctx->parray[i]->ci[patlen-1], wm_ctx->parray[i]->ci[patlen-2]); - else if (wm_ctx->search_hash_size == HASH14_SIZE) - idx = HASH14(wm_ctx->parray[i]->ci[patlen-1], wm_ctx->parray[i]->ci[patlen-2]); - else if (wm_ctx->search_hash_size == HASH15_SIZE) - idx = HASH15(wm_ctx->parray[i]->ci[patlen-1], wm_ctx->parray[i]->ci[patlen-2]); + u_int16_t patlen = ctx->search_shiftlen; + + if (ctx->search_hash_size == HASH9_SIZE) + idx = HASH9(ctx->parray[i]->ci[patlen-1], ctx->parray[i]->ci[patlen-2]); + else if (ctx->search_hash_size == HASH12_SIZE) + idx = HASH12(ctx->parray[i]->ci[patlen-1], ctx->parray[i]->ci[patlen-2]); + else if (ctx->search_hash_size == HASH14_SIZE) + idx = HASH14(ctx->parray[i]->ci[patlen-1], ctx->parray[i]->ci[patlen-2]); + else if (ctx->search_hash_size == HASH15_SIZE) + idx = HASH15(ctx->parray[i]->ci[patlen-1], ctx->parray[i]->ci[patlen-2]); else - idx = HASH16(wm_ctx->parray[i]->ci[patlen-1], wm_ctx->parray[i]->ci[patlen-2]); + idx = HASH16(ctx->parray[i]->ci[patlen-1], ctx->parray[i]->ci[patlen-2]); - if (wm_ctx->search_hash[idx] == NULL) { + if (ctx->search_hash[idx] == NULL) { WmHashItem *hi = WmAllocHashItem(mpm_ctx); hi->idx = i; hi->flags |= 0x01; - wm_ctx->search_hash[idx] = hi; + ctx->search_hash[idx] = hi; } else { WmHashItem *hi = WmAllocHashItem(mpm_ctx); hi->idx = i; hi->flags |= 0x01; /* Append this HashItem to the list */ - WmHashItem *thi = wm_ctx->search_hash[idx]; + WmHashItem *thi = ctx->search_hash[idx]; while (thi->nxt) thi = thi->nxt; thi->nxt = hi; } @@ -599,7 +669,7 @@ error: static void WmScanPrepareShiftTable(MpmCtx *mpm_ctx) { - WmCtx *wm_ctx = (WmCtx *)mpm_ctx->ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx->ctx; u_int16_t shift = 0, k = 0, idx = 0; u_int32_t i = 0; @@ -608,92 +678,92 @@ static void WmScanPrepareShiftTable(MpmCtx *mpm_ctx) if (smallest > 255) smallest = 255; if (smallest < 2) smallest = 2; - wm_ctx->scan_shiftlen = smallest; + ctx->scan_shiftlen = smallest; - wm_ctx->scan_shifttable = malloc(sizeof(u_int16_t) * wm_ctx->scan_hash_size); - if (wm_ctx->scan_shifttable == NULL) + ctx->scan_shifttable = malloc(sizeof(u_int16_t) * ctx->scan_hash_size); + if (ctx->scan_shifttable == NULL) return; mpm_ctx->memory_cnt++; - mpm_ctx->memory_size += (sizeof(u_int16_t) * wm_ctx->scan_hash_size); + mpm_ctx->memory_size += (sizeof(u_int16_t) * ctx->scan_hash_size); /* default shift table is set to minimal shift */ - for (i = 0; i < wm_ctx->scan_hash_size; i++) - wm_ctx->scan_shifttable[i] = wm_ctx->scan_shiftlen; + for (i = 0; i < ctx->scan_hash_size; i++) + ctx->scan_shifttable[i] = ctx->scan_shiftlen; for (i = 0; i < mpm_ctx->pattern_cnt; i++) { /* ignore one byte patterns */ - if (wm_ctx->parray[i]->len == 1) + if (ctx->parray[i]->len == 1) continue; /* ignore patterns that don't have the scan flag set */ - if (!(wm_ctx->parray[i]->flags & WUMANBER_SCAN)) + if (!(ctx->parray[i]->flags & WUMANBER_SCAN)) continue; //printf("WmPrepareShiftTable: i = %u ", i); - //prt(wm_ctx->parray[i].ci, wm_ctx->parray[i].len); + //prt(ctx->parray[i].ci, ctx->parray[i].len); /* add the first character of the pattern preceeded by * every possible other character. */ for (k = 0; k < 256; k++) { - shift = wm_ctx->scan_shiftlen - 1; + shift = ctx->scan_shiftlen - 1; if (shift > 255) shift = 255; - if (wm_ctx->scan_hash_size == HASH9_SIZE) { - idx = HASH9(wm_ctx->parray[i]->ci[0], (u_int8_t)k); + if (ctx->scan_hash_size == HASH9_SIZE) { + idx = HASH9(ctx->parray[i]->ci[0], (u_int8_t)k); //printf("HASH9 idx %u\n", idx); - } else if (wm_ctx->scan_hash_size == HASH12_SIZE) { - idx = HASH12(wm_ctx->parray[i]->ci[0], (u_int8_t)k); + } else if (ctx->scan_hash_size == HASH12_SIZE) { + idx = HASH12(ctx->parray[i]->ci[0], (u_int8_t)k); //printf("HASH12 idx %u\n", idx); - } else if (wm_ctx->scan_hash_size == HASH14_SIZE) { - idx = HASH14(wm_ctx->parray[i]->ci[0], (u_int8_t)k); + } else if (ctx->scan_hash_size == HASH14_SIZE) { + idx = HASH14(ctx->parray[i]->ci[0], (u_int8_t)k); //printf("HASH14 idx %u\n", idx); - } else if (wm_ctx->scan_hash_size == HASH15_SIZE) { - idx = HASH15(wm_ctx->parray[i]->ci[0], (u_int8_t)k); + } else if (ctx->scan_hash_size == HASH15_SIZE) { + idx = HASH15(ctx->parray[i]->ci[0], (u_int8_t)k); //printf("HASH15 idx %u\n", idx); } else { - idx = HASH16(wm_ctx->parray[i]->ci[0], (u_int8_t)k); + idx = HASH16(ctx->parray[i]->ci[0], (u_int8_t)k); //printf("HASH15 idx %u\n", idx); } - if (shift < wm_ctx->scan_shifttable[idx]) { - wm_ctx->scan_shifttable[idx] = shift; + if (shift < ctx->scan_shifttable[idx]) { + ctx->scan_shifttable[idx] = shift; } } - for (k = 0; k < wm_ctx->scan_shiftlen-1; k++) + for (k = 0; k < ctx->scan_shiftlen-1; k++) { - shift = (wm_ctx->scan_shiftlen - 2 - k); + shift = (ctx->scan_shiftlen - 2 - k); if (shift > 255) shift = 255; - if (wm_ctx->scan_hash_size == HASH9_SIZE) { - idx = HASH9(wm_ctx->parray[i]->ci[k+1], wm_ctx->parray[i]->ci[k]); + if (ctx->scan_hash_size == HASH9_SIZE) { + idx = HASH9(ctx->parray[i]->ci[k+1], ctx->parray[i]->ci[k]); //printf("HASH9 idx %u\n", idx); - } else if (wm_ctx->scan_hash_size == HASH12_SIZE) { - idx = HASH12(wm_ctx->parray[i]->ci[k+1], wm_ctx->parray[i]->ci[k]); + } else if (ctx->scan_hash_size == HASH12_SIZE) { + idx = HASH12(ctx->parray[i]->ci[k+1], ctx->parray[i]->ci[k]); //printf("HASH12 idx %u\n", idx); - } else if (wm_ctx->scan_hash_size == HASH14_SIZE) { - idx = HASH14(wm_ctx->parray[i]->ci[k+1], wm_ctx->parray[i]->ci[k]); + } else if (ctx->scan_hash_size == HASH14_SIZE) { + idx = HASH14(ctx->parray[i]->ci[k+1], ctx->parray[i]->ci[k]); //printf("HASH14 idx %u\n", idx); - } else if (wm_ctx->scan_hash_size == HASH15_SIZE) { - idx = HASH15(wm_ctx->parray[i]->ci[k+1], wm_ctx->parray[i]->ci[k]); + } else if (ctx->scan_hash_size == HASH15_SIZE) { + idx = HASH15(ctx->parray[i]->ci[k+1], ctx->parray[i]->ci[k]); //printf("HASH15 idx %u\n", idx); } else { - idx = HASH16(wm_ctx->parray[i]->ci[k+1], wm_ctx->parray[i]->ci[k]); + idx = HASH16(ctx->parray[i]->ci[k+1], ctx->parray[i]->ci[k]); //printf("HASH15 idx %u\n", idx); } - if (shift < wm_ctx->scan_shifttable[idx]) { - wm_ctx->scan_shifttable[idx] = shift; + if (shift < ctx->scan_shifttable[idx]) { + ctx->scan_shifttable[idx] = shift; } //printf("WmPrepareShiftTable: i %u, k %u, idx %u, shift set to %u: \"%c%c\"\n", - // i, k, idx, shift, wm_ctx->parray[i]->ci[k], wm_ctx->parray[i]->ci[k+1]); + // i, k, idx, shift, ctx->parray[i]->ci[k], ctx->parray[i]->ci[k+1]); } } } static void WmPrepareShiftTable(MpmCtx *mpm_ctx) { - WmCtx *wm_ctx = (WmCtx *)mpm_ctx->ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx->ctx; u_int16_t shift = 0, k = 0, idx = 0; u_int32_t i = 0; @@ -702,154 +772,154 @@ static void WmPrepareShiftTable(MpmCtx *mpm_ctx) if (smallest > 255) smallest = 255; if (smallest < 2) smallest = 2; - wm_ctx->search_shiftlen = smallest; + ctx->search_shiftlen = smallest; - wm_ctx->search_shifttable = malloc(sizeof(u_int16_t) * wm_ctx->search_hash_size); - if (wm_ctx->search_shifttable == NULL) + ctx->search_shifttable = malloc(sizeof(u_int16_t) * ctx->search_hash_size); + if (ctx->search_shifttable == NULL) return; mpm_ctx->memory_cnt++; - mpm_ctx->memory_size += (sizeof(u_int16_t) * wm_ctx->search_hash_size); + mpm_ctx->memory_size += (sizeof(u_int16_t) * ctx->search_hash_size); /* default shift table is set to minimal shift */ - for (i = 0; i < wm_ctx->search_hash_size; i++) - wm_ctx->search_shifttable[i] = wm_ctx->search_shiftlen; + for (i = 0; i < ctx->search_hash_size; i++) + ctx->search_shifttable[i] = ctx->search_shiftlen; for (i = 0; i < mpm_ctx->pattern_cnt; i++) { /* ignore one byte patterns */ - if (wm_ctx->parray[i]->len == 1) + if (ctx->parray[i]->len == 1) continue; /* ignore patterns that have the scan flag set */ - if (wm_ctx->parray[i]->flags & WUMANBER_SCAN) + if (ctx->parray[i]->flags & WUMANBER_SCAN) continue; //printf("WmPrepareShiftTable: i = %u ", i); - //prt(wm_ctx->parray[i].ci, wm_ctx->parray[i].len); + //prt(ctx->parray[i].ci, ctx->parray[i].len); /* add the first character of the pattern preceeded by * every possible other character. */ for (k = 0; k < 256; k++) { - shift = wm_ctx->search_shiftlen - 1; + shift = ctx->search_shiftlen - 1; if (shift > 255) shift = 255; - if (wm_ctx->search_hash_size == HASH9_SIZE) { - idx = HASH9(wm_ctx->parray[i]->ci[0], (u_int8_t)k); + if (ctx->search_hash_size == HASH9_SIZE) { + idx = HASH9(ctx->parray[i]->ci[0], (u_int8_t)k); //printf("HASH9 idx %u\n", idx); - } else if (wm_ctx->search_hash_size == HASH12_SIZE) { - idx = HASH12(wm_ctx->parray[i]->ci[0], (u_int8_t)k); + } else if (ctx->search_hash_size == HASH12_SIZE) { + idx = HASH12(ctx->parray[i]->ci[0], (u_int8_t)k); //printf("HASH12 idx %u\n", idx); - } else if (wm_ctx->search_hash_size == HASH14_SIZE) { - idx = HASH14(wm_ctx->parray[i]->ci[0], (u_int8_t)k); + } else if (ctx->search_hash_size == HASH14_SIZE) { + idx = HASH14(ctx->parray[i]->ci[0], (u_int8_t)k); //printf("HASH14 idx %u\n", idx); - } else if (wm_ctx->search_hash_size == HASH15_SIZE) { - idx = HASH15(wm_ctx->parray[i]->ci[0], (u_int8_t)k); + } else if (ctx->search_hash_size == HASH15_SIZE) { + idx = HASH15(ctx->parray[i]->ci[0], (u_int8_t)k); //printf("HASH15 idx %u\n", idx); } else { - idx = HASH16(wm_ctx->parray[i]->ci[0], (u_int8_t)k); + idx = HASH16(ctx->parray[i]->ci[0], (u_int8_t)k); //printf("HASH15 idx %u\n", idx); } - if (shift < wm_ctx->search_shifttable[idx]) { - wm_ctx->search_shifttable[idx] = shift; + if (shift < ctx->search_shifttable[idx]) { + ctx->search_shifttable[idx] = shift; } } - for (k = 0; k < wm_ctx->search_shiftlen - 1; k++) + for (k = 0; k < ctx->search_shiftlen - 1; k++) { - shift = (wm_ctx->search_shiftlen - 2 - k); + shift = (ctx->search_shiftlen - 2 - k); if (shift > 255) shift = 255; - if (wm_ctx->search_hash_size == HASH9_SIZE) { - idx = HASH9(wm_ctx->parray[i]->ci[k+1], wm_ctx->parray[i]->ci[k]); + if (ctx->search_hash_size == HASH9_SIZE) { + idx = HASH9(ctx->parray[i]->ci[k+1], ctx->parray[i]->ci[k]); //printf("HASH9 idx %u\n", idx); - } else if (wm_ctx->search_hash_size == HASH12_SIZE) { - idx = HASH12(wm_ctx->parray[i]->ci[k+1], wm_ctx->parray[i]->ci[k]); + } else if (ctx->search_hash_size == HASH12_SIZE) { + idx = HASH12(ctx->parray[i]->ci[k+1], ctx->parray[i]->ci[k]); //printf("HASH12 idx %u\n", idx); - } else if (wm_ctx->search_hash_size == HASH14_SIZE) { - idx = HASH14(wm_ctx->parray[i]->ci[k+1], wm_ctx->parray[i]->ci[k]); + } else if (ctx->search_hash_size == HASH14_SIZE) { + idx = HASH14(ctx->parray[i]->ci[k+1], ctx->parray[i]->ci[k]); //printf("HASH14 idx %u\n", idx); - } else if (wm_ctx->search_hash_size == HASH15_SIZE) { - idx = HASH15(wm_ctx->parray[i]->ci[k+1], wm_ctx->parray[i]->ci[k]); + } else if (ctx->search_hash_size == HASH15_SIZE) { + idx = HASH15(ctx->parray[i]->ci[k+1], ctx->parray[i]->ci[k]); //printf("HASH15 idx %u\n", idx); } else { - idx = HASH16(wm_ctx->parray[i]->ci[k+1], wm_ctx->parray[i]->ci[k]); + idx = HASH16(ctx->parray[i]->ci[k+1], ctx->parray[i]->ci[k]); //printf("HASH15 idx %u\n", idx); } - if (shift < wm_ctx->search_shifttable[idx]) { - wm_ctx->search_shifttable[idx] = shift; + if (shift < ctx->search_shifttable[idx]) { + ctx->search_shifttable[idx] = shift; } //printf("WmPrepareShiftTable: i %u, k %u, idx %u, shift set to %u: \"%c%c\"\n", - // i, k, idx, shift, wm_ctx->parray[i]->ci[k], wm_ctx->parray[i]->ci[k+1]); + // i, k, idx, shift, ctx->parray[i]->ci[k], ctx->parray[i]->ci[k+1]); } } } int WmPreparePatterns(MpmCtx *mpm_ctx) { - WmCtx *wm_ctx = (WmCtx *)mpm_ctx->ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx->ctx; /* alloc the pattern array */ - wm_ctx->parray = (WmPattern **)malloc(mpm_ctx->pattern_cnt * sizeof(WmPattern *)); - if (wm_ctx->parray == NULL) goto error; - memset(wm_ctx->parray, 0, mpm_ctx->pattern_cnt * sizeof(WmPattern *)); - //printf("mpm_ctx %p, parray %p\n", mpm_ctx,wm_ctx->parray); + ctx->parray = (WmPattern **)malloc(mpm_ctx->pattern_cnt * sizeof(WmPattern *)); + if (ctx->parray == NULL) goto error; + memset(ctx->parray, 0, mpm_ctx->pattern_cnt * sizeof(WmPattern *)); + //printf("mpm_ctx %p, parray %p\n", mpm_ctx,ctx->parray); mpm_ctx->memory_cnt++; mpm_ctx->memory_size += (mpm_ctx->pattern_cnt * sizeof(WmPattern *)); /* populate it with the patterns in the hash */ u_int32_t i = 0, p = 0; for (i = 0; i < INIT_HASH_SIZE; i++) { - WmPattern *node = wm_ctx->init_hash[i], *nnode = NULL; + WmPattern *node = ctx->init_hash[i], *nnode = NULL; for ( ; node != NULL; ) { nnode = node->next; node->next = NULL; - wm_ctx->parray[p] = node; + ctx->parray[p] = node; p++; node = nnode; } } /* we no longer need the hash, so free it's memory */ - free(wm_ctx->init_hash); - wm_ctx->init_hash = NULL; + free(ctx->init_hash); + ctx->init_hash = NULL; /* TODO VJ these values are chosen pretty much randomly, so * we should do some performance testing * */ /* scan */ - if (wm_ctx->scan_hash_size == 0) { + if (ctx->scan_hash_size == 0) { if (mpm_ctx->scan_pattern_cnt < 50) { - wm_ctx->scan_hash_size = HASH9_SIZE; + ctx->scan_hash_size = HASH9_SIZE; } else if(mpm_ctx->scan_pattern_cnt < 300) { - wm_ctx->scan_hash_size = HASH12_SIZE; + ctx->scan_hash_size = HASH12_SIZE; } else if(mpm_ctx->scan_pattern_cnt < 1200) { - wm_ctx->scan_hash_size = HASH14_SIZE; + ctx->scan_hash_size = HASH14_SIZE; } else if(mpm_ctx->scan_pattern_cnt < 2400) { - wm_ctx->scan_hash_size = HASH15_SIZE; + ctx->scan_hash_size = HASH15_SIZE; } else { - wm_ctx->scan_hash_size = HASH16_SIZE; + ctx->scan_hash_size = HASH16_SIZE; } } WmScanPrepareShiftTable(mpm_ctx); WmScanPrepareHash(mpm_ctx); - if (wm_ctx->scan_hash_size == HASH9_SIZE) { - wm_ctx->MBScan = WmScan2Hash9; + if (ctx->scan_hash_size == HASH9_SIZE) { + ctx->MBScan = WmScan2Hash9; mpm_ctx->Scan = WmScan2Hash9; - } else if (wm_ctx->scan_hash_size == HASH12_SIZE) { - wm_ctx->MBScan = WmScan2Hash12; + } else if (ctx->scan_hash_size == HASH12_SIZE) { + ctx->MBScan = WmScan2Hash12; mpm_ctx->Scan = WmScan2Hash12; - } else if (wm_ctx->scan_hash_size == HASH14_SIZE) { - wm_ctx->MBScan = WmScan2Hash14; + } else if (ctx->scan_hash_size == HASH14_SIZE) { + ctx->MBScan = WmScan2Hash14; mpm_ctx->Scan = WmScan2Hash14; - } else if (wm_ctx->scan_hash_size == HASH15_SIZE) { - wm_ctx->MBScan = WmScan2Hash15; + } else if (ctx->scan_hash_size == HASH15_SIZE) { + ctx->MBScan = WmScan2Hash15; mpm_ctx->Scan = WmScan2Hash15; } else { - wm_ctx->MBScan = WmScan2Hash16; + ctx->MBScan = WmScan2Hash16; mpm_ctx->Scan = WmScan2Hash16; } @@ -858,37 +928,37 @@ int WmPreparePatterns(MpmCtx *mpm_ctx) { } /* search XXX use search only pat cnt*/ - if (wm_ctx->search_hash_size == 0) { + if (ctx->search_hash_size == 0) { if (mpm_ctx->pattern_cnt < 50) { - wm_ctx->search_hash_size = HASH9_SIZE; + ctx->search_hash_size = HASH9_SIZE; } else if(mpm_ctx->pattern_cnt < 300) { - wm_ctx->search_hash_size = HASH12_SIZE; + ctx->search_hash_size = HASH12_SIZE; } else if(mpm_ctx->pattern_cnt < 1200) { - wm_ctx->search_hash_size = HASH14_SIZE; + ctx->search_hash_size = HASH14_SIZE; } else if(mpm_ctx->pattern_cnt < 2400) { - wm_ctx->search_hash_size = HASH15_SIZE; + ctx->search_hash_size = HASH15_SIZE; } else { - wm_ctx->search_hash_size = HASH16_SIZE; + ctx->search_hash_size = HASH16_SIZE; } } WmPrepareShiftTable(mpm_ctx); WmPrepareHash(mpm_ctx); - if (wm_ctx->search_hash_size == HASH9_SIZE) { - wm_ctx->MBSearch = WmSearch2Hash9; + if (ctx->search_hash_size == HASH9_SIZE) { + ctx->MBSearch = WmSearch2Hash9; mpm_ctx->Search = WmSearch2Hash9; - } else if (wm_ctx->search_hash_size == HASH12_SIZE) { - wm_ctx->MBSearch = WmSearch2Hash12; + } else if (ctx->search_hash_size == HASH12_SIZE) { + ctx->MBSearch = WmSearch2Hash12; mpm_ctx->Search = WmSearch2Hash12; - } else if (wm_ctx->search_hash_size == HASH14_SIZE) { - wm_ctx->MBSearch = WmSearch2Hash14; + } else if (ctx->search_hash_size == HASH14_SIZE) { + ctx->MBSearch = WmSearch2Hash14; mpm_ctx->Search = WmSearch2Hash14; - } else if (wm_ctx->search_hash_size == HASH15_SIZE) { - wm_ctx->MBSearch = WmSearch2Hash15; + } else if (ctx->search_hash_size == HASH15_SIZE) { + ctx->MBSearch = WmSearch2Hash15; mpm_ctx->Search = WmSearch2Hash15; } else { - wm_ctx->MBSearch = WmSearch2Hash16; + ctx->MBSearch = WmSearch2Hash16; mpm_ctx->Search = WmSearch2Hash16; } @@ -903,19 +973,19 @@ error: void WmPrintSearchStats(MpmThreadCtx *mpm_thread_ctx) { #ifdef WUMANBER_COUNTERS - WmThreadCtx *wm_thread_ctx = (WmThreadCtx *)mpm_thread_ctx->ctx; - - printf("Shift 0: %u\n", wm_thread_ctx->scan_stat_shift_null); - printf("Loop match: %u\n", wm_thread_ctx->scan_stat_loop_match); - printf("Loop no match: %u\n", wm_thread_ctx->scan_stat_loop_no_match); - printf("Num shifts: %u\n", wm_thread_ctx->scan_stat_num_shift); - printf("Total shifts: %u\n", wm_thread_ctx->scan_stat_total_shift); - - printf("Shift 0: %u\n", wm_thread_ctx->search_stat_shift_null); - printf("Loop match: %u\n", wm_thread_ctx->search_stat_loop_match); - printf("Loop no match: %u\n", wm_thread_ctx->search_stat_loop_no_match); - printf("Num shifts: %u\n", wm_thread_ctx->search_stat_num_shift); - printf("Total shifts: %u\n", wm_thread_ctx->search_stat_total_shift); + WmThreadCtx *tctx = (WmThreadCtx *)mpm_thread_ctx->ctx; + + printf("Shift 0: %u\n", tctx->scan_stat_shift_null); + printf("Loop match: %u\n", tctx->scan_stat_loop_match); + printf("Loop no match: %u\n", tctx->scan_stat_loop_no_match); + printf("Num shifts: %u\n", tctx->scan_stat_num_shift); + printf("Total shifts: %u\n", tctx->scan_stat_total_shift); + + printf("Shift 0: %u\n", tctx->search_stat_shift_null); + printf("Loop match: %u\n", tctx->search_stat_loop_match); + printf("Loop no match: %u\n", tctx->search_stat_loop_no_match); + printf("Num shifts: %u\n", tctx->search_stat_num_shift); + printf("Total shifts: %u\n", tctx->search_stat_total_shift); #endif /* WUMANBER_COUNTERS */ } @@ -936,14 +1006,14 @@ memcmp_lowercase(u_int8_t *s1, u_int8_t *s2, u_int16_t n) { /* SCAN FUNCTIONS */ u_int32_t WmScan2Hash9(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *pmq, u_int8_t *buf, u_int16_t buflen) { - WmCtx *wm_ctx = (WmCtx *)mpm_ctx->ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx->ctx; #ifdef WUMANBER_COUNTERS - WmThreadCtx *wm_thread_ctx = (WmThreadCtx *)mpm_thread_ctx->ctx; + WmThreadCtx *tctx = (WmThreadCtx *)mpm_thread_ctx->ctx; #endif /* WUMANBER_COUNTERS */ u_int32_t cnt = 0; u_int8_t *bufmin = buf; u_int8_t *bufend = buf + buflen - 1; - u_int16_t sl = wm_ctx->scan_shiftlen; + u_int16_t sl = ctx->scan_shiftlen; u_int16_t h; u_int8_t shift; WmHashItem *thi, *hi; @@ -960,20 +1030,37 @@ u_int32_t WmScan2Hash9(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMat while (buf <= bufend) { h = HASH9(wm_tolower(*buf),(wm_tolower(*(buf-1)))); - shift = wm_ctx->scan_shifttable[h]; + shift = ctx->scan_shifttable[h]; //printf("%p %u search: h %u, shift %u\n", buf, buf - bufmin, h, shift); if (shift == 0) { - COUNT(wm_thread_ctx->scan_stat_shift_null++); - /* get our hash item */ - hi = wm_ctx->scan_hash[h]; + COUNT(tctx->scan_stat_shift_null++); + + hi = ctx->scan_hash[h]; + //printf("search: hi %p\n", hi); if (hi != NULL) { - //printf("buf-sl+1 %p, buf-sl+2 %p\n", buf-sl+1, buf-sl+2); + /* get our patterns from the hash */ + if (ctx->scan_bloom[h] != NULL) { + COUNT(tctx->scan_stat_pminlen_calls++); + COUNT(tctx->scan_stat_pminlen_total+=ctx->scan_pminlen[h]); + + if ((bufend - (buf-sl)) < ctx->scan_pminlen[h]) { + goto skip_loop; + } else { + COUNT(tctx->scan_stat_bloom_calls++); + + if (BloomFilterTest(ctx->scan_bloom[h], buf-sl+1, ctx->scan_pminlen[h]) == 0) { + COUNT(tctx->scan_stat_bloom_hits++); + goto skip_loop; + } + } + } + prefixci_buf = (u_int16_t)(wm_tolower(*(buf-sl+1)) + wm_tolower(*(buf-sl+2))); prefixcs_buf = (u_int16_t)(*(buf-sl+1) + *(buf-sl+2)); - //printf("WmSearch2: prefixci_buf %u, prefixcs_buf %u\n", prefixci_buf, prefixcs_buf); + for (thi = hi; thi != NULL; thi = thi->nxt) { - p = wm_ctx->parray[thi->idx]; + p = ctx->parray[thi->idx]; //printf("WmSearch2: p->prefix_ci %u, p->prefix_cs %u\n", // p->prefix_ci, p->prefix_cs); @@ -984,7 +1071,7 @@ u_int32_t WmScan2Hash9(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMat if (memcmp_lowercase(p->ci, buf-sl+1, p->len) == 0) { //printf("CI Exact match: "); prt(p->ci, p->len); printf("\n"); - COUNT(wm_thread_ctx->scan_stat_loop_match++); + COUNT(tctx->scan_stat_loop_match++); MpmEndMatch *em; for (em = p->em; em; em = em->next) { @@ -994,14 +1081,14 @@ u_int32_t WmScan2Hash9(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMat } } else { - COUNT(wm_thread_ctx->scan_stat_loop_no_match++); + COUNT(tctx->scan_stat_loop_no_match++); } } else { if (p->prefix_cs != prefixcs_buf || p->len > (bufend-(buf-sl))) continue; if (memcmp(p->cs, buf-sl+1, p->len) == 0) { //printf("CS Exact match: "); prt(p->cs, p->len); printf("\n"); - COUNT(wm_thread_ctx->scan_stat_loop_match++); + COUNT(tctx->scan_stat_loop_match++); MpmEndMatch *em; for (em = p->em; em; em = em->next) { @@ -1011,15 +1098,16 @@ u_int32_t WmScan2Hash9(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMat } } else { - COUNT(wm_thread_ctx->scan_stat_loop_no_match++); + COUNT(tctx->scan_stat_loop_no_match++); } } } } +skip_loop: shift = 1; } else { - COUNT(wm_thread_ctx->scan_stat_total_shift += shift); - COUNT(wm_thread_ctx->scan_stat_num_shift++); + COUNT(tctx->scan_stat_total_shift += shift); + COUNT(tctx->scan_stat_num_shift++); } buf += shift; } @@ -1029,14 +1117,14 @@ u_int32_t WmScan2Hash9(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMat } u_int32_t WmScan2Hash12(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *pmq, u_int8_t *buf, u_int16_t buflen) { - WmCtx *wm_ctx = (WmCtx *)mpm_ctx->ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx->ctx; #ifdef WUMANBER_COUNTERS - WmThreadCtx *wm_thread_ctx = (WmThreadCtx *)mpm_thread_ctx->ctx; + WmThreadCtx *tctx = (WmThreadCtx *)mpm_thread_ctx->ctx; #endif /* WUMANBER_COUNTERS */ u_int32_t cnt = 0; u_int8_t *bufmin = buf; u_int8_t *bufend = buf + buflen - 1; - u_int16_t sl = wm_ctx->scan_shiftlen; + u_int16_t sl = ctx->scan_shiftlen; u_int16_t h; u_int8_t shift; WmHashItem *thi, *hi; @@ -1055,20 +1143,37 @@ u_int32_t WmScan2Hash12(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMa while (buf <= bufend) { //h = (wm_tolower(*buf)<<8)+(wm_tolower(*(buf-1))); h = HASH12(wm_tolower(*buf),(wm_tolower(*(buf-1)))); - shift = wm_ctx->scan_shifttable[h]; + shift = ctx->scan_shifttable[h]; //printf("search: h %u, shift %u\n", h, shift); if (shift == 0) { - COUNT(wm_thread_ctx->scan_stat_shift_null++); + COUNT(tctx->scan_stat_shift_null++); /* get our hash item */ - hi = wm_ctx->scan_hash[h]; + hi = ctx->scan_hash[h]; //printf("search: hi %p\n", hi); if (hi != NULL) { + /* get our patterns from the hash */ + if (ctx->scan_bloom[h] != NULL) { + COUNT(tctx->scan_stat_pminlen_calls++); + COUNT(tctx->scan_stat_pminlen_total+=ctx->scan_pminlen[h]); + + if ((bufend - (buf-sl)) < ctx->scan_pminlen[h]) { + goto skip_loop; + } else { + COUNT(tctx->scan_stat_bloom_calls++); + + if (BloomFilterTest(ctx->scan_bloom[h], buf-sl+1, ctx->scan_pminlen[h]) == 0) { + COUNT(tctx->scan_stat_bloom_hits++); + goto skip_loop; + } + } + } + prefixci_buf = (u_int16_t)(wm_tolower(*(buf-sl+1)) + wm_tolower(*(buf-sl+2))); prefixcs_buf = (u_int16_t)(*(buf-sl+1) + *(buf-sl+2)); //printf("WmSearch2: prefixci_buf %u, prefixcs_buf %u\n", prefixci_buf, prefixcs_buf); for (thi = hi; thi != NULL; thi = thi->nxt) { - p = wm_ctx->parray[thi->idx]; + p = ctx->parray[thi->idx]; //printf("WmSearch2: p->prefix_ci %u, p->prefix_cs %u\n", // p->prefix_ci, p->prefix_cs); @@ -1079,7 +1184,7 @@ u_int32_t WmScan2Hash12(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMa if (memcmp_lowercase(p->ci, buf-sl+1, p->len) == 0) { //printf("CI Exact match: "); prt(p->ci, p->len); printf("\n"); - COUNT(wm_thread_ctx->scan_stat_loop_match++); + COUNT(tctx->scan_stat_loop_match++); MpmEndMatch *em; for (em = p->em; em; em = em->next) { @@ -1089,14 +1194,14 @@ u_int32_t WmScan2Hash12(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMa } } else { - COUNT(wm_thread_ctx->scan_stat_loop_no_match++); + COUNT(tctx->scan_stat_loop_no_match++); } } else { if (p->prefix_cs != prefixcs_buf || p->len > (bufend-(buf-sl))) continue; if (memcmp(p->cs, buf-sl+1, p->len) == 0) { //printf("CS Exact match: "); prt(p->cs, p->len); printf("\n"); - COUNT(wm_thread_ctx->scan_stat_loop_match++); + COUNT(tctx->scan_stat_loop_match++); MpmEndMatch *em; for (em = p->em; em; em = em->next) { @@ -1106,15 +1211,16 @@ u_int32_t WmScan2Hash12(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMa } } else { - COUNT(wm_thread_ctx->scan_stat_loop_no_match++); + COUNT(tctx->scan_stat_loop_no_match++); } } } } +skip_loop: shift = 1; } else { - COUNT(wm_thread_ctx->scan_stat_total_shift += shift); - COUNT(wm_thread_ctx->scan_stat_num_shift++); + COUNT(tctx->scan_stat_total_shift += shift); + COUNT(tctx->scan_stat_num_shift++); } buf += shift; } @@ -1123,14 +1229,14 @@ u_int32_t WmScan2Hash12(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMa } u_int32_t WmScan2Hash14(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *pmq, u_int8_t *buf, u_int16_t buflen) { - WmCtx *wm_ctx = (WmCtx *)mpm_ctx->ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx->ctx; #ifdef WUMANBER_COUNTERS - WmThreadCtx *wm_thread_ctx = (WmThreadCtx *)mpm_thread_ctx->ctx; + WmThreadCtx *tctx = (WmThreadCtx *)mpm_thread_ctx->ctx; #endif /* WUMANBER_COUNTERS */ u_int32_t cnt = 0; u_int8_t *bufmin = buf; u_int8_t *bufend = buf + buflen - 1; - u_int16_t sl = wm_ctx->scan_shiftlen; + u_int16_t sl = ctx->scan_shiftlen; u_int16_t h; u_int8_t shift; WmHashItem *thi, *hi; @@ -1149,20 +1255,37 @@ u_int32_t WmScan2Hash14(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMa while (buf <= bufend) { //h = (wm_tolower(*buf)<<8)+(wm_tolower(*(buf-1))); h = HASH14(wm_tolower(*buf),(wm_tolower(*(buf-1)))); - shift = wm_ctx->scan_shifttable[h]; + shift = ctx->scan_shifttable[h]; //printf("search: h %u, shift %u\n", h, shift); if (shift == 0) { - COUNT(wm_thread_ctx->scan_stat_shift_null++); + COUNT(tctx->scan_stat_shift_null++); /* get our hash item */ - hi = wm_ctx->scan_hash[h]; + hi = ctx->scan_hash[h]; //printf("search: hi %p\n", hi); if (hi != NULL) { + /* get our patterns from the hash */ + if (ctx->scan_bloom[h] != NULL) { + COUNT(tctx->scan_stat_pminlen_calls++); + COUNT(tctx->scan_stat_pminlen_total+=ctx->scan_pminlen[h]); + + if ((bufend - (buf-sl)) < ctx->scan_pminlen[h]) { + goto skip_loop; + } else { + COUNT(tctx->scan_stat_bloom_calls++); + + if (BloomFilterTest(ctx->scan_bloom[h], buf-sl+1, ctx->scan_pminlen[h]) == 0) { + COUNT(tctx->scan_stat_bloom_hits++); + goto skip_loop; + } + } + } + prefixci_buf = (u_int16_t)(wm_tolower(*(buf-sl+1)) + wm_tolower(*(buf-sl+2))); prefixcs_buf = (u_int16_t)(*(buf-sl+1) + *(buf-sl+2)); //printf("WmSearch2: prefixci_buf %u, prefixcs_buf %u\n", prefixci_buf, prefixcs_buf); for (thi = hi; thi != NULL; thi = thi->nxt) { - p = wm_ctx->parray[thi->idx]; + p = ctx->parray[thi->idx]; //printf("WmSearch2: p->prefix_ci %u, p->prefix_cs %u\n", // p->prefix_ci, p->prefix_cs); @@ -1173,7 +1296,7 @@ u_int32_t WmScan2Hash14(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMa if (memcmp_lowercase(p->ci, buf-sl+1, p->len) == 0) { //printf("CI Exact match: "); prt(p->ci, p->len); printf("\n"); - COUNT(wm_thread_ctx->scan_stat_loop_match++); + COUNT(tctx->scan_stat_loop_match++); MpmEndMatch *em; for (em = p->em; em; em = em->next) { @@ -1183,14 +1306,14 @@ u_int32_t WmScan2Hash14(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMa } } else { - COUNT(wm_thread_ctx->scan_stat_loop_no_match++); + COUNT(tctx->scan_stat_loop_no_match++); } } else { if (p->prefix_cs != prefixcs_buf || p->len > (bufend-(buf-sl))) continue; if (memcmp(p->cs, buf-sl+1, p->len) == 0) { //printf("CS Exact match: "); prt(p->cs, p->len); printf("\n"); - COUNT(wm_thread_ctx->scan_stat_loop_match++); + COUNT(tctx->scan_stat_loop_match++); MpmEndMatch *em; for (em = p->em; em; em = em->next) { @@ -1200,15 +1323,16 @@ u_int32_t WmScan2Hash14(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMa } } else { - COUNT(wm_thread_ctx->scan_stat_loop_no_match++); + COUNT(tctx->scan_stat_loop_no_match++); } } } } +skip_loop: shift = 1; } else { - COUNT(wm_thread_ctx->scan_stat_total_shift += shift); - COUNT(wm_thread_ctx->scan_stat_num_shift++); + COUNT(tctx->scan_stat_total_shift += shift); + COUNT(tctx->scan_stat_num_shift++); } buf += shift; } @@ -1217,14 +1341,14 @@ u_int32_t WmScan2Hash14(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMa } u_int32_t WmScan2Hash15(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *pmq, u_int8_t *buf, u_int16_t buflen) { - WmCtx *wm_ctx = (WmCtx *)mpm_ctx->ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx->ctx; #ifdef WUMANBER_COUNTERS - WmThreadCtx *wm_thread_ctx = (WmThreadCtx *)mpm_thread_ctx->ctx; + WmThreadCtx *tctx = (WmThreadCtx *)mpm_thread_ctx->ctx; #endif /* WUMANBER_COUNTERS */ u_int32_t cnt = 0; u_int8_t *bufmin = buf; u_int8_t *bufend = buf + buflen - 1; - u_int16_t sl = wm_ctx->scan_shiftlen; + u_int16_t sl = ctx->scan_shiftlen; u_int16_t h; u_int8_t shift; WmHashItem *thi, *hi; @@ -1243,20 +1367,37 @@ u_int32_t WmScan2Hash15(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMa while (buf <= bufend) { //h = (wm_tolower(*buf)<<8)+(wm_tolower(*(buf-1))); h = HASH15(wm_tolower(*buf),(wm_tolower(*(buf-1)))); - shift = wm_ctx->scan_shifttable[h]; + shift = ctx->scan_shifttable[h]; //printf("search: h %u, shift %u\n", h, shift); if (shift == 0) { - COUNT(wm_thread_ctx->scan_stat_shift_null++); + COUNT(tctx->scan_stat_shift_null++); /* get our hash item */ - hi = wm_ctx->scan_hash[h]; + hi = ctx->scan_hash[h]; //printf("search: hi %p\n", hi); if (hi != NULL) { + /* get our patterns from the hash */ + if (ctx->scan_bloom[h] != NULL) { + COUNT(tctx->scan_stat_pminlen_calls++); + COUNT(tctx->scan_stat_pminlen_total+=ctx->scan_pminlen[h]); + + if ((bufend - (buf-sl)) < ctx->scan_pminlen[h]) { + goto skip_loop; + } else { + COUNT(tctx->scan_stat_bloom_calls++); + + if (BloomFilterTest(ctx->scan_bloom[h], buf-sl+1, ctx->scan_pminlen[h]) == 0) { + COUNT(tctx->scan_stat_bloom_hits++); + goto skip_loop; + } + } + } + prefixci_buf = (u_int16_t)(wm_tolower(*(buf-sl+1)) + wm_tolower(*(buf-sl+2))); prefixcs_buf = (u_int16_t)(*(buf-sl+1) + *(buf-sl+2)); //printf("WmSearch2: prefixci_buf %u, prefixcs_buf %u\n", prefixci_buf, prefixcs_buf); for (thi = hi; thi != NULL; thi = thi->nxt) { - p = wm_ctx->parray[thi->idx]; + p = ctx->parray[thi->idx]; //printf("WmSearch2: p->prefix_ci %u, p->prefix_cs %u\n", // p->prefix_ci, p->prefix_cs); @@ -1267,7 +1408,7 @@ u_int32_t WmScan2Hash15(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMa if (memcmp_lowercase(p->ci, buf-sl+1, p->len) == 0) { //printf("CI Exact match: "); prt(p->ci, p->len); printf("\n"); - COUNT(wm_thread_ctx->scan_stat_loop_match++); + COUNT(tctx->scan_stat_loop_match++); MpmEndMatch *em; for (em = p->em; em; em = em->next) { @@ -1277,14 +1418,14 @@ u_int32_t WmScan2Hash15(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMa } } else { - COUNT(wm_thread_ctx->scan_stat_loop_no_match++); + COUNT(tctx->scan_stat_loop_no_match++); } } else { if (p->prefix_cs != prefixcs_buf || p->len > (bufend-(buf-sl))) continue; if (memcmp(p->cs, buf-sl+1, p->len) == 0) { //printf("CS Exact match: "); prt(p->cs, p->len); printf("\n"); - COUNT(wm_thread_ctx->scan_stat_loop_match++); + COUNT(tctx->scan_stat_loop_match++); MpmEndMatch *em; for (em = p->em; em; em = em->next) { @@ -1294,15 +1435,16 @@ u_int32_t WmScan2Hash15(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMa } } else { - COUNT(wm_thread_ctx->scan_stat_loop_no_match++); + COUNT(tctx->scan_stat_loop_no_match++); } } } } +skip_loop: shift = 1; } else { - COUNT(wm_thread_ctx->scan_stat_total_shift += shift); - COUNT(wm_thread_ctx->scan_stat_num_shift++); + COUNT(tctx->scan_stat_total_shift += shift); + COUNT(tctx->scan_stat_num_shift++); } buf += shift; } @@ -1311,14 +1453,14 @@ u_int32_t WmScan2Hash15(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMa } u_int32_t WmScan2Hash16(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *pmq, u_int8_t *buf, u_int16_t buflen) { - WmCtx *wm_ctx = (WmCtx *)mpm_ctx->ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx->ctx; #ifdef WUMANBER_COUNTERS - WmThreadCtx *wm_thread_ctx = (WmThreadCtx *)mpm_thread_ctx->ctx; + WmThreadCtx *tctx = (WmThreadCtx *)mpm_thread_ctx->ctx; #endif /* WUMANBER_COUNTERS */ u_int32_t cnt = 0; u_int8_t *bufmin = buf; u_int8_t *bufend = buf + buflen - 1; - u_int16_t sl = wm_ctx->scan_shiftlen; + u_int16_t sl = ctx->scan_shiftlen; u_int16_t h; u_int8_t shift; WmHashItem *thi, *hi; @@ -1337,20 +1479,37 @@ u_int32_t WmScan2Hash16(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMa while (buf <= bufend) { //h = (wm_tolower(*buf)<<8)+(wm_tolower(*(buf-1))); h = HASH16(wm_tolower(*buf),(wm_tolower(*(buf-1)))); - shift = wm_ctx->scan_shifttable[h]; + shift = ctx->scan_shifttable[h]; //printf("search: h %u, shift %u\n", h, shift); if (shift == 0) { - COUNT(wm_thread_ctx->scan_stat_shift_null++); + COUNT(tctx->scan_stat_shift_null++); /* get our hash item */ - hi = wm_ctx->scan_hash[h]; + hi = ctx->scan_hash[h]; //printf("search: hi %p\n", hi); if (hi != NULL) { + /* get our patterns from the hash */ + if (ctx->scan_bloom[h] != NULL) { + COUNT(tctx->scan_stat_pminlen_calls++); + COUNT(tctx->scan_stat_pminlen_total+=ctx->scan_pminlen[h]); + + if ((bufend - (buf-sl)) < ctx->scan_pminlen[h]) { + goto skip_loop; + } else { + COUNT(tctx->scan_stat_bloom_calls++); + + if (BloomFilterTest(ctx->scan_bloom[h], buf-sl+1, ctx->scan_pminlen[h]) == 0) { + COUNT(tctx->scan_stat_bloom_hits++); + goto skip_loop; + } + } + } + prefixci_buf = (u_int16_t)(wm_tolower(*(buf-sl+1)) + wm_tolower(*(buf-sl+2))); prefixcs_buf = (u_int16_t)(*(buf-sl+1) + *(buf-sl+2)); //printf("WmSearch2: prefixci_buf %u, prefixcs_buf %u\n", prefixci_buf, prefixcs_buf); for (thi = hi; thi != NULL; thi = thi->nxt) { - p = wm_ctx->parray[thi->idx]; + p = ctx->parray[thi->idx]; //printf("WmSearch2: p->prefix_ci %u, p->prefix_cs %u\n", // p->prefix_ci, p->prefix_cs); @@ -1361,7 +1520,7 @@ u_int32_t WmScan2Hash16(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMa if (memcmp_lowercase(p->ci, buf-sl+1, p->len) == 0) { //printf("CI Exact match: "); prt(p->ci, p->len); printf("\n"); - COUNT(wm_thread_ctx->scan_stat_loop_match++); + COUNT(tctx->scan_stat_loop_match++); MpmEndMatch *em; for (em = p->em; em; em = em->next) { @@ -1371,14 +1530,14 @@ u_int32_t WmScan2Hash16(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMa } } else { - COUNT(wm_thread_ctx->scan_stat_loop_no_match++); + COUNT(tctx->scan_stat_loop_no_match++); } } else { if (p->prefix_cs != prefixcs_buf || p->len > (bufend-(buf-sl))) continue; if (memcmp(p->cs, buf-sl+1, p->len) == 0) { //printf("CS Exact match: "); prt(p->cs, p->len); printf("\n"); - COUNT(wm_thread_ctx->scan_stat_loop_match++); + COUNT(tctx->scan_stat_loop_match++); MpmEndMatch *em; for (em = p->em; em; em = em->next) { @@ -1388,15 +1547,16 @@ u_int32_t WmScan2Hash16(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMa } } else { - COUNT(wm_thread_ctx->scan_stat_loop_no_match++); + COUNT(tctx->scan_stat_loop_no_match++); } } } } +skip_loop: shift = 1; } else { - COUNT(wm_thread_ctx->scan_stat_total_shift += shift); - COUNT(wm_thread_ctx->scan_stat_num_shift++); + COUNT(tctx->scan_stat_total_shift += shift); + COUNT(tctx->scan_stat_num_shift++); } buf += shift; } @@ -1405,7 +1565,7 @@ u_int32_t WmScan2Hash16(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMa } u_int32_t WmScan1(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *pmq, u_int8_t *buf, u_int16_t buflen) { - WmCtx *wm_ctx = (WmCtx *)mpm_ctx->ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx->ctx; u_int8_t *bufmin = buf; u_int8_t *bufend = buf + buflen - 1; u_int32_t cnt = 0; @@ -1421,11 +1581,11 @@ u_int32_t WmScan1(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQ if (mpm_ctx->scan_minlen == 1) { while (buf <= bufend) { u_int8_t h = wm_tolower(*buf); - hi = &wm_ctx->scan_hash1[h]; + hi = &ctx->scan_hash1[h]; if (hi->flags & 0x01) { for (thi = hi; thi != NULL; thi = thi->nxt) { - p = wm_ctx->parray[thi->idx]; + p = ctx->parray[thi->idx]; if (p->len != 1) continue; @@ -1456,7 +1616,7 @@ u_int32_t WmScan1(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQ if (mpm_ctx->scan_maxlen > 1) { /* Pass bufmin on because buf no longer points to the * start of the buffer. */ - cnt += wm_ctx->MBScan(mpm_ctx, mpm_thread_ctx, pmq, bufmin, buflen); + cnt += ctx->MBScan(mpm_ctx, mpm_thread_ctx, pmq, bufmin, buflen); //printf("WmScan1: after 2+byte cnt %u\n", cnt); } return cnt; @@ -1465,14 +1625,14 @@ u_int32_t WmScan1(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQ /* SEARCH FUNCTIONS */ u_int32_t WmSearch2Hash9(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *pmq, u_int8_t *buf, u_int16_t buflen) { - WmCtx *wm_ctx = (WmCtx *)mpm_ctx->ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx->ctx; #ifdef WUMANBER_COUNTERS - WmThreadCtx *wm_thread_ctx = (WmThreadCtx *)mpm_thread_ctx->ctx; + WmThreadCtx *tctx = (WmThreadCtx *)mpm_thread_ctx->ctx; #endif /* WUMANBER_COUNTERS */ u_int32_t cnt = 0; u_int8_t *bufmin = buf; u_int8_t *bufend = buf + buflen - 1; - u_int16_t sl = wm_ctx->search_shiftlen; + u_int16_t sl = ctx->search_shiftlen; u_int16_t h; u_int8_t shift; WmHashItem *thi, *hi; @@ -1491,20 +1651,20 @@ u_int32_t WmSearch2Hash9(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternM while (buf <= bufend) { //while (buf < bufend) { h = HASH9(wm_tolower(*buf),(wm_tolower(*(buf-1)))); - shift = wm_ctx->search_shifttable[h]; + shift = ctx->search_shifttable[h]; //printf("%p %u search: h %u, shift %u\n", buf, buf - bufmin, h, shift); if (shift == 0) { - COUNT(wm_thread_ctx->search_stat_shift_null++); + COUNT(tctx->search_stat_shift_null++); /* get our hash item */ - hi = wm_ctx->search_hash[h]; + hi = ctx->search_hash[h]; if (hi != NULL) { //printf("buf-sl+1 %p, buf-sl+2 %p\n", buf-sl+1, buf-sl+2); prefixci_buf = (u_int16_t)(wm_tolower(*(buf-sl+1)) + wm_tolower(*(buf-sl+2))); prefixcs_buf = (u_int16_t)(*(buf-sl+1) + *(buf-sl+2)); //printf("WmSearch2: prefixci_buf %u, prefixcs_buf %u\n", prefixci_buf, prefixcs_buf); for (thi = hi; thi != NULL; thi = thi->nxt) { - p = wm_ctx->parray[thi->idx]; + p = ctx->parray[thi->idx]; //printf("WmSearch2: p->prefix_ci %u, p->prefix_cs %u\n", // p->prefix_ci, p->prefix_cs); @@ -1515,7 +1675,7 @@ u_int32_t WmSearch2Hash9(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternM if (memcmp_lowercase(p->ci, buf-sl+1, p->len) == 0) { //printf("CI Exact match: "); prt(p->ci, p->len); printf("\n"); - COUNT(wm_thread_ctx->search_stat_loop_match++); + COUNT(tctx->search_stat_loop_match++); MpmEndMatch *em; for (em = p->em; em; em = em->next) { @@ -1525,14 +1685,14 @@ u_int32_t WmSearch2Hash9(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternM } } else { - COUNT(wm_thread_ctx->search_stat_loop_no_match++); + COUNT(tctx->search_stat_loop_no_match++); } } else { if (p->prefix_cs != prefixcs_buf || p->len > (bufend-(buf-sl))) continue; if (memcmp(p->cs, buf-sl+1, p->len) == 0) { //printf("CS Exact match: "); prt(p->cs, p->len); printf("\n"); - COUNT(wm_thread_ctx->search_stat_loop_match++); + COUNT(tctx->search_stat_loop_match++); MpmEndMatch *em; for (em = p->em; em; em = em->next) { @@ -1542,15 +1702,15 @@ u_int32_t WmSearch2Hash9(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternM } } else { - COUNT(wm_thread_ctx->search_stat_loop_no_match++); + COUNT(tctx->search_stat_loop_no_match++); } } } } shift = 1; } else { - COUNT(wm_thread_ctx->search_stat_total_shift += shift); - COUNT(wm_thread_ctx->search_stat_num_shift++); + COUNT(tctx->search_stat_total_shift += shift); + COUNT(tctx->search_stat_num_shift++); } buf += shift; } @@ -1560,14 +1720,14 @@ u_int32_t WmSearch2Hash9(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternM } u_int32_t WmSearch2Hash12(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *pmq, u_int8_t *buf, u_int16_t buflen) { - WmCtx *wm_ctx = (WmCtx *)mpm_ctx->ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx->ctx; #ifdef WUMANBER_COUNTERS - WmThreadCtx *wm_thread_ctx = (WmThreadCtx *)mpm_thread_ctx->ctx; + WmThreadCtx *tctx = (WmThreadCtx *)mpm_thread_ctx->ctx; #endif /* WUMANBER_COUNTERS */ u_int32_t cnt = 0; u_int8_t *bufmin = buf; u_int8_t *bufend = buf + buflen - 1; - u_int16_t sl = wm_ctx->search_shiftlen; + u_int16_t sl = ctx->search_shiftlen; u_int16_t h; u_int8_t shift; WmHashItem *thi, *hi; @@ -1586,20 +1746,20 @@ u_int32_t WmSearch2Hash12(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, Pattern while (buf <= bufend) { //h = (wm_tolower(*buf)<<8)+(wm_tolower(*(buf-1))); h = HASH12(wm_tolower(*buf),(wm_tolower(*(buf-1)))); - shift = wm_ctx->search_shifttable[h]; + shift = ctx->search_shifttable[h]; //printf("search: h %u, shift %u\n", h, shift); if (shift == 0) { - COUNT(wm_thread_ctx->search_stat_shift_null++); + COUNT(tctx->search_stat_shift_null++); /* get our hash item */ - hi = wm_ctx->search_hash[h]; + hi = ctx->search_hash[h]; //printf("search: hi %p\n", hi); if (hi != NULL) { prefixci_buf = (u_int16_t)(wm_tolower(*(buf-sl+1)) + wm_tolower(*(buf-sl+2))); prefixcs_buf = (u_int16_t)(*(buf-sl+1) + *(buf-sl+2)); //printf("WmSearch2: prefixci_buf %u, prefixcs_buf %u\n", prefixci_buf, prefixcs_buf); for (thi = hi; thi != NULL; thi = thi->nxt) { - p = wm_ctx->parray[thi->idx]; + p = ctx->parray[thi->idx]; //printf("WmSearch2: p->prefix_ci %u, p->prefix_cs %u\n", // p->prefix_ci, p->prefix_cs); @@ -1610,7 +1770,7 @@ u_int32_t WmSearch2Hash12(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, Pattern if (memcmp_lowercase(p->ci, buf-sl+1, p->len) == 0) { //printf("CI Exact match: "); prt(p->ci, p->len); printf("\n"); - COUNT(wm_thread_ctx->search_stat_loop_match++); + COUNT(tctx->search_stat_loop_match++); MpmEndMatch *em; for (em = p->em; em; em = em->next) { @@ -1620,14 +1780,14 @@ u_int32_t WmSearch2Hash12(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, Pattern } } else { - COUNT(wm_thread_ctx->search_stat_loop_no_match++); + COUNT(tctx->search_stat_loop_no_match++); } } else { if (p->prefix_cs != prefixcs_buf || p->len > (bufend-(buf-sl))) continue; if (memcmp(p->cs, buf-sl+1, p->len) == 0) { //printf("CS Exact match: "); prt(p->cs, p->len); printf("\n"); - COUNT(wm_thread_ctx->search_stat_loop_match++); + COUNT(tctx->search_stat_loop_match++); MpmEndMatch *em; for (em = p->em; em; em = em->next) { @@ -1637,15 +1797,15 @@ u_int32_t WmSearch2Hash12(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, Pattern } } else { - COUNT(wm_thread_ctx->search_stat_loop_no_match++); + COUNT(tctx->search_stat_loop_no_match++); } } } } shift = 1; } else { - COUNT(wm_thread_ctx->search_stat_total_shift += shift); - COUNT(wm_thread_ctx->search_stat_num_shift++); + COUNT(tctx->search_stat_total_shift += shift); + COUNT(tctx->search_stat_num_shift++); } buf += shift; } @@ -1654,14 +1814,14 @@ u_int32_t WmSearch2Hash12(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, Pattern } u_int32_t WmSearch2Hash14(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *pmq, u_int8_t *buf, u_int16_t buflen) { - WmCtx *wm_ctx = (WmCtx *)mpm_ctx->ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx->ctx; #ifdef WUMANBER_COUNTERS - WmThreadCtx *wm_thread_ctx = (WmThreadCtx *)mpm_thread_ctx->ctx; + WmThreadCtx *tctx = (WmThreadCtx *)mpm_thread_ctx->ctx; #endif /* WUMANBER_COUNTERS */ u_int32_t cnt = 0; u_int8_t *bufmin = buf; u_int8_t *bufend = buf + buflen - 1; - u_int16_t sl = wm_ctx->search_shiftlen; + u_int16_t sl = ctx->search_shiftlen; u_int16_t h; u_int8_t shift; WmHashItem *thi, *hi; @@ -1680,20 +1840,20 @@ u_int32_t WmSearch2Hash14(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, Pattern while (buf <= bufend) { //h = (wm_tolower(*buf)<<8)+(wm_tolower(*(buf-1))); h = HASH14(wm_tolower(*buf),(wm_tolower(*(buf-1)))); - shift = wm_ctx->search_shifttable[h]; + shift = ctx->search_shifttable[h]; //printf("search: h %u, shift %u\n", h, shift); if (shift == 0) { - COUNT(wm_thread_ctx->search_stat_shift_null++); + COUNT(tctx->search_stat_shift_null++); /* get our hash item */ - hi = wm_ctx->search_hash[h]; + hi = ctx->search_hash[h]; //printf("search: hi %p\n", hi); if (hi != NULL) { prefixci_buf = (u_int16_t)(wm_tolower(*(buf-sl+1)) + wm_tolower(*(buf-sl+2))); prefixcs_buf = (u_int16_t)(*(buf-sl+1) + *(buf-sl+2)); //printf("WmSearch2: prefixci_buf %u, prefixcs_buf %u\n", prefixci_buf, prefixcs_buf); for (thi = hi; thi != NULL; thi = thi->nxt) { - p = wm_ctx->parray[thi->idx]; + p = ctx->parray[thi->idx]; //printf("WmSearch2: p->prefix_ci %u, p->prefix_cs %u\n", // p->prefix_ci, p->prefix_cs); @@ -1704,7 +1864,7 @@ u_int32_t WmSearch2Hash14(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, Pattern if (memcmp_lowercase(p->ci, buf-sl+1, p->len) == 0) { //printf("CI Exact match: "); prt(p->ci, p->len); printf("\n"); - COUNT(wm_thread_ctx->search_stat_loop_match++); + COUNT(tctx->search_stat_loop_match++); MpmEndMatch *em; for (em = p->em; em; em = em->next) { @@ -1714,14 +1874,14 @@ u_int32_t WmSearch2Hash14(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, Pattern } } else { - COUNT(wm_thread_ctx->search_stat_loop_no_match++); + COUNT(tctx->search_stat_loop_no_match++); } } else { if (p->prefix_cs != prefixcs_buf || p->len > (bufend-(buf-sl))) continue; if (memcmp(p->cs, buf-sl+1, p->len) == 0) { //printf("CS Exact match: "); prt(p->cs, p->len); printf("\n"); - COUNT(wm_thread_ctx->search_stat_loop_match++); + COUNT(tctx->search_stat_loop_match++); MpmEndMatch *em; for (em = p->em; em; em = em->next) { @@ -1731,15 +1891,15 @@ u_int32_t WmSearch2Hash14(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, Pattern } } else { - COUNT(wm_thread_ctx->search_stat_loop_no_match++); + COUNT(tctx->search_stat_loop_no_match++); } } } } shift = 1; } else { - COUNT(wm_thread_ctx->search_stat_total_shift += shift); - COUNT(wm_thread_ctx->search_stat_num_shift++); + COUNT(tctx->search_stat_total_shift += shift); + COUNT(tctx->search_stat_num_shift++); } buf += shift; } @@ -1748,14 +1908,14 @@ u_int32_t WmSearch2Hash14(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, Pattern } u_int32_t WmSearch2Hash15(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *pmq, u_int8_t *buf, u_int16_t buflen) { - WmCtx *wm_ctx = (WmCtx *)mpm_ctx->ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx->ctx; #ifdef WUMANBER_COUNTERS - WmThreadCtx *wm_thread_ctx = (WmThreadCtx *)mpm_thread_ctx->ctx; + WmThreadCtx *tctx = (WmThreadCtx *)mpm_thread_ctx->ctx; #endif /* WUMANBER_COUNTERS */ u_int32_t cnt = 0; u_int8_t *bufmin = buf; u_int8_t *bufend = buf + buflen - 1; - u_int16_t sl = wm_ctx->search_shiftlen; + u_int16_t sl = ctx->search_shiftlen; u_int16_t h; u_int8_t shift; WmHashItem *thi, *hi; @@ -1772,20 +1932,20 @@ u_int32_t WmSearch2Hash15(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, Pattern while (buf <= bufend) { h = HASH15(wm_tolower(*buf),(wm_tolower(*(buf-1)))); - shift = wm_ctx->search_shifttable[h]; + shift = ctx->search_shifttable[h]; //printf("search: h %u, shift %u\n", h, shift); if (shift == 0) { - COUNT(wm_thread_ctx->search_stat_shift_null++); + COUNT(tctx->search_stat_shift_null++); /* get our hash item */ - hi = wm_ctx->search_hash[h]; + hi = ctx->search_hash[h]; //printf("search: hi %p\n", hi); if (hi != NULL) { prefixci_buf = (u_int16_t)(wm_tolower(*(buf-sl+1)) + wm_tolower(*(buf-sl+2))); prefixcs_buf = (u_int16_t)(*(buf-sl+1) + *(buf-sl+2)); //printf("WmSearch2: prefixci_buf %u, prefixcs_buf %u\n", prefixci_buf, prefixcs_buf); for (thi = hi; thi != NULL; thi = thi->nxt) { - p = wm_ctx->parray[thi->idx]; + p = ctx->parray[thi->idx]; //printf("WmSearch2: p->prefix_ci %u, p->prefix_cs %u\n", // p->prefix_ci, p->prefix_cs); @@ -1796,7 +1956,7 @@ u_int32_t WmSearch2Hash15(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, Pattern if (memcmp_lowercase(p->ci, buf-sl+1, p->len) == 0) { //printf("CI Exact match: "); prt(p->ci, p->len); printf("\n"); - COUNT(wm_thread_ctx->search_stat_loop_match++); + COUNT(tctx->search_stat_loop_match++); MpmEndMatch *em; for (em = p->em; em; em = em->next) { @@ -1806,14 +1966,14 @@ u_int32_t WmSearch2Hash15(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, Pattern } } else { - COUNT(wm_thread_ctx->search_stat_loop_no_match++); + COUNT(tctx->search_stat_loop_no_match++); } } else { if (p->prefix_cs != prefixcs_buf || p->len > (bufend-(buf-sl))) continue; if (memcmp(p->cs, buf-sl+1, p->len) == 0) { //printf("CS Exact match: "); prt(p->cs, p->len); printf("\n"); - COUNT(wm_thread_ctx->search_stat_loop_match++); + COUNT(tctx->search_stat_loop_match++); MpmEndMatch *em; for (em = p->em; em; em = em->next) { @@ -1823,15 +1983,15 @@ u_int32_t WmSearch2Hash15(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, Pattern } } else { - COUNT(wm_thread_ctx->search_stat_loop_no_match++); + COUNT(tctx->search_stat_loop_no_match++); } } } } shift = 1; } else { - COUNT(wm_thread_ctx->search_stat_total_shift += shift); - COUNT(wm_thread_ctx->search_stat_num_shift++); + COUNT(tctx->search_stat_total_shift += shift); + COUNT(tctx->search_stat_num_shift++); } buf += shift; } @@ -1840,14 +2000,14 @@ u_int32_t WmSearch2Hash15(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, Pattern } u_int32_t WmSearch2Hash16(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *pmq, u_int8_t *buf, u_int16_t buflen) { - WmCtx *wm_ctx = (WmCtx *)mpm_ctx->ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx->ctx; #ifdef WUMANBER_COUNTERS - WmThreadCtx *wm_thread_ctx = (WmThreadCtx *)mpm_thread_ctx->ctx; + WmThreadCtx *tctx = (WmThreadCtx *)mpm_thread_ctx->ctx; #endif /* WUMANBER_COUNTERS */ u_int32_t cnt = 0; u_int8_t *bufmin = buf; u_int8_t *bufend = buf + buflen - 1; - u_int16_t sl = wm_ctx->search_shiftlen; + u_int16_t sl = ctx->search_shiftlen; u_int16_t h; u_int8_t shift; WmHashItem *thi, *hi; @@ -1864,20 +2024,20 @@ u_int32_t WmSearch2Hash16(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, Pattern while (buf <= bufend) { h = HASH16(wm_tolower(*buf),(wm_tolower(*(buf-1)))); - shift = wm_ctx->search_shifttable[h]; + shift = ctx->search_shifttable[h]; //printf("search: h %u, shift %u\n", h, shift); if (shift == 0) { - COUNT(wm_thread_ctx->search_stat_shift_null++); + COUNT(tctx->search_stat_shift_null++); /* get our hash item */ - hi = wm_ctx->search_hash[h]; + hi = ctx->search_hash[h]; //printf("search: hi %p\n", hi); if (hi != NULL) { prefixci_buf = (u_int16_t)(wm_tolower(*(buf-sl+1)) + wm_tolower(*(buf-sl+2))); prefixcs_buf = (u_int16_t)(*(buf-sl+1) + *(buf-sl+2)); //printf("WmSearch2: prefixci_buf %u, prefixcs_buf %u\n", prefixci_buf, prefixcs_buf); for (thi = hi; thi != NULL; thi = thi->nxt) { - p = wm_ctx->parray[thi->idx]; + p = ctx->parray[thi->idx]; //printf("WmSearch2: p->prefix_ci %u, p->prefix_cs %u\n", // p->prefix_ci, p->prefix_cs); @@ -1888,7 +2048,7 @@ u_int32_t WmSearch2Hash16(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, Pattern if (memcmp_lowercase(p->ci, buf-sl+1, p->len) == 0) { //printf("CI Exact match: "); prt(p->ci, p->len); printf("\n"); - COUNT(wm_thread_ctx->search_stat_loop_match++); + COUNT(tctx->search_stat_loop_match++); MpmEndMatch *em; for (em = p->em; em; em = em->next) { @@ -1898,14 +2058,14 @@ u_int32_t WmSearch2Hash16(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, Pattern } } else { - COUNT(wm_thread_ctx->search_stat_loop_no_match++); + COUNT(tctx->search_stat_loop_no_match++); } } else { if (p->prefix_cs != prefixcs_buf || p->len > (bufend-(buf-sl))) continue; if (memcmp(p->cs, buf-sl+1, p->len) == 0) { //printf("CS Exact match: "); prt(p->cs, p->len); printf("\n"); - COUNT(wm_thread_ctx->search_stat_loop_match++); + COUNT(tctx->search_stat_loop_match++); MpmEndMatch *em; for (em = p->em; em; em = em->next) { @@ -1915,15 +2075,15 @@ u_int32_t WmSearch2Hash16(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, Pattern } } else { - COUNT(wm_thread_ctx->search_stat_loop_no_match++); + COUNT(tctx->search_stat_loop_no_match++); } } } } shift = 1; } else { - COUNT(wm_thread_ctx->search_stat_total_shift += shift); - COUNT(wm_thread_ctx->search_stat_num_shift++); + COUNT(tctx->search_stat_total_shift += shift); + COUNT(tctx->search_stat_num_shift++); } buf += shift; } @@ -1932,7 +2092,7 @@ u_int32_t WmSearch2Hash16(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, Pattern } u_int32_t WmSearch1(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *pmq, u_int8_t *buf, u_int16_t buflen) { - WmCtx *wm_ctx = (WmCtx *)mpm_ctx->ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx->ctx; u_int8_t *bufmin = buf; u_int8_t *bufend = buf + buflen - 1; u_int32_t cnt = 0; @@ -1948,11 +2108,11 @@ u_int32_t WmSearch1(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatche if (mpm_ctx->search_minlen == 1) { while (buf <= bufend) { u_int8_t h = wm_tolower(*buf); - hi = &wm_ctx->search_hash1[h]; + hi = &ctx->search_hash1[h]; if (hi->flags & 0x01) { for (thi = hi; thi != NULL; thi = thi->nxt) { - p = wm_ctx->parray[thi->idx]; + p = ctx->parray[thi->idx]; if (p->len != 1) continue; @@ -1983,7 +2143,7 @@ u_int32_t WmSearch1(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatche if (mpm_ctx->search_maxlen > 1) { /* Pass bufmin on because buf no longer points to the * start of the buffer. */ - cnt += wm_ctx->MBSearch(mpm_ctx, mpm_thread_ctx, pmq, bufmin, buflen); + cnt += ctx->MBSearch(mpm_ctx, mpm_thread_ctx, pmq, bufmin, buflen); //printf("WmSearch1: after 2+byte cnt %u\n", cnt); } return cnt; @@ -2004,60 +2164,60 @@ void WmInitCtx (MpmCtx *mpm_ctx) { mpm_ctx->memory_size += sizeof(WmCtx); /* initialize the hash we use to speed up pattern insertions */ - WmCtx *wm_ctx = (WmCtx *)mpm_ctx->ctx; - wm_ctx->init_hash = malloc(sizeof(WmPattern *) * INIT_HASH_SIZE); - if (wm_ctx->init_hash == NULL) + WmCtx *ctx = (WmCtx *)mpm_ctx->ctx; + ctx->init_hash = malloc(sizeof(WmPattern *) * INIT_HASH_SIZE); + if (ctx->init_hash == NULL) return; - memset(wm_ctx->init_hash, 0, sizeof(WmPattern *) * INIT_HASH_SIZE); + memset(ctx->init_hash, 0, sizeof(WmPattern *) * INIT_HASH_SIZE); } void WmDestroyCtx(MpmCtx *mpm_ctx) { - WmCtx *wm_ctx = (WmCtx *)mpm_ctx->ctx; - if (wm_ctx == NULL) + WmCtx *ctx = (WmCtx *)mpm_ctx->ctx; + if (ctx == NULL) return; - if (wm_ctx->init_hash) { - free(wm_ctx->init_hash); + if (ctx->init_hash) { + free(ctx->init_hash); mpm_ctx->memory_cnt--; mpm_ctx->memory_size -= (INIT_HASH_SIZE * sizeof(WmPattern *)); } - if (wm_ctx->parray) { + if (ctx->parray) { u_int32_t i; for (i = 0; i < mpm_ctx->pattern_cnt; i++) { - if (wm_ctx->parray[i] != NULL) { - WmFreePattern(mpm_ctx, wm_ctx->parray[i]); + if (ctx->parray[i] != NULL) { + WmFreePattern(mpm_ctx, ctx->parray[i]); } } - free(wm_ctx->parray); + free(ctx->parray); mpm_ctx->memory_cnt--; mpm_ctx->memory_size -= (mpm_ctx->pattern_cnt * sizeof(WmPattern)); } - if (wm_ctx->scan_hash) { - free(wm_ctx->scan_hash); + if (ctx->scan_hash) { + free(ctx->scan_hash); mpm_ctx->memory_cnt--; - mpm_ctx->memory_size -= (sizeof(WmHashItem) * wm_ctx->scan_hash_size); + mpm_ctx->memory_size -= (sizeof(WmHashItem) * ctx->scan_hash_size); } - if (wm_ctx->scan_shifttable) { - free(wm_ctx->scan_shifttable); + if (ctx->scan_shifttable) { + free(ctx->scan_shifttable); mpm_ctx->memory_cnt--; - mpm_ctx->memory_size -= (sizeof(u_int16_t) * wm_ctx->scan_hash_size); + mpm_ctx->memory_size -= (sizeof(u_int16_t) * ctx->scan_hash_size); } - if (wm_ctx->search_hash) { - free(wm_ctx->search_hash); + if (ctx->search_hash) { + free(ctx->search_hash); mpm_ctx->memory_cnt--; - mpm_ctx->memory_size -= (sizeof(WmHashItem) * wm_ctx->search_hash_size); + mpm_ctx->memory_size -= (sizeof(WmHashItem) * ctx->search_hash_size); } - if (wm_ctx->search_shifttable) { - free(wm_ctx->search_shifttable); + if (ctx->search_shifttable) { + free(ctx->search_shifttable); mpm_ctx->memory_cnt--; - mpm_ctx->memory_size -= (sizeof(u_int16_t) * wm_ctx->search_hash_size); + mpm_ctx->memory_size -= (sizeof(u_int16_t) * ctx->search_hash_size); } free(mpm_ctx->ctx); @@ -2097,8 +2257,8 @@ void WmThreadInitCtx(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, u_int32_t ma } void WmThreadDestroyCtx(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx) { - WmThreadCtx *wm_ctx = (WmThreadCtx *)mpm_thread_ctx->ctx; - if (wm_ctx) { + WmThreadCtx *ctx = (WmThreadCtx *)mpm_thread_ctx->ctx; + if (ctx) { if (mpm_thread_ctx->match != NULL) { mpm_thread_ctx->memory_cnt--; mpm_thread_ctx->memory_size -= ((mpm_ctx->max_pattern_id + 1) * sizeof(MpmMatchBucket)); @@ -2136,9 +2296,9 @@ int WmTestInitCtx02 (void) { MpmCtx mpm_ctx; WmInitCtx(&mpm_ctx); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; - if (wm_ctx->parray == NULL) + if (ctx->parray == NULL) result = 1; WmDestroyCtx(&mpm_ctx); @@ -2182,9 +2342,9 @@ int WmTestThreadInitCtx02 (void) { MpmInitCtx(&mpm_ctx, MPM_WUMANBER); WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1); - WmThreadCtx *wm_thread_ctx = (WmThreadCtx *)mpm_thread_ctx.ctx; + WmThreadCtx *tctx = (WmThreadCtx *)mpm_thread_ctx.ctx; - if (wm_thread_ctx->search_stat_shift_null == 0) + if (tctx->search_stat_shift_null == 0) result = 1; WmThreadDestroyCtx(&mpm_ctx, &mpm_thread_ctx); @@ -2219,10 +2379,10 @@ int WmTestInitAddPattern02 (void) { MpmInitCtx(&mpm_ctx, MPM_WUMANBER); WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; WmAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 1, 0, 1234, 0); - if (wm_ctx->init_hash != NULL) + if (ctx->init_hash != NULL) result = 1; WmThreadDestroyCtx(&mpm_ctx, &mpm_thread_ctx); @@ -2237,10 +2397,10 @@ int WmTestInitAddPattern03 (void) { MpmInitCtx(&mpm_ctx, MPM_WUMANBER); WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; WmAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 1, 0, 1234, 0); - WmPattern *pat = WmInitHashLookup(wm_ctx, (u_int8_t *)"abcd", 4, 1); + WmPattern *pat = WmInitHashLookup(ctx, (u_int8_t *)"abcd", 4, 1); if (pat != NULL) { if (pat->len == 4) result = 1; @@ -2258,10 +2418,10 @@ int WmTestInitAddPattern04 (void) { MpmInitCtx(&mpm_ctx, MPM_WUMANBER); WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; WmAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 1, 0, 1234, 0); - WmPattern *pat = WmInitHashLookup(wm_ctx, (u_int8_t *)"abcd", 4, 1); + WmPattern *pat = WmInitHashLookup(ctx, (u_int8_t *)"abcd", 4, 1); if (pat != NULL) { if (pat->flags & WUMANBER_NOCASE) result = 1; @@ -2279,10 +2439,10 @@ int WmTestInitAddPattern05 (void) { MpmInitCtx(&mpm_ctx, MPM_WUMANBER); WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; WmAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 0, 0, 1234, 0); - WmPattern *pat = WmInitHashLookup(wm_ctx, (u_int8_t *)"abcd", 4, 0); + WmPattern *pat = WmInitHashLookup(ctx, (u_int8_t *)"abcd", 4, 0); if (pat != NULL) { if (!(pat->flags & WUMANBER_NOCASE)) result = 1; @@ -2300,10 +2460,10 @@ int WmTestInitAddPattern06 (void) { MpmInitCtx(&mpm_ctx, MPM_WUMANBER); WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; WmAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 1, 0, 1234, 0); - WmPattern *pat = WmInitHashLookup(wm_ctx, (u_int8_t *)"abcd", 4, 1); + WmPattern *pat = WmInitHashLookup(ctx, (u_int8_t *)"abcd", 4, 1); if (pat != NULL) { if (memcmp(pat->cs, "abcd", 4) == 0) result = 1; @@ -2354,9 +2514,9 @@ int WmTestPrepare02 (void) { WmAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 1, 0, 0, 0); WmPreparePatterns(&mpm_ctx); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; - if (wm_ctx->search_shiftlen == 4) + if (ctx->search_shiftlen == 4) result = 1; WmDestroyCtx(&mpm_ctx); @@ -2370,12 +2530,12 @@ int WmTestPrepare03 (void) { WmAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 1, 0, 0, 0); WmPreparePatterns(&mpm_ctx); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; - if (wm_ctx->search_shifttable[1] == 4) + if (ctx->search_shifttable[1] == 4) result = 1; else - printf("4 != %u: ", wm_ctx->search_shifttable[1]); + printf("4 != %u: ", ctx->search_shifttable[1]); WmDestroyCtx(&mpm_ctx); return result; @@ -2403,9 +2563,9 @@ int WmTestPrepare05 (void) { WmAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 1, 1, 0, 0); WmPreparePatterns(&mpm_ctx); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; - if (wm_ctx->scan_shiftlen == 4) + if (ctx->scan_shiftlen == 4) result = 1; WmDestroyCtx(&mpm_ctx); @@ -2419,12 +2579,12 @@ int WmTestPrepare06 (void) { WmAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 1, 1, 0, 0); WmPreparePatterns(&mpm_ctx); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; - if (wm_ctx->scan_shifttable[1] == 4) + if (ctx->scan_shifttable[1] == 4) result = 1; else - printf("4 != %u: ", wm_ctx->scan_shifttable[1]); + printf("4 != %u: ", ctx->scan_shifttable[1]); WmDestroyCtx(&mpm_ctx); return result; @@ -2461,11 +2621,11 @@ int WmTestSearch01Hash12 (void) { MpmThreadCtx mpm_thread_ctx; MpmInitCtx(&mpm_ctx, MPM_WUMANBER); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; WmAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 1, 0, 0, 0); - wm_ctx->search_hash_size = HASH12_SIZE; + ctx->search_hash_size = HASH12_SIZE; WmPreparePatterns(&mpm_ctx); WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1); //mpm_ctx.PrintCtx(&mpm_ctx); @@ -2489,11 +2649,11 @@ int WmTestSearch01Hash14 (void) { MpmThreadCtx mpm_thread_ctx; MpmInitCtx(&mpm_ctx, MPM_WUMANBER); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; WmAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 1, 0, 0, 0); - wm_ctx->search_hash_size = HASH14_SIZE; + ctx->search_hash_size = HASH14_SIZE; WmPreparePatterns(&mpm_ctx); WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1); //mpm_ctx.PrintCtx(&mpm_ctx); @@ -2517,11 +2677,11 @@ int WmTestSearch01Hash15 (void) { MpmThreadCtx mpm_thread_ctx; MpmInitCtx(&mpm_ctx, MPM_WUMANBER); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; WmAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 1, 0, 0, 0); - wm_ctx->search_hash_size = HASH15_SIZE; + ctx->search_hash_size = HASH15_SIZE; WmPreparePatterns(&mpm_ctx); WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1); //mpm_ctx.PrintCtx(&mpm_ctx); @@ -2545,11 +2705,11 @@ int WmTestSearch01Hash16 (void) { MpmThreadCtx mpm_thread_ctx; MpmInitCtx(&mpm_ctx, MPM_WUMANBER); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; WmAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0, 1, 0, 0, 0); - wm_ctx->search_hash_size = HASH16_SIZE; + ctx->search_hash_size = HASH16_SIZE; WmPreparePatterns(&mpm_ctx); WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1); //mpm_ctx.PrintCtx(&mpm_ctx); @@ -2937,10 +3097,10 @@ int WmTestSearch18Hash12 (void) { MpmCtx mpm_ctx; MpmThreadCtx mpm_thread_ctx; MpmInitCtx(&mpm_ctx, MPM_WUMANBER); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; WmAddPatternCS(&mpm_ctx, (u_int8_t *)"/VideoAccessCodecInstall.exe", 28, 0, 0, 0, 0); - wm_ctx->search_hash_size = HASH12_SIZE; + ctx->search_hash_size = HASH12_SIZE; WmPreparePatterns(&mpm_ctx); WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1); @@ -2963,10 +3123,10 @@ int WmTestSearch18Hash14 (void) { MpmCtx mpm_ctx; MpmThreadCtx mpm_thread_ctx; MpmInitCtx(&mpm_ctx, MPM_WUMANBER); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; WmAddPatternCS(&mpm_ctx, (u_int8_t *)"/VideoAccessCodecInstall.exe", 28, 0, 0, 0, 0); - wm_ctx->search_hash_size = HASH14_SIZE; + ctx->search_hash_size = HASH14_SIZE; WmPreparePatterns(&mpm_ctx); WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1); @@ -2989,10 +3149,10 @@ int WmTestSearch18Hash15 (void) { MpmCtx mpm_ctx; MpmThreadCtx mpm_thread_ctx; MpmInitCtx(&mpm_ctx, MPM_WUMANBER); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; WmAddPatternCS(&mpm_ctx, (u_int8_t *)"/VideoAccessCodecInstall.exe", 28, 0, 0, 0, 0); - wm_ctx->search_hash_size = HASH15_SIZE; + ctx->search_hash_size = HASH15_SIZE; WmPreparePatterns(&mpm_ctx); WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1); @@ -3015,10 +3175,10 @@ int WmTestSearch18 (void) { MpmCtx mpm_ctx; MpmThreadCtx mpm_thread_ctx; MpmInitCtx(&mpm_ctx, MPM_WUMANBER); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; WmAddPatternCS(&mpm_ctx, (u_int8_t *)"/VideoAccessCodecInstall.exe", 28, 0, 0, 0, 0); - wm_ctx->search_hash_size = HASH16_SIZE; + ctx->search_hash_size = HASH16_SIZE; WmPreparePatterns(&mpm_ctx); WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1); @@ -3041,10 +3201,10 @@ int WmTestSearch18Hash16 (void) { MpmCtx mpm_ctx; MpmThreadCtx mpm_thread_ctx; MpmInitCtx(&mpm_ctx, MPM_WUMANBER); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; WmAddPatternCS(&mpm_ctx, (u_int8_t *)"/VideoAccessCodecInstall.exe", 28, 0, 0, 0, 0); - wm_ctx->search_hash_size = HASH16_SIZE; + ctx->search_hash_size = HASH16_SIZE; WmPreparePatterns(&mpm_ctx); WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1); @@ -3091,10 +3251,10 @@ int WmTestSearch19Hash12 (void) { MpmCtx mpm_ctx; MpmThreadCtx mpm_thread_ctx; MpmInitCtx(&mpm_ctx, MPM_WUMANBER); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; WmAddPatternCI(&mpm_ctx, (u_int8_t *)"/VideoAccessCodecInstall.exe", 28, 0, 0, 0, 0); - wm_ctx->search_hash_size = HASH12_SIZE; + ctx->search_hash_size = HASH12_SIZE; WmPreparePatterns(&mpm_ctx); WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1); @@ -3117,10 +3277,10 @@ int WmTestSearch19Hash14 (void) { MpmCtx mpm_ctx; MpmThreadCtx mpm_thread_ctx; MpmInitCtx(&mpm_ctx, MPM_WUMANBER); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; WmAddPatternCI(&mpm_ctx, (u_int8_t *)"/VideoAccessCodecInstall.exe", 28, 0, 0, 0, 0); - wm_ctx->search_hash_size = HASH14_SIZE; + ctx->search_hash_size = HASH14_SIZE; WmPreparePatterns(&mpm_ctx); WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1); @@ -3143,10 +3303,10 @@ int WmTestSearch19Hash15 (void) { MpmCtx mpm_ctx; MpmThreadCtx mpm_thread_ctx; MpmInitCtx(&mpm_ctx, MPM_WUMANBER); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; WmAddPatternCI(&mpm_ctx, (u_int8_t *)"/VideoAccessCodecInstall.exe", 28, 0, 0, 0, 0); - wm_ctx->search_hash_size = HASH15_SIZE; + ctx->search_hash_size = HASH15_SIZE; WmPreparePatterns(&mpm_ctx); WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1); @@ -3169,10 +3329,10 @@ int WmTestSearch19Hash16 (void) { MpmCtx mpm_ctx; MpmThreadCtx mpm_thread_ctx; MpmInitCtx(&mpm_ctx, MPM_WUMANBER); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; WmAddPatternCI(&mpm_ctx, (u_int8_t *)"/VideoAccessCodecInstall.exe", 28, 0, 0, 0, 0); - wm_ctx->search_hash_size = HASH16_SIZE; + ctx->search_hash_size = HASH16_SIZE; WmPreparePatterns(&mpm_ctx); WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1); @@ -3219,10 +3379,10 @@ int WmTestSearch20Hash12 (void) { MpmCtx mpm_ctx; MpmThreadCtx mpm_thread_ctx; MpmInitCtx(&mpm_ctx, MPM_WUMANBER); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; WmAddPatternCS(&mpm_ctx, (u_int8_t *)"/videoaccesscodecinstall.exe", 28, 0, 0, 0, 0); - wm_ctx->search_hash_size = HASH12_SIZE; /* force hash12 */ + ctx->search_hash_size = HASH12_SIZE; /* force hash12 */ WmPreparePatterns(&mpm_ctx); WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1); @@ -3245,10 +3405,10 @@ int WmTestSearch20Hash14 (void) { MpmCtx mpm_ctx; MpmThreadCtx mpm_thread_ctx; MpmInitCtx(&mpm_ctx, MPM_WUMANBER); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; WmAddPatternCS(&mpm_ctx, (u_int8_t *)"/videoaccesscodecinstall.exe", 28, 0, 0, 0, 0); - wm_ctx->search_hash_size = HASH14_SIZE; /* force hash14 */ + ctx->search_hash_size = HASH14_SIZE; /* force hash14 */ WmPreparePatterns(&mpm_ctx); WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1); @@ -3271,10 +3431,10 @@ int WmTestSearch20Hash15 (void) { MpmCtx mpm_ctx; MpmThreadCtx mpm_thread_ctx; MpmInitCtx(&mpm_ctx, MPM_WUMANBER); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; WmAddPatternCS(&mpm_ctx, (u_int8_t *)"/videoaccesscodecinstall.exe", 28, 0, 0, 0, 0); - wm_ctx->search_hash_size = HASH15_SIZE; /* force hash15 */ + ctx->search_hash_size = HASH15_SIZE; /* force hash15 */ WmPreparePatterns(&mpm_ctx); WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1); @@ -3297,10 +3457,10 @@ int WmTestSearch20Hash16 (void) { MpmCtx mpm_ctx; MpmThreadCtx mpm_thread_ctx; MpmInitCtx(&mpm_ctx, MPM_WUMANBER); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; WmAddPatternCS(&mpm_ctx, (u_int8_t *)"/videoaccesscodecinstall.exe", 28, 0, 0, 0, 0); - wm_ctx->search_hash_size = HASH16_SIZE; /* force hash16 */ + ctx->search_hash_size = HASH16_SIZE; /* force hash16 */ WmPreparePatterns(&mpm_ctx); WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1); @@ -3347,10 +3507,10 @@ static int WmTestSearch21Hash12 (void) { MpmCtx mpm_ctx; MpmThreadCtx mpm_thread_ctx; MpmInitCtx(&mpm_ctx, MPM_WUMANBER); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; WmAddPatternCS(&mpm_ctx, (u_int8_t *)"/videoaccesscodecinstall.exe", 28, 0, 0, 0, 0); - wm_ctx->search_hash_size = HASH12_SIZE; /* force hash16 */ + ctx->search_hash_size = HASH12_SIZE; /* force hash16 */ WmPreparePatterns(&mpm_ctx); WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1); //WmPrintInfo(&mpm_ctx); @@ -3373,10 +3533,10 @@ static int WmTestSearch21Hash14 (void) { MpmCtx mpm_ctx; MpmThreadCtx mpm_thread_ctx; MpmInitCtx(&mpm_ctx, MPM_WUMANBER); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; WmAddPatternCS(&mpm_ctx, (u_int8_t *)"/videoaccesscodecinstall.exe", 28, 0, 0, 0, 0); - wm_ctx->search_hash_size = HASH14_SIZE; /* force hash16 */ + ctx->search_hash_size = HASH14_SIZE; /* force hash16 */ WmPreparePatterns(&mpm_ctx); WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1); //WmPrintInfo(&mpm_ctx); @@ -3399,10 +3559,10 @@ static int WmTestSearch21Hash15 (void) { MpmCtx mpm_ctx; MpmThreadCtx mpm_thread_ctx; MpmInitCtx(&mpm_ctx, MPM_WUMANBER); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; WmAddPatternCS(&mpm_ctx, (u_int8_t *)"/videoaccesscodecinstall.exe", 28, 0, 0, 0, 0); - wm_ctx->search_hash_size = HASH15_SIZE; /* force hash16 */ + ctx->search_hash_size = HASH15_SIZE; /* force hash16 */ WmPreparePatterns(&mpm_ctx); WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1); //WmPrintInfo(&mpm_ctx); @@ -3425,10 +3585,10 @@ static int WmTestSearch21Hash16 (void) { MpmCtx mpm_ctx; MpmThreadCtx mpm_thread_ctx; MpmInitCtx(&mpm_ctx, MPM_WUMANBER); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; WmAddPatternCS(&mpm_ctx, (u_int8_t *)"/videoaccesscodecinstall.exe", 28, 0, 0, 0, 0); - wm_ctx->search_hash_size = HASH16_SIZE; /* force hash16 */ + ctx->search_hash_size = HASH16_SIZE; /* force hash16 */ WmPreparePatterns(&mpm_ctx); WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1); //WmPrintInfo(&mpm_ctx); @@ -3451,7 +3611,7 @@ static int WmTestSearch22Hash9 (void) { MpmCtx mpm_ctx; MpmThreadCtx mpm_thread_ctx; MpmInitCtx(&mpm_ctx, MPM_WUMANBER); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; WmAddPatternCS(&mpm_ctx, (u_int8_t *)"A", 1, 0, 0, 0, 0); /* should match 30 times */ WmAddPatternCS(&mpm_ctx, (u_int8_t *)"AA", 2, 0, 0, 1, 0); /* should match 29 times */ @@ -3461,7 +3621,7 @@ static int WmTestSearch22Hash9 (void) { WmAddPatternCS(&mpm_ctx, (u_int8_t *)"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", 30, 0, 0, 5, 0); /* 1 */ /* total matches: 135 */ - wm_ctx->search_hash_size = HASH9_SIZE; /* force hash size */ + ctx->search_hash_size = HASH9_SIZE; /* force hash size */ WmPreparePatterns(&mpm_ctx); WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 6 /* 6 patterns */); @@ -3484,7 +3644,7 @@ static int WmTestSearch22Hash12 (void) { MpmCtx mpm_ctx; MpmThreadCtx mpm_thread_ctx; MpmInitCtx(&mpm_ctx, MPM_WUMANBER); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; WmAddPatternCS(&mpm_ctx, (u_int8_t *)"A", 1, 0, 0, 0, 0); /* should match 30 times */ WmAddPatternCS(&mpm_ctx, (u_int8_t *)"AA", 2, 0, 0, 1, 0); /* should match 29 times */ @@ -3494,7 +3654,7 @@ static int WmTestSearch22Hash12 (void) { WmAddPatternCS(&mpm_ctx, (u_int8_t *)"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", 30, 0, 0, 5, 0); /* 1 */ /* total matches: 135 */ - wm_ctx->search_hash_size = HASH12_SIZE; /* force hash size */ + ctx->search_hash_size = HASH12_SIZE; /* force hash size */ WmPreparePatterns(&mpm_ctx); WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 6 /* 6 patterns */); @@ -3517,7 +3677,7 @@ static int WmTestSearch22Hash14 (void) { MpmCtx mpm_ctx; MpmThreadCtx mpm_thread_ctx; MpmInitCtx(&mpm_ctx, MPM_WUMANBER); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; WmAddPatternCS(&mpm_ctx, (u_int8_t *)"A", 1, 0, 0, 0, 0); /* should match 30 times */ WmAddPatternCS(&mpm_ctx, (u_int8_t *)"AA", 2, 0, 0, 1, 0); /* should match 29 times */ @@ -3527,7 +3687,7 @@ static int WmTestSearch22Hash14 (void) { WmAddPatternCS(&mpm_ctx, (u_int8_t *)"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", 30, 0, 0, 5, 0); /* 1 */ /* total matches: 135 */ - wm_ctx->search_hash_size = HASH14_SIZE; /* force hash size */ + ctx->search_hash_size = HASH14_SIZE; /* force hash size */ WmPreparePatterns(&mpm_ctx); WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 6 /* 6 patterns */); @@ -3550,7 +3710,7 @@ static int WmTestSearch22Hash15 (void) { MpmCtx mpm_ctx; MpmThreadCtx mpm_thread_ctx; MpmInitCtx(&mpm_ctx, MPM_WUMANBER); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; WmAddPatternCS(&mpm_ctx, (u_int8_t *)"A", 1, 0, 0, 0, 0); /* should match 30 times */ WmAddPatternCS(&mpm_ctx, (u_int8_t *)"AA", 2, 0, 0, 1, 0); /* should match 29 times */ @@ -3560,7 +3720,7 @@ static int WmTestSearch22Hash15 (void) { WmAddPatternCS(&mpm_ctx, (u_int8_t *)"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", 30, 0, 0, 5, 0); /* 1 */ /* total matches: 135 */ - wm_ctx->search_hash_size = HASH15_SIZE; /* force hash size */ + ctx->search_hash_size = HASH15_SIZE; /* force hash size */ WmPreparePatterns(&mpm_ctx); WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 6 /* 6 patterns */); @@ -3583,7 +3743,7 @@ static int WmTestSearch22Hash16 (void) { MpmCtx mpm_ctx; MpmThreadCtx mpm_thread_ctx; MpmInitCtx(&mpm_ctx, MPM_WUMANBER); - WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx; + WmCtx *ctx = (WmCtx *)mpm_ctx.ctx; WmAddPatternCS(&mpm_ctx, (u_int8_t *)"A", 1, 0, 0, 0, 0); /* should match 30 times */ WmAddPatternCS(&mpm_ctx, (u_int8_t *)"AA", 2, 0, 0, 1, 0); /* should match 29 times */ @@ -3593,7 +3753,7 @@ static int WmTestSearch22Hash16 (void) { WmAddPatternCS(&mpm_ctx, (u_int8_t *)"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", 30, 0, 0, 5, 0); /* 1 */ /* total matches: 135 */ - wm_ctx->search_hash_size = HASH16_SIZE; /* force hash size */ + ctx->search_hash_size = HASH16_SIZE; /* force hash size */ WmPreparePatterns(&mpm_ctx); WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 6 /* 6 patterns */); diff --git a/src/util-mpm-wumanber.h b/src/util-mpm-wumanber.h index 52fb8b7db8..6474b981b0 100644 --- a/src/util-mpm-wumanber.h +++ b/src/util-mpm-wumanber.h @@ -4,10 +4,13 @@ #define __UTIL_MPM_WUMANBER_H__ #include "util-mpm.h" +#include "util-bloomfilter.h" #define WUMANBER_NOCASE 0x01 #define WUMANBER_SCAN 0x02 +#define WUMANBER_BLOOMSIZE 1024 + //#define WUMANBER_COUNTERS typedef struct _WmPattern { @@ -36,6 +39,10 @@ typedef struct _WmCtx { u_int32_t scan_hash_size; WmHashItem **scan_hash; + BloomFilter **scan_bloom; + u_int8_t *scan_pminlen; /* array containing the minimal length + of the patters in a hash bucket. Used + for the BloomFilter. */ WmHashItem scan_hash1[256]; u_int32_t search_hash_size; WmHashItem **search_hash; @@ -56,6 +63,10 @@ typedef struct _WmCtx { typedef struct _WmThreadCtx { #ifdef WUMANBER_COUNTERS + u_int32_t scan_stat_pminlen_calls; + u_int32_t scan_stat_pminlen_total; + u_int32_t scan_stat_bloom_calls; + u_int32_t scan_stat_bloom_hits; u_int32_t scan_stat_shift_null; u_int32_t scan_stat_loop_match; u_int32_t scan_stat_loop_no_match; diff --git a/src/util-mpm.c b/src/util-mpm.c index 572f6c811a..134f565dce 100644 --- a/src/util-mpm.c +++ b/src/util-mpm.c @@ -11,6 +11,7 @@ #include "util-mpm-trie.h" #include "util-mpm-wumanber.h" #include "util-mpm-b2g.h" +#include "util-mpm-b3g.h" /* cleanup list with all matches * @@ -223,6 +224,7 @@ void MpmTableSetup(void) { //MpmTrieRegister(); MpmWuManberRegister(); MpmB2gRegister(); + MpmB3gRegister(); } void MpmRegisterTests(void) { diff --git a/src/util-mpm.h b/src/util-mpm.h index 096624bb13..de08c00ee7 100644 --- a/src/util-mpm.h +++ b/src/util-mpm.h @@ -11,6 +11,7 @@ enum { MPM_TRIE, MPM_WUMANBER, MPM_B2G, + MPM_B3G, /* table size */ MPM_TABLE_SIZE, diff --git a/src/vips.c b/src/vips.c index 2ca5d1bac6..7e92fd9f96 100644 --- a/src/vips.c +++ b/src/vips.c @@ -208,11 +208,11 @@ int main(int argc, char **argv) UtRunSelftest(); /* inits and cleans up again */ UtInitialize(); TmModuleRegisterTests(); - MpmRegisterTests(); SigTableRegisterTests(); HashTableRegisterTests(); BloomFilterRegisterTests(); BloomFilterCountingRegisterTests(); + MpmRegisterTests(); SigRegisterTests(); UtRunTests(); UtCleanup(); @@ -252,6 +252,9 @@ int main(int argc, char **argv) SigLoadSignatures(); +//#define PERF_THREADS +#define MANY_THREADS +#ifndef PERF_THREADS /* create the threads */ ThreadVars *tv_receivenfq = TmThreadCreate("ReceiveNFQ","packetpool","packetpool","pickup-queue","simple","1slot_noinout"); if (tv_receivenfq == NULL) { @@ -269,7 +272,7 @@ int main(int argc, char **argv) printf("ERROR: TmThreadSpawn failed\n"); exit(1); } -#define MANY_THREADS + #ifdef MANY_THREADS ThreadVars *tv_decode1 = TmThreadCreate("Decode1","pickup-queue","simple","decode-queue1","simple","1slot"); if (tv_decode1 == NULL) { @@ -440,6 +443,7 @@ int main(int argc, char **argv) printf("ERROR: TmThreadSpawn failed\n"); exit(1); } + #else /* MANY_THREADS */ ThreadVars *tv_main = TmThreadCreate("MainThread","pickup-queue","simple","packetpool","packetpool","varslot"); if (tv_main == NULL) { @@ -466,7 +470,7 @@ int main(int argc, char **argv) exit(1); } TmVarSlotSetFuncAppend(tv_main,tm_module); - +/* tm_module = TmModuleGetByName("RespondReject"); if (tm_module == NULL) { printf("ERROR: TmModuleGetByName for RespondReject failed\n"); @@ -508,13 +512,364 @@ int main(int argc, char **argv) exit(1); } TmVarSlotSetFuncAppend(tv_main,tm_module); - +*/ if (TmThreadSpawn(tv_main) != 0) { printf("ERROR: TmThreadSpawn failed\n"); exit(1); } #endif /* MANY_THREADS */ +#else /* PERF_THREADS */ +#if 0 + /* create the threads */ + ThreadVars *tv_receivenfq = TmThreadCreate("ReceiveNFQ1","packetpool","packetpool","pickup-queue","simple","1slot_noinout"); + if (tv_receivenfq == NULL) { + printf("ERROR: TmThreadsCreate failed\n"); + exit(1); + } + TmModule *tm_module = TmModuleGetByName("ReceiveNFQ"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName failed for ReceiveNFQ\n"); + exit(1); + } + Tm1SlotSetFunc(tv_receivenfq,tm_module); + + if (TmThreadSpawn(tv_receivenfq) != 0) { + printf("ERROR: TmThreadSpawn failed\n"); + exit(1); + } + + ThreadVars *tv_decode1 = TmThreadCreate("Decode1","pickup-queue","simple","verdict-queue","simple","2slot"); + if (tv_decode1 == NULL) { + printf("ERROR: TmThreadsCreate failed for Decode1\n"); + exit(1); + } + tm_module = TmModuleGetByName("DecodeNFQ"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName DecodeNFQ failed\n"); + exit(1); + } + Tm2SlotSetFunc1(tv_decode1,tm_module); + + tm_module = TmModuleGetByName("Detect"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName DecodeNFQ failed\n"); + exit(1); + } + Tm2SlotSetFunc2(tv_decode1,tm_module); + + TmThreadSetCPUAffinity(tv_decode1,0); + + if (TmThreadSpawn(tv_decode1) != 0) { + printf("ERROR: TmThreadSpawn failed\n"); + exit(1); + } + + ThreadVars *tv_decode2 = TmThreadCreate("Decode2","pickup-queue","simple","verdict-queue","simple","2slot"); + if (tv_decode2 == NULL) { + printf("ERROR: TmThreadsCreate failed\n"); + exit(1); + } + tm_module = TmModuleGetByName("DecodeNFQ"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName DecodeNFQ failed\n"); + exit(1); + } + Tm2SlotSetFunc1(tv_decode2,tm_module); + + tm_module = TmModuleGetByName("Detect"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName DecodeNFQ failed\n"); + exit(1); + } + Tm2SlotSetFunc2(tv_decode2,tm_module); + + TmThreadSetCPUAffinity(tv_decode2,1); + + if (TmThreadSpawn(tv_decode2) != 0) { + printf("ERROR: TmThreadSpawn failed\n"); + exit(1); + } +/* + ThreadVars *tv_decode3 = TmThreadCreate("Decode3","pickup-queue","simple","verdict-queue","simple","2slot"); + if (tv_decode3 == NULL) { + printf("ERROR: TmThreadsCreate failed for Decode1\n"); + exit(1); + } + tm_module = TmModuleGetByName("DecodeNFQ"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName DecodeNFQ failed\n"); + exit(1); + } + Tm2SlotSetFunc1(tv_decode3,tm_module); + + tm_module = TmModuleGetByName("Detect"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName DecodeNFQ failed\n"); + exit(1); + } + Tm2SlotSetFunc2(tv_decode3,tm_module); + + TmThreadSetCPUAffinity(tv_decode3,0); + + if (TmThreadSpawn(tv_decode3) != 0) { + printf("ERROR: TmThreadSpawn failed\n"); + exit(1); + } + + ThreadVars *tv_decode4 = TmThreadCreate("Decode4","pickup-queue","simple","verdict-queue","simple","2slot"); + if (tv_decode4 == NULL) { + printf("ERROR: TmThreadsCreate failed\n"); + exit(1); + } + tm_module = TmModuleGetByName("DecodeNFQ"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName DecodeNFQ failed\n"); + exit(1); + } + Tm2SlotSetFunc1(tv_decode4,tm_module); + + tm_module = TmModuleGetByName("Detect"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName DecodeNFQ failed\n"); + exit(1); + } + Tm2SlotSetFunc2(tv_decode4,tm_module); + + TmThreadSetCPUAffinity(tv_decode4,1); + + if (TmThreadSpawn(tv_decode4) != 0) { + printf("ERROR: TmThreadSpawn failed\n"); + exit(1); + } +*/ + //ThreadVars *tv_verdict = TmThreadCreate("Verdict","verdict-queue","simple","respond-queue","simple","1slot"); + ThreadVars *tv_verdict = TmThreadCreate("Verdict","verdict-queue","simple","packetpool","packetpool","1slot"); + if (tv_verdict == NULL) { + printf("ERROR: TmThreadsCreate failed\n"); + exit(1); + } + tm_module = TmModuleGetByName("VerdictNFQ"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName VerdictNFQ failed\n"); + exit(1); + } + Tm1SlotSetFunc(tv_verdict,tm_module); + + if (TmThreadSpawn(tv_verdict) != 0) { + printf("ERROR: TmThreadSpawn failed\n"); + exit(1); + } +#endif + + /* create the threads */ + ThreadVars *tv_receivenfq = TmThreadCreate("ReceiveNFQ1","packetpool","packetpool","pickup-queue1","simple","1slot_noinout"); + if (tv_receivenfq == NULL) { + printf("ERROR: TmThreadsCreate failed\n"); + exit(1); + } + TmModule *tm_module = TmModuleGetByName("ReceiveNFQ"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName failed for ReceiveNFQ\n"); + exit(1); + } + Tm1SlotSetFunc(tv_receivenfq,tm_module); + TmThreadSetCPUAffinity(tv_receivenfq,0); + + if (TmThreadSpawn(tv_receivenfq) != 0) { + printf("ERROR: TmThreadSpawn failed\n"); + exit(1); + } + /* create the threads */ + ThreadVars *tv_receivenfq2 = TmThreadCreate("ReceiveNFQ2","packetpool","packetpool","pickup-queue2","simple","1slot_noinout"); + if (tv_receivenfq2 == NULL) { + printf("ERROR: TmThreadsCreate failed\n"); + exit(1); + } + tm_module = TmModuleGetByName("ReceiveNFQ"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName failed for ReceiveNFQ\n"); + exit(1); + } + Tm1SlotSetFunc(tv_receivenfq2,tm_module); + TmThreadSetCPUAffinity(tv_receivenfq2,1); + + if (TmThreadSpawn(tv_receivenfq2) != 0) { + printf("ERROR: TmThreadSpawn failed\n"); + exit(1); + } + + ThreadVars *tv_decode1 = TmThreadCreate("Decode1","pickup-queue1","simple","packetpool","packetpool","3slot"); + if (tv_decode1 == NULL) { + printf("ERROR: TmThreadsCreate failed for Decode1\n"); + exit(1); + } + tm_module = TmModuleGetByName("DecodeNFQ"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName DecodeNFQ failed\n"); + exit(1); + } + Tm3SlotSetFunc1(tv_decode1,tm_module); + + tm_module = TmModuleGetByName("Detect"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName DecodeNFQ failed\n"); + exit(1); + } + Tm3SlotSetFunc2(tv_decode1,tm_module); + + tm_module = TmModuleGetByName("VerdictNFQ"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName VerdictNFQ failed\n"); + exit(1); + } + Tm3SlotSetFunc3(tv_decode1,tm_module); + TmThreadSetCPUAffinity(tv_decode1,0); + + if (TmThreadSpawn(tv_decode1) != 0) { + printf("ERROR: TmThreadSpawn failed\n"); + exit(1); + } + + + ThreadVars *tv_decode2 = TmThreadCreate("Decode2","pickup-queue2","simple","packetpool","packetpool","3slot"); + if (tv_decode2 == NULL) { + printf("ERROR: TmThreadsCreate failed\n"); + exit(1); + } + tm_module = TmModuleGetByName("DecodeNFQ"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName DecodeNFQ failed\n"); + exit(1); + } + Tm3SlotSetFunc1(tv_decode2,tm_module); + + tm_module = TmModuleGetByName("Detect"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName DecodeNFQ failed\n"); + exit(1); + } + Tm3SlotSetFunc2(tv_decode2,tm_module); + + tm_module = TmModuleGetByName("VerdictNFQ"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName VerdictNFQ failed\n"); + exit(1); + } + Tm3SlotSetFunc3(tv_decode2,tm_module); + TmThreadSetCPUAffinity(tv_decode2,1); + + if (TmThreadSpawn(tv_decode2) != 0) { + printf("ERROR: TmThreadSpawn failed\n"); + exit(1); + } + +/* + //ThreadVars *tv_verdict = TmThreadCreate("Verdict","verdict-queue","simple","respond-queue","simple","1slot"); + ThreadVars *tv_verdict = TmThreadCreate("Verdict","verdict-queue","simple","packetpool","packetpool","1slot"); + if (tv_verdict == NULL) { + printf("ERROR: TmThreadsCreate failed\n"); + exit(1); + } + tm_module = TmModuleGetByName("VerdictNFQ"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName VerdictNFQ failed\n"); + exit(1); + } + Tm1SlotSetFunc(tv_verdict,tm_module); + + if (TmThreadSpawn(tv_verdict) != 0) { + printf("ERROR: TmThreadSpawn failed\n"); + exit(1); + } +*/ +/* + ThreadVars *tv_rreject = TmThreadCreate("RespondReject","respond-queue","simple","alert-queue1","simple","1slot"); + if (tv_rreject == NULL) { + printf("ERROR: TmThreadsCreate failed\n"); + exit(1); + } + tm_module = TmModuleGetByName("RespondReject"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName for RespondReject failed\n"); + exit(1); + } + Tm1SlotSetFunc(tv_rreject,tm_module); + + if (TmThreadSpawn(tv_rreject) != 0) { + printf("ERROR: TmThreadSpawn failed\n"); + exit(1); + } + + ThreadVars *tv_alert = TmThreadCreate("AlertFastlog&Httplog","alert-queue1","simple","alert-queue2","simple","2slot"); + if (tv_alert == NULL) { + printf("ERROR: TmThreadsCreate failed\n"); + exit(1); + } + tm_module = TmModuleGetByName("AlertFastlog"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName for AlertFastlog failed\n"); + exit(1); + } + Tm2SlotSetFunc1(tv_alert,tm_module); + + tm_module = TmModuleGetByName("LogHttplog"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName failed\n"); + exit(1); + } + Tm2SlotSetFunc2(tv_alert,tm_module); + + if (TmThreadSpawn(tv_alert) != 0) { + printf("ERROR: TmThreadSpawn failed\n"); + exit(1); + } + + //ThreadVars *tv_unified = TmThreadCreate("AlertUnifiedLog","alert-queue2","simple","packetpool","packetpool","2slot"); + ThreadVars *tv_unified = TmThreadCreate("AlertUnifiedLog","alert-queue2","simple","alert-queue3","simple","2slot"); + if (tv_unified == NULL) { + printf("ERROR: TmThreadsCreate failed\n"); + exit(1); + } + + tm_module = TmModuleGetByName("AlertUnifiedLog"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName for AlertUnifiedLog failed\n"); + exit(1); + } + Tm2SlotSetFunc1(tv_unified,tm_module); + + tm_module = TmModuleGetByName("AlertUnifiedAlert"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName for AlertUnifiedAlert failed\n"); + exit(1); + } + Tm2SlotSetFunc2(tv_unified,tm_module); + + if (TmThreadSpawn(tv_unified) != 0) { + printf("ERROR: TmThreadSpawn failed\n"); + exit(1); + } + + + ThreadVars *tv_debugalert = TmThreadCreate("AlertDebuglog","alert-queue3","simple","packetpool","packetpool","1slot"); + if (tv_debugalert == NULL) { + printf("ERROR: TmThreadsCreate failed\n"); + exit(1); + } + tm_module = TmModuleGetByName("AlertDebuglog"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName failed\n"); + exit(1); + } + Tm1SlotSetFunc(tv_debugalert,tm_module); + + if (TmThreadSpawn(tv_debugalert) != 0) { + printf("ERROR: TmThreadSpawn failed\n"); + exit(1); + } +*/ +#endif /* PERF_THREADS */ ThreadVars tv_flowmgr; memset(&tv_flowmgr, 0, sizeof(ThreadVars)); diff --git a/src/vips.h b/src/vips.h index e5bf815134..1f21bc8f12 100644 --- a/src/vips.h +++ b/src/vips.h @@ -14,7 +14,7 @@ /* maximum number of simultanious threads. */ #define NUM_THREADS 256 /* max packets processed simultaniously */ -#define MAX_PENDING 500 +#define MAX_PENDING 512 #define TRUE 1 #define FALSE 0