From e49d0a592428ab9fa3cc129f4ad0582fd2c75db9 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Mon, 18 Aug 2014 13:19:07 +0200 Subject: [PATCH] MPM: build sid list from MPM matches Pmq add rule list: Array of uint32_t's to store (internal) sids from the MPM. AC: store sids in the pattern list, append to Pmq::rule_id_array on match. Detect: sort rule_id_array after it was set up by the MPM. Rule id's (Signature::num) are ordered, and the rule's with the lowest id are to be inspected first. As the MPM doesn't fill the array in order, but instead 'randomly' we need this sort step to assure proper inspection order. --- src/detect.c | 12 +++++++++++ src/util-mpm-ac.c | 55 +++++++++++++++++++++++++++++++++++++++++++++++ src/util-mpm-ac.h | 8 +++++++ src/util-mpm.c | 22 +++++++++++++++++++ src/util-mpm.h | 6 ++++++ 5 files changed, 103 insertions(+) diff --git a/src/detect.c b/src/detect.c index 54d035af6c..9dd4b4e444 100644 --- a/src/detect.c +++ b/src/detect.c @@ -1019,6 +1019,18 @@ static inline void DetectMpmPrefilter(DetectEngineCtx *de_ctx, } } } + + /* Sort the rule list to lets look at pmq. + * NOTE due to merging of 'stream' pmqs we *MAY* have duplicate entries */ + if (det_ctx->pmq.rule_id_array_cnt) { + int DoSort(const void *a, const void *b) { + uint32_t x = *(uint32_t *)a; + uint32_t y = *(uint32_t *)b; + return x - y; + } + qsort(det_ctx->pmq.rule_id_array, det_ctx->pmq.rule_id_array_cnt, + sizeof(uint32_t), DoSort); + } } #ifdef DEBUG diff --git a/src/util-mpm-ac.c b/src/util-mpm-ac.c index f8e2199a39..ec16e415cf 100644 --- a/src/util-mpm-ac.c +++ b/src/util-mpm-ac.c @@ -357,6 +357,34 @@ static int SCACAddPattern(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen, /* we need the max pat id */ if (pid > ctx->max_pat_id) ctx->max_pat_id = pid; + + p->sids_size = 1; + p->sids = SCMalloc(p->sids_size * sizeof(uint32_t)); + BUG_ON(p->sids == NULL); + p->sids[0] = sid; + //SCLogInfo("MPM added %u:%u", pid, sid); + } else { + /* TODO figure out how we can be called multiple times for the same CTX with the same sid */ + + int found = 0; + uint32_t x = 0; + for (x = 0; x < p->sids_size; x++) { + if (p->sids[x] == sid) { + found = 1; + break; + } + } + if (!found) { + uint32_t *sids = SCRealloc(p->sids, (sizeof(uint32_t) * (p->sids_size + 1))); + BUG_ON(sids == NULL); + p->sids = sids; + p->sids[p->sids_size] = sid; + p->sids_size++; + //SCLogInfo("p->sids_size %u", p->sids_size); + //SCLogInfo("MPM added %u:%u (append)", pid, sid); + } else { + //SCLogInfo("rule %u already part of pid %u", sid, pid); + } } return 0; @@ -1056,6 +1084,11 @@ int SCACPreparePatterns(MpmCtx *mpm_ctx) ctx->parray[i]->original_pat, ctx->parray[i]->len); ctx->pid_pat_list[ctx->parray[i]->id].patlen = ctx->parray[i]->len; } + + /* ACPatternList now owns this memory */ + //SCLogInfo("ctx->parray[i]->sids_size %u", ctx->parray[i]->sids_size); + ctx->pid_pat_list[ctx->parray[i]->id].sids_size = ctx->parray[i]->sids_size; + ctx->pid_pat_list[ctx->parray[i]->id].sids = ctx->parray[i]->sids; } /* prepare the state table required by AC */ @@ -1237,6 +1270,8 @@ void SCACDestroyCtx(MpmCtx *mpm_ctx) for (i = 0; i < (ctx->max_pat_id + 1); i++) { if (ctx->pid_pat_list[i].cs != NULL) SCFree(ctx->pid_pat_list[i].cs); + if (ctx->pid_pat_list[i].sids != NULL) + SCFree(ctx->pid_pat_list[i].sids); } SCFree(ctx->pid_pat_list); } @@ -1294,6 +1329,11 @@ uint32_t SCACSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, } else { pmq->pattern_id_bitarray[(pids[k] & 0x0000FFFF) / 8] |= (1 << ((pids[k] & 0x0000FFFF) % 8)); pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = pids[k] & 0x0000FFFF; + + uint32_t x; + for (x = 0; x < pid_pat_list[pids[k] & 0x0000FFFF].sids_size; x++) { + pmq->rule_id_array[pmq->rule_id_array_cnt++] = pid_pat_list[pids[k] & 0x0000FFFF].sids[x]; + } } matches++; } else { @@ -1302,6 +1342,11 @@ uint32_t SCACSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, } else { pmq->pattern_id_bitarray[pids[k] / 8] |= (1 << (pids[k] % 8)); pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = pids[k]; + + uint32_t x; + for (x = 0; x < pid_pat_list[pids[k]].sids_size; x++) { + pmq->rule_id_array[pmq->rule_id_array_cnt++] = pid_pat_list[pids[k]].sids[x]; + } } matches++; } @@ -1333,6 +1378,11 @@ uint32_t SCACSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, } else { pmq->pattern_id_bitarray[(pids[k] & 0x0000FFFF) / 8] |= (1 << ((pids[k] & 0x0000FFFF) % 8)); pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = pids[k] & 0x0000FFFF; + + uint32_t x; + for (x = 0; x < pid_pat_list[pids[k] & 0x0000FFFF].sids_size; x++) { + pmq->rule_id_array[pmq->rule_id_array_cnt++] = pid_pat_list[pids[k] & 0x0000FFFF].sids[x]; + } } matches++; } else { @@ -1341,6 +1391,11 @@ uint32_t SCACSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, } else { pmq->pattern_id_bitarray[pids[k] / 8] |= (1 << (pids[k] % 8)); pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = pids[k]; + + uint32_t x; + for (x = 0; x < pid_pat_list[pids[k]].sids_size; x++) { + pmq->rule_id_array[pmq->rule_id_array_cnt++] = pid_pat_list[pids[k]].sids[x]; + } } matches++; } diff --git a/src/util-mpm-ac.h b/src/util-mpm-ac.h index 760fb70495..7519fffd62 100644 --- a/src/util-mpm-ac.h +++ b/src/util-mpm-ac.h @@ -52,12 +52,20 @@ typedef struct SCACPattern_ { /* pattern id */ uint32_t id; + /* sid(s) for this pattern */ + uint32_t sids_size; + uint32_t *sids; + struct SCACPattern_ *next; } SCACPattern; typedef struct SCACPatternList_ { uint8_t *cs; uint16_t patlen; + + /* sid(s) for this pattern */ + uint32_t sids_size; + uint32_t *sids; } SCACPatternList; typedef struct SCACOutputTable_ { diff --git a/src/util-mpm.c b/src/util-mpm.c index df81e87a26..cc40e0edc8 100644 --- a/src/util-mpm.c +++ b/src/util-mpm.c @@ -452,6 +452,16 @@ int PmqSetup(PatternMatcherQueue *pmq, uint32_t patmaxid) SCLogDebug("pmq->pattern_id_array %p, pmq->pattern_id_bitarray %p", pmq->pattern_id_array, pmq->pattern_id_bitarray); + + pmq->rule_id_array_size = 65536 * sizeof(uint32_t); + + pmq->rule_id_array = SCMalloc(pmq->rule_id_array_size); + if (pmq->rule_id_array == NULL) { + SCReturnInt(-1); + } + memset(pmq->rule_id_array, 0, pmq->rule_id_array_size); + pmq->rule_id_array_cnt = 0; + } SCReturnInt(0); @@ -510,6 +520,12 @@ void PmqMerge(PatternMatcherQueue *src, PatternMatcherQueue *dst) } /** \todo now set merged flag? */ + + if (src->rule_id_array && dst->rule_id_array) { + for (u = 0; u < src->rule_id_array_cnt; u++) { + dst->rule_id_array[dst->rule_id_array_cnt++] = src->rule_id_array[u]; + } + } } /** \brief Reset a Pmq for reusage. Meant to be called after a single search. @@ -532,6 +548,7 @@ void PmqReset(PatternMatcherQueue *pmq) } pmq->pattern_id_array_cnt = 0; */ + pmq->rule_id_array_cnt = 0; } /** \brief Cleanup a Pmq @@ -552,6 +569,11 @@ void PmqCleanup(PatternMatcherQueue *pmq) pmq->pattern_id_bitarray = NULL; } + if (pmq->rule_id_array != NULL) { + SCFree(pmq->rule_id_array); + pmq->rule_id_array = NULL; + } + pmq->pattern_id_array_cnt = 0; } diff --git a/src/util-mpm.h b/src/util-mpm.h index f45ecfa4cb..395c5cc8fb 100644 --- a/src/util-mpm.h +++ b/src/util-mpm.h @@ -106,6 +106,12 @@ typedef struct PatternMatcherQueue_ { uint8_t *pattern_id_bitarray; /** bitarray with pattern id matches */ uint32_t pattern_id_bitarray_size; /**< size in bytes */ + + /* used for storing rule id's */ + uint32_t rule_id_array_size; + uint32_t *rule_id_array; + uint32_t rule_id_array_cnt; + } PatternMatcherQueue; typedef struct MpmCtx_ {