aiden
/
suricata
mirror of https://github.com/OISF/suricata
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Switch to pattern id based results checking in the mpm. Move app layer proto detection towards a more signature based approach.

Browse Source
remotes/origin/master-1.0.x
Victor Julien 16 years ago
parent 41172f0024
commit 7a427ec7f4
20 changed files with 796 additions and 413 deletions
Show Stats Download Patch File Download Diff File

309
src/app-layer-detect-proto.c
Unescape Escape View File

@ -53,40 +53,20 @@
#include "app-layer-parser.h"
#include "app-layer-detect-proto.h"
#include "util-spm.h"
#include "util-cuda.h"
#include "util-cuda-handlers.h"
#include "util-mpm-b2g-cuda.h"
#include "util-debug.h"
#define INSPECT_BYTES 32
#define ALP_DETECT_MAX 256
/* undef __SC_CUDA_SUPPORT__. We will get back to this later. Need to
* analyze the performance of cuda support for app layer */
#undef __SC_CUDA_SUPPORT__
typedef struct AlpProtoDetectDirection_ {
MpmCtx mpm_ctx;
uint32_t id;
uint16_t map[ALP_DETECT_MAX]; /**< a mapping between condition id's and
protocol */
uint16_t max_len; /**< max length of all patterns, so we can
limit the search */
uint16_t min_len; /**< min length of all patterns, so we can
tell the stream engine to feed data
to app layer as soon as it has min
size data */
} AlpProtoDetectDirection;
typedef struct AlpProtoDetectCtx_ {
AlpProtoDetectDirection toserver;
AlpProtoDetectDirection toclient;
int alp_content_module_handle;
} AlpProtoDetectCtx;
/** global app layer detection context */
static AlpProtoDetectCtx alp_proto_ctx;
AlpProtoDetectCtx alp_proto_ctx;
/** \brief Initialize the app layer proto detection */
void AlpProtoInit(AlpProtoDetectCtx *ctx) {
@ -108,21 +88,87 @@ void AlpProtoInit(AlpProtoDetectCtx *ctx) {
ctx->toclient.id = 0;
ctx->toclient.min_len = INSPECT_BYTES;
ctx->toserver.min_len = INSPECT_BYTES;
ctx->mpm_pattern_id_store = MpmPatternIdTableInitHash();
}
void AlpProtoTestDestroy(AlpProtoDetectCtx *ctx) {
mpm_table[ctx->toserver.mpm_ctx.mpm_type].DestroyCtx(&ctx->toserver.mpm_ctx);
mpm_table[ctx->toclient.mpm_ctx.mpm_type].DestroyCtx(&ctx->toclient.mpm_ctx);
/**
* \brief Turn a proto detection into a AlpProtoSignature and store it
* in the ctx.
*
* \param ctx the contex
* \param co the content match
* \param proto the proto id
*/
static void AlpProtoAddSignature(AlpProtoDetectCtx *ctx, DetectContentData *co, uint16_t proto) {
AlpProtoSignature *s = SCMalloc(sizeof(AlpProtoSignature));
if (s == NULL) {
return;
}
memset(s, 0x00, sizeof(AlpProtoSignature));
s->proto = proto;
s->co = co;
if (ctx->head == NULL) {
ctx->head = s;
} else {
s->next = ctx->head;
ctx->head = s;
}
ctx->sigs++;
}
void AlpProtoDestroy() {
SCEnter();
mpm_table[alp_proto_ctx.toserver.mpm_ctx.mpm_type].DestroyCtx(&alp_proto_ctx.toserver.mpm_ctx);
mpm_table[alp_proto_ctx.toclient.mpm_ctx.mpm_type].DestroyCtx(&alp_proto_ctx.toclient.mpm_ctx);
SCReturn;
/** \brief free a AlpProtoSignature, recursively free any next sig */
static void AlpProtoFreeSignature(AlpProtoSignature *s) {
if (s == NULL)
return;
DetectContentFree(s->co);
s->co = NULL;
s->proto = 0;
AlpProtoSignature *next_s = s->next;
SCFree(s);
AlpProtoFreeSignature(next_s);
}
/**
* \brief Match a AlpProtoSignature against a buffer
*
* \param s signature
* \param buf pointer to buffer
* \param buflen length of the buffer
*
* \retval proto the detected proto or ALPROTO_UNKNOWN if no match
*/
static uint16_t AlpProtoMatchSignature(AlpProtoSignature *s, uint8_t *buf, uint16_t buflen) {
uint16_t proto = ALPROTO_UNKNOWN;
if (s->co->offset > buflen)
goto end;
if (s->co->depth > buflen)
goto end;
uint8_t *sbuf = buf + s->co->offset;
uint16_t sbuflen = s->co->depth - s->co->offset;
uint8_t *found = SpmSearch(sbuf, sbuflen, s->co->content, s->co->content_len);
if (found != NULL) {
proto = s->proto;
}
end:
return proto;
}
/** \brief Add a proto detection string to the detection ctx.
/**
* \brief Add a proto detection string to the detection ctx.
*
* \param ctx The detection ctx
* \param ip_proto The IP proto (TCP, UDP, etc)
* \param al_proto Application layer proto
@ -139,6 +185,8 @@ void AlpProtoAdd(AlpProtoDetectCtx *ctx, uint16_t ip_proto, uint16_t al_proto, c
cd->depth = depth;
cd->offset = offset;
cd->id = DetectContentGetId(ctx->mpm_pattern_id_store, cd);
//PrintRawDataFp(stdout,cd->content,cd->content_len);
AlpProtoDetectDirection *dir;
@ -161,24 +209,40 @@ void AlpProtoAdd(AlpProtoDetectCtx *ctx, uint16_t ip_proto, uint16_t al_proto, c
if (depth < dir->min_len)
dir->min_len = depth;
/* no longer need the cd */
DetectContentFree(cd);
/* finally turn into a signature and add to the ctx */
AlpProtoAddSignature(ctx, cd, al_proto);
}
static void AlpProtoTestDestroy(AlpProtoDetectCtx *ctx) {
mpm_table[ctx->toserver.mpm_ctx.mpm_type].DestroyCtx(&ctx->toserver.mpm_ctx);
mpm_table[ctx->toclient.mpm_ctx.mpm_type].DestroyCtx(&ctx->toclient.mpm_ctx);
AlpProtoFreeSignature(ctx->head);
}
void AlpProtoDestroy() {
SCEnter();
mpm_table[alp_proto_ctx.toserver.mpm_ctx.mpm_type].DestroyCtx(&alp_proto_ctx.toserver.mpm_ctx);
mpm_table[alp_proto_ctx.toclient.mpm_ctx.mpm_type].DestroyCtx(&alp_proto_ctx.toclient.mpm_ctx);
SCReturn;
}
void AlpProtoFinalizeThread(AlpProtoDetectCtx *ctx, AlpProtoDetectThreadCtx *tctx) {
uint32_t maxid;
uint32_t sig_maxid = 0;
uint32_t pat_maxid = ctx->mpm_pattern_id_store ? ctx->mpm_pattern_id_store->max_id : 0;
memset(tctx, 0x00, sizeof(AlpProtoDetectThreadCtx));
if (ctx->toclient.id > 0) {
maxid = ctx->toclient.id;
mpm_table[ctx->toclient.mpm_ctx.mpm_type].InitThreadCtx(&ctx->toclient.mpm_ctx, &tctx->toclient.mpm_ctx, maxid);
PmqSetup(&tctx->toclient.pmq, maxid);
//sig_maxid = ctx->toclient.id;
mpm_table[ctx->toclient.mpm_ctx.mpm_type].InitThreadCtx(&ctx->toclient.mpm_ctx, &tctx->toclient.mpm_ctx, sig_maxid);
PmqSetup(&tctx->toclient.pmq, sig_maxid, pat_maxid);
}
if (ctx->toserver.id > 0) {
maxid = ctx->toserver.id;
mpm_table[ctx->toserver.mpm_ctx.mpm_type].InitThreadCtx(&ctx->toserver.mpm_ctx, &tctx->toserver.mpm_ctx, maxid);
PmqSetup(&tctx->toserver.pmq, maxid);
//sig_maxid = ctx->toserver.id;
mpm_table[ctx->toserver.mpm_ctx.mpm_type].InitThreadCtx(&ctx->toserver.mpm_ctx, &tctx->toserver.mpm_ctx, sig_maxid);
PmqSetup(&tctx->toserver.pmq, sig_maxid, pat_maxid);
}
}
void AlpProtoDeFinalize2Thread(AlpProtoDetectThreadCtx *tctx) {
@ -220,6 +284,21 @@ void AlpProtoFinalizeGlobal(AlpProtoDetectCtx *ctx) {
min_len */
StreamMsgQueueSetMinInitChunkLen(FLOW_PKT_TOCLIENT, ctx->toclient.min_len);
StreamMsgQueueSetMinInitChunkLen(FLOW_PKT_TOSERVER, ctx->toserver.min_len);
/* allocate and initialize the mapping between pattern id and signature */
ctx->map = (AlpProtoSignature **)SCMalloc(ctx->sigs * sizeof(AlpProtoSignature *));
if (ctx->map == NULL) {
SCLogError(SC_ERR_MEM_ALLOC, "%s", strerror(errno));
return;
}
memset(ctx->map, 0x00, ctx->sigs * sizeof(AlpProtoSignature *));
AlpProtoSignature *s = ctx->head;
for ( ; s != NULL; s = s->next) {
BUG_ON(s->co == NULL);
ctx->map[s->co->id] = s;
}
}
void AppLayerDetectProtoThreadInit(void) {
@ -312,7 +391,8 @@ void AppLayerDetectProtoThreadInit(void) {
AlpProtoFinalizeGlobal(&alp_proto_ctx);
}
/** \brief Get the app layer proto based on a buffer
/**
* \brief Get the app layer proto based on a buffer
*
* \param ctx Global app layer detection context
* \param tctx Thread app layer detection context
@ -327,6 +407,7 @@ uint16_t AppLayerDetectGetProto(AlpProtoDetectCtx *ctx, AlpProtoDetectThreadCtx
AlpProtoDetectDirection *dir;
AlpProtoDetectDirectionThread *tdir;
if (flags & STREAM_TOSERVER) {
dir = &ctx->toserver;
tdir = &tctx->toserver;
@ -346,6 +427,8 @@ uint16_t AppLayerDetectGetProto(AlpProtoDetectCtx *ctx, AlpProtoDetectThreadCtx
uint16_t proto = ALPROTO_UNKNOWN;
uint32_t cnt = 0;
/* do the mpm search */
#ifndef __SC_CUDA_SUPPORT__
cnt = mpm_table[dir->mpm_ctx.mpm_type].Search(&dir->mpm_ctx,
&tdir->mpm_ctx,
@ -378,9 +461,15 @@ uint16_t AppLayerDetectGetProto(AlpProtoDetectCtx *ctx, AlpProtoDetectThreadCtx
goto end;
}
/** We just return the first match
* \todo what if we have more? */
proto = dir->map[tdir->pmq.sig_id_array[0]];
/* We just work with the first match */
uint16_t patid = tdir->pmq.pattern_id_array[0];
AlpProtoSignature *s = ctx->map[patid];
if (s == NULL) {
goto end;
}
proto = AlpProtoMatchSignature(s, buf, buflen);
end:
PmqReset(&tdir->pmq);
@ -446,92 +535,6 @@ end:
SCReturnUInt(proto);
}
/** \brief Handle a app layer message
*
* If the protocol is yet unknown, the proto detection code is run first.
*
* \param dp_ctx Thread app layer detect context
* \param smsg Stream message
*
* \retval 0 ok
* \retval -1 error
*/
int AppLayerHandleMsg(AlpProtoDetectThreadCtx *dp_ctx, StreamMsg *smsg)
{
SCEnter();
uint16_t alproto = ALPROTO_UNKNOWN;
int r = 0;
TcpSession *ssn = smsg->flow->protoctx;
if (ssn != NULL) {
alproto = ssn->alproto;
/* if we don't know the proto yet and we have received a stream
* initializer message, we run proto detection.
* We receive 2 stream init msgs (one for each direction) but we
* only run the proto detection once. */
if (alproto == ALPROTO_UNKNOWN && smsg->flags & STREAM_START) {
SCLogDebug("Stream initializer (len %" PRIu32 " (%" PRIu32 "))",
smsg->data.data_len, MSG_DATA_SIZE);
//printf("=> Init Stream Data -- start\n");
//PrintRawDataFp(stdout, smsg->init.data, smsg->init.data_len);
//printf("=> Init Stream Data -- end\n");
alproto = AppLayerDetectGetProto(&alp_proto_ctx, dp_ctx,
smsg->data.data, smsg->data.data_len, smsg->flags);
if (alproto != ALPROTO_UNKNOWN) {
/* store the proto and setup the L7 data array */
StreamL7DataPtrInit(ssn);
ssn->alproto = alproto;
ssn->flags |= STREAMTCP_FLAG_APPPROTO_DETECTION_COMPLETED;
r = AppLayerParse(smsg->flow, alproto, smsg->flags,
smsg->data.data, smsg->data.data_len);
} else {
if (smsg->flags & STREAM_TOSERVER) {
if (smsg->data.data_len >= alp_proto_ctx.toserver.max_len) {
ssn->flags |= STREAMTCP_FLAG_APPPROTO_DETECTION_COMPLETED;
SCLogDebug("ALPROTO_UNKNOWN flow %p", smsg->flow);
StreamTcpSetSessionNoReassemblyFlag(ssn, 0);
}
} else if (smsg->flags & STREAM_TOCLIENT) {
if (smsg->data.data_len >= alp_proto_ctx.toclient.max_len) {
ssn->flags |= STREAMTCP_FLAG_APPPROTO_DETECTION_COMPLETED;
SCLogDebug("ALPROTO_UNKNOWN flow %p", smsg->flow);
StreamTcpSetSessionNoReassemblyFlag(ssn, 1);
}
}
}
} else {
SCLogDebug("stream data (len %" PRIu32 " (%" PRIu32 ")), alproto "
"%"PRIu16" (flow %p)", smsg->data.data_len, MSG_DATA_SIZE,
alproto, smsg->flow);
//printf("=> Stream Data -- start\n");
//PrintRawDataFp(stdout, smsg->data.data, smsg->data.data_len);
//printf("=> Stream Data -- end\n");
/* if we don't have a data object here we are not getting it
* a start msg should have gotten us one */
if (alproto != ALPROTO_UNKNOWN) {
r = AppLayerParse(smsg->flow, alproto, smsg->flags,
smsg->data.data, smsg->data.data_len);
} else {
SCLogDebug(" smsg not start, but no l7 data? Weird");
}
}
}
/* flow is free again */
smsg->flow->use_cnt--;
/* return the used message to the queue */
StreamMsgReturnToPool(smsg);
SCReturnInt(r);
}
/* VJ Originally I thought of having separate app layer
* handling threads, leaving this here in case we'll revisit that */
#if 0
@ -789,9 +792,9 @@ int AlpDetectTest04(void) {
}
#endif
uint32_t cnt = mpm_table[ctx.toclient.mpm_ctx.mpm_type].Search(&ctx.toclient.mpm_ctx, &tctx.toclient.mpm_ctx, NULL, l7data, sizeof(l7data));
if (cnt != 0) {
printf("cnt %u != 0: ", cnt);
uint32_t cnt = mpm_table[ctx.toclient.mpm_ctx.mpm_type].Search(&ctx.toclient.mpm_ctx, &tctx.toclient.mpm_ctx, &tctx.toclient.pmq, l7data, sizeof(l7data));
if (cnt != 1) {
printf("cnt %u != 1: ", cnt);
r = 0;
}
@ -1243,6 +1246,45 @@ int AlpDetectTest11(void) {
return r;
}
/** \test AlpProtoSignature test */
int AlpDetectTest12(void) {
AlpProtoDetectCtx ctx;
int r = 0;
AlpProtoInit(&ctx);
AlpProtoAdd(&ctx, IPPROTO_TCP, ALPROTO_HTTP, "HTTP", 4, 0, STREAM_TOSERVER);
AlpProtoFinalizeGlobal(&ctx);
if (ctx.head == NULL) {
printf("ctx.head == NULL: ");
goto end;
}
if (ctx.head->proto != ALPROTO_HTTP) {
printf("ctx.head->proto != ALPROTO_HTTP: ");
goto end;
}
if (ctx.sigs != 1) {
printf("ctx.sigs %"PRIu16", expected 1: ", ctx.sigs);
goto end;
}
if (ctx.map == NULL) {
printf("no mapping: ");
goto end;
}
if (ctx.map[ctx.head->co->id] != ctx.head) {
printf("wrong sig: ");
goto end;
}
r = 1;
end:
return r;
}
#endif /* UNITTESTS */
void AlpDetectRegisterTests(void) {
@ -1258,5 +1300,6 @@ void AlpDetectRegisterTests(void) {
UtRegisterTest("AlpDetectTest09", AlpDetectTest09, 1);
UtRegisterTest("AlpDetectTest10", AlpDetectTest10, 1);
UtRegisterTest("AlpDetectTest11", AlpDetectTest11, 1);
UtRegisterTest("AlpDetectTest12", AlpDetectTest12, 1);
#endif /* UNITTESTS */
}

50
src/app-layer-detect-proto.h
Unescape Escape View File

@ -25,6 +25,7 @@
#define __APP_LAYER_DETECT_PROTO_H__
#include "stream.h"
#include "detect-content.h"
typedef struct AlpProtoDetectDirectionThread_ {
MpmThreadCtx mpm_ctx;
@ -36,11 +37,58 @@ typedef struct AlpProtoDetectThreadCtx_ {
AlpProtoDetectDirectionThread toclient;
} AlpProtoDetectThreadCtx;
int AppLayerHandleMsg(AlpProtoDetectThreadCtx *, StreamMsg *smsg);
/** \brief Signature for proto detection
* \todo we might just use SigMatch here
*/
typedef struct AlpProtoSignature_ {
uint16_t proto; /**< protocol */
DetectContentData *co; /**< content match that needs to match */
struct AlpProtoSignature_ *next; /**< next signature */
} AlpProtoSignature;
#define ALP_DETECT_MAX 256
typedef struct AlpProtoDetectDirection_ {
MpmCtx mpm_ctx;
uint32_t id;
uint16_t map[ALP_DETECT_MAX]; /**< a mapping between condition id's and
protocol */
uint16_t max_len; /**< max length of all patterns, so we can
limit the search */
uint16_t min_len; /**< min length of all patterns, so we can
tell the stream engine to feed data
to app layer as soon as it has min
size data */
} AlpProtoDetectDirection;
typedef struct AlpProtoDetectCtx_ {
AlpProtoDetectDirection toserver;
AlpProtoDetectDirection toclient;
MpmPatternIdStore *mpm_pattern_id_store; /** pattern id store */
int alp_content_module_handle;
/** mapping between proto id's and pattern id's: this will
* be used to look up a proto by the pattern id. The pattern
* id is returned by the mpm */
//uint16_t *proto_map;
/** Mapping between pattern id and signature. As each signature has a
* unique pattern with a unique id, we can lookup the signature by
* the pattern id. */
AlpProtoSignature **map;
AlpProtoSignature *head; /**< list of sigs */
uint16_t sigs; /**< number of sigs */
} AlpProtoDetectCtx;
void *AppLayerDetectProtoThread(void *td);
void AppLayerDetectProtoThreadInit(void);
uint16_t AppLayerDetectGetProto(AlpProtoDetectCtx *, AlpProtoDetectThreadCtx *, uint8_t *, uint16_t, uint8_t);
void AppLayerDetectProtoThreadSpawn(void);
void AlpDetectRegisterTests(void);

93
src/app-layer.c
Unescape Escape View File

@ -25,6 +25,8 @@
#include "suricata-common.h"
#include "app-layer.h"
#include "app-layer-detect-proto.h"
#include "stream-tcp-reassemble.h"
#include "stream-tcp-private.h"
#include "util-debug.h"
@ -93,3 +95,94 @@ void *AppLayerGetProtoStateFromFlow(Flow *f) {
SCReturnPtr(alstate, "void");
}
/** global app layer detection context */
extern AlpProtoDetectCtx alp_proto_ctx;
/**
* \brief Handle a app layer message
*
* If the protocol is yet unknown, the proto detection code is run first.
*
* \param dp_ctx Thread app layer detect context
* \param smsg Stream message
*
* \retval 0 ok
* \retval -1 error
*/
int AppLayerHandleMsg(AlpProtoDetectThreadCtx *dp_ctx, StreamMsg *smsg)
{
SCEnter();
uint16_t alproto = ALPROTO_UNKNOWN;
int r = 0;
TcpSession *ssn = smsg->flow->protoctx;
if (ssn != NULL) {
alproto = ssn->alproto;
/* if we don't know the proto yet and we have received a stream
* initializer message, we run proto detection.
* We receive 2 stream init msgs (one for each direction) but we
* only run the proto detection once. */
if (alproto == ALPROTO_UNKNOWN && smsg->flags & STREAM_START) {
SCLogDebug("Stream initializer (len %" PRIu32 " (%" PRIu32 "))",
smsg->data.data_len, MSG_DATA_SIZE);
//printf("=> Init Stream Data -- start\n");
//PrintRawDataFp(stdout, smsg->init.data, smsg->init.data_len);
//printf("=> Init Stream Data -- end\n");
alproto = AppLayerDetectGetProto(&alp_proto_ctx, dp_ctx,
smsg->data.data, smsg->data.data_len, smsg->flags);
if (alproto != ALPROTO_UNKNOWN) {
/* store the proto and setup the L7 data array */
StreamL7DataPtrInit(ssn);
ssn->alproto = alproto;
ssn->flags |= STREAMTCP_FLAG_APPPROTO_DETECTION_COMPLETED;
r = AppLayerParse(smsg->flow, alproto, smsg->flags,
smsg->data.data, smsg->data.data_len);
} else {
if (smsg->flags & STREAM_TOSERVER) {
if (smsg->data.data_len >= alp_proto_ctx.toserver.max_len) {
ssn->flags |= STREAMTCP_FLAG_APPPROTO_DETECTION_COMPLETED;
SCLogDebug("ALPROTO_UNKNOWN flow %p", smsg->flow);
StreamTcpSetSessionNoReassemblyFlag(ssn, 0);
}
} else if (smsg->flags & STREAM_TOCLIENT) {
if (smsg->data.data_len >= alp_proto_ctx.toclient.max_len) {
ssn->flags |= STREAMTCP_FLAG_APPPROTO_DETECTION_COMPLETED;
SCLogDebug("ALPROTO_UNKNOWN flow %p", smsg->flow);
StreamTcpSetSessionNoReassemblyFlag(ssn, 1);
}
}
}
} else {
SCLogDebug("stream data (len %" PRIu32 " (%" PRIu32 ")), alproto "
"%"PRIu16" (flow %p)", smsg->data.data_len, MSG_DATA_SIZE,
alproto, smsg->flow);
//printf("=> Stream Data -- start\n");
//PrintRawDataFp(stdout, smsg->data.data, smsg->data.data_len);
//printf("=> Stream Data -- end\n");
/* if we don't have a data object here we are not getting it
* a start msg should have gotten us one */
if (alproto != ALPROTO_UNKNOWN) {
r = AppLayerParse(smsg->flow, alproto, smsg->flags,
smsg->data.data, smsg->data.data_len);
} else {
SCLogDebug(" smsg not start, but no l7 data? Weird");
}
}
}
/* flow is free again */
smsg->flow->use_cnt--;
/* return the used message to the queue */
StreamMsgReturnToPool(smsg);
SCReturnInt(r);
}

2
src/app-layer.h
Unescape Escape View File

@ -29,11 +29,13 @@
#include "app-layer-protos.h"
#include "app-layer-parser.h"
#include "app-layer-detect-proto.h"
#include "stream.h"
uint16_t AppLayerGetProtoFromPacket(Packet *);
void *AppLayerGetProtoStateFromPacket(Packet *);
void *AppLayerGetProtoStateFromFlow(Flow *);
int AppLayerHandleMsg(AlpProtoDetectThreadCtx *, StreamMsg *smsg);
#endif /* __APP_LAYER_H__ */

30
src/decode.h
Unescape Escape View File

@ -404,38 +404,10 @@ typedef struct DecodeThreadVars_
* memset or bzero
*/
#define CLEAR_PACKET(p) { \
CLEAR_ADDR(&p->src); \
CLEAR_ADDR(&p->dst); \
if ((p)->tcph != NULL) { \
CLEAR_TCP_PACKET((p)); \
} \
(p)->ethh = NULL; \
(p)->ppph = NULL; \
(p)->greh = NULL; \
(p)->vlanh = NULL; \
(p)->ip4h = NULL; \
(p)->ip6h = NULL; \
(p)->action = 0; \
(p)->pktlen = 0; \
(p)->tunnel_pkt = 0; \
(p)->tunnel_verdicted = 0; \
(p)->rtv_cnt = 0; \
(p)->tpr_cnt = 0; \
(p)->root = NULL; \
(p)->proto = 0; \
(p)->sp = 0; \
(p)->dp = 0; \
(p)->flow = NULL; \
(p)->flowflags = 0; \
(p)->flags = 0; \
(p)->alerts.cnt = 0; \
if ((p)->pktvar != NULL) { \
PktVarFree((p)->pktvar); \
} \
(p)->pktvar = NULL; \
(p)->recursion_level = 0; \
(p)->ts.tv_sec = 0; \
(p)->ts.tv_usec = 0; \
memset((p), 0x00, sizeof(Packet)); \
}
/* reset these to -1(indicates that the packet is fresh from the queue) */

120
src/detect-content.c
Unescape Escape View File

@ -58,8 +58,7 @@ void DetectContentRegister (void) {
/* pass on the content_max_id */
uint32_t DetectContentMaxId(DetectEngineCtx *de_ctx) {
//SCLogDebug("DetectContentMaxId: %" PRIu32 "", de_ctx->content_max_id);
return de_ctx->content_max_id;
return MpmPatternIdStoreGetMaxId(de_ctx->mpm_pattern_id_store);
}
DetectContentData *DetectContentParse (char *contentstr)
@ -433,7 +432,7 @@ static int DetectContentSetup (DetectEngineCtx *de_ctx, Signature *s, char *cont
sm->type = DETECT_CONTENT;
sm->ctx = (void *)cd;
cd->id = DetectContentGetId(de_ctx, cd);
cd->id = DetectContentGetId(de_ctx->mpm_pattern_id_store, cd);
DetectContentPrint(cd);
@ -463,121 +462,6 @@ void DetectContentFree(void *ptr) {
SCFree(cd);
}
/* content hash
* A per detection engine hash to make sure each pattern has a unique global id
* but pattern that are the same share id's.
*/
typedef struct DetectContentTableElmt_ {
uint8_t *pattern; /**< ptr to the pattern */
uint16_t pattern_len; /**< pattern len */
uint32_t id; /**< pattern id */
} DetectContentTableElmt;
static char DetectContentTableCompare(void *p1, uint16_t len1, void *p2, uint16_t len2) {
SCEnter();
BUG_ON(len1 < sizeof(DetectContentTableElmt));
BUG_ON(len2 < sizeof(DetectContentTableElmt));
DetectContentTableElmt *e1 = (DetectContentTableElmt *)p1;
DetectContentTableElmt *e2 = (DetectContentTableElmt *)p2;
if (e1->pattern_len != e2->pattern_len) {
SCReturnInt(0);
}
if (memcmp(e1->pattern, e2->pattern, e1->pattern_len) != 0) {
SCReturnInt(0);
}
SCReturnInt(1);
}
static uint32_t DetectContentTableHash(HashTable *ht, void *p, uint16_t len) {
SCEnter();
BUG_ON(len < sizeof(DetectContentTableElmt));
DetectContentTableElmt *e = (DetectContentTableElmt *)p;
uint32_t hash = e->pattern_len;
uint16_t u = 0;
for (u = 0; u < e->pattern_len; u++) {
hash += e->pattern[u];
}
SCReturnUInt(hash % ht->array_size);
}
static void DetectContentTableElmtFree(void *e) {
DetectContentTableElmt *c = (DetectContentTableElmt *)e;
free(c->pattern);
free(e);
}
int DetectContentTableInitHash(DetectEngineCtx *de_ctx) {
SCEnter();
BUG_ON(de_ctx == NULL);
de_ctx->content_hash = HashTableInit(65536, DetectContentTableHash, DetectContentTableCompare, DetectContentTableElmtFree);
BUG_ON(de_ctx->content_hash == NULL);
SCReturnInt(0);
}
void DetectContentTableFreeHash(DetectEngineCtx *de_ctx) {
SCEnter();
if (de_ctx == NULL || de_ctx->content_hash == NULL) {
SCReturn;
}
HashTableFree(de_ctx->content_hash);
SCReturn;
}
uint32_t DetectContentGetId(DetectEngineCtx *de_ctx, DetectContentData *co) {
SCEnter();
BUG_ON(de_ctx == NULL || de_ctx->content_hash == NULL);
DetectContentTableElmt *e = NULL;
DetectContentTableElmt *r = NULL;
uint32_t id = 0;
e = malloc(sizeof(DetectContentTableElmt));
BUG_ON(e == NULL);
e->pattern = SCMalloc(co->content_len);
BUG_ON(e->pattern == NULL);
memcpy(e->pattern, co->content, co->content_len);
e->pattern_len = co->content_len;
e->id = 0;
r = HashTableLookup(de_ctx->content_hash, (void *)e, sizeof(DetectContentTableElmt));
if (r == NULL) {
e->id = de_ctx->content_max_id;
de_ctx->content_max_id++;
id = e->id;
int ret = HashTableAdd(de_ctx->content_hash, e, sizeof(DetectContentTableElmt));
BUG_ON(ret != 0);
e = NULL;
de_ctx->content_hash_unique++;
} else {
id = r->id;
de_ctx->content_hash_shared++;
}
if (e != NULL)
free(e);
SCReturnUInt(id);
}
#ifdef UNITTESTS /* UNITTESTS */
/**

4
src/detect-content.h
Unescape Escape View File

@ -88,8 +88,4 @@ SigMatch *SigMatchGetLastPattern(Signature *s);
void DetectContentFree(void *);
int DetectContentTableInitHash(DetectEngineCtx *);
void DetectContentTableFreeHash(DetectEngineCtx *);
uint32_t DetectContentGetId(DetectEngineCtx *, DetectContentData *);
#endif /* __DETECT_CONTENT_H__ */

200
src/detect-engine-mpm.c
Unescape Escape View File

@ -551,6 +551,8 @@ static int PatternMatchPreprarePopulateMpm(DetectEngineCtx *de_ctx, SigGroupHead
mpm_table[sgh->mpm_ctx->mpm_type].AddPattern(sgh->mpm_ctx, co->content, co->content_len, offset, depth, co->id, s->num, flags);
}
s->mpm_pattern_id = co->id;
SCLogDebug("%"PRIu32" adding co->id %"PRIu32" to the mpm phase (s->num %"PRIu32")", s->id, co->id, s->num);
} else {
SCLogDebug("%"PRIu32" no mpm pattern selected", s->id);
@ -612,7 +614,7 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
for (sm = s->umatch; sm != NULL; sm = sm->next) {
if (sm->type == DETECT_URICONTENT) {
ur_cnt++;
s->flags |= SIG_FLAG_MPM;
s->flags |= SIG_FLAG_MPM_URI;
}
}
}
@ -859,6 +861,8 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
mpm_table[sh->mpm_uri_ctx->mpm_type].AddPattern(sh->mpm_uri_ctx, ud->uricontent, ud->uricontent_len, 0, 0, ud->id, s->num, flags);
}
uricontent_mpmadded = 1;
s->mpm_uripattern_id = ud->id;
}
}
}
@ -906,3 +910,197 @@ error:
return -1;
}
/** \brief Pattern ID Hash for sharing pattern id's
*
* A per detection engine hash to make sure each pattern has a unique
* global id but patterns that are the same share id's.
*/
typedef struct MpmPatternIdTableElmt_ {
uint8_t *pattern; /**< ptr to the pattern */
uint16_t pattern_len; /**< pattern len */
uint32_t id; /**< pattern id */
} MpmPatternIdTableElmt;
/** \brief Hash compare func for MpmPatternId api
* \retval 1 patterns are the same
* \retval 0 patterns are not the same
**/
static char MpmPatternIdCompare(void *p1, uint16_t len1, void *p2, uint16_t len2) {
SCEnter();
BUG_ON(len1 < sizeof(MpmPatternIdTableElmt));
BUG_ON(len2 < sizeof(MpmPatternIdTableElmt));
MpmPatternIdTableElmt *e1 = (MpmPatternIdTableElmt *)p1;
MpmPatternIdTableElmt *e2 = (MpmPatternIdTableElmt *)p2;
if (e1->pattern_len != e2->pattern_len) {
SCReturnInt(0);
}
if (memcmp(e1->pattern, e2->pattern, e1->pattern_len) != 0) {
SCReturnInt(0);
}
SCReturnInt(1);
}
/** \brief Hash func for MpmPatternId api
* \retval hash hash value
*/
static uint32_t MpmPatternIdHashFunc(HashTable *ht, void *p, uint16_t len) {
SCEnter();
BUG_ON(len < sizeof(MpmPatternIdTableElmt));
MpmPatternIdTableElmt *e = (MpmPatternIdTableElmt *)p;
uint32_t hash = e->pattern_len;
uint16_t u = 0;
for (u = 0; u < e->pattern_len; u++) {
hash += e->pattern[u];
}
SCReturnUInt(hash % ht->array_size);
}
/** \brief free a MpmPatternIdTableElmt */
static void MpmPatternIdTableElmtFree(void *e) {
MpmPatternIdTableElmt *c = (MpmPatternIdTableElmt *)e;
free(c->pattern);
free(e);
}
/** \brief alloc initialize the MpmPatternIdHash */
MpmPatternIdStore *MpmPatternIdTableInitHash(void) {
SCEnter();
MpmPatternIdStore *ht = SCMalloc(sizeof(MpmPatternIdStore));
BUG_ON(ht == NULL);
memset(ht, 0x00, sizeof(MpmPatternIdStore));
ht->hash = HashTableInit(65536, MpmPatternIdHashFunc, MpmPatternIdCompare, MpmPatternIdTableElmtFree);
BUG_ON(ht->hash == NULL);
SCReturnPtr(ht, "MpmPatternIdStore");
}
void MpmPatternIdTableFreeHash(MpmPatternIdStore *ht) {
SCEnter();
if (ht == NULL) {
SCReturn;
}
if (ht->hash != NULL) {
HashTableFree(ht->hash);
}
SCFree(ht);
SCReturn;
}
uint32_t MpmPatternIdStoreGetMaxId(MpmPatternIdStore *ht) {
if (ht == NULL) {
return 0;
}
return ht->max_id;
}
/**
* \brief Get the pattern id for a content pattern
*
* \param ht mpm pattern id hash table store
* \param co content pattern data
*
* \retval id pattern id
*/
uint32_t DetectContentGetId(MpmPatternIdStore *ht, DetectContentData *co) {
SCEnter();
BUG_ON(ht == NULL || ht->hash == NULL);
MpmPatternIdTableElmt *e = NULL;
MpmPatternIdTableElmt *r = NULL;
uint32_t id = 0;
e = malloc(sizeof(MpmPatternIdTableElmt));
BUG_ON(e == NULL);
e->pattern = SCMalloc(co->content_len);
BUG_ON(e->pattern == NULL);
memcpy(e->pattern, co->content, co->content_len);
e->pattern_len = co->content_len;
e->id = 0;
r = HashTableLookup(ht->hash, (void *)e, sizeof(MpmPatternIdTableElmt));
if (r == NULL) {
e->id = ht->max_id;
ht->max_id++;
id = e->id;
int ret = HashTableAdd(ht->hash, e, sizeof(MpmPatternIdTableElmt));
BUG_ON(ret != 0);
e = NULL;
ht->unique_patterns++;
} else {
id = r->id;
ht->shared_patterns++;
}
if (e != NULL)
free(e);
SCReturnUInt(id);
}
/**
* \brief Get the pattern id for a uricontent pattern
*
* \param ht mpm pattern id hash table store
* \param co content pattern data
*
* \retval id pattern id
*/
uint32_t DetectUricontentGetId(MpmPatternIdStore *ht, DetectUricontentData *co) {
SCEnter();
BUG_ON(ht == NULL || ht->hash == NULL);
MpmPatternIdTableElmt *e = NULL;
MpmPatternIdTableElmt *r = NULL;
uint32_t id = 0;
e = malloc(sizeof(MpmPatternIdTableElmt));
BUG_ON(e == NULL);
e->pattern = SCMalloc(co->uricontent_len);
BUG_ON(e->pattern == NULL);
memcpy(e->pattern, co->uricontent, co->uricontent_len);
e->pattern_len = co->uricontent_len;
e->id = 0;
r = HashTableLookup(ht->hash, (void *)e, sizeof(MpmPatternIdTableElmt));
if (r == NULL) {
e->id = ht->max_id;
ht->max_id++;
id = e->id;
int ret = HashTableAdd(ht->hash, e, sizeof(MpmPatternIdTableElmt));
BUG_ON(ret != 0);
e = NULL;
ht->unique_patterns++;
} else {
id = r->id;
ht->shared_patterns++;
}
if (e != NULL)
free(e);
SCReturnUInt(id);
}

9
src/detect-engine-mpm.h
Unescape Escape View File

@ -26,6 +26,9 @@
#include "tm-modules.h"
#include "detect-content.h"
#include "detect-uricontent.h"
uint16_t PatternMatchDefaultMatcher(void);
uint32_t PacketPatternSearch(ThreadVars *, DetectEngineThreadCtx *, Packet *);
@ -49,5 +52,11 @@ TmEcode DetectEngineThreadCtxDeinit(ThreadVars *, void *);
void DbgPrintSearchStats();
MpmPatternIdStore *MpmPatternIdTableInitHash(void);
void MpmPatternIdTableFreeHash(MpmPatternIdStore *);
uint32_t MpmPatternIdStoreGetMaxId(MpmPatternIdStore *);
uint32_t DetectContentGetId(MpmPatternIdStore *, DetectContentData *);
uint32_t DetectUricontentGetId(MpmPatternIdStore *, DetectUricontentData *);
#endif /* __DETECT_ENGINE_MPM_H__ */

6
src/detect-engine-siggroup.c
Unescape Escape View File

@ -883,12 +883,16 @@ int SigGroupHeadAppendSig(DetectEngineCtx *de_ctx, SigGroupHead **sgh,
SCLogDebug("(%p)->mpm_content_maxlen %u", *sgh, (*sgh)->mpm_content_maxlen);
}
}
if (s->flags & SIG_FLAG_MPM) {
if (s->mpm_uricontent_maxlen > 0) {
if ((*sgh)->mpm_uricontent_maxlen == 0)
(*sgh)->mpm_uricontent_maxlen = s->mpm_uricontent_maxlen;
if ((*sgh)->mpm_uricontent_maxlen > s->mpm_uricontent_maxlen)
(*sgh)->mpm_uricontent_maxlen = s->mpm_uricontent_maxlen;
SCLogDebug("(%p)->mpm_uricontent_maxlen %u", *sgh, (*sgh)->mpm_uricontent_maxlen);
}
}
return 0;
@ -1241,7 +1245,7 @@ int SigGroupHeadLoadUricontent(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
if (s == NULL)
continue;
if (!(s->flags & SIG_FLAG_MPM))
if (!(s->flags & SIG_FLAG_MPM_URI))
continue;
sm = s->umatch;

11
src/detect-engine.c
Unescape Escape View File

@ -79,7 +79,10 @@ DetectEngineCtx *DetectEngineCtxInit(void) {
ThresholdHashInit(de_ctx);
VariableNameInitHash(de_ctx);
DetectContentTableInitHash(de_ctx);
de_ctx->mpm_pattern_id_store = MpmPatternIdTableInitHash();
if (de_ctx->mpm_pattern_id_store == NULL) {
goto error;
}
return de_ctx;
error:
@ -95,7 +98,8 @@ void DetectEngineCtxFree(DetectEngineCtx *de_ctx) {
/* Normally the hashes are freed elsewhere, but
* to be sure look at them again here.
*/
DetectContentTableFreeHash(de_ctx); /* normally cleaned up in SigGroupBuild */
MpmPatternIdTableFreeHash(de_ctx->mpm_pattern_id_store); /* normally cleaned up in SigGroupBuild */
SigGroupHeadHashFree(de_ctx);
SigGroupHeadMpmHashFree(de_ctx);
SigGroupHeadMpmUriHashFree(de_ctx);
@ -447,7 +451,8 @@ TmEcode DetectEngineThreadCtxInit(ThreadVars *tv, void *initdata, void **data) {
PatternMatchThreadPrepare(&det_ctx->mtc, de_ctx->mpm_matcher, DetectContentMaxId(de_ctx));
PatternMatchThreadPrepare(&det_ctx->mtcu, de_ctx->mpm_matcher, DetectUricontentMaxId(de_ctx));
PmqSetup(&det_ctx->pmq, DetectEngineGetMaxSigId(de_ctx));
//PmqSetup(&det_ctx->pmq, DetectEngineGetMaxSigId(de_ctx), DetectContentMaxId(de_ctx));
PmqSetup(&det_ctx->pmq, 0, DetectContentMaxId(de_ctx));
/* IP-ONLY */
DetectEngineIPOnlyThreadInit(de_ctx,&det_ctx->io_ctx);

61
src/detect-fast-pattern.c
Unescape Escape View File

@ -438,8 +438,10 @@ int DetectFastPatternTest08(void)
p.proto = IPPROTO_TCP;
DetectEngineCtx *de_ctx = DetectEngineCtxInit();
if (de_ctx == NULL)
if (de_ctx == NULL) {
printf("de_ctx init: ");
goto end;
}
de_ctx->flags |= DE_QUIET;
@ -448,27 +450,30 @@ int DetectFastPatternTest08(void)
"content:string2; content:strings3; fast_pattern; "
"content:strings_str4; content:strings_string5; "
"sid:1;)");
if (de_ctx->sig_list == NULL)
if (de_ctx->sig_list == NULL) {
printf("sig parse failed: ");
goto end;
}
SigGroupBuild(de_ctx);
DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
/* start the search phase */
det_ctx->sgh = SigMatchSignaturesGetSgh(&th_v, de_ctx, det_ctx, &p);
if (PacketPatternSearch(&th_v, det_ctx, &p) == 1)
result = 1;
uint32_t r = PacketPatternSearch(&th_v, det_ctx, &p);
if (r != 1) {
printf("expected 1, got %"PRIu32": ", r);
goto end;
}
result = 1;
end:
SigGroupCleanup(de_ctx);
SigCleanSignatures(de_ctx);
DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
end:
DetectEngineCtxFree(de_ctx);
return result;
}
/**
* \test Checks that a fast_pattern is used in the mpm phase, when the payload
* doesn't contain the fast_pattern string within it.
@ -549,8 +554,10 @@ int DetectFastPatternTest10(void)
p.proto = IPPROTO_TCP;
DetectEngineCtx *de_ctx = DetectEngineCtxInit();
if (de_ctx == NULL)
if (de_ctx == NULL) {
printf("de_ctx init: ");
goto end;
}
de_ctx->flags |= DE_QUIET;
@ -559,23 +566,27 @@ int DetectFastPatternTest10(void)
"content:string2; content:strings3; fast_pattern; "
"content:strings4_imp; fast_pattern; "
"content:strings_string5; sid:1;)");
if (de_ctx->sig_list == NULL)
if (de_ctx->sig_list == NULL) {
printf("sig parse failed: ");
goto end;
}
SigGroupBuild(de_ctx);
DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
/* start the search phase */
det_ctx->sgh = SigMatchSignaturesGetSgh(&th_v, de_ctx, det_ctx, &p);
if (PacketPatternSearch(&th_v, det_ctx, &p) == 1)
result = 1;
uint32_t r = PacketPatternSearch(&th_v, det_ctx, &p);
if (r != 1) {
printf("expected 1, got %"PRIu32": ", r);
goto end;
}
result = 1;
end:
SigGroupCleanup(de_ctx);
SigCleanSignatures(de_ctx);
DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
end:
DetectEngineCtxFree(de_ctx);
return result;
}
@ -718,8 +729,10 @@ int DetectFastPatternTest13(void)
p.proto = IPPROTO_TCP;
DetectEngineCtx *de_ctx = DetectEngineCtxInit();
if (de_ctx == NULL)
if (de_ctx == NULL) {
printf("de_ctx init: ");
goto end;
}
de_ctx->flags |= DE_QUIET;
@ -728,23 +741,27 @@ int DetectFastPatternTest13(void)
"content:string2; content:strings3; "
"content:strings4_imp; "
"content:strings_string5; sid:1;)");
if (de_ctx->sig_list == NULL)
if (de_ctx->sig_list == NULL) {
printf("sig parse failed: ");
goto end;
}
SigGroupBuild(de_ctx);
DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
/* start the search phase */
det_ctx->sgh = SigMatchSignaturesGetSgh(&th_v, de_ctx, det_ctx, &p);
if (PacketPatternSearch(&th_v, det_ctx, &p) == 1)
result = 1;
uint32_t r = PacketPatternSearch(&th_v, det_ctx, &p);
if (r != 1) {
printf("expected 1 result, got %"PRIu32": ", r);
goto end;
}
result = 1;
end:
SigGroupCleanup(de_ctx);
SigCleanSignatures(de_ctx);
DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
end:
DetectEngineCtxFree(de_ctx);
return result;
}

26
src/detect-parse.c
Unescape Escape View File

@ -841,7 +841,11 @@ Signature *SigInit(DetectEngineCtx *de_ctx, char *sigstr) {
if (ud == NULL)
continue;
sig->flags |= SIG_FLAG_MPM;
sig->flags |= SIG_FLAG_MPM_URI;
if (ud->flags & DETECT_URICONTENT_NEGATED) {
sig->flags |= SIG_FLAG_MPM_URI_NEG;
}
}
}
@ -850,7 +854,6 @@ Signature *SigInit(DetectEngineCtx *de_ctx, char *sigstr) {
/* determine the length of the longest pattern in the sig */
if (sig->flags & SIG_FLAG_MPM) {
sig->mpm_content_maxlen = 0;
sig->mpm_uricontent_maxlen = 0;
for (sm = sig->pmatch; sm != NULL; sm = sm->next) {
if (sm->type == DETECT_CONTENT) {
@ -864,6 +867,9 @@ Signature *SigInit(DetectEngineCtx *de_ctx, char *sigstr) {
sig->mpm_content_maxlen = cd->content_len;
}
}
}
if (sig->flags & SIG_FLAG_MPM_URI) {
sig->mpm_uricontent_maxlen = 0;
for (sm = sig->umatch; sm != NULL; sm = sm->next) {
if (sm->type == DETECT_URICONTENT) {
@ -963,7 +969,11 @@ Signature *SigInitReal(DetectEngineCtx *de_ctx, char *sigstr) {
if (ud == NULL)
continue;
sig->flags |= SIG_FLAG_MPM;
sig->flags |= SIG_FLAG_MPM_URI;
if (ud->flags & DETECT_URICONTENT_NEGATED) {
sig->flags |= SIG_FLAG_MPM_URI_NEG;
}
}
}
@ -972,7 +982,6 @@ Signature *SigInitReal(DetectEngineCtx *de_ctx, char *sigstr) {
/* determine the length of the longest pattern in the sig */
if (sig->flags & SIG_FLAG_MPM) {
sig->mpm_content_maxlen = 0;
sig->mpm_uricontent_maxlen = 0;
for (sm = sig->pmatch; sm != NULL; sm = sm->next) {
if (sm->type == DETECT_CONTENT) {
@ -986,6 +995,9 @@ Signature *SigInitReal(DetectEngineCtx *de_ctx, char *sigstr) {
sig->mpm_content_maxlen = cd->content_len;
}
}
}
if (sig->flags & SIG_FLAG_MPM_URI) {
sig->mpm_uricontent_maxlen = 0;
for (sm = sig->umatch; sm != NULL; sm = sm->next) {
if (sm->type == DETECT_URICONTENT) {
@ -1018,7 +1030,6 @@ Signature *SigInitReal(DetectEngineCtx *de_ctx, char *sigstr) {
/* determine the length of the longest pattern in the sig */
if (sig->next->flags & SIG_FLAG_MPM) {
sig->next->mpm_content_maxlen = 0;
sig->next->mpm_uricontent_maxlen = 0;
SigMatch *sm;
for (sm = sig->next->pmatch; sm != NULL; sm = sm->next) {
@ -1033,11 +1044,16 @@ Signature *SigInitReal(DetectEngineCtx *de_ctx, char *sigstr) {
sig->next->mpm_content_maxlen = cd->content_len;
}
}
}
if (sig->next->flags & SIG_FLAG_MPM_URI) {
sig->next->mpm_uricontent_maxlen = 0;
for (sm = sig->next->umatch; sm != NULL; sm = sm->next) {
if (sm->type == DETECT_URICONTENT) {
DetectUricontentData *ud = (DetectUricontentData *)sm->ctx;
if (ud == NULL)
continue;
if (sig->next->mpm_uricontent_maxlen == 0)
sig->next->mpm_uricontent_maxlen = ud->uricontent_len;
if (sig->next->mpm_uricontent_maxlen < ud->uricontent_len)

8
src/detect-uricontent.c
Unescape Escape View File

@ -82,7 +82,7 @@ void DetectUricontentRegister (void)
*/
uint32_t DetectUricontentMaxId(DetectEngineCtx *de_ctx)
{
return de_ctx->uricontent_max_id;
return MpmPatternIdStoreGetMaxId(de_ctx->mpm_pattern_id_store);
}
/**
@ -323,11 +323,7 @@ int DetectUricontentSetup (DetectEngineCtx *de_ctx, Signature *s, char *contents
sm->type = DETECT_URICONTENT;
sm->ctx = (void *)cd;
//SigMatchAppendAppLayer(s, sm);
/** \todo use unique id here as well */
cd->id = de_ctx->uricontent_max_id;
de_ctx->uricontent_max_id++;
cd->id = DetectUricontentGetId(de_ctx->mpm_pattern_id_store, cd);
/* Flagged the signature as to inspect the app layer data */
s->flags |= SIG_FLAG_APPLAYER;

39
src/detect.c
Unescape Escape View File

@ -612,13 +612,40 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
continue;
}
/* filter out sigs that want pattern matches, but
* have no matches */
if (!(det_ctx->pmq.sig_bitarray[(sig / 8)] & (1<<(sig % 8))) &&
(s->flags & SIG_FLAG_MPM) && !(s->flags & SIG_FLAG_MPM_NEGCONTENT)) {
SCLogDebug("mpm sig without matches.");
continue;
if (s->flags & SIG_FLAG_MPM) {
if (det_ctx->pmq.pattern_id_bitarray != NULL) {
/* filter out sigs that want pattern matches, but
* have no matches */
if (!(det_ctx->pmq.pattern_id_bitarray[(s->mpm_pattern_id / 8)] & (1<<(s->mpm_pattern_id % 8))) &&
(s->flags & SIG_FLAG_MPM) && !(s->flags & SIG_FLAG_MPM_NEGCONTENT)) {
SCLogDebug("mpm sig without matches (pat id check in content).");
continue;
}
}
}
if (s->flags & SIG_FLAG_MPM_URI) {
if (det_ctx->pmq.pattern_id_bitarray != NULL) {
/* filter out sigs that want pattern matches, but
* have no matches */
if (!(det_ctx->pmq.pattern_id_bitarray[(s->mpm_uripattern_id / 8)] & (1<<(s->mpm_uripattern_id % 8))) &&
(s->flags & SIG_FLAG_MPM_URI) && !(s->flags & SIG_FLAG_MPM_URI_NEG)) {
SCLogDebug("mpm sig without matches (pat id check in uri).");
continue;
}
}
}
#if 0
} else {
/* filter out sigs that want pattern matches, but
* have no matches */
if (!(det_ctx->pmq.sig_bitarray[(sig / 8)] & (1<<(sig % 8))) &&
(s->flags & SIG_FLAG_MPM) && !(s->flags & SIG_FLAG_MPM_NEGCONTENT)) {
SCLogDebug("mpm sig without matches (sig id check).");
continue;
}
}
#endif
/* if the sig has alproto and the session as well they should match */
if (s->alproto != ALPROTO_UNKNOWN && alproto != ALPROTO_UNKNOWN) {

67
src/detect.h
Unescape Escape View File

@ -177,25 +177,28 @@ typedef struct DetectPort_ {
} DetectPort;
/* Signature flags */
#define SIG_FLAG_RECURSIVE 0x0001 /**< recursive capturing enabled */
#define SIG_FLAG_SRC_ANY 0x0002 /**< source is any */
#define SIG_FLAG_DST_ANY 0x0004 /**< destination is any */
#define SIG_FLAG_SP_ANY 0x0008 /**< source port is any */
#define SIG_FLAG_DP_ANY 0x0010 /**< destination port is any */
#define SIG_FLAG_NOALERT 0x0020 /**< no alert flag is set */
#define SIG_FLAG_IPONLY 0x0040 /**< ip only signature */
#define SIG_FLAG_MPM 0x0080 /**< sig has mpm portion (content, uricontent, etc) */
#define SIG_FLAG_DEONLY 0x0100 /**< decode event only signature */
#define SIG_FLAG_PAYLOAD 0x0200 /**< signature is inspecting the packet payload */
#define SIG_FLAG_DSIZE 0x0400 /**< signature has a dsize setting */
#define SIG_FLAG_FLOW 0x0800 /**< signature has a flow setting */
#define SIG_FLAG_MPM_NEGCONTENT 0x1000 /**< sig has negative mpm portion(!content) */
#define SIG_FLAG_APPLAYER 0x2000 /**< signature applies to app layer instead of packets */
#define SIG_FLAG_BIDIREC 0x4000 /**< signature has bidirectional operator */
#define SIG_FLAG_PACKET 0x8000 /**< signature has matches against a packet (as opposed to app layer) */
#define SIG_FLAG_RECURSIVE 0x00000001 /**< recursive capturing enabled */
#define SIG_FLAG_SRC_ANY 0x00000002 /**< source is any */
#define SIG_FLAG_DST_ANY 0x00000004 /**< destination is any */
#define SIG_FLAG_SP_ANY 0x00000008 /**< source port is any */
#define SIG_FLAG_DP_ANY 0x00000010 /**< destination port is any */
#define SIG_FLAG_NOALERT 0x00000020 /**< no alert flag is set */
#define SIG_FLAG_IPONLY 0x00000040 /**< ip only signature */
#define SIG_FLAG_DEONLY 0x00000080 /**< decode event only signature */
#define SIG_FLAG_MPM 0x00000100 /**< sig has mpm portion (content) */
#define SIG_FLAG_MPM_NEGCONTENT 0x00000200 /**< sig has negative mpm portion(!content) */
#define SIG_FLAG_MPM_URI 0x00000400 /**< sig has mpm portion (uricontent) */
#define SIG_FLAG_MPM_URI_NEG 0x00000800 /**< sig has negative mpm portion(!uricontent) */
#define SIG_FLAG_PAYLOAD 0x00001000 /**< signature is inspecting the packet payload */
#define SIG_FLAG_DSIZE 0x00002000 /**< signature has a dsize setting */
#define SIG_FLAG_FLOW 0x00004000 /**< signature has a flow setting */
#define SIG_FLAG_APPLAYER 0x00008000 /**< signature applies to app layer instead of packets */
#define SIG_FLAG_BIDIREC 0x00010000 /**< signature has bidirectional operator */
#define SIG_FLAG_PACKET 0x00020000 /**< signature has matches against a packet (as opposed to app layer) */
/* Detection Engine flags */
#define DE_QUIET 0x01 /**< DE is quiet (esp for unittests) */
@ -218,7 +221,7 @@ typedef struct IPOnlyCIDRItem_ {
/** \brief Signature container */
typedef struct Signature_ {
uint16_t flags;
uint32_t flags;
uint8_t rev;
int prio;
@ -267,6 +270,11 @@ typedef struct Signature_ {
uint16_t sm_cnt;
SigIntId order_id;
/** pattern in the mpm matcher */
uint32_t mpm_pattern_id;
uint32_t mpm_uripattern_id;
} Signature;
typedef struct DetectEngineIPOnlyThreadCtx_ {
@ -325,6 +333,15 @@ typedef struct DetectEngineLookupDsize_ {
*/
#define DSIZE_STATES 2
/* mpm pattern id api */
typedef struct MpmPatternIdStore_ {
HashTable *hash;
uint32_t max_id;
uint32_t unique_patterns;
uint32_t shared_patterns;
} MpmPatternIdStore;
/** \brief threshold ctx */
typedef struct ThresholdCtx_ {
HashListTable *threshold_hash_table_dst; /**< Ipv4 dst hash table */
@ -365,10 +382,6 @@ typedef struct DetectEngineCtx_ {
uint32_t mpm_max_patcnt, mpm_min_patcnt, mpm_tot_patcnt,
mpm_uri_max_patcnt, mpm_uri_min_patcnt, mpm_uri_tot_patcnt;
/* content and uricontent vars */
uint32_t content_max_id;
uint32_t uricontent_max_id;
/* init phase vars */
HashListTable *sgh_hash_table;
@ -421,9 +434,9 @@ typedef struct DetectEngineCtx_ {
uint16_t max_uniq_small_toserver_sp_groups;
uint16_t max_uniq_small_toserver_dp_groups;
HashTable *content_hash;
uint32_t content_hash_unique;
uint32_t content_hash_shared;
/** hash table for looking up patterns for
* id sharing and id tracking. */
MpmPatternIdStore *mpm_pattern_id_store;
} DetectEngineCtx;
/* Engine groups profiles (low, medium, high, custom) */

1
src/stream-tcp-reassemble.c
Unescape Escape View File

@ -50,6 +50,7 @@
#include "util-debug.h"
#include "app-layer-protos.h"
#include "app-layer.h"
//#define DEBUG
#ifdef DEBUG

2
src/util-mpm-b2g.c
Unescape Escape View File

@ -383,10 +383,10 @@ static int B2gAddPattern(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen, uint16_
if (mpm_ctx->maxlen < patlen) mpm_ctx->maxlen = patlen;
if (mpm_ctx->minlen == 0) mpm_ctx->minlen = patlen;
else if (mpm_ctx->minlen > patlen) mpm_ctx->minlen = patlen;
}
/* we need a match */
B2gEndMatchAppend(mpm_ctx, p, offset, depth, pid, sid);
}
mpm_ctx->total_pattern_cnt++;
return 0;

153
src/util-mpm.c
Unescape Escape View File

@ -34,15 +34,19 @@
#include "util-mpm-b3g.h"
#include "util-hashlist.h"
/** \brief Setup a pmq
* \param pmq Pattern matcher queue to be initialized
* \param maxid Max id to be matched on
* \retval -1 error
* \retval 0 ok
*/
int PmqSetup(PatternMatcherQueue *pmq, uint32_t maxid) {
/**
* \brief Setup a pmq
*
* \param pmq Pattern matcher queue to be initialized
* \param maxid Max sig id to be matched on
* \param patmaxid Max pattern id to be matched on
*
* \retval -1 error
* \retval 0 ok
*/
int PmqSetup(PatternMatcherQueue *pmq, uint32_t sig_maxid, uint32_t patmaxid) {
SCEnter();
SCLogDebug("maxid %u", maxid);
SCLogDebug("sig_maxid %u, patmaxid %u", sig_maxid, patmaxid);
if (pmq == NULL) {
SCReturnInt(-1);
@ -50,25 +54,47 @@ int PmqSetup(PatternMatcherQueue *pmq, uint32_t maxid) {
memset(pmq, 0, sizeof(PatternMatcherQueue));
if (maxid == 0) {
SCReturnInt(0);
}
if (sig_maxid > 0) {
pmq->sig_id_array = SCMalloc(sig_maxid * sizeof(uint32_t));
if (pmq->sig_id_array == NULL) {
SCLogError(SC_ERR_MEM_ALLOC, "memory alloc failed");
SCReturnInt(-1);
}
memset(pmq->sig_id_array, 0, sig_maxid * sizeof(uint32_t));
pmq->sig_id_array_cnt = 0;
/* lookup bitarray */
pmq->sig_bitarray = SCMalloc((sig_maxid / 8) + 1);
if (pmq->sig_bitarray == NULL) {
SCLogError(SC_ERR_MEM_ALLOC, "memory alloc failed");
SCReturnInt(-1);
}
memset(pmq->sig_bitarray, 0, (sig_maxid / 8) + 1);
pmq->sig_id_array = SCMalloc(maxid * sizeof(uint32_t));
if (pmq->sig_id_array == NULL) {
SCLogError(SC_ERR_MEM_ALLOC, "memory alloc failed");
SCReturnInt(-1);
SCLogDebug("pmq->sig_id_array %p, pmq->sig_bitarray %p",
pmq->sig_id_array, pmq->sig_bitarray);
}
memset(pmq->sig_id_array, 0, maxid * sizeof(uint32_t));
pmq->sig_id_array_cnt = 0;
/* lookup bitarray */
pmq->sig_bitarray = SCMalloc(maxid / 8 + 1);
if (pmq->sig_bitarray == NULL) {
SCLogError(SC_ERR_MEM_ALLOC, "memory alloc failed");
SCReturnInt(-1);
if (patmaxid > 0) {
pmq->pattern_id_array = SCMalloc(patmaxid * sizeof(uint32_t));
if (pmq->pattern_id_array == NULL) {
SCLogError(SC_ERR_MEM_ALLOC, "memory alloc failed");
SCReturnInt(-1);
}
memset(pmq->pattern_id_array, 0, patmaxid * sizeof(uint32_t));
pmq->pattern_id_array_cnt = 0;
/* lookup bitarray */
pmq->pattern_id_bitarray = SCMalloc((patmaxid / 8) + 1);
if (pmq->pattern_id_bitarray == NULL) {
SCLogError(SC_ERR_MEM_ALLOC, "memory alloc failed");
SCReturnInt(-1);
}
memset(pmq->pattern_id_bitarray, 0, (patmaxid / 8) + 1);
SCLogDebug("pmq->pattern_id_array %p, pmq->pattern_id_bitarray %p",
pmq->pattern_id_array, pmq->pattern_id_bitarray);
}
memset(pmq->sig_bitarray, 0, maxid / 8 + 1);
SCReturnInt(0);
}
@ -94,38 +120,60 @@ MpmVerifyMatch(MpmThreadCtx *thread_ctx, PatternMatcherQueue *pmq, MpmEndMatch *
MpmEndMatch *em = list;
int ret = 0;
for ( ; em != NULL; em = em->next) {
SCLogDebug("em->sig_id %u", em->sig_id);
/* check offset */
if (offset < em->offset)
continue;
/* check depth */
if (em->depth && (offset+patlen) > em->depth)
continue;
if (pmq != NULL) {
/* make sure we only append a sig with a matching pattern once,
* so we won't inspect it more than once. For this we keep a
* bitarray of sig internal id's and flag each sig that matched */
if (!(pmq->sig_bitarray[(em->sig_id / 8)] & (1<<(em->sig_id % 8)))) {
/* flag this sig_id as being added now */
pmq->sig_bitarray[(em->sig_id / 8)] |= (1<<(em->sig_id % 8));
/* append the sig_id to the array with matches */
pmq->sig_id_array[pmq->sig_id_array_cnt] = em->sig_id;
pmq->sig_id_array_cnt++;
}
/* Handle pattern id storage */
if (pmq != NULL && pmq->pattern_id_bitarray != NULL) {
SCLogDebug("using pattern id arrays");
/* nosearch flag */
if (!(em->flags & MPM_ENDMATCH_NOSEARCH)) {
pmq->searchable++;
}
if (!(pmq->pattern_id_bitarray[(em->id / 8)] & (1<<(em->id % 8)))) {
/* flag this pattern id as being added now */
pmq->pattern_id_bitarray[(em->id / 8)] |= (1<<(em->id % 8));
/* append the pattern_id to the array with matches */
pmq->pattern_id_array[pmq->pattern_id_array_cnt] = em->id;
pmq->pattern_id_array_cnt++;
}
ret++;
ret = 1;
} else {
SCLogDebug("not using pattern id arrays, pmq %p, pmq->pattern_id_bitarray %p", pmq, pmq ? pmq->pattern_id_bitarray:NULL);
}
if (pmq != NULL && pmq->sig_bitarray != NULL) {
for ( ; em != NULL; em = em->next) {
SCLogDebug("em->sig_id %u", em->sig_id);
/* check offset */
if (offset < em->offset)
continue;
/* check depth */
if (em->depth && (offset+patlen) > em->depth)
continue;
if (pmq != NULL) {
/* make sure we only append a sig with a matching pattern once,
* so we won't inspect it more than once. For this we keep a
* bitarray of sig internal id's and flag each sig that matched */
if (!(pmq->sig_bitarray[(em->sig_id / 8)] & (1<<(em->sig_id % 8)))) {
/* flag this sig_id as being added now */
pmq->sig_bitarray[(em->sig_id / 8)] |= (1<<(em->sig_id % 8));
/* append the sig_id to the array with matches */
pmq->sig_id_array[pmq->sig_id_array_cnt] = em->sig_id;
pmq->sig_id_array_cnt++;
}
/* nosearch flag */
if (!(em->flags & MPM_ENDMATCH_NOSEARCH)) {
pmq->searchable++;
}
}
ret++;
}
}
if (pmq == NULL)
ret = 1;
SCReturnInt(ret);
}
@ -138,6 +186,11 @@ void PmqReset(PatternMatcherQueue *pmq) {
pmq->sig_bitarray[(pmq->sig_id_array[u] / 8)] &= ~(1<<(pmq->sig_id_array[u] % 8));
}
pmq->sig_id_array_cnt = 0;
for (u = 0; u < pmq->pattern_id_array_cnt; u++) {
pmq->pattern_id_bitarray[(pmq->pattern_id_array[u] / 8)] &= ~(1<<(pmq->pattern_id_array[u] % 8));
}
pmq->pattern_id_array_cnt = 0;
}
/** \brief Cleanup a Pmq

18
src/util-mpm.h
Unescape Escape View File

@ -89,13 +89,19 @@ typedef struct MpmThreadCtx_ {
* thread has this and passes a pointer to it to the pattern matcher.
* The actual pattern matcher will fill the structure. */
typedef struct PatternMatcherQueue_ {
uint32_t *sig_id_array; /* array with internal sig id's that had a
pattern match. These will be inspected
futher by the detection engine. */
uint32_t *sig_id_array; /** array with internal sig id's that had a
pattern match. These will be inspected
futher by the detection engine. */
uint32_t sig_id_array_cnt;
uint8_t *sig_bitarray;
uint32_t searchable; /* counter of the number of matches that
require a search-followup */
uint32_t searchable; /** counter of the number of matches that
require a search-followup */
uint32_t *pattern_id_array; /** array with internal sig id's that had a
pattern match. These will be inspected
futher by the detection engine. */
uint32_t pattern_id_array_cnt;
uint8_t *pattern_id_bitarray; /** bitarray with pattern id matches */
} PatternMatcherQueue;
typedef struct MpmCtx_ {
@ -155,7 +161,7 @@ typedef struct MpmTableElmt_ {
MpmTableElmt mpm_table[MPM_TABLE_SIZE];
int PmqSetup(PatternMatcherQueue *, uint32_t);
int PmqSetup(PatternMatcherQueue *, uint32_t, uint32_t);
void PmqReset(PatternMatcherQueue *);
void PmqCleanup(PatternMatcherQueue *);
void PmqFree(PatternMatcherQueue *);

Write Preview
Loading…
Cancel
Save