Big update:

- Implement "closing" state in flow.
- Add protocol specific timeouts.
- Lots of stream tracking updates, fixing a lot of out of window issues.
- Stream reassembly fixes.
- Implement a new IDS runmode with 4 stream and detect threads.
- Added a BUG_ON macro that aborts the engine if the expression is true.
- Better balance the flow queue handler for traffic that doesn't have flow (like icmp currently).
- Simplify application level protocol in the Tcp Session.
- Add some debugging memory counters.
remotes/origin/master-1.0.x
Victor Julien 17 years ago
parent b4f0e82463
commit b102ea2123

@ -34,7 +34,7 @@
#include "app-layer-protos.h"
#include "app-layer-parser.h"
#define INSPECT_BYTES 32
#define INSPECT_BYTES 32
#define ALP_DETECT_MAX 256
typedef struct AlpProtoDetectDirectionThread_ {
@ -61,25 +61,7 @@ typedef struct AlpProtoDetectCtx_ {
static AlpProtoDetectCtx alp_proto_ctx;
static AlpProtoDetectThreadCtx alp_proto_tctx;
static uint8_t al_proto_id = 0;
/** \brief data stored in the stream */
typedef struct AppLayerDetectProtoData_ {
uint8_t proto;
} AppLayerDetectProtoData;
static Pool *al_detect_proto_pool = NULL;
void *AppLayerDetectProtoAlloc(void *null) {
AppLayerDetectProtoData *d = malloc(sizeof(AppLayerDetectProtoData));
if (d == NULL) {
return NULL;
}
d->proto = ALPROTO_UNKNOWN;
return d;
}
#define AppLayerDetectProtoFree free
void AlpProtoInit(AlpProtoDetectCtx *ctx) {
memset(ctx, 0x00, sizeof(AlpProtoDetectCtx));
@ -108,7 +90,7 @@ void AlpProtoDestroy(AlpProtoDetectCtx *ctx) {
* \param offset Offset setting for the content. E.g. 4 mean that the content has to match after the first 4 bytes of the stream.
* \param flags Set STREAM_TOCLIENT or STREAM_TOSERVER for the direction in which to try to match the content.
*/
void AlpProtoAdd(AlpProtoDetectCtx *ctx, uint16_t ip_proto, uint8_t al_proto, char *content, uint16_t depth, uint16_t offset, uint8_t flags) {
void AlpProtoAdd(AlpProtoDetectCtx *ctx, uint16_t ip_proto, uint16_t al_proto, char *content, uint16_t depth, uint16_t offset, uint8_t flags) {
DetectContentData *cd = DetectContentParse(content);
if (cd == NULL) {
return;
@ -159,13 +141,6 @@ void AlpProtoFinalizeGlobal(AlpProtoDetectCtx *ctx) {
}
void AppLayerDetectProtoThreadInit(void) {
al_proto_id = StreamL7RegisterModule();
al_detect_proto_pool = PoolInit(262144, 32768, AppLayerDetectProtoAlloc, NULL, AppLayerDetectProtoFree);
if (al_detect_proto_pool == NULL) {
exit(1);
}
AlpProtoInit(&alp_proto_ctx);
/** \todo register these in the protocol parser api */
@ -267,12 +242,17 @@ end:
if (dir->mpm_ctx.Cleanup != NULL) {
dir->mpm_ctx.Cleanup(&tdir->mpm_ctx);
}
//#define DEBUG
#ifdef DEBUG
printf("AppLayerDetectGetProto: returning %" PRIu16 " (%s): ", proto, flags & STREAM_TOCLIENT ? "TOCLIENT" : "TOSERVER");
switch (proto) {
case ALPROTO_HTTP:
printf("HTTP\n");
printf("HTTP: ");
/* print the first 32 bytes */
if (buflen > 0) {
PrintRawUriFp(stdout,buf,(buflen>32)?32:buflen);
}
printf("\n");
break;
case ALPROTO_FTP:
printf("FTP\n");
@ -283,6 +263,9 @@ end:
case ALPROTO_SSH:
printf("SSH\n");
break;
case ALPROTO_TLS:
printf("TLS\n");
break;
case ALPROTO_IMAP:
printf("IMAP\n");
break;
@ -297,8 +280,12 @@ end:
break;
case ALPROTO_UNKNOWN:
default:
printf("UNKNOWN\n");
PrintRawDataFp(stdout,buf,buflen);
printf("UNKNOWN (%u): cnt was %u (", proto, cnt);
/* print the first 32 bytes */
if (buflen > 0) {
PrintRawUriFp(stdout,buf,(buflen>32)?32:buflen);
}
printf(")\n");
break;
}
#endif
@ -309,9 +296,7 @@ void *AppLayerDetectProtoThread(void *td)
{
ThreadVars *tv = (ThreadVars *)td;
char run = TRUE;
AppLayerDetectProtoData *al_proto = NULL;
char store = 0;
void *al_data_ptr = NULL;
uint16_t alproto = ALPROTO_UNKNOWN;
/* get the stream msg queue for this thread */
StreamMsgQueue *stream_q = StreamMsgQueueGetByPort(0);
@ -327,20 +312,13 @@ void *AppLayerDetectProtoThread(void *td)
StreamMsg *smsg = StreamMsgGetFromQueue(stream_q);
if (smsg != NULL) {
mutex_lock(&smsg->flow->m);
TcpSession *ssn = smsg->flow->stream;
TcpSession *ssn = smsg->flow->protoctx;
if (ssn != NULL) {
if (ssn->l7data == NULL) {
/* XXX we can use a pool here,
or make it part of the stream setup */
StreamL7DataPtrInit(ssn,StreamL7GetStorageSize());
}
if (ssn->l7data != NULL) {
al_data_ptr = ssn->l7data[al_proto_id];
}
alproto = ssn->alproto;
}
mutex_unlock(&smsg->flow->m);
if (ssn != NULL && ssn->l7data != NULL) {
if (ssn != NULL) {
if (smsg->flags & STREAM_START) {
//printf("L7AppDetectThread: stream initializer (len %" PRIu32 " (%" PRIu32 "))\n", smsg->data.data_len, MSG_DATA_SIZE);
@ -348,17 +326,18 @@ void *AppLayerDetectProtoThread(void *td)
//PrintRawDataFp(stdout, smsg->init.data, smsg->init.data_len);
//printf("=> Init Stream Data -- end\n");
if (al_data_ptr == NULL) {
al_proto = (AppLayerDetectProtoData *)PoolGet(al_detect_proto_pool);
if (al_proto != NULL) {
al_proto->proto = AppLayerDetectGetProto(&alp_proto_ctx, &alp_proto_tctx, smsg->data.data, smsg->data.data_len, smsg->flags);
store = 1;
alproto = AppLayerDetectGetProto(&alp_proto_ctx, &alp_proto_tctx, smsg->data.data, smsg->data.data_len, smsg->flags);
if (alproto != ALPROTO_UNKNOWN) {
/* store the proto and setup the L7 data array */
mutex_lock(&smsg->flow->m);
StreamL7DataPtrInit(ssn,StreamL7GetStorageSize());
ssn->alproto = alproto;
mutex_unlock(&smsg->flow->m);
AppLayerParse(smsg->flow, al_proto->proto, smsg->flags, smsg->data.data, smsg->data.data_len);
}
AppLayerParse(smsg->flow, alproto, smsg->flags, smsg->data.data, smsg->data.data_len);
}
} else {
//printf("AppLayerDetectThread: stream data (len %" PRIu32 " (%" PRIu32 "))\n", smsg->data.data_len, MSG_DATA_SIZE);
//printf("AppLayerDetectThread: stream data (len %" PRIu32 " (%" PRIu32 ")), alproto %"PRIu16"\n", smsg->data.data_len, MSG_DATA_SIZE, alproto);
//printf("=> Stream Data -- start\n");
//PrintRawDataFp(stdout, smsg->data.data, smsg->data.data_len);
@ -366,11 +345,8 @@ void *AppLayerDetectProtoThread(void *td)
/* if we don't have a data object here we are not getting it
* a start msg should have gotten us one */
if (al_data_ptr != NULL) {
al_proto = (AppLayerDetectProtoData *)al_data_ptr;
//printf("AppLayerDetectThread: already established that the proto is %" PRIu32 "\n", al_proto->proto);
AppLayerParse(smsg->flow, al_proto->proto, smsg->flags, smsg->data.data, smsg->data.data_len);
if (alproto != ALPROTO_UNKNOWN) {
AppLayerParse(smsg->flow, alproto, smsg->flags, smsg->data.data, smsg->data.data_len);
} else {
//printf("AppLayerDetectThread: smsg not start, but no l7 data? Weird\n");
}
@ -378,19 +354,9 @@ void *AppLayerDetectProtoThread(void *td)
}
mutex_lock(&smsg->flow->m);
if (store == 1) {
/* store */
if (ssn != NULL && ssn->l7data != NULL) {
ssn->l7data[al_proto_id] = (void *)al_proto;
} else {
al_proto->proto = 0;
PoolReturn(al_detect_proto_pool,(void *)al_proto);
}
store = 0;
}
/* XXX we need to improve this logic */
smsg->flow->use_cnt--;
mutex_unlock(&smsg->flow->m);
/* return the used message to the queue */
StreamMsgReturnToPool(smsg);
}

@ -208,7 +208,8 @@ static int HTTPParseRequest(void *http_state, AppLayerParserState *pstate, uint8
}
break;
}
case 2:
#if 0
case 2: /* BODY */
{
//printf("HTTPParseRequest: request body\n");
@ -223,6 +224,7 @@ static int HTTPParseRequest(void *http_state, AppLayerParserState *pstate, uint8
break;
}
#endif
}
}
@ -347,6 +349,7 @@ static int HTTPParseResponse(void *http_state, AppLayerParserState *pstate, uint
}
break;
}
#if 0
case 2:
{
//printf("HTTPParseResponse: response body\n");
@ -362,6 +365,7 @@ static int HTTPParseResponse(void *http_state, AppLayerParserState *pstate, uint
break;
}
#endif
}
}
@ -370,17 +374,36 @@ static int HTTPParseResponse(void *http_state, AppLayerParserState *pstate, uint
return 1;
}
#ifdef DEBUG
static pthread_mutex_t http_state_mem_lock = PTHREAD_MUTEX_INITIALIZER;
static uint64_t http_state_memuse = 0;
static uint64_t http_state_memcnt = 0;
#endif
static void *HTTPStateAlloc(void) {
void *s = malloc(sizeof(HttpState));
if (s == NULL)
return NULL;
memset(s, 0, sizeof(HttpState));
#ifdef DEBUG
mutex_lock(&http_state_mem_lock);
http_state_memcnt++;
http_state_memuse+=sizeof(HttpState);
mutex_unlock(&http_state_mem_lock);
#endif
return s;
}
static void HTTPStateFree(void *s) {
free(s);
#ifdef DEBUG
mutex_lock(&http_state_mem_lock);
http_state_memcnt--;
http_state_memuse-=sizeof(HttpState);
mutex_unlock(&http_state_mem_lock);
#endif
}
void RegisterHTTPParsers(void) {
@ -396,6 +419,14 @@ void RegisterHTTPParsers(void) {
AppLayerRegisterStateFuncs(ALPROTO_HTTP, HTTPStateAlloc, HTTPStateFree);
}
void HTTPAtExitPrintStats(void) {
#ifdef DEBUG
mutex_lock(&http_state_mem_lock);
printf("HTTPAtExitPrintStats: http_state_memcnt %"PRIu64", http_state_memuse %"PRIu64"\n", http_state_memcnt, http_state_memuse);
mutex_unlock(&http_state_mem_lock);
#endif
}
/* UNITTESTS */
#ifdef UNITTESTS
@ -410,7 +441,7 @@ int HTTPParserTest01(void) {
memset(&f, 0, sizeof(f));
memset(&ssn, 0, sizeof(ssn));
StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize());
f.stream = (void *)&ssn;
f.protoctx = (void *)&ssn;
int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_EOF, httpbuf, httplen);
if (r != 0) {
@ -419,7 +450,7 @@ int HTTPParserTest01(void) {
goto end;
}
HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)];
HttpState *http_state = ssn.aldata[AlpGetStateIdx(ALPROTO_HTTP)];
if (http_state == NULL) {
printf("no http state: ");
result = 0;
@ -447,7 +478,7 @@ int HTTPParserTest02(void) {
memset(&f, 0, sizeof(f));
memset(&ssn, 0, sizeof(ssn));
StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize());
f.stream = (void *)&ssn;
f.protoctx = (void *)&ssn;
int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_EOF, httpbuf, httplen);
if (r != 0) {
@ -456,7 +487,7 @@ int HTTPParserTest02(void) {
goto end;
}
HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)];
HttpState *http_state = ssn.aldata[AlpGetStateIdx(ALPROTO_HTTP)];
if (http_state == NULL) {
result = 0;
goto end;
@ -487,7 +518,7 @@ int HTTPParserTest03(void) {
memset(&f, 0, sizeof(f));
memset(&ssn, 0, sizeof(ssn));
StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize());
f.stream = (void *)&ssn;
f.protoctx = (void *)&ssn;
int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START, httpbuf1, httplen1);
if (r != 0) {
@ -510,7 +541,7 @@ int HTTPParserTest03(void) {
goto end;
}
HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)];
HttpState *http_state = ssn.aldata[AlpGetStateIdx(ALPROTO_HTTP)];
if (http_state == NULL) {
printf("no http state: ");
result = 0;
@ -542,7 +573,7 @@ int HTTPParserTest04(void) {
memset(&f, 0, sizeof(f));
memset(&ssn, 0, sizeof(ssn));
StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize());
f.stream = (void *)&ssn;
f.protoctx = (void *)&ssn;
int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START, httpbuf1, httplen1);
if (r != 0) {
@ -565,7 +596,7 @@ int HTTPParserTest04(void) {
goto end;
}
HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)];
HttpState *http_state = ssn.aldata[AlpGetStateIdx(ALPROTO_HTTP)];
if (http_state == NULL) {
printf("no http state: ");
result = 0;
@ -598,7 +629,7 @@ int HTTPParserTest05(void) {
memset(&ssn, 0, sizeof(ssn));
StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize());
f.stream = (void *)&ssn;
f.protoctx = (void *)&ssn;
int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START, httpbuf1, httplen1);
if (r != 0) {
@ -621,7 +652,7 @@ int HTTPParserTest05(void) {
goto end;
}
HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)];
HttpState *http_state = ssn.aldata[AlpGetStateIdx(ALPROTO_HTTP)];
if (http_state == NULL) {
printf("no http state: ");
result = 0;
@ -650,7 +681,7 @@ int HTTPParserTest06(void) {
memset(&ssn, 0, sizeof(ssn));
StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize());
f.stream = (void *)&ssn;
f.protoctx = (void *)&ssn;
int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START|STREAM_EOF, httpbuf1, httplen1);
if (r != 0) {
@ -659,7 +690,7 @@ int HTTPParserTest06(void) {
goto end;
}
HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)];
HttpState *http_state = ssn.aldata[AlpGetStateIdx(ALPROTO_HTTP)];
if (http_state == NULL) {
printf("no http state: ");
result = 0;
@ -690,7 +721,7 @@ int HTTPParserTest07(void) {
memset(&ssn, 0, sizeof(ssn));
StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize());
f.stream = (void *)&ssn;
f.protoctx = (void *)&ssn;
int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START, httpbuf1, httplen1);
if (r != 0) {
@ -706,7 +737,7 @@ int HTTPParserTest07(void) {
goto end;
}
HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)];
HttpState *http_state = ssn.aldata[AlpGetStateIdx(ALPROTO_HTTP)];
if (http_state == NULL) {
printf("no http state: ");
result = 0;
@ -747,7 +778,7 @@ int HTTPParserTest08(void) {
memset(&ssn, 0, sizeof(ssn));
StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize());
f.stream = (void *)&ssn;
f.protoctx = (void *)&ssn;
int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START, httpbuf1, httplen1);
if (r != 0) {
@ -791,7 +822,7 @@ int HTTPParserTest08(void) {
goto end;
}
HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)];
HttpState *http_state = ssn.aldata[AlpGetStateIdx(ALPROTO_HTTP)];
if (http_state == NULL) {
printf("no http state: ");
result = 0;
@ -827,7 +858,7 @@ int HTTPParserTest09(void) {
memset(&f, 0, sizeof(f));
memset(&ssn, 0, sizeof(ssn));
StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize());
f.stream = (void *)&ssn;
f.protoctx = (void *)&ssn;
uint32_t u;
for (u = 0; u < httplen1; u++) {
@ -860,7 +891,7 @@ int HTTPParserTest09(void) {
}
}
HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)];
HttpState *http_state = ssn.aldata[AlpGetStateIdx(ALPROTO_HTTP)];
if (http_state == NULL) {
printf("no http state: ");
result = 0;
@ -894,7 +925,7 @@ int HTTPParserTest10(void) {
memset(&f, 0, sizeof(f));
memset(&ssn, 0, sizeof(ssn));
StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize());
f.stream = (void *)&ssn;
f.protoctx = (void *)&ssn;
uint32_t u;
for (u = 0; u < httplen1; u++) {
@ -912,7 +943,7 @@ int HTTPParserTest10(void) {
}
}
HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)];
HttpState *http_state = ssn.aldata[AlpGetStateIdx(ALPROTO_HTTP)];
if (http_state == NULL) {
printf("no http state: ");
result = 0;

@ -3,6 +3,7 @@
void RegisterHTTPParsers(void);
void HTTPParserRegisterTests(void);
void HTTPAtExitPrintStats(void);
#endif /* __APP_LAYER_HTTP_H__ */

@ -402,7 +402,7 @@ uint16_t AlpGetStateIdx(uint16_t proto) {
return al_proto_table[proto].storage_id;
}
AppLayerParserStateStore* AppLayerParserStateStoreAlloc(void) {
AppLayerParserStateStore *AppLayerParserStateStoreAlloc(void) {
AppLayerParserStateStore *s = (AppLayerParserStateStore *)malloc(sizeof(AppLayerParserStateStore));
if (s == NULL)
return NULL;
@ -411,6 +411,17 @@ AppLayerParserStateStore* AppLayerParserStateStoreAlloc(void) {
return s;
}
/** \brief free a AppLayerParserStateStore structure
* \param s AppLayerParserStateStore structure to free */
void AppLayerParserStateStoreFree(AppLayerParserStateStore *s) {
if (s->to_server.store != NULL)
free(s->to_server.store);
if (s->to_client.store != NULL)
free(s->to_client.store);
free(s);
}
static void AppLayerParserResultCleanup(AppLayerParserResult *result) {
AppLayerParserResultElmt *e = result->head;
while (e != NULL) {
@ -493,22 +504,21 @@ int AppLayerParse(Flow *f, uint8_t proto, uint8_t flags, uint8_t *input, uint32_
uint16_t parser_idx = 0;
AppLayerProto *p = &al_proto_table[proto];
TcpSession *ssn = f->stream;
TcpSession *ssn = f->protoctx;
if (ssn == NULL) {
printf("AppLayerParse: no stream\n");
printf("AppLayerParse: no session\n");
return -1;
}
/* Get the parser state (if any) */
AppLayerParserStateStore *parser_state_store = (AppLayerParserStateStore *)ssn->l7data[app_layer_sid];
AppLayerParserStateStore *parser_state_store = (AppLayerParserStateStore *)ssn->aldata[app_layer_sid];
if (parser_state_store == NULL) {
parser_state_store = AppLayerParserStateStoreAlloc();
if (parser_state_store == NULL)
return -1;
mutex_lock(&f->m);
if (ssn->l7data != NULL) /** \todo remove once we fixed ssn timeouts */
ssn->l7data[app_layer_sid] = (void *)parser_state_store;
ssn->aldata[app_layer_sid] = (void *)parser_state_store;
mutex_unlock(&f->m);
}
@ -546,8 +556,7 @@ int AppLayerParse(Flow *f, uint8_t proto, uint8_t flags, uint8_t *input, uint32_
/* See if we already have a 'app layer' state */
void *app_layer_state = NULL;
mutex_lock(&f->m);
if (ssn->l7data != NULL) /** \todo remove once we fixed ssn timeouts */
app_layer_state = ssn->l7data[p->storage_id];
app_layer_state = ssn->aldata[p->storage_id];
mutex_unlock(&f->m);
if (app_layer_state == NULL) {
app_layer_state = p->StateAlloc();
@ -555,8 +564,7 @@ int AppLayerParse(Flow *f, uint8_t proto, uint8_t flags, uint8_t *input, uint32_
return -1;
mutex_lock(&f->m);
if (ssn->l7data != NULL) /** \todo remove once we fixed ssn timeouts */
ssn->l7data[p->storage_id] = app_layer_state;
ssn->aldata[p->storage_id] = app_layer_state;
mutex_unlock(&f->m);
}
@ -580,6 +588,39 @@ void RegisterAppLayerParsers(void) {
al_result_pool = PoolInit(100,10,AlpResultElmtPoolAlloc,NULL,AlpResultElmtPoolFree);
}
void AppLayerParserCleanupState(TcpSession *ssn) {
if (ssn == NULL) {
//printf("AppLayerParserCleanupState: no ssn\n");
return;
}
AppLayerProto *p = &al_proto_table[ssn->alproto];
if (p == NULL) {
//printf("AppLayerParserCleanupState: no parser state for %"PRIu16"\n", ssn->alproto);
return;
}
/* free the parser protocol state */
if (p->StateFree != NULL) {
if (ssn->aldata[p->storage_id] != NULL) {
//printf("AppLayerParserCleanupState: calling StateFree\n");
p->StateFree(ssn->aldata[p->storage_id]);
ssn->aldata[p->storage_id] = NULL;
}
}
if (ssn->aldata != NULL) {
if (ssn->aldata[app_layer_sid] != NULL) {
//printf("AppLayerParserCleanupState: calling AppLayerParserStateStoreFree\n");
AppLayerParserStateStoreFree(ssn->aldata[app_layer_sid]);
ssn->aldata[app_layer_sid] = NULL;
}
free(ssn->aldata);
ssn->aldata = NULL;
}
}
/** \brief Create a mapping between the individual parsers local field id's
* and the global field parser id's.
*

@ -87,5 +87,8 @@ int AlpParseFieldByEOF(AppLayerParserResult *, AppLayerParserState *, uint16_t,
int AlpParseFieldByDelimiter(AppLayerParserResult *, AppLayerParserState *, uint16_t, const uint8_t *, uint8_t, uint8_t *, uint32_t, uint32_t *);
uint16_t AlpGetStateIdx(uint16_t);
#include "stream-tcp-private.h"
void AppLayerParserCleanupState(TcpSession *);
#endif /* __APP_LAYER_PARSER_H__ */

@ -149,7 +149,7 @@ static int TLSParserTest01(void) {
memset(&f, 0, sizeof(f));
memset(&ssn, 0, sizeof(ssn));
StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize());
f.stream = (void *)&ssn;
f.protoctx = (void *)&ssn;
int r = AppLayerParse(&f, ALPROTO_TLS, STREAM_TOSERVER|STREAM_EOF, tlsbuf, tlslen);
if (r != 0) {
@ -158,7 +158,7 @@ static int TLSParserTest01(void) {
goto end;
}
TlsState *tls_state = ssn.l7data[AlpGetStateIdx(ALPROTO_TLS)];
TlsState *tls_state = ssn.aldata[AlpGetStateIdx(ALPROTO_TLS)];
if (tls_state == NULL) {
printf("no tls state: ");
result = 0;
@ -193,7 +193,7 @@ static int TLSParserTest02(void) {
memset(&f, 0, sizeof(f));
memset(&ssn, 0, sizeof(ssn));
StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize());
f.stream = (void *)&ssn;
f.protoctx = (void *)&ssn;
int r = AppLayerParse(&f, ALPROTO_TLS, STREAM_TOSERVER, tlsbuf1, tlslen1);
if (r != 0) {
@ -209,7 +209,7 @@ static int TLSParserTest02(void) {
goto end;
}
TlsState *tls_state = ssn.l7data[AlpGetStateIdx(ALPROTO_TLS)];
TlsState *tls_state = ssn.aldata[AlpGetStateIdx(ALPROTO_TLS)];
if (tls_state == NULL) {
printf("no tls state: ");
result = 0;
@ -246,7 +246,7 @@ static int TLSParserTest03(void) {
memset(&f, 0, sizeof(f));
memset(&ssn, 0, sizeof(ssn));
StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize());
f.stream = (void *)&ssn;
f.protoctx = (void *)&ssn;
int r = AppLayerParse(&f, ALPROTO_TLS, STREAM_TOSERVER, tlsbuf1, tlslen1);
if (r != 0) {
@ -269,7 +269,7 @@ static int TLSParserTest03(void) {
goto end;
}
TlsState *tls_state = ssn.l7data[AlpGetStateIdx(ALPROTO_TLS)];
TlsState *tls_state = ssn.aldata[AlpGetStateIdx(ALPROTO_TLS)];
if (tls_state == NULL) {
printf("no tls state: ");
result = 0;
@ -308,7 +308,7 @@ static int TLSParserTest04(void) {
memset(&f, 0, sizeof(f));
memset(&ssn, 0, sizeof(ssn));
StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize());
f.stream = (void *)&ssn;
f.protoctx = (void *)&ssn;
int r = AppLayerParse(&f, ALPROTO_TLS, STREAM_TOSERVER, tlsbuf1, tlslen1);
if (r != 0) {
@ -338,7 +338,7 @@ static int TLSParserTest04(void) {
goto end;
}
TlsState *tls_state = ssn.l7data[AlpGetStateIdx(ALPROTO_TLS)];
TlsState *tls_state = ssn.aldata[AlpGetStateIdx(ALPROTO_TLS)];
if (tls_state == NULL) {
printf("no tls state: ");
result = 0;

@ -13,5 +13,8 @@
#include <stdint.h>
#include <inttypes.h>
#include <assert.h>
#define BUG_ON(x) assert(!(x))
#endif /* __EIDPS_COMMON_H__ */

@ -624,6 +624,320 @@ int RunModeIdsPcap2(DetectEngineCtx *de_ctx, char *iface) {
return 0;
}
/** \brief Live pcap mode with 4 stream tracking and reassembly threads, testing the flow queuehandler */
int RunModeIdsPcap3(DetectEngineCtx *de_ctx, char *iface) {
TimeModeSetLive();
/* create the threads */
ThreadVars *tv_receivepcap = TmThreadCreatePacketHandler("ReceivePcap","packetpool","packetpool","pickup-queue","simple","1slot_noinout");
if (tv_receivepcap == NULL) {
printf("ERROR: TmThreadsCreate failed\n");
exit(EXIT_FAILURE);
}
TmModule *tm_module = TmModuleGetByName("ReceivePcap");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName failed for ReceivePcap\n");
exit(EXIT_FAILURE);
}
Tm1SlotSetFunc(tv_receivepcap,tm_module,(void *)iface);
if (TmThreadSpawn(tv_receivepcap) != 0) {
printf("ERROR: TmThreadSpawn failed\n");
exit(EXIT_FAILURE);
}
ThreadVars *tv_decode1 = TmThreadCreatePacketHandler("Decode1","pickup-queue","simple","decode-queue1,decode-queue2,decode-queue3,decode-queue4","flow","1slot");
if (tv_decode1 == NULL) {
printf("ERROR: TmThreadsCreate failed for Decode1\n");
exit(EXIT_FAILURE);
}
tm_module = TmModuleGetByName("DecodePcap");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName DecodePcap failed\n");
exit(EXIT_FAILURE);
}
Tm1SlotSetFunc(tv_decode1,tm_module,NULL);
if (TmThreadSpawn(tv_decode1) != 0) {
printf("ERROR: TmThreadSpawn failed\n");
exit(EXIT_FAILURE);
}
ThreadVars *tv;
tv = TmThreadCreatePacketHandler("Stream1","decode-queue1","simple","packetpool","packetpool","varslot");
if (tv == NULL) {
printf("ERROR: TmThreadsCreate failed for Stream1\n");
exit(EXIT_FAILURE);
}
tm_module = TmModuleGetByName("StreamTcp");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName StreamTcp failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("Detect");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName StreamTcp failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,(void *)de_ctx);
tm_module = TmModuleGetByName("RespondReject");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for RespondReject failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("AlertFastlog");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for AlertFastlog failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("LogHttplog");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("AlertUnifiedLog");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for AlertUnifiedLog failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("AlertUnifiedAlert");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for AlertUnifiedAlert failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("AlertDebuglog");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
TmThreadSetCPUAffinity(tv, 0);
if (TmThreadSpawn(tv) != 0) {
printf("ERROR: TmThreadSpawn failed\n");
exit(EXIT_FAILURE);
}
tv = TmThreadCreatePacketHandler("Stream2","decode-queue2","simple","packetpool","packetpool","varslot");
if (tv == NULL) {
printf("ERROR: TmThreadsCreate failed for Stream1\n");
exit(EXIT_FAILURE);
}
tm_module = TmModuleGetByName("StreamTcp");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName StreamTcp failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("Detect");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName StreamTcp failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,(void *)de_ctx);
tm_module = TmModuleGetByName("RespondReject");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for RespondReject failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("AlertFastlog");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for AlertFastlog failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("LogHttplog");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("AlertUnifiedLog");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for AlertUnifiedLog failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("AlertUnifiedAlert");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for AlertUnifiedAlert failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("AlertDebuglog");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
TmThreadSetCPUAffinity(tv, 0);
if (TmThreadSpawn(tv) != 0) {
printf("ERROR: TmThreadSpawn failed\n");
exit(EXIT_FAILURE);
}
tv = TmThreadCreatePacketHandler("Stream3","decode-queue3","simple","packetpool","packetpool","varslot");
if (tv == NULL) {
printf("ERROR: TmThreadsCreate failed for Stream1\n");
exit(EXIT_FAILURE);
}
tm_module = TmModuleGetByName("StreamTcp");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName StreamTcp failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("Detect");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName StreamTcp failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,(void *)de_ctx);
tm_module = TmModuleGetByName("RespondReject");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for RespondReject failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("AlertFastlog");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for AlertFastlog failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("LogHttplog");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("AlertUnifiedLog");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for AlertUnifiedLog failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("AlertUnifiedAlert");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for AlertUnifiedAlert failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("AlertDebuglog");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
TmThreadSetCPUAffinity(tv, 1);
if (TmThreadSpawn(tv) != 0) {
printf("ERROR: TmThreadSpawn failed\n");
exit(EXIT_FAILURE);
}
tv = TmThreadCreatePacketHandler("Stream4","decode-queue4","simple","packetpool","packetpool","varslot");
if (tv == NULL) {
printf("ERROR: TmThreadsCreate failed for Stream1\n");
exit(EXIT_FAILURE);
}
tm_module = TmModuleGetByName("StreamTcp");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName StreamTcp failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("Detect");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName StreamTcp failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,(void *)de_ctx);
tm_module = TmModuleGetByName("RespondReject");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for RespondReject failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("AlertFastlog");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for AlertFastlog failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("LogHttplog");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("AlertUnifiedLog");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for AlertUnifiedLog failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("AlertUnifiedAlert");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for AlertUnifiedAlert failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("AlertDebuglog");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
TmThreadSetCPUAffinity(tv, 1);
if (TmThreadSpawn(tv) != 0) {
printf("ERROR: TmThreadSpawn failed\n");
exit(EXIT_FAILURE);
}
return 0;
}
int RunModeIpsNFQ(DetectEngineCtx *de_ctx) {
TimeModeSetLive();
@ -1256,7 +1570,8 @@ int main(int argc, char **argv)
gettimeofday(&start_time, NULL);
if (mode == MODE_PCAP_DEV) {
RunModeIdsPcap2(de_ctx, pcap_dev);
RunModeIdsPcap3(de_ctx, pcap_dev);
//RunModeIdsPcap2(de_ctx, pcap_dev);
//RunModeIdsPcap(de_ctx, pcap_dev);
}
else if (mode == MODE_PCAP_FILE) {
@ -1337,6 +1652,8 @@ int main(int argc, char **argv)
FlowShutdown();
FlowPrintQueueInfo();
StreamTcpFreeConfig(STREAM_VERBOSE);
HTTPAtExitPrintStats();
/** \todo review whats needed here */
SigGroupCleanup(de_ctx);

@ -98,7 +98,7 @@ Flow *FlowGetFlowFromHash (Packet *p)
/* got one, now lock, initialize and return */
mutex_lock(&f->m);
FlowInit(f,p);
FlowRequeue(f, NULL, &flow_new_q);
FlowRequeue(f, NULL, &flow_new_q[f->protomap]);
f->flags |= FLOW_NEW_LIST;
f->fb = fb;
@ -139,14 +139,14 @@ Flow *FlowGetFlowFromHash (Packet *p)
/* lock, initialize and return */
mutex_lock(&f->m);
FlowInit(f,p);
FlowRequeue(f, NULL, &flow_new_q);
FlowRequeue(f, NULL, &flow_new_q[f->protomap]);
f->flags |= FLOW_NEW_LIST;
f->fb = fb;
mutex_unlock(&fb->m);
return f;
}
}
mutex_lock(&f->m);

@ -9,10 +9,13 @@
/* per flow flags */
#define FLOW_TO_SRC_SEEN 0x01
#define FLOW_TO_DST_SEEN 0x02
#define FLOW_NEW_LIST 0x04
#define FLOW_EST_LIST 0x08
#define FLOW_TOSERVER_IPONLY_SET 0x10
#define FLOW_TOCLIENT_IPONLY_SET 0x20
#define FLOW_CLOSED_LIST 0x10
#define FLOW_TOSERVER_IPONLY_SET 0x20
#define FLOW_TOCLIENT_IPONLY_SET 0x40
/* global flow flags */
#define FLOW_EMERGENCY 0x01
@ -38,16 +41,41 @@
#define FLOW_IPPROTO_ICMP_EMERG_NEW_TIMEOUT 10
#define FLOW_IPPROTO_ICMP_EMERG_EST_TIMEOUT 100
enum {
FLOW_PROTO_DEFAULT = 0,
FLOW_PROTO_TCP,
FLOW_PROTO_UDP,
FLOW_PROTO_ICMP,
/* should be last */
FLOW_PROTO_MAX,
};
/*
* Variables
*/
FlowQueue flow_spare_q; /* Spare flow's. Prealloced flows in here */
FlowQueue flow_new_q; /* Flows in the unreplied state live here */
FlowQueue flow_est_q; /* All other flows live here, the top holds the
* last recently used (lru) flow, so we can remove
* that in case of memory problems and check it for
* timeouts. */
/** FlowProto specific timeouts and free/state functions */
FlowProto flow_proto[FLOW_PROTO_MAX];
/** spare/unused/prealloced flows live here */
FlowQueue flow_spare_q;
/** Flows in the new/unreplied state live here */
FlowQueue flow_new_q[FLOW_PROTO_MAX];
/** All "established" flows live here, the top holds the
* last recently used (lru) flow, so we can remove
* that in case of memory problems and check it for
* timeouts. */
FlowQueue flow_est_q[FLOW_PROTO_MAX];
/** All "closing" flows live here, the top holds the
* last recently used (lru) flow, so we can remove
* that in case of memory problems and check it for
* timeouts. */
FlowQueue flow_close_q[FLOW_PROTO_MAX];
FlowBucket *flow_hash;
FlowConfig flow_config;

@ -49,6 +49,26 @@ void FlowFree(Flow *f)
free(f);
}
/**
* \brief Function to map the protocol to the defined FLOW_PROTO_* enumeration.
*
* \param proto protocol which is needed to be mapped
*/
int FlowGetProtoMapping(uint8_t proto) {
switch (proto) {
case IPPROTO_TCP:
return FLOW_PROTO_TCP;
case IPPROTO_UDP:
return FLOW_PROTO_UDP;
case IPPROTO_ICMP:
return FLOW_PROTO_ICMP;
default:
return FLOW_PROTO_DEFAULT;
}
}
/* initialize the flow from the first packet
* we see from it. */
void FlowInit(Flow *f, Packet *p)
@ -81,5 +101,7 @@ void FlowInit(Flow *f, Packet *p)
}
COPY_TIMESTAMP(&p->ts, &f->startts);
f->protomap = FlowGetProtoMapping(f->proto);
}

@ -3,9 +3,6 @@
#ifndef __FLOW_UTIL_H__
#define __FLOW_UTIL_H__
/** FlowProto specific timeouts and free/state functions */
FlowProto flow_proto[FLOW_PROTO_MAX];
#define COPY_TIMESTAMP(src,dst) ((dst)->tv_sec = (src)->tv_sec, (dst)->tv_usec = (src)->tv_usec)
/* only clear the parts that won't be overwritten
@ -19,12 +16,13 @@ FlowProto flow_proto[FLOW_PROTO_MAX];
(f)->lastts.tv_usec = 0; \
GenericVarFree((f)->flowvar); \
(f)->flowvar = NULL; \
(f)->stream = NULL; \
(f)->protoctx = NULL; \
(f)->use_cnt = 0; \
}
Flow *FlowAlloc(void);
void FlowFree(Flow *);
int FlowGetProtoMapping(uint8_t);
void FlowInit(Flow *, Packet *);
#endif /* __FLOW_UTIL_H__ */

@ -25,7 +25,6 @@
#include "flow-var.h"
#include "flow-private.h"
#include "util-unittest.h"
#include "stream-tcp-private.h"
//#define FLOW_DEFAULT_HASHSIZE 262144
#define FLOW_DEFAULT_HASHSIZE 65536
@ -40,32 +39,51 @@ static int FlowUpdateSpareFlows(void);
int FlowSetProtoTimeout(uint8_t , uint32_t ,uint32_t ,uint32_t);
int FlowSetProtoEmergencyTimeout(uint8_t , uint32_t ,uint32_t ,uint32_t);
static int FlowClearMemory(Flow *,uint8_t );
static int FlowGetProtoMapping(uint8_t);
int FlowSetProtoFreeFunc(uint8_t, void (*Free)(void *));
int FlowSetFlowStateFunc (uint8_t , int (*GetProtoState)(void *));
/** \brief Update the flows position in the queue's
* \param f Flow to requeue.
* \todo if we have a flow state func rely on that soly
*
* In-use flows are either in the flow_new_q or flow_est_q lists.
* In-use flows are in the flow_new_q, flow_est_q lists or flow_close_q lists.
*/
static void FlowUpdateQueue(Flow *f)
void FlowUpdateQueue(Flow *f)
{
if (f->flags & FLOW_NEW_LIST) {
/* in the new list -- we consider a flow no longer
* new if we have seen at least 2 pkts in both ways. */
if (f->todstpktcnt && f->tosrcpktcnt) {
FlowRequeue(f, &flow_new_q, &flow_est_q);
FlowRequeue(f, &flow_new_q[f->protomap], &flow_est_q[f->protomap]);
f->flags |= FLOW_EST_LIST; /* transition */
f->flags &= ~FLOW_NEW_LIST;
} else {
FlowRequeue(f, &flow_new_q, &flow_new_q);
FlowRequeue(f, &flow_new_q[f->protomap], &flow_new_q[f->protomap]);
}
} else if (f->flags & FLOW_EST_LIST) {
if (flow_proto[f->protomap].GetProtoState != NULL) {
uint8_t state = flow_proto[f->protomap].GetProtoState(f->protoctx);
if (state == FLOW_STATE_CLOSED) {
f->flags |= FLOW_CLOSED_LIST; /* transition */
f->flags &= ~FLOW_EST_LIST;
//printf("FlowUpdateQueue %p was put into closing queue ts %"PRIuMAX"\n", f, (uintmax_t)f->lastts.tv_sec);
FlowRequeue(f, &flow_est_q[f->protomap], &flow_close_q[f->protomap]);
} else {
/* Pull and put back -- this way the flows on
* top of the list are least recently used. */
FlowRequeue(f, &flow_est_q[f->protomap], &flow_est_q[f->protomap]);
}
} else {
/* Pull and put back -- this way the flows on
* top of the list are least recently used. */
FlowRequeue(f, &flow_est_q[f->protomap], &flow_est_q[f->protomap]);
}
} else if (f->flags & FLOW_CLOSED_LIST){
/* Pull and put back -- this way the flows on
* top of the list are least recently used. */
FlowRequeue(f, &flow_est_q, &flow_est_q);
FlowRequeue(f, &flow_close_q[f->protomap], &flow_close_q[f->protomap]);
}
}
@ -111,70 +129,69 @@ static int FlowPrune (FlowQueue *q, struct timeval *ts)
/*set the timeout value according to the flow operating mode, flow's state
and protocol.*/
uint32_t timeout = 0;
uint8_t proto_map;
proto_map = FlowGetProtoMapping(f->proto);
if (flow_flags & FLOW_EMERGENCY) {
if (flow_proto[proto_map].GetProtoState != NULL) {
switch(flow_proto[proto_map].GetProtoState(f->stream)) {
if (flow_proto[f->protomap].GetProtoState != NULL) {
switch(flow_proto[f->protomap].GetProtoState(f->protoctx)) {
case FLOW_STATE_NEW:
timeout = flow_proto[proto_map].emerg_new_timeout;
timeout = flow_proto[f->protomap].emerg_new_timeout;
break;
case FLOW_STATE_ESTABLISHED:
timeout = flow_proto[proto_map].emerg_est_timeout;
timeout = flow_proto[f->protomap].emerg_est_timeout;
break;
case FLOW_STATE_CLOSED:
timeout = flow_proto[proto_map].emerg_closed_timeout;
timeout = flow_proto[f->protomap].emerg_closed_timeout;
break;
}
} else {
if (f->flags & FLOW_EST_LIST)
timeout = flow_proto[proto_map].emerg_est_timeout;
timeout = flow_proto[f->protomap].emerg_est_timeout;
else
timeout = flow_proto[proto_map].emerg_new_timeout;
timeout = flow_proto[f->protomap].emerg_new_timeout;
}
} else {
if (flow_proto[proto_map].GetProtoState != NULL) {
switch(flow_proto[proto_map].GetProtoState(f->stream)) {
} else { /* impliet not emergency */
if (flow_proto[f->protomap].GetProtoState != NULL) {
switch(flow_proto[f->protomap].GetProtoState(f->protoctx)) {
case FLOW_STATE_NEW:
timeout = flow_proto[proto_map].new_timeout;
timeout = flow_proto[f->protomap].new_timeout;
break;
case FLOW_STATE_ESTABLISHED:
timeout = flow_proto[proto_map].est_timeout;
timeout = flow_proto[f->protomap].est_timeout;
break;
case FLOW_STATE_CLOSED:
timeout = flow_proto[proto_map].closed_timeout;
timeout = flow_proto[f->protomap].closed_timeout;
break;
}
} else {
if (f->flags & FLOW_EST_LIST)
timeout = flow_proto[proto_map].est_timeout;
timeout = flow_proto[f->protomap].est_timeout;
else
timeout = flow_proto[proto_map].new_timeout;
timeout = flow_proto[f->protomap].new_timeout;
}
}
DEBUGPRINT("got lock, now check: %" PRId64 "+%" PRIu32 "=(%" PRId64 ") < %" PRId64 "", f->lastts.tv_sec,
timeout, f->lastts.tv_sec + timeout, ts->tv_sec);
/** never prune a flow that is used by a packet or stream msg
* we are currently processing in one of the threads */
if (f->use_cnt > 0) {
/* do the timeout check */
if ((f->lastts.tv_sec + timeout) >= ts->tv_sec) {
mutex_unlock(&f->fb->m);
mutex_unlock(&f->m);
return 0;
}
/* do the timeout check */
if ((f->lastts.tv_sec + timeout) >= ts->tv_sec) {
/** never prune a flow that is used by a packet or stream msg
* we are currently processing in one of the threads */
if (f->use_cnt > 0) {
printf("FlowPrune: timed out but use_cnt > 0: %"PRIu16", %p, proto %"PRIu8"\n", f->use_cnt, f, f->proto);
mutex_unlock(&f->fb->m);
mutex_unlock(&f->m);
return 0;
}
//printf("timed out %" PRIuMAX "+%" PRIu32 "=(%" PRIuMAX ") < %" PRIuMAX ": %p, proto %"PRIu8"\n", (uintmax_t)f->lastts.tv_sec,
// timeout, (uintmax_t)(f->lastts.tv_sec + timeout), (uintmax_t)ts->tv_sec, f, f->proto);
/* remove from the hash */
if (f->hprev)
f->hprev->hnext = f->hnext;
@ -189,7 +206,7 @@ static int FlowPrune (FlowQueue *q, struct timeval *ts)
mutex_unlock(&f->fb->m);
f->fb = NULL;
FlowClearMemory (f, proto_map);
FlowClearMemory (f, f->protomap);
/* move to spare list */
FlowRequeue(f, q, &flow_spare_q);
@ -396,20 +413,26 @@ void FlowInitConfig (char quiet)
* \warning Not thread safe */
void FlowPrintQueueInfo (void)
{
int i;
printf("* Flow Queue info:\n");
printf(" - SPARE %" PRIu32 " (", flow_spare_q.len);
#ifdef DBG_PERF
printf("flow_spare_q.dbg_maxlen %" PRIu32 ")\n", flow_spare_q.dbg_maxlen);
#endif
printf(" - NEW %" PRIu32 " (", flow_new_q.len);
for (i = 0; i < FLOW_PROTO_MAX; i++) {
printf(" - NEW %" PRIu32 " (", flow_new_q[i].len);
#ifdef DBG_PERF
printf("flow_new_q.dbg_maxlen %" PRIu32 ")\n", flow_new_q.dbg_maxlen);
printf("flow_new_q.dbg_maxlen %" PRIu32 ")\n", flow_new_q[i].dbg_maxlen);
#endif
printf(" - ESTABLISHED %" PRIu32 " (", flow_est_q.len);
printf(" - ESTABLISHED %" PRIu32 " (", flow_est_q[i].len);
#ifdef DBG_PERF
printf("flow_est_q.dbg_maxlen %" PRIu32 ")\n", flow_est_q.dbg_maxlen);
printf("flow_est_q.dbg_maxlen %" PRIu32 ")\n", flow_est_q[i].dbg_maxlen);
#endif
printf(" - CLOSING %" PRIu32 " (", flow_close_q[i].len);
#ifdef DBG_PERF
printf("flow_closing_q.dbg_maxlen %" PRIu32 ")\n", flow_close_q[i].dbg_maxlen);
#endif
}
#ifdef FLOWBITS_STATS
printf("* Flowbits added: %" PRIu32 ", removed: %" PRIu32 ", ", flowbits_added, flowbits_removed);
printf("max memory usage: %" PRIu32 "\n", flowbits_memuse_max);
@ -420,19 +443,27 @@ void FlowPrintQueueInfo (void)
* \warning Not thread safe */
void FlowShutdown(void) {
Flow *f;
int i;
while((f = FlowDequeue(&flow_spare_q))) {
FlowFree(f);
}
while((f = FlowDequeue(&flow_new_q))) {
uint8_t proto_map = FlowGetProtoMapping(f->proto);
FlowClearMemory(f, proto_map);
FlowFree(f);
}
while((f = FlowDequeue(&flow_est_q))) {
uint8_t proto_map = FlowGetProtoMapping(f->proto);
FlowClearMemory(f, proto_map);
FlowFree(f);
for (i = 0; i < FLOW_PROTO_MAX; i++) {
while((f = FlowDequeue(&flow_new_q[i]))) {
uint8_t proto_map = FlowGetProtoMapping(f->proto);
FlowClearMemory(f, proto_map);
FlowFree(f);
}
while((f = FlowDequeue(&flow_est_q[i]))) {
uint8_t proto_map = FlowGetProtoMapping(f->proto);
FlowClearMemory(f, proto_map);
FlowFree(f);
}
while((f = FlowDequeue(&flow_close_q[i]))) {
uint8_t proto_map = FlowGetProtoMapping(f->proto);
FlowClearMemory(f, proto_map);
FlowFree(f);
}
}
free(flow_hash);
@ -458,7 +489,7 @@ void *FlowManagerThread(void *td)
{
ThreadVars *th_v = (ThreadVars *)td;
struct timeval ts;
uint32_t established_cnt = 0, new_cnt = 0, nowcnt;
uint32_t established_cnt = 0, new_cnt = 0, closing_cnt = 0, nowcnt;
uint32_t sleeping = 0;
uint8_t emerg = FALSE;
@ -484,23 +515,30 @@ void *FlowManagerThread(void *td)
DEBUGPRINT("ts %" PRId64 "", ts.tv_sec);
/* see if we still have enough spare flows */
if (!(FlowUpdateSpareFlows()) && emerg == TRUE) {
/*timeout_new = flow_config.emerg_timeout_new;
timeout_est = flow_config.emerg_timeout_est;*/
}
/* prune new list */
nowcnt = FlowPruneFlows(&flow_new_q, &ts);
if (nowcnt) {
DEBUGPRINT("Pruned %" PRIu32 " new flows...\n", nowcnt);
new_cnt += nowcnt;
}
/* prune established list */
nowcnt = FlowPruneFlows(&flow_est_q, &ts);
if (nowcnt) {
DEBUGPRINT("Pruned %" PRIu32 " established flows...\n", nowcnt);
established_cnt += nowcnt;
FlowUpdateSpareFlows();
int i;
for (i = 0; i < FLOW_PROTO_MAX; i++) {
/* prune closing list */
nowcnt = FlowPruneFlows(&flow_close_q[i], &ts);
if (nowcnt) {
DEBUGPRINT("Pruned %" PRIu32 " closing flows...\n", nowcnt);
closing_cnt += nowcnt;
}
/* prune new list */
nowcnt = FlowPruneFlows(&flow_new_q[i], &ts);
if (nowcnt) {
DEBUGPRINT("Pruned %" PRIu32 " new flows...\n", nowcnt);
new_cnt += nowcnt;
}
/* prune established list */
nowcnt = FlowPruneFlows(&flow_est_q[i], &ts);
if (nowcnt) {
DEBUGPRINT("Pruned %" PRIu32 " established flows...\n", nowcnt);
established_cnt += nowcnt;
}
}
sleeping = 0;
@ -589,26 +627,6 @@ void FlowInitFlowProto(void) {
flow_proto[FLOW_PROTO_ICMP].GetProtoState = NULL;
}
/**
* \brief Function to map the protocol to the defined FLOW_PROTO_* enumeration.
*
* \param proto protocol which is needed to be mapped
*/
static int FlowGetProtoMapping(uint8_t proto) {
switch (proto) {
case IPPROTO_TCP:
return FLOW_PROTO_TCP;
case IPPROTO_UDP:
return FLOW_PROTO_UDP;
case IPPROTO_ICMP:
return FLOW_PROTO_ICMP;
default:
return FLOW_PROTO_DEFAULT;
}
}
/**
* \brief Function clear the flow memory before queueing it to spare flow
* queue.
@ -620,11 +638,10 @@ static int FlowGetProtoMapping(uint8_t proto) {
static int FlowClearMemory(Flow* f, uint8_t proto_map) {
/* call the protocol specific free function if we have one */
if (flow_proto[proto_map].Freefunc != NULL) {
flow_proto[proto_map].Freefunc(f->stream);
flow_proto[proto_map].Freefunc(f->protoctx);
}
f->stream = NULL;
f->protoctx = NULL;
//memset(f, 0, sizeof(Flow));
CLEAR_FLOW(f);
return 1;
}
@ -705,6 +722,9 @@ int FlowSetProtoEmergencyTimeout(uint8_t proto, uint32_t emerg_new_timeout, uint
return 1;
}
#ifdef UNITTESTS
#include "stream-tcp-private.h"
/**
* \test Test the setting of the per protocol timeouts.
*
@ -813,7 +833,7 @@ static int FlowTestPrune(Flow *f, struct timeval *ts) {
return 0;
}
if (f->stream != NULL){
if (f->protoctx != NULL){
printf("Failed in freeing the TcpSession\n");
return 0;
}
@ -841,7 +861,7 @@ static int FlowTest03 (void) {
TimeGet(&ts);
f.lastts.tv_sec = ts.tv_sec - 5000;
f.stream = &ssn;
f.protoctx = &ssn;
f.fb = &fb;
f.proto = IPPROTO_TCP;
@ -885,7 +905,7 @@ static int FlowTest04 (void) {
ssn.server = client;
ssn.state = TCP_ESTABLISHED;
f.lastts.tv_sec = ts.tv_sec - 5000;
f.stream = &ssn;
f.protoctx = &ssn;
f.fb = &fb;
f.proto = IPPROTO_TCP;
@ -918,7 +938,7 @@ static int FlowTest05 (void) {
TimeGet(&ts);
ssn.state = TCP_SYN_SENT;
f.lastts.tv_sec = ts.tv_sec - 300;
f.stream = &ssn;
f.protoctx = &ssn;
f.fb = &fb;
f.proto = IPPROTO_TCP;
f.flags = FLOW_EMERGENCY;
@ -963,7 +983,7 @@ static int FlowTest06 (void) {
ssn.server = client;
ssn.state = TCP_ESTABLISHED;
f.lastts.tv_sec = ts.tv_sec - 5000;
f.stream = &ssn;
f.protoctx = &ssn;
f.fb = &fb;
f.proto = IPPROTO_TCP;
f.flags = FLOW_EMERGENCY;
@ -974,15 +994,18 @@ static int FlowTest06 (void) {
return 1;
}
#endif /* UNITTESTS */
/**
* \brief Function to register the Flow Unitests.
*/
void FlowRegisterTests (void) {
#ifdef UNITTESTS
UtRegisterTest("FlowTest01 -- Protocol Specific Timeouts", FlowTest01, 1);
UtRegisterTest("FlowTest02 -- Setting Protocol Specific Free Function", FlowTest02, 1);
UtRegisterTest("FlowTest03 -- Timeout a flow having fresh TcpSession", FlowTest03, 1);
UtRegisterTest("FlowTest04 -- Timeout a flow having TcpSession with segments", FlowTest04, 1);
UtRegisterTest("FlowTest05 -- Timeout a flow in emergency having fresh TcpSession", FlowTest05, 1);
UtRegisterTest("FlowTest06 -- Timeout a flow in emergency having TcpSession with segments", FlowTest06, 1);
#endif /* UNITTESTS */
}

@ -67,9 +67,15 @@ typedef struct Flow_
uint32_t tosrcpktcnt;
uint64_t bytecnt;
void *stream;
uint16_t use_cnt; /** how many pkts and stream msgs are
using the flow *right now* */
/** mapping to Flow's protocol specific protocols for timeouts
and state and free functions. */
uint8_t protomap;
/** protocol specific data pointer, e.g. for TcpSession */
void *protoctx;
/** how many pkts and stream msgs are using the flow *right now* */
uint16_t use_cnt;
pthread_mutex_t m;
@ -84,16 +90,6 @@ typedef struct Flow_
struct FlowBucket_ *fb;
} Flow;
enum {
FLOW_PROTO_DEFAULT = 0,
FLOW_PROTO_TCP,
FLOW_PROTO_UDP,
FLOW_PROTO_ICMP,
/* should be last */
FLOW_PROTO_MAX,
};
enum {
FLOW_STATE_NEW = 0,
FLOW_STATE_ESTABLISHED,
@ -126,6 +122,7 @@ int FlowSetProtoTimeout(uint8_t ,uint32_t ,uint32_t ,uint32_t);
int FlowSetProtoEmergencyTimeout(uint8_t ,uint32_t ,uint32_t ,uint32_t);
int FlowSetProtoFreeFunc (uint8_t , void (*Free)(void *));
int FlowSetFlowStateFunc (uint8_t , int (*GetProtoState)(void *));
void FlowUpdateQueue(Flow *);
#endif /* __FLOW_H__ */

@ -11,33 +11,34 @@ typedef struct TcpSegment_ {
} TcpSegment;
typedef struct TcpStream_ {
uint32_t isn; /* initial sequence number */
uint32_t next_seq; /* next expected sequence number */
uint32_t last_ack; /* last ack'd sequence number */
uint32_t next_win; /* next max seq within window */
uint8_t wscale;
uint16_t window;
uint32_t isn; /**< initial sequence number */
uint32_t next_seq; /**< next expected sequence number */
uint32_t last_ack; /**< last ack'd sequence number in this stream */
uint32_t next_win; /**< next max seq within window */
uint32_t window; /**< current window setting */
uint8_t wscale; /**< wscale setting in this direction */
/* reassembly */
uint32_t ra_base_seq; /* reassembled seq. We've reassembled up to this point. */
TcpSegment *seg_list;
uint8_t os_policy; /* target based OS policy used for reassembly and handling packets*/
uint32_t ra_base_seq; /**< reassembled seq. We've reassembled up to this point. */
TcpSegment *seg_list; /**< list of TCP segments that are not yet (fully) used in reassembly */
uint8_t os_policy; /**< target based OS policy used for reassembly and handling packets*/
} TcpStream;
/* from /usr/include/netinet/tcp.h */
enum
{
TCP_ESTABLISHED = 1,
TCP_NONE,
TCP_LISTEN,
TCP_SYN_SENT,
TCP_SYN_RECV,
TCP_ESTABLISHED,
TCP_FIN_WAIT1,
TCP_FIN_WAIT2,
TCP_TIME_WAIT,
TCP_CLOSED,
TCP_CLOSE_WAIT,
TCP_LAST_ACK,
TCP_LISTEN,
TCP_CLOSING /* now a valid state */
TCP_CLOSE_WAIT,
TCP_CLOSING,
TCP_CLOSED,
};
#define STREAMTCP_FLAG_MIDSTREAM 0x01 /*Flag for mid stream session*/
@ -53,9 +54,10 @@ enum
typedef struct TcpSession_ {
uint8_t state;
uint8_t flags;
uint16_t alproto; /**< application level protocol */
TcpStream server;
TcpStream client;
void **l7data;
u_int8_t flags;
void **aldata; /**< application level storage ptrs */
} TcpSession;
#endif /* __STREAM_TCP_PRIVATE_H__ */

File diff suppressed because it is too large Load Diff

@ -31,6 +31,7 @@ enum
int StreamTcpReassembleHandleSegment(TcpSession *, TcpStream *, Packet *);
int StreamTcpReassembleInit(void);
void StreamTcpReassembleFree(void);
void StreamTcpReassembleRegisterTests(void);
void StreamTcpCreateTestPacket(u_int8_t *, u_int8_t, u_int8_t);

File diff suppressed because it is too large Load Diff

@ -16,6 +16,7 @@ typedef struct TcpStreamCnf_ {
TcpStreamCnf stream_config;
void TmModuleStreamTcpRegister (void);
void StreamTcpInitConfig (char);
void StreamTcpFreeConfig(char);
void StreamTcpRegisterTests (void);
#endif /* __STREAM_TCP_H__ */

@ -8,6 +8,10 @@
#include "util-pool.h"
static pthread_mutex_t stream_pool_memuse_mutex;
static uint64_t stream_pool_memuse = 0;
static uint64_t stream_pool_memcnt = 0;
static StreamMsgQueue stream_q;
/* per queue setting */
@ -25,6 +29,11 @@ void *StreamMsgAlloc(void *null) {
return NULL;
memset(s, 0, sizeof(StreamMsg));
mutex_lock(&stream_pool_memuse_mutex);
stream_pool_memuse += sizeof(StreamMsg);
stream_pool_memcnt ++;
mutex_unlock(&stream_pool_memuse_mutex);
return s;
}
@ -140,6 +149,14 @@ void StreamMsgQueuesInit(void) {
stream_msg_pool = PoolInit(5000,250,StreamMsgAlloc,NULL,StreamMsgFree);
if (stream_msg_pool == NULL)
exit(1); /* XXX */
pthread_mutex_init(&stream_pool_memuse_mutex, NULL);
}
void StreamMsgQueuesDeinit(void) {
PoolFree(stream_msg_pool);
printf("StreamMsgQueuesDeinit: stream_pool_memuse %"PRIu64", stream_pool_memcnt %"PRIu64"\n", stream_pool_memuse, stream_pool_memcnt);
}
StreamMsgQueue *StreamMsgQueueGetByPort(uint16_t port) {

@ -49,6 +49,7 @@ typedef struct StreamMsgQueue_ {
/* prototypes */
void StreamMsgQueuesInit(void);
void StreamMsgQueuesDeinit(void);
StreamMsg *StreamMsgGetFromPool(void);
void StreamMsgReturnToPool(StreamMsg *);

@ -27,6 +27,7 @@
typedef struct TmqhFlowCtx_ {
uint16_t size;
uint16_t *queues;
uint16_t last;
} TmqhFlowCtx;
Packet *TmqhInputFlow(ThreadVars *t);
@ -167,7 +168,12 @@ void TmqhOutputFlow(ThreadVars *tv, Packet *p)
uint16_t idx = addr % ctx->size;
qid = ctx->queues[idx];
} else {
qid = ctx->queues[0];
ctx->last++;
if (ctx->last == ctx->size)
ctx->last = 0;
qid = ctx->queues[ctx->last];
}
PacketQueue *q = &trans_q[qid];

@ -113,6 +113,11 @@ void *PoolGet(Pool *p) {
} else {
if (p->allocated < p->max_buckets) {
p->allocated++;
p->outstanding++;
if (p->outstanding > p->max_outstanding)
p->max_outstanding = p->outstanding;
return p->Alloc(p->AllocData);
} else {
return NULL;
@ -121,6 +126,9 @@ void *PoolGet(Pool *p) {
void *ptr = pb->data;
pb->data = NULL;
p->outstanding++;
if (p->outstanding > p->max_outstanding)
p->max_outstanding = p->outstanding;
return ptr;
}
@ -141,9 +149,14 @@ void PoolReturn(Pool *p, void *data) {
p->alloc_list_size++;
pb->data = data;
p->outstanding--;
return;
}
void PoolPrintSaturation(Pool *p) {
printf("PoolPrintSaturation: Pool %p is using %"PRIu32" out of %"PRIu32" items (%02.1f%%), max %"PRIu32" (%02.1f%%): pool struct memory %"PRIu64".\n", p, p->outstanding, p->max_buckets, (float)(p->outstanding/(float)(p->max_buckets))*100, p->max_outstanding, (float)(p->max_outstanding/(float)(p->max_buckets))*100, (uint64_t)(p->max_buckets * sizeof(PoolBucket)));
}
/*
* ONLY TESTS BELOW THIS COMMENT
*/

@ -23,12 +23,16 @@ typedef struct Pool_ {
void *(*Alloc)(void *);
void *AllocData;
void (*Free)(void *);
uint32_t outstanding;
uint32_t max_outstanding;
} Pool;
/* prototypes */
Pool* PoolInit(uint32_t, uint32_t, void *(*Alloc)(void *), void *, void (*Free)(void *));
void PoolFree(Pool *);
void PoolPrint(Pool *);
void PoolPrintSaturation(Pool *p);
void *PoolGet(Pool *);
void PoolReturn(Pool *, void *);

@ -4,6 +4,8 @@
#include "detect.h"
#include "threads.h"
//#define DEBUG
static struct timeval current_time = { 0,0 };
static pthread_mutex_t current_time_mutex = PTHREAD_MUTEX_INITIALIZER;
static char live = TRUE;
@ -24,10 +26,13 @@ void TimeSet(struct timeval *tv) {
return;
mutex_lock(&current_time_mutex);
current_time.tv_sec = tv->tv_sec;
current_time.tv_sec = tv->tv_sec;
current_time.tv_usec = tv->tv_usec;
//printf("TimeSet: time set to %" PRIu64 " sec, %" PRIu64 " usec\n",
// current_time.tv_sec, current_time.tv_usec);
#ifdef DEBUG
printf("TimeSet: time set to %" PRIuMAX " sec, %" PRIuMAX " usec\n",
(uintmax_t)current_time.tv_sec, (uintmax_t)current_time.tv_usec);
#endif
mutex_unlock(&current_time_mutex);
}
@ -40,12 +45,14 @@ void TimeGet(struct timeval *tv) {
gettimeofday(tv, NULL);
} else {
mutex_lock(&current_time_mutex);
tv->tv_sec = current_time.tv_sec;
tv->tv_sec = current_time.tv_sec;
tv->tv_usec = current_time.tv_usec;
mutex_unlock(&current_time_mutex);
}
//printf("TimeGet: time we got is %" PRIu64 " sec, %" PRIu64 " usec\n",
// tv->tv_sec, tv->tv_usec);
#ifdef DEBUG
printf("TimeGet: time we got is %" PRIuMAX " sec, %" PRIuMAX " usec\n",
(uintmax_t)tv->tv_sec, (uintmax_t)tv->tv_usec);
#endif
}

Loading…
Cancel
Save