From a4a4d17ad02ce23d6b79bd5c3372d367287f6dd3 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Tue, 26 Feb 2019 10:56:53 +0100 Subject: [PATCH] app-layer/files: optimize GetFiles calls Remove FlowGetProtoMapping calls from the GetFiles wrapper and get the alstate from the flow directly. --- src/app-layer-parser.c | 31 ++++++++++++++++++++++--------- src/app-layer-parser.h | 3 +-- src/detect-engine-file.c | 10 ++++------ src/detect-engine-state.c | 33 +++++++++++---------------------- src/detect-file-data.c | 6 ++---- src/detect-filemagic.c | 6 ++---- src/detect-filename.c | 6 ++---- src/detect-filestore.c | 3 +-- src/flow-worker.c | 4 ++-- src/output-file.c | 6 ++---- src/output-filedata.c | 6 ++---- src/util-file.c | 14 +++++++------- 12 files changed, 58 insertions(+), 70 deletions(-) diff --git a/src/app-layer-parser.c b/src/app-layer-parser.c index 00b16a29e0..da54b6fddd 100644 --- a/src/app-layer-parser.c +++ b/src/app-layer-parser.c @@ -875,18 +875,16 @@ AppLayerDecoderEvents *AppLayerParserGetEventsByTx(uint8_t ipproto, AppProto alp SCReturnPtr(ptr, "AppLayerDecoderEvents *"); } -FileContainer *AppLayerParserGetFiles(uint8_t ipproto, AppProto alproto, - void *alstate, uint8_t direction) +FileContainer *AppLayerParserGetFiles(const Flow *f, const uint8_t direction) { SCEnter(); FileContainer *ptr = NULL; - if (alp_ctx.ctxs[FlowGetProtoMapping(ipproto)][alproto]. - StateGetFiles != NULL) + if (alp_ctx.ctxs[f->protomap][f->alproto].StateGetFiles != NULL) { - ptr = alp_ctx.ctxs[FlowGetProtoMapping(ipproto)][alproto]. - StateGetFiles(alstate, direction); + ptr = alp_ctx.ctxs[f->protomap][f->alproto]. + StateGetFiles(f->alstate, direction); } SCReturnPtr(ptr, "FileContainer *"); @@ -2020,12 +2018,23 @@ static void TestProtocolStateFree(void *s) SCFree(s); } -static uint64_t TestProtocolGetTxCnt(void *state) +static uint64_t TestGetTxCnt(void *state) { /* single tx */ return 1; } +static void TestStateTransactionFree(void *state, uint64_t tx_id) +{ + /* do nothing */ +} + +static void *TestGetTx(void *state, uint64_t tx_id) +{ + TestState *test_state = (TestState *)state; + return test_state; +} + void AppLayerParserRegisterProtocolUnittests(uint8_t ipproto, AppProto alproto, void (*RegisterUnittests)(void)) { @@ -2073,7 +2082,9 @@ static int AppLayerParserTest01(void) TestProtocolParser); AppLayerParserRegisterStateFuncs(IPPROTO_TCP, ALPROTO_TEST, TestProtocolStateAlloc, TestProtocolStateFree); - AppLayerParserRegisterGetTxCnt(IPPROTO_TCP, ALPROTO_TEST, TestProtocolGetTxCnt); + AppLayerParserRegisterTxFreeFunc(IPPROTO_TCP, ALPROTO_TEST, TestStateTransactionFree); + AppLayerParserRegisterGetTx(IPPROTO_TCP, ALPROTO_TEST, TestGetTx); + AppLayerParserRegisterGetTxCnt(IPPROTO_TCP, ALPROTO_TEST, TestGetTxCnt); f = UTHBuildFlow(AF_INET, "1.2.3.4", "4.3.2.1", 20, 40); if (f == NULL) @@ -2128,7 +2139,9 @@ static int AppLayerParserTest02(void) TestProtocolParser); AppLayerParserRegisterStateFuncs(IPPROTO_UDP, ALPROTO_TEST, TestProtocolStateAlloc, TestProtocolStateFree); - AppLayerParserRegisterGetTxCnt(IPPROTO_UDP, ALPROTO_TEST, TestProtocolGetTxCnt); + AppLayerParserRegisterTxFreeFunc(IPPROTO_UDP, ALPROTO_TEST, TestStateTransactionFree); + AppLayerParserRegisterGetTx(IPPROTO_UDP, ALPROTO_TEST, TestGetTx); + AppLayerParserRegisterGetTxCnt(IPPROTO_UDP, ALPROTO_TEST, TestGetTxCnt); f = UTHBuildFlow(AF_INET, "1.2.3.4", "4.3.2.1", 20, 40); if (f == NULL) diff --git a/src/app-layer-parser.h b/src/app-layer-parser.h index 0b08768309..fc367d2ff3 100644 --- a/src/app-layer-parser.h +++ b/src/app-layer-parser.h @@ -198,8 +198,7 @@ void AppLayerParserSetTransactionInspectId(const Flow *f, AppLayerParserState *p AppLayerDecoderEvents *AppLayerParserGetDecoderEvents(AppLayerParserState *pstate); void AppLayerParserSetDecoderEvents(AppLayerParserState *pstate, AppLayerDecoderEvents *devents); AppLayerDecoderEvents *AppLayerParserGetEventsByTx(uint8_t ipproto, AppProto alproto, void *tx); -FileContainer *AppLayerParserGetFiles(uint8_t ipproto, AppProto alproto, - void *alstate, uint8_t direction); +FileContainer *AppLayerParserGetFiles(const Flow *f, const uint8_t direction); int AppLayerParserGetStateProgress(uint8_t ipproto, AppProto alproto, void *alstate, uint8_t direction); uint64_t AppLayerParserGetTxCnt(const Flow *, void *alstate); diff --git a/src/detect-engine-file.c b/src/detect-engine-file.c index a6f8cfa058..4e9e9de021 100644 --- a/src/detect-engine-file.c +++ b/src/detect-engine-file.c @@ -49,6 +49,7 @@ #include "util-unittest.h" #include "util-unittest-helper.h" #include "util-profiling.h" +#include "util-validate.h" /** @@ -218,16 +219,13 @@ static int DetectFileInspect(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, int DetectFileInspectGeneric(ThreadVars *tv, DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, const Signature *s, const SigMatchData *smd, - Flow *f, uint8_t flags, void *alstate, void *tx, uint64_t tx_id) + Flow *f, uint8_t flags, void *_alstate, void *tx, uint64_t tx_id) { SCEnter(); - - if (alstate == NULL) { - SCReturnInt(DETECT_ENGINE_INSPECT_SIG_NO_MATCH); - } + DEBUG_VALIDATE_BUG_ON(f->alstate != _alstate); const uint8_t direction = flags & (STREAM_TOSERVER|STREAM_TOCLIENT); - FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto, alstate, direction); + FileContainer *ffc = AppLayerParserGetFiles(f, direction); if (ffc == NULL || ffc->head == NULL) { SCReturnInt(DETECT_ENGINE_INSPECT_SIG_NO_MATCH); } diff --git a/src/detect-engine-state.c b/src/detect-engine-state.c index 2f2c2b4c41..008f9c861e 100644 --- a/src/detect-engine-state.c +++ b/src/detect-engine-state.c @@ -663,8 +663,7 @@ static int DeStateSigTest03(void) SigMatchSignatures(&th_v, de_ctx, det_ctx, p); FAIL_IF(!(PacketAlertCheck(p, 1))); - FileContainer *files = AppLayerParserGetFiles(p->flow->proto, p->flow->alproto, - p->flow->alstate, STREAM_TOSERVER); + FileContainer *files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER); FAIL_IF_NULL(files); File *file = files->head; @@ -741,8 +740,7 @@ static int DeStateSigTest04(void) FAIL_IF_NULL(http_state); FAIL_IF_NULL(http_state->files_ts); - FileContainer *files = AppLayerParserGetFiles(p->flow->proto, p->flow->alproto, - p->flow->alstate, STREAM_TOSERVER); + FileContainer *files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER); FAIL_IF_NULL(files); File *file = files->head; FAIL_IF_NULL(file); @@ -818,8 +816,7 @@ static int DeStateSigTest05(void) FAIL_IF_NULL(http_state); FAIL_IF_NULL(http_state->files_ts); - FileContainer *files = AppLayerParserGetFiles(p->flow->proto, p->flow->alproto, - p->flow->alstate, STREAM_TOSERVER); + FileContainer *files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER); FAIL_IF_NULL(files); File *file = files->head; FAIL_IF_NULL(file); @@ -896,8 +893,7 @@ static int DeStateSigTest06(void) FAIL_IF_NULL(http_state); FAIL_IF_NULL(http_state->files_ts); - FileContainer *files = AppLayerParserGetFiles(p->flow->proto, p->flow->alproto, - p->flow->alstate, STREAM_TOSERVER); + FileContainer *files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER); FAIL_IF_NULL(files); File *file = files->head; FAIL_IF_NULL(file); @@ -979,8 +975,7 @@ static int DeStateSigTest07(void) FAIL_IF_NULL(http_state); FAIL_IF_NULL(http_state->files_ts); - FileContainer *files = AppLayerParserGetFiles(p->flow->proto, p->flow->alproto, - p->flow->alstate, STREAM_TOSERVER); + FileContainer *files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER); FAIL_IF_NULL(files); File *file = files->head; FAIL_IF_NULL(file); @@ -1077,8 +1072,7 @@ static int DeStateSigTest08(void) FAIL_IF_NULL(http_state); FAIL_IF_NULL(http_state->files_ts); - FileContainer *files = AppLayerParserGetFiles(p->flow->proto, p->flow->alproto, - p->flow->alstate, STREAM_TOSERVER); + FileContainer *files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER); FAIL_IF_NULL(files); File *file = files->head; FAIL_IF_NULL(file); @@ -1102,8 +1096,7 @@ static int DeStateSigTest08(void) FAIL_IF_NULL(http_state); FAIL_IF_NULL(http_state->files_ts); - files = AppLayerParserGetFiles(p->flow->proto, p->flow->alproto, - p->flow->alstate, STREAM_TOSERVER); + files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER); FAIL_IF_NULL(files); file = files->head; FAIL_IF_NULL(file); @@ -1202,8 +1195,7 @@ static int DeStateSigTest09(void) FAIL_IF_NULL(http_state); FAIL_IF_NULL(http_state->files_ts); - FileContainer *files = AppLayerParserGetFiles(p->flow->proto, p->flow->alproto, - p->flow->alstate, STREAM_TOSERVER); + FileContainer *files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER); FAIL_IF_NULL(files); File *file = files->head; FAIL_IF_NULL(file); @@ -1227,8 +1219,7 @@ static int DeStateSigTest09(void) FAIL_IF_NULL(http_state); FAIL_IF_NULL(http_state->files_ts); - files = AppLayerParserGetFiles(p->flow->proto, p->flow->alproto, - p->flow->alstate, STREAM_TOSERVER); + files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER); FAIL_IF_NULL(files); file = files->head; FAIL_IF_NULL(file); @@ -1325,8 +1316,7 @@ static int DeStateSigTest10(void) FAIL_IF_NULL(http_state); FAIL_IF_NULL(http_state->files_ts); - FileContainer *files = AppLayerParserGetFiles(p->flow->proto, p->flow->alproto, - p->flow->alstate, STREAM_TOSERVER); + FileContainer *files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER); FAIL_IF_NULL(files); File *file = files->head; FAIL_IF_NULL(file); @@ -1350,8 +1340,7 @@ static int DeStateSigTest10(void) FAIL_IF_NULL(http_state); FAIL_IF_NULL(http_state->files_ts); - files = AppLayerParserGetFiles(p->flow->proto, p->flow->alproto, - p->flow->alstate, STREAM_TOSERVER); + files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER); FAIL_IF_NULL(files); file = files->head; FAIL_IF_NULL(file); diff --git a/src/detect-file-data.c b/src/detect-file-data.c index 6558668f5d..0479315728 100644 --- a/src/detect-file-data.c +++ b/src/detect-file-data.c @@ -415,8 +415,7 @@ static int DetectEngineInspectFiledata( transforms = engine->v2.transforms; } - FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto, - f->alstate, flags); + FileContainer *ffc = AppLayerParserGetFiles(f, flags); if (ffc == NULL) { return DETECT_ENGINE_INSPECT_SIG_NO_MATCH; } @@ -484,8 +483,7 @@ static void PrefilterTxFiledata(DetectEngineThreadCtx *det_ctx, const MpmCtx *mpm_ctx = ctx->mpm_ctx; const int list_id = ctx->list_id; - FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto, - f->alstate, flags); + FileContainer *ffc = AppLayerParserGetFiles(f, flags); int local_file_id = 0; if (ffc != NULL) { File *file = ffc->head; diff --git a/src/detect-filemagic.c b/src/detect-filemagic.c index 0b3b994320..003082e79a 100644 --- a/src/detect-filemagic.c +++ b/src/detect-filemagic.c @@ -535,8 +535,7 @@ static int DetectEngineInspectFilemagic( transforms = engine->v2.transforms; } - FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto, - f->alstate, flags); + FileContainer *ffc = AppLayerParserGetFiles(f, flags); if (ffc == NULL) { return DETECT_ENGINE_INSPECT_SIG_NO_MATCH; } @@ -596,8 +595,7 @@ static void PrefilterTxFilemagic(DetectEngineThreadCtx *det_ctx, const MpmCtx *mpm_ctx = ctx->mpm_ctx; const int list_id = ctx->list_id; - FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto, - f->alstate, flags); + FileContainer *ffc = AppLayerParserGetFiles(f, flags); int local_file_id = 0; if (ffc != NULL) { File *file = ffc->head; diff --git a/src/detect-filename.c b/src/detect-filename.c index bbfc3abaaa..a900968044 100644 --- a/src/detect-filename.c +++ b/src/detect-filename.c @@ -381,8 +381,7 @@ static int DetectEngineInspectFilename( transforms = engine->v2.transforms; } - FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto, - f->alstate, flags); + FileContainer *ffc = AppLayerParserGetFiles(f, flags); if (ffc == NULL) { return DETECT_ENGINE_INSPECT_SIG_NO_MATCH; } @@ -442,8 +441,7 @@ static void PrefilterTxFilename(DetectEngineThreadCtx *det_ctx, const MpmCtx *mpm_ctx = ctx->mpm_ctx; const int list_id = ctx->list_id; - FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto, - f->alstate, flags); + FileContainer *ffc = AppLayerParserGetFiles(f, flags); int local_file_id = 0; if (ffc != NULL) { File *file = ffc->head; diff --git a/src/detect-filestore.c b/src/detect-filestore.c index c2d1340c22..614579634f 100644 --- a/src/detect-filestore.c +++ b/src/detect-filestore.c @@ -226,8 +226,7 @@ static int DetectFilestorePostMatch(DetectEngineThreadCtx *det_ctx, flags); } - FileContainer *ffc = AppLayerParserGetFiles(p->flow->proto, p->flow->alproto, - p->flow->alstate, flags); + FileContainer *ffc = AppLayerParserGetFiles(p->flow, flags); /* filestore for single files only */ if (s->filestore_ctx == NULL) { diff --git a/src/flow-worker.c b/src/flow-worker.c index 74e81c57de..35bfe36700 100644 --- a/src/flow-worker.c +++ b/src/flow-worker.c @@ -179,8 +179,8 @@ static void FlowPruneFiles(Packet *p) { if (p->flow && p->flow->alstate) { Flow *f = p->flow; - FileContainer *fc = AppLayerParserGetFiles(p->proto, f->alproto, - f->alstate, PKT_IS_TOSERVER(p) ? STREAM_TOSERVER : STREAM_TOCLIENT); + FileContainer *fc = AppLayerParserGetFiles(f, + PKT_IS_TOSERVER(p) ? STREAM_TOSERVER : STREAM_TOCLIENT); if (fc != NULL) { FilePrune(fc); } diff --git a/src/output-file.c b/src/output-file.c index 97aec6c2a8..9eccf48b60 100644 --- a/src/output-file.c +++ b/src/output-file.c @@ -170,10 +170,8 @@ static TmEcode OutputFileLog(ThreadVars *tv, Packet *p, void *thread_data) (p->flowflags & FLOW_PKT_TOCLIENT)); const bool file_trunc = StreamTcpReassembleDepthReached(p); - FileContainer *ffc_ts = AppLayerParserGetFiles(p->proto, f->alproto, - f->alstate, STREAM_TOSERVER); - FileContainer *ffc_tc = AppLayerParserGetFiles(p->proto, f->alproto, - f->alstate, STREAM_TOCLIENT); + FileContainer *ffc_ts = AppLayerParserGetFiles(f, STREAM_TOSERVER); + FileContainer *ffc_tc = AppLayerParserGetFiles(f, STREAM_TOCLIENT); OutputFileLogFfc(tv, op_thread_data, p, ffc_ts, file_close_ts, file_trunc, STREAM_TOSERVER); OutputFileLogFfc(tv, op_thread_data, p, ffc_tc, file_close_tc, file_trunc, STREAM_TOCLIENT); diff --git a/src/output-filedata.c b/src/output-filedata.c index 893818234c..9414513759 100644 --- a/src/output-filedata.c +++ b/src/output-filedata.c @@ -227,10 +227,8 @@ static TmEcode OutputFiledataLog(ThreadVars *tv, Packet *p, void *thread_data) (p->flowflags & FLOW_PKT_TOCLIENT)); const bool file_trunc = StreamTcpReassembleDepthReached(p); - FileContainer *ffc_ts = AppLayerParserGetFiles(p->proto, f->alproto, - f->alstate, STREAM_TOSERVER); - FileContainer *ffc_tc = AppLayerParserGetFiles(p->proto, f->alproto, - f->alstate, STREAM_TOCLIENT); + FileContainer *ffc_ts = AppLayerParserGetFiles(f, STREAM_TOSERVER); + FileContainer *ffc_tc = AppLayerParserGetFiles(f, STREAM_TOCLIENT); SCLogDebug("ffc_ts %p", ffc_ts); OutputFiledataLogFfc(tv, store, p, ffc_ts, STREAM_TOSERVER, file_close_ts, file_trunc, STREAM_TOSERVER); SCLogDebug("ffc_tc %p", ffc_tc); diff --git a/src/util-file.c b/src/util-file.c index f148dff88a..fd11244c30 100644 --- a/src/util-file.c +++ b/src/util-file.c @@ -1051,7 +1051,7 @@ void FileDisableStoring(Flow *f, uint8_t direction) else f->file_flags |= FLOWFILE_NO_STORE_TC; - FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto, f->alstate, direction); + FileContainer *ffc = AppLayerParserGetFiles(f, f->alstate, direction); if (ffc != NULL) { for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) { /* if we're already storing, we'll continue */ @@ -1083,7 +1083,7 @@ void FileDisableMagic(Flow *f, uint8_t direction) else f->file_flags |= FLOWFILE_NO_MAGIC_TC; - FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto, f->alstate, direction); + FileContainer *ffc = AppLayerParserGetFiles(f, f->alstate, direction); if (ffc != NULL) { for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) { SCLogDebug("disabling magic for file %p from direction %s", @@ -1114,7 +1114,7 @@ void FileDisableMd5(Flow *f, uint8_t direction) else f->file_flags |= FLOWFILE_NO_MD5_TC; - FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto, f->alstate, direction); + FileContainer *ffc = AppLayerParserGetFiles(f, f->alstate, direction); if (ffc != NULL) { for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) { SCLogDebug("disabling md5 for file %p from direction %s", @@ -1153,7 +1153,7 @@ void FileDisableSha1(Flow *f, uint8_t direction) else f->file_flags |= FLOWFILE_NO_SHA1_TC; - FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto, f->alstate, direction); + FileContainer *ffc = AppLayerParserGetFiles(f, f->alstate, direction); if (ffc != NULL) { for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) { SCLogDebug("disabling sha1 for file %p from direction %s", @@ -1192,7 +1192,7 @@ void FileDisableSha256(Flow *f, uint8_t direction) else f->file_flags |= FLOWFILE_NO_SHA256_TC; - FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto, f->alstate, direction); + FileContainer *ffc = AppLayerParserGetFiles(f, f->alstate, direction); if (ffc != NULL) { for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) { SCLogDebug("disabling sha256 for file %p from direction %s", @@ -1231,7 +1231,7 @@ void FileDisableFilesize(Flow *f, uint8_t direction) else f->file_flags |= FLOWFILE_NO_SIZE_TC; - FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto, f->alstate, direction); + FileContainer *ffc = AppLayerParserGetFiles(f, f->alstate, direction); if (ffc != NULL) { for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) { SCLogDebug("disabling size tracking for file %p from direction %s", @@ -1284,7 +1284,7 @@ void FileDisableStoringForTransaction(Flow *f, uint8_t direction, uint64_t tx_id SCEnter(); - FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto, f->alstate, direction); + FileContainer *ffc = AppLayerParserGetFiles(f, direction); if (ffc != NULL) { for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) { if (ptr->txid == tx_id) {