From e1022ee5ae4f2583acf4ab8fec5936059a760379 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Tue, 27 Sep 2011 22:44:51 +0200 Subject: [PATCH] file-extraction: Disconnect file handling from flow and move into the app layer state. --- src/Makefile.am | 2 +- src/app-layer-htp-body.c | 1 - src/app-layer-htp-file.c | 135 +++++------ src/app-layer-htp-file.h | 6 +- src/app-layer-htp.c | 30 ++- src/app-layer-htp.h | 2 + src/app-layer-parser.c | 22 ++ src/app-layer-parser.h | 10 +- src/detect-engine-file.c | 14 +- src/detect-engine-state.c | 2 +- src/detect-fileext.c | 2 +- src/detect-filemagic.c | 8 +- src/detect-filename.c | 2 +- src/detect-filestore.c | 4 +- src/detect.c | 4 +- src/flow-file.c | 498 -------------------------------------- src/flow-file.h | 121 --------- src/flow-util.h | 13 - src/flow.c | 1 - src/flow.h | 5 - src/log-file.c | 22 +- 21 files changed, 146 insertions(+), 758 deletions(-) diff --git a/src/Makefile.am b/src/Makefile.am index fd1c825655..01daf44ae5 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -47,7 +47,6 @@ flow-manager.c flow-manager.h \ flow-queue.c flow-queue.h \ flow-hash.c flow-hash.h \ flow-util.c flow-util.h \ -flow-file.c flow-file.h \ util-mem.h \ flow-var.c flow-var.h \ flow-bit.c flow-bit.h \ @@ -220,6 +219,7 @@ util-vector.h \ util-device.c util-device.h \ util-checksum.c util-checksum.h \ util-runmodes.c util-runmodes.h \ +util-file.c util-file.h \ tm-modules.c tm-modules.h \ tm-queues.c tm-queues.h \ tm-queuehandlers.c tm-queuehandlers.h \ diff --git a/src/app-layer-htp-body.c b/src/app-layer-htp-body.c index 99e8eb0863..ff0d27077a 100644 --- a/src/app-layer-htp-body.c +++ b/src/app-layer-htp-body.c @@ -55,7 +55,6 @@ #include "util-unittest.h" #include "util-unittest-helper.h" #include "flow-util.h" -#include "flow-file.h" #include "detect-engine.h" #include "detect-engine-state.h" diff --git a/src/app-layer-htp-file.c b/src/app-layer-htp-file.c index c9700eb5c4..ed57349dbd 100644 --- a/src/app-layer-htp-file.c +++ b/src/app-layer-htp-file.c @@ -52,7 +52,6 @@ #include "util-unittest.h" #include "util-unittest-helper.h" #include "flow-util.h" -#include "flow-file.h" #include "detect-engine.h" #include "detect-engine-state.h" @@ -76,44 +75,40 @@ * \retval -1 error * \retval -2 not handling files on this flow */ -int HTPFileOpen(Flow *f, uint8_t *filename, uint16_t filename_len, +int HTPFileOpen(HtpState *s, uint8_t *filename, uint16_t filename_len, uint8_t *data, uint32_t data_len) { int retval = 0; uint16_t txid; uint8_t flags = 0; - if (f == NULL) { + if (s == NULL) { SCReturnInt(-1); } - txid = AppLayerTransactionGetAvailId(f) - 1; + txid = AppLayerTransactionGetAvailId(s->f) - 1; - SCMutexLock(&f->files_m); - { - if (f->files == NULL) { - f->files = FlowFileContainerAlloc(); - if (f->files == NULL) { - retval = -1; - goto end; - } - } - - if (f->flags & FLOW_FILE_NO_HANDLING) { - flags |= FLOW_FILE_NOSTORE; - } - - if (FlowFileOpenFile(f->files, filename, filename_len, - data, data_len, flags) == NULL) - { + if (s->files == NULL) { + s->files = FileContainerAlloc(); + if (s->files == NULL) { retval = -1; + goto end; } + } - FlowFileSetTx(f->files->tail, txid); + if (s->f->flags & FLOW_FILE_NO_HANDLING) { + flags |= FILE_NOSTORE; } + if (FileOpenFile(s->files, filename, filename_len, + data, data_len, flags) == NULL) + { + retval = -1; + } + + FileSetTx(s->files->tail, txid); + end: - SCMutexUnlock(&f->files_m); SCReturnInt(retval); } @@ -128,35 +123,31 @@ end: * \retval -1 error * \retval -2 file doesn't need storing */ -int HTPFileStoreChunk(Flow *f, uint8_t *data, uint32_t data_len) { +int HTPFileStoreChunk(HtpState *s, uint8_t *data, uint32_t data_len) { SCEnter(); int retval = 0; int result = 0; - if (f == NULL) { + if (s == NULL) { SCReturnInt(-1); } - SCMutexLock(&f->files_m); - { - if (f->files == NULL) { - SCLogDebug("no files in flow"); - retval = -1; - goto end; - } + if (s->files == NULL) { + SCLogDebug("no files in state"); + retval = -1; + goto end; + } - result = FlowFileAppendData(f->files, data, data_len); - if (result == -1) { - SCLogDebug("appending data failed"); - retval = -1; - } else if (result == -2) { - retval = -2; - } + result = FileAppendData(s->files, data, data_len); + if (result == -1) { + SCLogDebug("appending data failed"); + retval = -1; + } else if (result == -2) { + retval = -2; } end: - SCMutexUnlock(&f->files_m); SCReturnInt(retval); } @@ -175,31 +166,27 @@ end: * \retval -1 error * \retval -2 not storing files on this flow/tx */ -int HTPFileClose(Flow *f, uint8_t *data, uint32_t data_len, uint8_t flags) { +int HTPFileClose(HtpState *s, uint8_t *data, uint32_t data_len, uint8_t flags) { int retval = 0; int result = 0; - if (f == NULL) { + if (s == NULL) { SCReturnInt(-1); } - SCMutexLock(&f->files_m); - { - if (f->files == NULL) { - retval = -1; - goto end; - } + if (s->files == NULL) { + retval = -1; + goto end; + } - result = FlowFileCloseFile(f->files, data, data_len, flags); - if (result == -1) { - retval = -1; - } else if (result == -2) { - retval = -2; - } + result = FileCloseFile(s->files, data, data_len, flags); + if (result == -1) { + retval = -1; + } else if (result == -2) { + retval = -2; } end: - SCMutexUnlock(&f->files_m); SCReturnInt(retval); } @@ -365,7 +352,7 @@ static int HTPFileParserTest02(void) { goto end; } - if (f.files == NULL || f.files->tail == NULL || f.files->tail->state != FLOWFILE_STATE_CLOSED) { + if (http_state->files == NULL || http_state->files->tail == NULL || http_state->files->tail->state != FILE_STATE_CLOSED) { goto end; } @@ -487,11 +474,11 @@ static int HTPFileParserTest03(void) { goto end; } - if (f.files == NULL || f.files->tail == NULL || f.files->tail->state != FLOWFILE_STATE_CLOSED) { + if (http_state->files == NULL || http_state->files->tail == NULL || http_state->files->tail->state != FILE_STATE_CLOSED) { goto end; } - if (f.files->head->chunks_head->len != 11) { + if (http_state->files->head->chunks_head->len != 11) { goto end; } @@ -613,7 +600,7 @@ static int HTPFileParserTest04(void) { goto end; } - if (f.files == NULL || f.files->tail == NULL || f.files->tail->state != FLOWFILE_STATE_CLOSED) { + if (http_state->files == NULL || http_state->files->tail == NULL || http_state->files->tail->state != FILE_STATE_CLOSED) { goto end; } @@ -694,39 +681,39 @@ static int HTPFileParserTest05(void) { goto end; } - if (f.files == NULL || f.files->tail == NULL || f.files->tail->state != FLOWFILE_STATE_CLOSED) { + if (http_state->files == NULL || http_state->files->tail == NULL || http_state->files->tail->state != FILE_STATE_CLOSED) { goto end; } - if (f.files->head == f.files->tail) + if (http_state->files->head == http_state->files->tail) goto end; - if (f.files->head->next != f.files->tail) + if (http_state->files->head->next != http_state->files->tail) goto end; - if (f.files->head->chunks_head->len != 11) { + if (http_state->files->head->chunks_head->len != 11) { printf("expected 11 but file is %u bytes instead\n", - f.files->head->chunks_head->len); - PrintRawDataFp(stdout, f.files->head->chunks_head->data, - f.files->head->chunks_head->len); + http_state->files->head->chunks_head->len); + PrintRawDataFp(stdout, http_state->files->head->chunks_head->data, + http_state->files->head->chunks_head->len); goto end; } - if (memcmp("filecontent", f.files->head->chunks_head->data, - f.files->head->chunks_head->len) != 0) { + if (memcmp("filecontent", http_state->files->head->chunks_head->data, + http_state->files->head->chunks_head->len) != 0) { goto end; } - if (f.files->tail->chunks_head->len != 11) { + if (http_state->files->tail->chunks_head->len != 11) { printf("expected 11 but file is %u bytes instead\n", - f.files->tail->chunks_head->len); - PrintRawDataFp(stdout, f.files->tail->chunks_head->data, - f.files->tail->chunks_head->len); + http_state->files->tail->chunks_head->len); + PrintRawDataFp(stdout, http_state->files->tail->chunks_head->data, + http_state->files->tail->chunks_head->len); goto end; } - if (memcmp("FILECONTENT", f.files->tail->chunks_head->data, - f.files->tail->chunks_head->len) != 0) { + if (memcmp("FILECONTENT", http_state->files->tail->chunks_head->data, + http_state->files->tail->chunks_head->len) != 0) { goto end; } result = 1; diff --git a/src/app-layer-htp-file.h b/src/app-layer-htp-file.h index 300ec96de3..84e4114a65 100644 --- a/src/app-layer-htp-file.h +++ b/src/app-layer-htp-file.h @@ -25,9 +25,9 @@ #ifndef __APP_LAYER_HTP_FILE_H__ #define __APP_LAYER_HTP_FILE_H__ -int HTPFileOpen(Flow *, uint8_t *, uint16_t, uint8_t *, uint32_t); -int HTPFileStoreChunk(Flow *, uint8_t *, uint32_t); -int HTPFileClose(Flow *, uint8_t *, uint32_t, uint8_t); +int HTPFileOpen(HtpState *, uint8_t *, uint16_t, uint8_t *, uint32_t); +int HTPFileStoreChunk(HtpState *, uint8_t *, uint32_t); +int HTPFileClose(HtpState *, uint8_t *, uint32_t, uint8_t); void HTPFileParserRegisterTests(void); diff --git a/src/app-layer-htp.c b/src/app-layer-htp.c index c25541d4a7..695ec7cc4b 100644 --- a/src/app-layer-htp.c +++ b/src/app-layer-htp.c @@ -41,6 +41,7 @@ #include "util-print.h" #include "util-pool.h" #include "util-radix-tree.h" +#include "util-file.h" #include "stream-tcp-private.h" #include "stream-tcp-reassemble.h" @@ -61,7 +62,6 @@ #include "util-unittest.h" #include "util-unittest-helper.h" #include "flow-util.h" -#include "flow-file.h" #include "detect-engine.h" #include "detect-engine-state.h" @@ -229,6 +229,7 @@ void HTPStateFree(void *state) htp_connp_destroy_all(s->connp); } + FileContainerFree(s->files); SCFree(s); #ifdef DEBUG @@ -1065,7 +1066,7 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, SCHtpTxUserData *htud, filedata_len = form_end - filedata - 2; /* 0d 0a */ } else if (htud->flags & HTP_BODY_COMPLETE) { filedata_len = chunks_buffer_len; - flags = FLOW_FILE_TRUNCATED; + flags = FILE_TRUNCATED; } BUG_ON(filedata_len > chunks_buffer_len); @@ -1075,7 +1076,7 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, SCHtpTxUserData *htud, printf("FILEDATA (final chunk) END: \n"); #endif if (!(htud->flags & HTP_DONTSTORE)) { - if (HTPFileClose(hstate->f, filedata, filedata_len, flags) == -1) + if (HTPFileClose(hstate, filedata, filedata_len, flags) == -1) { goto end; } @@ -1097,7 +1098,7 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, SCHtpTxUserData *htud, #endif if (!(htud->flags & HTP_DONTSTORE)) { - result = HTPFileStoreChunk(hstate->f, filedata, filedata_len); + result = HTPFileStoreChunk(hstate, filedata, filedata_len); if (result == -1) { goto end; } else if (result == -2) { @@ -1167,14 +1168,14 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, SCHtpTxUserData *htud, printf("FILEDATA END: \n"); #endif - result = HTPFileOpen(hstate->f, filename, filename_len, + result = HTPFileOpen(hstate, filename, filename_len, filedata, filedata_len); if (result == -1) { goto end; } else if (result == -2) { htud->flags |= HTP_DONTSTORE; } else { - if (HTPFileClose(hstate->f, NULL, 0, 0) == -1) { + if (HTPFileClose(hstate, NULL, 0, 0) == -1) { goto end; } } @@ -1185,7 +1186,7 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, SCHtpTxUserData *htud, SCLogDebug("offset %u", offset); htud->body_parsed = offset; - result = HTPFileOpen(hstate->f, filename, filename_len, + result = HTPFileOpen(hstate, filename, filename_len, filedata, filedata_len); if (result == -1) { goto end; @@ -1249,7 +1250,7 @@ int HtpRequestBodyHandlePUT(HtpState *hstate, SCHtpTxUserData *htud, filename_len = bstr_len(tx->parsed_uri->path); } - result = HTPFileOpen(hstate->f, filename, filename_len, + result = HTPFileOpen(hstate, filename, filename_len, data, data_len); if (result == -1) { goto end; @@ -1265,7 +1266,7 @@ int HtpRequestBodyHandlePUT(HtpState *hstate, SCHtpTxUserData *htud, /* otherwise, just store the data */ if (!(htud->flags & HTP_DONTSTORE)) { - result = HTPFileStoreChunk(hstate->f, data, data_len); + result = HTPFileStoreChunk(hstate, data, data_len); if (result == -1) { goto end; } else if (result == -2) { @@ -1440,7 +1441,7 @@ static int HTPCallbackRequest(htp_connp_t *connp) { if (htud != NULL) { if (htud->flags & HTP_FILENAME_SET) { SCLogDebug("closing file that was being stored"); - (void)HTPFileClose(hstate->f, NULL, 0, 0); + (void)HTPFileClose(hstate, NULL, 0, 0); htud->flags &= ~HTP_FILENAME_SET; } } @@ -1776,6 +1777,14 @@ void AppLayerHtpPrintStats(void) { #endif } +static FileContainer *HTPStateGetFiles(void *state) { + if (state == NULL) + return NULL; + + HtpState *http_state = (HtpState *)state; + SCReturnPtr(http_state->files, "FileContainer"); +} + /** * \brief Register the HTTP protocol and state handling functions to APP layer * of the engine. @@ -1802,6 +1811,7 @@ void RegisterHTPParsers(void) AppLayerRegisterStateFuncs(ALPROTO_HTTP, HTPStateAlloc, HTPStateFree); AppLayerRegisterTransactionIdFuncs(ALPROTO_HTTP, HTPStateUpdateTransactionId, HTPStateTransactionFree); + AppLayerRegisterGetFilesFunc(ALPROTO_HTTP, HTPStateGetFiles); AppLayerRegisterProto("http", ALPROTO_HTTP, STREAM_TOSERVER, HTPHandleRequestData); diff --git a/src/app-layer-htp.h b/src/app-layer-htp.h index ad39ca4a35..a9772dde3b 100644 --- a/src/app-layer-htp.h +++ b/src/app-layer-htp.h @@ -34,6 +34,7 @@ #define __APP_LAYER_HTP_H__ #include "util-radix-tree.h" +#include "util-file.h" #include @@ -147,6 +148,7 @@ typedef struct HtpState_ { uint16_t transaction_cnt; uint16_t transaction_done; uint32_t request_body_limit; + FileContainer *files; } HtpState; void RegisterHTPParsers(void); diff --git a/src/app-layer-parser.c b/src/app-layer-parser.c index 37fd2eb912..d22a48061c 100644 --- a/src/app-layer-parser.c +++ b/src/app-layer-parser.c @@ -41,6 +41,7 @@ #include "stream.h" #include "stream-tcp-reassemble.h" +#include "app-layer.h" #include "app-layer-protos.h" #include "app-layer-parser.h" #include "app-layer-smb.h" @@ -71,6 +72,21 @@ static SCMutex al_result_pool_mutex = PTHREAD_MUTEX_INITIALIZER; static uint32_t al_result_pool_elmts = 0; #endif /* DEBUG */ +/** \brief Get the file container flow + * \param f flow pointer to a LOCKED flow + * \retval files void pointer to the state + * \retval NULL in case we have no state */ +FileContainer *AppLayerGetFilesFromFlow(Flow *f) { + uint16_t alproto = f->alproto; + + if (alproto == ALPROTO_UNKNOWN) + return NULL; + + if (al_proto_table[alproto].StateGetFiles != NULL) + return al_proto_table[alproto].StateGetFiles(AppLayerGetProtoStateFromFlow(f)); + else + return NULL; +} /** \brief Alloc a AppLayerParserResultElmt func for the pool */ static void *AlpResultElmtPoolAlloc(void *null) @@ -618,6 +634,12 @@ void *AppLayerGetProtocolParserLocalStorage(uint16_t proto) return NULL; } +void AppLayerRegisterGetFilesFunc(uint16_t proto, + FileContainer *(*StateGetFiles)(void *state)) +{ + al_proto_table[proto].StateGetFiles = StateGetFiles; +} + /** \brief Indicate to the app layer parser that a logger is active * for this protocol. */ diff --git a/src/app-layer-parser.h b/src/app-layer-parser.h index a5028757eb..ac273b4994 100644 --- a/src/app-layer-parser.h +++ b/src/app-layer-parser.h @@ -24,6 +24,8 @@ #ifndef __APP_LAYER_PARSER_H__ #define __APP_LAYER_PARSER_H__ +#include "util-file.h" + /** Mapping between local parser id's (e.g. HTTP_FIELD_REQUEST_URI) and * the dynamically assigned (at registration) global parser id. */ typedef struct AppLayerLocalMap_ { @@ -50,6 +52,7 @@ typedef struct AppLayerProto_ { void (*StateTransactionFree)(void *, uint16_t); void *(*LocalStorageAlloc)(void); void (*LocalStorageFree)(void *); + FileContainer *(*StateGetFiles)(void *); } AppLayerProto; @@ -245,6 +248,8 @@ void AppLayerRegisterLocalStorageFunc(uint16_t proto, void *(*LocalStorageAlloc)(void), void (*LocalStorageFree)(void *)); void *AppLayerGetProtocolParserLocalStorage(uint16_t); +void AppLayerRegisterGetFilesFunc(uint16_t proto, + FileContainer *(*StateGetFile)(void *)); void AppLayerRegisterLogger(uint16_t proto); uint16_t AppLayerGetProtoByName(const char *); @@ -269,8 +274,6 @@ int AppLayerTransactionGetBaseId(Flow *f); int AppLayerTransactionGetInspectId(Flow *f); uint16_t AppLayerTransactionGetAvailId(Flow *f); -uint16_t AppLayerGetStateVersion(Flow *f); - void AppLayerSetEOF(Flow *); /* cleanup */ @@ -279,4 +282,7 @@ void AppLayerFreeProbingParsers(AppLayerProbingParser *); void AppLayerFreeProbingParsersInfo(AppLayerProbingParserInfo *); void AppLayerPrintProbingParsers(AppLayerProbingParser *); +uint16_t AppLayerGetStateVersion(Flow *f); +FileContainer *AppLayerGetFilesFromFlow(Flow *); + #endif /* __APP_LAYER_PARSER_H__ */ diff --git a/src/detect-engine-file.c b/src/detect-engine-file.c index 081b4082ce..8a7ccb5d54 100644 --- a/src/detect-engine-file.c +++ b/src/detect-engine-file.c @@ -70,21 +70,20 @@ * * \note flow is not locked at this time */ -static int DetectFileInspect(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Flow *f, Signature *s) { +static int DetectFileInspect(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Flow *f, Signature *s, FileContainer *ffc) { SigMatch *sm = NULL; int r = 0; int match = 0; SCLogDebug("file inspection..."); - SCMutexLock(&f->files_m); - if (f->files != NULL) { - FlowFile *file = f->files->head; + if (ffc != NULL) { + File *file = ffc->head; for (; file != NULL; file = file->next) { SCLogDebug("file"); - if (file->state == FLOWFILE_STATE_NONE) { - SCLogDebug("file state FLOWFILE_STATE_NONE"); + if (file->state == FILE_STATE_NONE) { + SCLogDebug("file state FILE_STATE_NONE"); continue; } @@ -143,7 +142,6 @@ static int DetectFileInspect(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Flo } } - SCMutexUnlock(&f->files_m); SCReturnInt(r); } @@ -178,7 +176,7 @@ int DetectFileInspectHttp(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Flow * /* inspect files for this transaction */ det_ctx->tx_id = (uint16_t)idx; - match = DetectFileInspect(tv, det_ctx, f, s); + match = DetectFileInspect(tv, det_ctx, f, s, htp_state->files); if (match == 1) { r = 1; } else if (match == 2) { diff --git a/src/detect-engine-state.c b/src/detect-engine-state.c index 2dfb59f4c7..99ae9cc9b3 100644 --- a/src/detect-engine-state.c +++ b/src/detect-engine-state.c @@ -883,7 +883,7 @@ next_sig: if (!(f->de_state->flags & DE_STATE_FILE_STORE_DISABLED)) { if (DeStateStoreFilestoreSigsCantMatch(det_ctx->sgh, f->de_state, flags) == 1) { SCLogDebug("disabling file storage for transaction"); - FlowFileDisableStoringForTransaction(f, det_ctx->tx_id); + FileDisableStoringForTransaction(f, det_ctx->tx_id); f->de_state->flags |= DE_STATE_FILE_STORE_DISABLED; } } diff --git a/src/detect-fileext.c b/src/detect-fileext.c index 710cecfaea..e6c9ec9d51 100644 --- a/src/detect-fileext.c +++ b/src/detect-fileext.c @@ -89,7 +89,7 @@ int DetectFileextMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, Flow *f, int ret = 0; DetectFileextData *fileext = (DetectFileextData *)m->ctx; - FlowFile *file = (FlowFile *)state; + File *file = (File *)state; if (file->name == NULL) SCReturnInt(0); diff --git a/src/detect-filemagic.c b/src/detect-filemagic.c index 034d26fcea..ed89eb84ac 100644 --- a/src/detect-filemagic.c +++ b/src/detect-filemagic.c @@ -82,7 +82,7 @@ void DetectFilemagicRegister(void) { * \retval -1 error * \retval 0 ok */ -static int FilemagicLookup(FlowFile *file) { +static int FilemagicLookup(File *file) { if (file == NULL || file->chunks_head == NULL) { SCReturnInt(-1); } @@ -94,7 +94,7 @@ static int FilemagicLookup(FlowFile *file) { uint32_t size = 0; if (buf != NULL) { - FlowFileData *ffd = file->chunks_head; + FileData *ffd = file->chunks_head; for ( ; ffd != NULL; ffd = ffd->next) { uint32_t copy_len = ffd->len; @@ -110,7 +110,7 @@ static int FilemagicLookup(FlowFile *file) { } /* file is done but smaller than FILEMAGIC_MIN_SIZE */ - if (ffd->next == NULL && file->state >= FLOWFILE_STATE_CLOSED) { + if (ffd->next == NULL && file->state >= FILE_STATE_CLOSED) { file->magic = MagicLookup(buf, size); break; } @@ -140,7 +140,7 @@ int DetectFilemagicMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, Flow *f int ret = 0; DetectFilemagicData *filemagic = m->ctx; - FlowFile *file = (FlowFile *)state; + File *file = (File *)state; if (file->txid < det_ctx->tx_id) SCReturnInt(0); diff --git a/src/detect-filename.c b/src/detect-filename.c index dbc1f7d497..2ec9c8bb3f 100644 --- a/src/detect-filename.c +++ b/src/detect-filename.c @@ -88,7 +88,7 @@ int DetectFilenameMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, Flow *f, int ret = 0; DetectFilenameData *filename = m->ctx; - FlowFile *file = (FlowFile *)state; + File *file = (File *)state; if (file->name == NULL) SCReturnInt(0); diff --git a/src/detect-filestore.c b/src/detect-filestore.c index 67534e76ca..95cc238962 100644 --- a/src/detect-filestore.c +++ b/src/detect-filestore.c @@ -83,8 +83,8 @@ int DetectFilestoreMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, Flow *f uint8_t flags, void *state, Signature *s, SigMatch *m) { SCEnter(); - FlowFile *file = (FlowFile *)state; - FlowFileStore(file); + File *file = (File *)state; + FileStore(file); SCReturnInt(1); } diff --git a/src/detect.c b/src/detect.c index 9e8e39a6d9..c983b847ee 100644 --- a/src/detect.c +++ b/src/detect.c @@ -1785,7 +1785,7 @@ end: (p->flow->sgh_toclient == NULL || p->flow->sgh_toclient->filestore_cnt == 0)) { - FlowFileDisableStoring(p->flow); + FileDisableStoring(p->flow); } } @@ -3787,7 +3787,7 @@ int SigAddressPrepareStage4(DetectEngineCtx *de_ctx) { SigGroupHeadBuildHeadArray(de_ctx, sgh); SigGroupHeadSetFilemagicFlag(de_ctx, sgh); SigGroupHeadSetFilestoreCount(de_ctx, sgh); - SCLogInfo("filestore count %u", sgh->filestore_cnt); + SCLogDebug("filestore count %u", sgh->filestore_cnt); } if (de_ctx->decoder_event_sgh != NULL) { diff --git a/src/flow-file.c b/src/flow-file.c index 332cd4b185..8fc0b002ff 100644 --- a/src/flow-file.c +++ b/src/flow-file.c @@ -27,506 +27,8 @@ #include "suricata.h" #include "debug.h" #include "flow.h" -#include "flow-file.h" #include "util-hash.h" #include "util-debug.h" #include "util-memcmp.h" #include "util-print.h" -/* prototypes */ -static void FlowFileFree(FlowFile *); - -int FlowFileMagicSize(void) { - return 512; -} - -/** - * \brief check if we have stored enough - * - * \param ff file - * - * \retval 0 limit not reached yet - * \retval 1 limit reached - */ -static int FlowFileStoreNoStoreCheck(FlowFile *ff) { - SCEnter(); - - if (ff == NULL) { - SCReturnInt(0); - } - - - if (ff->store == -1) { - if (ff->state == FLOWFILE_STATE_OPENED && - ff->size >= (uint64_t)FlowFileMagicSize()) - { - SCReturnInt(1); - } - } - - SCReturnInt(0); -} - -/** - * \brief allocate a FlowFileContainer - * - * \retval new newly allocated FlowFileContainer - * \retval NULL error - */ -FlowFileContainer *FlowFileContainerAlloc(void) { - FlowFileContainer *new = SCMalloc(sizeof(FlowFileContainer)); - if (new == NULL) { - SCLogError(SC_ERR_MEM_ALLOC, "Error allocating mem"); - return NULL; - } - memset(new, 0, sizeof(FlowFileContainer)); - new->head = new->tail = NULL; - return new; -} - -/** - * \brief Recycle a FlowFileContainer - * - * \param ffc FlowFileContainer - */ -void FlowFileContainerRecycle(FlowFileContainer *ffc) { - if (ffc == NULL) - return; - - FlowFile *cur = ffc->head; - FlowFile *next = NULL; - for (;cur != NULL; cur = next) { - next = cur->next; - FlowFileFree(cur); - } - ffc->head = ffc->tail = NULL; -} - -/** - * \brief Free a FlowFileContainer - * - * \param ffc FlowFileContainer - */ -void FlowFileContainerFree(FlowFileContainer *ffc) { - if (ffc == NULL) - return; - - FlowFile *ptr = ffc->head; - FlowFile *next = NULL; - for (;ptr != NULL; ptr = next) { - next = ptr->next; - FlowFileFree(ptr); - } - ffc->head = ffc->tail = NULL; - SCFree(ffc); -} - -/** - * \internal - * - * \brief allocate a FlowFileData chunk and set it up - * - * \param data data chunk to store in the FlowFileData - * \param data_len lenght of the data - * - * \retval new FlowFileData object - */ -static FlowFileData *FlowFileDataAlloc(uint8_t *data, uint32_t data_len) { - FlowFileData *new = SCMalloc(sizeof(FlowFileData)); - if (new == NULL) { - return NULL; - } - memset(new, 0, sizeof(FlowFileData)); - - new->data = SCMalloc(data_len); - if (new->data == NULL) { - SCFree(new); - return NULL; - } - - new->len = data_len; - memcpy(new->data, data, data_len); - - new->next = NULL; - return new; -} - -/** - * \internal - * - * \brief free a FlowFileData object - * - * \param ffd the flow file data object to free - */ -static void FlowFileDataFree(FlowFileData *ffd) { - if (ffd == NULL) - return; - - if (ffd->data != NULL) { - SCFree(ffd->data); - } - - SCFree(ffd); -} - -static int FlowFileAppendFlowFileDataFilePtr(FlowFile *ff, FlowFileData *ffd) { - SCEnter(); - - if (ff == NULL) { - SCReturnInt(-1); - } - - if (ff->chunks_tail == NULL) { - ff->chunks_head = ffd; - ff->chunks_tail = ffd; - } else { - ff->chunks_tail->next = ffd; - ff->chunks_tail = ffd; - } - - ff->size += ffd->len; - SCLogDebug("file size %"PRIu64, ff->size); - - SCReturnInt(0); -} - -static int FlowFileAppendFlowFileData(FlowFileContainer *ffc, FlowFileData *ffd) { - SCEnter(); - - if (ffc == NULL) { - SCReturnInt(-1); - } - - if (FlowFileAppendFlowFileDataFilePtr(ffc->tail, ffd) == -1) - { - SCReturnInt(-1); - } - - SCReturnInt(0); -} - -/** - * \brief Alloc a new FlowFile - * - * \param name character array containing the name (not a string) - * \param name_len length in bytes of the name - * - * \retval new FlowFile object or NULL on error - */ -static FlowFile *FlowFileAlloc(uint8_t *name, uint16_t name_len) { - FlowFile *new = SCMalloc(sizeof(FlowFile)); - if (new == NULL) { - SCLogError(SC_ERR_MEM_ALLOC, "Error allocating mem"); - return NULL; - } - memset(new, 0, sizeof(FlowFile)); - - new->fd = -1; - - new->name = SCMalloc(name_len); - if (new->name == NULL) { - SCFree(new); - return NULL; - } - - new->name_len = name_len; - memcpy(new->name, name, name_len); - - return new; -} - -static void FlowFileFree(FlowFile *ff) { - if (ff == NULL) - return; - - if (ff->fd != -1) - close(ff->fd); - - if (ff->name != NULL) - SCFree(ff->name); - SCFree(ff); -} - -void FlowFileContainerAdd(FlowFileContainer *ffc, FlowFile *ff) { - if (ffc->head == NULL) { - ffc->head = ffc->tail = ff; - } else { - ffc->tail->next = ff; - ffc->tail = ff; - } -} - -/** - * \brief Open a new FlowFile - * - * \param ffc flow container - * \param name filename character array - * \param name_len filename len - * \param data initial data - * \param data_len initial data len - * \param flags open flags - * - * \retval ff flowfile object - * - * \note filename is not a string, so it's not nul terminated. - */ -FlowFile *FlowFileOpenFile(FlowFileContainer *ffc, uint8_t *name, - uint16_t name_len, uint8_t *data, uint32_t data_len, uint8_t flags) -{ - SCEnter(); - - //PrintRawDataFp(stdout, name, name_len); - - FlowFile *ff = FlowFileAlloc(name, name_len); - if (ff == NULL) { - SCReturnPtr(NULL, "FlowFile"); - } - - if (flags & FLOW_FILE_NOSTORE) { - ff->store = -1; - } - - ff->state = FLOWFILE_STATE_OPENED; - SCLogDebug("flowfile state transitioned to FLOWFILE_STATE_OPENED"); - - FlowFileContainerAdd(ffc, ff); - - if (data != NULL) { - //PrintRawDataFp(stdout, data, data_len); - - FlowFileData *ffd = FlowFileDataAlloc(data, data_len); - if (ffd == NULL) { - ff->state = FLOWFILE_STATE_ERROR; - SCReturnPtr(NULL, "FlowFile"); - } - - /* append the data */ - if (FlowFileAppendFlowFileData(ffc, ffd) < 0) { - ff->state = FLOWFILE_STATE_ERROR; - FlowFileDataFree(ffd); - SCReturnPtr(NULL, "FlowFile"); - } - } - - SCReturnPtr(ff, "FlowFile"); -} - -static int FlowFileCloseFilePtr(FlowFile *ff, uint8_t *data, - uint32_t data_len, uint8_t flags) -{ - SCEnter(); - - if (ff == NULL) { - SCReturnInt(-1); - } - - if (ff->state != FLOWFILE_STATE_OPENED) { - SCReturnInt(-1); - } - - if (data != NULL) { - //PrintRawDataFp(stdout, data, data_len); - - FlowFileData *ffd = FlowFileDataAlloc(data, data_len); - if (ffd == NULL) { - ff->state = FLOWFILE_STATE_ERROR; - SCReturnInt(-1); - } - - /* append the data */ - if (FlowFileAppendFlowFileDataFilePtr(ff, ffd) < 0) { - ff->state = FLOWFILE_STATE_ERROR; - FlowFileDataFree(ffd); - SCReturnInt(-1); - } - } - - if (flags & FLOW_FILE_TRUNCATED) { - ff->state = FLOWFILE_STATE_TRUNCATED; - SCLogDebug("flowfile state transitioned to FLOWFILE_STATE_TRUNCATED"); - - if (flags & FLOW_FILE_NOSTORE) { - ff->store = -1; - } - } else { - ff->state = FLOWFILE_STATE_CLOSED; - SCLogDebug("flowfile state transitioned to FLOWFILE_STATE_CLOSED"); - } - - SCReturnInt(0); -} - -/** - * \brief Close a FlowFile - * - * \param ffc the container - * \param data final data if any - * \param data_len data len if any - * \param flags flags - * - * \retval 0 ok - * \retval -1 error - */ -int FlowFileCloseFile(FlowFileContainer *ffc, uint8_t *data, - uint32_t data_len, uint8_t flags) -{ - SCEnter(); - - if (ffc == NULL || ffc->tail == NULL) { - SCReturnInt(-1); - } - - if (FlowFileCloseFilePtr(ffc->tail, data, data_len, flags) == -1) { - SCReturnInt(-1); - } - - SCReturnInt(0); -} - -/** - * \brief Store a chunk of file data in the flow. The open "flowfile" - * will be used. - * - * \param ffc the container - * \param data data chunk - * \param data_len data chunk len - * - * \retval 0 ok - * \retval -1 error - * \retval -2 no store for this file - */ -int FlowFileAppendData(FlowFileContainer *ffc, uint8_t *data, uint32_t data_len) { - SCEnter(); - - if (ffc == NULL || ffc->tail == NULL || data == NULL || data_len == 0) { - SCReturnInt(-1); - } - - if (ffc->tail->state != FLOWFILE_STATE_OPENED) { - if (ffc->tail->store == -1) { - SCReturnInt(-2); - } - SCReturnInt(-1); - } - - if (FlowFileStoreNoStoreCheck(ffc->tail) == 1) { - ffc->tail->state = FLOWFILE_STATE_CLOSED; - SCLogDebug("flowfile state transitioned to FLOWFILE_STATE_CLOSED"); - SCReturnInt(-2); - } - - SCLogDebug("appending %"PRIu32" bytes", data_len); - - FlowFileData *ffd = FlowFileDataAlloc(data, data_len); - if (ffd == NULL) { - ffc->tail->state = FLOWFILE_STATE_ERROR; - SCReturnInt(-1); - } - - /* append the data */ - if (FlowFileAppendFlowFileData(ffc, ffd) < 0) { - ffc->tail->state = FLOWFILE_STATE_ERROR; - FlowFileDataFree(ffd); - SCReturnInt(-1); - } - SCReturnInt(0); -} - -/** - * \brief Tag a file for storing - * - * \param ff The file to store - */ -int FlowFileStore(FlowFile *ff) { - ff->store = 1; - SCReturnInt(0); -} - -/** - * \brief Set the TX id for a file - * - * \param ff The file to store - * \param txid the tx id - */ -int FlowFileSetTx(FlowFile *ff, uint16_t txid) { - ff->txid = txid; - SCReturnInt(0); -} - -/** - * \brief disable file storage for a flow - * - * \param f *LOCKED* flow - */ -void FlowFileDisableStoring(Flow *f) { - FlowFile *ptr = NULL; - - SCEnter(); - - f->flags |= FLOW_FILE_NO_HANDLING; - - SCMutexLock(&f->files_m); - if (f->files != NULL) { - for (ptr = f->files->head; ptr != NULL; ptr = ptr->next) { - if (ptr->state == FLOWFILE_STATE_OPENED) { - - if (ptr->store == 0) { - ptr->store = -1; - } -// ptr->state = FLOWFILE_STATE_CLOSED; -// SCLogDebug("flowfile state transitioned to FLOWFILE_STATE_CLOSED"); - } - } - } - SCMutexUnlock(&f->files_m); - SCReturn; -} - -/** - * \brief set no store flag, close file if needed - * - * \param ff file - */ -static void FlowFileDisableStoringForFile(FlowFile *ff) { - SCEnter(); - - if (ff == NULL) { - SCReturn; - } - - ff->store = -1; - - if (ff->state == FLOWFILE_STATE_OPENED && ff->size >= (uint64_t)FlowFileMagicSize()) { - (void)FlowFileCloseFilePtr(ff, NULL, 0, - (FLOW_FILE_TRUNCATED|FLOW_FILE_NOSTORE)); - } -} - -/** - * \brief disable file storing for files in a transaction - * - * \param f flow - * \param tx_id transaction id - */ -void FlowFileDisableStoringForTransaction(struct Flow_ *f, uint16_t tx_id) { - FlowFile *ptr = NULL; - - SCEnter(); - - SCMutexLock(&f->files_m); - if (f->files != NULL) { - for (ptr = f->files->head; ptr != NULL; ptr = ptr->next) { - if (ptr->state == FLOWFILE_STATE_OPENED && ptr->txid == tx_id) { - if (ptr->store == 1) { - /* weird, already storing -- let it continue*/ - SCLogDebug("file is already being stored"); - } else { - FlowFileDisableStoringForFile(ptr); - } - } - } - } - SCMutexUnlock(&f->files_m); - - SCReturn; -} diff --git a/src/flow-file.h b/src/flow-file.h index 45fa6b059d..9ecd93a383 100644 --- a/src/flow-file.h +++ b/src/flow-file.h @@ -28,125 +28,4 @@ #include "flow.h" #include "util-hash.h" -#define FLOW_FILE_TRUNCATED 0x01 -#define FLOW_FILE_NOSTORE 0x02 - -typedef enum FlowFileState_ { - FLOWFILE_STATE_NONE = 0, /**< no state */ - FLOWFILE_STATE_OPENED, /**< flow file is opened */ - FLOWFILE_STATE_CLOSED, /**< flow file is completed, - there will be no more data. */ - FLOWFILE_STATE_TRUNCATED, /**< flow file is not complete, but - there will be no more data. */ - FLOWFILE_STATE_STORED, /**< all fully written to disk */ - FLOWFILE_STATE_ERROR, /**< file is in an error state */ - FLOWFILE_STATE_MAX -} FlowFileState; - -typedef struct FlowFileData_ { - uint8_t *data; - uint32_t len; - int stored; /* true if this chunk has been stored already - * false otherwise */ - struct FlowFileData_ *next; -} FlowFileData; - -typedef struct FlowFile_ { - int16_t store; /**< need storing? 0: no, 1: yes, -1: won't */ - uint16_t txid; /**< tx this file is part of */ - uint8_t *name; - uint16_t name_len; - int16_t state; - int fd; /**< file discriptor for storing files */ - uint64_t size; /**< size tracked so far */ - const char *magic; - FlowFileData *chunks_head; - FlowFileData *chunks_tail; - struct FlowFile_ *next; -} FlowFile; - -typedef struct FlowFileContainer_ { - FlowFile *head; - FlowFile *tail; -} FlowFileContainer; - -FlowFileContainer *FlowFileContainerAlloc(); -void FlowFileContainerFree(FlowFileContainer *); - -void FlowFileContainerRecycle(FlowFileContainer *); - -void FlowFileContainerAdd(FlowFileContainer *, FlowFile *); - -/** - * \brief Open a new FlowFile - * - * \param ffc flow container - * \param name filename character array - * \param name_len filename len - * \param data initial data - * \param data_len initial data len - * \param flags open flags - * - * \retval ff flowfile object - * - * \note filename is not a string, so it's not nul terminated. - */ -FlowFile *FlowFileOpenFile(FlowFileContainer *, uint8_t *name, uint16_t name_len, - uint8_t *data, uint32_t data_len, uint8_t flags); -/** - * \brief Close a FlowFile - * - * \param ffc the container - * \param data final data if any - * \param data_len data len if any - * \param flags flags - * - * \retval 0 ok - * \retval -1 error - */ -int FlowFileCloseFile(FlowFileContainer *, uint8_t *data, uint32_t data_len, uint8_t flags); - -/** - * \brief Store a chunk of file data in the flow. The open "flowfile" - * will be used. - * - * \param ffc the container - * \param data data chunk - * \param data_len data chunk len - * - * \retval 0 ok - * \retval -1 error - */ -int FlowFileAppendData(FlowFileContainer *, uint8_t *data, uint32_t data_len); - -/** - * \brief Tag a file for storing - * - * \param ff The file to store - */ -int FlowFileStore(FlowFile *); - -/** - * \brief Set the TX id for a file - * - * \param ff The file to store - * \param txid the tx id - */ -int FlowFileSetTx(FlowFile *, uint16_t txid); - -/** - * \brief disable file storage for a flow - * - * \param f *LOCKED* flow - */ -void FlowFileDisableStoring(struct Flow_ *); - -/** - * \brief disable file storing for a transaction - * - * \param f flow - * \param tx_id transaction id - */ -void FlowFileDisableStoringForTransaction(struct Flow_ *, uint16_t); - #endif /* __FLOW_FILE_H__ */ diff --git a/src/flow-util.h b/src/flow-util.h index b3bbe410fa..c517540099 100644 --- a/src/flow-util.h +++ b/src/flow-util.h @@ -62,8 +62,6 @@ (f)->lnext = NULL; \ (f)->lprev = NULL; \ RESET_COUNTERS((f)); \ - (f)->files = NULL; \ - SCMutexInit(&(f)->files_m, NULL); \ } while (0) /** \brief macro to recycle a flow before it goes into the spare queue for reuse. @@ -94,10 +92,6 @@ GenericVarFree((f)->flowvar); \ (f)->flowvar = NULL; \ RESET_COUNTERS((f)); \ - SCMutexLock(&(f)->files_m); \ - if ((f)->files != NULL) \ - FlowFileContainerRecycle((f)->files); \ - SCMutexUnlock(&(f)->files_m); \ } while(0) #define FLOW_DESTROY(f) do { \ @@ -112,13 +106,6 @@ GenericVarFree((f)->flowvar); \ SCMutexDestroy(&(f)->de_state_m); \ (f)->tag_list = NULL; \ - SCMutexLock(&(f)->files_m); \ - if ((f)->files != NULL) {\ - FlowFileContainerFree((f)->files); \ - SCMutexDestroy(&(f)->files_m); \ - (f)->files = NULL; \ - } \ - SCMutexUnlock(&(f)->files_m); \ } while(0) Flow *FlowAlloc(void); diff --git a/src/flow.c b/src/flow.c index 70980d0a86..93ce1d3a8e 100644 --- a/src/flow.c +++ b/src/flow.c @@ -41,7 +41,6 @@ #include "flow-queue.h" #include "flow-hash.h" #include "flow-util.h" -#include "flow-file.h" #include "flow-var.h" #include "flow-private.h" #include "flow-timeout.h" diff --git a/src/flow.h b/src/flow.h index 1f47dda0fc..d737216f84 100644 --- a/src/flow.h +++ b/src/flow.h @@ -28,7 +28,6 @@ #include "util-var.h" #include "util-atomic.h" #include "detect-tag.h" -#include "flow-file.h" #define FLOW_QUIET TRUE #define FLOW_VERBOSE FALSE @@ -314,10 +313,6 @@ typedef struct Flow_ uint32_t tosrcpktcnt; uint64_t bytecnt; #endif - - FlowFileContainer *files; - SCMutex files_m; - } Flow; enum { diff --git a/src/log-file.c b/src/log-file.c index 4dded34179..679a8630ec 100644 --- a/src/log-file.c +++ b/src/log-file.c @@ -33,6 +33,8 @@ #include "threads.h" +#include "app-layer-parser.h" + #include "util-print.h" #include "util-unittest.h" #include "util-privs.h" @@ -100,20 +102,20 @@ TmEcode LogFileLogIPv4(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, P SCReturnInt(TM_ECODE_OK); } - SCMutexLock(&p->flow->files_m); + SCMutexLock(&p->flow->m); - FlowFileContainer *ffc = p->flow->files; + FileContainer *ffc = AppLayerGetFilesFromFlow(p->flow); if (ffc != NULL) { - FlowFile *ff; + File *ff; for (ff = ffc->head; ff != NULL; ff = ff->next) { - if (ff->state == FLOWFILE_STATE_STORED) + if (ff->state == FILE_STATE_STORED) continue; if (ff->store != 1) { continue; } - FlowFileData *ffd; + FileData *ffd; for (ffd = ff->chunks_head; ffd != NULL; ffd = ffd->next) { if (ffd->stored == 1) continue; @@ -176,12 +178,12 @@ TmEcode LogFileLogIPv4(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, P continue; } - if (ff->state == FLOWFILE_STATE_CLOSED || - ff->state == FLOWFILE_STATE_TRUNCATED || - ff->state == FLOWFILE_STATE_ERROR) + if (ff->state == FILE_STATE_CLOSED || + ff->state == FILE_STATE_TRUNCATED || + ff->state == FILE_STATE_ERROR) { if (ffd->next == NULL) { - ff->state = FLOWFILE_STATE_STORED; + ff->state = FILE_STATE_STORED; close(ff->fd); ff->fd = -1; } @@ -191,7 +193,7 @@ TmEcode LogFileLogIPv4(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, P } } } - SCMutexUnlock(&p->flow->files_m); + SCMutexUnlock(&p->flow->m); SCReturnInt(TM_ECODE_OK); }