From 9878eca0860e3b59a48030f2a894fb51e13c4eae Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Thu, 1 Dec 2011 18:08:15 +0100 Subject: [PATCH] file handling: expand filestore keyword Filestore keyword by default (... filestore; ... ) marks only the file in the same direction as the rule match for storing. This makes sense when inspecting individual files (filemagic, filename, etc) but not so much when looking at suspicious file requests, where the actual file is in the response. The filestore keyword now takes 2 optional options: filestore:,; By default the direction is "same as rule match", and scope is "currently inspected file". For direction the following values are possible: "request" and "to_server", "response" and "to_client", "both". For scope the following values are possible: "tx" for all files in the current HTTP/1.1 transation, "ssn" and "flow" for all files in the session/flow. For the above case, where a suspious request should lead to a response file download, this would work: alert http ... content:"/suspicious/"; http_uri; filestore:response; ... --- src/app-layer-htp-file.c | 12 ++- src/app-layer-htp.h | 18 ++-- src/detect-engine-file.c | 24 ++++- src/detect-filestore.c | 212 ++++++++++++++++++++++++++++++++++++++- src/detect-filestore.h | 14 +++ src/util-file.c | 31 +++++- src/util-file.h | 4 + 7 files changed, 301 insertions(+), 14 deletions(-) diff --git a/src/app-layer-htp-file.c b/src/app-layer-htp-file.c index a3adf3bab7..f54b5d9e43 100644 --- a/src/app-layer-htp-file.c +++ b/src/app-layer-htp-file.c @@ -97,6 +97,11 @@ int HTPFileOpen(HtpState *s, uint8_t *filename, uint16_t filename_len, } files = s->files_tc; + + if (s->flags & HTP_FLAG_STORE_FILES_TS || + (s->flags & HTP_FLAG_STORE_FILES_TX_TS && txid == s->store_tx_id)) { + flags |= FILE_STORE; + } } else { if (s->files_ts == NULL) { s->files_ts = FileContainerAlloc(); @@ -107,6 +112,11 @@ int HTPFileOpen(HtpState *s, uint8_t *filename, uint16_t filename_len, } files = s->files_ts; + + if (s->flags & HTP_FLAG_STORE_FILES_TC || + (s->flags & HTP_FLAG_STORE_FILES_TX_TC && txid == s->store_tx_id)) { + flags |= FILE_STORE; + } } /* if the previous file is in the same txid, we @@ -117,7 +127,7 @@ int HTPFileOpen(HtpState *s, uint8_t *filename, uint16_t filename_len, DeStateResetFileInspection(s->f, direction); } - if (s->f->flags & FLOW_FILE_NO_STORE) { + if (!(flags & FILE_STORE) && s->f->flags & FLOW_FILE_NO_STORE) { flags |= FILE_NOSTORE; } if (s->f->flags & FLOW_FILE_NO_MAGIC) { diff --git a/src/app-layer-htp.h b/src/app-layer-htp.h index 36bc493df4..5dd3912097 100644 --- a/src/app-layer-htp.h +++ b/src/app-layer-htp.h @@ -45,18 +45,23 @@ /** a boundary should be smaller in size */ #define HTP_BOUNDARY_MAX 200U -#define HTP_FLAG_STATE_OPEN 0x01 /**< Flag to indicate that HTTP +#define HTP_FLAG_STATE_OPEN 0x0001 /**< Flag to indicate that HTTP connection is open */ -#define HTP_FLAG_STATE_CLOSED 0x02 /**< Flag to indicate that HTTP +#define HTP_FLAG_STATE_CLOSED 0x0002 /**< Flag to indicate that HTTP connection is closed */ -#define HTP_FLAG_STATE_DATA 0x04 /**< Flag to indicate that HTTP +#define HTP_FLAG_STATE_DATA 0x0004 /**< Flag to indicate that HTTP connection needs more data */ -#define HTP_FLAG_STATE_ERROR 0x08 /**< Flag to indicate that an error +#define HTP_FLAG_STATE_ERROR 0x0008 /**< Flag to indicate that an error has been occured on HTTP connection */ -#define HTP_FLAG_NEW_BODY_SET 0x10 /**< Flag to indicate that HTTP +#define HTP_FLAG_NEW_BODY_SET 0x0010 /**< Flag to indicate that HTTP has parsed a new body (for pcre) */ +#define HTP_FLAG_STORE_FILES_TS 0x0020 +#define HTP_FLAG_STORE_FILES_TC 0x0040 +#define HTP_FLAG_STORE_FILES_TX_TS 0x0080 +#define HTP_FLAG_STORE_FILES_TX_TC 0x0100 + enum { HTP_BODY_NONE = 0, /**< Flag to indicate the current operation */ @@ -146,9 +151,10 @@ typedef struct HtpState_ { htp_connp_t *connp; /**< Connection parser structure for each connection */ Flow *f; /**< Needed to retrieve the original flow when usin HTPLib callbacks */ - uint8_t flags; + uint16_t flags; uint16_t transaction_cnt; uint16_t transaction_done; + uint16_t store_tx_id; uint32_t request_body_limit; uint32_t response_body_limit; FileContainer *files_ts; diff --git a/src/detect-engine-file.c b/src/detect-engine-file.c index fac18f555e..7e994ace81 100644 --- a/src/detect-engine-file.c +++ b/src/detect-engine-file.c @@ -30,6 +30,8 @@ #include "detect-parse.h" #include "detect-engine-state.h" +#include "detect-filestore.h" + #include "detect-engine-uri.h" #include "detect-engine-hcbd.h" #include "detect-engine-hhd.h" @@ -70,7 +72,9 @@ * * \note flow is not locked at this time */ -static int DetectFileInspect(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Flow *f, Signature *s, FileContainer *ffc) { +static int DetectFileInspect(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, + Flow *f, Signature *s, uint8_t flags, FileContainer *ffc) +{ SigMatch *sm = NULL; int r = 0; int match = 0; @@ -122,7 +126,7 @@ static int DetectFileInspect(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Flo if (sigmatch_table[sm->type].AppLayerMatch != NULL) { match = sigmatch_table[sm->type]. - AppLayerMatch(tv, det_ctx, f, 0, (void *)file, s, sm); + AppLayerMatch(tv, det_ctx, f, flags, (void *)file, s, sm); if (match == 0) { r = 2; break; @@ -147,6 +151,20 @@ static int DetectFileInspect(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Flo /* continue, this file may (or may not) be unable to match * maybe we have more that can :) */ } + } else { + /* if we have a filestore sm with a scope > file (so tx, ssn) we + * run it here */ + sm = s->sm_lists[DETECT_SM_LIST_FILEMATCH]; + if (sm != NULL && sm->next == NULL && sm->type == DETECT_FILESTORE) { + DetectFilestoreData *fd = sm->ctx; + if (fd->scope > FILESTORE_SCOPE_DEFAULT) { + match = sigmatch_table[sm->type]. + AppLayerMatch(tv, det_ctx, f, flags, NULL, s, sm); + if (match == 1) { + r = 1; + } + } + } } if (store_r == 1) @@ -211,7 +229,7 @@ int DetectFileInspectHttp(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Flow * /* inspect files for this transaction */ det_ctx->tx_id = (uint16_t)idx; - match = DetectFileInspect(tv, det_ctx, f, s, ffc); + match = DetectFileInspect(tv, det_ctx, f, s, flags, ffc); if (match == 1) { r = 1; } else if (match == 2) { diff --git a/src/detect-filestore.c b/src/detect-filestore.c index 95cc238962..2fa3b4b829 100644 --- a/src/detect-filestore.c +++ b/src/detect-filestore.c @@ -49,6 +49,14 @@ #include "detect-filestore.h" +/** + * \brief Regex for parsing our flow options + */ +#define PARSE_REGEX "^\\s*([A-z_]+)\\s*(?:,\\s*([A-z_]+))?\\s*(?:,\\s*([A-z_]+))?\\s*$" + +static pcre *parse_regex; +static pcre_extra *parse_regex_study; + int DetectFilestoreMatch (ThreadVars *, DetectEngineThreadCtx *, Flow *, uint8_t, void *, Signature *, SigMatch *); static int DetectFilestoreSetup (DetectEngineCtx *, Signature *, char *); @@ -64,8 +72,28 @@ void DetectFilestoreRegister(void) { sigmatch_table[DETECT_FILESTORE].Free = NULL; sigmatch_table[DETECT_FILESTORE].RegisterTests = NULL; + const char *eb; + int eo; + int opts = 0; + + parse_regex = pcre_compile(PARSE_REGEX, opts, &eb, &eo, NULL); + if(parse_regex == NULL) + { + SCLogError(SC_ERR_PCRE_COMPILE, "pcre compile of \"%s\" failed at offset %" PRId32 ": %s", PARSE_REGEX, eo, eb); + goto error; + } + + parse_regex_study = pcre_study(parse_regex, 0, &eb); + if(eb != NULL) + { + SCLogError(SC_ERR_PCRE_STUDY, "pcre study failed: %s", eb); + goto error; + } SCLogDebug("registering filestore rule option"); return; +error: + /* XXX */ + return; } /** @@ -78,13 +106,96 @@ void DetectFilestoreRegister(void) { * * \retval 0 no match * \retval 1 match + * + * \todo when we start supporting more protocols, the logic in this function + * needs to be put behind a api. */ int DetectFilestoreMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, Flow *f, uint8_t flags, void *state, Signature *s, SigMatch *m) { SCEnter(); - File *file = (File *)state; - FileStore(file); + DetectFilestoreData *filestore = m->ctx; + if (filestore != NULL) { + int this_file = 0; + int this_tx = 0; + int this_flow = 0; + int rule_dir = 0; + int toserver_dir = 0; + int toclient_dir = 0; + + switch (filestore->direction) { + case FILESTORE_DIR_DEFAULT: + rule_dir = 1; + break; + case FILESTORE_DIR_BOTH: + toserver_dir = 1; + toclient_dir = 1; + break; + case FILESTORE_DIR_TOSERVER: + toserver_dir = 1; + break; + case FILESTORE_DIR_TOCLIENT: + toclient_dir = 1; + break; + } + + switch (filestore->scope) { + case FILESTORE_SCOPE_DEFAULT: + if (rule_dir) { + this_file = 1; + } else if (flags & STREAM_TOCLIENT && toclient_dir) { + this_file = 1; + } else if (flags & STREAM_TOSERVER && toserver_dir) { + this_file = 1; + } + break; + case FILESTORE_SCOPE_TX: + this_tx = 1; + break; + case FILESTORE_SCOPE_SSN: + this_flow = 1; + break; + } + + if (this_file) { + File *file = (File *)state; + FileStore(file); + } else if (this_tx) { + /* flag tx all files will be stored */ + if (f->alproto == ALPROTO_HTTP && f->alstate != NULL) { + HtpState *htp_state = f->alstate; + if (toserver_dir) { + htp_state->flags |= HTP_FLAG_STORE_FILES_TX_TS; + FileStoreAllFilesForTx(htp_state->files_ts, det_ctx->tx_id); + } + if (toclient_dir) { + htp_state->flags |= HTP_FLAG_STORE_FILES_TX_TC; + FileStoreAllFilesForTx(htp_state->files_tc, det_ctx->tx_id); + } + htp_state->store_tx_id = det_ctx->tx_id; + } + + } else if (this_flow) { + /* flag flow all files will be stored */ + if (f->alproto == ALPROTO_HTTP && f->alstate != NULL) { + HtpState *htp_state = f->alstate; + if (toserver_dir) { + htp_state->flags |= HTP_FLAG_STORE_FILES_TS; + FileStoreAllFiles(htp_state->files_ts); + } + if (toclient_dir) { + htp_state->flags |= HTP_FLAG_STORE_FILES_TC; + FileStoreAllFiles(htp_state->files_tc); + } + } + } else { + File *file = (File *)state; + FileStore(file); + } + } else { + File *file = (File *)state; + FileStore(file); + } SCReturnInt(1); } @@ -101,14 +212,109 @@ int DetectFilestoreMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, Flow *f */ static int DetectFilestoreSetup (DetectEngineCtx *de_ctx, Signature *s, char *str) { + SCEnter(); + + DetectFilestoreData *fd = NULL; SigMatch *sm = NULL; + char *args[3] = {NULL,NULL,NULL}; +#define MAX_SUBSTRINGS 30 + int ret = 0, res = 0; + int ov[MAX_SUBSTRINGS]; sm = SigMatchAlloc(); if (sm == NULL) goto error; sm->type = DETECT_FILESTORE; - sm->ctx = NULL; + + if (str != NULL && strlen(str) > 0) { + SCLogDebug("str %s", str); + + ret = pcre_exec(parse_regex, parse_regex_study, str, strlen(str), 0, 0, ov, MAX_SUBSTRINGS); + if (ret < 1 || ret > 4) { + SCLogError(SC_ERR_PCRE_MATCH, "parse error, ret %" PRId32 ", string %s", ret, str); + goto error; + } + + if (ret > 1) { + const char *str_ptr; + res = pcre_get_substring((char *)str, ov, MAX_SUBSTRINGS, 1, &str_ptr); + if (res < 0) { + SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed"); + goto error; + } + args[0] = (char *)str_ptr; + + if (ret > 2) { + res = pcre_get_substring((char *)str, ov, MAX_SUBSTRINGS, 2, &str_ptr); + if (res < 0) { + SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed"); + goto error; + } + args[1] = (char *)str_ptr; + } + if (ret > 3) { + res = pcre_get_substring((char *)str, ov, MAX_SUBSTRINGS, 3, &str_ptr); + if (res < 0) { + SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed"); + goto error; + } + args[2] = (char *)str_ptr; + } + } + + fd = SCMalloc(sizeof(DetectFilestoreData)); + if (fd == NULL) + goto error; + memset(fd, 0x00, sizeof(DetectFilestoreData)); + + if (args[0] != NULL) { + SCLogDebug("first arg %s", args[0]); + + if (strcasecmp(args[0], "request") == 0 || + strcasecmp(args[0], "to_server") == 0) + { + fd->direction = FILESTORE_DIR_TOSERVER; + fd->scope = FILESTORE_SCOPE_TX; + } + else if (strcasecmp(args[0], "response") == 0 || + strcasecmp(args[0], "to_client") == 0) + { + fd->direction = FILESTORE_DIR_TOCLIENT; + fd->scope = FILESTORE_SCOPE_TX; + } + else if (strcasecmp(args[0], "both") == 0) + { + fd->direction = FILESTORE_DIR_BOTH; + fd->scope = FILESTORE_SCOPE_TX; + } + } else { + fd->direction = FILESTORE_DIR_DEFAULT; + } + + if (args[1] != NULL) { + SCLogDebug("second arg %s", args[1]); + + if (strcasecmp(args[1], "file") == 0) + { + fd->scope = FILESTORE_SCOPE_DEFAULT; + } else if (strcasecmp(args[1], "tx") == 0) + { + fd->scope = FILESTORE_SCOPE_TX; + } else if (strcasecmp(args[1], "ssn") == 0 || + strcasecmp(args[1], "flow") == 0) + { + fd->scope = FILESTORE_SCOPE_SSN; + } + } else { + if (fd->scope == 0) + fd->scope = FILESTORE_SCOPE_DEFAULT; + } + + sm->ctx = fd; + } else { + sm->ctx = NULL; + } SigMatchAppendSMToList(s, sm, DETECT_SM_LIST_FILEMATCH); diff --git a/src/detect-filestore.h b/src/detect-filestore.h index bacc1e6d4b..f394842fbd 100644 --- a/src/detect-filestore.h +++ b/src/detect-filestore.h @@ -24,6 +24,20 @@ #ifndef __DETECT_FILESTORE_H__ #define __DETECT_FILESTORE_H__ +#define FILESTORE_DIR_DEFAULT 0 /* rule dir */ +#define FILESTORE_DIR_TOSERVER 1 +#define FILESTORE_DIR_TOCLIENT 2 +#define FILESTORE_DIR_BOTH 3 + +#define FILESTORE_SCOPE_DEFAULT 0 /* per file */ +#define FILESTORE_SCOPE_TX 1 /* per transaction */ +#define FILESTORE_SCOPE_SSN 2 /* per flow/ssn */ + +typedef struct DetectFilestoreData_ { + int16_t direction; + int16_t scope; +} DetectFilestoreData; + /* prototypes */ void DetectFilestoreRegister (void); diff --git a/src/util-file.c b/src/util-file.c index 5f5b86dbf7..3b3b68d9cd 100644 --- a/src/util-file.c +++ b/src/util-file.c @@ -434,7 +434,9 @@ File *FileOpenFile(FileContainer *ffc, uint8_t *name, SCReturnPtr(NULL, "File"); } - if (flags & FILE_NOSTORE) { + if (flags & FILE_STORE) { + ff->store = 1; + } else if (flags & FILE_NOSTORE) { ff->store = -1; } if (flags & FILE_NOMAGIC) { @@ -633,3 +635,30 @@ void FileDisableStoringForTransaction(Flow *f, uint8_t direction, uint16_t tx_id SCReturn; } + +void FileStoreAllFilesForTx(FileContainer *fc, uint16_t tx_id) { + File *ptr = NULL; + + SCEnter(); + + if (fc != NULL) { + for (ptr = fc->head; ptr != NULL; ptr = ptr->next) { + if (ptr->txid == tx_id) { + ptr->store = 1; + } + } + } +} + +void FileStoreAllFiles(FileContainer *fc) { + File *ptr = NULL; + + SCEnter(); + + if (fc != NULL) { + for (ptr = fc->head; ptr != NULL; ptr = ptr->next) { + ptr->store = 1; + } + } +} + diff --git a/src/util-file.h b/src/util-file.h index 2d5a82d0e2..ba692107d3 100644 --- a/src/util-file.h +++ b/src/util-file.h @@ -28,6 +28,7 @@ #define FILE_TRUNCATED 0x01 #define FILE_NOSTORE 0x02 #define FILE_NOMAGIC 0x04 +#define FILE_STORE 0x08 typedef enum FileState_ { FILE_STATE_NONE = 0, /**< no state */ @@ -160,4 +161,7 @@ void FilePrune(FileContainer *ffc); void FileForceMagicEnable(void); int FileForceMagic(void); +void FileStoreAllFiles(FileContainer *); +void FileStoreAllFilesForTx(FileContainer *, uint16_t); + #endif /* __UTIL_FILE_H__ */