diff --git a/src/app-layer-htp-file.c b/src/app-layer-htp-file.c index a3adf3bab7..f54b5d9e43 100644 --- a/src/app-layer-htp-file.c +++ b/src/app-layer-htp-file.c @@ -97,6 +97,11 @@ int HTPFileOpen(HtpState *s, uint8_t *filename, uint16_t filename_len, } files = s->files_tc; + + if (s->flags & HTP_FLAG_STORE_FILES_TS || + (s->flags & HTP_FLAG_STORE_FILES_TX_TS && txid == s->store_tx_id)) { + flags |= FILE_STORE; + } } else { if (s->files_ts == NULL) { s->files_ts = FileContainerAlloc(); @@ -107,6 +112,11 @@ int HTPFileOpen(HtpState *s, uint8_t *filename, uint16_t filename_len, } files = s->files_ts; + + if (s->flags & HTP_FLAG_STORE_FILES_TC || + (s->flags & HTP_FLAG_STORE_FILES_TX_TC && txid == s->store_tx_id)) { + flags |= FILE_STORE; + } } /* if the previous file is in the same txid, we @@ -117,7 +127,7 @@ int HTPFileOpen(HtpState *s, uint8_t *filename, uint16_t filename_len, DeStateResetFileInspection(s->f, direction); } - if (s->f->flags & FLOW_FILE_NO_STORE) { + if (!(flags & FILE_STORE) && s->f->flags & FLOW_FILE_NO_STORE) { flags |= FILE_NOSTORE; } if (s->f->flags & FLOW_FILE_NO_MAGIC) { diff --git a/src/app-layer-htp.h b/src/app-layer-htp.h index 36bc493df4..5dd3912097 100644 --- a/src/app-layer-htp.h +++ b/src/app-layer-htp.h @@ -45,18 +45,23 @@ /** a boundary should be smaller in size */ #define HTP_BOUNDARY_MAX 200U -#define HTP_FLAG_STATE_OPEN 0x01 /**< Flag to indicate that HTTP +#define HTP_FLAG_STATE_OPEN 0x0001 /**< Flag to indicate that HTTP connection is open */ -#define HTP_FLAG_STATE_CLOSED 0x02 /**< Flag to indicate that HTTP +#define HTP_FLAG_STATE_CLOSED 0x0002 /**< Flag to indicate that HTTP connection is closed */ -#define HTP_FLAG_STATE_DATA 0x04 /**< Flag to indicate that HTTP +#define HTP_FLAG_STATE_DATA 0x0004 /**< Flag to indicate that HTTP connection needs more data */ -#define HTP_FLAG_STATE_ERROR 0x08 /**< Flag to indicate that an error +#define HTP_FLAG_STATE_ERROR 0x0008 /**< Flag to indicate that an error has been occured on HTTP connection */ -#define HTP_FLAG_NEW_BODY_SET 0x10 /**< Flag to indicate that HTTP +#define HTP_FLAG_NEW_BODY_SET 0x0010 /**< Flag to indicate that HTTP has parsed a new body (for pcre) */ +#define HTP_FLAG_STORE_FILES_TS 0x0020 +#define HTP_FLAG_STORE_FILES_TC 0x0040 +#define HTP_FLAG_STORE_FILES_TX_TS 0x0080 +#define HTP_FLAG_STORE_FILES_TX_TC 0x0100 + enum { HTP_BODY_NONE = 0, /**< Flag to indicate the current operation */ @@ -146,9 +151,10 @@ typedef struct HtpState_ { htp_connp_t *connp; /**< Connection parser structure for each connection */ Flow *f; /**< Needed to retrieve the original flow when usin HTPLib callbacks */ - uint8_t flags; + uint16_t flags; uint16_t transaction_cnt; uint16_t transaction_done; + uint16_t store_tx_id; uint32_t request_body_limit; uint32_t response_body_limit; FileContainer *files_ts; diff --git a/src/detect-engine-file.c b/src/detect-engine-file.c index fac18f555e..7e994ace81 100644 --- a/src/detect-engine-file.c +++ b/src/detect-engine-file.c @@ -30,6 +30,8 @@ #include "detect-parse.h" #include "detect-engine-state.h" +#include "detect-filestore.h" + #include "detect-engine-uri.h" #include "detect-engine-hcbd.h" #include "detect-engine-hhd.h" @@ -70,7 +72,9 @@ * * \note flow is not locked at this time */ -static int DetectFileInspect(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Flow *f, Signature *s, FileContainer *ffc) { +static int DetectFileInspect(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, + Flow *f, Signature *s, uint8_t flags, FileContainer *ffc) +{ SigMatch *sm = NULL; int r = 0; int match = 0; @@ -122,7 +126,7 @@ static int DetectFileInspect(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Flo if (sigmatch_table[sm->type].AppLayerMatch != NULL) { match = sigmatch_table[sm->type]. - AppLayerMatch(tv, det_ctx, f, 0, (void *)file, s, sm); + AppLayerMatch(tv, det_ctx, f, flags, (void *)file, s, sm); if (match == 0) { r = 2; break; @@ -147,6 +151,20 @@ static int DetectFileInspect(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Flo /* continue, this file may (or may not) be unable to match * maybe we have more that can :) */ } + } else { + /* if we have a filestore sm with a scope > file (so tx, ssn) we + * run it here */ + sm = s->sm_lists[DETECT_SM_LIST_FILEMATCH]; + if (sm != NULL && sm->next == NULL && sm->type == DETECT_FILESTORE) { + DetectFilestoreData *fd = sm->ctx; + if (fd->scope > FILESTORE_SCOPE_DEFAULT) { + match = sigmatch_table[sm->type]. + AppLayerMatch(tv, det_ctx, f, flags, NULL, s, sm); + if (match == 1) { + r = 1; + } + } + } } if (store_r == 1) @@ -211,7 +229,7 @@ int DetectFileInspectHttp(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Flow * /* inspect files for this transaction */ det_ctx->tx_id = (uint16_t)idx; - match = DetectFileInspect(tv, det_ctx, f, s, ffc); + match = DetectFileInspect(tv, det_ctx, f, s, flags, ffc); if (match == 1) { r = 1; } else if (match == 2) { diff --git a/src/detect-filestore.c b/src/detect-filestore.c index 95cc238962..2fa3b4b829 100644 --- a/src/detect-filestore.c +++ b/src/detect-filestore.c @@ -49,6 +49,14 @@ #include "detect-filestore.h" +/** + * \brief Regex for parsing our flow options + */ +#define PARSE_REGEX "^\\s*([A-z_]+)\\s*(?:,\\s*([A-z_]+))?\\s*(?:,\\s*([A-z_]+))?\\s*$" + +static pcre *parse_regex; +static pcre_extra *parse_regex_study; + int DetectFilestoreMatch (ThreadVars *, DetectEngineThreadCtx *, Flow *, uint8_t, void *, Signature *, SigMatch *); static int DetectFilestoreSetup (DetectEngineCtx *, Signature *, char *); @@ -64,8 +72,28 @@ void DetectFilestoreRegister(void) { sigmatch_table[DETECT_FILESTORE].Free = NULL; sigmatch_table[DETECT_FILESTORE].RegisterTests = NULL; + const char *eb; + int eo; + int opts = 0; + + parse_regex = pcre_compile(PARSE_REGEX, opts, &eb, &eo, NULL); + if(parse_regex == NULL) + { + SCLogError(SC_ERR_PCRE_COMPILE, "pcre compile of \"%s\" failed at offset %" PRId32 ": %s", PARSE_REGEX, eo, eb); + goto error; + } + + parse_regex_study = pcre_study(parse_regex, 0, &eb); + if(eb != NULL) + { + SCLogError(SC_ERR_PCRE_STUDY, "pcre study failed: %s", eb); + goto error; + } SCLogDebug("registering filestore rule option"); return; +error: + /* XXX */ + return; } /** @@ -78,13 +106,96 @@ void DetectFilestoreRegister(void) { * * \retval 0 no match * \retval 1 match + * + * \todo when we start supporting more protocols, the logic in this function + * needs to be put behind a api. */ int DetectFilestoreMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, Flow *f, uint8_t flags, void *state, Signature *s, SigMatch *m) { SCEnter(); - File *file = (File *)state; - FileStore(file); + DetectFilestoreData *filestore = m->ctx; + if (filestore != NULL) { + int this_file = 0; + int this_tx = 0; + int this_flow = 0; + int rule_dir = 0; + int toserver_dir = 0; + int toclient_dir = 0; + + switch (filestore->direction) { + case FILESTORE_DIR_DEFAULT: + rule_dir = 1; + break; + case FILESTORE_DIR_BOTH: + toserver_dir = 1; + toclient_dir = 1; + break; + case FILESTORE_DIR_TOSERVER: + toserver_dir = 1; + break; + case FILESTORE_DIR_TOCLIENT: + toclient_dir = 1; + break; + } + + switch (filestore->scope) { + case FILESTORE_SCOPE_DEFAULT: + if (rule_dir) { + this_file = 1; + } else if (flags & STREAM_TOCLIENT && toclient_dir) { + this_file = 1; + } else if (flags & STREAM_TOSERVER && toserver_dir) { + this_file = 1; + } + break; + case FILESTORE_SCOPE_TX: + this_tx = 1; + break; + case FILESTORE_SCOPE_SSN: + this_flow = 1; + break; + } + + if (this_file) { + File *file = (File *)state; + FileStore(file); + } else if (this_tx) { + /* flag tx all files will be stored */ + if (f->alproto == ALPROTO_HTTP && f->alstate != NULL) { + HtpState *htp_state = f->alstate; + if (toserver_dir) { + htp_state->flags |= HTP_FLAG_STORE_FILES_TX_TS; + FileStoreAllFilesForTx(htp_state->files_ts, det_ctx->tx_id); + } + if (toclient_dir) { + htp_state->flags |= HTP_FLAG_STORE_FILES_TX_TC; + FileStoreAllFilesForTx(htp_state->files_tc, det_ctx->tx_id); + } + htp_state->store_tx_id = det_ctx->tx_id; + } + + } else if (this_flow) { + /* flag flow all files will be stored */ + if (f->alproto == ALPROTO_HTTP && f->alstate != NULL) { + HtpState *htp_state = f->alstate; + if (toserver_dir) { + htp_state->flags |= HTP_FLAG_STORE_FILES_TS; + FileStoreAllFiles(htp_state->files_ts); + } + if (toclient_dir) { + htp_state->flags |= HTP_FLAG_STORE_FILES_TC; + FileStoreAllFiles(htp_state->files_tc); + } + } + } else { + File *file = (File *)state; + FileStore(file); + } + } else { + File *file = (File *)state; + FileStore(file); + } SCReturnInt(1); } @@ -101,14 +212,109 @@ int DetectFilestoreMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, Flow *f */ static int DetectFilestoreSetup (DetectEngineCtx *de_ctx, Signature *s, char *str) { + SCEnter(); + + DetectFilestoreData *fd = NULL; SigMatch *sm = NULL; + char *args[3] = {NULL,NULL,NULL}; +#define MAX_SUBSTRINGS 30 + int ret = 0, res = 0; + int ov[MAX_SUBSTRINGS]; sm = SigMatchAlloc(); if (sm == NULL) goto error; sm->type = DETECT_FILESTORE; - sm->ctx = NULL; + + if (str != NULL && strlen(str) > 0) { + SCLogDebug("str %s", str); + + ret = pcre_exec(parse_regex, parse_regex_study, str, strlen(str), 0, 0, ov, MAX_SUBSTRINGS); + if (ret < 1 || ret > 4) { + SCLogError(SC_ERR_PCRE_MATCH, "parse error, ret %" PRId32 ", string %s", ret, str); + goto error; + } + + if (ret > 1) { + const char *str_ptr; + res = pcre_get_substring((char *)str, ov, MAX_SUBSTRINGS, 1, &str_ptr); + if (res < 0) { + SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed"); + goto error; + } + args[0] = (char *)str_ptr; + + if (ret > 2) { + res = pcre_get_substring((char *)str, ov, MAX_SUBSTRINGS, 2, &str_ptr); + if (res < 0) { + SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed"); + goto error; + } + args[1] = (char *)str_ptr; + } + if (ret > 3) { + res = pcre_get_substring((char *)str, ov, MAX_SUBSTRINGS, 3, &str_ptr); + if (res < 0) { + SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed"); + goto error; + } + args[2] = (char *)str_ptr; + } + } + + fd = SCMalloc(sizeof(DetectFilestoreData)); + if (fd == NULL) + goto error; + memset(fd, 0x00, sizeof(DetectFilestoreData)); + + if (args[0] != NULL) { + SCLogDebug("first arg %s", args[0]); + + if (strcasecmp(args[0], "request") == 0 || + strcasecmp(args[0], "to_server") == 0) + { + fd->direction = FILESTORE_DIR_TOSERVER; + fd->scope = FILESTORE_SCOPE_TX; + } + else if (strcasecmp(args[0], "response") == 0 || + strcasecmp(args[0], "to_client") == 0) + { + fd->direction = FILESTORE_DIR_TOCLIENT; + fd->scope = FILESTORE_SCOPE_TX; + } + else if (strcasecmp(args[0], "both") == 0) + { + fd->direction = FILESTORE_DIR_BOTH; + fd->scope = FILESTORE_SCOPE_TX; + } + } else { + fd->direction = FILESTORE_DIR_DEFAULT; + } + + if (args[1] != NULL) { + SCLogDebug("second arg %s", args[1]); + + if (strcasecmp(args[1], "file") == 0) + { + fd->scope = FILESTORE_SCOPE_DEFAULT; + } else if (strcasecmp(args[1], "tx") == 0) + { + fd->scope = FILESTORE_SCOPE_TX; + } else if (strcasecmp(args[1], "ssn") == 0 || + strcasecmp(args[1], "flow") == 0) + { + fd->scope = FILESTORE_SCOPE_SSN; + } + } else { + if (fd->scope == 0) + fd->scope = FILESTORE_SCOPE_DEFAULT; + } + + sm->ctx = fd; + } else { + sm->ctx = NULL; + } SigMatchAppendSMToList(s, sm, DETECT_SM_LIST_FILEMATCH); diff --git a/src/detect-filestore.h b/src/detect-filestore.h index bacc1e6d4b..f394842fbd 100644 --- a/src/detect-filestore.h +++ b/src/detect-filestore.h @@ -24,6 +24,20 @@ #ifndef __DETECT_FILESTORE_H__ #define __DETECT_FILESTORE_H__ +#define FILESTORE_DIR_DEFAULT 0 /* rule dir */ +#define FILESTORE_DIR_TOSERVER 1 +#define FILESTORE_DIR_TOCLIENT 2 +#define FILESTORE_DIR_BOTH 3 + +#define FILESTORE_SCOPE_DEFAULT 0 /* per file */ +#define FILESTORE_SCOPE_TX 1 /* per transaction */ +#define FILESTORE_SCOPE_SSN 2 /* per flow/ssn */ + +typedef struct DetectFilestoreData_ { + int16_t direction; + int16_t scope; +} DetectFilestoreData; + /* prototypes */ void DetectFilestoreRegister (void); diff --git a/src/util-file.c b/src/util-file.c index 5f5b86dbf7..3b3b68d9cd 100644 --- a/src/util-file.c +++ b/src/util-file.c @@ -434,7 +434,9 @@ File *FileOpenFile(FileContainer *ffc, uint8_t *name, SCReturnPtr(NULL, "File"); } - if (flags & FILE_NOSTORE) { + if (flags & FILE_STORE) { + ff->store = 1; + } else if (flags & FILE_NOSTORE) { ff->store = -1; } if (flags & FILE_NOMAGIC) { @@ -633,3 +635,30 @@ void FileDisableStoringForTransaction(Flow *f, uint8_t direction, uint16_t tx_id SCReturn; } + +void FileStoreAllFilesForTx(FileContainer *fc, uint16_t tx_id) { + File *ptr = NULL; + + SCEnter(); + + if (fc != NULL) { + for (ptr = fc->head; ptr != NULL; ptr = ptr->next) { + if (ptr->txid == tx_id) { + ptr->store = 1; + } + } + } +} + +void FileStoreAllFiles(FileContainer *fc) { + File *ptr = NULL; + + SCEnter(); + + if (fc != NULL) { + for (ptr = fc->head; ptr != NULL; ptr = ptr->next) { + ptr->store = 1; + } + } +} + diff --git a/src/util-file.h b/src/util-file.h index 2d5a82d0e2..ba692107d3 100644 --- a/src/util-file.h +++ b/src/util-file.h @@ -28,6 +28,7 @@ #define FILE_TRUNCATED 0x01 #define FILE_NOSTORE 0x02 #define FILE_NOMAGIC 0x04 +#define FILE_STORE 0x08 typedef enum FileState_ { FILE_STATE_NONE = 0, /**< no state */ @@ -160,4 +161,7 @@ void FilePrune(FileContainer *ffc); void FileForceMagicEnable(void); int FileForceMagic(void); +void FileStoreAllFiles(FileContainer *); +void FileStoreAllFilesForTx(FileContainer *, uint16_t); + #endif /* __UTIL_FILE_H__ */