From 3f214b506a66e390ecf7821a8ac51cd1c0916ca5 Mon Sep 17 00:00:00 2001 From: Giuseppe Longo Date: Tue, 1 Dec 2015 11:21:24 +0100 Subject: [PATCH] file-store: add depth setting When a rules match and fired filestore we may want to increase the stream reassembly depth for this specific. This add the 'depth' setting in file-store config, which permits to specify how much data we want to reassemble into a stream. --- .../file-extraction/file-extraction.rst | 3 ++- src/detect-filestore.c | 4 +++ src/log-filestore.c | 16 ++++++++++++ src/util-file.c | 25 +++++++++++++++++++ src/util-file.h | 2 ++ suricata.yaml.in | 3 ++- 6 files changed, 51 insertions(+), 2 deletions(-) diff --git a/doc/userguide/file-extraction/file-extraction.rst b/doc/userguide/file-extraction/file-extraction.rst index 001ea525cd..f4bc439ff4 100644 --- a/doc/userguide/file-extraction/file-extraction.rst +++ b/doc/userguide/file-extraction/file-extraction.rst @@ -16,7 +16,7 @@ Settings *stream.checksum_validation* controls whether or not the stream engine rejects packets with invalid checksums. A good idea normally, but the network interface performs checksum offloading a lot of packets may seem to be broken. This setting is enabled by default, and can be disabled by setting to "no". Note that the checksum handling can be controlled per interface, see "checksum_checks" in example configuration. -*stream.reassembly.depth* controls how far into a stream reassembly is done. Beyond this value no reassembly will be done. This means that after this value the HTTP session will no longer be tracked. By default a settings of 1 Megabyte is used. 0 sets it to unlimited. +*file-store.stream-depth* controls how far into a stream reassembly is done. Beyond this value no reassembly will be done. This means that after this value the HTTP session will no longer be tracked. By default a settings of 1 Megabyte is used. 0 sets it to unlimited. If set to no, it is disabled and stream.reassembly.depth is considered. *libhtp.default-config.request-body-limit* / *libhtp.server-config..request-body-limit* controls how much of the HTTP request body is tracked for inspection by the http_client_body keyword, but also used to limit file inspection. A value of 0 means unlimited. @@ -40,6 +40,7 @@ drop dir must be configured. log-dir: files # directory to store the files force-magic: no # force logging magic on all stored files force-md5: no # force logging of md5 checksums + stream-depth: 1mb # reassemble 1mb into a stream, set to no to disable waldo: file.waldo # waldo file to store the file_id across runs Each file that is stored with have a name "file.". The id will be reset and files will be overwritten unless the waldo option is used. diff --git a/src/detect-filestore.c b/src/detect-filestore.c index 3e6957586c..efbeda0dec 100644 --- a/src/detect-filestore.c +++ b/src/detect-filestore.c @@ -198,6 +198,10 @@ int DetectFilestorePostMatch(ThreadVars *t, DetectEngineThreadCtx *det_ctx, Pack #endif } + /* set filestore depth for stream reassembling */ + TcpSession *ssn = (TcpSession *)p->flow->protoctx; + TcpSessionSetReassemblyDepth(ssn, FileReassemblyDepth()); + if (p->flowflags & FLOW_PKT_TOCLIENT) flags |= STREAM_TOCLIENT; else diff --git a/src/log-filestore.c b/src/log-filestore.c index d793d341e2..d57b589246 100644 --- a/src/log-filestore.c +++ b/src/log-filestore.c @@ -46,6 +46,7 @@ #include "util-atomic.h" #include "util-file.h" #include "util-time.h" +#include "util-misc.h" #include "output.h" @@ -484,6 +485,21 @@ static OutputCtx *LogFilestoreLogInitCtx(ConfNode *conf) FileForceHashParseCfg(conf); SCLogInfo("storing files in %s", g_logfile_base_dir); + const char *stream_depth_str = ConfNodeLookupChildValue(conf, "stream-depth"); + if (stream_depth_str != NULL && strcmp(stream_depth_str, "no")) { + uint32_t stream_depth = 0; + if (ParseSizeStringU32(stream_depth_str, + &stream_depth) < 0) { + SCLogError(SC_ERR_SIZE_PARSE, "Error parsing " + "file-store.stream-depth " + "from conf file - %s. Killing engine", + stream_depth_str); + exit(EXIT_FAILURE); + } else { + FileReassemblyDepthEnable(stream_depth); + } + } + SCReturnPtr(output_ctx, "OutputCtx"); } diff --git a/src/util-file.c b/src/util-file.c index 5089210216..f2ffa8a9f0 100644 --- a/src/util-file.c +++ b/src/util-file.c @@ -28,6 +28,7 @@ #include "debug.h" #include "flow.h" #include "stream.h" +#include "stream-tcp.h" #include "runmodes.h" #include "util-hash.h" #include "util-debug.h" @@ -66,6 +67,16 @@ static int g_file_force_sha256 = 0; */ static int g_file_force_tracking = 0; +/** \brief switch to use g_file_store_reassembly_depth + * to reassembly files + */ +static int g_file_store_enable = 0; + +/** \brief stream_config.reassembly_depth equivalent + * for files + */ +static uint32_t g_file_store_reassembly_depth = 0; + /* prototypes */ static void FileFree(File *); @@ -99,6 +110,20 @@ int FileForceFilestore(void) return g_file_force_filestore; } +void FileReassemblyDepthEnable(uint32_t size) +{ + g_file_store_enable = 1; + g_file_store_reassembly_depth = size; +} + +uint32_t FileReassemblyDepth(void) +{ + if (g_file_store_enable == 1) + return g_file_store_reassembly_depth; + else + return stream_config.reassembly_depth; +} + int FileForceMagic(void) { return g_file_force_magic; diff --git a/src/util-file.h b/src/util-file.h index 3b9ee9daee..8e9b1a3c8d 100644 --- a/src/util-file.h +++ b/src/util-file.h @@ -182,6 +182,8 @@ void FilePrune(FileContainer *ffc); void FileForceFilestoreEnable(void); int FileForceFilestore(void); +void FileReassemblyDepthEnable(uint32_t size); +uint32_t FileReassemblyDepth(void); void FileDisableMagic(Flow *f, uint8_t); void FileForceMagicEnable(void); diff --git a/suricata.yaml.in b/suricata.yaml.in index 0920b2da98..9d6f65231c 100644 --- a/suricata.yaml.in +++ b/suricata.yaml.in @@ -394,7 +394,7 @@ outputs: # file "file..meta" is created. # # File extraction depends on a lot of things to be fully done: - # - stream reassembly depth. For optimal results, set this to 0 (unlimited) + # - file-store stream-depth. For optimal results, set this to 0 (unlimited) # - http request / response body sizes. Again set to 0 for optimal results. # - rules that contain the "filestore" keyword. - file-store: @@ -405,6 +405,7 @@ outputs: # sha1 and sha256 #force-hash: [md5] force-filestore: no # force storing of all files + stream-depth: 1mb # reassemble 1mb into a stream, set to no to disable #waldo: file.waldo # waldo file to store the file_id across runs # output module to log files tracked in a easily parsable json format