diff --git a/doc/userguide/file-extraction/file-extraction.rst b/doc/userguide/file-extraction/file-extraction.rst index b010531e79..a85567bf42 100644 --- a/doc/userguide/file-extraction/file-extraction.rst +++ b/doc/userguide/file-extraction/file-extraction.rst @@ -44,8 +44,9 @@ drop dir must be configured. waldo: file.waldo # waldo file to store the file_id across runs max-open-files: 0 # how many files to keep open (O means none) write-meta: yes # write a .meta file if set to yes + include-pid: yes # include the pid in filenames if set to yes. -Each file that is stored with have a name "file.". The id will be reset and files will be overwritten unless the waldo option is used. A "file..meta" file is generated containing file metadata if write-meta is set to yes (default). +Each file that is stored will have a name "file.". The id will be reset and files will be overwritten unless the waldo option is used. A "file..meta" file is generated containing file metadata if write-meta is set to yes (default). If the include-pid option is set, the files will instead have a name "file..", and metafiles will be "file...meta". Files will additionally have the suffix ".tmp" while they are open, which is only removed when they are finalized. :: diff --git a/src/log-filestore.c b/src/log-filestore.c index 8777b4a60f..6b2c0f55e5 100644 --- a/src/log-filestore.c +++ b/src/log-filestore.c @@ -63,6 +63,7 @@ #define MODULE_NAME "LogFilestoreLog" static char g_logfile_base_dir[PATH_MAX] = "/tmp"; +static char g_working_file_suffix[PATH_MAX] = ".tmp"; SC_ATOMIC_DECLARE(uint32_t, filestore_open_file_cnt); /**< Atomic counter of simultaneously open files */ @@ -203,12 +204,25 @@ static uint32_t FileGetMaxOpenFiles(void) return g_file_store_max_open_files; } -static void LogFilestoreLogCreateMetaFile(const Packet *p, const File *ff, char *filename, int ipver) { +static int g_file_store_include_pid = 0; + +static void FileIncludePidEnable(void) +{ + g_file_store_include_pid = 1; +} + +static int FileIncludePid(void) +{ + return g_file_store_include_pid; +} + +static void LogFilestoreLogCreateMetaFile(const Packet *p, const File *ff, char *base_filename, int ipver) { if (!FileWriteMeta()) return; char metafilename[PATH_MAX] = ""; - snprintf(metafilename, sizeof(metafilename), "%s.meta", filename); + snprintf(metafilename, sizeof(metafilename), "%s.meta%s", base_filename, + g_working_file_suffix); FILE *fp = fopen(metafilename, "w+"); if (fp != NULL) { char timebuf[64]; @@ -279,15 +293,19 @@ static void LogFilestoreLogCreateMetaFile(const Packet *p, const File *ff, char static void LogFilestoreLogCloseMetaFile(const File *ff) { - if (!FileWriteMeta()) - return; - - char filename[PATH_MAX] = ""; - snprintf(filename, sizeof(filename), "%s/file.%u", - g_logfile_base_dir, ff->file_store_id); - char metafilename[PATH_MAX] = ""; - snprintf(metafilename, sizeof(metafilename), "%s.meta", filename); - FILE *fp = fopen(metafilename, "a"); + char pid_expression[PATH_MAX] = ""; + if (FileIncludePid()) + snprintf(pid_expression, sizeof(pid_expression), ".%d", getpid()); + char final_filename[PATH_MAX] = ""; + snprintf(final_filename, sizeof(final_filename), "%s/file%s.%u", + g_logfile_base_dir, pid_expression, ff->file_store_id); + char final_metafilename[PATH_MAX] = ""; + snprintf(final_metafilename, sizeof(final_metafilename), + "%s.meta", final_filename); + char working_metafilename[PATH_MAX] = ""; + snprintf(working_metafilename, sizeof(working_metafilename), + "%s%s", final_metafilename, g_working_file_suffix); + FILE *fp = fopen(working_metafilename, "a"); if (fp != NULL) { #ifdef HAVE_MAGIC fprintf(fp, "MAGIC: %s\n", @@ -337,7 +355,39 @@ static void LogFilestoreLogCloseMetaFile(const File *ff) fclose(fp); } else { - SCLogInfo("opening %s failed: %s", metafilename, strerror(errno)); + SCLogInfo("opening %s failed: %s", working_metafilename, + strerror(errno)); + } +} + +static void LogFilestoreFinalizeFiles(const File *ff) { + char pid_expression[PATH_MAX] = ""; + if (FileIncludePid()) + snprintf(pid_expression, sizeof(pid_expression), ".%d", getpid()); + char final_filename[PATH_MAX] = ""; + snprintf(final_filename, sizeof(final_filename), "%s/file%s.%u", + g_logfile_base_dir, pid_expression, ff->file_store_id); + char working_filename[PATH_MAX] = ""; + snprintf(working_filename, sizeof(working_filename), "%s%s", + final_filename, g_working_file_suffix); + if (rename(working_filename, final_filename) != 0) { + SCLogWarning(SC_WARN_RENAMING_FILE, "renaming file %s to %s failed", + working_filename, final_filename); + return; + } + if (FileWriteMeta()) { + LogFilestoreLogCloseMetaFile(ff); + char final_metafilename[PATH_MAX] = ""; + snprintf(final_metafilename, sizeof(final_metafilename), + "%s.meta", final_filename); + char working_metafilename[PATH_MAX] = ""; + snprintf(working_metafilename, sizeof(working_metafilename), + "%s%s", final_metafilename, g_working_file_suffix); + if (rename(working_metafilename, final_metafilename) != 0) { + SCLogWarning(SC_WARN_RENAMING_FILE, + "renaming metafile %s to %s failed", working_metafilename, + final_metafilename); + } } } @@ -365,14 +415,20 @@ static int LogFilestoreLogger(ThreadVars *tv, void *thread_data, const Packet *p SCLogDebug("ff %p, data %p, data_len %u", ff, data, data_len); - snprintf(filename, sizeof(filename), "%s/file.%u", - g_logfile_base_dir, ff->file_store_id); + char pid_expression[PATH_MAX] = ""; + if (FileIncludePid()) + snprintf(pid_expression, sizeof(pid_expression), ".%d", getpid()); + char base_filename[PATH_MAX] = ""; + snprintf(base_filename, sizeof(base_filename), "%s/file%s.%u", + g_logfile_base_dir, pid_expression, ff->file_store_id); + snprintf(filename, sizeof(filename), "%s%s", base_filename, + g_working_file_suffix); if (flags & OUTPUT_FILEDATA_FLAG_OPEN) { aft->file_cnt++; /* create a .meta file that contains time, src/dst/sp/dp/proto */ - LogFilestoreLogCreateMetaFile(p, ff, filename, ipver); + LogFilestoreLogCreateMetaFile(p, ff, base_filename, ipver); if (SC_ATOMIC_GET(filestore_open_file_cnt) < FileGetMaxOpenFiles()) { SC_ATOMIC_ADD(filestore_open_file_cnt, 1); @@ -425,7 +481,7 @@ static int LogFilestoreLogger(ThreadVars *tv, void *thread_data, const Packet *p ff->fd = -1; SC_ATOMIC_SUB(filestore_open_file_cnt, 1); } - LogFilestoreLogCloseMetaFile(ff); + LogFilestoreFinalizeFiles(ff); } return 0; @@ -598,6 +654,12 @@ static OutputCtx *LogFilestoreLogInitCtx(ConfNode *conf) } } + const char *include_pid = ConfNodeLookupChildValue(conf, "include-pid"); + if (include_pid != NULL && ConfValIsTrue(include_pid)) { + FileIncludePidEnable(); + SCLogInfo("enabling pid as a part of all file names"); + } + SCReturnPtr(output_ctx, "OutputCtx"); } diff --git a/src/util-error.c b/src/util-error.c index 1e4075cd83..15410359c9 100644 --- a/src/util-error.c +++ b/src/util-error.c @@ -343,6 +343,7 @@ const char * SCErrorToString(SCError err) CASE_CODE (SC_ERR_NO_REDIS_ASYNC); CASE_CODE (SC_ERR_REDIS_CONFIG); CASE_CODE (SC_ERR_BYPASS_NOT_SUPPORTED); + CASE_CODE (SC_WARN_RENAMING_FILE); } return "UNKNOWN_ERROR"; diff --git a/src/util-error.h b/src/util-error.h index cc73272a1f..12ce9aa19b 100644 --- a/src/util-error.h +++ b/src/util-error.h @@ -332,7 +332,8 @@ typedef enum { SC_WARN_EVENT_DROPPED, SC_ERR_NO_REDIS_ASYNC, SC_ERR_REDIS_CONFIG, - SC_ERR_BYPASS_NOT_SUPPORTED + SC_ERR_BYPASS_NOT_SUPPORTED, + SC_WARN_RENAMING_FILE } SCError; const char *SCErrorToString(SCError); diff --git a/suricata.yaml.in b/suricata.yaml.in index 04055abf05..e0669e6755 100644 --- a/suricata.yaml.in +++ b/suricata.yaml.in @@ -425,7 +425,11 @@ outputs: # # The files are stored to the log-dir in a format "file." where is # an incrementing number starting at 1. For each file "file." a meta - # file "file..meta" is created. + # file "file..meta" is created. Before they are finalized, they will + # have a ".tmp" suffix to indicate that they are still being processed. + # + # If include-pid is yes, then the files are instead "file..", with + # meta files named as "file...meta" # # File extraction depends on a lot of things to be fully done: # - file-store stream-depth. For optimal results, set this to 0 (unlimited) @@ -449,6 +453,7 @@ outputs: # remain open for filestore by Suricata. Default value is 0 which # means files get closed after each write #max-open-files: 1000 + include-pid: no # set to yes to include pid in file names # output module to log files tracked in a easily parsable json format - file-log: