Adds options to mark when a file is final.

This takes the form of an option to add the pid of the process to file
names. Additionally, it adds a suffix to the file name to indicate it is
not finalized.

Adding the pid to the file name reduces the likelihood that a file is
overwritten when suricata is unexpectedly killed. The number in the
waldo file is only written out during a clean shutdown. In the event
of an improper shutdown, extracted files will be written using the old
number and existing files with the same name will be overwritten.

Writes extracted files and their metadata to a temporary file suffixed
with '.tmp'. Renames the files when they are completely done being
written. As-is there is no way to know that a file on disk is still
being written to by suricata.

doc/userguide/file-extraction/file-extraction.rst

@@ -44,8 +44,9 @@ drop dir must be configured.
       waldo: file.waldo # waldo file to store the file_id across runs
       max-open-files: 0 # how many files to keep open (O means none)
       write-meta: yes   # write a .meta file if set to yes
+      include-pid: yes  # include the pid in filenames if set to yes.
 
-Each file that is stored with have a name "file.<id>". The id will be reset and files will be overwritten unless the waldo option is used. A "file.<id>.meta" file is generated containing file metadata if write-meta is set to yes (default).
+Each file that is stored will have a name "file.<id>". The id will be reset and files will be overwritten unless the waldo option is used. A "file.<id>.meta" file is generated containing file metadata if write-meta is set to yes (default). If the include-pid option is set, the files will instead have a name "file.<pid>.<id>", and metafiles will be "file.<pid>.<id>.meta". Files will additionally have the suffix ".tmp" while they are open, which is only removed when they are finalized.
 
 
 ::

src/log-filestore.c

@@ -63,6 +63,7 @@
 #define MODULE_NAME "LogFilestoreLog"
 
 static char g_logfile_base_dir[PATH_MAX] = "/tmp";
+static char g_working_file_suffix[PATH_MAX] = ".tmp";
 
 SC_ATOMIC_DECLARE(uint32_t, filestore_open_file_cnt);  /**< Atomic counter of simultaneously open files */
 
@@ -203,12 +204,25 @@ static uint32_t FileGetMaxOpenFiles(void)
     return g_file_store_max_open_files;
 }
 
-static void LogFilestoreLogCreateMetaFile(const Packet *p, const File *ff, char *filename, int ipver) {
+static int g_file_store_include_pid = 0;
+
+static void FileIncludePidEnable(void)
+{
+    g_file_store_include_pid = 1;
+}
+
+static int FileIncludePid(void)
+{
+    return g_file_store_include_pid;
+}
+
+static void LogFilestoreLogCreateMetaFile(const Packet *p, const File *ff, char *base_filename, int ipver) {
     if (!FileWriteMeta())
         return;
 
     char metafilename[PATH_MAX] = "";
-    snprintf(metafilename, sizeof(metafilename), "%s.meta", filename);
+    snprintf(metafilename, sizeof(metafilename), "%s.meta%s", base_filename,
+            g_working_file_suffix);
     FILE *fp = fopen(metafilename, "w+");
     if (fp != NULL) {
         char timebuf[64];
@@ -279,15 +293,19 @@ static void LogFilestoreLogCreateMetaFile(const Packet *p, const File *ff, char
 
 static void LogFilestoreLogCloseMetaFile(const File *ff)
 {
-    if (!FileWriteMeta())
-        return;
-
-    char filename[PATH_MAX] = "";
-    snprintf(filename, sizeof(filename), "%s/file.%u",
-            g_logfile_base_dir, ff->file_store_id);
-    char metafilename[PATH_MAX] = "";
-    snprintf(metafilename, sizeof(metafilename), "%s.meta", filename);
-    FILE *fp = fopen(metafilename, "a");
+    char pid_expression[PATH_MAX] = "";
+    if (FileIncludePid())
+        snprintf(pid_expression, sizeof(pid_expression), ".%d", getpid());
+    char final_filename[PATH_MAX] = "";
+    snprintf(final_filename, sizeof(final_filename), "%s/file%s.%u",
+            g_logfile_base_dir, pid_expression, ff->file_store_id);
+    char final_metafilename[PATH_MAX] = "";
+    snprintf(final_metafilename, sizeof(final_metafilename),
+            "%s.meta", final_filename);
+    char working_metafilename[PATH_MAX] = "";
+    snprintf(working_metafilename, sizeof(working_metafilename),
+            "%s%s", final_metafilename, g_working_file_suffix);
+    FILE *fp = fopen(working_metafilename, "a");
     if (fp != NULL) {
 #ifdef HAVE_MAGIC
         fprintf(fp, "MAGIC:             %s\n",
@@ -337,7 +355,39 @@ static void LogFilestoreLogCloseMetaFile(const File *ff)
 
         fclose(fp);
     } else {
-        SCLogInfo("opening %s failed: %s", metafilename, strerror(errno));
+        SCLogInfo("opening %s failed: %s", working_metafilename,
+                strerror(errno));
+    }
+}
+
+static void LogFilestoreFinalizeFiles(const File *ff) {
+    char pid_expression[PATH_MAX] = "";
+    if (FileIncludePid())
+        snprintf(pid_expression, sizeof(pid_expression), ".%d", getpid());
+    char final_filename[PATH_MAX] = "";
+    snprintf(final_filename, sizeof(final_filename), "%s/file%s.%u",
+            g_logfile_base_dir, pid_expression, ff->file_store_id);
+    char working_filename[PATH_MAX] = "";
+    snprintf(working_filename, sizeof(working_filename), "%s%s",
+            final_filename, g_working_file_suffix);
+    if (rename(working_filename, final_filename) != 0) {
+        SCLogWarning(SC_WARN_RENAMING_FILE, "renaming file %s to %s failed",
+                working_filename, final_filename);
+        return;
+    }
+    if (FileWriteMeta()) {
+        LogFilestoreLogCloseMetaFile(ff);
+        char final_metafilename[PATH_MAX] = "";
+        snprintf(final_metafilename, sizeof(final_metafilename),
+                "%s.meta", final_filename);
+        char working_metafilename[PATH_MAX] = "";
+        snprintf(working_metafilename, sizeof(working_metafilename),
+                "%s%s", final_metafilename, g_working_file_suffix);
+        if (rename(working_metafilename, final_metafilename) != 0) {
+            SCLogWarning(SC_WARN_RENAMING_FILE,
+                    "renaming metafile %s to %s failed", working_metafilename,
+                    final_metafilename);
+        }
     }
 }
 
@@ -365,14 +415,20 @@ static int LogFilestoreLogger(ThreadVars *tv, void *thread_data, const Packet *p
 
     SCLogDebug("ff %p, data %p, data_len %u", ff, data, data_len);
 
-    snprintf(filename, sizeof(filename), "%s/file.%u",
-            g_logfile_base_dir, ff->file_store_id);
+    char pid_expression[PATH_MAX] = "";
+    if (FileIncludePid())
+        snprintf(pid_expression, sizeof(pid_expression), ".%d", getpid());
+    char base_filename[PATH_MAX] = "";
+    snprintf(base_filename, sizeof(base_filename), "%s/file%s.%u",
+            g_logfile_base_dir, pid_expression, ff->file_store_id);
+    snprintf(filename, sizeof(filename), "%s%s", base_filename,
+            g_working_file_suffix);
 
     if (flags & OUTPUT_FILEDATA_FLAG_OPEN) {
         aft->file_cnt++;
 
         /* create a .meta file that contains time, src/dst/sp/dp/proto */
-        LogFilestoreLogCreateMetaFile(p, ff, filename, ipver);
+        LogFilestoreLogCreateMetaFile(p, ff, base_filename, ipver);
 
         if (SC_ATOMIC_GET(filestore_open_file_cnt) < FileGetMaxOpenFiles()) {
             SC_ATOMIC_ADD(filestore_open_file_cnt, 1);
@@ -425,7 +481,7 @@ static int LogFilestoreLogger(ThreadVars *tv, void *thread_data, const Packet *p
             ff->fd = -1;
             SC_ATOMIC_SUB(filestore_open_file_cnt, 1);
         }
-        LogFilestoreLogCloseMetaFile(ff);
+        LogFilestoreFinalizeFiles(ff);
     }
 
     return 0;
@@ -598,6 +654,12 @@ static OutputCtx *LogFilestoreLogInitCtx(ConfNode *conf)
         }
     }
 
+    const char *include_pid = ConfNodeLookupChildValue(conf, "include-pid");
+    if (include_pid != NULL && ConfValIsTrue(include_pid)) {
+        FileIncludePidEnable();
+        SCLogInfo("enabling pid as a part of all file names");
+    }
+
     SCReturnPtr(output_ctx, "OutputCtx");
 }
 

src/util-error.c

@@ -343,6 +343,7 @@ const char * SCErrorToString(SCError err)
         CASE_CODE (SC_ERR_NO_REDIS_ASYNC);
         CASE_CODE (SC_ERR_REDIS_CONFIG);
         CASE_CODE (SC_ERR_BYPASS_NOT_SUPPORTED);
+        CASE_CODE (SC_WARN_RENAMING_FILE);
     }
 
     return "UNKNOWN_ERROR";

src/util-error.h

@@ -332,7 +332,8 @@ typedef enum {
     SC_WARN_EVENT_DROPPED,
     SC_ERR_NO_REDIS_ASYNC,
     SC_ERR_REDIS_CONFIG,
-    SC_ERR_BYPASS_NOT_SUPPORTED
+    SC_ERR_BYPASS_NOT_SUPPORTED,
+    SC_WARN_RENAMING_FILE
 } SCError;
 
 const char *SCErrorToString(SCError);

suricata.yaml.in

@@ -425,7 +425,11 @@ outputs:
   #
   # The files are stored to the log-dir in a format "file.<id>" where <id> is
   # an incrementing number starting at 1. For each file "file.<id>" a meta
-  # file "file.<id>.meta" is created.
+  # file "file.<id>.meta" is created. Before they are finalized, they will
+  # have a ".tmp" suffix to indicate that they are still being processed.
+  #
+  # If include-pid is yes, then the files are instead "file.<pid>.<id>", with
+  # meta files named as "file.<pid>.<id>.meta"
   #
   # File extraction depends on a lot of things to be fully done:
   # - file-store stream-depth. For optimal results, set this to 0 (unlimited)
@@ -449,6 +453,7 @@ outputs:
       # remain open for filestore by Suricata. Default value is 0 which
       # means files get closed after each write
       #max-open-files: 1000
+      include-pid: no # set to yes to include pid in file names
 
   # output module to log files tracked in a easily parsable json format
   - file-log: