app-layer/transactions: track files opens and logs

To make sure a transaction is not evicted before all file logging is complete.

rust/src/applayer.rs

@@ -1,4 +1,4 @@
-/* Copyright (C) 2017-2020 Open Information Security Foundation
+/* Copyright (C) 2017-2021 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -59,6 +59,10 @@ pub struct AppLayerTxData {
     /// logger flags for tx logging api
     logged: LoggerFlags,
 
+    /// track file open/logs so we can know how long to keep the tx
+    pub files_opened: u32,
+    pub files_logged: u32,
+
     /// detection engine flags for use by detection engine
     detect_flags_ts: u64,
     detect_flags_tc: u64,
@@ -69,10 +73,18 @@ impl AppLayerTxData {
         Self {
             config: AppLayerTxConfig::new(),
             logged: LoggerFlags::new(),
+            files_opened: 0,
+            files_logged: 0,
             detect_flags_ts: 0,
             detect_flags_tc: 0,
         }
     }
+    pub fn init_files_opened(&mut self) {
+        self.files_opened = 1;
+    }
+    pub fn incr_files_opened(&mut self) {
+        self.files_opened += 1;
+    }
 }
 
 #[macro_export]

src/app-layer-parser.c

@@ -876,6 +876,8 @@ FileContainer *AppLayerParserGetFiles(const Flow *f, const uint8_t direction)
 #define IS_DISRUPTED(flags) ((flags) & (STREAM_DEPTH | STREAM_GAP))
 
 extern int g_detect_disabled;
+extern bool g_file_logger_enabled;
+
 /**
  * \brief remove obsolete (inspected and logged) transactions
  */
@@ -994,6 +996,15 @@ void AppLayerParserTransactionsCleanup(Flow *f)
             }
         }
 
+        /* if file logging is enabled, we keep a tx active while some of the files aren't
+         * logged yet. */
+        if (txd && txd->files_opened && g_file_logger_enabled) {
+            if (txd->files_opened != txd->files_logged) {
+                skipped = true;
+                goto next;
+            }
+        }
+
         /* if we are here, the tx can be freed. */
         p->StateTransactionFree(alstate, i);
         SCLogDebug("%p/%"PRIu64" freed", tx, i);

src/output-file.c

@@ -34,6 +34,8 @@
 #include "util-validate.h"
 #include "util-magic.h"
 
+bool g_file_logger_enabled = false;
+
 typedef struct OutputLoggerThreadStore_ {
     void *thread_data;
     struct OutputLoggerThreadStore_ *next;
@@ -92,9 +94,22 @@ int OutputRegisterFileLogger(LoggerId id, const char *name, FileLogger LogFunc,
     }
 
     SCLogDebug("OutputRegisterFileLogger happy");
+
+    g_file_logger_enabled = true;
     return 0;
 }
 
+static void CloseFile(const Packet *p, Flow *f, File *file)
+{
+    void *txv = AppLayerParserGetTx(p->proto, f->alproto, f->alstate, file->txid);
+    if (txv) {
+        AppLayerTxData *txd = AppLayerParserGetTxData(p->proto, f->alproto, txv);
+        if (txd)
+            txd->files_logged++;
+    }
+    file->flags |= FILE_LOGGED;
+}
+
 static void OutputFileLogFfc(ThreadVars *tv,
         OutputLoggerThreadData *op_thread_data,
         Packet *p,
@@ -144,7 +159,7 @@ static void OutputFileLogFfc(ThreadVars *tv,
                 }
 
                 if (file_logged) {
-                    ff->flags |= FILE_LOGGED;
+                    CloseFile(p, p->flow, ff);
                 }
             }
         }