output/alert: optionally log file_data

This patch adds file data to alerts so an analyst can directly
understand what is the file. It can also be used to do some
detection on the file outside of Suricata without doing file
extraction.

Ticket: #7347

etc/schema.json

@@ -171,6 +171,12 @@
                         "items": {
                             "type": "integer"
                         }
+                    },
+                    "data": {
+                        "type": "string"
+                    },
+                    "offset": {
+                        "type": "integer"
                     }
                 }
             }

src/output-json-alert.c

@@ -67,6 +67,7 @@
 #include "util-validate.h"
 
 #include "action-globals.h"
+#include <stdint.h>
 
 #define MODULE_NAME "JsonAlertLog"
 
@@ -85,6 +86,7 @@
 #define LOG_JSON_WEBSOCKET_PAYLOAD_BASE64 BIT_U16(12)
 #define LOG_JSON_PAYLOAD_LENGTH           BIT_U16(13)
 #define LOG_JSON_REFERENCE                BIT_U16(14)
+#define LOG_JSON_FILE_DATA                BIT_U16(15)
 
 #define METADATA_DEFAULTS ( LOG_JSON_FLOW |                        \
             LOG_JSON_APP_LAYER  |                                  \
@@ -431,7 +433,8 @@ static void AlertAddAppLayer(const Packet *p, JsonBuilder *jb,
     }
 }
 
-static void AlertAddFiles(const Packet *p, JsonBuilder *jb, const uint64_t tx_id)
+static void AlertAddFiles(
+        const Packet *p, JsonBuilder *jb, const uint64_t tx_id, int32_t file_dump_length)
 {
     const uint8_t direction =
             (p->flowflags & FLOW_PKT_TOSERVER) ? STREAM_TOSERVER : STREAM_TOCLIENT;
@@ -452,7 +455,7 @@ static void AlertAddFiles(const Packet *p, JsonBuilder *jb, const uint64_t tx_id
                 jb_open_array(jb, "files");
             }
             jb_start_object(jb);
-            EveFileInfo(jb, file, tx_id, file->flags);
+            EveFileInfo(jb, file, tx_id, file->flags, file_dump_length);
             jb_close(jb);
             file = file->next;
         }
@@ -680,7 +683,10 @@ static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p)
                 }
                 /* including fileinfo data is configured by the metadata setting */
                 if (json_output_ctx->flags & LOG_JSON_RULE_METADATA) {
-                    AlertAddFiles(p, jb, pa->tx_id);
+                    AlertAddFiles(p, jb, pa->tx_id,
+                            json_output_ctx->flags & LOG_JSON_FILE_DATA
+                                    ? json_output_ctx->payload_buffer_size
+                                    : -1);
                 }
             }
 
@@ -945,6 +951,7 @@ static void JsonAlertLogSetupMetadata(AlertJsonOutputCtx *json_output_ctx,
         SetFlag(conf, "websocket-payload", LOG_JSON_WEBSOCKET_PAYLOAD_BASE64, &flags);
         SetFlag(conf, "verdict", LOG_JSON_VERDICT, &flags);
         SetFlag(conf, "payload-length", LOG_JSON_PAYLOAD_LENGTH, &flags);
+        SetFlag(conf, "file-data", LOG_JSON_FILE_DATA, &flags);
 
         /* Check for obsolete flags and warn that they have no effect. */
         static const char *deprecated_flags[] = { "http", "tls", "ssh", "smtp", "dnp3", "app-layer",

src/output-json-file.c

@@ -195,9 +195,9 @@ JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, void *tx,
     if (stored) {
         // the file has just been stored on disk cf OUTPUT_FILEDATA_FLAG_CLOSE
         // but the flag is not set until the loggers have been called
-        EveFileInfo(js, ff, tx_id, ff->flags | FILE_STORED);
+        EveFileInfo(js, ff, tx_id, ff->flags | FILE_STORED, -1);
     } else {
-        EveFileInfo(js, ff, tx_id, ff->flags);
+        EveFileInfo(js, ff, tx_id, ff->flags, -1);
     }
     jb_close(js);
 

src/output-json.c

@@ -29,6 +29,7 @@
 #include "conf.h"
 
 #include "util-debug.h"
+#include "util-streaming-buffer.h"
 #include "util-time.h"
 #include "util-var-name.h"
 #include "util-macset.h"
@@ -121,7 +122,8 @@ json_t *SCJsonString(const char *val)
 /* Default Sensor ID value */
 static int64_t sensor_id = -1; /* -1 = not defined */
 
-void EveFileInfo(JsonBuilder *jb, const File *ff, const uint64_t tx_id, const uint16_t flags)
+void EveFileInfo(JsonBuilder *jb, const File *ff, const uint64_t tx_id, const uint16_t flags,
+        int32_t dump_length)
 {
     jb_set_string_from_bytes(jb, "filename", ff->name, ff->name_len);
 
@@ -179,6 +181,25 @@ void EveFileInfo(JsonBuilder *jb, const File *ff, const uint64_t tx_id, const ui
         jb_set_uint(jb, "end", ff->end);
     }
     jb_set_uint(jb, "tx_id", tx_id);
+
+    if (dump_length >= 0) {
+        const uint8_t *data;
+        uint32_t data_len;
+        uint64_t data_offset;
+        StreamingBufferGetData(ff->sb, &data, &data_len, &data_offset);
+        if (data == NULL || data_len == 0)
+            return;
+        if (dump_length == 0) {
+            jb_set_base64(jb, "data", data, data_len);
+        } else {
+            if (data_len < (uint32_t)dump_length) {
+                jb_set_base64(jb, "data", data, data_len);
+            } else {
+                jb_set_base64(jb, "data", data, dump_length);
+            }
+        }
+        jb_set_uint(jb, "offset", data_offset);
+    }
 }
 
 static void EveAddPacketVars(const Packet *p, JsonBuilder *js_vars)

src/output-json.h

@@ -95,7 +95,8 @@ typedef struct OutputJsonThreadCtx_ {
 json_t *SCJsonString(const char *val);
 
 void CreateEveFlowId(JsonBuilder *js, const Flow *f);
-void EveFileInfo(JsonBuilder *js, const File *file, const uint64_t tx_id, const uint16_t flags);
+void EveFileInfo(JsonBuilder *js, const File *file, const uint64_t tx_id, const uint16_t flags,
+        int32_t file_dump);
 void EveTcpFlags(uint8_t flags, JsonBuilder *js);
 void EvePacket(const Packet *p, JsonBuilder *js, uint32_t max_length);
 JsonBuilder *CreateEveHeader(const Packet *p, enum OutputJsonLogDirection dir,

suricata.yaml.in

@@ -194,6 +194,7 @@ outputs:
             # http-body-printable: yes # Requires metadata; enable dumping of HTTP body in printable format
             # websocket-payload: yes   # Requires metadata; enable dumping of WebSocket Payload in Base64
             # websocket-payload-printable: yes # Requires metadata; enable dumping of WebSocket Payload in printable format
+            # file-data: yes # Requires metadata; enable dumping of the file data in base64 format
 
             # Enable the logging of tagged packets for rules using the
             # "tag" keyword.