output/json: Multi-threaded EVE logging support

This commit modifies the JSON loggers with changes necessary to support
multi-threaded EVE output.

Each "thread-init" function sets up the per-thread log file context for
subsequent calls to the JSON output to buffer function.

src/output-json-alert.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2013-2014 Open Information Security Foundation
+/* Copyright (C) 2013-2020 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -35,6 +35,7 @@
 #include "threadvars.h"
 #include "util-debug.h"
 
+#include "util-logopenfile.h"
 #include "util-misc.h"
 #include "util-unittest.h"
 #include "util-unittest-helper.h"
@@ -716,37 +717,47 @@ static int JsonAlertLogCondition(ThreadVars *tv, const Packet *p)
 
 static TmEcode JsonAlertLogThreadInit(ThreadVars *t, const void *initdata, void **data)
 {
-    JsonAlertLogThread *aft = SCMalloc(sizeof(JsonAlertLogThread));
+    JsonAlertLogThread *aft = SCCalloc(1, sizeof(JsonAlertLogThread));
     if (unlikely(aft == NULL))
         return TM_ECODE_FAILED;
-    memset(aft, 0, sizeof(JsonAlertLogThread));
-    if(initdata == NULL)
+
+    if (initdata == NULL)
     {
         SCLogDebug("Error getting context for EveLogAlert.  \"initdata\" argument NULL");
-        SCFree(aft);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
     aft->json_buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE);
     if (aft->json_buffer == NULL) {
-        SCFree(aft);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
     /** Use the Output Context (file pointer and mutex) */
     AlertJsonOutputCtx *json_output_ctx = ((OutputCtx *)initdata)->data;
-    aft->file_ctx = json_output_ctx->file_ctx;
-    aft->json_output_ctx = json_output_ctx;
 
     aft->payload_buffer = MemBufferCreateNew(json_output_ctx->payload_buffer_size);
     if (aft->payload_buffer == NULL) {
-        MemBufferFree(aft->json_buffer);
-        SCFree(aft);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
+    aft->file_ctx = LogFileEnsureExists(json_output_ctx->file_ctx, t->id);
+    if (!aft->file_ctx) {
+        goto error_exit;
+    }
+
+    aft->json_output_ctx = json_output_ctx;
 
     *data = (void *)aft;
     return TM_ECODE_OK;
+
+error_exit:
+    if (aft->json_buffer != NULL) {
+        MemBufferFree(aft->json_buffer);
+    }
+    if (aft->payload_buffer != NULL) {
+        MemBufferFree(aft->payload_buffer);
+    }
+    SCFree(aft);
+    return TM_ECODE_FAILED;
 }
 
 static TmEcode JsonAlertLogThreadDeinit(ThreadVars *t, void *data)

src/output-json-anomaly.c

@@ -287,23 +287,32 @@ static TmEcode JsonAnomalyLogThreadInit(ThreadVars *t, const void *initdata, voi
 
     if (initdata == NULL) {
         SCLogDebug("Error getting context for EveLogAnomaly.  \"initdata\" argument NULL");
-        SCFree(aft);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
     aft->json_buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE);
     if (aft->json_buffer == NULL) {
-        SCFree(aft);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
     /** Use the Output Context (file pointer and mutex) */
     AnomalyJsonOutputCtx *json_output_ctx = ((OutputCtx *)initdata)->data;
-    aft->file_ctx = json_output_ctx->file_ctx;
+
+    aft->file_ctx = LogFileEnsureExists(json_output_ctx->file_ctx, t->id);
+    if (!aft->file_ctx) {
+        goto error_exit;
+    }
     aft->json_output_ctx = json_output_ctx;
 
     *data = (void *)aft;
     return TM_ECODE_OK;
+
+error_exit:
+    if (aft->json_buffer != NULL) {
+        MemBufferFree(aft->json_buffer);
+    }
+    SCFree(aft);
+    return TM_ECODE_FAILED;
 }
 
 static TmEcode JsonAnomalyLogThreadDeinit(ThreadVars *t, void *data)

src/output-json-common.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2018 Open Information Security Foundation
+/* Copyright (C) 2018-2020 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -85,13 +85,25 @@ TmEcode JsonLogThreadInit(ThreadVars *t, const void *initdata, void **data)
 
     thread->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE);
     if (unlikely(thread->buffer == NULL)) {
-        SCFree(thread);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
     thread->ctx = ((OutputCtx *)initdata)->data;
+    LogFileEnsureExists(thread->ctx->file_ctx, t->id);
+    thread->file_ctx = LogFileEnsureExists(thread->ctx->file_ctx, t->id);
+    if (!thread->file_ctx) {
+        goto error_exit;
+    }
+
     *data = (void *)thread;
     return TM_ECODE_OK;
+
+error_exit:
+    if (thread->buffer) {
+        MemBufferFree(thread->buffer);
+    }
+    SCFree(thread);
+    return TM_ECODE_FAILED;
 }
 
 TmEcode JsonLogThreadDeinit(ThreadVars *t, void *data)

src/output-json-dcerpc.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2017-2018 Open Information Security Foundation
+/* Copyright (C) 2017-2020 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -58,7 +58,7 @@ static int JsonDCERPCLogger(ThreadVars *tv, void *thread_data,
     jb_close(jb);
 
     MemBufferReset(thread->buffer);
-    OutputJsonBuilderBuffer(jb, thread->ctx->file_ctx, &thread->buffer);
+    OutputJsonBuilderBuffer(jb, thread->file_ctx, &thread->buffer);
 
     jb_free(jb);
     return TM_ECODE_OK;

src/output-json-dhcp.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2015 Open Information Security Foundation
+/* Copyright (C) 2015-2020 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -56,6 +56,7 @@ typedef struct LogDHCPFileCtx_ {
 typedef struct LogDHCPLogThread_ {
     LogDHCPFileCtx *dhcplog_ctx;
     MemBuffer      *buffer;
+    LogFileCtx *file_ctx;
 } LogDHCPLogThread;
 
 static int JsonDHCPLogger(ThreadVars *tv, void *thread_data,
@@ -79,7 +80,7 @@ static int JsonDHCPLogger(ThreadVars *tv, void *thread_data,
 
     EveAddCommonOptions(&thread->dhcplog_ctx->cfg, p, f, js);
     MemBufferReset(thread->buffer);
-    OutputJsonBuilderBuffer(js, thread->dhcplog_ctx->file_ctx, &thread->buffer);
+    OutputJsonBuilderBuffer(js, thread->file_ctx, &thread->buffer);
     jb_free(js);
 
     return TM_ECODE_OK;
@@ -133,20 +134,29 @@ static TmEcode JsonDHCPLogThreadInit(ThreadVars *t, const void *initdata, void *
 
     if (initdata == NULL) {
         SCLogDebug("Error getting context for EveLogDHCP.  \"initdata\" is NULL.");
-        SCFree(thread);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
     thread->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE);
     if (unlikely(thread->buffer == NULL)) {
-        SCFree(thread);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
     thread->dhcplog_ctx = ((OutputCtx *)initdata)->data;
-    *data = (void *)thread;
+    thread->file_ctx = LogFileEnsureExists(thread->dhcplog_ctx->file_ctx, t->id);
+    if (!thread->file_ctx) {
+        goto error_exit;
+    }
 
+    *data = (void *)thread;
     return TM_ECODE_OK;
+
+error_exit:
+    if (unlikely(thread->buffer != NULL)) {
+        MemBufferFree(thread->buffer);
+    }
+    SCFree(thread);
+    return TM_ECODE_FAILED;
 }
 
 static TmEcode JsonDHCPLogThreadDeinit(ThreadVars *t, void *data)

src/output-json-dnp3.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2015 Open Information Security Foundation
+/* Copyright (C) 2015-2020 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -51,6 +51,7 @@ typedef struct LogDNP3FileCtx_ {
 } LogDNP3FileCtx;
 
 typedef struct LogDNP3LogThread_ {
+    LogFileCtx *file_ctx;
     LogDNP3FileCtx *dnp3log_ctx;
     MemBuffer      *buffer;
 } LogDNP3LogThread;
@@ -231,7 +232,7 @@ static int JsonDNP3LoggerToServer(ThreadVars *tv, void *thread_data,
         jb_open_object(js, "dnp3");
         JsonDNP3LogRequest(js, tx);
         jb_close(js);
-        OutputJsonBuilderBuffer(js, thread->dnp3log_ctx->file_ctx, &buffer);
+        OutputJsonBuilderBuffer(js, thread->file_ctx, &buffer);
         jb_free(js);
     }
 
@@ -258,7 +259,7 @@ static int JsonDNP3LoggerToClient(ThreadVars *tv, void *thread_data,
         jb_open_object(js, "dnp3");
         JsonDNP3LogResponse(js, tx);
         jb_close(js);
-        OutputJsonBuilderBuffer(js, thread->dnp3log_ctx->file_ctx, &buffer);
+        OutputJsonBuilderBuffer(js, thread->file_ctx, &buffer);
         jb_free(js);
     }
 
@@ -314,20 +315,30 @@ static TmEcode JsonDNP3LogThreadInit(ThreadVars *t, const void *initdata, void *
 
     if (initdata == NULL) {
         SCLogDebug("Error getting context for DNP3.  \"initdata\" is NULL.");
-        SCFree(thread);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
     thread->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE);
     if (unlikely(thread->buffer == NULL)) {
-        SCFree(thread);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
     thread->dnp3log_ctx = ((OutputCtx *)initdata)->data;
+    thread->file_ctx = LogFileEnsureExists(thread->dnp3log_ctx->file_ctx, t->id);
+    if (!thread->file_ctx) {
+        goto error_exit;
+    }
+
     *data = (void *)thread;
 
     return TM_ECODE_OK;
+
+error_exit:
+    if (thread->buffer != NULL) {
+        MemBufferFree(thread->buffer);
+    }
+    SCFree(thread);
+    return TM_ECODE_FAILED;
 }
 
 static TmEcode JsonDNP3LogThreadDeinit(ThreadVars *t, void *data)

src/output-json-dns.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2013 Open Information Security Foundation
+/* Copyright (C) 2007-2020 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -266,6 +266,7 @@ typedef struct LogDnsFileCtx_ {
 typedef struct LogDnsLogThread_ {
     LogDnsFileCtx *dnslog_ctx;
     /** LogFileCtx has the pointer to the file and a mutex to allow multithreading */
+    LogFileCtx *file_ctx;
     MemBuffer *buffer;
 } LogDnsLogThread;
 
@@ -330,7 +331,7 @@ static int JsonDnsLoggerToServer(ThreadVars *tv, void *thread_data,
         jb_close(jb);
 
         MemBufferReset(td->buffer);
-        OutputJsonBuilderBuffer(jb, td->dnslog_ctx->file_ctx, &td->buffer);
+        OutputJsonBuilderBuffer(jb, td->file_ctx, &td->buffer);
         jb_free(jb);
     }
 
@@ -361,7 +362,7 @@ static int JsonDnsLoggerToClient(ThreadVars *tv, void *thread_data,
             rs_dns_log_json_answer(txptr, td->dnslog_ctx->flags, jb);
             jb_close(jb);
             MemBufferReset(td->buffer);
-            OutputJsonBuilderBuffer(jb, td->dnslog_ctx->file_ctx, &td->buffer);
+            OutputJsonBuilderBuffer(jb, td->file_ctx, &td->buffer);
             jb_free(jb);
         }
     } else {
@@ -382,7 +383,7 @@ static int JsonDnsLoggerToClient(ThreadVars *tv, void *thread_data,
             jb_set_object(jb, "dns", answer);
 
             MemBufferReset(td->buffer);
-            OutputJsonBuilderBuffer(jb, td->dnslog_ctx->file_ctx, &td->buffer);
+            OutputJsonBuilderBuffer(jb, td->file_ctx, &td->buffer);
             jb_free(jb);
         }
         /* Log authorities. */
@@ -402,7 +403,7 @@ static int JsonDnsLoggerToClient(ThreadVars *tv, void *thread_data,
             jb_set_object(jb, "dns", answer);
 
             MemBufferReset(td->buffer);
-            OutputJsonBuilderBuffer(jb, td->dnslog_ctx->file_ctx, &td->buffer);
+            OutputJsonBuilderBuffer(jb, td->file_ctx, &td->buffer);
             jb_free(jb);
         }
     }
@@ -412,29 +413,37 @@ static int JsonDnsLoggerToClient(ThreadVars *tv, void *thread_data,
 
 static TmEcode LogDnsLogThreadInit(ThreadVars *t, const void *initdata, void **data)
 {
-    LogDnsLogThread *aft = SCMalloc(sizeof(LogDnsLogThread));
+    LogDnsLogThread *aft = SCCalloc(1, sizeof(LogDnsLogThread));
     if (unlikely(aft == NULL))
         return TM_ECODE_FAILED;
-    memset(aft, 0, sizeof(LogDnsLogThread));
 
     if(initdata == NULL)
     {
         SCLogDebug("Error getting context for EveLogDNS.  \"initdata\" argument NULL");
-        SCFree(aft);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
     aft->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE);
     if (aft->buffer == NULL) {
-        SCFree(aft);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
     /* Use the Ouptut Context (file pointer and mutex) */
     aft->dnslog_ctx= ((OutputCtx *)initdata)->data;
+    aft->file_ctx = LogFileEnsureExists(aft->dnslog_ctx->file_ctx, t->id);
+    if (!aft->file_ctx) {
+        goto error_exit;
+    }
 
     *data = (void *)aft;
     return TM_ECODE_OK;
+
+error_exit:
+    if (aft->buffer != NULL) {
+        MemBufferFree(aft->buffer);
+    }
+    SCFree(aft);
+    return TM_ECODE_FAILED;
 }
 
 static TmEcode LogDnsLogThreadDeinit(ThreadVars *t, void *data)

src/output-json-drop.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2013 Open Information Security Foundation
+/* Copyright (C) 2007-2020 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -67,6 +67,7 @@ typedef struct JsonDropOutputCtx_ {
 } JsonDropOutputCtx;
 
 typedef struct JsonDropLogThread_ {
+    LogFileCtx *file_ctx;
     JsonDropOutputCtx *drop_ctx;
     MemBuffer *buffer;
 } JsonDropLogThread;
@@ -173,7 +174,7 @@ static int DropLogJSON (JsonDropLogThread *aft, const Packet *p)
         }
     }
 
-    OutputJsonBuilderBuffer(js, aft->drop_ctx->file_ctx, &aft->buffer);
+    OutputJsonBuilderBuffer(js, aft->file_ctx, &aft->buffer);
     jb_free(js);
 
     return TM_ECODE_OK;
@@ -181,29 +182,37 @@ static int DropLogJSON (JsonDropLogThread *aft, const Packet *p)
 
 static TmEcode JsonDropLogThreadInit(ThreadVars *t, const void *initdata, void **data)
 {
-    JsonDropLogThread *aft = SCMalloc(sizeof(JsonDropLogThread));
+    JsonDropLogThread *aft = SCCalloc(1, sizeof(JsonDropLogThread));
     if (unlikely(aft == NULL))
         return TM_ECODE_FAILED;
-    memset(aft, 0, sizeof(*aft));
 
     if(initdata == NULL)
     {
         SCLogDebug("Error getting context for EveLogDrop.  \"initdata\" argument NULL");
-        SCFree(aft);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
     aft->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE);
     if (aft->buffer == NULL) {
-        SCFree(aft);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
     /** Use the Ouptut Context (file pointer and mutex) */
     aft->drop_ctx = ((OutputCtx *)initdata)->data;
+    aft->file_ctx = LogFileEnsureExists(aft->drop_ctx->file_ctx, t->id);
+    if (!aft->file_ctx) {
+        goto error_exit;
+    }
 
     *data = (void *)aft;
     return TM_ECODE_OK;
+
+error_exit:
+    if (aft->buffer != NULL) {
+        MemBufferFree(aft->buffer);
+    }
+    SCFree(aft);
+    return TM_ECODE_FAILED;
 }
 
 static TmEcode JsonDropLogThreadDeinit(ThreadVars *t, void *data)

src/output-json-email-common.h

@@ -32,6 +32,7 @@ typedef struct OutputJsonEmailCtx_ {
 } OutputJsonEmailCtx;
 
 typedef struct JsonEmailLogThread_ {
+    LogFileCtx *file_ctx;
     OutputJsonEmailCtx *emaillog_ctx;
     MemBuffer *buffer;
 } JsonEmailLogThread;

src/output-json-file.c

@@ -77,6 +77,7 @@ typedef struct OutputFileCtx_ {
 
 typedef struct JsonFileLogThread_ {
     OutputFileCtx *filelog_ctx;
+    LogFileCtx *file_ctx;
     MemBuffer *buffer;
 } JsonFileLogThread;
 
@@ -203,7 +204,7 @@ static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p,
     }
 
     MemBufferReset(aft->buffer);
-    OutputJsonBuilderBuffer(js, aft->filelog_ctx->file_ctx, &aft->buffer);
+    OutputJsonBuilderBuffer(js, aft->file_ctx, &aft->buffer);
     jb_free(js);
 }
 
@@ -224,29 +225,37 @@ static int JsonFileLogger(ThreadVars *tv, void *thread_data, const Packet *p,
 
 static TmEcode JsonFileLogThreadInit(ThreadVars *t, const void *initdata, void **data)
 {
-    JsonFileLogThread *aft = SCMalloc(sizeof(JsonFileLogThread));
+    JsonFileLogThread *aft = SCCalloc(1, sizeof(JsonFileLogThread));
     if (unlikely(aft == NULL))
         return TM_ECODE_FAILED;
-    memset(aft, 0, sizeof(JsonFileLogThread));
 
     if(initdata == NULL)
     {
         SCLogDebug("Error getting context for EveLogFile.  \"initdata\" argument NULL");
-        SCFree(aft);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
-    /* Use the Ouptut Context (file pointer and mutex) */
-    aft->filelog_ctx = ((OutputCtx *)initdata)->data;
-
     aft->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE);
     if (aft->buffer == NULL) {
-        SCFree(aft);
-        return TM_ECODE_FAILED;
+        goto error_exit;
+    }
+
+    /* Use the Ouptut Context (file pointer and mutex) */
+    aft->filelog_ctx = ((OutputCtx *)initdata)->data;
+    aft->file_ctx = LogFileEnsureExists(aft->filelog_ctx->file_ctx, t->id);
+    if (!aft->file_ctx) {
+        goto error_exit;
     }
 
     *data = (void *)aft;
     return TM_ECODE_OK;
+
+error_exit:
+    if (aft->buffer != NULL) {
+        MemBufferFree(aft->buffer);
+    }
+    SCFree(aft);
+    return TM_ECODE_FAILED;
 }
 
 static TmEcode JsonFileLogThreadDeinit(ThreadVars *t, void *data)

src/output-json-flow.c

@@ -60,6 +60,7 @@ typedef struct LogJsonFileCtx_ {
 typedef struct JsonFlowLogThread_ {
     LogJsonFileCtx *flowlog_ctx;
     /** LogFileCtx has the pointer to the file and a mutex to allow multithreading */
+    LogFileCtx *file_ctx;
     MemBuffer *buffer;
 } JsonFlowLogThread;
 
@@ -333,12 +334,12 @@ static int JsonFlowLogger(ThreadVars *tv, void *thread_data, Flow *f)
 
     JsonBuilder *jb = CreateEveHeaderFromFlow(f);
     if (unlikely(jb == NULL)) {
-        return TM_ECODE_OK;
+        SCReturnInt(TM_ECODE_OK);
     }
 
     EveFlowLogJSON(jhl, jb, f);
 
-    OutputJsonBuilderBuffer(jb, jhl->flowlog_ctx->file_ctx, &jhl->buffer);
+    OutputJsonBuilderBuffer(jb, jhl->file_ctx, &jhl->buffer);
     jb_free(jb);
 
     SCReturnInt(TM_ECODE_OK);
@@ -425,29 +426,38 @@ static OutputInitResult OutputFlowLogInitSub(ConfNode *conf, OutputCtx *parent_c
 
 static TmEcode JsonFlowLogThreadInit(ThreadVars *t, const void *initdata, void **data)
 {
-    JsonFlowLogThread *aft = SCMalloc(sizeof(JsonFlowLogThread));
+    JsonFlowLogThread *aft = SCCalloc(1, sizeof(JsonFlowLogThread));
     if (unlikely(aft == NULL))
         return TM_ECODE_FAILED;
-    memset(aft, 0, sizeof(JsonFlowLogThread));
 
     if(initdata == NULL)
     {
         SCLogDebug("Error getting context for EveLogFlow.  \"initdata\" argument NULL");
-        SCFree(aft);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
-    /* Use the Ouptut Context (file pointer and mutex) */
+    /* Use the Outptut Context (file pointer and mutex) */
     aft->flowlog_ctx = ((OutputCtx *)initdata)->data; //TODO
 
     aft->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE);
     if (aft->buffer == NULL) {
-        SCFree(aft);
-        return TM_ECODE_FAILED;
+        goto error_exit;
+    }
+
+    aft->file_ctx = LogFileEnsureExists(aft->flowlog_ctx->file_ctx, t->id);
+    if (!aft->file_ctx) {
+        goto error_exit;
     }
 
     *data = (void *)aft;
     return TM_ECODE_OK;
+
+error_exit:
+    if (aft->buffer != NULL) {
+        MemBufferFree(aft->buffer);
+    }
+    SCFree(aft);
+    return TM_ECODE_FAILED;
 }
 
 static TmEcode JsonFlowLogThreadDeinit(ThreadVars *t, void *data)

src/output-json-ftp.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2017 Open Information Security Foundation
+/* Copyright (C) 2017-2020 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -54,6 +54,7 @@ typedef struct LogFTPFileCtx_ {
 } LogFTPFileCtx;
 
 typedef struct LogFTPLogThread_ {
+    LogFileCtx *file_ctx;
     LogFTPFileCtx *ftplog_ctx;
     MemBuffer          *buffer;
 } LogFTPLogThread;
@@ -175,7 +176,7 @@ static int JsonFTPLogger(ThreadVars *tv, void *thread_data,
         }
 
         MemBufferReset(thread->buffer);
-        OutputJsonBuilderBuffer(jb, thread->ftplog_ctx->file_ctx, &thread->buffer);
+        OutputJsonBuilderBuffer(jb, thread->file_ctx, &thread->buffer);
 
         jb_free(jb);
     }
@@ -234,20 +235,30 @@ static TmEcode JsonFTPLogThreadInit(ThreadVars *t, const void *initdata, void **
 
     if (initdata == NULL) {
         SCLogDebug("Error getting context for EveLogFTP.  \"initdata\" is NULL.");
-        SCFree(thread);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
     thread->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE);
     if (unlikely(thread->buffer == NULL)) {
-        SCFree(thread);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
     thread->ftplog_ctx = ((OutputCtx *)initdata)->data;
+    thread->file_ctx = LogFileEnsureExists(thread->ftplog_ctx->file_ctx, t->id);
+    if (!thread->file_ctx) {
+        goto error_exit;
+    }
+
     *data = (void *)thread;
 
     return TM_ECODE_OK;
+
+error_exit:
+    if (thread->buffer != NULL) {
+        MemBufferFree(thread->buffer);
+    }
+    SCFree(thread);
+    return TM_ECODE_FAILED;
 }
 
 static TmEcode JsonFTPLogThreadDeinit(ThreadVars *t, void *data)

src/output-json-http.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2013 Open Information Security Foundation
+/* Copyright (C) 2007-2020 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -66,6 +66,7 @@ typedef struct LogHttpFileCtx_ {
 
 typedef struct JsonHttpLogThread_ {
     LogHttpFileCtx *httplog_ctx;
+    LogFileCtx *file_ctx;
     /** LogFileCtx has the pointer to the file and a mutex to allow multithreading */
     uint32_t uri_cnt;
 
@@ -538,7 +539,7 @@ static int JsonHttpLogger(ThreadVars *tv, void *thread_data, const Packet *p, Fl
         }
     }
 
-    OutputJsonBuilderBuffer(js, jhl->httplog_ctx->file_ctx, &jhl->buffer);
+    OutputJsonBuilderBuffer(js, jhl->file_ctx, &jhl->buffer);
     jb_free(js);
 
     SCReturnInt(TM_ECODE_OK);
@@ -741,29 +742,38 @@ static OutputInitResult OutputHttpLogInitSub(ConfNode *conf, OutputCtx *parent_c
 
 static TmEcode JsonHttpLogThreadInit(ThreadVars *t, const void *initdata, void **data)
 {
-    JsonHttpLogThread *aft = SCMalloc(sizeof(JsonHttpLogThread));
+    JsonHttpLogThread *aft = SCCalloc(1, sizeof(JsonHttpLogThread));
     if (unlikely(aft == NULL))
         return TM_ECODE_FAILED;
-    memset(aft, 0, sizeof(JsonHttpLogThread));
 
     if(initdata == NULL)
     {
         SCLogDebug("Error getting context for EveLogHTTP.  \"initdata\" argument NULL");
-        SCFree(aft);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
-    /* Use the Ouptut Context (file pointer and mutex) */
+    /* Use the Output Context (file pointer and mutex) */
     aft->httplog_ctx = ((OutputCtx *)initdata)->data; //TODO
 
     aft->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE);
     if (aft->buffer == NULL) {
-        SCFree(aft);
-        return TM_ECODE_FAILED;
+        goto error_exit;
+    }
+
+    aft->file_ctx = LogFileEnsureExists(aft->httplog_ctx->file_ctx, t->id);
+    if (!aft->file_ctx) {
+        goto error_exit;
     }
 
     *data = (void *)aft;
     return TM_ECODE_OK;
+
+error_exit:
+    if (aft->buffer != NULL) {
+        MemBufferFree(aft->buffer);
+    }
+    SCFree(aft);
+    return TM_ECODE_FAILED;
 }
 
 static TmEcode JsonHttpLogThreadDeinit(ThreadVars *t, void *data)

src/output-json-ikev2.c

@@ -55,6 +55,7 @@ typedef struct LogIKEv2FileCtx_ {
 } LogIKEv2FileCtx;
 
 typedef struct LogIKEv2LogThread_ {
+    LogFileCtx *file_ctx;
     LogIKEv2FileCtx *ikev2log_ctx;
     MemBuffer          *buffer;
 } LogIKEv2LogThread;
@@ -79,7 +80,7 @@ static int JsonIKEv2Logger(ThreadVars *tv, void *thread_data,
     jb_close(jb);
 
     MemBufferReset(thread->buffer);
-    OutputJsonBuilderBuffer(jb, thread->ikev2log_ctx->file_ctx, &thread->buffer);
+    OutputJsonBuilderBuffer(jb, thread->file_ctx, &thread->buffer);
 
     jb_free(jb);
     return TM_ECODE_OK;
@@ -135,20 +136,29 @@ static TmEcode JsonIKEv2LogThreadInit(ThreadVars *t, const void *initdata, void
 
     if (initdata == NULL) {
         SCLogDebug("Error getting context for EveLogIKEv2.  \"initdata\" is NULL.");
-        SCFree(thread);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
     thread->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE);
     if (unlikely(thread->buffer == NULL)) {
-        SCFree(thread);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
     thread->ikev2log_ctx = ((OutputCtx *)initdata)->data;
-    *data = (void *)thread;
+    thread->file_ctx = LogFileEnsureExists(thread->ikev2log_ctx->file_ctx, t->id);
+    if (!thread->file_ctx) {
+        goto error_exit;
+    }
 
+    *data = (void *)thread;
     return TM_ECODE_OK;
+
+error_exit:
+    if (thread->buffer != NULL) {
+        MemBufferFree(thread->buffer);
+    }
+    SCFree(thread);
+    return TM_ECODE_FAILED;
 }
 
 static TmEcode JsonIKEv2LogThreadDeinit(ThreadVars *t, void *data)

src/output-json-krb5.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2018 Open Information Security Foundation
+/* Copyright (C) 2018-2020 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -55,6 +55,7 @@ typedef struct LogKRB5FileCtx_ {
 } LogKRB5FileCtx;
 
 typedef struct LogKRB5LogThread_ {
+    LogFileCtx *file_ctx;
     LogKRB5FileCtx *krb5log_ctx;
     MemBuffer          *buffer;
 } LogKRB5LogThread;
@@ -79,7 +80,7 @@ static int JsonKRB5Logger(ThreadVars *tv, void *thread_data,
     jb_close(jb);
 
     MemBufferReset(thread->buffer);
-    OutputJsonBuilderBuffer(jb, thread->krb5log_ctx->file_ctx, &thread->buffer);
+    OutputJsonBuilderBuffer(jb, thread->file_ctx, &thread->buffer);
 
     jb_free(jb);
     return TM_ECODE_OK;
@@ -136,20 +137,29 @@ static TmEcode JsonKRB5LogThreadInit(ThreadVars *t, const void *initdata, void *
 
     if (initdata == NULL) {
         SCLogDebug("Error getting context for EveLogKRB5.  \"initdata\" is NULL.");
-        SCFree(thread);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
     thread->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE);
     if (unlikely(thread->buffer == NULL)) {
-        SCFree(thread);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
     thread->krb5log_ctx = ((OutputCtx *)initdata)->data;
+    thread->file_ctx = LogFileEnsureExists(thread->krb5log_ctx->file_ctx, t->id);
+    if (!thread->file_ctx) {
+        goto error_exit;
+    }
     *data = (void *)thread;
 
     return TM_ECODE_OK;
+
+error_exit:
+    if (thread->buffer != NULL) {
+        MemBufferFree(thread->buffer);
+    }
+    SCFree(thread);
+    return TM_ECODE_FAILED;
 }
 
 static TmEcode JsonKRB5LogThreadDeinit(ThreadVars *t, void *data)

src/output-json-metadata.c

@@ -106,30 +106,37 @@ static int JsonMetadataLogCondition(ThreadVars *tv, const Packet *p)
 
 static TmEcode JsonMetadataLogThreadInit(ThreadVars *t, const void *initdata, void **data)
 {
-    JsonMetadataLogThread *aft = SCMalloc(sizeof(JsonMetadataLogThread));
+    JsonMetadataLogThread *aft = SCCalloc(1, sizeof(JsonMetadataLogThread));
     if (unlikely(aft == NULL))
         return TM_ECODE_FAILED;
-    memset(aft, 0, sizeof(JsonMetadataLogThread));
-    if(initdata == NULL)
-    {
+
+    if(initdata == NULL) {
         SCLogDebug("Error getting context for EveLogMetadata.  \"initdata\" argument NULL");
-        SCFree(aft);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
     aft->json_buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE);
     if (aft->json_buffer == NULL) {
-        SCFree(aft);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
     /** Use the Output Context (file pointer and mutex) */
     MetadataJsonOutputCtx *json_output_ctx = ((OutputCtx *)initdata)->data;
-    aft->file_ctx = json_output_ctx->file_ctx;
+    aft->file_ctx = LogFileEnsureExists(json_output_ctx->file_ctx, t->id);
+    if (!aft->file_ctx) {
+        goto error_exit;
+    }
     aft->json_output_ctx = json_output_ctx;
 
     *data = (void *)aft;
     return TM_ECODE_OK;
+
+error_exit:
+    if (aft->json_buffer != NULL) {
+        MemBufferFree(aft->json_buffer);
+    }
+    SCFree(aft);
+    return TM_ECODE_FAILED;
 }
 
 static TmEcode JsonMetadataLogThreadDeinit(ThreadVars *t, void *data)

src/output-json-netflow.c

@@ -55,6 +55,7 @@ typedef struct LogJsonFileCtx_ {
 } LogJsonFileCtx;
 
 typedef struct JsonNetFlowLogThread_ {
+    LogFileCtx *file_ctx;
     LogJsonFileCtx *flowlog_ctx;
     /** LogFileCtx has the pointer to the file and a mutex to allow multithreading */
 
@@ -287,7 +288,7 @@ static int JsonNetFlowLogger(ThreadVars *tv, void *thread_data, Flow *f)
         return TM_ECODE_OK;
     NetFlowLogEveToServer(jhl, jb, f);
     EveAddCommonOptions(&netflow_ctx->cfg, NULL, f, jb);
-    OutputJsonBuilderBuffer(jb, jhl->flowlog_ctx->file_ctx, &jhl->buffer);
+    OutputJsonBuilderBuffer(jb, jhl->file_ctx, &jhl->buffer);
     jb_free(jb);
 
     /* only log a response record if we actually have seen response packets */
@@ -299,7 +300,7 @@ static int JsonNetFlowLogger(ThreadVars *tv, void *thread_data, Flow *f)
             return TM_ECODE_OK;
         NetFlowLogEveToClient(jhl, jb, f);
         EveAddCommonOptions(&netflow_ctx->cfg, NULL, f, jb);
-        OutputJsonBuilderBuffer(jb, jhl->flowlog_ctx->file_ctx, &jhl->buffer);
+        OutputJsonBuilderBuffer(jb, jhl->file_ctx, &jhl->buffer);
         jb_free(jb);
     }
     SCReturnInt(TM_ECODE_OK);
@@ -386,16 +387,13 @@ static OutputInitResult OutputNetFlowLogInitSub(ConfNode *conf, OutputCtx *paren
 
 static TmEcode JsonNetFlowLogThreadInit(ThreadVars *t, const void *initdata, void **data)
 {
-    JsonNetFlowLogThread *aft = SCMalloc(sizeof(JsonNetFlowLogThread));
+    JsonNetFlowLogThread *aft = SCCalloc(1, sizeof(JsonNetFlowLogThread));
     if (unlikely(aft == NULL))
         return TM_ECODE_FAILED;
-    memset(aft, 0, sizeof(JsonNetFlowLogThread));
 
-    if(initdata == NULL)
-    {
+    if(initdata == NULL) {
         SCLogDebug("Error getting context for EveLogNetflow.  \"initdata\" argument NULL");
-        SCFree(aft);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
     /* Use the Ouptut Context (file pointer and mutex) */
@@ -403,12 +401,23 @@ static TmEcode JsonNetFlowLogThreadInit(ThreadVars *t, const void *initdata, voi
 
     aft->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE);
     if (aft->buffer == NULL) {
-        SCFree(aft);
-        return TM_ECODE_FAILED;
+        goto error_exit;
+    }
+
+    aft->file_ctx = LogFileEnsureExists(aft->flowlog_ctx->file_ctx, t->id);
+    if (!aft->file_ctx) {
+        goto error_exit;
     }
 
     *data = (void *)aft;
     return TM_ECODE_OK;
+
+error_exit:
+    if (aft->buffer != NULL) {
+        MemBufferFree(aft->buffer);
+    }
+    SCFree(aft);
+    return TM_ECODE_FAILED;
 }
 
 static TmEcode JsonNetFlowLogThreadDeinit(ThreadVars *t, void *data)

src/output-json-nfs.c

@@ -96,7 +96,7 @@ static int JsonNFSLogger(ThreadVars *tv, void *thread_data,
     jb_close(jb);
 
     MemBufferReset(thread->buffer);
-    OutputJsonBuilderBuffer(jb, thread->ctx->file_ctx, &thread->buffer);
+    OutputJsonBuilderBuffer(jb, thread->file_ctx, &thread->buffer);
     jb_free(jb);
     return TM_ECODE_OK;
 }

src/output-json-rdp.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2019 Open Information Security Foundation
+/* Copyright (C) 2019-2020 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -51,6 +51,7 @@ typedef struct LogRdpFileCtx_ {
 
 typedef struct LogRdpLogThread_ {
     LogRdpFileCtx *rdplog_ctx;
+    LogFileCtx *file_ctx;
     MemBuffer       *buffer;
 } LogRdpLogThread;
 
@@ -69,7 +70,7 @@ static int JsonRdpLogger(ThreadVars *tv, void *thread_data,
         return TM_ECODE_FAILED;
     }
     MemBufferReset(thread->buffer);
-    OutputJsonBuilderBuffer(js, thread->rdplog_ctx->file_ctx, &thread->buffer);
+    OutputJsonBuilderBuffer(js, thread->file_ctx, &thread->buffer);
 
     jb_free(js);
     return TM_ECODE_OK;
@@ -126,14 +127,24 @@ static TmEcode JsonRdpLogThreadInit(ThreadVars *t, const void *initdata, void **
 
     thread->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE);
     if (unlikely(thread->buffer == NULL)) {
-        SCFree(thread);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
     thread->rdplog_ctx = ((OutputCtx *)initdata)->data;
-    *data = (void *)thread;
+    thread->file_ctx = LogFileEnsureExists(thread->rdplog_ctx->file_ctx, t->id);
+    if (!thread->file_ctx) {
+        goto error_exit;
+    }
 
+    *data = (void *)thread;
     return TM_ECODE_OK;
+
+error_exit:
+    if (thread->buffer != NULL) {
+        MemBufferFree(thread->buffer);
+    }
+    SCFree(thread);
+    return TM_ECODE_FAILED;
 }
 
 static TmEcode JsonRdpLogThreadDeinit(ThreadVars *t, void *data)

src/output-json-rfb.c

@@ -53,6 +53,7 @@ typedef struct LogRFBFileCtx_ {
 
 typedef struct LogRFBLogThread_ {
     LogRFBFileCtx *rfblog_ctx;
+    LogFileCtx *file_ctx;
     MemBuffer          *buffer;
 } LogRFBLogThread;
 
@@ -84,7 +85,7 @@ static int JsonRFBLogger(ThreadVars *tv, void *thread_data,
     }
 
     MemBufferReset(thread->buffer);
-    OutputJsonBuilderBuffer(js, thread->rfblog_ctx->file_ctx, &thread->buffer);
+    OutputJsonBuilderBuffer(js, thread->file_ctx, &thread->buffer);
     jb_free(js);
 
     return TM_ECODE_OK;
@@ -142,14 +143,24 @@ static TmEcode JsonRFBLogThreadInit(ThreadVars *t, const void *initdata, void **
 
     thread->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE);
     if (unlikely(thread->buffer == NULL)) {
-        SCFree(thread);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
     thread->rfblog_ctx = ((OutputCtx *)initdata)->data;
+    thread->file_ctx = LogFileEnsureExists(thread->rfblog_ctx->file_ctx, t->id);
+    if (!thread->file_ctx) {
+        goto error_exit;
+    }
     *data = (void *)thread;
 
     return TM_ECODE_OK;
+
+error_exit:
+    if (thread->buffer != NULL) {
+        MemBufferFree(thread->buffer);
+    }
+    SCFree(thread);
+    return TM_ECODE_FAILED;
 }
 
 static TmEcode JsonRFBLogThreadDeinit(ThreadVars *t, void *data)

src/output-json-sip.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2018 Open Information Security Foundation
+/* Copyright (C) 2018-2020 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -55,6 +55,7 @@ typedef struct LogSIPFileCtx_ {
 } LogSIPFileCtx;
 
 typedef struct LogSIPLogThread_ {
+    LogFileCtx *file_ctx;
     LogSIPFileCtx *siplog_ctx;
     MemBuffer          *buffer;
 } LogSIPLogThread;
@@ -87,7 +88,7 @@ static int JsonSIPLogger(ThreadVars *tv, void *thread_data,
     }
 
     MemBufferReset(thread->buffer);
-    OutputJsonBuilderBuffer(js, thread->siplog_ctx->file_ctx, &thread->buffer);
+    OutputJsonBuilderBuffer(js, thread->file_ctx, &thread->buffer);
     jb_free(js);
 
     return TM_ECODE_OK;
@@ -145,20 +146,30 @@ static TmEcode JsonSIPLogThreadInit(ThreadVars *t, const void *initdata, void **
 
     if (initdata == NULL) {
         SCLogDebug("Error getting context for EveLogSIP.  \"initdata\" is NULL.");
-        SCFree(thread);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
     thread->buffer = MemBufferCreateNew(OUTPUT_BUFFER_SIZE);
     if (unlikely(thread->buffer == NULL)) {
-        SCFree(thread);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
     thread->siplog_ctx = ((OutputCtx *)initdata)->data;
+
+    thread->file_ctx = LogFileEnsureExists(thread->siplog_ctx->file_ctx, t->id);
+    if (!thread->file_ctx) {
+        goto error_exit;
+    }
     *data = (void *)thread;
 
     return TM_ECODE_OK;
+
+error_exit:
+    if (thread->buffer != NULL) {
+        MemBufferFree(thread->buffer);
+    }
+    SCFree(thread);
+    return TM_ECODE_FAILED;
 }
 
 static TmEcode JsonSIPLogThreadDeinit(ThreadVars *t, void *data)

src/output-json-smb.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2017-2018 Open Information Security Foundation
+/* Copyright (C) 2017-2020 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -77,7 +77,7 @@ static int JsonSMBLogger(ThreadVars *tv, void *thread_data,
 
     EveAddCommonOptions(&thread->ctx->cfg, p, f, jb);
     MemBufferReset(thread->buffer);
-    OutputJsonBuilderBuffer(jb, thread->ctx->file_ctx, &thread->buffer);
+    OutputJsonBuilderBuffer(jb, thread->file_ctx, &thread->buffer);
 
     jb_free(jb);
     return TM_ECODE_OK;

src/output-json-smtp.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2015 Open Information Security Foundation
+/* Copyright (C) 2007-2020 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -89,7 +89,7 @@ static int JsonSmtpLogger(ThreadVars *tv, void *thread_data, const Packet *p, Fl
     jb_close(jb);
 
     if (EveEmailLogJson(jhl, jb, p, f, state, tx, tx_id) == TM_ECODE_OK) {
-        OutputJsonBuilderBuffer(jb, jhl->emaillog_ctx->file_ctx, &jhl->buffer);
+        OutputJsonBuilderBuffer(jb, jhl->file_ctx, &jhl->buffer);
     }
 
     jb_free(jb);
@@ -206,29 +206,36 @@ static OutputInitResult OutputSmtpLogInitSub(ConfNode *conf, OutputCtx *parent_c
 
 static TmEcode JsonSmtpLogThreadInit(ThreadVars *t, const void *initdata, void **data)
 {
-    JsonEmailLogThread *aft = SCMalloc(sizeof(JsonEmailLogThread));
+    JsonEmailLogThread *aft = SCCalloc(1, sizeof(JsonEmailLogThread));
     if (unlikely(aft == NULL))
         return TM_ECODE_FAILED;
-    memset(aft, 0, sizeof(JsonEmailLogThread));
 
-    if(initdata == NULL)
-    {
+    if(initdata == NULL) {
         SCLogDebug("Error getting context for EveLogSMTP.  \"initdata\" argument NULL");
-        SCFree(aft);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
-    /* Use the Ouptut Context (file pointer and mutex) */
+    /* Use the Output Context (file pointer and mutex) */
     aft->emaillog_ctx = ((OutputCtx *)initdata)->data;
 
     aft->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE);
     if (aft->buffer == NULL) {
-        SCFree(aft);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
+    aft->file_ctx = LogFileEnsureExists(aft->emaillog_ctx->file_ctx, t->id);
+    if (!aft->file_ctx) {
+        goto error_exit;
+    }
     *data = (void *)aft;
     return TM_ECODE_OK;
+
+error_exit:
+    if (aft->buffer != NULL) {
+        MemBufferFree(aft->buffer);
+    }
+    SCFree(aft);
+    return TM_ECODE_FAILED;
 }
 
 static TmEcode JsonSmtpLogThreadDeinit(ThreadVars *t, void *data)

src/output-json-snmp.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2018-2019 Open Information Security Foundation
+/* Copyright (C) 2018-2020 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -55,6 +55,7 @@ typedef struct LogSNMPFileCtx_ {
 } LogSNMPFileCtx;
 
 typedef struct LogSNMPLogThread_ {
+    LogFileCtx *file_ctx;
     LogSNMPFileCtx *snmplog_ctx;
     MemBuffer          *buffer;
 } LogSNMPLogThread;
@@ -79,7 +80,7 @@ static int JsonSNMPLogger(ThreadVars *tv, void *thread_data,
     jb_close(jb);
 
     MemBufferReset(thread->buffer);
-    OutputJsonBuilderBuffer(jb, thread->snmplog_ctx->file_ctx, &thread->buffer);
+    OutputJsonBuilderBuffer(jb, thread->file_ctx, &thread->buffer);
 
     jb_free(jb);
     return TM_ECODE_OK;
@@ -135,20 +136,30 @@ static TmEcode JsonSNMPLogThreadInit(ThreadVars *t, const void *initdata, void *
 
     if (initdata == NULL) {
         SCLogDebug("Error getting context for EveLogSNMP.  \"initdata\" is NULL.");
-        SCFree(thread);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
     thread->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE);
     if (unlikely(thread->buffer == NULL)) {
-        SCFree(thread);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
     thread->snmplog_ctx = ((OutputCtx *)initdata)->data;
-    *data = (void *)thread;
 
+    thread->file_ctx = LogFileEnsureExists(thread->snmplog_ctx->file_ctx, t->id);
+    if (!thread->file_ctx) {
+        goto error_exit;
+    }
+
+    *data = (void *)thread;
     return TM_ECODE_OK;
+
+error_exit:
+    if (thread->buffer != NULL) {
+        MemBufferFree(thread->buffer);
+    }
+    SCFree(thread);
+    return TM_ECODE_FAILED;
 }
 
 static TmEcode JsonSNMPLogThreadDeinit(ThreadVars *t, void *data)

src/output-json-ssh.c

@@ -61,6 +61,7 @@ typedef struct OutputSshCtx_ {
 
 typedef struct JsonSshLogThread_ {
     OutputSshCtx *sshlog_ctx;
+    LogFileCtx *file_ctx;
     MemBuffer *buffer;
 } JsonSshLogThread;
 
@@ -103,22 +104,32 @@ static TmEcode JsonSshLogThreadInit(ThreadVars *t, const void *initdata, void **
         return TM_ECODE_FAILED;
     }
 
-    JsonSshLogThread *aft = SCMalloc(sizeof(JsonSshLogThread));
+    JsonSshLogThread *aft = SCCalloc(1, sizeof(JsonSshLogThread));
     if (unlikely(aft == NULL))
         return TM_ECODE_FAILED;
-    memset(aft, 0, sizeof(JsonSshLogThread));
 
-    /* Use the Ouptut Context (file pointer and mutex) */
+    /* Use the Output Context (file pointer and mutex) */
     aft->sshlog_ctx = ((OutputCtx *)initdata)->data;
 
     aft->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE);
     if (aft->buffer == NULL) {
-        SCFree(aft);
-        return TM_ECODE_FAILED;
+        goto error_exit;
+    }
+
+    aft->file_ctx = LogFileEnsureExists(aft->sshlog_ctx->file_ctx, t->id);
+    if (!aft->file_ctx) {
+        goto error_exit;
     }
 
     *data = (void *)aft;
     return TM_ECODE_OK;
+
+error_exit:
+    if (aft->buffer != NULL) {
+        MemBufferFree(aft->buffer);
+    }
+    SCFree(aft);
+    return TM_ECODE_FAILED;
 }
 
 static TmEcode JsonSshLogThreadDeinit(ThreadVars *t, void *data)

src/output-json-stats.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2014 Open Information Security Foundation
+/* Copyright (C) 2014-2020 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -70,6 +70,7 @@ typedef struct OutputStatsCtx_ {
 
 typedef struct JsonStatsLogThread_ {
     OutputStatsCtx *statslog_ctx;
+    LogFileCtx *file_ctx;
     MemBuffer *buffer;
 } JsonStatsLogThread;
 
@@ -313,7 +314,7 @@ static int JsonStatsLogger(ThreadVars *tv, void *thread_data, const StatsTable *
 
     json_object_set_new(js, "stats", js_stats);
 
-    OutputJSONBuffer(js, aft->statslog_ctx->file_ctx, &aft->buffer);
+    OutputJSONBuffer(js, aft->file_ctx, &aft->buffer);
     MemBufferReset(aft->buffer);
 
     json_object_clear(js_stats);
@@ -326,29 +327,38 @@ static int JsonStatsLogger(ThreadVars *tv, void *thread_data, const StatsTable *
 
 static TmEcode JsonStatsLogThreadInit(ThreadVars *t, const void *initdata, void **data)
 {
-    JsonStatsLogThread *aft = SCMalloc(sizeof(JsonStatsLogThread));
+    JsonStatsLogThread *aft = SCCalloc(1, sizeof(JsonStatsLogThread));
     if (unlikely(aft == NULL))
         return TM_ECODE_FAILED;
-    memset(aft, 0, sizeof(JsonStatsLogThread));
 
     if(initdata == NULL)
     {
         SCLogDebug("Error getting context for EveLogStats.  \"initdata\" argument NULL");
-        SCFree(aft);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
-    /* Use the Ouptut Context (file pointer and mutex) */
-    aft->statslog_ctx = ((OutputCtx *)initdata)->data;
-
     aft->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE);
     if (aft->buffer == NULL) {
-        SCFree(aft);
-        return TM_ECODE_FAILED;
+        goto error_exit;
+    }
+
+    /* Use the Output Context (file pointer and mutex) */
+    aft->statslog_ctx = ((OutputCtx *)initdata)->data;
+
+    aft->file_ctx = LogFileEnsureExists(aft->statslog_ctx->file_ctx, t->id);
+    if (!aft->file_ctx) {
+        goto error_exit;
     }
 
     *data = (void *)aft;
     return TM_ECODE_OK;
+
+error_exit:
+    if (aft->buffer != NULL) {
+        MemBufferFree(aft->buffer);
+    }
+    SCFree(aft);
+    return TM_ECODE_FAILED;
 }
 
 static TmEcode JsonStatsLogThreadDeinit(ThreadVars *t, void *data)

src/output-json-template-rust.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2018 Open Information Security Foundation
+/* Copyright (C) 2018-2020 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -61,6 +61,7 @@ typedef struct LogTemplateFileCtx_ {
 
 typedef struct LogTemplateLogThread_ {
     LogTemplateFileCtx *templatelog_ctx;
+    LogFileCtx *file_ctx;
     MemBuffer          *buffer;
 } LogTemplateLogThread;
 
@@ -82,7 +83,7 @@ static int JsonTemplateLogger(ThreadVars *tv, void *thread_data,
     jb_close(js);
 
     MemBufferReset(thread->buffer);
-    OutputJsonBuilderBuffer(js, thread->templatelog_ctx->file_ctx, &thread->buffer);
+    OutputJsonBuilderBuffer(js, thread->file_ctx, &thread->buffer);
     jb_free(js);
 
     return TM_ECODE_OK;
@@ -137,20 +138,29 @@ static TmEcode JsonTemplateLogThreadInit(ThreadVars *t, const void *initdata, vo
 
     if (initdata == NULL) {
         SCLogDebug("Error getting context for EveLogTemplate.  \"initdata\" is NULL.");
-        SCFree(thread);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
     thread->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE);
     if (unlikely(thread->buffer == NULL)) {
-        SCFree(thread);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
     thread->templatelog_ctx = ((OutputCtx *)initdata)->data;
+    thread->file_ctx = LogFileEnsureExists(thread->templatelog_ctx->file_ctx, t->id);
+    if (!thread->file_ctx) {
+        goto error_exit;
+    }
     *data = (void *)thread;
 
     return TM_ECODE_OK;
+
+error_exit:
+    if (thread->buffer != NULL) {
+        MemBufferFree(thread->buffer);
+    }
+    SCFree(thread);
+    return TM_ECODE_FAILED;
 }
 
 static TmEcode JsonTemplateLogThreadDeinit(ThreadVars *t, void *data)

src/output-json-template.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2015 Open Information Security Foundation
+/* Copyright (C) 2015-2020 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -59,6 +59,7 @@ typedef struct LogTemplateFileCtx_ {
 } LogTemplateFileCtx;
 
 typedef struct LogTemplateLogThread_ {
+    LogFileCtx *file_ctx;
     LogTemplateFileCtx *templatelog_ctx;
     MemBuffer          *buffer;
 } LogTemplateLogThread;
@@ -94,7 +95,7 @@ static int JsonTemplateLogger(ThreadVars *tv, void *thread_data,
     jb_close(js);
 
     MemBufferReset(thread->buffer);
-    OutputJsonBuilderBuffer(js, thread->templatelog_ctx->file_ctx, &thread->buffer);
+    OutputJsonBuilderBuffer(js, thread->file_ctx, &thread->buffer);
 
     jb_free(js);
     return TM_ECODE_OK;
@@ -145,20 +146,29 @@ static TmEcode JsonTemplateLogThreadInit(ThreadVars *t, const void *initdata, vo
 
     if (initdata == NULL) {
         SCLogDebug("Error getting context for EveLogTemplate.  \"initdata\" is NULL.");
-        SCFree(thread);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
     thread->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE);
     if (unlikely(thread->buffer == NULL)) {
-        SCFree(thread);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
     thread->templatelog_ctx = ((OutputCtx *)initdata)->data;
+    thread->file_ctx = LogFileEnsureExists(thread->templatelog_ctx->file_ctx, t->id);
+    if (!thread->file_ctx) {
+        goto error_exit;
+    }
     *data = (void *)thread;
 
     return TM_ECODE_OK;
+
+error_exit:
+    if (thread->buffer != NULL) {
+        MemBufferFree(thread->buffer);
+    }
+    SCFree(thread);
+    return TM_ECODE_FAILED;
 }
 
 static TmEcode JsonTemplateLogThreadDeinit(ThreadVars *t, void *data)

src/output-json-tftp.c

@@ -57,6 +57,7 @@ typedef struct LogTFTPFileCtx_ {
 } LogTFTPFileCtx;
 
 typedef struct LogTFTPLogThread_ {
+    LogFileCtx *file_ctx;
     LogTFTPFileCtx *tftplog_ctx;
     MemBuffer          *buffer;
 } LogTFTPLogThread;
@@ -79,7 +80,7 @@ static int JsonTFTPLogger(ThreadVars *tv, void *thread_data,
 
     EveAddCommonOptions(&thread->tftplog_ctx->cfg, p, f, jb);
     MemBufferReset(thread->buffer);
-    OutputJsonBuilderBuffer(jb, thread->tftplog_ctx->file_ctx, &thread->buffer);
+    OutputJsonBuilderBuffer(jb, thread->file_ctx, &thread->buffer);
 
     jb_free(jb);
     return TM_ECODE_OK;
@@ -135,20 +136,29 @@ static TmEcode JsonTFTPLogThreadInit(ThreadVars *t, const void *initdata, void *
 
     if (initdata == NULL) {
         SCLogDebug("Error getting context for EveLogTFTP.  \"initdata\" is NULL.");
-        SCFree(thread);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
     thread->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE);
     if (unlikely(thread->buffer == NULL)) {
-        SCFree(thread);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
     thread->tftplog_ctx = ((OutputCtx *)initdata)->data;
+    thread->file_ctx = LogFileEnsureExists(thread->tftplog_ctx->file_ctx, t->id);
+    if (!thread->file_ctx) {
+        goto error_exit;
+    }
     *data = (void *)thread;
 
     return TM_ECODE_OK;
+
+error_exit:
+    if (thread->buffer != NULL) {
+        MemBufferFree(thread->buffer);
+    }
+    SCFree(thread);
+    return TM_ECODE_FAILED;
 }
 
 static TmEcode JsonTFTPLogThreadDeinit(ThreadVars *t, void *data)

src/output-json-tls.c

@@ -106,6 +106,7 @@ typedef struct OutputTlsCtx_ {
 
 
 typedef struct JsonTlsLogThread_ {
+    LogFileCtx *file_ctx;
     OutputTlsCtx *tlslog_ctx;
     MemBuffer *buffer;
 } JsonTlsLogThread;
@@ -440,7 +441,7 @@ static int JsonTlsLogger(ThreadVars *tv, void *thread_data, const Packet *p,
     /* Close the tls object. */
     jb_close(js);
 
-    OutputJsonBuilderBuffer(js, tls_ctx->file_ctx, &aft->buffer);
+    OutputJsonBuilderBuffer(js, aft->file_ctx, &aft->buffer);
     jb_free(js);
 
     return 0;
@@ -448,30 +449,37 @@ static int JsonTlsLogger(ThreadVars *tv, void *thread_data, const Packet *p,
 
 static TmEcode JsonTlsLogThreadInit(ThreadVars *t, const void *initdata, void **data)
 {
-    JsonTlsLogThread *aft = SCMalloc(sizeof(JsonTlsLogThread));
+    JsonTlsLogThread *aft = SCCalloc(1, sizeof(JsonTlsLogThread));
     if (unlikely(aft == NULL)) {
         return TM_ECODE_FAILED;
     }
 
-    memset(aft, 0, sizeof(JsonTlsLogThread));
-
     if (initdata == NULL) {
         SCLogDebug("Error getting context for eve-log tls 'initdata' argument NULL");
-        SCFree(aft);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
-    /* use the Output Context (file pointer and mutex) */
-    aft->tlslog_ctx = ((OutputCtx *)initdata)->data;
-
     aft->buffer = MemBufferCreateNew(JSON_OUTPUT_BUFFER_SIZE);
     if (aft->buffer == NULL) {
-        SCFree(aft);
-        return TM_ECODE_FAILED;
+        goto error_exit;
     }
 
+    /* use the Output Context (file pointer and mutex) */
+    aft->tlslog_ctx = ((OutputCtx *)initdata)->data;
+
+    aft->file_ctx = LogFileEnsureExists(aft->tlslog_ctx->file_ctx, t->id);
+    if (!aft->file_ctx) {
+        goto error_exit;
+    }
     *data = (void *)aft;
     return TM_ECODE_OK;
+
+error_exit:
+    if (aft->buffer != NULL) {
+        MemBufferFree(aft->buffer);
+    }
+    SCFree(aft);
+    return TM_ECODE_FAILED;
 }
 
 static TmEcode JsonTlsLogThreadDeinit(ThreadVars *t, void *data)

src/output-json.c

@@ -1022,6 +1022,17 @@ OutputInitResult OutputJsonInitCtx(ConfNode *conf)
             json_ctx->json_out == LOGFILE_TYPE_UNIX_DGRAM ||
             json_ctx->json_out == LOGFILE_TYPE_UNIX_STREAM)
         {
+            if (json_ctx->json_out == LOGFILE_TYPE_FILE) {
+                /* Threaded file output */
+                const ConfNode *threaded = ConfNodeLookupChild(conf, "threaded");
+                if (threaded && threaded->val && ConfValIsTrue(threaded->val)) {
+                    SCLogConfig("Enabling threaded eve logging.");
+                    json_ctx->file_ctx->threaded = true;
+                } else {
+                    json_ctx->file_ctx->threaded = false;
+                }
+            }
+
             if (SCConfLogOpenGeneric(conf, json_ctx->file_ctx, DEFAULT_LOG_FILENAME, 1) < 0) {
                 LogFileFreeCtx(json_ctx->file_ctx);
                 SCFree(json_ctx);
@@ -1029,6 +1040,7 @@ OutputInitResult OutputJsonInitCtx(ConfNode *conf)
                 return result;
             }
             OutputRegisterFileRotationFlag(&json_ctx->file_ctx->rotation_flag);
+
         }
 #ifndef OS_WIN32
 	else if (json_ctx->json_out == LOGFILE_TYPE_SYSLOG) {

src/output-json.h

@@ -105,6 +105,7 @@ typedef struct OutputJsonCtx_ {
 
 typedef struct OutputJsonThreadCtx_ {
     OutputJsonCtx *ctx;
+    LogFileCtx *file_ctx;
     MemBuffer *buffer;
 } OutputJsonThreadCtx;
 

suricata.yaml.in

@@ -84,6 +84,9 @@ outputs:
       enabled: @e_enable_evelog@
       filetype: regular #regular|syslog|unix_dgram|unix_stream|redis
       filename: eve.json
+      # Enable for multi-threaded eve.json output; output files are suffixed
+      # with an identifier, e.g., eve.json.9.
+      #threaded: false
       #prefix: "@cee: " # prefix to prepend to each log entry
       # the following are valid when type: syslog above
       #identity: "suricata"