logging: Anomaly logging

This changeset adds anomaly logging to suricata for issue 2282.

Anomaly logging is controlled via the `anomaly` section within eve-log.
There is a single option -- `packethdr` -- for including the packet header
in the anomaly.

src/Makefile.am

@@ -299,6 +299,7 @@ output-filedata.c output-filedata.h \
 output-filestore.c output-filestore.h \
 output-flow.c output-flow.h \
 output-json-alert.c output-json-alert.h \
+output-json-anomaly.c output-json-anomaly.h \
 output-json-dns.c output-json-dns.h \
 output-json-dnp3.c output-json-dnp3.h \
 output-json-dnp3-objects.c output-json-dnp3-objects.h \

src/output-json-anomaly.c

@@ -0,0 +1,348 @@
+/* Copyright (C) 2019 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Jeff Lucovsky <jeff@lucovsky.org>
+ *
+ * Logs anomalies in JSON format.
+ *
+ */
+
+#include "suricata-common.h"
+#include "debug.h"
+#include "detect.h"
+#include "flow.h"
+#include "conf.h"
+
+#include "threads.h"
+#include "tm-threads.h"
+#include "threadvars.h"
+#include "util-debug.h"
+
+#include "util-misc.h"
+
+#include "detect-parse.h"
+#include "detect-engine.h"
+#include "util-logopenfile.h"
+
+#include "output.h"
+#include "output-json.h"
+#include "output-json-anomaly.h"
+
+#include "util-byte.h"
+#include "util-byte.h"
+#include "util-privs.h"
+#include "util-print.h"
+#include "util-proto-name.h"
+#include "util-optimize.h"
+#include "util-buffer.h"
+#include "util-validate.h"
+
+#define MODULE_NAME "JsonAnomalyLog"
+
+#ifdef HAVE_LIBJANSSON
+
+#define LOG_JSON_PACKET            BIT_U16(0)
+#define JSON_STREAM_BUFFER_SIZE 4096
+
+typedef struct AnomalyJsonOutputCtx_ {
+    LogFileCtx* file_ctx;
+    uint16_t flags;
+    OutputJsonCommonSettings cfg;
+} AnomalyJsonOutputCtx;
+
+typedef struct JsonAnomalyLogThread_ {
+    /** LogFileCtx has the pointer to the file and a mutex to allow multithreading */
+    LogFileCtx* file_ctx;
+    MemBuffer *json_buffer;
+    AnomalyJsonOutputCtx* json_output_ctx;
+} JsonAnomalyLogThread;
+
+static int AnomalyJson(ThreadVars *tv, JsonAnomalyLogThread *aft, const Packet *p)
+{
+    bool is_IP_pkt = PKT_IS_IPV4(p) || PKT_IS_IPV6(p);
+
+    char timebuf[64];
+    CreateIsoTimeString(&p->ts, timebuf, sizeof(timebuf));
+
+    for (int i = 0; i < p->events.cnt; i++) {
+        MemBufferReset(aft->json_buffer);
+
+        json_t *js;
+        if (is_IP_pkt) {
+            js = CreateJSONHeader(p, LOG_DIR_PACKET, "anomaly");
+        } else {
+            js = json_object();
+        }
+
+        if (unlikely(js == NULL)) {
+            return TM_ECODE_OK;
+        }
+
+        json_t *ajs = json_object();
+        if (unlikely(ajs == NULL)) {
+            json_decref(js);
+            return TM_ECODE_OK;
+        }
+
+        if (!is_IP_pkt) {
+            json_object_set_new(js, "timestamp", json_string(timebuf));
+        } else {
+            JsonFiveTuple((const Packet *)p, LOG_DIR_PACKET, js);
+            JsonAddCommonOptions(&aft->json_output_ctx->cfg, p, p->flow, js);
+        }
+
+        if (aft->json_output_ctx->flags & LOG_JSON_PACKET) {
+            char buf[(32 * 3) + 1];
+            PrintRawLineHexBuf(buf, sizeof(buf), GET_PKT_DATA(p),
+                               GET_PKT_LEN(p) < 32 ? GET_PKT_LEN(p) : 32);
+            json_object_set_new(js, "packethdr", json_string((char *)buf));
+
+            json_object_set_new(js, "linktype", json_integer(p->datalink));
+        }
+
+        uint8_t event_code = p->events.events[i];
+        if (EVENT_IS_DECODER_PACKET_ERROR(event_code)) {
+            const char *event = DEvents[event_code].event_name;
+            json_object_set_new(ajs, "event", json_string(event));
+        } else {
+            /* include event code with unrecognized events */
+            uint32_t offset = 0;
+            char unknown_event_buf[32];
+            PrintBufferData(unknown_event_buf, &offset, 32, "%s(%d)", "Unknown", event_code);
+            json_object_set_new(ajs, "event", json_string(unknown_event_buf));
+        }
+
+        /* anomaly */
+        json_object_set_new(js, "anomaly", ajs);
+        OutputJSONBuffer(js, aft->file_ctx, &aft->json_buffer);
+
+        json_object_clear(js);
+        json_decref(js);
+    }
+
+    return TM_ECODE_OK;
+}
+
+
+static int JsonAnomalyLogger(ThreadVars *tv, void *thread_data, const Packet *p)
+{
+    JsonAnomalyLogThread *aft = thread_data;
+    return AnomalyJson(tv, aft, p);
+}
+
+static int JsonAnomalyLogCondition(ThreadVars *tv, const Packet *p)
+{
+    return p->events.cnt > 0 ? TRUE : FALSE;
+}
+
+#define OUTPUT_BUFFER_SIZE 65535
+static TmEcode JsonAnomalyLogThreadInit(ThreadVars *t, const void *initdata, void **data)
+{
+    JsonAnomalyLogThread *aft = SCCalloc(1, sizeof(JsonAnomalyLogThread));
+    if (unlikely(aft == NULL)) {
+        return TM_ECODE_FAILED;
+    }
+
+    if(initdata == NULL) {
+        SCLogDebug("Error getting context for EveLogAnomaly.  \"initdata\" argument NULL");
+        SCFree(aft);
+        return TM_ECODE_FAILED;
+    }
+
+    aft->json_buffer = MemBufferCreateNew(OUTPUT_BUFFER_SIZE);
+    if (aft->json_buffer == NULL) {
+        SCFree(aft);
+        return TM_ECODE_FAILED;
+    }
+
+    /** Use the Output Context (file pointer and mutex) */
+    AnomalyJsonOutputCtx *json_output_ctx = ((OutputCtx *)initdata)->data;
+    aft->file_ctx = json_output_ctx->file_ctx;
+    aft->json_output_ctx = json_output_ctx;
+
+    *data = (void *)aft;
+    return TM_ECODE_OK;
+}
+
+static TmEcode JsonAnomalyLogThreadDeinit(ThreadVars *t, void *data)
+{
+    JsonAnomalyLogThread *aft = (JsonAnomalyLogThread *)data;
+    if (aft == NULL) {
+        return TM_ECODE_OK;
+    }
+
+    MemBufferFree(aft->json_buffer);
+
+    /* clear memory */
+    memset(aft, 0, sizeof(JsonAnomalyLogThread));
+
+    SCFree(aft);
+    return TM_ECODE_OK;
+}
+
+static void JsonAnomalyLogDeInitCtx(OutputCtx *output_ctx)
+{
+    AnomalyJsonOutputCtx *json_output_ctx = (AnomalyJsonOutputCtx *) output_ctx->data;
+    if (json_output_ctx != NULL) {
+        LogFileFreeCtx(json_output_ctx->file_ctx);
+        SCFree(json_output_ctx);
+    }
+    SCFree(output_ctx);
+}
+
+static void JsonAnomalyLogDeInitCtxSub(OutputCtx *output_ctx)
+{
+    SCLogDebug("cleaning up sub output_ctx %p", output_ctx);
+
+    AnomalyJsonOutputCtx *json_output_ctx = (AnomalyJsonOutputCtx *) output_ctx->data;
+
+    if (json_output_ctx != NULL) {
+        SCFree(json_output_ctx);
+    }
+    SCFree(output_ctx);
+}
+
+#define DEFAULT_LOG_FILENAME "anomaly.json"
+static void SetFlag(const ConfNode *conf, const char *name, uint16_t flag, uint16_t *out_flags)
+{
+    DEBUG_VALIDATE_BUG_ON(conf == NULL);
+    const char *setting = ConfNodeLookupChildValue(conf, name);
+    if (setting != NULL) {
+        if (ConfValIsTrue(setting)) {
+            *out_flags |= flag;
+        } else {
+            *out_flags &= ~flag;
+        }
+    }
+}
+
+static void JsonAnomalyLogConf(AnomalyJsonOutputCtx *json_output_ctx,
+        ConfNode *conf)
+{
+    uint16_t flags = 0;
+    if (conf != NULL) {
+        SetFlag(conf, "packethdr", LOG_JSON_PACKET, &flags);
+    }
+    json_output_ctx->flags |= flags;
+}
+
+/**
+ * \brief Create a new LogFileCtx for "fast" output style.
+ * \param conf The configuration node for this output.
+ * \return A LogFileCtx pointer on success, NULL on failure.
+ */
+static OutputInitResult JsonAnomalyLogInitCtx(ConfNode *conf)
+{
+    OutputInitResult result = { NULL, false };
+    AnomalyJsonOutputCtx *json_output_ctx = NULL;
+    LogFileCtx *logfile_ctx = LogFileNewCtx();
+    if (logfile_ctx == NULL) {
+        SCLogDebug("JsonAnomalyLogInitCtx: Could not create new LogFileCtx");
+        return result;
+    }
+
+    if (SCConfLogOpenGeneric(conf, logfile_ctx, DEFAULT_LOG_FILENAME, 1) < 0) {
+        LogFileFreeCtx(logfile_ctx);
+        return result;
+    }
+
+    OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
+    if (unlikely(output_ctx == NULL)) {
+        LogFileFreeCtx(logfile_ctx);
+        return result;
+    }
+
+    json_output_ctx = SCCalloc(1, sizeof(AnomalyJsonOutputCtx));
+    if (unlikely(json_output_ctx == NULL)) {
+        LogFileFreeCtx(logfile_ctx);
+        SCFree(output_ctx);
+        return result;
+    }
+
+    json_output_ctx->file_ctx = logfile_ctx;
+    JsonAnomalyLogConf(json_output_ctx, conf);
+
+    output_ctx->data = json_output_ctx;
+    output_ctx->DeInit = JsonAnomalyLogDeInitCtx;
+
+    result.ctx = output_ctx;
+    result.ok = true;
+    return result;
+}
+
+/**
+ * \brief Create a new LogFileCtx for "fast" output style.
+ * \param conf The configuration node for this output.
+ * \return A LogFileCtx pointer on success, NULL on failure.
+ */
+static OutputInitResult JsonAnomalyLogInitCtxSub(ConfNode *conf, OutputCtx *parent_ctx)
+{
+    OutputInitResult result = { NULL, false };
+    OutputJsonCtx *ajt = parent_ctx->data;
+    AnomalyJsonOutputCtx *json_output_ctx = NULL;
+
+    OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
+    if (unlikely(output_ctx == NULL))
+        return result;
+
+    json_output_ctx = SCCalloc(1, sizeof(AnomalyJsonOutputCtx));
+    if (unlikely(json_output_ctx == NULL)) {
+        goto error;
+    }
+
+    json_output_ctx->file_ctx = ajt->file_ctx;
+    JsonAnomalyLogConf(json_output_ctx, conf);
+    json_output_ctx->cfg = ajt->cfg;
+
+    output_ctx->data = json_output_ctx;
+    output_ctx->DeInit = JsonAnomalyLogDeInitCtxSub;
+
+    result.ctx = output_ctx;
+    result.ok = true;
+    return result;
+
+error:
+    if (json_output_ctx != NULL) {
+        SCFree(json_output_ctx);
+    }
+
+    SCFree(output_ctx);
+
+    return result;
+}
+
+void JsonAnomalyLogRegister (void)
+{
+    OutputRegisterPacketModule(LOGGER_JSON_ANOMALY, MODULE_NAME, "anomaly-json-log",
+        JsonAnomalyLogInitCtx, JsonAnomalyLogger, JsonAnomalyLogCondition,
+        JsonAnomalyLogThreadInit, JsonAnomalyLogThreadDeinit, NULL);
+    OutputRegisterPacketSubModule(LOGGER_JSON_ANOMALY, "eve-log", MODULE_NAME,
+        "eve-log.anomaly", JsonAnomalyLogInitCtxSub, JsonAnomalyLogger,
+        JsonAnomalyLogCondition, JsonAnomalyLogThreadInit, JsonAnomalyLogThreadDeinit,
+        NULL);
+}
+
+#else
+
+void JsonAnomalyLogRegister (void)
+{
+}
+
+#endif

src/output-json-anomaly.h

@@ -0,0 +1,37 @@
+/* Copyright (C) 2019 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Jeff Lucovsky <jeff@lucovsky.org>
+ *
+ * Logs anomalies in JSON format.
+ *
+ */
+
+#ifndef __OUTPUT_JSON_ANOMALY_H__
+#define __OUTPUT_JSON_ANOMALY_H__
+
+void JsonAnomalyLogRegister(void);
+#ifdef HAVE_LIBJANSSON
+void AnomalyJsonHeader(void *ctx, const Packet *p, const PacketAlert *pa, json_t *js,
+                     uint16_t flags);
+#endif /* HAVE_LIBJANSSON */
+
+#endif /* __OUTPUT_JSON_ALERT_H__ */
+

src/output.c

@@ -46,6 +46,7 @@
 #include "alert-prelude.h"
 #include "alert-syslog.h"
 #include "output-json-alert.h"
+#include "output-json-anomaly.h"
 #include "output-json-flow.h"
 #include "output-json-netflow.h"
 #include "log-cf-common.h"
@@ -1079,6 +1080,7 @@ void OutputRegisterLoggers(void)
     LogStatsLogRegister();
 
     JsonAlertLogRegister();
+    JsonAnomalyLogRegister();
     /* flow/netflow */
     JsonFlowLogRegister();
     JsonNetFlowLogRegister();

src/suricata-common.h

@@ -447,6 +447,7 @@ typedef enum {
     LOGGER_ALERT_SYSLOG,
     LOGGER_DROP,
     LOGGER_JSON_ALERT,
+    LOGGER_JSON_ANOMALY,
     LOGGER_JSON_DROP,
     LOGGER_FILE_STORE,
     LOGGER_JSON_FILE,

src/util-profiling.c

@@ -1297,6 +1297,7 @@ const char * PacketProfileLoggertIdToString(LoggerId id)
         CASE_CODE (LOGGER_ALERT_SYSLOG);
         CASE_CODE (LOGGER_DROP);
         CASE_CODE (LOGGER_JSON_ALERT);
+        CASE_CODE (LOGGER_JSON_ANOMALY);
         CASE_CODE (LOGGER_JSON_DROP);
         CASE_CODE (LOGGER_JSON_SSH);
         CASE_CODE (LOGGER_DNS_TS);

suricata.yaml.in

@@ -154,6 +154,8 @@ outputs:
             # Enable the logging of tagged packets for rules using the
             # "tag" keyword.
             tagged-packets: yes
+        - anomaly:
+            # packethdr: no            # enable dumping of packet header
         - http:
             extended: yes     # enable this for extended logging information
             # custom allows additional http fields to be included in eve-log