eve: user callbacks for adding additional data

Provide a way for library/plugin users to register a callback that
will be called prior to an EVE record being closed. The callback will
be passed ThreadVars, Packet, and Flow pointers if available, as well
as private user data.
pull/12117/head
Jason Ish 5 months ago committed by Victor Julien
parent 30bd2a27ff
commit b660ff8aff

@ -425,7 +425,7 @@ static int EveStreamLogger(ThreadVars *tv, void *thread_data, const Packet *p)
/* Close stream. */
jb_close(js);
OutputJsonBuilderBuffer(js, td->ctx);
OutputJsonBuilderBuffer(tv, p, p->flow, js, td->ctx);
jb_free(js);
return TM_ECODE_OK;

@ -15,11 +15,50 @@
* 02110-1301, USA.
*/
#include "suricata-common.h"
#include "output-eve.h"
#include "util-debug.h"
#include "rust.h"
typedef struct EveUserCallback_ {
SCEveUserCallbackFn Callback;
void *user;
struct EveUserCallback_ *next;
} EveUserCallback;
static EveUserCallback *eve_user_callbacks = NULL;
static TAILQ_HEAD(, SCEveFileType_) output_types = TAILQ_HEAD_INITIALIZER(output_types);
bool SCEveRegisterCallback(SCEveUserCallbackFn fn, void *user)
{
EveUserCallback *cb = SCCalloc(1, sizeof(*cb));
if (cb == NULL) {
return false;
}
cb->Callback = fn;
cb->user = user;
if (eve_user_callbacks == NULL) {
eve_user_callbacks = cb;
} else {
EveUserCallback *current = eve_user_callbacks;
while (current->next != NULL) {
current = current->next;
}
current->next = cb;
}
return true;
}
void SCEveRunCallbacks(ThreadVars *tv, const Packet *p, Flow *f, JsonBuilder *jb)
{
EveUserCallback *cb = eve_user_callbacks;
while (cb != NULL) {
cb->Callback(tv, p, f, jb, cb->user);
cb = cb->next;
}
}
static bool IsBuiltinTypeName(const char *name)
{
const char *builtin[] = {

@ -31,6 +31,7 @@
#define SURICATA_OUTPUT_EVE_H
#include "suricata-common.h"
#include "rust.h"
#include "conf.h"
typedef uint32_t ThreadId;
@ -173,4 +174,46 @@ bool SCRegisterEveFileType(SCEveFileType *);
SCEveFileType *SCEveFindFileType(const char *name);
/** \brief Function type for EVE callbacks.
*
* The function type for callbacks registered with
* SCEveRegisterCallback. This function will be called with the
* JsonBuilder just prior to the top-level object being closed. New
* fields maybe added, however there is no way to alter existing
* objects already added to the JsonBuilder.
*
* \param tv The ThreadVars for the thread performing the logging.
* \param p Packet if available.
* \param f Flow if available.
* \param user User data provided during callback registration.
*/
typedef void (*SCEveUserCallbackFn)(
ThreadVars *tv, const Packet *p, Flow *f, JsonBuilder *jb, void *user);
/** \brief Register a callback for adding extra information to EVE logs.
*
* Allow users to register a callback for each EVE log. The callback
* is called just before the root object on the JsonBuilder is to be
* closed.
*
* New objects and fields can be append, but exist entries cannot be modified.
*
* Packet and Flow will be provided if available, but will other be
* NULL.
*
* Limitations: At this time the callbacks will only be called for EVE
* loggers that use JsonBuilder, notably this means it won't be called
* for stats records at this time.
*
* \returns true if callback is registered, false is not due to memory
* allocation error.
*/
bool SCEveRegisterCallback(SCEveUserCallbackFn fn, void *user);
/** \internal
*
* Run EVE callbacks.
*/
void SCEveRunCallbacks(ThreadVars *tv, const Packet *p, Flow *f, JsonBuilder *jb);
#endif

@ -757,7 +757,7 @@ static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p)
EveAddVerdict(jb, p);
}
OutputJsonBuilderBuffer(jb, aft->ctx);
OutputJsonBuilderBuffer(tv, p, p->flow, jb, aft->ctx);
jb_free(jb);
}
@ -767,7 +767,7 @@ static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p)
CreateEveHeader(p, LOG_DIR_PACKET, "packet", NULL, json_output_ctx->eve_ctx);
if (unlikely(packetjs != NULL)) {
EvePacket(p, packetjs, 0);
OutputJsonBuilderBuffer(packetjs, aft->ctx);
OutputJsonBuilderBuffer(tv, p, p->flow, packetjs, aft->ctx);
jb_free(packetjs);
}
}
@ -801,7 +801,7 @@ static int AlertJsonDecoderEvent(ThreadVars *tv, JsonAlertLogThread *aft, const
AlertJsonHeader(p, pa, jb, json_output_ctx->flags, NULL, NULL);
OutputJsonBuilderBuffer(jb, aft->ctx);
OutputJsonBuilderBuffer(tv, p, p->flow, jb, aft->ctx);
jb_free(jb);
}

@ -143,16 +143,16 @@ static int AnomalyDecodeEventJson(ThreadVars *tv, JsonAnomalyLogThread *aft,
EvePacket(p, js, GET_PKT_LEN(p) < 32 ? GET_PKT_LEN(p) : 32);
}
OutputJsonBuilderBuffer(js, aft->ctx);
OutputJsonBuilderBuffer(tv, p, p->flow, js, aft->ctx);
jb_free(js);
}
return TM_ECODE_OK;
}
static int AnomalyAppLayerDecoderEventJson(JsonAnomalyLogThread *aft,
const Packet *p, AppLayerDecoderEvents *decoder_events,
bool is_pktlayer, const char *layer, uint64_t tx_id)
static int AnomalyAppLayerDecoderEventJson(ThreadVars *tv, JsonAnomalyLogThread *aft,
const Packet *p, AppLayerDecoderEvents *decoder_events, bool is_pktlayer, const char *layer,
uint64_t tx_id)
{
const char *alprotoname = AppLayerGetProtoName(p->flow->alproto);
@ -201,7 +201,7 @@ static int AnomalyAppLayerDecoderEventJson(JsonAnomalyLogThread *aft,
/* anomaly */
jb_close(js);
OutputJsonBuilderBuffer(js, aft->ctx);
OutputJsonBuilderBuffer(tv, p, p->flow, js, aft->ctx);
jb_free(js);
/* Current implementation assumes a single owner for this value */
@ -223,8 +223,7 @@ static int JsonAnomalyTxLogger(ThreadVars *tv, void *thread_data, const Packet *
decoder_events = AppLayerParserGetEventsByTx(f->proto, f->alproto, tx);
if (decoder_events && decoder_events->event_last_logged < decoder_events->cnt) {
SCLogDebug("state %p, tx: %p, tx_id: %"PRIu64, state, tx, tx_id);
AnomalyAppLayerDecoderEventJson(aft, p, decoder_events, false,
"proto_parser", tx_id);
AnomalyAppLayerDecoderEventJson(tv, aft, p, decoder_events, false, "proto_parser", tx_id);
}
return TM_ECODE_OK;
}
@ -255,8 +254,8 @@ static int AnomalyJson(ThreadVars *tv, JsonAnomalyLogThread *aft, const Packet *
if (aft->json_output_ctx->flags & LOG_JSON_APPLAYER_TYPE) {
/* app layer proto detect events */
if (rc == TM_ECODE_OK && AnomalyHasPacketAppLayerEvents(p)) {
rc = AnomalyAppLayerDecoderEventJson(aft, p, p->app_layer_events,
true, "proto_detect", TX_ID_UNUSED);
rc = AnomalyAppLayerDecoderEventJson(
tv, aft, p, p->app_layer_events, true, "proto_detect", TX_ID_UNUSED);
}
/* parser state events */
@ -264,8 +263,8 @@ static int AnomalyJson(ThreadVars *tv, JsonAnomalyLogThread *aft, const Packet *
SCLogDebug("Checking for anomaly events; alproto %d", p->flow->alproto);
AppLayerDecoderEvents *parser_events = AppLayerParserGetDecoderEvents(p->flow->alparser);
if (parser_events && (parser_events->event_last_logged < parser_events->cnt)) {
rc = AnomalyAppLayerDecoderEventJson(aft, p, parser_events,
false, "parser", TX_ID_UNUSED);
rc = AnomalyAppLayerDecoderEventJson(
tv, aft, p, parser_events, false, "parser", TX_ID_UNUSED);
}
}
}

@ -90,7 +90,7 @@ static int JsonArpLogger(ThreadVars *tv, void *thread_data, const Packet *p)
JSONFormatAndAddMACAddr(jb, "dest_mac", arph->dest_mac, false);
jb_set_string(jb, "dest_ip", dstip);
jb_close(jb); /* arp */
OutputJsonBuilderBuffer(jb, thread);
OutputJsonBuilderBuffer(tv, p, p->flow, jb, thread);
jb_free(jb);
return TM_ECODE_OK;

@ -47,7 +47,7 @@ static int JsonDCERPCLogger(ThreadVars *tv, void *thread_data,
jb_close(jb);
MemBufferReset(thread->buffer);
OutputJsonBuilderBuffer(jb, thread);
OutputJsonBuilderBuffer(tv, p, p->flow, jb, thread);
jb_free(jb);
return TM_ECODE_OK;

@ -72,7 +72,7 @@ static int JsonDHCPLogger(ThreadVars *tv, void *thread_data,
rs_dhcp_logger_log(ctx->rs_logger, tx, js);
OutputJsonBuilderBuffer(js, thread->thread);
OutputJsonBuilderBuffer(tv, p, p->flow, js, thread->thread);
jb_free(js);
return TM_ECODE_OK;

@ -246,7 +246,7 @@ static int JsonDNP3LoggerToServer(ThreadVars *tv, void *thread_data,
jb_open_object(js, "dnp3");
JsonDNP3LogRequest(js, tx);
jb_close(js);
OutputJsonBuilderBuffer(js, thread->ctx);
OutputJsonBuilderBuffer(tv, p, p->flow, js, thread->ctx);
jb_free(js);
SCReturnInt(TM_ECODE_OK);
@ -267,7 +267,7 @@ static int JsonDNP3LoggerToClient(ThreadVars *tv, void *thread_data,
jb_open_object(js, "dnp3");
JsonDNP3LogResponse(js, tx);
jb_close(js);
OutputJsonBuilderBuffer(js, thread->ctx);
OutputJsonBuilderBuffer(tv, p, p->flow, js, thread->ctx);
jb_free(js);
SCReturnInt(TM_ECODE_OK);

@ -331,7 +331,7 @@ static int JsonDoh2Logger(ThreadVars *tv, void *thread_data, const Packet *p, Fl
}
out:
if (r || r2) {
OutputJsonBuilderBuffer(jb, td->ctx);
OutputJsonBuilderBuffer(tv, p, p->flow, jb, td->ctx);
}
jb_free(jb);
return TM_ECODE_OK;
@ -363,7 +363,7 @@ static int JsonDnsLoggerToServer(ThreadVars *tv, void *thread_data,
}
jb_close(jb);
OutputJsonBuilderBuffer(jb, td->ctx);
OutputJsonBuilderBuffer(tv, p, p->flow, jb, td->ctx);
jb_free(jb);
}
@ -392,7 +392,7 @@ static int JsonDnsLoggerToClient(ThreadVars *tv, void *thread_data,
jb_set_int(jb, "version", 2);
SCDnsLogJsonAnswer(txptr, td->dnslog_ctx->flags, jb);
jb_close(jb);
OutputJsonBuilderBuffer(jb, td->ctx);
OutputJsonBuilderBuffer(tv, p, p->flow, jb, td->ctx);
jb_free(jb);
}
@ -432,7 +432,7 @@ static int JsonDnsLogger(ThreadVars *tv, void *thread_data, const Packet *p, Flo
}
if (SCDnsLogJson(txptr, td->dnslog_ctx->flags, jb)) {
OutputJsonBuilderBuffer(jb, td->ctx);
OutputJsonBuilderBuffer(tv, p, p->flow, jb, td->ctx);
}
jb_free(jb);
}

@ -85,7 +85,7 @@ static int g_droplog_flows_start = 1;
*
* \return return TM_ECODE_OK on success
*/
static int DropLogJSON (JsonDropLogThread *aft, const Packet *p)
static int DropLogJSON(ThreadVars *tv, JsonDropLogThread *aft, const Packet *p)
{
JsonDropOutputCtx *drop_ctx = aft->drop_ctx;
@ -191,7 +191,7 @@ static int DropLogJSON (JsonDropLogThread *aft, const Packet *p)
}
}
OutputJsonBuilderBuffer(js, aft->ctx);
OutputJsonBuilderBuffer(tv, p, p->flow, js, aft->ctx);
jb_free(js);
return TM_ECODE_OK;
@ -326,7 +326,7 @@ static OutputInitResult JsonDropLogInitCtxSub(ConfNode *conf, OutputCtx *parent_
static int JsonDropLogger(ThreadVars *tv, void *thread_data, const Packet *p)
{
JsonDropLogThread *td = thread_data;
int r = DropLogJSON(td, p);
int r = DropLogJSON(tv, td, p);
if (r < 0)
return -1;

@ -213,8 +213,8 @@ JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, void *tx,
* \internal
* \brief Write meta data on a single line json record
*/
static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const File *ff, void *tx,
const uint64_t tx_id, uint8_t dir, OutputJsonCtx *eve_ctx)
static void FileWriteJsonRecord(ThreadVars *tv, JsonFileLogThread *aft, const Packet *p,
const File *ff, void *tx, const uint64_t tx_id, uint8_t dir, OutputJsonCtx *eve_ctx)
{
HttpXFFCfg *xff_cfg = aft->filelog_ctx->xff_cfg != NULL ? aft->filelog_ctx->xff_cfg
: aft->filelog_ctx->parent_xff_cfg;
@ -223,7 +223,7 @@ static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const F
return;
}
OutputJsonBuilderBuffer(js, aft->ctx);
OutputJsonBuilderBuffer(tv, p, p->flow, js, aft->ctx);
jb_free(js);
}
@ -237,7 +237,7 @@ static int JsonFileLogger(ThreadVars *tv, void *thread_data, const Packet *p, co
SCLogDebug("ff %p", ff);
FileWriteJsonRecord(aft, p, ff, tx, tx_id, dir, aft->filelog_ctx->eve_ctx);
FileWriteJsonRecord(tv, aft, p, ff, tx, tx_id, dir, aft->filelog_ctx->eve_ctx);
return 0;
}

@ -340,7 +340,7 @@ static int JsonFlowLogger(ThreadVars *tv, void *thread_data, Flow *f)
EveFlowLogJSON(thread, jb, f);
OutputJsonBuilderBuffer(jb, thread);
OutputJsonBuilderBuffer(tv, NULL, f, jb, thread);
jb_free(jb);
SCReturnInt(TM_ECODE_OK);

@ -287,8 +287,8 @@ void FrameJsonLogOneFrame(const uint8_t ipproto, const Frame *frame, Flow *f,
jb_close(jb);
}
static int FrameJsonUdp(
JsonFrameLogThread *aft, const Packet *p, Flow *f, FramesContainer *frames_container)
static int FrameJsonUdp(ThreadVars *tv, JsonFrameLogThread *aft, const Packet *p, Flow *f,
FramesContainer *frames_container)
{
FrameJsonOutputCtx *json_output_ctx = aft->json_output_ctx;
@ -315,7 +315,7 @@ static int FrameJsonUdp(
jb_set_string(jb, "app_proto", AppProtoToString(f->alproto));
FrameJsonLogOneFrame(IPPROTO_UDP, frame, p->flow, NULL, p, jb, aft->payload_buffer);
OutputJsonBuilderBuffer(jb, aft->ctx);
OutputJsonBuilderBuffer(tv, p, p->flow, jb, aft->ctx);
jb_free(jb);
frame->flags |= FRAME_FLAG_LOGGED;
}
@ -333,7 +333,7 @@ static int FrameJson(ThreadVars *tv, JsonFrameLogThread *aft, const Packet *p)
return TM_ECODE_OK;
if (p->proto == IPPROTO_UDP) {
return FrameJsonUdp(aft, p, p->flow, frames_container);
return FrameJsonUdp(tv, aft, p, p->flow, frames_container);
}
BUG_ON(p->proto != IPPROTO_TCP);
@ -387,7 +387,7 @@ static int FrameJson(ThreadVars *tv, JsonFrameLogThread *aft, const Packet *p)
jb_set_string(jb, "app_proto", AppProtoToString(p->flow->alproto));
FrameJsonLogOneFrame(IPPROTO_TCP, frame, p->flow, stream, p, jb, aft->payload_buffer);
OutputJsonBuilderBuffer(jb, aft->ctx);
OutputJsonBuilderBuffer(tv, p, p->flow, jb, aft->ctx);
jb_free(jb);
frame->flags |= FRAME_FLAG_LOGGED;
} else if (frame != NULL) {

@ -493,7 +493,7 @@ static int JsonHttpLogger(ThreadVars *tv, void *thread_data, const Packet *p, Fl
}
}
OutputJsonBuilderBuffer(js, jhl->ctx);
OutputJsonBuilderBuffer(tv, p, p->flow, js, jhl->ctx);
jb_free(js);
SCReturnInt(TM_ECODE_OK);

@ -90,7 +90,7 @@ static int JsonIKELogger(ThreadVars *tv, void *thread_data, const Packet *p, Flo
goto error;
}
OutputJsonBuilderBuffer(jb, thread->ctx);
OutputJsonBuilderBuffer(tv, p, p->flow, jb, thread->ctx);
jb_free(jb);
return TM_ECODE_OK;

@ -74,7 +74,7 @@ static int MetadataJson(ThreadVars *tv, OutputJsonThreadCtx *aft, const Packet *
if (!aft->ctx->cfg.include_metadata) {
EveAddMetadata(p, p->flow, js);
}
OutputJsonBuilderBuffer(js, aft);
OutputJsonBuilderBuffer(tv, p, p->flow, js, aft);
jb_free(js);
return TM_ECODE_OK;

@ -85,7 +85,7 @@ static int JsonMQTTLogger(ThreadVars *tv, void *thread_data,
if (!rs_mqtt_logger_log(tx, thread->mqttlog_ctx->flags, thread->mqttlog_ctx->max_log_len, js))
goto error;
OutputJsonBuilderBuffer(js, thread->ctx);
OutputJsonBuilderBuffer(tv, p, p->flow, js, thread->ctx);
jb_free(js);
return TM_ECODE_OK;

@ -276,7 +276,7 @@ static int JsonNetFlowLogger(ThreadVars *tv, void *thread_data, Flow *f)
return TM_ECODE_OK;
NetFlowLogEveToServer(jb, f);
EveAddCommonOptions(&jhl->ctx->cfg, NULL, f, jb, LOG_DIR_FLOW_TOSERVER);
OutputJsonBuilderBuffer(jb, jhl);
OutputJsonBuilderBuffer(tv, NULL, f, jb, jhl);
jb_free(jb);
/* only log a response record if we actually have seen response packets */
@ -286,7 +286,7 @@ static int JsonNetFlowLogger(ThreadVars *tv, void *thread_data, Flow *f)
return TM_ECODE_OK;
NetFlowLogEveToClient(jb, f);
EveAddCommonOptions(&jhl->ctx->cfg, NULL, f, jb, LOG_DIR_FLOW_TOCLIENT);
OutputJsonBuilderBuffer(jb, jhl);
OutputJsonBuilderBuffer(tv, NULL, f, jb, jhl);
jb_free(jb);
}
SCReturnInt(TM_ECODE_OK);

@ -94,7 +94,7 @@ static int JsonNFSLogger(ThreadVars *tv, void *thread_data,
jb_close(jb);
MemBufferReset(thread->buffer);
OutputJsonBuilderBuffer(jb, thread);
OutputJsonBuilderBuffer(tv, p, p->flow, jb, thread);
jb_free(jb);
return TM_ECODE_OK;
}

@ -80,7 +80,7 @@ static int JsonPgsqlLogger(ThreadVars *tv, void *thread_data, const Packet *p, F
goto error;
}
OutputJsonBuilderBuffer(jb, thread->ctx);
OutputJsonBuilderBuffer(tv, p, p->flow, jb, thread->ctx);
jb_free(jb);
return TM_ECODE_OK;

@ -59,7 +59,7 @@ static int JsonSMBLogger(ThreadVars *tv, void *thread_data,
}
jb_close(jb);
OutputJsonBuilderBuffer(jb, thread);
OutputJsonBuilderBuffer(tv, p, p->flow, jb, thread);
jb_free(jb);
return TM_ECODE_OK;

@ -85,7 +85,7 @@ static int JsonSmtpLogger(ThreadVars *tv, void *thread_data, const Packet *p, Fl
jb_close(jb);
EveEmailLogJson(jhl, jb, p, f, state, tx, tx_id);
OutputJsonBuilderBuffer(jb, jhl->ctx);
OutputJsonBuilderBuffer(tv, p, p->flow, jb, jhl->ctx);
jb_free(jb);

@ -501,7 +501,7 @@ static int JsonTlsLogger(ThreadVars *tv, void *thread_data, const Packet *p,
/* Close the tls object. */
jb_close(js);
OutputJsonBuilderBuffer(js, aft->ctx);
OutputJsonBuilderBuffer(tv, p, p->flow, js, aft->ctx);
jb_free(js);
return 0;

@ -955,7 +955,8 @@ int OutputJSONBuffer(json_t *js, LogFileCtx *file_ctx, MemBuffer **buffer)
return 0;
}
int OutputJsonBuilderBuffer(JsonBuilder *js, OutputJsonThreadCtx *ctx)
int OutputJsonBuilderBuffer(
ThreadVars *tv, const Packet *p, Flow *f, JsonBuilder *js, OutputJsonThreadCtx *ctx)
{
LogFileCtx *file_ctx = ctx->file_ctx;
MemBuffer **buffer = &ctx->buffer;
@ -967,6 +968,8 @@ int OutputJsonBuilderBuffer(JsonBuilder *js, OutputJsonThreadCtx *ctx)
jb_set_string(js, "pcap_filename", PcapFileGetFilename());
}
SCEveRunCallbacks(tv, p, f, js);
jb_close(js);
MemBufferReset(*buffer);

@ -103,7 +103,8 @@ JsonBuilder *CreateEveHeader(const Packet *p, enum OutputJsonLogDirection dir,
JsonBuilder *CreateEveHeaderWithTxId(const Packet *p, enum OutputJsonLogDirection dir,
const char *event_type, JsonAddrInfo *addr, uint64_t tx_id, OutputJsonCtx *eve_ctx);
int OutputJSONBuffer(json_t *js, LogFileCtx *file_ctx, MemBuffer **buffer);
int OutputJsonBuilderBuffer(JsonBuilder *js, OutputJsonThreadCtx *ctx);
int OutputJsonBuilderBuffer(
ThreadVars *tv, const Packet *p, Flow *f, JsonBuilder *js, OutputJsonThreadCtx *ctx);
OutputInitResult OutputJsonInitCtx(ConfNode *);
OutputInitResult OutputJsonLogInitSub(ConfNode *conf, OutputCtx *parent_ctx);

@ -927,7 +927,7 @@ static int JsonGenericLogger(ThreadVars *tv, void *thread_data, const Packet *p,
goto error;
}
OutputJsonBuilderBuffer(js, thread);
OutputJsonBuilderBuffer(tv, p, p->flow, js, thread);
jb_free(js);
return TM_ECODE_OK;

Loading…
Cancel
Save