From bc71a43e089b5a028c13d7861b87c7c53fb97698 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Wed, 29 Jan 2014 14:36:22 +0100 Subject: [PATCH] http-json: separate module using tx api Turn HTTP json logger into a Tx Logger API logger. --- src/output-httplog.c | 216 ++++++++++++++++++++-------------------- src/output-httplog.h | 2 + src/output-json.c | 28 +++++- src/output-json.h | 4 + src/suricata.c | 2 + src/tm-threads-common.h | 1 + 6 files changed, 145 insertions(+), 108 deletions(-) diff --git a/src/output-httplog.c b/src/output-httplog.c index 228c88b677..607052ba3a 100644 --- a/src/output-httplog.c +++ b/src/output-httplog.c @@ -53,43 +53,28 @@ #ifdef HAVE_LIBJANSSON #include -#define LOG_HTTP_MAXN_NODES 64 -#define LOG_HTTP_NODE_STRLEN 256 -#define LOG_HTTP_NODE_MAXOUTPUTLEN 8192 - -#define TIMESTAMP_DEFAULT_FORMAT "%b %d, %Y; %H:%M:%S" -#define LOG_HTTP_CF_NONE "-" -#define LOG_HTTP_CF_LITERAL '%' -#define LOG_HTTP_CF_REQUEST_HOST 'h' -#define LOG_HTTP_CF_REQUEST_PROTOCOL 'H' -#define LOG_HTTP_CF_REQUEST_METHOD 'm' -#define LOG_HTTP_CF_REQUEST_URI 'u' -#define LOG_HTTP_CF_REQUEST_TIME 't' -#define LOG_HTTP_CF_REQUEST_HEADER 'i' -#define LOG_HTTP_CF_REQUEST_COOKIE 'C' -#define LOG_HTTP_CF_REQUEST_LEN 'b' -#define LOG_HTTP_CF_RESPONSE_STATUS 's' -#define LOG_HTTP_CF_RESPONSE_HEADER 'o' -#define LOG_HTTP_CF_RESPONSE_LEN 'B' -#define LOG_HTTP_CF_TIMESTAMP 't' -#define LOG_HTTP_CF_TIMESTAMP_U 'z' -#define LOG_HTTP_CF_CLIENT_IP 'a' -#define LOG_HTTP_CF_SERVER_IP 'A' -#define LOG_HTTP_CF_CLIENT_PORT 'p' -#define LOG_HTTP_CF_SERVER_PORT 'P' - -typedef struct OutputHttpCtx_ { +typedef struct LogHttpFileCtx_ { + LogFileCtx *file_ctx; uint32_t flags; /** Store mode */ -} OutputHttpCtx; +} LogHttpFileCtx; + +typedef struct JsonHttpLogThread_ { + LogHttpFileCtx *httplog_ctx; + /** LogFileCtx has the pointer to the file and a mutex to allow multithreading */ + uint32_t uri_cnt; + + MemBuffer *buffer; +} JsonHttpLogThread; + #define LOG_HTTP_DEFAULT 0 #define LOG_HTTP_EXTENDED 1 #define LOG_HTTP_CUSTOM 2 /* JSON format logging */ -static void LogHttpLogJSON(AlertJsonThread *aft, json_t *js, htp_tx_t *tx) +static void JsonHttpLogJSON(JsonHttpLogThread *aft, json_t *js, htp_tx_t *tx) { - OutputHttpCtx *http_ctx = aft->http_ctx->data; + LogHttpFileCtx *http_ctx = aft->httplog_ctx; json_t *hjs = json_object(); if (hjs == NULL) { return; @@ -162,6 +147,7 @@ static void LogHttpLogJSON(AlertJsonThread *aft, json_t *js, htp_tx_t *tx) SCFree(c); } +#if 1 if (http_ctx->flags & LOG_HTTP_EXTENDED) { /* referer */ htp_header_t *h_referer = NULL; @@ -215,107 +201,54 @@ static void LogHttpLogJSON(AlertJsonThread *aft, json_t *js, htp_tx_t *tx) /* length */ json_object_set_new(hjs, "length", json_integer(tx->response_message_len)); } +#endif json_object_set_new(js, "http", hjs); } -static TmEcode HttpJsonIPWrapper(ThreadVars *tv, Packet *p, void *data) +static int JsonHttpLogger(ThreadVars *tv, void *thread_data, const Packet *p, Flow *f, void *alstate, void *txptr, uint64_t tx_id) { SCEnter(); - uint64_t tx_id = 0; - uint64_t total_txs = 0; - htp_tx_t *tx = NULL; - HtpState *htp_state = NULL; - int tx_progress = 0; - int tx_progress_done_value_ts = 0; - int tx_progress_done_value_tc = 0; - AlertJsonThread *aft = (AlertJsonThread *)data; - MemBuffer *buffer = (MemBuffer *)aft->buffer; - - /* no flow, no htp state */ - if (p->flow == NULL) { - SCReturnInt(TM_ECODE_OK); - } - - /* check if we have HTTP state or not */ - FLOWLOCK_WRLOCK(p->flow); /* WRITE lock before we updated flow logged id */ - uint16_t proto = FlowGetAppProtocol(p->flow); - if (proto != ALPROTO_HTTP) - goto end; - - htp_state = (HtpState *)FlowGetAppState(p->flow); - if (htp_state == NULL) { - SCLogDebug("no http state, so no request logging"); - goto end; - } - - total_txs = AppLayerParserGetTxCnt(IPPROTO_TCP, ALPROTO_HTTP, htp_state); - tx_id = AppLayerParserGetTransactionLogId(p->flow->alparser); - tx_progress_done_value_ts = AppLayerParserGetStateProgressCompletionStatus(p->proto, ALPROTO_HTTP, 0); - tx_progress_done_value_tc = AppLayerParserGetStateProgressCompletionStatus(p->proto, ALPROTO_HTTP, 1); + htp_tx_t *tx = txptr; + JsonHttpLogThread *jhl = (JsonHttpLogThread *)thread_data; + MemBuffer *buffer = (MemBuffer *)jhl->buffer; - json_t *js = CreateJSONHeader(p, 1); + json_t *js = CreateJSONHeader((Packet *)p, 1); //TODO const if (unlikely(js == NULL)) return TM_ECODE_OK; - for (; tx_id < total_txs; tx_id++) - { - tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, tx_id); - if (tx == NULL) { - SCLogDebug("tx is NULL not logging !!"); - continue; - } - - if (!(AppLayerParserStateIssetFlag(p->flow->alparser, APP_LAYER_PARSER_EOF))) { - tx_progress = AppLayerParserGetStateProgress(p->proto, ALPROTO_HTTP, tx, 0); - if (tx_progress < tx_progress_done_value_ts) - break; - - tx_progress = AppLayerParserGetStateProgress(p->proto, ALPROTO_HTTP, tx, 1); - if (tx_progress < tx_progress_done_value_tc) - break; - } - - SCLogDebug("got a HTTP request and now logging !!"); + SCLogDebug("got a HTTP request and now logging !!"); - /* reset */ - MemBufferReset(buffer); + /* reset */ + MemBufferReset(buffer); - /* Maybe we'll do a "custom" later - if (http_ctx->flags & LOG_HTTP_CUSTOM) { - LogHttpLogJSONCustom(aft, js, tx, &p->ts); - } else { - */ - LogHttpLogJSON(aft, js, tx); - /* - } - */ + JsonHttpLogJSON(jhl, js, tx); - OutputJSON(js, aft, &aft->http_cnt); - json_object_del(js, "http"); + OutputJSONBuffer(js, jhl->httplog_ctx->file_ctx, buffer); + json_object_del(js, "http"); - AppLayerParserSetTransactionLogId(p->flow->alparser); - } json_object_clear(js); json_decref(js); -end: - FLOWLOCK_UNLOCK(p->flow); - SCReturnInt(TM_ECODE_OK); - -} - -TmEcode OutputHttpLog (ThreadVars *tv, Packet *p, void *data) -{ - SCEnter(); - HttpJsonIPWrapper(tv, p, data); SCReturnInt(TM_ECODE_OK); } +#define DEFAULT_LOG_FILENAME "http.json" OutputCtx *OutputHttpLogInit(ConfNode *conf) { - OutputHttpCtx *http_ctx = SCMalloc(sizeof(OutputHttpCtx)); + LogFileCtx *file_ctx = LogFileNewCtx(); + if(file_ctx == NULL) { + SCLogError(SC_ERR_HTTP_LOG_GENERIC, "couldn't create new file_ctx"); + return NULL; + } + + if (SCConfLogOpenGeneric(conf, file_ctx, DEFAULT_LOG_FILENAME) < 0) { + LogFileFreeCtx(file_ctx); + return NULL; + } + + LogHttpFileCtx *http_ctx = SCMalloc(sizeof(LogHttpFileCtx)); if (unlikely(http_ctx == NULL)) return NULL; @@ -323,6 +256,7 @@ OutputCtx *OutputHttpLogInit(ConfNode *conf) if (unlikely(output_ctx == NULL)) return NULL; + http_ctx->file_ctx = file_ctx; http_ctx->flags = LOG_HTTP_DEFAULT; if (conf) { @@ -340,4 +274,72 @@ OutputCtx *OutputHttpLogInit(ConfNode *conf) return output_ctx; } +#define OUTPUT_BUFFER_SIZE 65535 +static TmEcode JsonHttpLogThreadInit(ThreadVars *t, void *initdata, void **data) +{ + JsonHttpLogThread *aft = SCMalloc(sizeof(JsonHttpLogThread)); + if (unlikely(aft == NULL)) + return TM_ECODE_FAILED; + memset(aft, 0, sizeof(JsonHttpLogThread)); + + if(initdata == NULL) + { + SCLogDebug("Error getting context for HTTPLog. \"initdata\" argument NULL"); + SCFree(aft); + return TM_ECODE_FAILED; + } + + /* Use the Ouptut Context (file pointer and mutex) */ + aft->httplog_ctx = ((OutputCtx *)initdata)->data; //TODO + + aft->buffer = MemBufferCreateNew(OUTPUT_BUFFER_SIZE); + if (aft->buffer == NULL) { + SCFree(aft); + return TM_ECODE_FAILED; + } + + *data = (void *)aft; + return TM_ECODE_OK; +} + +static TmEcode JsonHttpLogThreadDeinit(ThreadVars *t, void *data) +{ + JsonHttpLogThread *aft = (JsonHttpLogThread *)data; + if (aft == NULL) { + return TM_ECODE_OK; + } + + MemBufferFree(aft->buffer); + /* clear memory */ + memset(aft, 0, sizeof(JsonHttpLogThread)); + + SCFree(aft); + return TM_ECODE_OK; +} + +void TmModuleJsonHttpLogRegister (void) { + tmm_modules[TMM_JSONHTTPLOG].name = "JsonHttpLog"; + tmm_modules[TMM_JSONHTTPLOG].ThreadInit = JsonHttpLogThreadInit; + tmm_modules[TMM_JSONHTTPLOG].ThreadDeinit = JsonHttpLogThreadDeinit; + tmm_modules[TMM_JSONHTTPLOG].RegisterTests = NULL; + tmm_modules[TMM_JSONHTTPLOG].cap_flags = 0; + + OutputRegisterTxModule("JsonHttpLog", "http-json-log", OutputHttpLogInit, + ALPROTO_HTTP, JsonHttpLogger); +} + +#else + +static TmEcode OutputJsonThreadInit(ThreadVars *t, void *initdata, void **data) +{ + SCLogInfo("Can't init JSON output - JSON support was disabled during build."); + return TM_ECODE_FAILED; +} + +void TmModuleJsonHttpLogRegister (void) +{ + tmm_modules[TMM_JSONHTTPLOG].name = "JsonHttpLog"; + tmm_modules[TMM_JSONHTTPLOG].ThreadInit = OutputJsonThreadInit; +} + #endif diff --git a/src/output-httplog.h b/src/output-httplog.h index c2bb6a6a61..dc1434aaf3 100644 --- a/src/output-httplog.h +++ b/src/output-httplog.h @@ -27,5 +27,7 @@ TmEcode OutputHttpLog (ThreadVars *tv, Packet *p, void *data); OutputCtx *OutputHttpLogInit(ConfNode *); +void TmModuleJsonHttpLogRegister (void); + #endif /* __OUTPUT_HTTPLOG_H__ */ diff --git a/src/output-json.c b/src/output-json.c index f864270716..ba4bc51b41 100644 --- a/src/output-json.c +++ b/src/output-json.c @@ -303,6 +303,30 @@ json_t *CreateJSONHeader(Packet *p, int direction_sensitive) return js; } +int OutputJSONBuffer(json_t *js, LogFileCtx *file_ctx, MemBuffer *buffer) { + char *js_s = json_dumps(js, + JSON_PRESERVE_ORDER|JSON_COMPACT|JSON_ENSURE_ASCII| +#ifdef JSON_ESCAPE_SLASH + JSON_ESCAPE_SLASH +#else + 0 +#endif + ); + if (unlikely(js_s == NULL)) + return TM_ECODE_OK; + + SCMutexLock(&file_ctx->fp_mutex); + if (json_out == ALERT_SYSLOG) { + syslog(alert_syslog_level, "%s", js_s); + } else if (json_out == ALERT_FILE) { + MemBufferWriteString(buffer, "%s\n", js_s); + (void)MemBufferPrintToFPAsString(buffer, file_ctx->fp); + fflush(file_ctx->fp); + } + SCMutexUnlock(&file_ctx->fp_mutex); + return 0; +} + TmEcode OutputJSON(json_t *js, void *data, uint64_t *count) { AlertJsonThread *aft = (AlertJsonThread *)data; @@ -482,7 +506,7 @@ TmEcode OutputJson (ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, Pack } if (output_flags & OUTPUT_HTTP) { - OutputHttpLog(tv, p, data); +// OutputHttpLog(tv, p, data); } if (output_flags & OUTPUT_TLS) { @@ -679,6 +703,7 @@ OutputCtx *OutputJsonInitCtx(ConfNode *conf) output_flags |= OUTPUT_FILES; continue; } +#if 0 if (strcmp(output->val, "http") == 0) { SCLogDebug("Enabling HTTP output"); ConfNode *child = ConfNodeLookupChild(output, "http"); @@ -687,6 +712,7 @@ OutputCtx *OutputJsonInitCtx(ConfNode *conf) output_flags |= OUTPUT_HTTP; continue; } +#endif if (strcmp(output->val, "tls") == 0) { SCLogDebug("Enabling TLS output"); ConfNode *child = ConfNodeLookupChild(output, "tls"); diff --git a/src/output-json.h b/src/output-json.h index c0129a3f9e..fade780d03 100644 --- a/src/output-json.h +++ b/src/output-json.h @@ -28,8 +28,12 @@ void TmModuleOutputJsonRegister (void); #ifdef HAVE_LIBJANSSON +#include "suricata-common.h" +#include "util-buffer.h" + json_t *CreateJSONHeader(Packet *p, int direction_sensative); TmEcode OutputJSON(json_t *js, void *data, uint64_t *count); +int OutputJSONBuffer(json_t *js, LogFileCtx *file_ctx, MemBuffer *buffer); OutputCtx *OutputJsonInitCtx(ConfNode *); diff --git a/src/suricata.c b/src/suricata.c index c0cef0d0ba..e0f11a7ccb 100644 --- a/src/suricata.c +++ b/src/suricata.c @@ -81,6 +81,7 @@ #include "log-droplog.h" #include "log-httplog.h" +#include "output-httplog.h" #include "log-dnslog.h" #include "log-tlslog.h" #include "log-pcap.h" @@ -797,6 +798,7 @@ void RegisterAllModules() TmModuleOutputJsonRegister(); /* http log */ TmModuleLogHttpLogRegister(); + TmModuleJsonHttpLogRegister(); TmModuleLogTlsLogRegister(); /* pcap log */ TmModulePcapLogRegister(); diff --git a/src/tm-threads-common.h b/src/tm-threads-common.h index 1fda423402..716ad9573b 100644 --- a/src/tm-threads-common.h +++ b/src/tm-threads-common.h @@ -84,6 +84,7 @@ typedef enum { TMM_TXLOGGER, TMM_FILELOGGER, TMM_FILEDATALOGGER, + TMM_JSONHTTPLOG, TMM_SIZE, } TmmId;