aiden
/
suricata
mirror of https://github.com/OISF/suricata
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

http-json: separate module using tx api

Browse Source 
Turn HTTP json logger into a Tx Logger API logger.
pull/805/head
Victor Julien 12 years ago
parent 4874d5abbb
commit bc71a43e08
6 changed files with 145 additions and 108 deletions
Show Stats Download Patch File Download Diff File

216
src/output-httplog.c
Unescape Escape View File

@ -53,43 +53,28 @@
#ifdef HAVE_LIBJANSSON
#include <jansson.h>
#define LOG_HTTP_MAXN_NODES 64
#define LOG_HTTP_NODE_STRLEN 256
#define LOG_HTTP_NODE_MAXOUTPUTLEN 8192
#define TIMESTAMP_DEFAULT_FORMAT "%b %d, %Y; %H:%M:%S"
#define LOG_HTTP_CF_NONE "-"
#define LOG_HTTP_CF_LITERAL '%'
#define LOG_HTTP_CF_REQUEST_HOST 'h'
#define LOG_HTTP_CF_REQUEST_PROTOCOL 'H'
#define LOG_HTTP_CF_REQUEST_METHOD 'm'
#define LOG_HTTP_CF_REQUEST_URI 'u'
#define LOG_HTTP_CF_REQUEST_TIME 't'
#define LOG_HTTP_CF_REQUEST_HEADER 'i'
#define LOG_HTTP_CF_REQUEST_COOKIE 'C'
#define LOG_HTTP_CF_REQUEST_LEN 'b'
#define LOG_HTTP_CF_RESPONSE_STATUS 's'
#define LOG_HTTP_CF_RESPONSE_HEADER 'o'
#define LOG_HTTP_CF_RESPONSE_LEN 'B'
#define LOG_HTTP_CF_TIMESTAMP 't'
#define LOG_HTTP_CF_TIMESTAMP_U 'z'
#define LOG_HTTP_CF_CLIENT_IP 'a'
#define LOG_HTTP_CF_SERVER_IP 'A'
#define LOG_HTTP_CF_CLIENT_PORT 'p'
#define LOG_HTTP_CF_SERVER_PORT 'P'
typedef struct OutputHttpCtx_ {
typedef struct LogHttpFileCtx_ {
LogFileCtx *file_ctx;
uint32_t flags; /** Store mode */
} OutputHttpCtx;
} LogHttpFileCtx;
typedef struct JsonHttpLogThread_ {
LogHttpFileCtx *httplog_ctx;
/** LogFileCtx has the pointer to the file and a mutex to allow multithreading */
uint32_t uri_cnt;
MemBuffer *buffer;
} JsonHttpLogThread;
#define LOG_HTTP_DEFAULT 0
#define LOG_HTTP_EXTENDED 1
#define LOG_HTTP_CUSTOM 2
/* JSON format logging */
static void LogHttpLogJSON(AlertJsonThread *aft, json_t *js, htp_tx_t *tx)
static void JsonHttpLogJSON(JsonHttpLogThread *aft, json_t *js, htp_tx_t *tx)
{
OutputHttpCtx *http_ctx = aft->http_ctx->data;
LogHttpFileCtx *http_ctx = aft->httplog_ctx;
json_t *hjs = json_object();
if (hjs == NULL) {
return;
@ -162,6 +147,7 @@ static void LogHttpLogJSON(AlertJsonThread *aft, json_t *js, htp_tx_t *tx)
SCFree(c);
}
#if 1
if (http_ctx->flags & LOG_HTTP_EXTENDED) {
/* referer */
htp_header_t *h_referer = NULL;
@ -215,107 +201,54 @@ static void LogHttpLogJSON(AlertJsonThread *aft, json_t *js, htp_tx_t *tx)
/* length */
json_object_set_new(hjs, "length", json_integer(tx->response_message_len));
}
#endif
json_object_set_new(js, "http", hjs);
}
static TmEcode HttpJsonIPWrapper(ThreadVars *tv, Packet *p, void *data)
static int JsonHttpLogger(ThreadVars *tv, void *thread_data, const Packet *p, Flow *f, void *alstate, void *txptr, uint64_t tx_id)
{
SCEnter();
uint64_t tx_id = 0;
uint64_t total_txs = 0;
htp_tx_t *tx = NULL;
HtpState *htp_state = NULL;
int tx_progress = 0;
int tx_progress_done_value_ts = 0;
int tx_progress_done_value_tc = 0;
AlertJsonThread *aft = (AlertJsonThread *)data;
MemBuffer *buffer = (MemBuffer *)aft->buffer;
/* no flow, no htp state */
if (p->flow == NULL) {
SCReturnInt(TM_ECODE_OK);
}
/* check if we have HTTP state or not */
FLOWLOCK_WRLOCK(p->flow); /* WRITE lock before we updated flow logged id */
uint16_t proto = FlowGetAppProtocol(p->flow);
if (proto != ALPROTO_HTTP)
goto end;
htp_state = (HtpState *)FlowGetAppState(p->flow);
if (htp_state == NULL) {
SCLogDebug("no http state, so no request logging");
goto end;
}
total_txs = AppLayerParserGetTxCnt(IPPROTO_TCP, ALPROTO_HTTP, htp_state);
tx_id = AppLayerParserGetTransactionLogId(p->flow->alparser);
tx_progress_done_value_ts = AppLayerParserGetStateProgressCompletionStatus(p->proto, ALPROTO_HTTP, 0);
tx_progress_done_value_tc = AppLayerParserGetStateProgressCompletionStatus(p->proto, ALPROTO_HTTP, 1);
htp_tx_t *tx = txptr;
JsonHttpLogThread *jhl = (JsonHttpLogThread *)thread_data;
MemBuffer *buffer = (MemBuffer *)jhl->buffer;
json_t *js = CreateJSONHeader(p, 1);
json_t *js = CreateJSONHeader((Packet *)p, 1); //TODO const
if (unlikely(js == NULL))
return TM_ECODE_OK;
for (; tx_id < total_txs; tx_id++)
{
tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP, htp_state, tx_id);
if (tx == NULL) {
SCLogDebug("tx is NULL not logging !!");
continue;
}
if (!(AppLayerParserStateIssetFlag(p->flow->alparser, APP_LAYER_PARSER_EOF))) {
tx_progress = AppLayerParserGetStateProgress(p->proto, ALPROTO_HTTP, tx, 0);
if (tx_progress < tx_progress_done_value_ts)
break;
tx_progress = AppLayerParserGetStateProgress(p->proto, ALPROTO_HTTP, tx, 1);
if (tx_progress < tx_progress_done_value_tc)
break;
}
SCLogDebug("got a HTTP request and now logging !!");
SCLogDebug("got a HTTP request and now logging !!");
/* reset */
MemBufferReset(buffer);
/* reset */
MemBufferReset(buffer);
/* Maybe we'll do a "custom" later
if (http_ctx->flags & LOG_HTTP_CUSTOM) {
LogHttpLogJSONCustom(aft, js, tx, &p->ts);
} else {
*/
LogHttpLogJSON(aft, js, tx);
/*
}
*/
JsonHttpLogJSON(jhl, js, tx);
OutputJSON(js, aft, &aft->http_cnt);
json_object_del(js, "http");
OutputJSONBuffer(js, jhl->httplog_ctx->file_ctx, buffer);
json_object_del(js, "http");
AppLayerParserSetTransactionLogId(p->flow->alparser);
}
json_object_clear(js);
json_decref(js);
end:
FLOWLOCK_UNLOCK(p->flow);
SCReturnInt(TM_ECODE_OK);
}
TmEcode OutputHttpLog (ThreadVars *tv, Packet *p, void *data)
{
SCEnter();
HttpJsonIPWrapper(tv, p, data);
SCReturnInt(TM_ECODE_OK);
}
#define DEFAULT_LOG_FILENAME "http.json"
OutputCtx *OutputHttpLogInit(ConfNode *conf)
{
OutputHttpCtx *http_ctx = SCMalloc(sizeof(OutputHttpCtx));
LogFileCtx *file_ctx = LogFileNewCtx();
if(file_ctx == NULL) {
SCLogError(SC_ERR_HTTP_LOG_GENERIC, "couldn't create new file_ctx");
return NULL;
}
if (SCConfLogOpenGeneric(conf, file_ctx, DEFAULT_LOG_FILENAME) < 0) {
LogFileFreeCtx(file_ctx);
return NULL;
}
LogHttpFileCtx *http_ctx = SCMalloc(sizeof(LogHttpFileCtx));
if (unlikely(http_ctx == NULL))
return NULL;
@ -323,6 +256,7 @@ OutputCtx *OutputHttpLogInit(ConfNode *conf)
if (unlikely(output_ctx == NULL))
return NULL;
http_ctx->file_ctx = file_ctx;
http_ctx->flags = LOG_HTTP_DEFAULT;
if (conf) {
@ -340,4 +274,72 @@ OutputCtx *OutputHttpLogInit(ConfNode *conf)
return output_ctx;
}
#define OUTPUT_BUFFER_SIZE 65535
static TmEcode JsonHttpLogThreadInit(ThreadVars *t, void *initdata, void **data)
{
JsonHttpLogThread *aft = SCMalloc(sizeof(JsonHttpLogThread));
if (unlikely(aft == NULL))
return TM_ECODE_FAILED;
memset(aft, 0, sizeof(JsonHttpLogThread));
if(initdata == NULL)
{
SCLogDebug("Error getting context for HTTPLog. \"initdata\" argument NULL");
SCFree(aft);
return TM_ECODE_FAILED;
}
/* Use the Ouptut Context (file pointer and mutex) */
aft->httplog_ctx = ((OutputCtx *)initdata)->data; //TODO
aft->buffer = MemBufferCreateNew(OUTPUT_BUFFER_SIZE);
if (aft->buffer == NULL) {
SCFree(aft);
return TM_ECODE_FAILED;
}
*data = (void *)aft;
return TM_ECODE_OK;
}
static TmEcode JsonHttpLogThreadDeinit(ThreadVars *t, void *data)
{
JsonHttpLogThread *aft = (JsonHttpLogThread *)data;
if (aft == NULL) {
return TM_ECODE_OK;
}
MemBufferFree(aft->buffer);
/* clear memory */
memset(aft, 0, sizeof(JsonHttpLogThread));
SCFree(aft);
return TM_ECODE_OK;
}
void TmModuleJsonHttpLogRegister (void) {
tmm_modules[TMM_JSONHTTPLOG].name = "JsonHttpLog";
tmm_modules[TMM_JSONHTTPLOG].ThreadInit = JsonHttpLogThreadInit;
tmm_modules[TMM_JSONHTTPLOG].ThreadDeinit = JsonHttpLogThreadDeinit;
tmm_modules[TMM_JSONHTTPLOG].RegisterTests = NULL;
tmm_modules[TMM_JSONHTTPLOG].cap_flags = 0;
OutputRegisterTxModule("JsonHttpLog", "http-json-log", OutputHttpLogInit,
ALPROTO_HTTP, JsonHttpLogger);
}
#else
static TmEcode OutputJsonThreadInit(ThreadVars *t, void *initdata, void **data)
{
SCLogInfo("Can't init JSON output - JSON support was disabled during build.");
return TM_ECODE_FAILED;
}
void TmModuleJsonHttpLogRegister (void)
{
tmm_modules[TMM_JSONHTTPLOG].name = "JsonHttpLog";
tmm_modules[TMM_JSONHTTPLOG].ThreadInit = OutputJsonThreadInit;
}
#endif

2
src/output-httplog.h
Unescape Escape View File

@ -27,5 +27,7 @@
TmEcode OutputHttpLog (ThreadVars *tv, Packet *p, void *data);
OutputCtx *OutputHttpLogInit(ConfNode *);
void TmModuleJsonHttpLogRegister (void);
#endif /* __OUTPUT_HTTPLOG_H__ */

28
src/output-json.c
Unescape Escape View File

@ -303,6 +303,30 @@ json_t *CreateJSONHeader(Packet *p, int direction_sensitive)
return js;
}
int OutputJSONBuffer(json_t *js, LogFileCtx *file_ctx, MemBuffer *buffer) {
char *js_s = json_dumps(js,
JSON_PRESERVE_ORDER|JSON_COMPACT|JSON_ENSURE_ASCII|
#ifdef JSON_ESCAPE_SLASH
JSON_ESCAPE_SLASH
#else
0
#endif
);
if (unlikely(js_s == NULL))
return TM_ECODE_OK;
SCMutexLock(&file_ctx->fp_mutex);
if (json_out == ALERT_SYSLOG) {
syslog(alert_syslog_level, "%s", js_s);
} else if (json_out == ALERT_FILE) {
MemBufferWriteString(buffer, "%s\n", js_s);
(void)MemBufferPrintToFPAsString(buffer, file_ctx->fp);
fflush(file_ctx->fp);
}
SCMutexUnlock(&file_ctx->fp_mutex);
return 0;
}
TmEcode OutputJSON(json_t *js, void *data, uint64_t *count)
{
AlertJsonThread *aft = (AlertJsonThread *)data;
@ -482,7 +506,7 @@ TmEcode OutputJson (ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, Pack
}
if (output_flags & OUTPUT_HTTP) {
OutputHttpLog(tv, p, data);
// OutputHttpLog(tv, p, data);
}
if (output_flags & OUTPUT_TLS) {
@ -679,6 +703,7 @@ OutputCtx *OutputJsonInitCtx(ConfNode *conf)
output_flags |= OUTPUT_FILES;
continue;
}
#if 0
if (strcmp(output->val, "http") == 0) {
SCLogDebug("Enabling HTTP output");
ConfNode *child = ConfNodeLookupChild(output, "http");
@ -687,6 +712,7 @@ OutputCtx *OutputJsonInitCtx(ConfNode *conf)
output_flags |= OUTPUT_HTTP;
continue;
}
#endif
if (strcmp(output->val, "tls") == 0) {
SCLogDebug("Enabling TLS output");
ConfNode *child = ConfNodeLookupChild(output, "tls");

4
src/output-json.h
Unescape Escape View File

@ -28,8 +28,12 @@ void TmModuleOutputJsonRegister (void);
#ifdef HAVE_LIBJANSSON
#include "suricata-common.h"
#include "util-buffer.h"
json_t *CreateJSONHeader(Packet *p, int direction_sensative);
TmEcode OutputJSON(json_t *js, void *data, uint64_t *count);
int OutputJSONBuffer(json_t *js, LogFileCtx *file_ctx, MemBuffer *buffer);
OutputCtx *OutputJsonInitCtx(ConfNode *);

2
src/suricata.c
Unescape Escape View File

@ -81,6 +81,7 @@
#include "log-droplog.h"
#include "log-httplog.h"
#include "output-httplog.h"
#include "log-dnslog.h"
#include "log-tlslog.h"
#include "log-pcap.h"
@ -797,6 +798,7 @@ void RegisterAllModules()
TmModuleOutputJsonRegister();
/* http log */
TmModuleLogHttpLogRegister();
TmModuleJsonHttpLogRegister();
TmModuleLogTlsLogRegister();
/* pcap log */
TmModulePcapLogRegister();

1
src/tm-threads-common.h
Unescape Escape View File

@ -84,6 +84,7 @@ typedef enum {
TMM_TXLOGGER,
TMM_FILELOGGER,
TMM_FILEDATALOGGER,
TMM_JSONHTTPLOG,
TMM_SIZE,
} TmmId;

Write Preview
Loading…
Cancel
Save