aiden
/
suricata
mirror of https://github.com/OISF/suricata
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

flowbits: analyze and dump to json

Browse Source 
Analyze flowbits to find which bits are only checked.

Track whether they are set and checked on the same level of 'statefulness'
for later used.

Dump flowbits to json including the sids that set/check etc the bit.
pull/3182/head
Victor Julien 8 years ago
parent d05355db3d
commit 51d429b3b1
7 changed files with 377 additions and 4 deletions
Show Stats Download Patch File Download Diff File

2
src/detect-engine-build.c
Unescape Escape View File

@ -1509,6 +1509,8 @@ int SigAddressPrepareStage1(DetectEngineCtx *de_ctx)
SCLogConfig("building signature grouping structure, stage 1: "
"preprocessing rules... complete");
}
DetectFlowbitsAnalyze(de_ctx);
return 0;
error:

356
src/detect-flowbits.c
Unescape Escape View File

@ -1,4 +1,4 @@
/* Copyright (C) 2007-2010 Open Information Security Foundation
/* Copyright (C) 2007-2017 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
@ -247,6 +247,7 @@ int DetectFlowbitSetup (DetectEngineCtx *de_ctx, Signature *s, const char *rawst
goto error;
cd->idx = VarNameStoreSetupAdd(fb_name, VAR_TYPE_FLOW_BIT);
de_ctx->max_fb_id = MAX(cd->idx, de_ctx->max_fb_id);
cd->cmd = fb_cmd;
SCLogDebug("idx %" PRIu32 ", cmd %s, name %s",
@ -298,6 +299,359 @@ void DetectFlowbitFree (void *ptr)
SCFree(fd);
}
struct FBAnalyze {
uint16_t cnts[DETECT_FLOWBITS_CMD_MAX];
uint16_t state_cnts[DETECT_FLOWBITS_CMD_MAX];
uint32_t *set_sids;
uint32_t set_sids_idx;
uint32_t set_sids_size;
uint32_t *isset_sids;
uint32_t isset_sids_idx;
uint32_t isset_sids_size;
uint32_t *isnotset_sids;
uint32_t isnotset_sids_idx;
uint32_t isnotset_sids_size;
uint32_t *unset_sids;
uint32_t unset_sids_idx;
uint32_t unset_sids_size;
uint32_t *toggle_sids;
uint32_t toggle_sids_idx;
uint32_t toggle_sids_size;
};
#ifdef PROFILING
#ifdef HAVE_LIBJANSSON
static void DetectFlowbitsAnalyzeDump(const DetectEngineCtx *de_ctx,
struct FBAnalyze *array, uint32_t elements);
#endif
#endif
void DetectFlowbitsAnalyze(DetectEngineCtx *de_ctx)
{
const int nlists = DetectBufferTypeMaxId();
const uint32_t max_fb_id = de_ctx->max_fb_id;
if (max_fb_id == 0)
return;
#define MAX_SIDS 8
uint32_t array_size = max_fb_id + 1;
struct FBAnalyze array[array_size];
memset(&array, 0, array_size * sizeof(struct FBAnalyze));
SCLogDebug("fb analyzer array size: %"PRIu64,
(uint64_t)(array_size * sizeof(struct FBAnalyze)));
/* fill flowbit array, updating counters per sig */
for (uint32_t i = 0; i < de_ctx->sig_array_len; i++) {
const Signature *s = de_ctx->sig_array[i];
bool has_state = false;
/* see if the signature uses stateful matching */
for (int x = DETECT_SM_LIST_DYNAMIC_START; x < nlists; x++) {
if (s->init_data->smlists[x] == NULL)
continue;
has_state = true;
break;
}
for (const SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_MATCH] ; sm != NULL; sm = sm->next) {
switch (sm->type) {
case DETECT_FLOWBITS:
{
/* figure out the flowbit action */
const DetectFlowbitsData *fb = (DetectFlowbitsData *)sm->ctx;
array[fb->idx].cnts[fb->cmd]++;
if (has_state)
array[fb->idx].state_cnts[fb->cmd]++;
if (fb->cmd == DETECT_FLOWBITS_CMD_ISSET) {
if (array[fb->idx].isset_sids_idx >= array[fb->idx].isset_sids_size) {
uint32_t old_size = array[fb->idx].isset_sids_size;
uint32_t new_size = MAX(2 * old_size, MAX_SIDS);
void *ptr = SCRealloc(array[fb->idx].isset_sids, new_size * sizeof(uint32_t));
if (ptr == NULL)
goto end;
array[fb->idx].isset_sids_size = new_size;
array[fb->idx].isset_sids = ptr;
}
array[fb->idx].isset_sids[array[fb->idx].isset_sids_idx] = s->num;
array[fb->idx].isset_sids_idx++;
} else if (fb->cmd == DETECT_FLOWBITS_CMD_ISNOTSET){
if (array[fb->idx].isnotset_sids_idx >= array[fb->idx].isnotset_sids_size) {
uint32_t old_size = array[fb->idx].isnotset_sids_size;
uint32_t new_size = MAX(2 * old_size, MAX_SIDS);
void *ptr = SCRealloc(array[fb->idx].isnotset_sids, new_size * sizeof(uint32_t));
if (ptr == NULL)
goto end;
array[fb->idx].isnotset_sids_size = new_size;
array[fb->idx].isnotset_sids = ptr;
}
array[fb->idx].isnotset_sids[array[fb->idx].isnotset_sids_idx] = s->num;
array[fb->idx].isnotset_sids_idx++;
}
}
}
}
for (const SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_POSTMATCH] ; sm != NULL; sm = sm->next) {
switch (sm->type) {
case DETECT_FLOWBITS:
{
/* figure out what flowbit action */
const DetectFlowbitsData *fb = (DetectFlowbitsData *)sm->ctx;
array[fb->idx].cnts[fb->cmd]++;
if (has_state)
array[fb->idx].state_cnts[fb->cmd]++;
if (fb->cmd == DETECT_FLOWBITS_CMD_SET) {
if (array[fb->idx].set_sids_idx >= array[fb->idx].set_sids_size) {
uint32_t old_size = array[fb->idx].set_sids_size;
uint32_t new_size = MAX(2 * old_size, MAX_SIDS);
void *ptr = SCRealloc(array[fb->idx].set_sids, new_size * sizeof(uint32_t));
if (ptr == NULL)
goto end;
array[fb->idx].set_sids_size = new_size;
array[fb->idx].set_sids = ptr;
}
array[fb->idx].set_sids[array[fb->idx].set_sids_idx] = s->num;
array[fb->idx].set_sids_idx++;
}
else if (fb->cmd == DETECT_FLOWBITS_CMD_UNSET) {
if (array[fb->idx].unset_sids_idx >= array[fb->idx].unset_sids_size) {
uint32_t old_size = array[fb->idx].unset_sids_size;
uint32_t new_size = MAX(2 * old_size, MAX_SIDS);
void *ptr = SCRealloc(array[fb->idx].unset_sids, new_size * sizeof(uint32_t));
if (ptr == NULL)
goto end;
array[fb->idx].unset_sids_size = new_size;
array[fb->idx].unset_sids = ptr;
}
array[fb->idx].unset_sids[array[fb->idx].unset_sids_idx] = s->num;
array[fb->idx].unset_sids_idx++;
}
else if (fb->cmd == DETECT_FLOWBITS_CMD_TOGGLE) {
if (array[fb->idx].toggle_sids_idx >= array[fb->idx].toggle_sids_size) {
uint32_t old_size = array[fb->idx].toggle_sids_size;
uint32_t new_size = MAX(2 * old_size, MAX_SIDS);
void *ptr = SCRealloc(array[fb->idx].toggle_sids, new_size * sizeof(uint32_t));
if (ptr == NULL)
goto end;
array[fb->idx].toggle_sids_size = new_size;
array[fb->idx].toggle_sids = ptr;
}
array[fb->idx].toggle_sids[array[fb->idx].toggle_sids_idx] = s->num;
array[fb->idx].toggle_sids_idx++;
}
}
}
}
}
/* walk array to see if all bits make sense */
for (uint32_t i = 0; i < array_size; i++) {
char *varname = VarNameStoreSetupLookup(i, VAR_TYPE_FLOW_BIT);
if (varname == NULL)
continue;
if (array[i].cnts[DETECT_FLOWBITS_CMD_ISSET] &&
array[i].cnts[DETECT_FLOWBITS_CMD_TOGGLE] == 0 &&
array[i].cnts[DETECT_FLOWBITS_CMD_SET] == 0) {
const Signature *s = de_ctx->sig_array[array[i].isset_sids[0]];
SCLogWarning(SC_WARN_FLOWBIT, "flowbit '%s' is checked but not "
"set. Checked in %u and %u other sigs",
varname, s->id, array[i].isset_sids_idx - 1);
}
if (array[i].state_cnts[DETECT_FLOWBITS_CMD_ISSET] &&
array[i].state_cnts[DETECT_FLOWBITS_CMD_SET] == 0)
{
SCLogDebug("flowbit %s/%u: isset in state, set not in state", varname, i);
}
if (array[i].cnts[DETECT_FLOWBITS_CMD_ISSET] > 0 &&
array[i].state_cnts[DETECT_FLOWBITS_CMD_ISSET] == 0 &&
array[i].state_cnts[DETECT_FLOWBITS_CMD_SET])
{
SCLogDebug("flowbit %s/%u: isset not in state, set in state", varname, i);
}
SCLogDebug("ALL flowbit %s/%u: sets %u toggles %u unsets %u isnotsets %u issets %u", varname, i,
array[i].cnts[DETECT_FLOWBITS_CMD_SET], array[i].cnts[DETECT_FLOWBITS_CMD_TOGGLE],
array[i].cnts[DETECT_FLOWBITS_CMD_UNSET], array[i].cnts[DETECT_FLOWBITS_CMD_ISNOTSET],
array[i].cnts[DETECT_FLOWBITS_CMD_ISSET]);
SCLogDebug("STATE flowbit %s/%u: sets %u toggles %u unsets %u isnotsets %u issets %u", varname, i,
array[i].state_cnts[DETECT_FLOWBITS_CMD_SET], array[i].state_cnts[DETECT_FLOWBITS_CMD_TOGGLE],
array[i].state_cnts[DETECT_FLOWBITS_CMD_UNSET], array[i].state_cnts[DETECT_FLOWBITS_CMD_ISNOTSET],
array[i].state_cnts[DETECT_FLOWBITS_CMD_ISSET]);
for (uint32_t x = 0; x < array[i].set_sids_idx; x++) {
SCLogDebug("SET flowbit %s/%u: SID %u", varname, i,
de_ctx->sig_array[array[i].set_sids[x]]->id);
}
for (uint32_t x = 0; x < array[i].isset_sids_idx; x++) {
SCLogDebug("GET flowbit %s/%u: SID %u", varname, i,
de_ctx->sig_array[array[i].isset_sids[x]]->id);
}
SCFree(varname);
}
#ifdef PROFILING
#ifdef HAVE_LIBJANSSON
DetectFlowbitsAnalyzeDump(de_ctx, array, array_size);
#endif
#endif
end:
for (uint32_t i = 0; i < array_size; i++) {
SCFree(array[i].set_sids);
SCFree(array[i].unset_sids);
SCFree(array[i].isset_sids);
SCFree(array[i].isnotset_sids);
SCFree(array[i].toggle_sids);
}
}
#ifdef PROFILING
#ifdef HAVE_LIBJANSSON
#include "output-json.h"
#include "util-buffer.h"
SCMutex g_flowbits_dump_write_m = SCMUTEX_INITIALIZER;
static void DetectFlowbitsAnalyzeDump(const DetectEngineCtx *de_ctx,
struct FBAnalyze *array, uint32_t elements)
{
json_t *js = json_object();
if (js == NULL)
return;
json_t *js_array = json_array();
uint32_t x;
for (x = 0; x < elements; x++)
{
char *varname = VarNameStoreSetupLookup(x, VAR_TYPE_FLOW_BIT);
if (varname == NULL)
continue;
const struct FBAnalyze *e = &array[x];
json_t *js_fb = json_object();
if (unlikely(js_fb != NULL)) {
json_object_set_new(js_fb, "name", json_string(varname));
json_object_set_new(js_fb, "internal_id", json_integer(x));
json_object_set_new(js_fb, "set_cnt", json_integer(e->cnts[DETECT_FLOWBITS_CMD_SET]));
json_object_set_new(js_fb, "unset_cnt", json_integer(e->cnts[DETECT_FLOWBITS_CMD_UNSET]));
json_object_set_new(js_fb, "toggle_cnt", json_integer(e->cnts[DETECT_FLOWBITS_CMD_TOGGLE]));
json_object_set_new(js_fb, "isset_cnt", json_integer(e->cnts[DETECT_FLOWBITS_CMD_ISSET]));
json_object_set_new(js_fb, "isnotset_cnt", json_integer(e->cnts[DETECT_FLOWBITS_CMD_ISNOTSET]));
// sets
if (e->cnts[DETECT_FLOWBITS_CMD_SET]) {
json_t *js_set_array = json_array();
if (js_set_array) {
for(uint32_t i = 0; i < e->set_sids_idx; i++) {
const Signature *s = de_ctx->sig_array[e->set_sids[i]];
json_array_append_new(js_set_array, json_integer(s->id));
}
json_object_set_new(js_fb, "sets", js_set_array);
}
}
// gets
if (e->cnts[DETECT_FLOWBITS_CMD_ISSET]) {
json_t *js_isset_array = json_array();
if (js_isset_array) {
for(uint32_t i = 0; i < e->isset_sids_idx; i++) {
const Signature *s = de_ctx->sig_array[e->isset_sids[i]];
json_array_append_new(js_isset_array, json_integer(s->id));
}
json_object_set_new(js_fb, "isset", js_isset_array);
}
}
// isnotset
if (e->cnts[DETECT_FLOWBITS_CMD_ISNOTSET]) {
json_t *js_isnotset_array = json_array();
if (js_isnotset_array) {
for(uint32_t i = 0; i < e->isnotset_sids_idx; i++) {
const Signature *s = de_ctx->sig_array[e->isnotset_sids[i]];
json_array_append_new(js_isnotset_array, json_integer(s->id));
}
json_object_set_new(js_fb, "isnotset", js_isnotset_array);
}
}
// unset
if (e->cnts[DETECT_FLOWBITS_CMD_UNSET]) {
json_t *js_unset_array = json_array();
if (js_unset_array) {
for(uint32_t i = 0; i < e->unset_sids_idx; i++) {
const Signature *s = de_ctx->sig_array[e->unset_sids[i]];
json_array_append_new(js_unset_array, json_integer(s->id));
}
json_object_set_new(js_fb, "unset", js_unset_array);
}
}
// toggle
if (e->cnts[DETECT_FLOWBITS_CMD_TOGGLE]) {
json_t *js_toggle_array = json_array();
if (js_toggle_array) {
for(uint32_t i = 0; i < e->toggle_sids_idx; i++) {
const Signature *s = de_ctx->sig_array[e->toggle_sids[i]];
json_array_append_new(js_toggle_array, json_integer(s->id));
}
json_object_set_new(js_fb, "toggle", js_toggle_array);
}
}
json_array_append_new(js_array, js_fb);
}
SCFree(varname);
}
json_object_set_new(js, "flowbits", js_array);
const char *filename = "flowbits.json";
const char *log_dir = ConfigGetLogDirectory();
char log_path[PATH_MAX] = "";
snprintf(log_path, sizeof(log_path), "%s/%s", log_dir, filename);
MemBuffer *mbuf = NULL;
mbuf = MemBufferCreateNew(4096);
BUG_ON(mbuf == NULL);
OutputJSONMemBufferWrapper wrapper = {
.buffer = &mbuf,
.expand_by = 4096,
};
int r = json_dump_callback(js, OutputJSONMemBufferCallback, &wrapper,
JSON_PRESERVE_ORDER|JSON_COMPACT|JSON_ENSURE_ASCII|
JSON_ESCAPE_SLASH);
if (r != 0) {
SCLogWarning(SC_ERR_SOCKET, "unable to serialize JSON object");
} else {
MemBufferWriteString(mbuf, "\n");
SCMutexLock(&g_flowbits_dump_write_m);
FILE *fp = fopen(log_path, "w");
if (fp != NULL) {
MemBufferPrintToFPAsString(mbuf, fp);
fclose(fp);
}
SCMutexUnlock(&g_flowbits_dump_write_m);
}
MemBufferFree(mbuf);
json_object_clear(js);
json_decref(js);
}
#endif /* HAVE_LIBJANSSON */
#endif /* PROFILING */
#ifdef UNITTESTS
static int FlowBitsTestParse01(void)

5
src/detect.h
Unescape Escape View File

@ -672,6 +672,9 @@ typedef struct DetectEngineCtx_ {
/* specify the configuration for mpm context factory */
uint8_t sgh_mpm_context;
/* max flowbit id that is used */
uint32_t max_fb_id;
uint32_t max_fp_id;
MpmCtxFactoryContainer *mpm_ctx_factory_container;
@ -1247,6 +1250,8 @@ int SigMatchSignaturesRunPostMatch(ThreadVars *tv,
const Signature *s);
void DetectSignatureApplyActions(Packet *p, const Signature *s, const uint8_t);
void DetectFlowbitsAnalyze(DetectEngineCtx *de_ctx);
#include "detect-engine-build.h"
#include "detect-engine-register.h"

2
src/util-error.c
Unescape Escape View File

@ -346,6 +346,8 @@ const char * SCErrorToString(SCError err)
CASE_CODE (SC_WARN_RENAMING_FILE);
CASE_CODE (SC_ERR_PF_RING_VLAN);
CASE_CODE (SC_ERR_CREATE_DIRECTORY);
CASE_CODE (SC_WARN_FLOWBIT);
CASE_CODE (SC_ERR_MAX);
}

2
src/util-error.h
Unescape Escape View File

@ -336,6 +336,8 @@ typedef enum {
SC_WARN_RENAMING_FILE,
SC_ERR_PF_RING_VLAN,
SC_ERR_CREATE_DIRECTORY,
SC_WARN_FLOWBIT,
SC_ERR_MAX,
} SCError;

13
src/util-var-name.c
Unescape Escape View File

@ -203,7 +203,6 @@ error:
return 0;
}
#if 0
/** \brief Get a name from the idx.
* \param idx index of the variable whose name is to be fetched
* \param type variable type
@ -211,7 +210,7 @@ error:
* \retval name of the variable if successful.
* \todo no alloc on lookup
*/
static const char *VariableIdxGetName(VarNameStore *v, uint32_t idx, enum VarTypes type)
static char *VariableIdxGetName(VarNameStore *v, uint32_t idx, enum VarTypes type)
{
VariableName *fn = SCMalloc(sizeof(VariableName));
if (unlikely(fn == NULL))
@ -239,7 +238,7 @@ error:
VariableNameFree(fn);
return NULL;
}
#endif
/** \brief setup staging store. Include current store if there is one.
*/
int VarNameStoreSetupStaging(uint32_t de_ctx_version)
@ -327,6 +326,14 @@ uint32_t VarNameStoreSetupAdd(const char *name, const enum VarTypes type)
return id;
}
char *VarNameStoreSetupLookup(uint32_t idx, const enum VarTypes type)
{
SCMutexLock(&g_varnamestore_staging_m);
char *name = VariableIdxGetName(g_varnamestore_staging, idx, type);
SCMutexUnlock(&g_varnamestore_staging_m);
return name;
}
void VarNameStoreActivateStaging(void)
{
SCMutexLock(&g_varnamestore_staging_m);

1
src/util-var-name.h
Unescape Escape View File

@ -28,6 +28,7 @@ int VarNameStoreSetupStaging(uint32_t de_ctx_version);
const char *VarNameStoreLookupById(const uint32_t id, const enum VarTypes type);
uint32_t VarNameStoreLookupByName(const char *name, const enum VarTypes type);
uint32_t VarNameStoreSetupAdd(const char *name, const enum VarTypes type);
char *VarNameStoreSetupLookup(uint32_t idx, const enum VarTypes type);
void VarNameStoreActivateStaging(void);
void VarNameStoreFreeOld(void);
void VarNameStoreFree(uint32_t de_ctx_version);

Write Preview
Loading…
Cancel
Save