aiden
/
suricata
mirror of https://github.com/OISF/suricata
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

File carving -- enable reponse file extraction

Browse Source 
- Enable response body tracking
- Enable file extraction for responses
- File store meta file includes magic, close reason.
- Option to force magic lookup for all stored files.
- Fix libmagic calls thead safety.
remotes/origin/master-1.2.x
Victor Julien 14 years ago
parent 66a3cd96a8
commit b402d97179
16 changed files with 485 additions and 135 deletions
Show Stats Download Patch File Download Diff File

26
src/app-layer-htp-body.c
Unescape Escape View File

@ -104,14 +104,14 @@ int HtpBodyAppendChunk(HtpTxUserData *htud, HtpBody *body, uint8_t *data, uint32
body->first = body->last = bd;
body->nchunks++;
htud->content_len_so_far = len;
body->content_len_so_far = len;
} else {
bd = (HtpBodyChunk *)SCMalloc(sizeof(HtpBodyChunk));
if (bd == NULL)
goto error;
bd->len = len;
bd->stream_offset = htud->content_len_so_far;
bd->stream_offset = body->content_len_so_far;
bd->next = NULL;
bd->id = body->nchunks + 1;
@ -125,7 +125,7 @@ int HtpBodyAppendChunk(HtpTxUserData *htud, HtpBody *body, uint8_t *data, uint32
body->last = bd;
body->nchunks++;
htud->content_len_so_far += len;
body->content_len_so_far += len;
}
SCLogDebug("Body %p; Chunk id: %"PRIu32", data %p, len %"PRIu32"", body,
bd->id, bd->data, (uint32_t)bd->len);
@ -198,7 +198,6 @@ void HtpBodyFree(HtpBody *body)
prev = cur;
}
body->first = body->last = NULL;
body->operation = HTP_BODY_NONE;
}
/**
@ -208,17 +207,15 @@ void HtpBodyFree(HtpBody *body)
*
* \retval none
*/
void HtpBodyPrune(HtpTxUserData *htud)
void HtpBodyPrune(HtpBody *body)
{
SCEnter();
HtpBody *body = &htud->request_body;
if (body->nchunks == 0) {
if (body == NULL || body->nchunks == 0) {
SCReturn;
}
if (htud->body_parsed == 0) {
if (body->body_parsed == 0) {
SCReturn;
}
@ -226,18 +223,15 @@ void HtpBodyPrune(HtpTxUserData *htud)
" len %"PRIu32, body, body->last->id, body->last->data,
(uint32_t)body->last->len);
HtpBodyChunk *cur = NULL;
cur = body->first;
HtpBodyChunk *cur = body->first;
while (cur != NULL) {
HtpBodyChunk *next = cur->next;
SCLogDebug("cur->stream_offset %"PRIu64" + cur->len %u = %"PRIu64", "
"htud->body_parsed %"PRIu64, cur->stream_offset, cur->len,
cur->stream_offset + cur->len, htud->body_parsed);
"body->body_parsed %"PRIu64, cur->stream_offset, cur->len,
cur->stream_offset + cur->len, body->body_parsed);
if ((cur->stream_offset + cur->len) >= htud->body_parsed) {
if ((cur->stream_offset + cur->len) >= body->body_parsed) {
break;
}

2
src/app-layer-htp-body.h
Unescape Escape View File

@ -31,6 +31,6 @@
int HtpBodyAppendChunk(HtpTxUserData *, HtpBody *, uint8_t *, uint32_t);
void HtpBodyPrint(HtpBody *);
void HtpBodyFree(HtpBody *);
void HtpBodyPrune(HtpTxUserData *);
void HtpBodyPrune(HtpBody *);
#endif /* __APP_LAYER_HTP_BODY_H__ */

5
src/app-layer-htp-file.c
Unescape Escape View File

@ -76,18 +76,15 @@
* \retval -2 not handling files on this flow
*/
int HTPFileOpen(HtpState *s, uint8_t *filename, uint16_t filename_len,
uint8_t *data, uint32_t data_len)
uint8_t *data, uint32_t data_len, uint16_t txid)
{
int retval = 0;
uint16_t txid;
uint8_t flags = 0;
if (s == NULL) {
SCReturnInt(-1);
}
txid = AppLayerTransactionGetAvailId(s->f) - 1;
if (s->files == NULL) {
s->files = FileContainerAlloc();
if (s->files == NULL) {

2
src/app-layer-htp-file.h
Unescape Escape View File

@ -25,7 +25,7 @@
#ifndef __APP_LAYER_HTP_FILE_H__
#define __APP_LAYER_HTP_FILE_H__
int HTPFileOpen(HtpState *, uint8_t *, uint16_t, uint8_t *, uint32_t);
int HTPFileOpen(HtpState *, uint8_t *, uint16_t, uint8_t *, uint32_t, uint16_t);
int HTPFileStoreChunk(HtpState *, uint8_t *, uint32_t);
int HTPFileClose(HtpState *, uint8_t *, uint32_t, uint8_t);

247
src/app-layer-htp.c
Unescape Escape View File

@ -80,6 +80,7 @@ typedef struct HTPCfgRec_ {
/** max size of the client body we inspect */
uint32_t request_body_limit;
uint32_t response_body_limit;
} HTPCfgRec;
/** Fast lookup tree (radix) for the various HTP configurations */
@ -100,6 +101,8 @@ static uint8_t need_htp_request_body = 0;
static uint8_t need_htp_request_multipart_hdr = 0;
/** part of the engine needs the request file (e.g. log-file module) */
static uint8_t need_htp_request_file = 0;
/** part of the engine needs the request body (e.g. file_data keyword) */
static uint8_t need_htp_response_body = 0;
#ifdef DEBUG
/**
@ -220,9 +223,10 @@ void HTPStateFree(void *state)
HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
if (htud != NULL) {
HtpBodyFree(&htud->request_body);
HtpBodyFree(&htud->response_body);
SCFree(htud);
htp_tx_set_user_data(tx, NULL);
}
htp_tx_set_user_data(tx, NULL);
}
}
}
@ -297,6 +301,18 @@ void AppLayerHtpEnableRequestBodyCallback(void)
SCReturn;
}
/**
* \brief Sets a flag that informs the HTP app layer that some module in the
* engine needs the http request body data.
* \initonly
*/
void AppLayerHtpEnableResponseBodyCallback(void)
{
SCEnter();
need_htp_response_body = 1;
SCReturn;
}
/**
* \brief Sets a flag that informs the HTP app layer that some module in the
* engine needs the http request multi part header.
@ -321,6 +337,8 @@ void AppLayerHtpNeedFileInspection(void)
{
SCEnter();
AppLayerHtpNeedMultipartHeader();
AppLayerHtpEnableRequestBodyCallback();
AppLayerHtpEnableResponseBodyCallback();
need_htp_request_file = 1;
SCReturn;
@ -383,11 +401,13 @@ static int HTPHandleRequestData(Flow *f, void *htp_state,
SCLogDebug("LIBHTP using config: %p", htp);
hstate->request_body_limit = htp_cfg_rec->request_body_limit;
hstate->response_body_limit = htp_cfg_rec->response_body_limit;
}
} else {
SCLogDebug("Using default HTP config: %p", htp);
hstate->request_body_limit = cfglist.request_body_limit;
hstate->response_body_limit = cfglist.response_body_limit;
}
if (NULL == htp) {
@ -820,7 +840,7 @@ static int HTTPParseContentTypeHeader(uint8_t *name, size_t name_len,
static int HtpRequestBodySetupMultipart(htp_tx_data_t *d, HtpTxUserData *htud) {
htp_header_t *cl = table_getc(d->tx->request_headers, "content-length");
if (cl != NULL)
htud->content_len = htp_parse_content_length(cl->value);
htud->request_body.content_len = htp_parse_content_length(cl->value);
htp_header_t *h = (htp_header_t *)table_getc(d->tx->request_headers,
"Content-Type");
@ -988,14 +1008,14 @@ static void HtpRequestBodyReassemble(HtpTxUserData *htud,
for ( ; cur != NULL; cur = cur->next) {
/* skip body chunks entirely before what we parsed already */
if (cur->stream_offset + cur->len <= htud->body_parsed)
if (cur->stream_offset + cur->len <= htud->request_body.body_parsed)
continue;
if (cur->stream_offset < htud->body_parsed &&
cur->stream_offset + cur->len >= htud->body_parsed) {
if (cur->stream_offset < htud->request_body.body_parsed &&
cur->stream_offset + cur->len >= htud->request_body.body_parsed) {
uint32_t toff = htud->body_parsed - cur->stream_offset;
uint32_t tlen = (cur->stream_offset + cur->len) - htud->body_parsed;
uint32_t toff = htud->request_body.body_parsed - cur->stream_offset;
uint32_t tlen = (cur->stream_offset + cur->len) - htud->request_body.body_parsed;
buf_len += tlen;
if ((buf = SCRealloc(buf, buf_len)) == NULL) {
@ -1107,7 +1127,7 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
}
}
htud->body_parsed += filedata_len;
htud->request_body.body_parsed += filedata_len;
} else {
SCLogDebug("chunk too small to already process in part");
}
@ -1169,7 +1189,7 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
#endif
result = HTPFileOpen(hstate, filename, filename_len,
filedata, filedata_len);
filedata, filedata_len, hstate->transaction_cnt);
if (result == -1) {
goto end;
} else if (result == -2) {
@ -1184,10 +1204,10 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
uint32_t offset = (header_end + 4) - chunks_buffer;
SCLogDebug("offset %u", offset);
htud->body_parsed = offset;
htud->request_body.body_parsed = offset;
result = HTPFileOpen(hstate, filename, filename_len,
filedata, filedata_len);
filedata, filedata_len, hstate->transaction_cnt);
if (result == -1) {
goto end;
} else if (result == -2) {
@ -1251,7 +1271,60 @@ int HtpRequestBodyHandlePUT(HtpState *hstate, HtpTxUserData *htud,
}
result = HTPFileOpen(hstate, filename, filename_len,
data, data_len);
data, data_len, hstate->transaction_cnt);
if (result == -1) {
goto end;
} else if (result == -2) {
htud->flags |= HTP_DONTSTORE;
} else {
htud->flags |= HTP_FILENAME_SET;
htud->flags &= ~HTP_DONTSTORE;
}
}
else
{
/* otherwise, just store the data */
if (!(htud->flags & HTP_DONTSTORE)) {
result = HTPFileStoreChunk(hstate, data, data_len);
if (result == -1) {
goto end;
} else if (result == -2) {
/* we know for sure we're not storing the file */
htud->flags |= HTP_DONTSTORE;
}
}
}
return 0;
end:
return -1;
}
int HtpResponseBodyHandle(HtpState *hstate, HtpTxUserData *htud,
htp_tx_t *tx, uint8_t *data, uint32_t data_len)
{
SCEnter();
int result = 0;
/* see if we need to open the file */
if (!(htud->flags & HTP_FILENAME_SET))
{
SCLogDebug("setting up file name");
uint8_t *filename = NULL;
uint32_t filename_len = 0;
/* get the name */
if (tx->parsed_uri != NULL && tx->parsed_uri->path != NULL) {
filename = (uint8_t *)bstr_ptr(tx->parsed_uri->path);
filename_len = bstr_len(tx->parsed_uri->path);
}
result = HTPFileOpen(hstate, filename, filename_len,
data, data_len, hstate->transaction_cnt - 1);
SCLogDebug("result %d", result);
if (result == -1) {
goto end;
} else if (result == -2) {
@ -1267,6 +1340,7 @@ int HtpRequestBodyHandlePUT(HtpState *hstate, HtpTxUserData *htud,
if (!(htud->flags & HTP_DONTSTORE)) {
result = HTPFileStoreChunk(hstate, data, data_len);
SCLogDebug("result %d", result);
if (result == -1) {
goto end;
} else if (result == -2) {
@ -1305,7 +1379,7 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d)
SCReturnInt(HOOK_OK);
}
memset(htud, 0, sizeof(HtpTxUserData));
htud->request_body.operation = HTP_BODY_REQUEST;
htud->operation = HTP_BODY_REQUEST;
if (d->tx->request_method_number == M_POST) {
if (HtpRequestBodySetupMultipart(d, htud) == 0) {
@ -1321,18 +1395,18 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d)
htp_tx_set_user_data(d->tx, htud);
}
SCLogDebug("htud->content_len_so_far %"PRIu64, htud->content_len_so_far);
SCLogDebug("htud->request_body.content_len_so_far %"PRIu64, htud->request_body.content_len_so_far);
SCLogDebug("hstate->request_body_limit %u", hstate->request_body_limit);
/* within limits, add the body chunk to the state. */
if (hstate->request_body_limit == 0 || htud->content_len_so_far < hstate->request_body_limit)
if (hstate->request_body_limit == 0 || htud->request_body.content_len_so_far < hstate->request_body_limit)
{
uint32_t len = (uint32_t)d->len;
if (hstate->request_body_limit > 0 &&
(htud->content_len_so_far + len) > hstate->request_body_limit)
(htud->request_body.content_len_so_far + len) > hstate->request_body_limit)
{
len = hstate->request_body_limit - htud->content_len_so_far;
len = hstate->request_body_limit - htud->request_body.content_len_so_far;
BUG_ON(len > (uint32_t)d->len);
}
SCLogDebug("len %u", len);
@ -1341,10 +1415,10 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d)
if (r < 0) {
htud->flags |= HTP_BODY_COMPLETE;
} else if (hstate->request_body_limit > 0 &&
htud->content_len_so_far >= hstate->request_body_limit)
htud->request_body.content_len_so_far >= hstate->request_body_limit)
{
htud->flags |= HTP_BODY_COMPLETE;
} else if (htud->content_len_so_far == htud->content_len) {
} else if (htud->request_body.content_len_so_far == htud->request_body.content_len) {
htud->flags |= HTP_BODY_COMPLETE;
}
@ -1375,7 +1449,76 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d)
end:
/* see if we can get rid of htp body chunks */
HtpBodyPrune(htud);
HtpBodyPrune(&htud->request_body);
/* set the new chunk flag */
hstate->flags |= HTP_FLAG_NEW_BODY_SET;
SCReturnInt(HOOK_OK);
}
/**
* \brief Function callback to append chunks for Responses
* \param d pointer to the htp_tx_data_t structure (a chunk from htp lib)
* \retval int HOOK_OK if all goes well
*/
int HTPCallbackResponseBodyData(htp_tx_data_t *d)
{
SCEnter();
HtpState *hstate = (HtpState *)d->tx->connp->user_data;
if (hstate == NULL) {
SCReturnInt(HOOK_ERROR);
}
SCLogDebug("New response body data available at %p -> %p -> %p, bodylen "
"%"PRIu32"", hstate, d, d->data, (uint32_t)d->len);
HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(d->tx);
if (htud == NULL) {
htud = SCMalloc(sizeof(HtpTxUserData));
if (htud == NULL) {
SCReturnInt(HOOK_OK);
}
memset(htud, 0, sizeof(HtpTxUserData));
htud->operation = HTP_BODY_RESPONSE;
/* Set the user data for handling body chunks on this transaction */
htp_tx_set_user_data(d->tx, htud);
}
SCLogDebug("htud->response_body.content_len_so_far %"PRIu64, htud->response_body.content_len_so_far);
SCLogDebug("hstate->response_body_limit %u", hstate->response_body_limit);
/* within limits, add the body chunk to the state. */
if (hstate->response_body_limit == 0 || htud->response_body.content_len_so_far < hstate->response_body_limit)
{
uint32_t len = (uint32_t)d->len;
if (hstate->response_body_limit > 0 &&
(htud->response_body.content_len_so_far + len) > hstate->response_body_limit)
{
len = hstate->response_body_limit - htud->response_body.content_len_so_far;
BUG_ON(len > (uint32_t)d->len);
}
SCLogDebug("len %u", len);
int r = HtpBodyAppendChunk(htud, &htud->response_body, (uint8_t *)d->data, len);
if (r < 0) {
htud->flags |= HTP_BODY_COMPLETE;
} else if (hstate->response_body_limit > 0 &&
htud->response_body.content_len_so_far >= hstate->response_body_limit)
{
htud->flags |= HTP_BODY_COMPLETE;
} else if (htud->response_body.content_len_so_far == htud->response_body.content_len) {
htud->flags |= HTP_BODY_COMPLETE;
}
HtpResponseBodyHandle(hstate, htud, d->tx, (uint8_t *)d->data, (uint32_t)d->len);
}
/* see if we can get rid of htp body chunks */
HtpBodyPrune(&htud->response_body);
/* set the new chunk flag */
hstate->flags |= HTP_FLAG_NEW_BODY_SET;
@ -1470,9 +1613,22 @@ static int HTPCallbackResponse(htp_connp_t *connp) {
/* Unset the body inspection (if any) */
hstate->flags &=~ HTP_FLAG_NEW_BODY_SET;
if (connp->out_tx != NULL) {
HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(connp->out_tx);
if (htud != NULL) {
if (htud->flags & HTP_FILENAME_SET) {
SCLogDebug("closing file that was being stored");
(void)HTPFileClose(hstate, NULL, 0, 0);
htud->flags &= ~HTP_FILENAME_SET;
}
}
}
/* remove obsolete transactions */
size_t idx;
for (idx = 0; idx < hstate->transaction_done; idx++) {
SCLogDebug("idx %"PRIuMAX, (uintmax_t)idx);
htp_tx_t *tx = list_get(hstate->connp->conn->transactions, idx);
if (tx == NULL)
continue;
@ -1481,8 +1637,9 @@ static int HTPCallbackResponse(htp_connp_t *connp) {
HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
if (htud != NULL) {
HtpBodyFree(&htud->request_body);
htp_tx_set_user_data(tx, NULL);
HtpBodyFree(&htud->response_body);
SCFree(htud);
htp_tx_set_user_data(tx, NULL);
}
htp_tx_destroy(tx);
@ -1527,6 +1684,7 @@ static void HTPConfigure(void)
SCLogDebug("LIBHTP default config: %p", cfglist.cfg);
cfglist.request_body_limit = HTP_CONFIG_DEFAULT_REQUEST_BODY_LIMIT;
cfglist.response_body_limit = HTP_CONFIG_DEFAULT_RESPONSE_BODY_LIMIT;
htp_config_register_request(cfglist.cfg, HTPCallbackRequest);
htp_config_register_response(cfglist.cfg, HTPCallbackResponse);
#ifdef HAVE_HTP_URI_NORMALIZE_HOOK
@ -1593,12 +1751,31 @@ static void HTPConfigure(void)
}
else {
SCLogWarning(SC_ERR_UNKNOWN_VALUE,
"LIBHTP malformed request_body_limit "
"LIBHTP malformed request-body-limit "
"\"%s\", using default %u", p->val,
HTP_CONFIG_DEFAULT_REQUEST_BODY_LIMIT);
cfglist.request_body_limit = HTP_CONFIG_DEFAULT_REQUEST_BODY_LIMIT;
continue;
}
} else if (strcasecmp("response-body-limit", p->name) == 0) {
/* limit */
int limit = atoi(p->val);
if (limit >= 0) {
SCLogDebug("LIBHTP default: %s=%s (%d)",
p->name, p->val, limit);
cfglist.response_body_limit = (uint32_t)limit;
}
else {
SCLogWarning(SC_ERR_UNKNOWN_VALUE,
"LIBHTP malformed response-body-limit "
"\"%s\", using default %u", p->val,
HTP_CONFIG_DEFAULT_RESPONSE_BODY_LIMIT);
cfglist.response_body_limit = HTP_CONFIG_DEFAULT_RESPONSE_BODY_LIMIT;
continue;
}
} else {
SCLogWarning(SC_ERR_UNKNOWN_VALUE,
@ -1757,6 +1934,25 @@ static void HTPConfigure(void)
htprec->request_body_limit = HTP_CONFIG_DEFAULT_REQUEST_BODY_LIMIT;
continue;
}
} else if (strcasecmp("response-body-limit", p->name) == 0) {
/* limit */
int limit = atoi(p->val);
if (limit >= 0) {
SCLogDebug("LIBHTP default: %s=%s (%d)",
p->name, p->val, limit);
htprec->response_body_limit = (uint32_t)limit;
}
else {
SCLogWarning(SC_ERR_UNKNOWN_VALUE,
"LIBHTP malformed response-body-limit "
"\"%s\", using default %u", p->val,
HTP_CONFIG_DEFAULT_RESPONSE_BODY_LIMIT);
htprec->response_body_limit = HTP_CONFIG_DEFAULT_RESPONSE_BODY_LIMIT;
continue;
}
} else {
SCLogWarning(SC_ERR_UNKNOWN_VALUE,
"LIBHTP Ignoring unknown server config: %s",
@ -1835,8 +2031,11 @@ void AppLayerHtpRegisterExtraCallbacks(void) {
SCLogDebug("Registering callback htp_config_register_request_body_data on htp");
htp_config_register_request_body_data(cfglist.cfg,
HTPCallbackRequestBodyData);
} else {
SCLogDebug("No htp extra callback needed");
}
if (need_htp_response_body == 1) {
SCLogDebug("Registering callback htp_config_register_response_body_data on htp");
htp_config_register_response_body_data(cfglist.cfg,
HTPCallbackResponseBodyData);
}
SCReturn;
}

23
src/app-layer-htp.h
Unescape Escape View File

@ -40,6 +40,7 @@
/* default request body limit */
#define HTP_CONFIG_DEFAULT_REQUEST_BODY_LIMIT 4096U
#define HTP_CONFIG_DEFAULT_RESPONSE_BODY_LIMIT 4096U
/** a boundary should be smaller in size */
#define HTP_BOUNDARY_MAX 200U
@ -92,10 +93,15 @@ typedef struct HtpBody_ {
HtpBodyChunk *first; /**< Pointer to the first chunk */
HtpBodyChunk *last; /**< Pointer to the last chunk */
uint32_t nchunks; /**< Number of chunks in the current operation */
uint8_t operation; /**< This flag indicate if it's a request
or a response */
uint8_t type;
/* Holds the length of the htp request body */
uint64_t content_len;
/* Holds the length of the htp request body seen so far */
uint64_t content_len_so_far;
uint64_t body_parsed;
/* pahole: padding: 3 */
} HtpBody;
@ -122,14 +128,7 @@ typedef struct HtpBody_ {
typedef struct HtpTxUserData_ {
/* Body of the request (if any) */
HtpBody request_body;
//HtpBody response_body;
/* Holds the length of the htp request body */
uint64_t content_len;
/* Holds the length of the htp request body seen so far */
uint64_t content_len_so_far;
uint64_t body_parsed;
HtpBody response_body;
/** Holds the boundary identificator string if any (used on
* multipart/form-data only)
@ -138,6 +137,8 @@ typedef struct HtpTxUserData_ {
uint8_t boundary_len;
uint8_t flags;
int16_t operation;
} HtpTxUserData;
typedef struct HtpState_ {
@ -149,6 +150,7 @@ typedef struct HtpState_ {
uint16_t transaction_cnt;
uint16_t transaction_done;
uint32_t request_body_limit;
uint32_t response_body_limit;
FileContainer *files;
} HtpState;
@ -167,6 +169,7 @@ void AppLayerHtpRegisterExtraCallbacks(void);
/* To free the state from unittests using app-layer-htp */
void HTPStateFree(void *);
void AppLayerHtpEnableRequestBodyCallback(void);
void AppLayerHtpEnableResponseBodyCallback(void);
void AppLayerHtpNeedFileInspection(void);
void AppLayerHtpPrintStats(void);

39
src/detect-engine-file.c
Unescape Escape View File

@ -75,7 +75,7 @@ static int DetectFileInspect(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Flo
int r = 0;
int match = 0;
SCLogDebug("file inspection...");
SCLogDebug("file inspection... %p", ffc);
if (ffc != NULL) {
File *file = ffc->head;
@ -139,13 +139,33 @@ static int DetectFileInspect(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Flo
* return 3 so we can distinguish */
if (s->init_flags & SIG_FLAG_FILESTORE && r == 2)
r = 3;
/* continue, this file may (or may not) be unable to match
* maybe we have more that can :) */
}
}
SCReturnInt(r);
}
int DetectFileInspectHttp(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Flow *f, Signature *s, void *alstate) {
/**
* \brief Inspect the file inspecting keywords against the HTTP transactions.
*
* \param tv thread vars
* \param det_ctx detection engine thread ctx
* \param f flow
* \param s signature to inspect
* \param alstate state
* \param flags direction flag
*
* \retval 0 no match
* \retval 1 match
* \retval 2 can't match
* \retval 3 can't match filestore signature
*
* \note flow is not locked at this time
*/
int DetectFileInspectHttp(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Flow *f, Signature *s, void *alstate, uint8_t flags) {
SCEnter();
int r = 0;
@ -155,21 +175,24 @@ int DetectFileInspectHttp(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Flow *
size_t end_tx = 0;
int match = 0;
/* locking the flow, we will inspect the htp state */
SCMutexLock(&f->m);
htp_state = (HtpState *)alstate;
if (htp_state == NULL) {
SCLogDebug("no HTTP state");
SCReturnInt(0);
goto end;
}
/* locking the flow, we will inspect the htp state */
SCMutexLock(&f->m);
if (htp_state->connp != NULL && htp_state->connp->conn != NULL)
{
start_tx = AppLayerTransactionGetInspectId(f);
/* tx cnt is incremented after request finishes, so we need to inspect
* response one before the lowest. */
if (flags & STREAM_TOCLIENT && start_tx)
start_tx--;
end_tx = list_size(htp_state->connp->conn->transactions);
}
SCMutexUnlock(&f->m);
for (idx = start_tx ; idx < end_tx; idx++)
{
@ -192,5 +215,7 @@ int DetectFileInspectHttp(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Flow *
}
}
end:
SCMutexUnlock(&f->m);
SCReturnInt(r);
}

2
src/detect-engine-file.h
Unescape Escape View File

@ -24,6 +24,6 @@
#ifndef __DETECT_ENGINE_FILE_H__
#define __DETECT_ENGINE_FILE_H__
int DetectFileInspectHttp(ThreadVars *, DetectEngineThreadCtx *, Flow *, Signature *, void *);
int DetectFileInspectHttp(ThreadVars *, DetectEngineThreadCtx *, Flow *, Signature *, void *, uint8_t);
#endif /* __DETECT_ENGINE_FILE_H__ */

10
src/detect-engine-hcbd.c
Unescape Escape View File

@ -388,12 +388,6 @@ static void DetectEngineBufferHttpClientBodies(DetectEngineCtx *de_ctx,
continue;
}
/* this applies only for the client request body like the keyword name says */
if (htud->request_body.operation != HTP_BODY_REQUEST) {
SCLogDebug("htp chunk not a request chunk");
continue;
}
/* in case of chunked transfer encoding, we don't have the length
* of the request body until we see a chunk with length 0. This
* doesn't let us use the request body callback function to
@ -407,8 +401,8 @@ static void DetectEngineBufferHttpClientBodies(DetectEngineCtx *de_ctx,
* and running content validation on this buffer type of architecture
* to a stateful inspection, where we can inspect body chunks as and
* when they come */
if (htud->content_len == 0) {
if ((htud->content_len_so_far > 0) &&
if (htud->request_body.content_len == 0) {
if ((htud->request_body.content_len_so_far > 0) &&
tx->progress != TX_PROGRESS_REQ_BODY) {
/* final length of the body */
htud->flags |= HTP_BODY_COMPLETE;

70
src/detect-engine-state.c
Unescape Escape View File

@ -467,7 +467,7 @@ int DeStateDetectStartDetection(ThreadVars *tv, DetectEngineCtx *de_ctx,
if (s->sm_lists[DETECT_SM_LIST_FILEMATCH] != NULL) {
inspect_flags |= DE_STATE_FLAG_FILE_INSPECT;
match = DetectFileInspectHttp(tv, det_ctx, f, s, alstate);
match = DetectFileInspectHttp(tv, det_ctx, f, s, alstate, flags);
if (match == 1) {
match_flags |= DE_STATE_FLAG_FILE_MATCH;
} else if (match == 2) {
@ -519,6 +519,17 @@ int DeStateDetectStartDetection(ThreadVars *tv, DetectEngineCtx *de_ctx,
}
if (s->sm_lists[DETECT_SM_LIST_FILEMATCH] != NULL) {
inspect_flags |= DE_STATE_FLAG_FILE_INSPECT;
match = DetectFileInspectHttp(tv, det_ctx, f, s, alstate, flags);
SCLogDebug("match %d", match);
if (match == 1) {
match_flags |= DE_STATE_FLAG_FILE_MATCH;
} else if (match == 2) {
match_flags |= DE_STATE_FLAG_SIG_CANT_MATCH;
} else if (match == 3) {
match_flags |= DE_STATE_FLAG_SIG_CANT_MATCH;
file_no_match++;
}
}
}
} else if (alproto == ALPROTO_DCERPC || alproto == ALPROTO_SMB || alproto == ALPROTO_SMB2) {
@ -768,7 +779,62 @@ int DeStateDetectContinueDetection(ThreadVars *tv, DetectEngineCtx *de_ctx, Dete
if (!(item->flags & DE_STATE_FLAG_FILE_MATCH)) {
inspect_flags |= DE_STATE_FLAG_FILE_INSPECT;
match = DetectFileInspectHttp(tv, det_ctx, f, s, alstate);
match = DetectFileInspectHttp(tv, det_ctx, f, s, alstate, flags);
if (match == 1) {
match_flags |= DE_STATE_FLAG_FILE_MATCH;
} else if (match == 2) {
match_flags |= DE_STATE_FLAG_SIG_CANT_MATCH;
} else if (match == 3) {
match_flags |= DE_STATE_FLAG_SIG_CANT_MATCH;
file_no_match++;
}
}
}
} else if (alproto == ALPROTO_HTTP && (flags & STREAM_TOCLIENT)) {
/* For to client set the flags in inspect so it can't match
* if the sig requires something only the request has. The rest
* will be inspected in the opposite direction. */
if (s->sm_lists[DETECT_SM_LIST_UMATCH] != NULL) {
inspect_flags |= DE_STATE_FLAG_URI_INSPECT;
}
if (s->sm_lists[DETECT_SM_LIST_HCBDMATCH] != NULL) {
inspect_flags |= DE_STATE_FLAG_HCBD_INSPECT;
}
if (s->sm_lists[DETECT_SM_LIST_HHDMATCH] != NULL) {
inspect_flags |= DE_STATE_FLAG_HHD_INSPECT;
if (DetectEngineInspectHttpHeader(de_ctx, det_ctx, s, f,
flags, alstate) == 1) {
match_flags |= DE_STATE_FLAG_HHD_MATCH;
}
SCLogDebug("inspecting http header");
}
if (s->sm_lists[DETECT_SM_LIST_HRHDMATCH] != NULL) {
inspect_flags |= DE_STATE_FLAG_HRHD_INSPECT;
if (DetectEngineInspectHttpRawHeader(de_ctx, det_ctx, s, f,
flags, alstate) == 1) {
match_flags |= DE_STATE_FLAG_HRHD_MATCH;
}
SCLogDebug("inspecting http raw header");
}
if (s->sm_lists[DETECT_SM_LIST_HMDMATCH] != NULL) {
inspect_flags |= DE_STATE_FLAG_HMD_INSPECT;
}
if (s->sm_lists[DETECT_SM_LIST_HCDMATCH] != NULL) {
inspect_flags |= DE_STATE_FLAG_HCD_INSPECT;
if (DetectEngineInspectHttpCookie(de_ctx, det_ctx, s, f,
flags, alstate) == 1) {
match_flags |= DE_STATE_FLAG_HCD_MATCH;
}
SCLogDebug("inspecting http cookie");
}
if (s->sm_lists[DETECT_SM_LIST_HRUDMATCH] != NULL) {
inspect_flags |= DE_STATE_FLAG_HRUD_INSPECT;
}
if (s->sm_lists[DETECT_SM_LIST_FILEMATCH] != NULL) {
if (!(item->flags & DE_STATE_FLAG_FILE_MATCH)) {
inspect_flags |= DE_STATE_FLAG_FILE_INSPECT;
match = DetectFileInspectHttp(tv, det_ctx, f, s, alstate, flags);
if (match == 1) {
match_flags |= DE_STATE_FLAG_FILE_MATCH;
} else if (match == 2) {

6
src/detect-filemagic.c
Unescape Escape View File

@ -41,6 +41,7 @@
#include "util-debug.h"
#include "util-spm-bm.h"
#include "util-magic.h"
#include "util-print.h"
#include "util-unittest.h"
#include "util-unittest-helper.h"
@ -82,7 +83,7 @@ void DetectFilemagicRegister(void) {
* \retval -1 error
* \retval 0 ok
*/
static int FilemagicLookup(File *file) {
int FilemagicLookup(File *file) {
if (file == NULL || file->chunks_head == NULL) {
SCReturnInt(-1);
}
@ -108,7 +109,6 @@ static int FilemagicLookup(File *file) {
file->magic = MagicLookup(buf, size);
break;
}
/* file is done but smaller than FILEMAGIC_MIN_SIZE */
if (ffd->next == NULL && file->state >= FILE_STATE_CLOSED) {
file->magic = MagicLookup(buf, size);
@ -138,8 +138,8 @@ int DetectFilemagicMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, Flow *f
{
SCEnter();
int ret = 0;
DetectFilemagicData *filemagic = (DetectFilemagicData *)m->ctx;
DetectFilemagicData *filemagic = m->ctx;
File *file = (File *)state;
if (file->txid < det_ctx->tx_id)

1
src/detect-filemagic.h
Unescape Escape View File

@ -35,5 +35,6 @@ typedef struct DetectFilemagicData {
/* prototypes */
void DetectFilemagicRegister (void);
int FilemagicLookup(File *file);
#endif /* __DETECT_FILEMAGIC_H__ */

121
src/log-file.c
Unescape Escape View File

@ -35,11 +35,14 @@
#include "app-layer-parser.h"
#include "detect-filemagic.h"
#include "util-print.h"
#include "util-unittest.h"
#include "util-privs.h"
#include "util-debug.h"
#include "util-atomic.h"
#include "util-file.h"
#include "output.h"
@ -58,6 +61,7 @@ static void LogFileLogDeInitCtx(OutputCtx *);
SC_ATOMIC_DECLARE(unsigned int, file_id);
static char g_logfile_base_dir[PATH_MAX] = "/tmp";
static int g_logfile_force_magic = 0;
void TmModuleLogFileLogRegister (void) {
tmm_modules[TMM_FILELOG].name = MODULE_NAME;
@ -92,6 +96,70 @@ static void CreateTimeString (const struct timeval *ts, char *str, size_t size)
t->tm_min, t->tm_sec, (uint32_t) ts->tv_usec);
}
static void LogFileLogCreateMetaFile(Packet *p, File *ff, char *filename) {
char metafilename[PATH_MAX] = "";
snprintf(metafilename, sizeof(metafilename), "%s.meta", filename);
FILE *fp = fopen(metafilename, "w+");
if (fp != NULL) {
char timebuf[64];
CreateTimeString(&p->ts, timebuf, sizeof(timebuf));
fprintf(fp, "TIME: %s\n", timebuf);
if (p->pcap_cnt > 0) {
fprintf(fp, "PCAP PKT NUM: %"PRIu64"\n", p->pcap_cnt);
}
char srcip[16], dstip[16];
inet_ntop(AF_INET, (const void *)GET_IPV4_SRC_ADDR_PTR(p), srcip, sizeof(srcip));
inet_ntop(AF_INET, (const void *)GET_IPV4_DST_ADDR_PTR(p), dstip, sizeof(dstip));
fprintf(fp, "SRC IP: %s\n", srcip);
fprintf(fp, "DST IP: %s\n", dstip);
fprintf(fp, "PROTO: %" PRIu32 "\n", IPV4_GET_IPPROTO(p));
if (PKT_IS_TCP(p) || PKT_IS_UDP(p)) {
fprintf(fp, "SRC PORT: %" PRIu16 "\n", p->sp);
fprintf(fp, "DST PORT: %" PRIu16 "\n", p->dp);
}
fprintf(fp, "FILENAME: ");
PrintRawUriFp(fp, ff->name, ff->name_len);
fprintf(fp, "\n");
fclose(fp);
}
}
static void LogFileLogCloseMetaFile(File *ff) {
char filename[PATH_MAX] = "";
snprintf(filename, sizeof(filename), "%s/file.%u",
g_logfile_base_dir, ff->file_id);
char metafilename[PATH_MAX] = "";
snprintf(metafilename, sizeof(metafilename), "%s.meta", filename);
FILE *fp = fopen(metafilename, "a");
if (g_logfile_force_magic || ff->magic != NULL) {
if (g_logfile_force_magic && ff->magic == NULL) {
FilemagicLookup(ff);
}
fprintf(fp, "MAGIC: %s\n", ff->magic ? ff->magic : "<unknown>");
}
switch (ff->state) {
case FILE_STATE_CLOSED:
fprintf(fp, "STATE: CLOSED\n");
break;
case FILE_STATE_TRUNCATED:
fprintf(fp, "STATE: TRUNCATED\n");
break;
case FILE_STATE_ERROR:
fprintf(fp, "STATE: ERROR\n");
break;
}
fprintf(fp, "SIZE: %"PRIu64"\n", ff->size);
fclose(fp);
}
TmEcode LogFileLogIPv4(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq)
{
SCEnter();
@ -105,18 +173,24 @@ TmEcode LogFileLogIPv4(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, P
SCMutexLock(&p->flow->m);
FileContainer *ffc = AppLayerGetFilesFromFlow(p->flow);
SCLogDebug("ffc %p", ffc);
if (ffc != NULL) {
File *ff;
for (ff = ffc->head; ff != NULL; ff = ff->next) {
if (ff->state == FILE_STATE_STORED)
SCLogDebug("ff %p", ff);
if (ff->state == FILE_STATE_STORED) {
SCLogDebug("ff->state == FILE_STATE_STORED");
continue;
}
if (ff->store != 1) {
SCLogDebug("ff->store %d, so not 1", ff->store);
continue;
}
FileData *ffd;
for (ffd = ff->chunks_head; ffd != NULL; ffd = ffd->next) {
SCLogDebug("ffd %p", ffd);
if (ffd->stored == 1)
continue;
@ -127,7 +201,7 @@ TmEcode LogFileLogIPv4(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, P
char filename[PATH_MAX] = "";
snprintf(filename, sizeof(filename), "%s/file.%u",
g_logfile_base_dir, SC_ATOMIC_GET(file_id));
SC_ATOMIC_ADD(file_id, 1);
ff->file_id = SC_ATOMIC_ADD(file_id, 1);
ff->fd = open(filename, O_CREAT | O_TRUNC | O_NOFOLLOW | O_WRONLY, 0644);
if (ff->fd == -1) {
@ -136,36 +210,7 @@ TmEcode LogFileLogIPv4(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, P
}
/* create a .meta file that contains time, src/dst/sp/dp/proto */
char metafilename[PATH_MAX] = "";
snprintf(metafilename, sizeof(metafilename), "%s.meta", filename);
FILE *fp = fopen(metafilename, "w+");
if (fp != NULL) {
char timebuf[64];
CreateTimeString(&p->ts, timebuf, sizeof(timebuf));
fprintf(fp, "TIME: %s\n", timebuf);
if (p->pcap_cnt > 0) {
fprintf(fp, "PCAP PKT NUM: %"PRIu64"\n", p->pcap_cnt);
}
char srcip[16], dstip[16];
inet_ntop(AF_INET, (const void *)GET_IPV4_SRC_ADDR_PTR(p), srcip, sizeof(srcip));
inet_ntop(AF_INET, (const void *)GET_IPV4_DST_ADDR_PTR(p), dstip, sizeof(dstip));
fprintf(fp, "SRC IP: %s\n", srcip);
fprintf(fp, "DST IP: %s\n", dstip);
fprintf(fp, "PROTO: %" PRIu32 "\n", IPV4_GET_IPPROTO(p));
if (PKT_IS_TCP(p) || PKT_IS_UDP(p)) {
fprintf(fp, "SRC PORT: %" PRIu32 "\n", p->sp);
fprintf(fp, "DST PORT: %" PRIu32 "\n", p->dp);
}
fprintf(fp, "FILENAME: ");
PrintRawUriFp(fp, ff->name, ff->name_len);
fprintf(fp, "\n");
fclose(fp);
}
LogFileLogCreateMetaFile(p, ff, filename);
aft->file_cnt++;
} else {
@ -183,6 +228,8 @@ TmEcode LogFileLogIPv4(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, P
ff->state == FILE_STATE_ERROR)
{
if (ffd->next == NULL) {
LogFileLogCloseMetaFile(ff);
ff->state = FILE_STATE_STORED;
close(ff->fd);
ff->fd = -1;
@ -249,8 +296,6 @@ TmEcode LogFileLogThreadInit(ThreadVars *t, void *initdata, void **data)
/* Use the Ouptut Context (file pointer and mutex) */
aft->file_ctx = ((OutputCtx *)initdata)->data;
SCLogInfo("storing files in %s", g_logfile_base_dir);
struct stat stat_buf;
if (stat(g_logfile_base_dir, &stat_buf) != 0) {
SCLogError(SC_ERR_LOGDIR_CONFIG, "The file drop directory \"%s\" "
@ -318,6 +363,14 @@ OutputCtx *LogFileLogInitCtx(ConfNode *conf)
}
}
const char *force_magic = ConfNodeLookupChildValue(conf, "force-magic");
if (force_magic != NULL && ConfValIsTrue(force_magic)) {
g_logfile_force_magic = 1;
SCLogInfo("forcing magic lookup for stored files");
}
SCLogInfo("storing files in %s", g_logfile_base_dir);
SCReturnPtr(output_ctx, "OutputCtx");
}

59
src/util-magic.c
Unescape Escape View File

@ -21,6 +21,10 @@
* \author Victor Julien <victor@inliniac.net>
*
* Wrappers and tests for libmagic usage.
*
* Libmagic's API is not thread safe. The data the pointer returned by
* magic_buffer is overwritten by the next magic_buffer call. This is
* why we need to lock calls and copy the returned string.
*/
#include "suricata-common.h"
@ -31,40 +35,45 @@
#include <magic.h>
static magic_t g_magic_ctx = NULL;
static SCMutex g_magic_lock;
/**
* \brief Initialize the "magic" context.
*/
int MagicInit(void) {
BUG_ON(g_magic_ctx != NULL);
SCEnter();
char *filename = NULL;
FILE *fd = NULL;
SCEnter();
SCMutexInit(&g_magic_lock, NULL);
SCMutexLock(&g_magic_lock);
g_magic_ctx = magic_open(0);
if (g_magic_ctx == NULL) {
g_magic_ctx = magic_open(0);
if (g_magic_ctx == NULL) {
SCLogError(SC_ERR_MAGIC_OPEN, "magic_open failed: %s", magic_error(g_magic_ctx));
goto error;
}
ConfGet("magic-file", &filename);
if (filename != NULL) {
SCLogInfo("using magic-file %s", filename);
SCLogError(SC_ERR_MAGIC_OPEN, "magic_open failed: %s", magic_error(g_magic_ctx));
goto error;
}
if ( (fd = fopen(filename, "r")) == NULL) {
SCLogWarning(SC_ERR_FOPEN, "Error opening file: \"%s\": %s", filename, strerror(errno));
goto error;
}
fclose(fd);
}
ConfGet("magic-file", &filename);
if (filename != NULL) {
SCLogInfo("using magic-file %s", filename);
if (magic_load(g_magic_ctx, filename) != 0) {
SCLogError(SC_ERR_MAGIC_LOAD, "magic_load failed: %s", magic_error(g_magic_ctx));
if ( (fd = fopen(filename, "r")) == NULL) {
SCLogWarning(SC_ERR_FOPEN, "Error opening file: \"%s\": %s", filename, strerror(errno));
goto error;
}
fclose(fd);
}
if (magic_load(g_magic_ctx, filename) != 0) {
SCLogError(SC_ERR_MAGIC_LOAD, "magic_load failed: %s", magic_error(g_magic_ctx));
goto error;
}
SCMutexUnlock(&g_magic_lock);
SCReturnInt(0);
error:
@ -73,6 +82,7 @@ error:
g_magic_ctx = NULL;
}
SCMutexUnlock(&g_magic_lock);
SCReturnInt(-1);
}
@ -82,16 +92,23 @@ error:
* \param buf the buffer
* \param buflen length of the buffer
*
* \retval result pointer to const string
* \retval result pointer to null terminated string
*/
const char *MagicLookup(uint8_t *buf, uint32_t buflen) {
char *MagicLookup(uint8_t *buf, uint32_t buflen) {
const char *result = NULL;
char *magic = NULL;
SCMutexLock(&g_magic_lock);
if (buf != NULL && buflen > 0) {
result = magic_buffer(g_magic_ctx, (void *)buf, (size_t)buflen);
if (result != NULL) {
magic = SCStrdup(result);
}
}
SCReturnPtr(result, "const char");
SCMutexUnlock(&g_magic_lock);
SCReturnPtr(magic, "const char");
}
void MagicDeinit(void) {

4
src/util-magic.h
Unescape Escape View File

@ -26,9 +26,7 @@
int MagicInit(void);
void MagicDeinit(void);
const char *MagicLookup(uint8_t *, uint32_t);
char *MagicLookup(uint8_t *, uint32_t);
void MagicRegisterTests(void);
#endif /* __UTIL_MAGIC_H__ */

3
suricata.yaml
Unescape Escape View File

@ -756,6 +756,7 @@ libhtp:
default-config:
personality: IDS
request_body_limit: 3072
response_body_limit: 3072
server-config:
@ -763,6 +764,7 @@ libhtp:
address: [192.168.1.0/24, 127.0.0.0/8, "::1"]
personality: Apache_2_2
request_body_limit: 4096
response_body_limit: 4096
- iis7:
address:
@ -770,6 +772,7 @@ libhtp:
- 192.168.10.0/24
personality: IIS_7_0
request_body_limit: 4096
response_body_limit: 4096
# Profiling settings. Only effective if Suricata has been built with the
# the --enable-profiling configure flag.

Write Preview
Loading…
Cancel
Save