aiden
/
suricata
mirror of https://github.com/OISF/suricata
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

support for fast_pattern only and fast_pattern:offset,length. Also support the new option for engine-analysis

Browse Source
remotes/origin/master-1.1.x
Anoop Saldanha 15 years ago committed by Victor Julien
parent 0d741b9a55
commit a85fa6b792
12 changed files with 1627 additions and 62 deletions
Show Stats Download Patch File Download Diff File

28
src/detect-content.h
Unescape Escape View File

@ -26,17 +26,21 @@
/* Flags affecting this content */
#define DETECT_CONTENT_NOCASE 0x01
#define DETECT_CONTENT_DISTANCE 0x02
#define DETECT_CONTENT_WITHIN 0x04
#define DETECT_CONTENT_FAST_PATTERN 0x08
#define DETECT_CONTENT_NOCASE 0x0001
#define DETECT_CONTENT_DISTANCE 0x0002
#define DETECT_CONTENT_WITHIN 0x0004
#define DETECT_CONTENT_OFFSET 0x0008
#define DETECT_CONTENT_DEPTH 0x0010
#define DETECT_CONTENT_FAST_PATTERN 0x0020
#define DETECT_CONTENT_FAST_PATTERN_ONLY 0x0040
#define DETECT_CONTENT_FAST_PATTERN_CHOP 0x0080
/** content applies to a "raw"/undecoded field if applicable */
#define DETECT_CONTENT_RAWBYTES 0x10
#define DETECT_CONTENT_RAWBYTES 0x0100
/** content is negated */
#define DETECT_CONTENT_NEGATED 0x20
#define DETECT_CONTENT_NEGATED 0x0200
/** a relative match to this content is next, used in matching phase */
#define DETECT_CONTENT_RELATIVE_NEXT 0x40
#define DETECT_CONTENT_RELATIVE_NEXT 0x0400
#define DETECT_CONTENT_IS_SINGLE(c) (!((c)->flags & DETECT_CONTENT_DISTANCE || \
(c)->flags & DETECT_CONTENT_WITHIN || \
@ -49,7 +53,7 @@
typedef struct DetectContentData_ {
uint8_t *content; /**< ptr to chunk of memory containing the pattern */
uint8_t content_len;/**< length of the pattern (and size of the memory) */
uint8_t flags;
uint16_t flags;
PatIntId id; /**< unique pattern id */
uint16_t depth;
uint16_t offset;
@ -58,7 +62,13 @@ typedef struct DetectContentData_ {
int32_t distance;
int32_t within;
BmCtx *bm_ctx; /**< Boyer Moore context (for spm search) */
/* if someone wants to add an extra var to this structutre of size 1 byte
* you can reduce the below var to uint8_t. No problemo */
uint16_t avoid_double_check;
/* for chopped fast pattern, the offset */
uint16_t fp_chop_offset;
/* for chopped fast pattern, the length */
uint16_t fp_chop_len;
} DetectContentData;
/* prototypes */

33
src/detect-depth.c
Unescape Escape View File

@ -55,6 +55,7 @@ static int DetectDepthSetup (DetectEngineCtx *de_ctx, Signature *s, char *depths
char *str = depthstr;
char dubbed = 0;
SigMatch *pm = NULL;
DetectContentData *cd = NULL;
/* strip "'s */
if (depthstr[0] == '\"' && depthstr[strlen(depthstr)-1] == '\"') {
@ -115,13 +116,28 @@ static int DetectDepthSetup (DetectEngineCtx *de_ctx, Signature *s, char *depths
break;
case DETECT_CONTENT:
{
DetectContentData *cd = (DetectContentData *)pm->ctx;
cd = (DetectContentData *)pm->ctx;
if (cd == NULL) {
SCLogError(SC_ERR_INVALID_ARGUMENT, "invalid argument");
if (dubbed) SCFree(str);
return -1;
}
if (cd->flags & DETECT_CONTENT_NEGATED) {
if (cd->flags & DETECT_CONTENT_FAST_PATTERN) {
SCLogError(SC_ERR_INVALID_SIGNATURE, "You can't have a relative "
"negated keyword set along with a fast_pattern");
goto error;
}
} else {
if (cd->flags & DETECT_CONTENT_FAST_PATTERN_ONLY) {
SCLogError(SC_ERR_INVALID_SIGNATURE, "You can't have a relative "
"keyword set along with a fast_pattern:only;");
goto error;
}
}
cd->depth = (uint32_t)atoi(str);
if (cd->depth < cd->content_len) {
cd->depth = cd->content_len;
@ -130,8 +146,9 @@ static int DetectDepthSetup (DetectEngineCtx *de_ctx, Signature *s, char *depths
}
/* Now update the real limit, as depth is relative to the offset */
cd->depth += cd->offset;
}
break;
cd->flags |= DETECT_CONTENT_DEPTH;
break;
default:
SCLogError(SC_ERR_DEPTH_MISSING_CONTENT, "depth needs a preceeding "
@ -141,6 +158,12 @@ static int DetectDepthSetup (DetectEngineCtx *de_ctx, Signature *s, char *depths
break;
}
if (dubbed) SCFree(str);
if (dubbed)
SCFree(str);
return 0;
error:
if (dubbed)
SCFree(str);
return -1;
}

22
src/detect-distance.c
Unescape Escape View File

@ -250,6 +250,20 @@ static int DetectDistanceSetup (DetectEngineCtx *de_ctx, Signature *s,
goto error;
}
if (cd->flags & DETECT_CONTENT_NEGATED) {
if (cd->flags & DETECT_CONTENT_FAST_PATTERN) {
SCLogError(SC_ERR_INVALID_SIGNATURE, "You can't have a relative "
"negated keyword set along with a fast_pattern");
goto error;
}
} else {
if (cd->flags & DETECT_CONTENT_FAST_PATTERN_ONLY) {
SCLogError(SC_ERR_INVALID_SIGNATURE, "You can't have a relative "
"keyword set along with a fast_pattern:only;");
goto error;
}
}
cd->distance = strtol(str, NULL, 10);
cd->flags |= DETECT_CONTENT_DISTANCE;
if (cd->flags & DETECT_CONTENT_WITHIN) {
@ -284,6 +298,14 @@ static int DetectDistanceSetup (DetectEngineCtx *de_ctx, Signature *s,
}
cd->flags |= DETECT_CONTENT_RELATIVE_NEXT;
if (cd->flags & DETECT_CONTENT_FAST_PATTERN_ONLY) {
SCLogError(SC_ERR_INVALID_SIGNATURE, "Previous keyword "
"has a fast_pattern:only; set. You can't "
"have relative keywords around a fast_pattern "
"only content");
goto error;
}
break;
case DETECT_PCRE:

65
src/detect-engine-mpm.c
Unescape Escape View File

@ -34,6 +34,7 @@
#include "detect-engine-siggroup.h"
#include "detect-engine-mpm.h"
#include "detect-engine-iponly.h"
#include "detect-parse.h"
#include "util-mpm.h"
#include "conf.h"
@ -729,16 +730,62 @@ static int PatternMatchPreprarePopulateMpm(DetectEngineCtx *de_ctx, SigGroupHead
}
}
/* add the content to the "packet" mpm */
if (co->flags & DETECT_CONTENT_NOCASE) {
mpm_table[sgh->mpm_ctx->mpm_type].AddPatternNocase(sgh->mpm_ctx,
co->content, co->content_len, offset, depth, co->id,
s->num, flags);
if (co->flags & DETECT_CONTENT_FAST_PATTERN_CHOP) {
/* add the content to the "packet" mpm */
if (co->flags & DETECT_CONTENT_NOCASE) {
mpm_table[sgh->mpm_ctx->mpm_type].
AddPatternNocase(sgh->mpm_ctx,
co->content + co->fp_chop_offset,
co->fp_chop_len,
0, 0, co->id, s->num, flags);
} else {
mpm_table[sgh->mpm_ctx->mpm_type].
AddPattern(sgh->mpm_ctx,
co->content + co->fp_chop_offset,
co->fp_chop_len,
0, 0, co->id, s->num, flags);
}
} else {
mpm_table[sgh->mpm_ctx->mpm_type].AddPattern(sgh->mpm_ctx,
co->content, co->content_len, offset, depth, co->id,
s->num, flags);
}
if (co->flags & DETECT_CONTENT_FAST_PATTERN_ONLY) {
co->avoid_double_check = 1;
} else {
if (!(co->flags & DETECT_CONTENT_RELATIVE_NEXT)) {
SigMatch *tmp_sm = s->pmatch;
for ( ; tmp_sm != NULL; tmp_sm = tmp_sm->next) {
if (tmp_sm->type != DETECT_CONTENT)
continue;
DetectContentData *tmp_co = (DetectContentData *)tmpsm->ctx;
if (tmp_co == NULL)
continue;
if (co->id == tmp_co->id)
break;
}
SigMatch *prev_sm = SigMatchGetLastSMFromLists(s, 2,
DETECT_CONTENT, tmp_sm->prev);
if (prev_sm != NULL) {
DetectContentData *prev_co = (DetectContentData *)prev_sm->ctx;
if (!(prev_co->flags & DETECT_CONTENT_RELATIVE_NEXT))
co->avoid_double_check = 1;
}
}
} /* else - if (co->flags & DETECT_CONTENT_FAST_PATTERN_CHOP) */
/* add the content to the "packet" mpm */
if (co->flags & DETECT_CONTENT_NOCASE) {
mpm_table[sgh->mpm_ctx->mpm_type].
AddPatternNocase(sgh->mpm_ctx,
co->content, co->content_len,
offset, depth, co->id, s->num, flags);
} else {
mpm_table[sgh->mpm_ctx->mpm_type].
AddPattern(sgh->mpm_ctx,
co->content, co->content_len,
offset, depth, co->id, s->num, flags);
}
} /* else - if (co->flags & DETECT_CONTENT_FAST_PATTERN_CHOP) */
/* tell matcher we are inspecting packet */
s->flags |= SIG_FLAG_MPM_PACKET;

9
src/detect-engine-payload.c
Unescape Escape View File

@ -81,10 +81,15 @@ static int DoInspectPacketPayload(DetectEngineCtx *de_ctx,
switch(sm->type) {
case DETECT_CONTENT:
{
DetectContentData *cd = NULL;
cd = (DetectContentData *)sm->ctx;
DetectContentData *cd = (DetectContentData *)sm->ctx;
SCLogDebug("inspecting content %"PRIu32" payload_len %"PRIu32, cd->id, payload_len);
/* we might have already have this content matched by the mpm.
* (if there is any other reason why we'd want to avoid checking
* it here, please fill it in) */
if (cd->avoid_double_check)
goto match;
/* rule parsers should take care of this */
BUG_ON(cd->depth != 0 && cd->depth <= cd->offset);

1320
src/detect-fast-pattern.c
View File

File diff suppressed because it is too large Load Diff

23
src/detect-offset.c
Unescape Escape View File

@ -122,6 +122,21 @@ int DetectOffsetSetup (DetectEngineCtx *de_ctx, Signature *s, char *offsetstr)
if (dubbed) SCFree(str);
return -1;
}
if (cd->flags & DETECT_CONTENT_NEGATED) {
if (cd->flags & DETECT_CONTENT_FAST_PATTERN) {
SCLogError(SC_ERR_INVALID_SIGNATURE, "You can't have a relative "
"negated keyword set along with a fast_pattern");
goto error;
}
} else {
if (cd->flags & DETECT_CONTENT_FAST_PATTERN_ONLY) {
SCLogError(SC_ERR_INVALID_SIGNATURE, "You can't have a relative "
"keyword set along with a fast_pattern:only;");
goto error;
}
}
cd->offset = (uint32_t)atoi(str);
if (cd->depth != 0) {
if (cd->depth < cd->content_len) {
@ -132,6 +147,9 @@ int DetectOffsetSetup (DetectEngineCtx *de_ctx, Signature *s, char *offsetstr)
/* Updating the depth as is relative to the offset */
cd->depth += cd->offset;
}
cd->flags |= DETECT_CONTENT_OFFSET;
break;
default:
@ -146,5 +164,10 @@ int DetectOffsetSetup (DetectEngineCtx *de_ctx, Signature *s, char *offsetstr)
if (dubbed)
SCFree(str);
return 0;
error:
if (dubbed)
SCFree(str);
return -1;
}

2
src/detect-uricontent.c
Unescape Escape View File

@ -1196,7 +1196,7 @@ static int DetectUriSigTest04(void) {
s->pmatch == NULL ||
((DetectContentData*) s->pmatch->ctx)->depth != 15 ||
((DetectContentData*) s->pmatch->ctx)->offset != 5 ||
((DetectContentData*) s->umatch_tail->ctx)->distance != 30 ||
((DetectUricontentData*) s->umatch_tail->ctx)->distance != 30 ||
s->match != NULL)
{
printf("sig 10 failed to parse: ");

22
src/detect-within.c
Unescape Escape View File

@ -261,6 +261,20 @@ static int DetectWithinSetup (DetectEngineCtx *de_ctx, Signature *s, char *withi
goto error;
}
if (cd->flags & DETECT_CONTENT_NEGATED) {
if (cd->flags & DETECT_CONTENT_FAST_PATTERN) {
SCLogError(SC_ERR_INVALID_SIGNATURE, "You can't have a relative "
"negated keyword set along with a fast_pattern");
goto error;
}
} else {
if (cd->flags & DETECT_CONTENT_FAST_PATTERN_ONLY) {
SCLogError(SC_ERR_INVALID_SIGNATURE, "You can't have a relative "
"keyword set along with a fast_pattern:only;");
goto error;
}
}
cd->within = strtol(str, NULL, 10);
if (cd->within < (int32_t)cd->content_len) {
SCLogError(SC_ERR_WITHIN_INVALID, "within argument \"%"PRIi32"\" is "
@ -304,6 +318,14 @@ static int DetectWithinSetup (DetectEngineCtx *de_ctx, Signature *s, char *withi
}
cd->flags |= DETECT_CONTENT_RELATIVE_NEXT;
if (cd->flags & DETECT_CONTENT_FAST_PATTERN_ONLY) {
SCLogError(SC_ERR_INVALID_SIGNATURE, "Previous keyword "
"has a fast_pattern:only; set. You can't "
"have relative keywords around a fast_pattern "
"only content");
goto error;
}
break;
case DETECT_PCRE:

134
src/detect.c
Unescape Escape View File

@ -151,6 +151,10 @@
extern uint8_t engine_mode;
extern int engine_analysis;
static int fp_engine_analysis_set = 0;
static FILE *fp_engine_analysis_FD = NULL;
SigMatch *SigMatchAlloc(void);
void DetectExitPrintStats(ThreadVars *tv, void *data);
@ -243,6 +247,90 @@ char *DetectLoadCompleteSigPath(char *sig_file)
return path;
}
/**
* \brief Prints analysis of fast pattern for a signature.
*
* The code here mimics the logic to select fast_pattern from staging.
* If any changes are made to the staging logic, this should follow suit.
*
* \param s Pointer to the signature.
*/
void EngineAnalysisFastPattern(Signature *s)
{
int fast_pattern_set = 0;
int fast_pattern_only_set = 0;
int fast_pattern_chop_set = 0;
int content_maxlen = 0;
DetectContentData *cd = NULL;
DetectContentData *fp_cd = NULL;
SigMatch *sm = NULL;
for (sm = s->pmatch; sm != NULL; sm = sm->next) {
if (sm->type != DETECT_CONTENT)
continue;
cd = (DetectContentData *)sm->ctx;
if (cd->flags & DETECT_CONTENT_FAST_PATTERN) {
fast_pattern_set = 1;
if (cd->flags & DETECT_CONTENT_FAST_PATTERN_ONLY) {
fast_pattern_only_set = 1;
} else if (cd->flags & DETECT_CONTENT_FAST_PATTERN_CHOP) {
fast_pattern_chop_set = 1;
}
fp_cd = cd;
break;
} else if (cd->content_len <= content_maxlen) {
continue;
}
fp_cd = cd;
}
if (fp_cd == NULL) {
fprintf(fp_engine_analysis_FD, "== Sid: %u ==\n", s->id);
fprintf(fp_engine_analysis_FD, " No content present\n");
return;
}
fprintf(fp_engine_analysis_FD, "== Sid: %u ==\n", s->id);
fprintf(fp_engine_analysis_FD, " Fast pattern matcher: content\n");
fprintf(fp_engine_analysis_FD, " Fast pattern set: %s\n", fast_pattern_set ? "yes" : "no");
fprintf(fp_engine_analysis_FD, " Fast pattern only set: %s\n",
fast_pattern_only_set ? "yes" : "no");
fprintf(fp_engine_analysis_FD, " Fast pattern chop set: %s\n",
fast_pattern_chop_set ? "yes" : "no");
if (fast_pattern_chop_set) {
fprintf(fp_engine_analysis_FD, " Fast pattern offset, length: %u, %u\n",
fp_cd->fp_chop_offset, fp_cd->fp_chop_len);
}
fprintf(fp_engine_analysis_FD, " Content negated: %s\n",
(fp_cd->flags & DETECT_CONTENT_NEGATED) ? "yes" : "no");
uint8_t *pat = malloc(fp_cd->content_len + 1);
if (pat == NULL) {
SCLogError(SC_ERR_MEM_ALLOC, "Error allocating memory");
exit(EXIT_FAILURE);
}
memcpy(pat, cd->content, cd->content_len);
pat[cd->content_len] = '\0';
fprintf(fp_engine_analysis_FD, " Original content: %s\n", pat);
if (fast_pattern_chop_set) {
uint8_t *pat = malloc(fp_cd->fp_chop_len + 1);
if (pat == NULL) {
SCLogError(SC_ERR_MEM_ALLOC, "Error allocating memory");
exit(EXIT_FAILURE);
}
memcpy(pat, cd->content + fp_cd->fp_chop_offset, fp_cd->fp_chop_len);
pat[fp_cd->fp_chop_len] = '\0';
fprintf(fp_engine_analysis_FD, " Final content: %s\n", pat);
} else {
fprintf(fp_engine_analysis_FD, " Final content: %s\n", pat);
}
return;
}
/**
* \brief Load a file with signatures
* \param de_ctx Pointer to the detection engine context
@ -302,6 +390,9 @@ int DetectLoadSigFile(DetectEngineCtx *de_ctx, char *sig_file, int *sigs_tot) {
sig = DetectEngineAppendSig(de_ctx, line);
(*sigs_tot)++;
if (sig != NULL) {
if (fp_engine_analysis_set) {
EngineAnalysisFastPattern(sig);
}
SCLogDebug("signature %"PRIu32" loaded", sig->id);
good++;
} else {
@ -338,6 +429,40 @@ int SigLoadSignatures (DetectEngineCtx *de_ctx, char *sig_file)
int sigtotal = 0;
char *sfile = NULL;
if (engine_analysis) {
if ((ConfGetBool("engine-analysis.rules-fast-pattern",
&fp_engine_analysis_set)) == 0) {
fp_engine_analysis_set = 0;
}
if (fp_engine_analysis_set) {
char log_path[256], *log_dir;
if (ConfGet("default-log-dir", &log_dir) != 1)
log_dir = DEFAULT_LOG_DIR;
snprintf(log_path, 256, "%s/%s", log_dir, "rules_fast_pattern.txt");
fp_engine_analysis_FD = fopen(log_path, "w");
if (fp_engine_analysis_FD == NULL) {
SCLogError(SC_ERR_FOPEN, "ERROR: failed to open %s: %s", log_path,
strerror(errno));
return -1;
}
struct timeval tval;
struct tm *tms;
gettimeofday(&tval, NULL);
struct tm local_tm;
tms = (struct tm *)localtime_r(&tval.tv_sec, &local_tm);
fprintf(fp_engine_analysis_FD, "----------------------------------------------"
"---------------------\n");
fprintf(fp_engine_analysis_FD, "Date: %" PRId32 "/%" PRId32 "/%04d -- "
"%02d:%02d:%02d\n",
tms->tm_mday, tms->tm_mon + 1, tms->tm_year + 1900, tms->tm_hour,
tms->tm_min, tms->tm_sec);
fprintf(fp_engine_analysis_FD, "----------------------------------------------"
"---------------------\n");
}
}
/* ok, let's load signature files from the general config */
rule_files = ConfGetNode("rule-files");
if (rule_files != NULL) {
@ -418,6 +543,15 @@ int SigLoadSignatures (DetectEngineCtx *de_ctx, char *sig_file)
ret = 0;
end:
if (engine_analysis) {
if (fp_engine_analysis_set) {
if (fp_engine_analysis_FD != NULL) {
fclose(fp_engine_analysis_FD);
fp_engine_analysis_FD = NULL;
}
}
}
DetectParseDupSigHashFree(de_ctx);
SCReturnInt(ret);
}

22
src/suricata.c
Unescape Escape View File

@ -170,6 +170,10 @@ uint8_t suricata_ctl_flags = 0;
/** Run mode selected */
int run_mode = MODE_UNKNOWN;
/** engine_analysis. disabled(0) by default, unless enabled by the user by
* running the engine with --engine-analysis */
int engine_analysis = 0;
/** Engine mode: inline (ENGINE_MODE_IPS) or just
* detection mode (ENGINE_MODE_IDS by default) */
uint8_t engine_mode = ENGINE_MODE_IDS;
@ -330,6 +334,12 @@ void usage(const char *progname)
printf("\t--list-unittests : list unit tests\n");
printf("\t--fatal-unittests : enable fatal failure on unittest error\n");
#endif /* UNITTESTS */
#ifdef __SC_CUDA_SUPPORT__
printf("\t--list-cuda-cards : list cuda supported cards\n");
#endif
printf("\t--engine-analysis : print reports on analysis of different sections in the engine and exit.\n"
"\t Please have a look at the conf parameter engine-analysis on what reports\n"
"\t can be printed\n");
printf("\t--pidfile <file> : write pid to this file (only for daemon mode)\n");
printf("\t--init-errors-fatal : enable fatal failure on signature init error\n");
printf("\t--dump-config : show the running configuration\n");
@ -436,6 +446,7 @@ int main(int argc, char **argv)
{"unittest-filter", required_argument, 0, 'U'},
{"list-unittests", 0, &list_unittests, 1},
{"list-cuda-cards", 0, &list_cuda_cards, 1},
{"engine-analysis", 0, &engine_analysis, 1},
#ifdef OS_WIN32
{"service-install", 0, 0, 0},
{"service-remove", 0, 0, 0},
@ -513,6 +524,8 @@ int main(int argc, char **argv)
"--enable-cuda to configure when building.\n");
exit(EXIT_FAILURE);
#endif /* UNITTESTS */
} else if(strcmp((long_opts[option_index]).name, "engine-analysis") == 0) {
printf("==Carrying out engine analyis==\n");
}
#ifdef OS_WIN32
else if(strcmp((long_opts[option_index]).name, "service-install") == 0) {
@ -778,8 +791,10 @@ int main(int argc, char **argv)
SCHInfoLoadFromConfig();
if (run_mode == MODE_UNKNOWN) {
usage(argv[0]);
exit(EXIT_FAILURE);
if (!engine_analysis) {
usage(argv[0]);
exit(EXIT_FAILURE);
}
}
/* create table for O(1) lowercase conversion lookup */
@ -1046,6 +1061,9 @@ int main(int argc, char **argv)
exit(EXIT_FAILURE);
}
if (engine_analysis) {
exit(EXIT_SUCCESS);
}
#ifdef PROFILING
SCProfilingInitRuleCounters(de_ctx);

9
suricata.yaml
Unescape Escape View File

@ -78,6 +78,15 @@ defrag:
prealloc: yes
timeout: 60
# When run with the option --engine-analysis, the engine will read each of
# the parameters below, and print reports for each of the enabled sections
# and exit. The reports are printed to a file in the default log dir
# given by the parameter "default-log-dir", with engine reporting
# subsection below printing reports in its own report file.
engine-analysis:
# enables printing reports for fast-pattern for every rule.
rules-fast-pattern: yes
# You can specify a threshold config file by setting "threshold-file"
# to the path of the threshold config file:
# threshold-file: /etc/suricata/threshold.config

Write Preview
Loading…
Cancel
Save