aiden
/
suricata
mirror of https://github.com/OISF/suricata
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

pcre: migrate detect-pcre to pcre2

Browse Source 
pcre2 substrings need special free...
pull/6414/head
Philippe Antoine 5 years ago
parent 3633c48e6e
commit f0f3295ba0
4 changed files with 109 additions and 149 deletions
Show Stats Download Patch File Download Diff File

3
src/detect-parse.h
Unescape Escape View File

@ -42,9 +42,6 @@ enum {
typedef struct DetectParseRegex_ { typedef struct DetectParseRegex_ {
pcre *regex; pcre *regex;
pcre_extra *study; pcre_extra *study;
#ifdef PCRE_HAVE_JIT_EXEC
pcre_jit_stack *jit_stack;
#endif
struct DetectParseRegex_ *next; struct DetectParseRegex_ *next;
} DetectParseRegex; } DetectParseRegex;

238
src/detect-pcre.c
Unescape Escape View File

@ -74,36 +74,20 @@ static int pcre_match_limit_recursion = 0;
static DetectParseRegex parse_regex; static DetectParseRegex parse_regex;
static DetectParseRegex parse_capture_regex; static DetectParseRegex parse_capture_regex;
#ifdef PCRE_HAVE_JIT #ifdef PCRE2_HAVE_JIT
static int pcre_use_jit = 1; static int pcre2_use_jit = 1;
#endif #endif
#ifdef PCRE_HAVE_JIT_EXEC // TODOpcre2 pcre2_jit_stack_create ?
#define PCRE_JIT_MIN_STACK 32*1024
#define PCRE_JIT_MAX_STACK 512*1024
#endif
/* \brief Helper function for using pcre_exec with/without JIT /* \brief Helper function for using pcre_exec with/without JIT
*/ */
static inline int DetectPcreExec(DetectEngineThreadCtx *det_ctx, DetectPcreData *pd, DetectParseRegex *regex, static inline int DetectPcreExec(DetectEngineThreadCtx *det_ctx, DetectPcreData *pd,
const char *str, const size_t strlen, int start_offset, int options, int *ovector, int ovector_size) const char *str, const size_t strlen, int start_offset, int options)
{ {
#ifdef PCRE_HAVE_JIT_EXEC
if (pd->thread_ctx_jit_stack_id != -1) {
pcre_jit_stack *jit_stack = (pcre_jit_stack *)
DetectThreadCtxGetKeywordThreadCtx(det_ctx, pd->thread_ctx_jit_stack_id);
if (jit_stack) {
SCLogDebug("Using jit_stack %p", jit_stack);
return pcre_jit_exec(regex->regex, regex->study, str, strlen,
start_offset, options, ovector, ovector_size,
jit_stack);
}
}
#endif
/* Fallback if registration during setup failed */ /* Fallback if registration during setup failed */
return pcre_exec(regex->regex, regex->study, str, strlen, return pcre2_match(pd->parse_regex.regex, (PCRE2_SPTR8)str, strlen, start_offset, options,
start_offset, options, ovector, ovector_size); pd->parse_regex.match, NULL);
} }
static int DetectPcreSetup (DetectEngineCtx *, Signature *, const char *); static int DetectPcreSetup (DetectEngineCtx *, Signature *, const char *);
@ -163,10 +147,10 @@ void DetectPcreRegister (void)
FatalError(SC_ERR_PCRE_COMPILE, "pcre compile and study failed"); FatalError(SC_ERR_PCRE_COMPILE, "pcre compile and study failed");
} }
#ifdef PCRE_HAVE_JIT #ifdef PCRE2_HAVE_JIT
if (PageSupportsRWX() == 0) { if (PageSupportsRWX() == 0) {
SCLogConfig("PCRE won't use JIT as OS doesn't allow RWX pages"); SCLogConfig("PCRE2 won't use JIT as OS doesn't allow RWX pages");
pcre_use_jit = 0; pcre2_use_jit = 0;
} }
#endif #endif
@ -193,10 +177,9 @@ int DetectPcrePayloadMatch(DetectEngineThreadCtx *det_ctx, const Signature *s,
{ {
SCEnter(); SCEnter();
int ret = 0; int ret = 0;
int ov[MAX_SUBSTRINGS];
const uint8_t *ptr = NULL; const uint8_t *ptr = NULL;
uint16_t len = 0; uint16_t len = 0;
uint16_t capture_len = 0; PCRE2_SIZE capture_len = 0;
DetectPcreData *pe = (DetectPcreData *)smd->ctx; DetectPcreData *pe = (DetectPcreData *)smd->ctx;
@ -214,10 +197,10 @@ int DetectPcrePayloadMatch(DetectEngineThreadCtx *det_ctx, const Signature *s,
} }
/* run the actual pcre detection */ /* run the actual pcre detection */
ret = DetectPcreExec(det_ctx, pe, &pe->parse_regex, (char *) ptr, len, start_offset, 0, ov, MAX_SUBSTRINGS); ret = DetectPcreExec(det_ctx, pe, (char *)ptr, len, start_offset, 0);
SCLogDebug("ret %d (negating %s)", ret, (pe->flags & DETECT_PCRE_NEGATE) ? "set" : "not set"); SCLogDebug("ret %d (negating %s)", ret, (pe->flags & DETECT_PCRE_NEGATE) ? "set" : "not set");
if (ret == PCRE_ERROR_NOMATCH) { if (ret == PCRE2_ERROR_NOMATCH) {
if (pe->flags & DETECT_PCRE_NEGATE) { if (pe->flags & DETECT_PCRE_NEGATE) {
/* regex didn't match with negate option means we /* regex didn't match with negate option means we
* consider it a match */ * consider it a match */
@ -241,28 +224,47 @@ int DetectPcrePayloadMatch(DetectEngineThreadCtx *det_ctx, const Signature *s,
uint8_t x; uint8_t x;
for (x = 0; x < pe->idx; x++) { for (x = 0; x < pe->idx; x++) {
SCLogDebug("capturing %u", x); SCLogDebug("capturing %u", x);
const char *str_ptr = NULL; const char *pcre2_str_ptr = NULL;
ret = pcre_get_substring((char *)ptr, ov, MAX_SUBSTRINGS, x+1, &str_ptr); ret = pcre2_substring_get_bynumber(pe->parse_regex.match, x + 1,
if (unlikely(ret == 0)) { (PCRE2_UCHAR8 **)&pcre2_str_ptr, &capture_len);
pcre_free_substring(str_ptr); if (unlikely(ret != 0)) {
pcre2_substring_free((PCRE2_UCHAR8 *)pcre2_str_ptr);
continue;
}
/* store max 64k. Errors are ignored */
capture_len = (capture_len < 0xffff) ? (uint16_t)capture_len : 0xffff;
uint8_t *str_ptr = SCMalloc(capture_len);
if (unlikely(str_ptr == NULL)) {
pcre2_substring_free((PCRE2_UCHAR8 *)pcre2_str_ptr);
continue; continue;
} }
memcpy(str_ptr, pcre2_str_ptr, capture_len);
pcre2_substring_free((PCRE2_UCHAR8 *)pcre2_str_ptr);
SCLogDebug("data %p/%u, type %u id %u p %p", SCLogDebug("data %p/%u, type %u id %u p %p",
str_ptr, ret, pe->captypes[x], pe->capids[x], p); str_ptr, ret, pe->captypes[x], pe->capids[x], p);
if (pe->captypes[x] == VAR_TYPE_PKT_VAR_KV) { if (pe->captypes[x] == VAR_TYPE_PKT_VAR_KV) {
/* get the value, as first capture is the key */ /* get the value, as first capture is the key */
const char *str_ptr2 = NULL; const char *pcre2_str_ptr2 = NULL;
int ret2 = pcre_get_substring((char *)ptr, ov, MAX_SUBSTRINGS, x+2, &str_ptr2); /* key length is limited to 256 chars */
if (unlikely(ret2 == 0)) { uint16_t key_len = (capture_len < 0xff) ? (uint16_t)capture_len : 0xff;
pcre_free_substring(str_ptr); int ret2 = pcre2_substring_get_bynumber(pe->parse_regex.match, x + 2,
pcre_free_substring(str_ptr2); (PCRE2_UCHAR8 **)&pcre2_str_ptr2, &capture_len);
if (unlikely(ret2 != 0)) {
SCFree(str_ptr);
pcre2_substring_free((PCRE2_UCHAR8 *)pcre2_str_ptr2);
break; break;
} }
/* key length is limited to 256 chars */ capture_len = (capture_len < 0xffff) ? (uint16_t)capture_len : 0xffff;
uint16_t key_len = (ret < 0xff) ? (uint16_t)ret : 0xff; uint8_t *str_ptr2 = SCMalloc(capture_len);
capture_len = (ret2 < 0xffff) ? (uint16_t)ret2 : 0xffff; if (unlikely(str_ptr2 == NULL)) {
pcre2_substring_free((PCRE2_UCHAR8 *)pcre2_str_ptr);
continue;
}
memcpy(str_ptr2, pcre2_str_ptr2, capture_len);
pcre2_substring_free((PCRE2_UCHAR8 *)pcre2_str_ptr2);
(void)DetectVarStoreMatchKeyValue(det_ctx, (void)DetectVarStoreMatchKeyValue(det_ctx,
(uint8_t *)str_ptr, key_len, (uint8_t *)str_ptr, key_len,
@ -270,15 +272,11 @@ int DetectPcrePayloadMatch(DetectEngineThreadCtx *det_ctx, const Signature *s,
DETECT_VAR_TYPE_PKT_POSTMATCH); DETECT_VAR_TYPE_PKT_POSTMATCH);
} else if (pe->captypes[x] == VAR_TYPE_PKT_VAR) { } else if (pe->captypes[x] == VAR_TYPE_PKT_VAR) {
/* store max 64k. Errors are ignored */
capture_len = (ret < 0xffff) ? (uint16_t)ret : 0xffff;
(void)DetectVarStoreMatch(det_ctx, pe->capids[x], (void)DetectVarStoreMatch(det_ctx, pe->capids[x],
(uint8_t *)str_ptr, capture_len, (uint8_t *)str_ptr, capture_len,
DETECT_VAR_TYPE_PKT_POSTMATCH); DETECT_VAR_TYPE_PKT_POSTMATCH);
} else if (pe->captypes[x] == VAR_TYPE_FLOW_VAR && f != NULL) { } else if (pe->captypes[x] == VAR_TYPE_FLOW_VAR && f != NULL) {
/* store max 64k. Errors are ignored */
capture_len = (ret < 0xffff) ? (uint16_t)ret : 0xffff;
(void)DetectVarStoreMatch(det_ctx, pe->capids[x], (void)DetectVarStoreMatch(det_ctx, pe->capids[x],
(uint8_t *)str_ptr, capture_len, (uint8_t *)str_ptr, capture_len,
DETECT_VAR_TYPE_FLOW_POSTMATCH); DETECT_VAR_TYPE_FLOW_POSTMATCH);
@ -286,6 +284,7 @@ int DetectPcrePayloadMatch(DetectEngineThreadCtx *det_ctx, const Signature *s,
} }
} }
PCRE2_SIZE *ov = pcre2_get_ovector_pointer(pe->parse_regex.match);
/* update offset for pcre RELATIVE */ /* update offset for pcre RELATIVE */
det_ctx->buffer_offset = (ptr + ov[1]) - payload; det_ctx->buffer_offset = (ptr + ov[1]) - payload;
det_ctx->pcre_match_start_offset = (ptr + ov[0] + 1) - payload; det_ctx->pcre_match_start_offset = (ptr + ov[0] + 1) - payload;
@ -346,9 +345,8 @@ static DetectPcreData *DetectPcreParse (DetectEngineCtx *de_ctx,
const char *regexstr, int *sm_list, char *capture_names, const char *regexstr, int *sm_list, char *capture_names,
size_t capture_names_size, bool negate, AppProto *alproto) size_t capture_names_size, bool negate, AppProto *alproto)
{ {
int ec; int en;
const char *eb; PCRE2_SIZE eo2;
int eo;
int opts = 0; int opts = 0;
DetectPcreData *pd = NULL; DetectPcreData *pd = NULL;
char *op = NULL; char *op = NULL;
@ -436,27 +434,27 @@ static DetectPcreData *DetectPcreParse (DetectEngineCtx *de_ctx,
switch (*op) { switch (*op) {
case 'A': case 'A':
opts |= PCRE_ANCHORED; opts |= PCRE2_ANCHORED;
break; break;
case 'E': case 'E':
opts |= PCRE_DOLLAR_ENDONLY; opts |= PCRE2_DOLLAR_ENDONLY;
break; break;
case 'G': case 'G':
opts |= PCRE_UNGREEDY; opts |= PCRE2_UNGREEDY;
break; break;
case 'i': case 'i':
opts |= PCRE_CASELESS; opts |= PCRE2_CASELESS;
pd->flags |= DETECT_PCRE_CASELESS; pd->flags |= DETECT_PCRE_CASELESS;
break; break;
case 'm': case 'm':
opts |= PCRE_MULTILINE; opts |= PCRE2_MULTILINE;
break; break;
case 's': case 's':
opts |= PCRE_DOTALL; opts |= PCRE2_DOTALL;
break; break;
case 'x': case 'x':
opts |= PCRE_EXTENDED; opts |= PCRE2_EXTENDED;
break; break;
case 'O': case 'O':
@ -627,70 +625,57 @@ static DetectPcreData *DetectPcreParse (DetectEngineCtx *de_ctx,
* PCRE will let us know. * PCRE will let us know.
*/ */
if (capture_names == NULL || strlen(capture_names) == 0) if (capture_names == NULL || strlen(capture_names) == 0)
opts |= PCRE_NO_AUTO_CAPTURE; opts |= PCRE2_NO_AUTO_CAPTURE;
pd->parse_regex.regex = pcre_compile2(re, opts, &ec, &eb, &eo, NULL); pd->parse_regex.regex =
if (pd->parse_regex.regex == NULL && ec == 15) { // reference to non-existent subpattern pcre2_compile((PCRE2_SPTR8)re, PCRE2_ZERO_TERMINATED, opts, &en, &eo2, NULL);
opts &= ~PCRE_NO_AUTO_CAPTURE; if (pd->parse_regex.regex == NULL && en == 115) { // reference to non-existent subpattern
pd->parse_regex.regex = pcre_compile(re, opts, &eb, &eo, NULL); opts &= ~PCRE2_NO_AUTO_CAPTURE;
pd->parse_regex.regex =
pcre2_compile((PCRE2_SPTR8)re, PCRE2_ZERO_TERMINATED, opts, &en, &eo2, NULL);
} }
if (pd->parse_regex.regex == NULL) { if (pd->parse_regex.regex == NULL) {
SCLogError(SC_ERR_PCRE_COMPILE, "pcre compile of \"%s\" failed " PCRE2_UCHAR errbuffer[256];
"at offset %" PRId32 ": %s", regexstr, eo, eb); pcre2_get_error_message(en, errbuffer, sizeof(errbuffer));
SCLogError(SC_ERR_PCRE_COMPILE,
"pcre2 compile of \"%s\" failed at "
"offset %d: %s",
regexstr, (int)eo2, errbuffer);
goto error; goto error;
} }
int options = 0; #ifdef PCRE2_HAVE_JIT
#ifdef PCRE_HAVE_JIT if (pcre2_use_jit) {
if (pcre_use_jit) ret = pcre2_jit_compile(pd->parse_regex.regex, PCRE2_JIT_COMPLETE);
options |= PCRE_STUDY_JIT_COMPILE; if (ret != 0) {
#endif /* warning, so we won't print the sig after this. Adding
pd->parse_regex.study = pcre_study(pd->parse_regex.regex, options, &eb); * file and line to the message so the admin can figure
if(eb != NULL) { * out what sig this is about */
SCLogError(SC_ERR_PCRE_STUDY, "pcre study failed : %s", eb); SCLogDebug("PCRE2 JIT compiler does not support: %s. "
goto error; "Falling back to regular PCRE2 handling (%s:%d)",
regexstr, de_ctx->rule_file, de_ctx->rule_line);
}
} }
#endif /*PCRE2_HAVE_JIT*/
#ifdef PCRE_HAVE_JIT pd->parse_regex.context = pcre2_match_context_create(NULL);
int jit = 0; if (pd->parse_regex.context == NULL) {
ret = pcre_fullinfo(pd->parse_regex.regex, pd->parse_regex.study, PCRE_INFO_JIT, &jit); SCLogError(SC_ERR_PCRE_COMPILE, "pcre2 could not create match context");
if (ret != 0 || jit != 1) { goto error;
/* warning, so we won't print the sig after this. Adding
* file and line to the message so the admin can figure
* out what sig this is about */
SCLogDebug("PCRE JIT compiler does not support: %s. "
"Falling back to regular PCRE handling (%s:%d)",
regexstr, de_ctx->rule_file, de_ctx->rule_line);
} }
pd->parse_regex.match = pcre2_match_data_create_from_pattern(pd->parse_regex.regex, NULL);
#endif /*PCRE_HAVE_JIT*/ if (pd->flags & DETECT_PCRE_MATCH_LIMIT) {
if (pcre_match_limit >= -1) {
if (pd->parse_regex.study == NULL) pcre2_set_match_limit(pd->parse_regex.context, pcre_match_limit);
pd->parse_regex.study = (pcre_extra *) SCCalloc(1,sizeof(pcre_extra)); }
if (pcre_match_limit_recursion >= -1) {
if (pd->parse_regex.study) { // pcre2_set_depth_limit unsupported on ubuntu 16.04
if(pd->flags & DETECT_PCRE_MATCH_LIMIT) { pcre2_set_recursion_limit(pd->parse_regex.context, pcre_match_limit_recursion);
if(pcre_match_limit >= -1) {
pd->parse_regex.study->match_limit = pcre_match_limit;
pd->parse_regex.study->flags |= PCRE_EXTRA_MATCH_LIMIT;
}
#ifndef NO_PCRE_MATCH_RLIMIT
if(pcre_match_limit_recursion >= -1) {
pd->parse_regex.study->match_limit_recursion = pcre_match_limit_recursion;
pd->parse_regex.study->flags |= PCRE_EXTRA_MATCH_LIMIT_RECURSION;
}
#endif /* NO_PCRE_MATCH_RLIMIT */
} else {
pd->parse_regex.study->match_limit = SC_MATCH_LIMIT_DEFAULT;
pd->parse_regex.study->flags |= PCRE_EXTRA_MATCH_LIMIT;
#ifndef NO_PCRE_MATCH_RLIMIT
pd->parse_regex.study->match_limit_recursion = SC_MATCH_LIMIT_RECURSION_DEFAULT;
pd->parse_regex.study->flags |= PCRE_EXTRA_MATCH_LIMIT_RECURSION;
#endif /* NO_PCRE_MATCH_RLIMIT */
} }
} else { } else {
goto error; pcre2_set_match_limit(pd->parse_regex.context, SC_MATCH_LIMIT_DEFAULT);
pcre2_set_recursion_limit(pd->parse_regex.context, PCRE_EXTRA_MATCH_LIMIT_RECURSION);
} }
return pd; return pd;
@ -717,7 +702,7 @@ static int DetectPcreParseCapture(const char *regexstr, DetectEngineCtx *de_ctx,
SCLogDebug("regexstr %s, pd %p", regexstr, pd); SCLogDebug("regexstr %s, pd %p", regexstr, pd);
ret = pcre_fullinfo(pd->parse_regex.regex, pd->parse_regex.study, PCRE_INFO_CAPTURECOUNT, &capture_cnt); ret = pcre2_pattern_info(pd->parse_regex.regex, PCRE2_INFO_CAPTURECOUNT, &capture_cnt);
SCLogDebug("ret %d capture_cnt %d", ret, capture_cnt); SCLogDebug("ret %d capture_cnt %d", ret, capture_cnt);
if (ret == 0 && capture_cnt && strlen(capture_names) > 0) if (ret == 0 && capture_cnt && strlen(capture_names) > 0)
{ {
@ -835,27 +820,6 @@ error:
return -1; return -1;
} }
#ifdef PCRE_HAVE_JIT_EXEC
static void *DetectPcreThreadInit(void *data /*@unused@*/)
{
pcre_jit_stack *jit_stack = pcre_jit_stack_alloc(PCRE_JIT_MIN_STACK, PCRE_JIT_MAX_STACK);
if (jit_stack == NULL) {
SCLogWarning(SC_WARN_PCRE_JITSTACK, "Unable to allocate PCRE JIT stack; will continue without JIT stack");
}
SCLogDebug("Using jit_stack %p", jit_stack);
return (void *)jit_stack;
}
static void DetectPcreThreadFree(void *ctx)
{
SCLogDebug("freeing jit_stack %p", ctx);
if (ctx != NULL)
pcre_jit_stack_free((pcre_jit_stack *)ctx);
}
#endif
static int DetectPcreSetup (DetectEngineCtx *de_ctx, Signature *s, const char *regexstr) static int DetectPcreSetup (DetectEngineCtx *de_ctx, Signature *s, const char *regexstr)
{ {
SCEnter(); SCEnter();
@ -873,14 +837,6 @@ static int DetectPcreSetup (DetectEngineCtx *de_ctx, Signature *s, const char *r
if (DetectPcreParseCapture(regexstr, de_ctx, pd, capture_names) < 0) if (DetectPcreParseCapture(regexstr, de_ctx, pd, capture_names) < 0)
goto error; goto error;
#ifdef PCRE_HAVE_JIT_EXEC
/* Deliberately silent on failures. Not having a context id means
* JIT will be bypassed */
pd->thread_ctx_jit_stack_id = DetectRegisterThreadCtxFuncs(de_ctx, "pcre",
DetectPcreThreadInit, (void *)pd,
DetectPcreThreadFree, 1);
#endif
int sm_list = -1; int sm_list = -1;
if (s->init_data->list != DETECT_SM_LIST_NOTSET) { if (s->init_data->list != DETECT_SM_LIST_NOTSET) {
if (parsed_sm_list != DETECT_SM_LIST_NOTSET && parsed_sm_list != s->init_data->list) { if (parsed_sm_list != DETECT_SM_LIST_NOTSET && parsed_sm_list != s->init_data->list) {
@ -967,7 +923,9 @@ static void DetectPcreFree(DetectEngineCtx *de_ctx, void *ptr)
return; return;
DetectPcreData *pd = (DetectPcreData *)ptr; DetectPcreData *pd = (DetectPcreData *)ptr;
DetectParseFreeRegex(&pd->parse_regex); pcre2_code_free(pd->parse_regex.regex);
pcre2_match_context_free(pd->parse_regex.context);
pcre2_match_data_free(pd->parse_regex.match);
SCFree(pd); SCFree(pd);
return; return;

15
src/detect-pcre.h
Unescape Escape View File

@ -36,14 +36,19 @@
#define DETECT_PCRE_CAPTURE_MAX 8 #define DETECT_PCRE_CAPTURE_MAX 8
#include <pcre2.h>
typedef struct DetectParseRegex2 {
pcre2_code *regex;
pcre2_match_context *context;
pcre2_match_data *match;
// struct DetectParseRegex2 *next;
} DetectParseRegex2;
typedef struct DetectPcreData_ { typedef struct DetectPcreData_ {
/* pcre options */ /* pcre options */
DetectParseRegex parse_regex; DetectParseRegex2 parse_regex;
#ifdef PCRE_HAVE_JIT_EXEC
/* JIT stack thread context id */
int thread_ctx_jit_stack_id;
#endif
int opts; int opts;
uint16_t flags; uint16_t flags;
uint8_t idx; uint8_t idx;

2
src/suricata.c
Unescape Escape View File

@ -694,7 +694,7 @@ static void PrintBuildInfo(void)
#ifdef HAVE_HTP_URI_NORMALIZE_HOOK #ifdef HAVE_HTP_URI_NORMALIZE_HOOK
strlcat(features, "HAVE_HTP_URI_NORMALIZE_HOOK ", sizeof(features)); strlcat(features, "HAVE_HTP_URI_NORMALIZE_HOOK ", sizeof(features));
#endif #endif
#ifdef PCRE_HAVE_JIT #ifdef PCRE2_HAVE_JIT
strlcat(features, "PCRE_JIT ", sizeof(features)); strlcat(features, "PCRE_JIT ", sizeof(features));
#endif #endif
/* For compatibility, just say we have HAVE_NSS. */ /* For compatibility, just say we have HAVE_NSS. */

Write Preview
Loading…
Cancel
Save