From 83b2c8abdb4ce5c25254151da1c40b37e278a8ec Mon Sep 17 00:00:00 2001
From: Victor Julien <victor@inliniac.net>
Date: Sun, 13 Jun 2010 20:07:23 +0200
Subject: [PATCH] Improve stateful uri detection code.

---
 src/app-layer-parser.c       |  43 +++++-
 src/app-layer-parser.h       |   1 +
 src/detect-engine-mpm.c      |  19 +--
 src/detect-engine-siggroup.c |  35 ++---
 src/detect-engine-uri.c      |   7 +-
 src/detect-engine.c          |   8 +
 src/detect-flow.c            |   8 +-
 src/detect-uricontent.c      |  11 +-
 src/detect-uricontent.h      |   5 +-
 src/detect.c                 | 287 ++++++++++++++++++++---------------
 src/detect.h                 |  12 +-
 11 files changed, 255 insertions(+), 181 deletions(-)

diff --git a/src/app-layer-parser.c b/src/app-layer-parser.c
index 22b0dc9182..11b12e066e 100644
--- a/src/app-layer-parser.c
+++ b/src/app-layer-parser.c
@@ -957,7 +957,9 @@ int AppLayerTransactionGetBaseId(Flow *f) {
         goto error;
     }
 
-    AppLayerParserStateStore *parser_state_store = parser_state_store = (AppLayerParserStateStore *)f->aldata[app_layer_sid];
+    AppLayerParserStateStore *parser_state_store =
+        (AppLayerParserStateStore *)f->aldata[app_layer_sid];
+
     if (parser_state_store == NULL) {
         SCLogDebug("no state store");
         goto error;
@@ -969,6 +971,30 @@ error:
     SCReturnInt(-1);
 }
 
+/** \brief get the base transaction id */
+int AppLayerTransactionGetInspectId(Flow *f) {
+    SCEnter();
+
+    /* Get the parser state (if any) */
+    if (f->aldata == NULL) {
+        SCLogDebug("no aldata");
+        goto error;
+    }
+
+    AppLayerParserStateStore *parser_state_store =
+        (AppLayerParserStateStore *)f->aldata[app_layer_sid];
+
+    if (parser_state_store == NULL) {
+        SCLogDebug("no state store");
+        goto error;
+    }
+
+    SCReturnInt((int)parser_state_store->inspect_id);
+
+error:
+    SCReturnInt(-1);
+}
+
 /** \brief get the highest loggable transaction id */
 int AppLayerTransactionGetLoggableId(Flow *f) {
     SCEnter();
@@ -979,7 +1005,9 @@ int AppLayerTransactionGetLoggableId(Flow *f) {
         goto error;
     }
 
-    AppLayerParserStateStore *parser_state_store = parser_state_store = (AppLayerParserStateStore *)f->aldata[app_layer_sid];
+    AppLayerParserStateStore *parser_state_store =
+        (AppLayerParserStateStore *)f->aldata[app_layer_sid];
+
     if (parser_state_store == NULL) {
         SCLogDebug("no state store");
         goto error;
@@ -1010,7 +1038,9 @@ void AppLayerTransactionUpdateLoggedId(Flow *f) {
         goto error;
     }
 
-    AppLayerParserStateStore *parser_state_store = parser_state_store = (AppLayerParserStateStore *)f->aldata[app_layer_sid];
+    AppLayerParserStateStore *parser_state_store =
+        (AppLayerParserStateStore *)f->aldata[app_layer_sid];
+
     if (parser_state_store == NULL) {
         SCLogDebug("no state store");
         goto error;
@@ -1032,7 +1062,9 @@ int AppLayerTransactionGetLoggedId(Flow *f) {
         goto error;
     }
 
-    AppLayerParserStateStore *parser_state_store = parser_state_store = (AppLayerParserStateStore *)f->aldata[app_layer_sid];
+    AppLayerParserStateStore *parser_state_store =
+        (AppLayerParserStateStore *)f->aldata[app_layer_sid];
+
     if (parser_state_store == NULL) {
         SCLogDebug("no state store");
         goto error;
@@ -1060,8 +1092,7 @@ int AppLayerTransactionUpdateInspectId(Flow *f)
     AppLayerParserStateStore *parser_state_store = NULL;
 
     if (f->aldata != NULL) {
-        parser_state_store = (AppLayerParserStateStore *)
-                                                    f->aldata[app_layer_sid];
+        parser_state_store = (AppLayerParserStateStore *)f->aldata[app_layer_sid];
         if (parser_state_store != NULL) {
             /* update inspect_id and see if it there are other transactions
              * as well */
diff --git a/src/app-layer-parser.h b/src/app-layer-parser.h
index 82b4a91bd8..3edf5ed442 100644
--- a/src/app-layer-parser.h
+++ b/src/app-layer-parser.h
@@ -168,6 +168,7 @@ void AppLayerTransactionUpdateLoggedId(Flow *);
 int AppLayerTransactionGetLoggableId(Flow *f);
 int AppLayerTransactionGetLoggedId(Flow *f);
 int AppLayerTransactionGetBaseId(Flow *f);
+int AppLayerTransactionGetInspectId(Flow *f);
 
 void AppLayerParserRegisterTests(void);
 
diff --git a/src/detect-engine-mpm.c b/src/detect-engine-mpm.c
index b699ea7cda..6bfd51355d 100644
--- a/src/detect-engine-mpm.c
+++ b/src/detect-engine-mpm.c
@@ -443,9 +443,7 @@ static int PatternMatchPreprarePopulateMpm(DetectEngineCtx *de_ctx, SigGroupHead
 
     /* add all the contents to a counting hash */
     for (sig = 0; sig < sgh->sig_cnt; sig++) {
-        uint32_t num = sgh->match_array[sig];
-
-        Signature *s = de_ctx->sig_array[num];
+        Signature *s = sgh->match_array[sig];
         if (s == NULL)
             continue;
 
@@ -543,8 +541,7 @@ static int PatternMatchPreprarePopulateMpm(DetectEngineCtx *de_ctx, SigGroupHead
 
     /* now determine which one to add to the mpm phase */
     for (sig = 0; sig < sgh->sig_cnt; sig++) {
-        uint32_t num = sgh->match_array[sig];
-        Signature *s = de_ctx->sig_array[num];
+        Signature *s = sgh->match_array[sig];
         if (s == NULL)
             continue;
 
@@ -694,9 +691,7 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
     /** see if this head has content and/or uricontent
      *  \todo we can move this to the signature init phase */
     for (sig = 0; sig < sh->sig_cnt; sig++) {
-        uint32_t num = sh->match_array[sig];
-
-        s = de_ctx->sig_array[num];
+        s = sh->match_array[sig];
         if (s == NULL)
             continue;
 
@@ -771,9 +766,7 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
 
     /* for each signature in this group do */
     for (sig = 0; sig < sh->sig_cnt; sig++) {
-        uint32_t num = sh->match_array[sig];
-
-        s = de_ctx->sig_array[num];
+        s = sh->match_array[sig];
         if (s == NULL)
             continue;
 
@@ -922,9 +915,7 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
 
     /* add the patterns for uricontent signatures */
     for (sig = 0; sig < sh->sig_cnt; sig++) {
-        uint32_t num = sh->match_array[sig];
-
-        s = de_ctx->sig_array[num];
+        s = sh->match_array[sig];
         if (s == NULL)
             continue;
 
diff --git a/src/detect-engine-siggroup.c b/src/detect-engine-siggroup.c
index e048a0f9b8..4dc17ac723 100644
--- a/src/detect-engine-siggroup.c
+++ b/src/detect-engine-siggroup.c
@@ -158,12 +158,13 @@ void SigGroupHeadFree(SigGroupHead *sgh)
 
     if (sgh->match_array != NULL) {
         detect_siggroup_matcharray_free_cnt++;
-        detect_siggroup_matcharray_memory -= (sgh->sig_cnt * sizeof(SigIntId));
+        detect_siggroup_matcharray_memory -= (sgh->sig_cnt * sizeof(Signature *));
         SCFree(sgh->match_array);
         sgh->match_array = NULL;
-        sgh->sig_cnt = 0;
     }
 
+    sgh->sig_cnt = 0;
+
     if (sgh->init != NULL) {
         SigGroupHeadInitDataFree(sgh->init);
         sgh->init = NULL;
@@ -1267,7 +1268,6 @@ int SigGroupHeadLoadContent(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
     Signature *s = NULL;
     SigMatch *sm = NULL;
     uint32_t sig = 0;
-    SigIntId num = 0;
     DetectContentData *co = NULL;
 
     if (sgh == NULL)
@@ -1286,9 +1286,7 @@ int SigGroupHeadLoadContent(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
     memset(sgh->init->content_array,0, sgh->init->content_size);
 
     for (sig = 0; sig < sgh->sig_cnt; sig++) {
-        num = sgh->match_array[sig];
-
-        s = de_ctx->sig_array[num];
+        s = sgh->match_array[sig];
         if (s == NULL)
             continue;
 
@@ -1353,7 +1351,6 @@ int SigGroupHeadLoadUricontent(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
     Signature *s = NULL;
     SigMatch *sm = NULL;
     uint32_t sig = 0;
-    SigIntId num = 0;
     DetectUricontentData *co = NULL;
 
     if (sgh == NULL)
@@ -1372,9 +1369,8 @@ int SigGroupHeadLoadUricontent(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
     memset(sgh->init->uri_content_array, 0, sgh->init->uri_content_size);
 
     for (sig = 0; sig < sgh->sig_cnt; sig++) {
-        num = sgh->match_array[sig];
+        s = sgh->match_array[sig];
 
-        s = de_ctx->sig_array[num];
         if (s == NULL)
             continue;
 
@@ -1438,7 +1434,6 @@ int SigGroupHeadLoadStreamContent(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
     Signature *s = NULL;
     SigMatch *sm = NULL;
     uint32_t sig = 0;
-    SigIntId num = 0;
     DetectContentData *co = NULL;
 
     if (sgh == NULL)
@@ -1457,9 +1452,7 @@ int SigGroupHeadLoadStreamContent(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
     memset(sgh->init->stream_content_array,0, sgh->init->stream_content_size);
 
     for (sig = 0; sig < sgh->sig_cnt; sig++) {
-        num = sgh->match_array[sig];
-
-        s = de_ctx->sig_array[num];
+        s = sgh->match_array[sig];
         if (s == NULL)
             continue;
 
@@ -1529,24 +1522,24 @@ int SigGroupHeadBuildMatchArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh,
 
     BUG_ON(sgh->match_array != NULL);
 
-    sgh->match_array = SCMalloc(sgh->sig_cnt * sizeof(SigIntId));
+    sgh->match_array = SCMalloc(sgh->sig_cnt * sizeof(Signature *));
     if (sgh->match_array == NULL)
         return -1;
 
-    memset(sgh->match_array,0, sgh->sig_cnt * sizeof(SigIntId));
+    memset(sgh->match_array,0, sgh->sig_cnt * sizeof(Signature *));
 
     detect_siggroup_matcharray_init_cnt++;
-    detect_siggroup_matcharray_memory += (sgh->sig_cnt * sizeof(SigIntId));
+    detect_siggroup_matcharray_memory += (sgh->sig_cnt * sizeof(Signature *));
 
     for (sig = 0; sig < max_idx + 1; sig++) {
-        if ( !(sgh->sig_array[(sig / 8)] & (1 << (sig % 8))) )
+        if (!(sgh->sig_array[(sig / 8)] & (1 << (sig % 8))) )
             continue;
 
         s = de_ctx->sig_array[sig];
         if (s == NULL)
             continue;
 
-        sgh->match_array[idx] = s->num;
+        sgh->match_array[idx] = s;
         idx++;
     }
 
@@ -2057,9 +2050,9 @@ static int SigGroupHeadTest09(void)
     SigGroupHeadSetSigCnt(sh, 4);
     SigGroupHeadBuildMatchArray(de_ctx, sh, 4);
 
-    result &= (sh->match_array[0] == 0);
-    result &= (sh->match_array[1] == 2);
-    result &= (sh->match_array[2] == 4);
+    result &= (sh->match_array[0] == de_ctx->sig_list);
+    result &= (sh->match_array[1] == de_ctx->sig_list->next->next);
+    result &= (sh->match_array[2] == de_ctx->sig_list->next->next->next->next);
 
     SigGroupHeadFree(sh);
 
diff --git a/src/detect-engine-uri.c b/src/detect-engine-uri.c
index e442d3f68d..138a68d4bd 100644
--- a/src/detect-engine-uri.c
+++ b/src/detect-engine-uri.c
@@ -283,7 +283,7 @@ int DetectEngineInspectPacketUris(DetectEngineCtx *de_ctx,
        state against those patterns */
     if (s->flags & SIG_FLAG_MPM_URI) {
         if (det_ctx->de_mpm_scanned_uri == FALSE) {
-            uint32_t cnt = DetectUricontentInspectMpm(det_ctx, alstate);
+            uint32_t cnt = DetectUricontentInspectMpm(det_ctx, f, htp_state);
 
             /* only consider uri sigs if we've seen at least one match */
             /** \warning when we start supporting negated uri content matches
@@ -328,11 +328,10 @@ int DetectEngineInspectPacketUris(DetectEngineCtx *de_ctx,
     SCLogDebug("co->id %"PRIu32, co->id);
 #endif
 
-    size_t idx = 0;
+    size_t idx = AppLayerTransactionGetInspectId(f);
     htp_tx_t *tx = NULL;
 
-    for (idx = 0;//htp_state->new_in_tx_index;
-         idx < list_size(htp_state->connp->conn->transactions); idx++)
+    for ( ; idx < list_size(htp_state->connp->conn->transactions); idx++)
     {
         tx = list_get(htp_state->connp->conn->transactions, idx);
         if (tx == NULL || tx->request_uri_normalized == NULL)
diff --git a/src/detect-engine.c b/src/detect-engine.c
index 3f934e1d5d..f53eba6e22 100644
--- a/src/detect-engine.c
+++ b/src/detect-engine.c
@@ -353,6 +353,14 @@ TmEcode DetectEngineThreadCtxInit(ThreadVars *tv, void *initdata, void **data) {
                     (uintmax_t)(det_ctx->de_state_sig_array_len * sizeof(uint8_t)), strerror(errno));
             return TM_ECODE_FAILED;
         }
+
+        det_ctx->match_array_len = de_ctx->sig_array_len;
+        det_ctx->match_array = SCMalloc(det_ctx->match_array_len * sizeof(Signature *));
+        if (det_ctx->match_array == NULL) {
+            SCLogError(SC_ERR_MEM_ALLOC, "malloc of %"PRIuMAX" failed: %s",
+                    (uintmax_t)(det_ctx->match_array_len * sizeof(Signature *)), strerror(errno));
+            return TM_ECODE_FAILED;
+        }
     }
 
     /** alert counter setup */
diff --git a/src/detect-flow.c b/src/detect-flow.c
index 687c4b7b72..6605c5e982 100644
--- a/src/detect-flow.c
+++ b/src/detect-flow.c
@@ -106,6 +106,8 @@ error:
  */
 int DetectFlowMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, Packet *p, Signature *s, SigMatch *m)
 {
+    SCEnter();
+
     uint8_t cnt = 0;
     DetectFlowData *fd = (DetectFlowData *)m->ctx;
 
@@ -122,9 +124,9 @@ int DetectFlowMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, Packet *p, S
     }
 
     int ret = (fd->match_cnt == cnt) ? 1 : 0;
-    //printf("DetectFlowMatch: returning %" PRId32 " cnt %" PRId32 " fd->match_cnt %" PRId32 " fd->flags 0x%02X p->flowflags 0x%02X \n", ret, cnt,
-              //fd->match_cnt, fd->flags, p->flowflags);
-    return ret;
+    SCLogDebug("returning %" PRId32 " cnt %" PRIu8 " fd->match_cnt %" PRId32 " fd->flags 0x%02X p->flowflags 0x%02X",
+        ret, cnt, fd->match_cnt, fd->flags, p->flowflags);
+    SCReturnInt(ret);
 }
 
 /**
diff --git a/src/detect-uricontent.c b/src/detect-uricontent.c
index 202702da3c..34b92c71a8 100644
--- a/src/detect-uricontent.c
+++ b/src/detect-uricontent.c
@@ -409,29 +409,32 @@ static inline int DoDetectAppLayerUricontentMatch (DetectEngineThreadCtx *det_ct
     return ret;
 }
 
-/** \brief Run the pattern matcher against the uri(s)
+/**
+ *  \brief Run the pattern matcher against the uri(s)
  *
  *  We run against _all_ uri(s) we have as the pattern matcher will
  *  flag each sig that has a match. We need to do this for all uri(s)
  *  to not miss possible events.
  *
+ *  \param f locked flow
+ *  \param htp_state initialized htp state
+ *
  *  \warning Make sure the flow/state is locked
  *  \todo what should we return? Just the fact that we matched?
  */
-uint32_t DetectUricontentInspectMpm(DetectEngineThreadCtx *det_ctx, void *alstate) {
+uint32_t DetectUricontentInspectMpm(DetectEngineThreadCtx *det_ctx, Flow *f, HtpState *htp_state) {
     SCEnter();
 
     uint32_t cnt = 0;
     size_t idx = 0;
     htp_tx_t *tx = NULL;
 
-    HtpState *htp_state = (HtpState *)alstate;
     if (htp_state == NULL || htp_state->connp == NULL) {
         SCLogDebug("no HTTP state / no connp");
         SCReturnUInt(0U);
     }
 
-    for (idx = 0;//htp_state->new_in_tx_index;
+    for (idx = AppLayerTransactionGetInspectId(f);
          idx < list_size(htp_state->connp->conn->transactions); idx++)
     {
         tx = list_get(htp_state->connp->conn->transactions, idx);
diff --git a/src/detect-uricontent.h b/src/detect-uricontent.h
index 0acb228ace..1b816b4bf3 100644
--- a/src/detect-uricontent.h
+++ b/src/detect-uricontent.h
@@ -43,6 +43,7 @@
                                        (c)->within > 0))
 
 #include "util-spm-bm.h"
+#include "app-layer-htp.h"
 
 typedef struct DetectUricontentData_ {
     uint8_t *uricontent;
@@ -62,9 +63,11 @@ typedef struct DetectUricontentData_ {
 /* prototypes */
 void DetectUricontentRegister (void);
 uint32_t DetectUricontentMaxId(DetectEngineCtx *);
-uint32_t DetectUricontentInspectMpm(DetectEngineThreadCtx *det_ctx, void *alstate);
+//uint32_t DetectUricontentInspectMpm(DetectEngineThreadCtx *det_ctx, void *alstate);
 SigMatch *DetectUricontentGetLastPattern(SigMatch *);
 void DetectUricontentPrint(DetectUricontentData *);
 
+uint32_t DetectUricontentInspectMpm(DetectEngineThreadCtx *, Flow *, HtpState *);
+
 #endif /* __DETECT_URICONTENT_H__ */
 
diff --git a/src/detect.c b/src/detect.c
index 5a9fd0d59c..a44891351d 100644
--- a/src/detect.c
+++ b/src/detect.c
@@ -430,6 +430,104 @@ int SigLoadSignatures (DetectEngineCtx *de_ctx, char *sig_file)
     SCReturnInt(0);
 }
 
+/**
+ *  \brief build an array of signatures that will be inspected
+ *
+ *  All signatures that can be filtered out on forehand are not added to it.
+ *
+ *  \param de_ctx detection engine ctx
+ *  \param det_ctx detection engine thread ctx -- array is stored here
+ *  \param de_state_start flag to indicate if we're at the start of a stateful run
+ *  \param p packet
+ *  \param alproto application layer protocol
+ */
+static void SigMatchSignaturesBuildMatchArray(DetectEngineCtx *de_ctx,
+        DetectEngineThreadCtx *det_ctx, char de_state_start, Packet *p,
+        uint16_t alproto)
+{
+    uint32_t i;
+
+    /* reset previous run */
+    det_ctx->match_array_cnt = 0;
+
+    for (i = 0; i < det_ctx->sgh->sig_cnt; i++) {
+        Signature *s = det_ctx->sgh->match_array[i];
+
+        if (s->flags & SIG_FLAG_MPM) {
+            /* filter out sigs that want pattern matches, but
+             * have no matches */
+            if (!(det_ctx->pmq.pattern_id_bitarray[(s->mpm_pattern_id / 8)] & (1<<(s->mpm_pattern_id % 8))) &&
+                    (s->flags & SIG_FLAG_MPM) && !(s->flags & SIG_FLAG_MPM_NEGCONTENT)) {
+                SCLogDebug("mpm sig without matches (pat id check in content).");
+                continue;
+            }
+        }
+
+        /* de_state check, filter out all signatures that already had a match before
+         * or just partially match */
+        if (de_state_start == FALSE) {
+            if (s->amatch != NULL || s->umatch != NULL) {
+                if (det_ctx->de_state_sig_array[s->num] != DE_STATE_MATCH_NEW) {
+                    continue;
+                }
+            }
+        }
+
+        /* filter out the sigs that inspect the payload, if packet
+           no payload inspection flag is set*/
+        if ((p->flags & PKT_NOPAYLOAD_INSPECTION) && (s->flags & SIG_FLAG_PAYLOAD)) {
+            SCLogDebug("no payload inspection enabled and sig has payload portion.");
+            continue;
+        }
+
+        /* if the sig has alproto and the session as well they should match */
+        if (s->alproto != ALPROTO_UNKNOWN && alproto != ALPROTO_UNKNOWN) {
+            if (s->alproto != alproto) {
+                continue;
+            }
+        }
+
+        /* check the source & dst port in the sig */
+        if (p->proto == IPPROTO_TCP || p->proto == IPPROTO_UDP) {
+            if (!(s->flags & SIG_FLAG_DP_ANY)) {
+                DetectPort *dport = DetectPortLookupGroup(s->dp,p->dp);
+                if (dport == NULL) {
+                    SCLogDebug("dport didn't match.");
+                    continue;
+                }
+            }
+            if (!(s->flags & SIG_FLAG_SP_ANY)) {
+                DetectPort *sport = DetectPortLookupGroup(s->sp,p->sp);
+                if (sport == NULL) {
+                    SCLogDebug("sport didn't match.");
+                    continue;
+                }
+            }
+        }
+
+        /* check the destination address */
+        if (!(s->flags & SIG_FLAG_DST_ANY)) {
+            DetectAddress *daddr = DetectAddressLookupInHead(&s->dst,&p->dst);
+            if (daddr == NULL) {
+                SCLogDebug("dst addr didn't match.");
+                    continue;
+            }
+        }
+        /* check the source address */
+        if (!(s->flags & SIG_FLAG_SRC_ANY)) {
+            DetectAddress *saddr = DetectAddressLookupInHead(&s->src,&p->src);
+            if (saddr == NULL) {
+                SCLogDebug("src addr didn't match.");
+                    continue;
+            }
+        }
+
+        /* okay, store it */
+        det_ctx->match_array[det_ctx->match_array_cnt] = s;
+        det_ctx->match_array_cnt++;
+    }
+}
+
 /**
  *  \brief Get the SigGroupHead for a packet.
  *
@@ -498,7 +596,7 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
     int match = 0, fmatch = 0;
     Signature *s = NULL;
     SigMatch *sm = NULL;
-    uint32_t idx,sig;
+    uint32_t idx;
     uint16_t alproto = ALPROTO_UNKNOWN;
     void *alstate = NULL;
     uint8_t flags = 0;
@@ -637,10 +735,12 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
 
     /* stateful app layer detection */
     char de_state_start = FALSE;
+    /* initialize to 0 (DE_STATE_MATCH_FULL) */
     memset(det_ctx->de_state_sig_array, 0x00, det_ctx->de_state_sig_array_len);
-    if (p->flow != NULL) {
+    if (alstate != NULL) {
         if (DeStateFlowHasState(p->flow)) {
-            DeStateDetectContinueDetection(th_v, de_ctx, det_ctx, p->flow, flags, alstate, alproto);
+            DeStateDetectContinueDetection(th_v, de_ctx, det_ctx, p->flow,
+                    flags, alstate, alproto);
         } else {
             de_state_start = TRUE;
         }
@@ -648,87 +748,36 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
         de_state_start = TRUE;
     }
 
+    /* build the match array */
+    SigMatchSignaturesBuildMatchArray(de_ctx, det_ctx, de_state_start, p, alproto);
+
     /* inspect the sigs against the packet */
-    for (idx = 0; idx < det_ctx->sgh->sig_cnt; idx++) {
+    for (idx = 0; idx < det_ctx->match_array_cnt; idx++) {
         PROFILING_START;
 
-        sig = det_ctx->sgh->match_array[idx];
-        s = de_ctx->sig_array[sig];
-
+        s = det_ctx->match_array[idx];
         SCLogDebug("inspecting signature id %"PRIu32"", s->id);
 
+        SCLogDebug("s->amatch %p, s->umatch %p", s->amatch, s->umatch);
         if ((s->amatch != NULL || s->umatch != NULL) && p->flow != NULL) {
-            if (de_state_start == FALSE) {
-                if (det_ctx->de_state_sig_array[s->num] != DE_STATE_MATCH_NEW) {
-                    goto next;
-                }
-            }
-        }
-
-        /* filter out the sigs that inspects the payload, if packet
-           no payload inspection flag is set*/
-        if ((p->flags & PKT_NOPAYLOAD_INSPECTION) && (s->flags & SIG_FLAG_PAYLOAD)) {
-            SCLogDebug("no payload inspection enabled and sig has payload portion.");
-            goto next;
-        }
-
-        if (s->flags & SIG_FLAG_MPM) {
-            if (det_ctx->pmq.pattern_id_bitarray != NULL) {
-                /* filter out sigs that want pattern matches, but
-                 * have no matches */
-                if (!(det_ctx->pmq.pattern_id_bitarray[(s->mpm_pattern_id / 8)] & (1<<(s->mpm_pattern_id % 8))) &&
-                        (s->flags & SIG_FLAG_MPM) && !(s->flags & SIG_FLAG_MPM_NEGCONTENT)) {
-                    SCLogDebug("mpm sig without matches (pat id check in content).");
-                    goto next;
-                }
-            }
-        }
-
-        /* if the sig has alproto and the session as well they should match */
-        if (s->alproto != ALPROTO_UNKNOWN && alproto != ALPROTO_UNKNOWN) {
-            if (s->alproto != alproto) {
-                goto next;
-            }
-        }
-
-        /* check the source & dst port in the sig */
-        if (p->proto == IPPROTO_TCP || p->proto == IPPROTO_UDP) {
-            if (!(s->flags & SIG_FLAG_DP_ANY)) {
-                DetectPort *dport = DetectPortLookupGroup(s->dp,p->dp);
-                if (dport == NULL) {
-                    SCLogDebug("dport didn't match.");
+            if (de_state_start == TRUE) {
+                SCLogDebug("stateful app layer match inspection starting");
+                if (DeStateDetectStartDetection(th_v, de_ctx, det_ctx, s,
+                            p->flow, flags, alstate, alproto) != 1)
                     goto next;
-                }
-            }
-            if (!(s->flags & SIG_FLAG_SP_ANY)) {
-                DetectPort *sport = DetectPortLookupGroup(s->sp,p->sp);
-                if (sport == NULL) {
-                    SCLogDebug("sport didn't match.");
+            } else {
+                SCLogDebug("signature %"PRIu32" (%"PRIuMAX"): %s",
+                        s->id, (uintmax_t)s->num, DeStateMatchResultToString(det_ctx->de_state_sig_array[s->num]));
+                if (det_ctx->de_state_sig_array[s->num] != DE_STATE_MATCH_NEW) {
                     goto next;
                 }
             }
         }
 
-        /* check the destination address */
-        if (!(s->flags & SIG_FLAG_DST_ANY)) {
-            DetectAddress *daddr = DetectAddressLookupInHead(&s->dst,&p->dst);
-            if (daddr == NULL) {
-                SCLogDebug("dst addr didn't match.");
-                goto next;
-            }
-        }
-        /* check the source address */
-        if (!(s->flags & SIG_FLAG_SRC_ANY)) {
-            DetectAddress *saddr = DetectAddressLookupInHead(&s->src,&p->src);
-            if (saddr == NULL) {
-                SCLogDebug("src addr didn't match.");
-                goto next;
-            }
-        }
-
         /* Check the payload keywords. If we are a MPM sig and we've made
          * to here, we've had at least one of the patterns match */
         if (s->pmatch != NULL) {
+            /* if we have stream msgs, inspect against those first */
             if (smsg != NULL) {
                 char pmatch = 0;
                 int i = 0;
@@ -769,22 +818,6 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
             }
         }
 
-        SCLogDebug("s->amatch %p", s->amatch);
-        if ((s->amatch != NULL || s->umatch != NULL) && p->flow != NULL) {
-            if (de_state_start == TRUE) {
-                SCLogDebug("stateful app layer match inspection starting");
-                if (DeStateDetectStartDetection(th_v, de_ctx, det_ctx, s,
-                            p->flow, flags, alstate, alproto) != 1)
-                    goto next;
-            } else {
-                SCLogDebug("signature %"PRIu32" (%"PRIuMAX"): %s",
-                        s->id, (uintmax_t)s->num, DeStateMatchResultToString(det_ctx->de_state_sig_array[s->num]));
-                if (det_ctx->de_state_sig_array[s->num] != DE_STATE_MATCH_NEW) {
-                    goto next;
-                }
-            }
-        }
-
         /* if we get here but have no sigmatches to match against,
          * we consider the sig matched. */
         if (s->match == NULL) {
@@ -865,7 +898,7 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
         break;
     }
 
-    if (p->flow != NULL) {
+    if (alstate != NULL) {
         SCLogDebug("getting de_state_status");
         int de_state_status = DeStateUpdateInspectTransactionId(p->flow);
         SCLogDebug("de_state_status %d", de_state_status);
@@ -923,8 +956,8 @@ end:
  *  \retval TM_ECODE_FAILED error
  *  \retval TM_ECODE_OK ok
  */
-TmEcode Detect(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq) {
-
+TmEcode Detect(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq)
+{
     /* No need to perform any detection on this packet, if the the given flag is set.*/
     if (p->flags & PKT_NOPACKET_INSPECTION)
         return 0;
@@ -2495,7 +2528,7 @@ void DbgPrintSigs(DetectEngineCtx *de_ctx, SigGroupHead *sgh) {
 
     uint32_t sig;
     for (sig = 0; sig < sgh->sig_cnt; sig++) {
-        printf("%" PRIu32 " ", de_ctx->sig_array[sgh->match_array[sig]]->id);
+        printf("%" PRIu32 " ", sgh->match_array[sig]->id);
     }
     printf("\n");
 }
@@ -2569,7 +2602,7 @@ int SigAddressPrepareStage5(DetectEngineCtx *de_ctx) {
                 if (global_src_gr->sh != NULL) {
                     printf(" - ");
                     for (u = 0; u < global_src_gr->sh->sig_cnt; u++) {
-                        Signature *s = de_ctx->sig_array[global_src_gr->sh->match_array[u]];
+                        Signature *s = global_src_gr->sh->match_array[u];
                         printf("%" PRIu32 " ", s->id);
                     }
                     printf("\n");
@@ -2601,7 +2634,7 @@ int SigAddressPrepareStage5(DetectEngineCtx *de_ctx) {
                     if (global_dst_gr->sh != NULL) {
                         printf(" - ");
                         for (u = 0; u < global_dst_gr->sh->sig_cnt; u++) {
-                            Signature *s = de_ctx->sig_array[global_dst_gr->sh->match_array[u]];
+                            Signature *s = global_dst_gr->sh->match_array[u];
                             printf("%" PRIu32 " ", s->id);
                         }
                         printf("\n");
@@ -2621,7 +2654,7 @@ int SigAddressPrepareStage5(DetectEngineCtx *de_ctx) {
 #ifdef PRINTSIGS
                             printf(" - ");
                             for (u = 0; u < dp->sh->sig_cnt; u++) {
-                                Signature *s = de_ctx->sig_array[dp->sh->match_array[u]];
+                                Signature *s = dp->sh->match_array[u];
                                 printf("%" PRIu32 " ", s->id);
                             }
 #endif
@@ -2652,7 +2685,7 @@ int SigAddressPrepareStage5(DetectEngineCtx *de_ctx) {
 #ifdef PRINTSIGS
                             printf(" - ");
                             for (u = 0; u < dp->sh->sig_cnt; u++) {
-                                Signature *s = de_ctx->sig_array[dp->sh->match_array[u]];
+                                Signature *s = dp->sh->match_array[u];
                                 printf("%" PRIu32 " ", s->id);
                             }
 #endif
@@ -7229,11 +7262,11 @@ static int SigTestSgh01 (void) {
         goto end;
     }
 
-    if (sgh->match_array[0] != 0) {
-        printf("sgh doesn't contain sid 1, should have (sgh->match_array[0] %u, expected 0): ", sgh->match_array[0]);
+    if (sgh->match_array[0] != de_ctx->sig_list) {
+        printf("sgh doesn't contain sid 1, should have (sgh->match_array[0] %p, expected %p): ", sgh->match_array[0], de_ctx->sig_list);
         goto end;
     }
-    if (sgh->match_array[1] != 2) {
+    if (sgh->match_array[1] != de_ctx->sig_list->next->next) {
         printf("sgh doesn't contain sid 3, should have: ");
         goto end;
     }
@@ -7256,8 +7289,9 @@ static int SigTestSgh01 (void) {
         goto end;
     }
 
-    if (sgh2->match_array[0] != 1) {
-        printf("sgh doesn't contain sid 2, should have (sgh2->match_array[0] %u, expected 0): ", sgh2->match_array[0]);
+    if (sgh2->match_array[0] != de_ctx->sig_list->next) {
+        printf("sgh doesn't contain sid 2, should have (sgh2->match_array[0] %p, expected %p): ",
+                sgh2->match_array[0], de_ctx->sig_list->next);
         goto end;
     }
 
@@ -7273,7 +7307,7 @@ static int SigTestSgh01 (void) {
         goto end;
     }
 
-    if (sgh2->match_array[0] != 1) {
+    if (sgh2->match_array[0] != de_ctx->sig_list->next) {
         printf("sgh2 doesn't contain sid 2, should have: ");
         goto end;
     }
@@ -7360,11 +7394,11 @@ static int SigTestSgh02 (void) {
         goto end;
     }
 
-    if (sgh->match_array[0] != 0) {
-        printf("sgh doesn't contain sid 1, should have (sgh->match_array[0] %u, expected 0): ", sgh->match_array[0]);
+    if (sgh->match_array[0] != de_ctx->sig_list) {
+        printf("sgh doesn't contain sid 1, should have (sgh->match_array[0] %p, expected %p): ", sgh->match_array[0], de_ctx->sig_list);
         goto end;
     }
-    if (sgh->match_array[1] != 2) {
+    if (sgh->match_array[1] != de_ctx->sig_list->next->next) {
         printf("sgh doesn't contain sid 3, should have: ");
         goto end;
     }
@@ -7392,16 +7426,16 @@ static int SigTestSgh02 (void) {
         printf("sgh sig cnt %u, expected 3: ", sgh->sig_cnt);
         goto end;
     }
-    if (sgh->match_array[0] != 0) {
+    if (sgh->match_array[0] != de_ctx->sig_list) {
         printf("sgh doesn't contain sid 1, should have: ");
         goto end;
     }
-    if (sgh->match_array[1] != 1) {
-        printf("sgh doesn't contain sid 1, should have: ");
+    if (sgh->match_array[1] != de_ctx->sig_list->next) {
+        printf("sgh doesn't contain sid 2, should have: ");
         goto end;
     }
-    if (sgh->match_array[2] != 2) {
-        printf("sgh doesn't contain sid 1, should have: ");
+    if (sgh->match_array[2] != de_ctx->sig_list->next->next) {
+        printf("sgh doesn't contain sid 3, should have: ");
         goto end;
     }
 #if 0
@@ -7429,7 +7463,7 @@ static int SigTestSgh02 (void) {
         goto end;
     }
 
-    if (sgh->match_array[0] != 0) {
+    if (sgh->match_array[0] != de_ctx->sig_list) {
         printf("sgh doesn't contain sid 1, should have: ");
         goto end;
     }
@@ -7459,7 +7493,7 @@ static int SigTestSgh02 (void) {
         goto end;
     }
 
-    if (sgh->match_array[0] != 0) {
+    if (sgh->match_array[0] != de_ctx->sig_list) {
         printf("sgh doesn't contain sid 1, should have: ");
         goto end;
     }
@@ -7560,11 +7594,11 @@ static int SigTestSgh03 (void) {
         goto end;
     }
 
-    if (sgh->match_array[0] != 0) {
-        printf("sgh doesn't contain sid 1, should have (sgh->match_array[0] %u, expected 0): ", sgh->match_array[0]);
+    if (sgh->match_array[0] != de_ctx->sig_list) {
+        printf("sgh doesn't contain sid 1, should have (sgh->match_array[0] %p, expected %p): ", sgh->match_array[0], de_ctx->sig_list);
         goto end;
     }
-    if (sgh->match_array[1] != 2) {
+    if (sgh->match_array[1] != de_ctx->sig_list->next->next) {
         printf("sgh doesn't contain sid 3, should have: ");
         goto end;
     }
@@ -7589,15 +7623,15 @@ static int SigTestSgh03 (void) {
         printf("sgh sig cnt %u, expected 3: ", sgh->sig_cnt);
         goto end;
     }
-    if (sgh->match_array[0] != 0) {
+    if (sgh->match_array[0] != de_ctx->sig_list) {
         printf("sgh doesn't contain sid 1, should have: ");
         goto end;
     }
-    if (sgh->match_array[1] != 1) {
+    if (sgh->match_array[1] != de_ctx->sig_list->next) {
         printf("sgh doesn't contain sid 1, should have: ");
         goto end;
     }
-    if (sgh->match_array[2] != 2) {
+    if (sgh->match_array[2] != de_ctx->sig_list->next->next) {
         printf("sgh doesn't contain sid 1, should have: ");
         goto end;
     }
@@ -7633,7 +7667,7 @@ static int SigTestSgh03 (void) {
         goto end;
     }
 
-    if (sgh->match_array[0] != 0) {
+    if (sgh->match_array[0] != de_ctx->sig_list) {
         printf("sgh doesn't contain sid 1, should have: ");
         goto end;
     }
@@ -7721,11 +7755,11 @@ static int SigTestSgh04 (void) {
         goto end;
     }
 
-    if (sgh->match_array[0] != 0) {
-        printf("sgh doesn't contain sid 1, should have (sgh->match_array[0] %u, expected 0): ", sgh->match_array[0]);
+    if (sgh->match_array[0] != de_ctx->sig_list) {
+        printf("sgh doesn't contain sid 1, should have (sgh->match_array[0] %p, expected %p): ", sgh->match_array[0], de_ctx->sig_list);
         goto end;
     }
-    if (sgh->match_array[1] != 2) {
+    if (sgh->match_array[1] != de_ctx->sig_list->next->next) {
         printf("sgh doesn't contain sid 3, should have: ");
         goto end;
     }
@@ -7753,16 +7787,16 @@ static int SigTestSgh04 (void) {
         printf("sgh sig cnt %u, expected 3: ", sgh->sig_cnt);
         goto end;
     }
-    if (sgh->match_array[0] != 0) {
+    if (sgh->match_array[0] != de_ctx->sig_list) {
         printf("sgh doesn't contain sid 1, should have: ");
         goto end;
     }
-    if (sgh->match_array[1] != 1) {
-        printf("sgh doesn't contain sid 1, should have: ");
+    if (sgh->match_array[1] != de_ctx->sig_list->next) {
+        printf("sgh doesn't contain sid 2, should have: ");
         goto end;
     }
-    if (sgh->match_array[2] != 2) {
-        printf("sgh doesn't contain sid 1, should have: ");
+    if (sgh->match_array[2] != de_ctx->sig_list->next->next) {
+        printf("sgh doesn't contain sid 3, should have: ");
         goto end;
     }
 #if 0
@@ -7790,7 +7824,7 @@ static int SigTestSgh04 (void) {
         goto end;
     }
 
-    if (sgh->match_array[0] != 0) {
+    if (sgh->match_array[0] != de_ctx->sig_list) {
         printf("sgh doesn't contain sid 1, should have: ");
         goto end;
     }
@@ -7821,7 +7855,7 @@ static int SigTestSgh04 (void) {
         goto end;
     }
 
-    if (sgh->match_array[0] != 0) {
+    if (sgh->match_array[0] != de_ctx->sig_list) {
         printf("sgh doesn't contain sid 1, should have: ");
         goto end;
     }
@@ -7891,6 +7925,7 @@ static int SigTestSgh05 (void) {
 end:
     return result;
 }
+
 static int SigTestContent01Real (int mpm_type) {
     uint8_t *buf = (uint8_t *)"01234567890123456789012345678901";
     uint16_t buflen = strlen((char *)buf);
diff --git a/src/detect.h b/src/detect.h
index 27785a9786..30e5a4d0e9 100644
--- a/src/detect.h
+++ b/src/detect.h
@@ -473,6 +473,14 @@ typedef struct DetectionEngineThreadCtx_ {
     char de_have_httpuri;
     char de_mpm_scanned_uri;
 
+    /** array of signature pointers we're going to inspect in the detection
+     *  loop. */
+    Signature **match_array;
+    /** size of the array in items (mem size if * sizeof(Signature *) */
+    uint32_t match_array_len;
+    /** size in use */
+    uint32_t match_array_cnt;
+
     /** Array of sigs that had a state change */
     uint8_t *de_state_sig_array;
     SigIntId de_state_sig_array_len;
@@ -594,8 +602,8 @@ typedef struct SigGroupHead_ {
     uint8_t *sig_array; /**< bit array of sig nums (internal id's) */
     uint32_t sig_size; /**< size in bytes */
 
-    /* Array with sig nums... size is sig_cnt * sizeof(SigIntId) */
-    SigIntId *match_array;
+    /** Array with sig ptrs... size is sig_cnt * sizeof(Signature *) */
+    Signature **match_array;
 
     /* ptr to our init data we only use at... init :) */
     SigGroupHeadInitData *init;