Don't scan TCP packet payload if it was added to the stream. Inspect the tcp stream with the correct packet. Should fix #184 and #185.

src/decode.h

@@ -672,6 +672,7 @@ void AddressDebugPrint(Address *);
 #define PKT_NOPAYLOAD_INSPECTION        0x02    /**< Flag to indicate that packet contents should not be inspected*/
 #define PKT_ALLOC                       0x04    /**< Packet was alloc'd this run, needs to be freed */
 #define PKT_HAS_TAG                     0x08    /**< Packet has matched a tag */
+#define PKT_STREAM_ADD                  0x10    /**< Packet payload was added to reassembled stream */
 
 #endif /* __DECODE_H__ */
 

src/detect-engine-mpm.c

@@ -179,12 +179,14 @@ uint32_t UriPatternSearch(DetectEngineThreadCtx *det_ctx,
  *
  *  \param tv threadvars
  *  \param det_ctx detection engine thread ctx
+ *  \param p packet
  *  \param smsg stream msg (reassembled stream data)
+ *  \param flags stream flags
  *
  *  \retval ret number of matches
  */
 uint32_t StreamPatternSearch(ThreadVars *tv, DetectEngineThreadCtx *det_ctx,
-                           StreamMsg *smsg)
+        Packet *p, StreamMsg *smsg, uint8_t flags)
 {
     SCEnter();
 

src/detect-engine-mpm.h

@@ -35,7 +35,7 @@ uint16_t PatternMatchDefaultMatcher(void);
 
 uint32_t PacketPatternSearch(ThreadVars *, DetectEngineThreadCtx *, Packet *);
 uint32_t UriPatternSearch(DetectEngineThreadCtx *, uint8_t *, uint16_t);
-uint32_t StreamPatternSearch(ThreadVars *, DetectEngineThreadCtx *, StreamMsg *);
+uint32_t StreamPatternSearch(ThreadVars *, DetectEngineThreadCtx *, Packet *, StreamMsg *, uint8_t);
 
 void PacketPatternCleanup(ThreadVars *, DetectEngineThreadCtx *);
 void StreamPatternCleanup(ThreadVars *t, DetectEngineThreadCtx *det_ctx, StreamMsg *smsg);

src/detect-tls-version.c

@@ -557,17 +557,22 @@ static int DetectTlsVersionTestDetect03(void) {
     Signature *s = NULL;
     ThreadVars th_v;
     DetectEngineThreadCtx *det_ctx = NULL;
+    TCPHdr tcp_hdr;
 
     memset(&th_v, 0, sizeof(th_v));
     memset(&p, 0, sizeof(p));
     memset(&f, 0, sizeof(f));
     memset(&ssn, 0, sizeof(ssn));
+    memset(&tcp_hdr, 0, sizeof(tcp_hdr));
+
+    tcp_hdr.th_seq = htonl(1000);
 
     p.src.family = AF_INET;
     p.dst.family = AF_INET;
     p.payload = tlsbuf4;
     p.payload_len = tlslen4;
     p.proto = IPPROTO_TCP;
+    p.tcph = &tcp_hdr;
 
     FLOW_INITIALIZE(&f);
     f.protoctx = (void *)&ssn;
@@ -575,6 +580,7 @@ static int DetectTlsVersionTestDetect03(void) {
     p.flowflags |= FLOW_PKT_TOSERVER;
     p.flowflags |= FLOW_PKT_ESTABLISHED;
     f.alproto = ALPROTO_TLS;
+    f.proto = p.proto;
 
     StreamTcpInitConfig(TRUE);
     FlowL7DataPtrInit(&f);

src/detect-uricontent.c

@@ -1247,17 +1247,22 @@ static int DetectUriSigTest05(void) {
     Signature *s = NULL;
     ThreadVars th_v;
     DetectEngineThreadCtx *det_ctx = NULL;
+    TCPHdr tcp_hdr;
 
     memset(&th_v, 0, sizeof(th_v));
     memset(&p, 0, sizeof(p));
     memset(&f, 0, sizeof(f));
     memset(&ssn, 0, sizeof(ssn));
+    memset(&tcp_hdr, 0, sizeof(tcp_hdr));
+
+    tcp_hdr.th_seq = htonl(1000);
 
     p.src.family = AF_INET;
     p.dst.family = AF_INET;
     p.payload = httpbuf1;
     p.payload_len = httplen1;
     p.proto = IPPROTO_TCP;
+    p.tcph = &tcp_hdr;
 
     FLOW_INITIALIZE(&f);
     f.protoctx = (void *)&ssn;
@@ -1268,6 +1273,7 @@ static int DetectUriSigTest05(void) {
     p.flowflags |= FLOW_PKT_TOSERVER;
     p.flowflags |= FLOW_PKT_ESTABLISHED;
     f.alproto = ALPROTO_HTTP;
+    f.proto = p.proto;
 
     StreamTcpInitConfig(TRUE);
     FlowL7DataPtrInit(&f);
@@ -1292,26 +1298,21 @@ static int DetectUriSigTest05(void) {
     de_ctx->flags |= DE_QUIET;
 
     s = de_ctx->sig_list = SigInit(de_ctx,"alert tcp any any -> any any (msg:"
-                                   "\" Test uricontent\"; "
+            "\" Test uricontent\"; uricontent:\"foo\"; sid:1;)");
-                                   "uricontent:\"foo\"; sid:1;)");
     if (s == NULL) {
         goto end;
     }
 
     s = s->next = SigInit(de_ctx,"alert tcp any any -> any any (msg:"
-                                   "\" Test uricontent\"; "
+            "\" Test uricontent\"; uricontent:\"one\"; content:\"two\"; sid:2;)");
-                                   "uricontent:\"one\"; content:\"two\"; sid:2;)");
     if (s == NULL) {
         goto end;
     }
 
     s = s->next = SigInit(de_ctx,"alert tcp any any -> any any (msg:"
-                                   "\" Test uricontent\"; "
+            "\" Test uricontent\"; uricontent:\"one\"; offset:1; depth:10; "
-                                   "uricontent:\"one\"; offset:1; depth:10; "
+            "uricontent:\"two\"; distance:1; within: 4; uricontent:\"three\"; "
-                                   "uricontent:\"two\"; distance:1; within: 4; "
+            "distance:1; within: 6; sid:3;)");
-                                   "uricontent:\"three\"; distance:1; within: 6; "
-                                   "sid:3;)");
-
     if (s == NULL) {
         goto end;
     }
@@ -1374,17 +1375,22 @@ static int DetectUriSigTest06(void) {
     Signature *s = NULL;
     ThreadVars th_v;
     DetectEngineThreadCtx *det_ctx = NULL;
+    TCPHdr tcp_hdr;
 
     memset(&th_v, 0, sizeof(th_v));
     memset(&p, 0, sizeof(p));
     memset(&f, 0, sizeof(f));
     memset(&ssn, 0, sizeof(ssn));
+    memset(&tcp_hdr, 0, sizeof(tcp_hdr));
+
+    tcp_hdr.th_seq = htonl(1000);
 
     p.src.family = AF_INET;
     p.dst.family = AF_INET;
     p.payload = httpbuf1;
     p.payload_len = httplen1;
     p.proto = IPPROTO_TCP;
+    p.tcph = &tcp_hdr;
 
     FLOW_INITIALIZE(&f);
     f.protoctx = (void *)&ssn;
@@ -1395,6 +1401,7 @@ static int DetectUriSigTest06(void) {
     p.flowflags |= FLOW_PKT_TOSERVER;
     p.flowflags |= FLOW_PKT_ESTABLISHED;
     f.alproto = ALPROTO_HTTP;
+    f.proto = p.proto;
 
     StreamTcpInitConfig(TRUE);
     FlowL7DataPtrInit(&f);

src/detect.c

@@ -543,6 +543,85 @@ SigGroupHead *SigMatchSignaturesGetSgh(DetectEngineCtx *de_ctx, DetectEngineThre
     SCReturnPtr(sgh, "SigGroupHead");
 }
 
+/** \brief Get the smsgs relevant to this packet
+ *
+ *  \param f LOCKED flow
+ *  \param p packet
+ *  \param flags stream flags
+ */
+static StreamMsg *SigMatchSignaturesGetSmsg(Flow *f, Packet *p, uint8_t flags) {
+    SCEnter();
+
+    StreamMsg *smsg = NULL;
+
+    if (p->proto == IPPROTO_TCP) {
+        TcpSession *ssn = (TcpSession *)f->protoctx;
+        if (ssn != NULL) {
+            /* at stream eof, inspect all smsg's */
+            if (flags & STREAM_EOF) {
+                if (p->flowflags & FLOW_PKT_TOSERVER) {
+                    smsg = ssn->toserver_smsg_head;
+                    /* deref from the ssn */
+                    ssn->toserver_smsg_head = NULL;
+                    ssn->toserver_smsg_tail = NULL;
+
+                    SCLogDebug("to_server smsg %p at stream eof", smsg);
+                } else {
+                    smsg = ssn->toclient_smsg_head;
+                    /* deref from the ssn */
+                    ssn->toclient_smsg_head = NULL;
+                    ssn->toclient_smsg_tail = NULL;
+
+                    SCLogDebug("to_client smsg %p at stream eof", smsg);
+                }
+            } else {
+                if (p->flowflags & FLOW_PKT_TOSERVER) {
+                    StreamMsg *head = ssn->toserver_smsg_head;
+                    if (head == NULL) {
+                        SCLogDebug("no smsgs in to_server direction");
+                        goto end;
+                    }
+
+                    /* if the smsg is bigger than the current packet, we will
+                     * process the smsg in a later run */
+                    if ((head->data.seq + head->data.data_len) > (TCP_GET_SEQ(p) + p->payload_len)) {
+                        SCLogDebug("smsg ends beyond current packet, skipping for now");
+                        goto end;
+                    }
+
+                    smsg = head;
+                    /* deref from the ssn */
+                    ssn->toserver_smsg_head = NULL;
+                    ssn->toserver_smsg_tail = NULL;
+
+                    SCLogDebug("to_server smsg %p", smsg);
+                } else {
+                    StreamMsg *head = ssn->toclient_smsg_head;
+                    if (head == NULL)
+                        goto end;
+
+                    /* if the smsg is bigger than the current packet, we will
+                     * process the smsg in a later run */
+                    if ((head->data.seq + head->data.data_len) > (TCP_GET_SEQ(p) + p->payload_len)) {
+                        SCLogDebug("smsg ends beyond current packet, skipping for now");
+                        goto end;
+                    }
+
+                    smsg = head;
+                    /* deref from the ssn */
+                    ssn->toclient_smsg_head = NULL;
+                    ssn->toclient_smsg_tail = NULL;
+
+                    SCLogDebug("to_client smsg %p", smsg);
+                }
+            }
+        }
+    }
+
+end:
+    SCReturnPtr(smsg, "StreamMsg");
+}
+
 /**
  *  \brief Signature match function
  *
@@ -586,6 +665,7 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
                 sgh = p->flow->sgh_toclient;
                 use_flow_sgh = TRUE;
             }
+            smsg = SigMatchSignaturesGetSmsg(p->flow, p, flags);
         } else {
             no_store_flow_sgh = TRUE;
         }
@@ -597,26 +677,6 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
             alproto = AppLayerGetProtoFromPacket(p);
             SCLogDebug("alstate %p, alproto %u", alstate, alproto);
 
-            if (p->proto == IPPROTO_TCP) {
-                TcpSession *ssn = (TcpSession *)p->flow->protoctx;
-                if (ssn != NULL) {
-                    if (p->flowflags & FLOW_PKT_TOSERVER) {
-                        smsg = ssn->toserver_smsg_head;
-                        /* deref from the ssn */
-                        ssn->toserver_smsg_head = NULL;
-                        ssn->toserver_smsg_tail = NULL;
-
-                        SCLogDebug("to_server smsg %p", smsg);
-                    } else {
-                        smsg = ssn->toclient_smsg_head;
-                        /* deref from the ssn */
-                        ssn->toclient_smsg_head = NULL;
-                        ssn->toclient_smsg_tail = NULL;
-
-                        SCLogDebug("to_client smsg %p", smsg);
-                    }
-                }
-            }
         } else {
             SCLogDebug("packet doesn't have established flag set");
         }
@@ -676,13 +736,13 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
     /* have a look at the reassembled stream (if any) */
     if (p->flowflags & FLOW_PKT_ESTABLISHED) {
         if (smsg != NULL && det_ctx->sgh->mpm_stream_ctx != NULL) {
-            cnt = StreamPatternSearch(th_v, det_ctx, smsg);
+            cnt = StreamPatternSearch(th_v, det_ctx, p, smsg, flags);
             SCLogDebug("cnt %u", cnt);
         }
     }
 
     if (p->payload_len > 0 && det_ctx->sgh->mpm_ctx != NULL &&
-        !(p->flags & PKT_NOPAYLOAD_INSPECTION))
+        (!(p->flags & PKT_NOPAYLOAD_INSPECTION) && !(p->flags & PKT_STREAM_ADD)))
     {
         /* run the multi packet matcher against the payload of the packet */
         if (det_ctx->sgh->mpm_content_maxlen > p->payload_len) {
@@ -962,7 +1022,7 @@ end:
         }
 
         /* if we have (a) smsg(s), return to the pool */
-        while(smsg != NULL) {
+        while (smsg != NULL) {
             StreamMsg *smsg_next = smsg->next;
             SCLogDebug("returning smsg %p to pool", smsg);
             smsg->next = NULL;

src/stream-tcp-reassemble.c

@@ -1297,6 +1297,7 @@ static void StreamTcpSetupMsg(TcpSession *ssn, TcpStream *stream, Packet *p,
 
     smsg->data.data_len = 0;
     smsg->flow = p->flow;
+    BUG_ON(smsg->flow == NULL);
 
     FlowIncrUsecnt(smsg->flow);
 
@@ -1630,7 +1631,7 @@ int StreamTcpReassembleHandleSegmentUpdateACK (TcpReassemblyThreadCtx *ra_ctx,
              * IDS should advance it's ra_base_seq and should not consider this
              * packet any longer, even if it is retransmitted, as end host will
              * drop it anyway */
-             ra_base_seq = seg->seq - 1;
+            ra_base_seq = seg->seq - 1;
 
             smsg->flags |= STREAM_GAP;
             smsg->gap.gap_size = gap_len;
@@ -1656,6 +1657,7 @@ int StreamTcpReassembleHandleSegmentUpdateACK (TcpReassemblyThreadCtx *ra_ctx,
 
                 StreamTcpSetupMsg(ssn, stream, p, smsg);
             }
+            smsg->data.seq = ra_base_seq;
 
             /* handle segments partly before ra_base_seq */
             if (SEQ_GT(ra_base_seq, seg->seq)) {
@@ -1764,6 +1766,7 @@ int StreamTcpReassembleHandleSegmentUpdateACK (TcpReassemblyThreadCtx *ra_ctx,
                     smsg_offset = 0;
 
                     StreamTcpSetupMsg(ssn, stream,p,smsg);
+                    smsg->data.seq = ra_base_seq;
 
                     copy_size = sizeof(smsg->data.data) - smsg_offset;
                     if (copy_size > (seg->payload_len - payload_offset)) {
@@ -1906,7 +1909,7 @@ int StreamTcpReassembleProcessAppLayer(TcpReassemblyThreadCtx *ra_ctx)
         do {
             smsg = StreamMsgGetFromQueue(ra_ctx->stream_q);
             if (smsg != NULL) {
-                SCLogDebug("smsg %p, next %p, prev %p, flow %p", smsg, smsg->next, smsg->prev, smsg->flow);
+                SCLogDebug("smsg %p, next %p, prev %p, flow %p, q->len %u", smsg, smsg->next, smsg->prev, smsg->flow, ra_ctx->stream_q->len);
 
                 BUG_ON(smsg->flow == NULL);
 
@@ -1959,6 +1962,8 @@ int StreamTcpReassembleHandleSegment(TcpReassemblyThreadCtx *ra_ctx,
             SCLogDebug("StreamTcpReassembleHandleSegmentHandleData error");
             SCReturnInt(-1);
         }
+
+        p->flags |= PKT_STREAM_ADD;
     }
 
     SCReturnInt(0);

src/stream.c

@@ -67,6 +67,8 @@ void StreamMsgFree(void *ptr) {
 }
 
 static void StreamMsgEnqueue (StreamMsgQueue *q, StreamMsg *s) {
+    SCEnter();
+    SCLogDebug("s %p", s);
     /* more packets in queue */
     if (q->top != NULL) {
         s->next = q->top;
@@ -82,13 +84,16 @@ static void StreamMsgEnqueue (StreamMsgQueue *q, StreamMsg *s) {
     if (q->len > q->dbg_maxlen)
         q->dbg_maxlen = q->len;
 #endif /* DBG_PERF */
+    SCReturn;
 }
 
 static StreamMsg *StreamMsgDequeue (StreamMsgQueue *q) {
+    SCEnter();
+
     /* if the queue is empty there are no packets left.
      * In that case we sleep and try again. */
     if (q->len == 0) {
-        return NULL;
+        SCReturnPtr(NULL, "StreamMsg");
     }
 
     /* pull the bottom packet from the queue */
@@ -107,7 +112,7 @@ static StreamMsg *StreamMsgDequeue (StreamMsgQueue *q) {
 
     s->next = NULL;
     s->prev = NULL;
-    return s;
+    SCReturnPtr(s, "StreamMsg");
 }
 
 /* Used by stream reassembler to get msgs */
@@ -121,6 +126,7 @@ StreamMsg *StreamMsgGetFromPool(void)
 
 /* Used by l7inspection to return msgs to pool */
 void StreamMsgReturnToPool(StreamMsg *s) {
+    SCLogDebug("s %p", s);
     SCMutexLock(&stream_msg_pool_mutex);
     PoolReturn(stream_msg_pool, (void *)s);
     SCMutexUnlock(&stream_msg_pool_mutex);

src/stream.h

@@ -38,17 +38,18 @@
 #define STREAMQUEUE_FLAG_INIT    0x01
 
 typedef struct StreamMsg_ {
-    uint32_t id; /* unique stream id */
+    uint32_t id;    /**< unique stream id */
-    uint8_t flags; /* msg flags */
+    uint8_t flags;  /**< msg flags */
-    Flow *flow; /* parent flow */
+    Flow *flow;     /**< parent flow */
 
     union {
-        /* case STREAM_START */
+        /* case !STREAM_EOF && !STREAM_GAP */
         struct {
-            Address src_ip, dst_ip;
+            Address src_ip, dst_ip;     /**< ipaddresses */
-            Port src_port, dst_port;
+            Port src_port, dst_port;    /**< ports */
-            uint8_t data[MSG_DATA_SIZE];
+            uint8_t data[MSG_DATA_SIZE];/**< reassembled data */
-            uint16_t data_len;
+            uint16_t data_len;          /**< length of the data */
+            uint32_t seq;               /**< sequence number */
         } data;
         /* case STREAM_GAP */
         struct {