Fix the pseudo packet having the wrong proto set, causing massive fp's. Flag packets to be part of the established phase of a tcp session, so we won't prematurely inspect the app layer state.

src/decode.h

@@ -281,7 +281,7 @@ typedef struct Packet_
     uint8_t recursion_level;
 
     /* Pkt Flags */
-    uint8_t flags;
+    uint16_t flags;
     /* flow */
     uint8_t flowflags;
     struct Flow_ *flow;
@@ -736,14 +736,15 @@ void AddressDebugPrint(Address *);
 #define VLAN_OVER_GRE       13
 
 /*Packet Flags*/
-#define PKT_NOPACKET_INSPECTION         0x01    /**< Flag to indicate that packet header or contents should not be inspected*/
+#define PKT_NOPACKET_INSPECTION         0x0001    /**< Flag to indicate that packet header or contents should not be inspected*/
-#define PKT_NOPAYLOAD_INSPECTION        0x02    /**< Flag to indicate that packet contents should not be inspected*/
+#define PKT_NOPAYLOAD_INSPECTION        0x0002    /**< Flag to indicate that packet contents should not be inspected*/
-#define PKT_ALLOC                       0x04    /**< Packet was alloc'd this run, needs to be freed */
+#define PKT_ALLOC                       0x0004    /**< Packet was alloc'd this run, needs to be freed */
-#define PKT_HAS_TAG                     0x08    /**< Packet has matched a tag */
+#define PKT_HAS_TAG                     0x0008    /**< Packet has matched a tag */
-#define PKT_STREAM_ADD                  0x10    /**< Packet payload was added to reassembled stream */
+#define PKT_STREAM_ADD                  0x0010    /**< Packet payload was added to reassembled stream */
-#define PKT_STREAM_EOF                  0x20    /**< Stream is in eof state */
+#define PKT_STREAM_EST                  0x0020    /**< Packet is part of establised stream */
-#define PKT_HAS_FLOW                    0x40
+#define PKT_STREAM_EOF                  0x0040    /**< Stream is in eof state */
-#define PKT_PSEUDO_STREAM_END           0x80    /**< Pseudo packet to end the stream */
+#define PKT_HAS_FLOW                    0x0080
+#define PKT_PSEUDO_STREAM_END           0x0100    /**< Pseudo packet to end the stream */
 
 #endif /* __DECODE_H__ */
 

src/detect-engine-state.c

@@ -488,7 +488,6 @@ int DeStateDetectStartDetection(ThreadVars *tv, DetectEngineCtx *de_ctx,
         DeStateSignatureAppend(f->de_state, s, sm, umatch, dmatch, hcbdmatch,
                                hhdmatch, hrhdmatch, hmdmatch, hcdmatch);
     }
-
     SCMutexUnlock(&f->de_state_m);
 
     SCReturnInt(r);

src/detect.c

@@ -1146,12 +1146,14 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
 
             /* Retrieve the app layer state and protocol and the tcp reassembled
              * stream chunks. */
-            if (p->flowflags & FLOW_PKT_ESTABLISHED) {
+            if ((IP_GET_IPPROTO(p) == IPPROTO_TCP && p->flags & PKT_STREAM_EST) ||
+                (IP_GET_IPPROTO(p) == IPPROTO_UDP && p->flowflags & FLOW_PKT_ESTABLISHED))
+            {
                 alstate = AppLayerGetProtoStateFromPacket(p);
                 alproto = AppLayerGetProtoFromPacket(p);
                 SCLogDebug("alstate %p, alproto %u", alstate, alproto);
             } else {
-                SCLogDebug("packet doesn't have established flag set");
+                SCLogDebug("packet doesn't have established flag set (proto %d)", IP_GET_IPPROTO(p));
             }
         }
         SCMutexUnlock(&p->flow->m);
@@ -1844,7 +1846,12 @@ PacketCreateMask(Packet *p, SignatureMask *mask, uint16_t alproto, void *alstate
                     SCLogDebug("packet/flow has dce state");
                     (*mask) |= SIG_MASK_REQUIRE_DCE_STATE;
                     break;
+                default:
+                    SCLogDebug("packet/flow has other state");
+                    break;
             }
+        } else {
+            SCLogDebug("no alstate");
         }
     }
 }

src/stream-tcp.c

@@ -53,6 +53,8 @@
 #include "stream-tcp.h"
 
 #include "app-layer-parser.h"
+#include "app-layer-protos.h"
+
 #include "util-host-os-info.h"
 #include "util-privs.h"
 
@@ -2758,7 +2760,9 @@ static int StreamTcpPacket (ThreadVars *tv, Packet *p, StreamTcpThread *stt,
                 break;
         }
 
-        if (ssn->state > TCP_ESTABLISHED) {
+        if (ssn->state == TCP_ESTABLISHED) {
+            p->flags |= PKT_STREAM_EST;
+        } else if (ssn->state > TCP_ESTABLISHED) {
             p->flags |= PKT_STREAM_EOF;
         }
     }
@@ -3297,6 +3301,7 @@ Packet *StreamTcpPseudoSetup(Packet *parent, uint8_t *pkt, uint32_t len)
 
     /* copy packet and set lenght, proto */
     p->tunnel_proto = parent->proto;
+    p->proto = parent->proto;
     p->pktlen = len;
     memcpy(&p->pkt, pkt, (len - parent->payload_len));
     p->recursion_level = parent->recursion_level + 1;
@@ -3398,12 +3403,13 @@ void StreamTcpPseudoPacketCreateStreamEndPacket(Packet *p, TcpSession *ssn, Pack
     /* Setup the IP and TCP headers */
     StreamTcpPseudoPacketSetupHeader(np,p);
 
+    np->flowflags = p->flowflags;
+
+    np->flags |= PKT_STREAM_EST;
     np->flags |= PKT_STREAM_EOF;
     np->flags |= PKT_HAS_FLOW;
     np->flags |= PKT_PSEUDO_STREAM_END;
 
-    np->flowflags = p->flowflags;
-
     if (PKT_IS_TOSERVER(p)) {
         SCLogDebug("original is to_server, so pseudo is to_client");
         np->flowflags &= ~FLOW_PKT_TOSERVER;