detect: do not run tx detection on non established packets

Follows commit 2fb5059

Ticket: 6775

src/detect.c

@@ -146,6 +146,9 @@ static void DetectRun(ThreadVars *th_v,
     /* run tx/state inspection. Don't call for ICMP error msgs. */
     if (pflow && pflow->alstate && likely(pflow->proto == p->proto)) {
         if (p->proto == IPPROTO_TCP) {
+            if ((p->flags & PKT_STREAM_EST) == 0) {
+                goto end;
+            }
             const TcpSession *ssn = p->flow->protoctx;
             if (ssn && (ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED) == 0) {
                 // PACKET_PROFILING_DETECT_START(p, PROF_DETECT_TX);

src/flow-worker.c

@@ -619,7 +619,7 @@ static TmEcode FlowWorker(ThreadVars *tv, Packet *p, void *data)
             if (p->proto == IPPROTO_TCP) {
                 StreamTcpSessionCleanup(p->flow->protoctx);
             }
-        } else if (p->proto == IPPROTO_TCP && p->flow->protoctx) {
+        } else if (p->proto == IPPROTO_TCP && p->flow->protoctx && p->flags & PKT_STREAM_EST) {
             FramesPrune(p->flow, p);
             FLOWWORKER_PROFILING_START(p, PROFILE_FLOWWORKER_TCPPRUNE);
             StreamTcpPruneSession(p->flow, p->flowflags & FLOW_PKT_TOSERVER ?
@@ -631,18 +631,19 @@ static TmEcode FlowWorker(ThreadVars *tv, Packet *p, void *data)
 
         if ((PKT_IS_PSEUDOPKT(p)) ||
                 (p->flow->flags & (FLOW_TS_APP_UPDATED | FLOW_TC_APP_UPDATED))) {
-            if (PKT_IS_TOSERVER(p)) {
-                if (PKT_IS_PSEUDOPKT(p) || (p->flow->flags & (FLOW_TS_APP_UPDATED))) {
-                    AppLayerParserTransactionsCleanup(p->flow, STREAM_TOSERVER);
-                    p->flow->flags &= ~FLOW_TS_APP_UPDATED;
-                }
-            } else {
-                if (PKT_IS_PSEUDOPKT(p) || (p->flow->flags & (FLOW_TC_APP_UPDATED))) {
-                    AppLayerParserTransactionsCleanup(p->flow, STREAM_TOCLIENT);
-                    p->flow->flags &= ~FLOW_TC_APP_UPDATED;
+            if ((p->flags & PKT_STREAM_EST) || p->proto != IPPROTO_TCP) {
+                if (PKT_IS_TOSERVER(p)) {
+                    if (PKT_IS_PSEUDOPKT(p) || (p->flow->flags & (FLOW_TS_APP_UPDATED))) {
+                        AppLayerParserTransactionsCleanup(p->flow, STREAM_TOSERVER);
+                        p->flow->flags &= ~FLOW_TS_APP_UPDATED;
+                    }
+                } else {
+                    if (PKT_IS_PSEUDOPKT(p) || (p->flow->flags & (FLOW_TC_APP_UPDATED))) {
+                        AppLayerParserTransactionsCleanup(p->flow, STREAM_TOCLIENT);
+                        p->flow->flags &= ~FLOW_TC_APP_UPDATED;
+                    }
                 }
             }
-
         } else {
             SCLogDebug("not pseudo, no app update: skip");
         }

src/output-tx.c

@@ -341,6 +341,9 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data)
         SCLogDebug("not pseudo, no app update: skip");
         return TM_ECODE_OK;
     }
+    if ((p->flags & PKT_STREAM_EST) == 0 && p->proto == IPPROTO_TCP) {
+        return TM_ECODE_OK;
+    }
     SCLogDebug("pseudo, or app update: run output");
 
     OutputTxLoggerThreadData *op_thread_data = (OutputTxLoggerThreadData *)thread_data;