From 42205006d1035c85aea43b5a76e2bb0ad0fc08dd Mon Sep 17 00:00:00 2001
From: Victor Julien <victor@inliniac.net>
Date: Fri, 3 Jul 2020 14:42:48 +0200
Subject: [PATCH] flow/timeout: flag last pseudo packet

Flag the last flow timeout pseudo packet so that we can force
TX logging w/o setting both app-layer flags.

Case this fixes:

1. flow times out when only TS TCP data received, but non of it is ACK'd.
   So there is no app-layer proto yet, or app state or Flow::alparser. So
   EOF flags can't be set.

2. Flow timeout sees no reason to create pseudo packet in TC direction.

3. TS pseudo packet finds HTTP, creates HTTP state, flag EOF TS.

4. TX logging skips HTTP logging because:
   - TC progress not reached
   - EOF TC flag not set.

The solution has been to flag the very last packet for the flow as such
and use it has a master-EOF flag.
---
 src/flow-timeout.c |  4 ++++
 src/flow.h         |  3 +++
 src/output-tx.c    | 11 ++++++++---
 3 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/src/flow-timeout.c b/src/flow-timeout.c
index 335ec9bf3d..8659a7402f 100644
--- a/src/flow-timeout.c
+++ b/src/flow-timeout.c
@@ -380,6 +380,9 @@ int FlowForceReassemblyForFlow(Flow *f, int server, int client)
                 goto done;
             }
             PKT_SET_SRC(p2, PKT_SRC_FFR);
+            p2->flowflags |= FLOW_PKT_LAST_PSEUDO;
+        } else {
+            p1->flowflags |= FLOW_PKT_LAST_PSEUDO;
         }
     } else {
         if (server == STREAM_HAS_UNPROCESSED_SEGMENTS_NEED_ONLY_DETECTION) {
@@ -388,6 +391,7 @@ int FlowForceReassemblyForFlow(Flow *f, int server, int client)
                 goto done;
             }
             PKT_SET_SRC(p1, PKT_SRC_FFR);
+            p1->flowflags |= FLOW_PKT_LAST_PSEUDO;
         } else {
             /* impossible */
             BUG_ON(1);
diff --git a/src/flow.h b/src/flow.h
index e362f8d40d..f625ada106 100644
--- a/src/flow.h
+++ b/src/flow.h
@@ -222,6 +222,9 @@ typedef struct AppLayerParserState_ AppLayerParserState;
 #define FLOW_PKT_TOCLIENT_IPONLY_SET    0x10
 #define FLOW_PKT_TOSERVER_FIRST         0x20
 #define FLOW_PKT_TOCLIENT_FIRST         0x40
+/** last pseudo packet in the flow. Can be used to trigger final clean,
+ *  logging, etc. */
+#define FLOW_PKT_LAST_PSEUDO            0x80
 
 #define FLOW_END_FLAG_STATE_NEW         0x01
 #define FLOW_END_FLAG_STATE_ESTABLISHED 0x02
diff --git a/src/output-tx.c b/src/output-tx.c
index 4294d40f4c..ff11b87944 100644
--- a/src/output-tx.c
+++ b/src/output-tx.c
@@ -260,15 +260,20 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data)
             if ((tx_logged_old & (1<<logger->logger_id)) == 0) {
                 SCLogDebug("alproto match %d, logging tx_id %"PRIu64, logger->alproto, tx_id);
 
+                const bool last_pseudo = (p->flowflags & FLOW_PKT_LAST_PSEUDO) != 0;
                 const bool ts_eof = AppLayerParserStateIssetFlag(f->alparser,
                         APP_LAYER_PARSER_EOF_TS) != 0;
                 const bool tc_eof = AppLayerParserStateIssetFlag(f->alparser,
                         APP_LAYER_PARSER_EOF_TC) != 0;
-                SCLogDebug("pcap_cnt %"PRIu64", tx_id %"PRIu64" logger %d. EOFs TS %s TC %s",
+                SCLogDebug("pcap_cnt %"PRIu64", tx_id %"PRIu64" logger %d. "
+                        "EOFs TS %s TC %s LAST PSEUDO %s",
                         p->pcap_cnt, tx_id, logger->logger_id,
-                        ts_eof ? "true" : "false", tc_eof ? "true" : "false");
+                        ts_eof ? "true" : "false", tc_eof ? "true" : "false",
+                        last_pseudo ? "true" : "false");
 
-                if (!(ts_eof && tc_eof)) {
+                if ((ts_eof && tc_eof) || last_pseudo) {
+                    SCLogDebug("EOF, so log now");
+                } else {
                     if (logger->LogCondition) {
                         int r = logger->LogCondition(tv, p, alstate, tx, tx_id);
                         if (r == FALSE) {