flow/output: log triggered exception policies

To accompany the Exception Policy stats, also add information about any
Exception Policy triggered and for which target to the flow log event.

Task #6215

(cherry picked from commit 08e928988f)

doc/userguide/configuration/exception-policies.rst

@@ -220,6 +220,54 @@ Notes:
 Log Output
 **********
 
+.. _eps_flow_event:
+
+Flow Event
+==========
+
+When an Exception Policy is triggered, this will be indicated in the flow log
+event for the associated flow, also indicating which target triggered that, and
+what policy was applied. If no exception policy is triggered, that field won't
+be present in the logs.
+
+Note that this is true even if the policy is applied only to certain packets from
+a flow.
+
+.. Note:: As exception policies targetting ``flow memcap`` and ``defrag
+      memcap`` act on a per-packet level, these are not shown in the flow logs.
+      Ticket #7884 (https://redmine.openinfosecfoundation.org/issues/7884)
+      tracks work on this.
+      For such cases, the exception policy stats counters are available.
+
+In the log sample below, the flow triggered the ``midstream policy``, leading
+to Suricata applying the behavior that had been configured for such scenario:
+*to pass the flow* (``pass_flow``). It also did trigger the ``app_layer_error``
+exception policy, but that is set up to ``ignore``::
+
+  "flow": {
+    "pkts_toserver": 4,
+    "pkts_toclient": 5,
+    "bytes_toserver": 495,
+    "bytes_toclient": 351,
+    "start": "2016-07-13T22:42:07.199672+0000",
+    "end": "2016-07-13T22:42:07.573174+0000",
+    "age": 0,
+    "state": "new",
+    "reason": "shutdown",
+    "alerted": false,
+    "action": "pass",
+    "exception_policy": [
+       {
+         "target": "stream_midstream",
+         "policy": "pass_flow"
+       },
+       {
+         "target": "app_layer_error",
+         "policy": "ignore"
+       }
+     ]
+  }
+
 .. _eps_stats:
 
 Available Stats

doc/userguide/output/eve/eve-json-format.rst

@@ -314,7 +314,13 @@ Event type: ``flow``::
         "age": 40,
         "state": "closed",
         "reason": "shutdown",
-        "alerted": true
+        "alerted": true,
+        "exception_policy": [
+          {
+            "target": "stream_midstream",
+            "policy": "ignore"
+          }
+        ]
       },
       "ether": {
         "dest_macs": [
@@ -1706,6 +1712,14 @@ Fields
 * "state": display state of the flow (include "new", "established", "closed", "bypassed")
 * "reason": mechanism that did trigger the end of the flow (include "timeout", "forced" and "shutdown")
 * "alerted": "true" or "false" depending if an alert has been seen on flow
+* "action": "pass" or "drop" depending if flow was PASS'ed or DROP'ed (no present if none)
+* "exception_policy": array consisting of exception policies that have been triggered by
+  the flow:
+
+    * "target": if an exception policy was triggered, what setting exceptions
+      led to this (cf. :ref:`Exception Policy - Specific Settings<eps_settings>`).
+    * "policy": if an exception policy was triggered, what policy was applied
+      (to the flow or to any packet(s) from it).
 
 Example ::
 
@@ -1726,7 +1740,14 @@ Example ::
     "bypass": "capture",
     "state": "bypassed",
     "reason": "timeout",
-    "alerted": false
+    "alerted": false,
+    "action": "pass",
+    "exception_policy": [
+      {
+        "target": "stream_midstream",
+        "policy": "pass_flow"
+      }
+    ]
   }
 
 Event type: RDP

etc/schema.json

@@ -1567,6 +1567,20 @@
                 "end": {
                     "type": "string"
                 },
+                "exception_policy": {
+                    "description": "The exception policy(ies) triggered by the flow. Not logged if none was triggered",
+                    "type": "array",
+                    "properties": {
+                        "target": {
+                            "description": "What triggered the exception",
+                            "type": "string"
+                        },
+                        "policy": {
+                            "description": "Which exception policy was applied",
+                            "type": "string"
+                        }
+                    }
+                },
                 "pkts_toclient": {
                     "type": "integer"
                 },

src/app-layer-parser.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2021 Open Information Security Foundation
+/* Copyright (C) 2007-2025 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -163,6 +163,11 @@ static void AppLayerConfig(void)
     g_applayerparser_error_policy = ExceptionPolicyParse("app-layer.error-policy", true);
 }
 
+enum ExceptionPolicy AppLayerErrorGetExceptionPolicy(void)
+{
+    return g_applayerparser_error_policy;
+}
+
 static void AppLayerParserFramesFreeContainer(FramesContainer *frames)
 {
     if (frames != NULL) {

src/app-layer-parser.h

@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2020 Open Information Security Foundation
+/* Copyright (C) 2007-2025 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -137,6 +137,8 @@ void AppLayerParserThreadCtxFree(AppLayerParserThreadCtx *tctx);
 int AppLayerParserConfParserEnabled(const char *ipproto,
                                     const char *alproto_name);
 
+enum ExceptionPolicy AppLayerErrorGetExceptionPolicy(void);
+
 /** \brief Prototype for parsing functions */
 typedef AppLayerResult (*AppLayerParserFPtr)(Flow *f, void *protocol_state,
         AppLayerParserState *pstate, StreamSlice stream_slice, void *local_storage);

src/flow.h

@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2024 Open Information Security Foundation
+/* Copyright (C) 2007-2025 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -467,6 +467,9 @@ typedef struct Flow_
     uint8_t min_ttl_toclient;
     uint8_t max_ttl_toclient;
 
+    /** which exception policies were applied, if any */
+    uint8_t applied_exception_policy;
+
     /** application level storage ptrs.
      *
      */

src/output-json-flow.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2020 Open Information Security Foundation
+/* Copyright (C) 2007-2025 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -50,6 +50,7 @@
 #include "stream-tcp.h"
 #include "stream-tcp-private.h"
 #include "flow-storage.h"
+#include "util-exception-policy.h"
 
 static JsonBuilder *CreateEveHeaderFromFlow(const Flow *f)
 {
@@ -214,6 +215,65 @@ void EveAddFlow(Flow *f, JsonBuilder *js)
     jb_set_string(js, "start", timebuf1);
 }
 
+static void EveExceptionPolicyLog(JsonBuilder *js, uint16_t flag)
+{
+    if (flag & EXCEPTION_TARGET_FLAG_DEFRAG_MEMCAP) {
+        jb_start_object(js);
+        jb_set_string(js, "target",
+                ExceptionPolicyTargetFlagToString(EXCEPTION_TARGET_FLAG_DEFRAG_MEMCAP));
+        jb_set_string(js, "policy",
+                ExceptionPolicyEnumToString(
+                        ExceptionPolicyTargetPolicy(EXCEPTION_TARGET_FLAG_DEFRAG_MEMCAP), true));
+        jb_close(js);
+    }
+    if (flag & EXCEPTION_TARGET_FLAG_SESSION_MEMCAP) {
+        jb_start_object(js);
+        jb_set_string(js, "target",
+                ExceptionPolicyTargetFlagToString(EXCEPTION_TARGET_FLAG_SESSION_MEMCAP));
+        jb_set_string(js, "policy",
+                ExceptionPolicyEnumToString(
+                        ExceptionPolicyTargetPolicy(EXCEPTION_TARGET_FLAG_SESSION_MEMCAP), true));
+        jb_close(js);
+    }
+    if (flag & EXCEPTION_TARGET_FLAG_REASSEMBLY_MEMCAP) {
+        jb_start_object(js);
+        jb_set_string(js, "target",
+                ExceptionPolicyTargetFlagToString(EXCEPTION_TARGET_FLAG_REASSEMBLY_MEMCAP));
+        jb_set_string(js, "policy",
+                ExceptionPolicyEnumToString(
+                        ExceptionPolicyTargetPolicy(EXCEPTION_TARGET_FLAG_REASSEMBLY_MEMCAP),
+                        true));
+        jb_close(js);
+    }
+    if (flag & EXCEPTION_TARGET_FLAG_FLOW_MEMCAP) {
+        jb_start_object(js);
+        jb_set_string(
+                js, "target", ExceptionPolicyTargetFlagToString(EXCEPTION_TARGET_FLAG_FLOW_MEMCAP));
+        jb_set_string(js, "policy",
+                ExceptionPolicyEnumToString(
+                        ExceptionPolicyTargetPolicy(EXCEPTION_TARGET_FLAG_FLOW_MEMCAP), true));
+        jb_close(js);
+    }
+    if (flag & EXCEPTION_TARGET_FLAG_MIDSTREAM) {
+        jb_start_object(js);
+        jb_set_string(
+                js, "target", ExceptionPolicyTargetFlagToString(EXCEPTION_TARGET_FLAG_MIDSTREAM));
+        jb_set_string(js, "policy",
+                ExceptionPolicyEnumToString(
+                        ExceptionPolicyTargetPolicy(EXCEPTION_TARGET_FLAG_MIDSTREAM), true));
+        jb_close(js);
+    }
+    if (flag & EXCEPTION_TARGET_FLAG_APPLAYER_ERROR) {
+        jb_start_object(js);
+        jb_set_string(js, "target",
+                ExceptionPolicyTargetFlagToString(EXCEPTION_TARGET_FLAG_APPLAYER_ERROR));
+        jb_set_string(js, "policy",
+                ExceptionPolicyEnumToString(
+                        ExceptionPolicyTargetPolicy(EXCEPTION_TARGET_FLAG_APPLAYER_ERROR), true));
+        jb_close(js);
+    }
+}
+
 /* Eve format logging */
 static void EveFlowLogJSON(OutputJsonThreadCtx *aft, JsonBuilder *jb, Flow *f)
 {
@@ -277,6 +337,11 @@ static void EveFlowLogJSON(OutputJsonThreadCtx *aft, JsonBuilder *jb, Flow *f)
     } else if (f->flags & FLOW_ACTION_PASS) {
         JB_SET_STRING(jb, "action", "pass");
     }
+    if (f->applied_exception_policy != 0) {
+        jb_open_array(jb, "exception_policy");
+        EveExceptionPolicyLog(jb, f->applied_exception_policy);
+        jb_close(jb); /* close array */
+    }
 
     /* Close flow. */
     jb_close(jb);

src/stream-tcp.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2024 Open Information Security Foundation
+/* Copyright (C) 2007-2025 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -900,6 +900,21 @@ static void StreamTcpSsnMemcapExceptionPolicyStatsIncr(
     }
 }
 
+enum ExceptionPolicy StreamTcpSsnMemcapGetExceptionPolicy(void)
+{
+    return stream_config.ssn_memcap_policy;
+}
+
+enum ExceptionPolicy StreamTcpReassemblyMemcapGetExceptionPolicy(void)
+{
+    return stream_config.reassembly_memcap_policy;
+}
+
+enum ExceptionPolicy StreamMidstreamGetExceptionPolicy(void)
+{
+    return stream_config.midstream_policy;
+}
+
 /** \internal
  *  \brief The function is used to fetch a TCP session from the
  *         ssn_pool, when a TCP SYN is received.

src/stream-tcp.h

@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2024 Open Information Security Foundation
+/* Copyright (C) 2007-2025 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -161,6 +161,10 @@ void StreamTcpDetectLogFlush(ThreadVars *tv, StreamTcpThread *stt, Flow *f, Pack
 const char *StreamTcpStateAsString(const enum TcpState);
 const char *StreamTcpSsnStateAsString(const TcpSession *ssn);
 
+enum ExceptionPolicy StreamTcpSsnMemcapGetExceptionPolicy(void);
+enum ExceptionPolicy StreamTcpReassemblyMemcapGetExceptionPolicy(void);
+enum ExceptionPolicy StreamMidstreamGetExceptionPolicy(void);
+
 /** ------- Inline functions: ------ */
 
 /**

src/util-exception-policy-types.h

@@ -1,4 +1,4 @@
-/* Copyright (C) 2024 Open Information Security Foundation
+/* Copyright (C) 2024-2025 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -40,6 +40,14 @@ enum ExceptionPolicy {
  * "tcp.reassembly_exception_policy.drop_packet" + 1 */
 #define EXCEPTION_POLICY_COUNTER_MAX_LEN 45
 
+/** Flags for possible scenario/ config settings for exception policies */
+#define EXCEPTION_TARGET_FLAG_DEFRAG_MEMCAP     BIT_U8(0)
+#define EXCEPTION_TARGET_FLAG_SESSION_MEMCAP    BIT_U8(1)
+#define EXCEPTION_TARGET_FLAG_REASSEMBLY_MEMCAP BIT_U8(2)
+#define EXCEPTION_TARGET_FLAG_FLOW_MEMCAP       BIT_U8(3)
+#define EXCEPTION_TARGET_FLAG_MIDSTREAM         BIT_U8(4)
+#define EXCEPTION_TARGET_FLAG_APPLAYER_ERROR    BIT_U8(5)
+
 typedef struct ExceptionPolicyCounters_ {
     /* Follows enum order */
     uint16_t eps_id[EXCEPTION_POLICY_MAX];

src/util-exception-policy.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2022-2024 Open Information Security Foundation
+/* Copyright (C) 2022-2025 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -19,13 +19,18 @@
  * \file
  */
 
+#include "util-exception-policy.h"
 #include "suricata-common.h"
 #include "suricata.h"
 #include "packet.h"
-#include "util-exception-policy.h"
 #include "util-misc.h"
 #include "stream-tcp-reassemble.h"
 #include "action-globals.h"
+#include "conf.h"
+#include "flow.h"
+#include "stream-tcp.h"
+#include "defrag-hash.h"
+#include "app-layer-parser.h"
 
 enum ExceptionPolicy g_eps_master_switch = EXCEPTION_POLICY_NOT_SET;
 /** true if exception policy was defined in config */
@@ -66,9 +71,75 @@ static enum ExceptionPolicy GetMasterExceptionPolicy(const char *option)
     return g_eps_master_switch;
 }
 
+static uint8_t ExceptionPolicyFlag(enum PacketDropReason drop_reason)
+{
+    switch (drop_reason) {
+        case PKT_DROP_REASON_DEFRAG_MEMCAP:
+            return EXCEPTION_TARGET_FLAG_DEFRAG_MEMCAP;
+        case PKT_DROP_REASON_STREAM_MEMCAP:
+            return EXCEPTION_TARGET_FLAG_SESSION_MEMCAP;
+        case PKT_DROP_REASON_STREAM_REASSEMBLY:
+            return EXCEPTION_TARGET_FLAG_REASSEMBLY_MEMCAP;
+        case PKT_DROP_REASON_FLOW_MEMCAP:
+            return EXCEPTION_TARGET_FLAG_FLOW_MEMCAP;
+        case PKT_DROP_REASON_STREAM_MIDSTREAM:
+            return EXCEPTION_TARGET_FLAG_MIDSTREAM;
+        case PKT_DROP_REASON_APPLAYER_ERROR:
+            return EXCEPTION_TARGET_FLAG_APPLAYER_ERROR;
+        default:
+            return 0;
+    }
+
+    return 0;
+}
+
+const char *ExceptionPolicyTargetFlagToString(uint8_t target_flag)
+{
+    switch (target_flag) {
+        case EXCEPTION_TARGET_FLAG_DEFRAG_MEMCAP:
+            return "defrag_memcap";
+        case EXCEPTION_TARGET_FLAG_SESSION_MEMCAP:
+            return "stream_memcap";
+        case EXCEPTION_TARGET_FLAG_REASSEMBLY_MEMCAP:
+            return "stream_reassembly_memcap";
+        case EXCEPTION_TARGET_FLAG_FLOW_MEMCAP:
+            return "flow_memcap";
+        case EXCEPTION_TARGET_FLAG_MIDSTREAM:
+            return "stream_midstream";
+        case EXCEPTION_TARGET_FLAG_APPLAYER_ERROR:
+            return "app_layer_error";
+        default:
+            return "none";
+    }
+}
+
+enum ExceptionPolicy ExceptionPolicyTargetPolicy(uint8_t target_flag)
+{
+    switch (target_flag) {
+        case EXCEPTION_TARGET_FLAG_DEFRAG_MEMCAP:
+            return DefragGetMemcapExceptionPolicy();
+        case EXCEPTION_TARGET_FLAG_SESSION_MEMCAP:
+            return StreamTcpSsnMemcapGetExceptionPolicy();
+        case EXCEPTION_TARGET_FLAG_REASSEMBLY_MEMCAP:
+            return StreamTcpReassemblyMemcapGetExceptionPolicy();
+        case EXCEPTION_TARGET_FLAG_FLOW_MEMCAP:
+            return FlowGetMemcapExceptionPolicy();
+        case EXCEPTION_TARGET_FLAG_MIDSTREAM:
+            return StreamMidstreamGetExceptionPolicy();
+        case EXCEPTION_TARGET_FLAG_APPLAYER_ERROR:
+            return AppLayerErrorGetExceptionPolicy();
+        default:
+            return EXCEPTION_POLICY_NOT_SET;
+    }
+    return EXCEPTION_POLICY_NOT_SET;
+}
+
 void ExceptionPolicyApply(Packet *p, enum ExceptionPolicy policy, enum PacketDropReason drop_reason)
 {
     SCLogDebug("start: pcap_cnt %" PRIu64 ", policy %u", p->pcap_cnt, policy);
+    if (p->flow) {
+        p->flow->applied_exception_policy |= ExceptionPolicyFlag(drop_reason);
+    }
     switch (policy) {
         case EXCEPTION_POLICY_AUTO:
             break;

src/util-exception-policy.h

@@ -1,4 +1,4 @@
-/* Copyright (C) 2022-2024 Open Information Security Foundation
+/* Copyright (C) 2022-2025 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -26,6 +26,8 @@
 #include "util-exception-policy-types.h"
 
 const char *ExceptionPolicyEnumToString(enum ExceptionPolicy policy, bool is_json);
+const char *ExceptionPolicyTargetFlagToString(uint8_t target_flag);
+enum ExceptionPolicy ExceptionPolicyTargetPolicy(uint8_t target_flag);
 void SetMasterExceptionPolicy(void);
 void ExceptionPolicyApply(
         Packet *p, enum ExceptionPolicy policy, enum PacketDropReason drop_reason);