frames: support UDP frames

UDP frames point to the UDP packet payloads.

The frames are removed after each packet.

Ticket: #4983.

src/app-layer-frames.c

@@ -391,20 +391,30 @@ static void FrameFreeSingleFrame(Frames *frames, Frame *r)
     FrameClean(r);
 }
 
-void FramesFree(Frames *frames)
+static void FramesClear(Frames *frames)
 {
     BUG_ON(frames == NULL);
 
+    SCLogDebug("frames %u", frames->cnt);
     for (uint16_t i = 0; i < frames->cnt; i++) {
         if (i < FRAMES_STATIC_CNT) {
             Frame *r = &frames->sframes[i];
+            SCLogDebug("removing frame %p", r);
             FrameFreeSingleFrame(frames, r);
         } else {
             const uint16_t o = i - FRAMES_STATIC_CNT;
             Frame *r = &frames->dframes[o];
+            SCLogDebug("removing frame %p", r);
             FrameFreeSingleFrame(frames, r);
         }
     }
+    frames->cnt = 0;
+}
+
+void FramesFree(Frames *frames)
+{
+    BUG_ON(frames == NULL);
+    FramesClear(frames);
     SCFree(frames->dframes);
     frames->dframes = NULL;
 }
@@ -419,7 +429,7 @@ Frame *AppLayerFrameNewByPointer(Flow *f, const StreamSlice *stream_slice,
 
     /* workarounds for many (unit|fuzz)tests not handling TCP data properly */
 #if defined(UNITTESTS) || defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION)
-    if (f->protoctx == NULL)
+    if (f->proto == IPPROTO_TCP && f->protoctx == NULL)
         return NULL;
     if (frame_start < stream_slice->input ||
             frame_start >= stream_slice->input + stream_slice->input_len)
@@ -427,8 +437,7 @@ Frame *AppLayerFrameNewByPointer(Flow *f, const StreamSlice *stream_slice,
 #endif
     BUG_ON(frame_start < stream_slice->input);
     BUG_ON(stream_slice->input == NULL);
-    BUG_ON(f->proto != IPPROTO_TCP);
-    BUG_ON(f->protoctx == NULL);
+    BUG_ON(f->proto == IPPROTO_TCP && f->protoctx == NULL);
 
     ptrdiff_t ptr_offset = frame_start - stream_slice->input;
 #ifdef DEBUG
@@ -464,6 +473,29 @@ Frame *AppLayerFrameNewByPointer(Flow *f, const StreamSlice *stream_slice,
     return r;
 }
 
+static Frame *AppLayerFrameUdp(Flow *f, const StreamSlice *stream_slice,
+        const uint32_t frame_start_rel, const int64_t len, int dir, uint8_t frame_type)
+{
+    BUG_ON(f->proto != IPPROTO_UDP);
+
+    FramesContainer *frames_container = AppLayerFramesSetupContainer(f);
+    if (frames_container == NULL)
+        return NULL;
+
+    Frames *frames;
+    if (dir == 0) {
+        frames = &frames_container->toserver;
+    } else {
+        frames = &frames_container->toclient;
+    }
+
+    Frame *r = FrameNew(frames, frame_start_rel, len);
+    if (r != NULL) {
+        r->type = frame_type;
+    }
+    return r;
+}
+
 /** \brief create new frame using a relative offset from the start of the stream slice
  */
 Frame *AppLayerFrameNewByRelativeOffset(Flow *f, const StreamSlice *stream_slice,
@@ -471,16 +503,19 @@ Frame *AppLayerFrameNewByRelativeOffset(Flow *f, const StreamSlice *stream_slice
 {
     /* workarounds for many (unit|fuzz)tests not handling TCP data properly */
 #if defined(UNITTESTS) || defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION)
-    if (f->protoctx == NULL)
+    if (f->proto == IPPROTO_TCP && f->protoctx == NULL)
         return NULL;
     if (stream_slice->input == NULL)
         return NULL;
 #endif
     BUG_ON(stream_slice->input == NULL);
-    BUG_ON(f->proto != IPPROTO_TCP);
-    BUG_ON(f->protoctx == NULL);
+    BUG_ON(f->proto == IPPROTO_TCP && f->protoctx == NULL);
     BUG_ON(f->alparser == NULL);
 
+    if (f->proto == IPPROTO_UDP) {
+        return AppLayerFrameUdp(f, stream_slice, frame_start_rel, len, dir, frame_type);
+    }
+
     FramesContainer *frames_container = AppLayerFramesSetupContainer(f);
     if (frames_container == NULL)
         return NULL;
@@ -545,14 +580,13 @@ Frame *AppLayerFrameNewByAbsoluteOffset(Flow *f, const StreamSlice *stream_slice
 {
     /* workarounds for many (unit|fuzz)tests not handling TCP data properly */
 #if defined(UNITTESTS) || defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION)
-    if (f->protoctx == NULL)
+    if (f->proto == IPPROTO_TCP && f->protoctx == NULL)
         return NULL;
     if (stream_slice->input == NULL)
         return NULL;
 #endif
     BUG_ON(stream_slice->input == NULL);
-    BUG_ON(f->proto != IPPROTO_TCP);
-    BUG_ON(f->protoctx == NULL);
+    BUG_ON(f->proto == IPPROTO_TCP && f->protoctx == NULL);
     BUG_ON(f->alparser == NULL);
     BUG_ON(frame_start < stream_slice->offset);
     BUG_ON(frame_start - stream_slice->offset >= (uint64_t)INT_MAX);
@@ -751,13 +785,25 @@ static void FramePrune(Frames *frames, const TcpStream *stream, const bool eof)
 
 void FramesPrune(Flow *f, Packet *p)
 {
-    if (f->protoctx == NULL)
+    if (f->proto == IPPROTO_TCP && f->protoctx == NULL)
         return;
     FramesContainer *frames_container = AppLayerFramesGetContainer(f);
     if (frames_container == NULL)
         return;
 
     Frames *frames;
+
+    if (p->proto == IPPROTO_UDP) {
+        SCLogDebug("clearing all UDP frames");
+        if (PKT_IS_TOSERVER(p)) {
+            frames = &frames_container->toserver;
+        } else {
+            frames = &frames_container->toclient;
+        }
+        FramesClear(frames);
+        return;
+    }
+
     TcpSession *ssn = f->protoctx;
 
     if (ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED) {

src/app-layer-parser.c

@@ -197,7 +197,7 @@ FramesContainer *AppLayerFramesGetContainer(Flow *f)
 FramesContainer *AppLayerFramesSetupContainer(Flow *f)
 {
 #ifdef UNITTESTS
-    if (f == NULL || f->alparser == NULL || f->protoctx == NULL)
+    if (f == NULL || f->alparser == NULL || (f->proto == IPPROTO_TCP && f->protoctx == NULL))
         return NULL;
 #endif
     DEBUG_VALIDATE_BUG_ON(f == NULL || f->alparser == NULL);

src/detect-engine-frame.c

@@ -156,6 +156,42 @@ int DetectRunFrameInspectRule(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, co
     return false;
 }
 
+/** \internal
+ *  \brief setup buffer based on frame in UDP payload
+ */
+static InspectionBuffer *DetectFrame2InspectBufferUdp(DetectEngineThreadCtx *det_ctx,
+        const DetectEngineTransforms *transforms, Packet *p, InspectionBuffer *buffer,
+        const Frames *frames, const Frame *frame, const int list_id, const uint32_t idx,
+        const bool first)
+{
+    DEBUG_VALIDATE_BUG_ON(frame->rel_offset >= p->payload_len);
+    if (frame->rel_offset >= p->payload_len)
+        return NULL;
+
+    int frame_len = frame->len != -1 ? frame->len : p->payload_len - frame->rel_offset;
+    uint8_t ci_flags = DETECT_CI_FLAGS_START;
+
+    if (frame->rel_offset + frame_len > p->payload_len) {
+        frame_len = p->payload_len - frame->rel_offset;
+    } else {
+        ci_flags |= DETECT_CI_FLAGS_END;
+    }
+    const uint8_t *data = p->payload + frame->rel_offset;
+    const uint32_t data_len = frame_len;
+
+    SCLogDebug("packet %" PRIu64 " -> frame %p/%" PRIi64 "/%s rel_offset %" PRIi64
+               " type %u len %" PRIi64,
+            p->pcap_cnt, frame, frame->id,
+            AppLayerParserGetFrameNameById(p->flow->proto, p->flow->alproto, frame->type),
+            frame->rel_offset, frame->type, frame->len);
+    // PrintRawDataFp(stdout, data, MIN(64,data_len));
+
+    InspectionBufferSetupMulti(buffer, transforms, data, data_len);
+    buffer->inspect_offset = 0;
+    buffer->flags = ci_flags;
+    return buffer;
+}
+
 InspectionBuffer *DetectFrame2InspectBuffer(DetectEngineThreadCtx *det_ctx,
         const DetectEngineTransforms *transforms, Packet *p, const Frames *frames,
         const Frame *frame, const int list_id, const uint32_t idx, const bool first)
@@ -168,6 +204,12 @@ InspectionBuffer *DetectFrame2InspectBuffer(DetectEngineThreadCtx *det_ctx,
         return buffer;
 
     BUG_ON(p->flow == NULL);
+
+    if (p->proto == IPPROTO_UDP) {
+        return DetectFrame2InspectBufferUdp(
+                det_ctx, transforms, p, buffer, frames, frame, list_id, idx, first);
+    }
+
     BUG_ON(p->flow->protoctx == NULL);
     TcpSession *ssn = p->flow->protoctx;
     TcpStream *stream;

src/detect-frame.c

@@ -64,14 +64,20 @@ static int DetectFrameSetup(DetectEngineCtx *de_ctx, Signature *s, const char *s
     char value[256] = "";
     strlcpy(value, str, sizeof(value));
 
-    if (!(DetectProtoContainsProto(&s->proto, IPPROTO_TCP))) {
-        SCLogError(SC_ERR_INVALID_RULE_ARGUMENT, "'frame' keyword only supported for TCP");
+    const bool is_tcp = DetectProtoContainsProto(&s->proto, IPPROTO_TCP);
+    const bool is_udp = DetectProtoContainsProto(&s->proto, IPPROTO_UDP);
+
+    if (!(is_tcp || is_udp)) {
+        SCLogError(SC_ERR_INVALID_RULE_ARGUMENT, "'frame' keyword only supported for TCP and UDP");
         return -1;
     }
 
-    int raw_frame_type;
+    int raw_frame_type = -1;
     if (AppProtoIsValid(s->alproto)) {
-        raw_frame_type = AppLayerParserGetFrameIdByName(IPPROTO_TCP, s->alproto, str);
+        if (is_tcp)
+            raw_frame_type = AppLayerParserGetFrameIdByName(IPPROTO_TCP, s->alproto, str);
+        if (is_udp && raw_frame_type < 0)
+            raw_frame_type = AppLayerParserGetFrameIdByName(IPPROTO_UDP, s->alproto, str);
         if (raw_frame_type < 0) {
             char *dot = strchr(value, '.');
             if (dot != NULL)
@@ -89,7 +95,10 @@ static int DetectFrameSetup(DetectEngineCtx *de_ctx, Signature *s, const char *s
                 if (DetectSignatureSetAppProto(s, keyword_alproto) < 0)
                     return -1;
             }
-            raw_frame_type = AppLayerParserGetFrameIdByName(IPPROTO_TCP, s->alproto, val);
+            if (is_tcp)
+                raw_frame_type = AppLayerParserGetFrameIdByName(IPPROTO_TCP, s->alproto, val);
+            if (is_udp && raw_frame_type < 0)
+                raw_frame_type = AppLayerParserGetFrameIdByName(IPPROTO_UDP, s->alproto, val);
             if (raw_frame_type < 0) {
                 SCLogError(SC_ERR_INVALID_RULE_ARGUMENT, "unknown frame '%s' for protocol '%s'",
                         val, proto);
@@ -115,7 +124,10 @@ static int DetectFrameSetup(DetectEngineCtx *de_ctx, Signature *s, const char *s
         if (DetectSignatureSetAppProto(s, alproto) < 0)
             return -1;
 
-        raw_frame_type = AppLayerParserGetFrameIdByName(IPPROTO_TCP, s->alproto, val);
+        if (is_tcp)
+            raw_frame_type = AppLayerParserGetFrameIdByName(IPPROTO_TCP, s->alproto, val);
+        if (is_udp && raw_frame_type < 0)
+            raw_frame_type = AppLayerParserGetFrameIdByName(IPPROTO_UDP, s->alproto, val);
         if (raw_frame_type < 0) {
             SCLogError(SC_ERR_INVALID_RULE_ARGUMENT, "unknown frame '%s' for protocol '%s'", val,
                     proto);

src/detect.c

@@ -145,6 +145,8 @@ static void DetectRun(ThreadVars *th_v,
                 DetectRunFrames(th_v, de_ctx, det_ctx, p, pflow, &scratch);
                 // PACKET_PROFILING_DETECT_END(p, PROF_DETECT_TX);
             }
+        } else if (p->proto == IPPROTO_UDP) {
+            DetectRunFrames(th_v, de_ctx, det_ctx, p, pflow, &scratch);
         }
 
         PACKET_PROFILING_DETECT_START(p, PROF_DETECT_TX);

src/flow-worker.c

@@ -574,6 +574,8 @@ static TmEcode FlowWorker(ThreadVars *tv, Packet *p, void *data)
             StreamTcpPruneSession(p->flow, p->flowflags & FLOW_PKT_TOSERVER ?
                     STREAM_TOSERVER : STREAM_TOCLIENT);
             FLOWWORKER_PROFILING_END(p, PROFILE_FLOWWORKER_TCPPRUNE);
+        } else if (p->proto == IPPROTO_UDP) {
+            FramesPrune(p->flow, p);
         }
 
         /* run tx cleanup last */

src/output-json-alert.c

@@ -592,27 +592,37 @@ static void AlertAddFiles(const Packet *p, JsonBuilder *jb, const uint64_t tx_id
 
 static void AlertAddFrame(const Packet *p, JsonBuilder *jb, const int64_t frame_id)
 {
-    if (p->flow == NULL || p->flow->protoctx == NULL)
+    if (p->flow == NULL || (p->proto == IPPROTO_TCP && p->flow->protoctx == NULL))
         return;
 
     FramesContainer *frames_container = AppLayerFramesGetContainer(p->flow);
     if (frames_container == NULL)
         return;
 
-    Frames *frames;
-    TcpSession *ssn = p->flow->protoctx;
-    TcpStream *stream;
-    if (PKT_IS_TOSERVER(p)) {
-        stream = &ssn->client;
-        frames = &frames_container->toserver;
-    } else {
-        stream = &ssn->server;
-        frames = &frames_container->toclient;
+    Frames *frames = NULL;
+    TcpStream *stream = NULL;
+    if (p->proto == IPPROTO_TCP) {
+        TcpSession *ssn = p->flow->protoctx;
+        if (PKT_IS_TOSERVER(p)) {
+            stream = &ssn->client;
+            frames = &frames_container->toserver;
+        } else {
+            stream = &ssn->server;
+            frames = &frames_container->toclient;
+        }
+    } else if (p->proto == IPPROTO_UDP) {
+        if (PKT_IS_TOSERVER(p)) {
+            frames = &frames_container->toserver;
+        } else {
+            frames = &frames_container->toclient;
+        }
     }
 
-    Frame *frame = FrameGetById(frames, frame_id);
-    if (frame != NULL) {
-        FrameJsonLogOneFrame(frame, p->flow, stream, p, jb);
+    if (frames) {
+        Frame *frame = FrameGetById(frames, frame_id);
+        if (frame != NULL) {
+            FrameJsonLogOneFrame(frame, p->flow, stream, p, jb);
+        }
     }
 }
 

src/output-json-frame.c

@@ -126,7 +126,7 @@ static void PayloadAsHex(const uint8_t *data, uint32_t data_len, char *str, size
 }
 #endif
 
-static void FrameAddPayload(JsonBuilder *js, const TcpStream *stream, const Frame *frame)
+static void FrameAddPayloadTCP(JsonBuilder *js, const TcpStream *stream, const Frame *frame)
 {
     uint32_t sb_data_len = 0;
     const uint8_t *data = NULL;
@@ -178,44 +178,123 @@ static void FrameAddPayload(JsonBuilder *js, const TcpStream *stream, const Fram
 #endif
 }
 
+static void FrameAddPayloadUDP(JsonBuilder *js, const Packet *p, const Frame *frame)
+{
+    DEBUG_VALIDATE_BUG_ON(frame->rel_offset >= p->payload_len);
+    if (frame->rel_offset >= p->payload_len)
+        return;
+
+    int frame_len = frame->len != -1 ? frame->len : p->payload_len - frame->rel_offset;
+
+    if (frame->rel_offset + frame_len > p->payload_len) {
+        frame_len = p->payload_len - frame->rel_offset;
+        JB_SET_FALSE(js, "complete");
+    } else {
+        JB_SET_TRUE(js, "complete");
+    }
+    const uint8_t *data = p->payload + frame->rel_offset;
+    const uint32_t data_len = frame_len;
+
+    const uint32_t log_data_len = MIN(data_len, 256);
+    jb_set_base64(js, "payload", data, log_data_len);
+
+    uint8_t printable_buf[log_data_len + 1];
+    uint32_t o = 0;
+    PrintStringsToBuffer(printable_buf, &o, log_data_len + 1, data, log_data_len);
+    printable_buf[log_data_len] = '\0';
+    jb_set_string(js, "payload_printable", (char *)printable_buf);
+#if 0
+    char pretty_buf[data_len * 4 + 1];
+    pretty_buf[0] = '\0';
+    PayloadAsHex(data, data_len, pretty_buf, data_len * 4 + 1);
+    jb_set_string(js, "payload_hex", pretty_buf);
+#endif
+}
+
 // TODO separate between stream_offset and frame_offset
 void FrameJsonLogOneFrame(const Frame *frame, const Flow *f, const TcpStream *stream,
         const Packet *p, JsonBuilder *jb)
 {
-    int64_t abs_offset = frame->rel_offset + (int64_t)STREAM_BASE_OFFSET(stream);
-
     jb_open_object(jb, "frame");
     jb_set_string(jb, "type", AppLayerParserGetFrameNameById(f->proto, f->alproto, frame->type));
     jb_set_uint(jb, "id", frame->id);
-    jb_set_uint(jb, "stream_offset", (uint64_t)abs_offset);
+    jb_set_string(jb, "direction", PKT_IS_TOSERVER(p) ? "toserver" : "toclient");
+
+    if (f->proto == IPPROTO_TCP) {
+        DEBUG_VALIDATE_BUG_ON(stream == NULL);
+        int64_t abs_offset = frame->rel_offset + (int64_t)STREAM_BASE_OFFSET(stream);
+        jb_set_uint(jb, "stream_offset", (uint64_t)abs_offset);
 
-    if (frame->len < 0) {
-        uint64_t usable = StreamTcpGetUsable(stream, true);
-        uint64_t len = usable - abs_offset;
-        jb_set_uint(jb, "length", len);
+        if (f->proto == IPPROTO_TCP && frame->len < 0) {
+            uint64_t usable = StreamTcpGetUsable(stream, true);
+            uint64_t len = usable - abs_offset;
+            jb_set_uint(jb, "length", len);
+        } else {
+            jb_set_uint(jb, "length", frame->len);
+        }
+        FrameAddPayloadTCP(jb, stream, frame);
     } else {
         jb_set_uint(jb, "length", frame->len);
+        FrameAddPayloadUDP(jb, p, frame);
     }
-    jb_set_string(jb, "direction", PKT_IS_TOSERVER(p) ? "toserver" : "toclient");
     if (frame->flags & FRAME_FLAG_TX_ID_SET) {
         jb_set_uint(jb, "tx_id", frame->tx_id);
     }
-    FrameAddPayload(jb, stream, frame);
     jb_close(jb);
 }
 
+static int FrameJsonUdp(
+        JsonFrameLogThread *aft, const Packet *p, Flow *f, FramesContainer *frames_container)
+{
+    FrameJsonOutputCtx *json_output_ctx = aft->json_output_ctx;
+
+    Frames *frames;
+    if (PKT_IS_TOSERVER(p)) {
+        frames = &frames_container->toserver;
+    } else {
+        frames = &frames_container->toclient;
+    }
+
+    for (uint32_t idx = 0; idx < frames->cnt; idx++) {
+        Frame *frame = FrameGetByIndex(frames, idx);
+        if (frame == NULL || frame->flags & FRAME_FLAG_LOGGED)
+            continue;
+
+        /* First initialize the address info (5-tuple). */
+        JsonAddrInfo addr = json_addr_info_zero;
+        JsonAddrInfoInit(p, LOG_DIR_PACKET, &addr);
+
+        JsonBuilder *jb =
+                CreateEveHeader(p, LOG_DIR_PACKET, "frame", &addr, json_output_ctx->eve_ctx);
+        if (unlikely(jb == NULL))
+            return TM_ECODE_OK;
+
+        jb_set_string(jb, "app_proto", AppProtoToString(f->alproto));
+        FrameJsonLogOneFrame(frame, p->flow, NULL, p, jb);
+        OutputJsonBuilderBuffer(jb, aft->ctx);
+        jb_free(jb);
+        frame->flags |= FRAME_FLAG_LOGGED;
+    }
+    return TM_ECODE_OK;
+}
+
 static int FrameJson(ThreadVars *tv, JsonFrameLogThread *aft, const Packet *p)
 {
     FrameJsonOutputCtx *json_output_ctx = aft->json_output_ctx;
 
-    BUG_ON(p->proto != IPPROTO_TCP);
     BUG_ON(p->flow == NULL);
-    BUG_ON(p->flow->protoctx == NULL);
 
     FramesContainer *frames_container = AppLayerFramesGetContainer(p->flow);
     if (frames_container == NULL)
         return TM_ECODE_OK;
 
+    if (p->proto == IPPROTO_UDP) {
+        return FrameJsonUdp(aft, p, p->flow, frames_container);
+    }
+
+    BUG_ON(p->proto != IPPROTO_TCP);
+    BUG_ON(p->flow->protoctx == NULL);
+
     /* TODO can we set these EOF flags once per packet? We have them in detect, tx, file, filedata,
      * etc */
     const bool last_pseudo = (p->flowflags & FLOW_PKT_LAST_PSEUDO) != 0;
@@ -288,7 +367,7 @@ static int JsonFrameLogCondition(ThreadVars *tv, const Packet *p)
     if (p->flow == NULL || p->flow->alproto == ALPROTO_UNKNOWN)
         return FALSE;
 
-    if (p->proto == IPPROTO_TCP && p->flow->alparser != NULL) {
+    if ((p->proto == IPPROTO_TCP || p->proto == IPPROTO_UDP) && p->flow->alparser != NULL) {
         FramesContainer *frames_container = AppLayerFramesGetContainer(p->flow);
         if (frames_container == NULL)
             return FALSE;