Initial version of a inline raw reassembly function that reassembles in a sliding window. Introduce new unittest helpers for stream reassembly.

src/Makefile.am

@@ -208,6 +208,7 @@ stream.c stream.h \
 stream-tcp.c stream-tcp.h stream-tcp-private.h \
 stream-tcp-reassemble.c stream-tcp-reassemble.h \
 stream-tcp-inline.c stream-tcp-inline.h \
+stream-tcp-util.c stream-tcp-util.h \
 respond-reject.c respond-reject.h \
 respond-reject-libnet11.h respond-reject-libnet11.c \
 conf.c conf.h \

src/stream-tcp-reassemble.h

@@ -102,5 +102,11 @@ void StreamTcpReassemblePause (TcpSession *, char );
 void StreamTcpReassembleUnPause (TcpSession *, char );
 int StreamTcpCheckStreamContents(uint8_t *, uint16_t , TcpStream *);
 
+int StreamTcpReassembleInsertSegment(ThreadVars *, TcpReassemblyThreadCtx *, TcpStream *, TcpSegment *, Packet *);
+TcpSegment* StreamTcpGetSegment(ThreadVars *, TcpReassemblyThreadCtx *, uint16_t);
+
+void StreamTcpReturnStreamSegments(TcpStream *);
+void StreamTcpSegmentReturntoPool(TcpSegment *);
+
 #endif /* __STREAM_TCP_REASSEMBLE_H__ */
 

src/stream-tcp-util.c

@@ -0,0 +1,217 @@
+/* Copyright (C) 2007-2011 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Victor Julien <victor@inliniac.net>
+ *
+ *  Helper functions for the stream engine.
+ */
+
+#include "suricata-common.h"
+
+#include "stream-tcp-reassemble.h"
+#include "stream-tcp-inline.h"
+#include "stream-tcp.h"
+#include "stream-tcp-util.h"
+
+#include "util-memcmp.h"
+#include "util-print.h"
+
+#include "util-unittest.h"
+#include "util-unittest-helper.h"
+
+#ifdef UNITTESTS
+
+/* unittest helper functions */
+
+void StreamTcpUTInit(TcpReassemblyThreadCtx **ra_ctx) {
+    StreamTcpInitConfig(TRUE);
+    *ra_ctx = StreamTcpReassembleInitThreadCtx();
+}
+
+void StreamTcpUTDeinit(TcpReassemblyThreadCtx *ra_ctx) {
+    StreamTcpReassembleFreeThreadCtx(ra_ctx);
+    StreamTcpFreeConfig(TRUE);
+}
+
+void StreamTcpUTSetupSession(TcpSession *ssn) {
+    memset(ssn, 0x00, sizeof(TcpSession));
+}
+
+void StreamTcpUTClearSession(TcpSession *ssn) {
+    StreamTcpUTClearStream(&ssn->client);
+    StreamTcpUTClearStream(&ssn->server);
+}
+
+void StreamTcpUTSetupStream(TcpStream *s, uint32_t isn) {
+    memset(s, 0x00, sizeof(TcpStream));
+
+    s->isn = isn;
+    STREAMTCP_SET_RA_BASE_SEQ(s, isn);
+}
+
+void StreamTcpUTClearStream(TcpStream *s) {
+    StreamTcpReturnStreamSegments(s);
+}
+
+int StreamTcpUTAddSegmentWithByte(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t byte, uint16_t len) {
+    TcpSegment *s = StreamTcpGetSegment(tv, ra_ctx, len);
+    if (s == NULL) {
+        return -1;
+    }
+
+    s->seq = seq;
+    s->payload_len = len;
+    memset(s->payload, byte, len);
+
+    Packet *p = UTHBuildPacketReal(s->payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
+    if (StreamTcpReassembleInsertSegment(tv, ra_ctx, stream, s, p) < 0)
+        return -1;
+    UTHFreePacket(p);
+    return 0;
+}
+
+/* tests */
+
+int StreamTcpUtilTest01(void) {
+    int ret = 0;
+    TcpReassemblyThreadCtx *ra_ctx = NULL;
+
+    StreamTcpUTInit(&ra_ctx);
+
+    if (ra_ctx == NULL) {
+        printf("ra_ctx is NULL: ");
+        goto end;
+    }
+
+    ret = 1;
+end:
+    StreamTcpUTDeinit(ra_ctx);
+    return ret;
+}
+
+
+int StreamTcpUtilStreamTest01(void) {
+    int ret = 0;
+    TcpReassemblyThreadCtx *ra_ctx = NULL;
+    ThreadVars tv;
+    TcpStream stream;
+
+    memset(&tv, 0x00, sizeof(tv));
+
+    StreamTcpUTInit(&ra_ctx);
+    StreamTcpUTSetupStream(&stream, 1);
+
+    if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &stream,  2, 'A', 5) == -1) {
+        printf("failed to add segment 1: ");
+        goto end;
+    }
+    if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &stream,  7, 'B', 5) == -1) {
+        printf("failed to add segment 2: ");
+        goto end;
+    }
+    if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &stream, 12, 'C', 5) == -1) {
+        printf("failed to add segment 3: ");
+        goto end;
+    }
+
+    TcpSegment *seg = stream.seg_list;
+    if (seg->seq != 2) {
+        printf("first seg in the list should have seq 2: ");
+        goto end;
+    }
+
+    seg = seg->next;
+    if (seg->seq != 7) {
+        printf("first seg in the list should have seq 7: ");
+        goto end;
+    }
+
+    seg = seg->next;
+    if (seg->seq != 12) {
+        printf("first seg in the list should have seq 12: ");
+        goto end;
+    }
+
+    ret = 1;
+end:
+    StreamTcpUTClearStream(&stream);
+    StreamTcpUTDeinit(ra_ctx);
+    return ret;
+}
+
+int StreamTcpUtilStreamTest02(void) {
+    int ret = 0;
+    TcpReassemblyThreadCtx *ra_ctx = NULL;
+    ThreadVars tv;
+    TcpStream stream;
+
+    memset(&tv, 0x00, sizeof(tv));
+
+    StreamTcpUTInit(&ra_ctx);
+    StreamTcpUTSetupStream(&stream, 1);
+
+    if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &stream,  7, 'B', 5) == -1) {
+        printf("failed to add segment 2: ");
+        goto end;
+    }
+    if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &stream, 12, 'C', 5) == -1) {
+        printf("failed to add segment 3: ");
+        goto end;
+    }
+    if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &stream,  2, 'A', 5) == -1) {
+        printf("failed to add segment 1: ");
+        goto end;
+    }
+
+    TcpSegment *seg = stream.seg_list;
+    if (seg->seq != 2) {
+        printf("first seg in the list should have seq 2: ");
+        goto end;
+    }
+
+    seg = seg->next;
+    if (seg->seq != 7) {
+        printf("first seg in the list should have seq 7: ");
+        goto end;
+    }
+
+    seg = seg->next;
+    if (seg->seq != 12) {
+        printf("first seg in the list should have seq 12: ");
+        goto end;
+    }
+
+    ret = 1;
+end:
+    StreamTcpUTClearStream(&stream);
+    StreamTcpUTDeinit(ra_ctx);
+    return ret;
+}
+
+#endif
+
+void StreamTcpUtilRegisterTests(void) {
+#ifdef UNITTESTS
+    UtRegisterTest("StreamTcpUtilTest01", StreamTcpUtilTest01, 1);
+    UtRegisterTest("StreamTcpUtilStreamTest01", StreamTcpUtilStreamTest01, 1);
+    UtRegisterTest("StreamTcpUtilStreamTest02", StreamTcpUtilStreamTest02, 1);
+#endif /* UNITTESTS */
+}
+

src/stream-tcp-util.h

@@ -0,0 +1,45 @@
+/* Copyright (C) 2007-2011 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Victor Julien <victor@inliniac.net>
+ */
+
+#ifndef __STREAM_TCP_UTIL_H__
+#define __STREAM_TCP_UTIL_H__
+
+#include "stream-tcp-private.h"
+
+void StreamTcpUTInit(TcpReassemblyThreadCtx **);
+void StreamTcpUTDeinit(TcpReassemblyThreadCtx *);
+
+void StreamTcpUTSetupSession(TcpSession *);
+void StreamTcpUTClearSession(TcpSession *);
+
+void StreamTcpUTSetupStream(TcpStream *, uint32_t isn);
+void StreamTcpUTClearStream(TcpStream *);
+
+int StreamTcpUTAddSegmentWithByte(ThreadVars *, TcpReassemblyThreadCtx *, TcpStream *, uint32_t, uint8_t, uint16_t);
+int StreamTcpUTAddSegmentWithPayload(ThreadVars *, TcpReassemblyThreadCtx *, TcpStream *, uint32_t, uint8_t *, uint16_t);
+
+
+void StreamTcpUtilRegisterTests(void);
+
+#endif /* __STREAM_TCP_UTIL_H__ */
+

src/stream-tcp.c

@@ -49,8 +49,8 @@
 #include "stream-tcp-private.h"
 #include "stream-tcp-reassemble.h"
 #include "stream-tcp.h"
+#include "stream-tcp-util.h"
 #include "stream.h"
-#include "stream-tcp.h"
 
 #include "app-layer-parser.h"
 #include "app-layer-protos.h"
@@ -64,6 +64,7 @@
 #define STREAMTCP_DEFAULT_PREALLOC              32768
 #define STREAMTCP_DEFAULT_MEMCAP                32 * 1024 * 1024 /* 32mb */
 #define STREAMTCP_DEFAULT_REASSEMBLY_MEMCAP     64 * 1024 * 1024 /* 64mb */
+#define STREAMTCP_DEFAULT_REASSEMBLY_WINDOW     3000
 
 #define STREAMTCP_NEW_TIMEOUT                   60
 #define STREAMTCP_EST_TIMEOUT                   3600
@@ -98,7 +99,6 @@ static int StreamTcpHandleFin(ThreadVars *tv, StreamTcpThread *, TcpSession *, P
 void StreamTcpRegisterTests (void);
 void StreamTcpReturnStreamSegments (TcpStream *);
 void StreamTcpInitConfig(char);
-extern void StreamTcpSegmentReturntoPool(TcpSegment *);
 int StreamTcpGetFlowState(void *);
 void StreamTcpSetOSPolicy(TcpStream*, Packet*);
 void StreamTcpPseudoPacketCreateStreamEndPacket(Packet *, TcpSession *, PacketQueue *);
@@ -169,24 +169,6 @@ int StreamTcpCheckMemcap(uint32_t size) {
     SCReturnInt(ret);
 }
 
-void StreamTcpReturnStreamSegments (TcpStream *stream)
-{
-    TcpSegment *seg = stream->seg_list;
-    TcpSegment *next_seg;
-
-    if (seg == NULL)
-        return;
-
-    while (seg != NULL) {
-        next_seg = seg->next;
-        StreamTcpSegmentReturntoPool(seg);
-        seg = next_seg;
-    }
-
-    stream->seg_list = NULL;
-    stream->seg_list_tail = NULL;
-}
-
 /** \brief Function to return the stream back to the pool. It returns the
  *         segments in the stream to the segment pool.
  *
@@ -438,6 +420,9 @@ void StreamTcpInitConfig(char quiet)
         SCLogInfo("stream.\"inline\": %s", stream_inline ? "enabled" : "disabled");
     }
 
+    /** \todo yaml part */
+    stream_config.reassembly_inline_window = STREAMTCP_DEFAULT_REASSEMBLY_WINDOW;
+
     /* init the memcap and it's lock */
     stream_memuse = 0;
     stream_memuse_max = 0;

src/stream-tcp.h

@@ -41,13 +41,24 @@
 
 /*global flow data*/
 typedef struct TcpStreamCnf_ {
-    uint32_t memcap; /** max stream mem usage */
+    /** stream tracking
+     *
+     * max stream mem usage
+     */
+    uint32_t memcap;
+
     uint32_t max_sessions;
     uint32_t prealloc_sessions;
     int midstream;
     int async_oneside;
     uint32_t reassembly_memcap; /**< max memory usage for stream reassembly */
     uint32_t reassembly_depth;  /**< Depth until when we reassemble the stream */
+
+    /** reassembly -- inline mode
+     *
+     *  sliding window size for raw stream reassembly
+     */
+    uint32_t reassembly_inline_window;
     uint8_t flags;
 } TcpStreamCnf;