diff --git a/src/Makefile.am b/src/Makefile.am index 20c4dd04c0..7238ccb4c7 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -208,6 +208,7 @@ stream.c stream.h \ stream-tcp.c stream-tcp.h stream-tcp-private.h \ stream-tcp-reassemble.c stream-tcp-reassemble.h \ stream-tcp-inline.c stream-tcp-inline.h \ +stream-tcp-util.c stream-tcp-util.h \ respond-reject.c respond-reject.h \ respond-reject-libnet11.h respond-reject-libnet11.c \ conf.c conf.h \ diff --git a/src/stream-tcp-reassemble.c b/src/stream-tcp-reassemble.c index 83c00d521a..e9a5aa292a 100644 --- a/src/stream-tcp-reassemble.c +++ b/src/stream-tcp-reassemble.c @@ -50,6 +50,7 @@ #include "stream-tcp-private.h" #include "stream-tcp-reassemble.h" #include "stream-tcp-inline.h" +#include "stream-tcp-util.h" #include "stream.h" @@ -57,6 +58,8 @@ #include "app-layer-protos.h" #include "app-layer.h" +#include "detect-engine-state.h" + #define PSEUDO_PACKET_PAYLOAD_SIZE 65416 /* 64 Kb minus max IP and TCP header */ #ifdef DEBUG @@ -105,7 +108,6 @@ static int HandleSegmentStartsAfterListSegment(ThreadVars *, TcpReassemblyThread void StreamTcpSegmentDataReplace(TcpSegment *, TcpSegment *, uint32_t, uint16_t); void StreamTcpSegmentDataCopy(TcpSegment *, TcpSegment *); TcpSegment* StreamTcpGetSegment(ThreadVars *tv, TcpReassemblyThreadCtx *, uint16_t); -void StreamTcpSegmentReturntoPool(TcpSegment *); void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t); void StreamTcpReassemblePseudoPacketCreate(TcpStream *, Packet *, PacketQueue *); @@ -223,6 +225,56 @@ void TcpSegmentPoolFree(void *ptr) { return; } +/** + * \brief Function to return the segment back to the pool. + * + * \param seg Segment which will be returned back to the pool. + */ +void StreamTcpSegmentReturntoPool(TcpSegment *seg) +{ + if (seg == NULL) + return; + + seg->next = NULL; + seg->prev = NULL; + + uint16_t idx = segment_pool_idx[seg->pool_size]; + SCMutexLock(&segment_pool_mutex[idx]); + PoolReturn(segment_pool[idx], (void *) seg); + SCLogDebug("segment_pool[%"PRIu16"]->empty_list_size %"PRIu32"", + idx,segment_pool[idx]->empty_list_size); + SCMutexUnlock(&segment_pool_mutex[idx]); + +#ifdef DEBUG + SCMutexLock(&segment_pool_cnt_mutex); + segment_pool_cnt--; + SCMutexUnlock(&segment_pool_cnt_mutex); +#endif +} + +/** + * \brief return all segments in this stream into the pool(s) + * + * \param stream the stream to cleanup + */ +void StreamTcpReturnStreamSegments (TcpStream *stream) +{ + TcpSegment *seg = stream->seg_list; + TcpSegment *next_seg; + + if (seg == NULL) + return; + + while (seg != NULL) { + next_seg = seg->next; + StreamTcpSegmentReturntoPool(seg); + seg = next_seg; + } + + stream->seg_list = NULL; + stream->seg_list_tail = NULL; +} + int StreamTcpReassembleInit(char quiet) { StreamMsgQueuesInit(); @@ -332,8 +384,14 @@ TcpReassemblyThreadCtx *StreamTcpReassembleInitThreadCtx(void) void StreamTcpReassembleFreeThreadCtx(TcpReassemblyThreadCtx *ra_ctx) { SCEnter(); - if (ra_ctx->stream_q != NULL) + if (ra_ctx->stream_q != NULL) { + StreamMsg *smsg; + while ((smsg = StreamMsgGetFromQueue(ra_ctx->stream_q)) != NULL) { + StreamMsgReturnToPool(smsg); + } + StreamMsgQueueFree(ra_ctx->stream_q); + } ra_ctx->stream_q = NULL; AlpProtoDeFinalize2Thread(&ra_ctx->dp_ctx); @@ -472,8 +530,7 @@ static inline uint32_t StreamTcpReassembleGetRaBaseSeq(TcpStream *stream) * \retval -1 error -- either we hit a memory issue (OOM/memcap) or we received * a segment before ra_base_seq. */ - -static int ReassembleInsertSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, +int StreamTcpReassembleInsertSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, TcpSegment *seg, Packet *p) { SCEnter(); @@ -1531,8 +1588,8 @@ int StreamTcpReassembleHandleSegmentHandleData(ThreadVars *tv, TcpReassemblyThre seg->payload_len = size; seg->seq = TCP_GET_SEQ(p); - if (ReassembleInsertSegment(tv, ra_ctx, stream, seg, p) != 0) { - SCLogDebug("ReassembleInsertSegment failed"); + if (StreamTcpReassembleInsertSegment(tv, ra_ctx, stream, seg, p) != 0) { + SCLogDebug("StreamTcpReassembleInsertSegment failed"); SCReturnInt(-1); } @@ -1754,7 +1811,7 @@ static int StreamTcpReassembleInlineAppLayer (TcpReassemblyThreadCtx *ra_ctx, SCLogDebug("stream->seg_list %p", stream->seg_list); #ifdef DEBUG PrintList(stream->seg_list); - PrintRawDataFp(stdout, p->payload, p->payload_len); + //PrintRawDataFp(stdout, p->payload, p->payload_len); #endif if (stream->seg_list == NULL) { @@ -2180,54 +2237,29 @@ static int StreamTcpReassembleInlineAppLayer (TcpReassemblyThreadCtx *ra_ctx, } /** - * \brief Update the stream reassembly upon receiving an ACK packet. + * \brief Update the stream reassembly upon receiving a data segment * - * Stream is in the opposite direction of the packet, as the ACK-packet - * is ACK'ing the stream. + * | left edge | right edge based on sliding window size + * [aaa] + * [aaabbb] + * ... + * [aaabbbcccdddeeefff] + * [bbbcccdddeeefffggg] <- cut off aaa to adhere to the window size + * + * GAP situation: each chunk that is uninterrupted has it's own smsg + * [aaabbb].[dddeeefff] + * [aaa].[ccc].[eeefff] + * + * A flag will be set to indicate where the *NEW* payload starts. This + * is to aid the detection code for alert only sigs. * * \todo this function is too long, we need to break it up. It needs it BAD */ -static int StreamTcpReassembleAppLayer (TcpReassemblyThreadCtx *ra_ctx, +static int StreamTcpReassembleInlineRaw (TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p) { SCEnter(); - - uint8_t flags = 0; - - /* check if toserver reassembly has started before reassembling toclient. */ - if (PKT_IS_TOSERVER(p) && - !(ssn->flags & STREAMTCP_FLAG_TOSERVER_REASSEMBLY_STARTED)) - { - SCLogDebug("toserver reassembling is not done yet, so " - "skipping reassembling at the moment for to_client"); - SCReturnInt(0); - } - - SCLogDebug("stream->seg_list %p", stream->seg_list); -#ifdef DEBUG - PrintList(stream->seg_list); -#endif - - if (stream->seg_list == NULL) { - /* send an empty EOF msg if we have no segments but TCP state - * is beyond ESTABLISHED */ - if (ssn->state > TCP_ESTABLISHED) { - SCLogDebug("sending empty eof message"); - /* send EOF to app layer */ - STREAM_SET_FLAGS(ssn, stream, p, flags); - AppLayerHandleTCPData(&ra_ctx->dp_ctx, p->flow, ssn, - NULL, 0, flags|STREAM_EOF); - - /* even if app layer detection failed, we will now move on to - * release reassembly for both directions. */ - ssn->flags |= STREAMTCP_FLAG_TOSERVER_REASSEMBLY_STARTED; - - } else { - SCLogDebug("no segments in the list to reassemble"); - } - - SCReturnInt(0); - } + SCLogDebug("start p %p, seq %"PRIu32, p, TCP_GET_SEQ(p)); /* check if reassembling has been paused for the moment or not */ if (stream->flags & STREAMTCP_STREAM_FLAG_PAUSE_REASSEMBLY) { @@ -2236,100 +2268,76 @@ static int StreamTcpReassembleAppLayer (TcpReassemblyThreadCtx *ra_ctx, SCReturnInt(0); } - if (stream->flags & STREAMTCP_STREAM_FLAG_GAP) { + if (stream->seg_list == NULL) { SCReturnInt(0); } - uint32_t ra_base_seq; - - /* check if we have detected the app layer protocol or not. If it has been - detected then, process data normally, as we have sent one smsg from - toserver side already to the app layer */ - if (ssn->state <= TCP_ESTABLISHED) { - if (!(ssn->flags & STREAMTCP_FLAG_APPPROTO_DETECTION_COMPLETED)) { - /* Do not perform reassembling of data from server, until the app layer - proto has been detected and we have sent atleast one smsg from client - data to app layer */ - if (PKT_IS_TOSERVER(p)) { - SCLogDebug("we didn't detected the app layer protocol till " - "yet, so not doing toclient reassembling"); - SCReturnInt(0); - } - /* initialize the tmp_ra_base_seq for each new run */ - stream->tmp_ra_app_base_seq = stream->ra_app_base_seq; - ra_base_seq = stream->tmp_ra_app_base_seq; - /* if app layer protocol has been detected, then restore the reassembled - seq. to the value till reassembling has been done and unset the queue - init flag permanently for this tcp session */ - } else if (SEQ_GT(stream->tmp_ra_app_base_seq, stream->ra_app_base_seq)) { - stream->ra_app_base_seq = stream->tmp_ra_app_base_seq; - ra_base_seq = stream->ra_app_base_seq; - SCLogDebug("the app layer protocol has been detected"); - } else { - ra_base_seq = stream->ra_app_base_seq; - } - /* set the ra_bas_seq to stream->ra_base_seq as now app layer protocol - has been detected */ - } else { - if (SEQ_GT(stream->tmp_ra_app_base_seq, stream->ra_app_base_seq)) { - stream->ra_app_base_seq = stream->tmp_ra_app_base_seq; - ra_base_seq = stream->ra_app_base_seq; - } else { - ra_base_seq = stream->ra_app_base_seq; - } - } - - SCLogDebug("ra_base_seq %u", ra_base_seq); - - uint8_t data[4096]; - uint32_t data_len = 0; + uint32_t ra_base_seq = stream->ra_raw_base_seq; + StreamMsg *smsg = NULL; + uint16_t smsg_offset = 0; uint16_t payload_offset = 0; uint16_t payload_len = 0; + TcpSegment *seg = stream->seg_list; uint32_t next_seq = ra_base_seq + 1; + /* determine the left edge and right edge */ + uint32_t right_edge = TCP_GET_SEQ(p) + p->payload_len; + uint32_t left_edge = right_edge - stream_config.reassembly_inline_window; + + /* shift the window to the right if the left edge doesn't cover segments */ + if (SEQ_GT(seg->seq,left_edge)) { + right_edge += (seg->seq - left_edge); + left_edge = seg->seq; + } + + SCLogDebug("left_edge %"PRIu32", right_edge %"PRIu32, left_edge, right_edge); + /* loop through the segments and fill one or more msgs */ - TcpSegment *seg = stream->seg_list; - SCLogDebug("pre-loop seg %p", seg); - for (; seg != NULL && SEQ_LT(seg->seq, stream->last_ack);) { + for (; seg != NULL && SEQ_LT(seg->seq, right_edge); ) { SCLogDebug("seg %p", seg); - /* if app layer protocol has been detected, then remove all the segments - which has been previously processed and reassembled */ - if ((ssn->flags & STREAMTCP_FLAG_APPPROTO_DETECTION_COMPLETED) && - (seg->flags & SEGMENTTCP_FLAG_RAW_PROCESSED) && - (seg->flags & SEGMENTTCP_FLAG_APPLAYER_PROCESSED)) - { - SCLogDebug("segment(%p) of length %"PRIu16" has been processed," - " so return it to pool", seg, seg->payload_len); - TcpSegment *next_seg = seg->next; - StreamTcpRemoveSegmentFromStream(stream, seg); - StreamTcpSegmentReturntoPool(seg); - seg = next_seg; - continue; - } - /* If packets are fully before ra_base_seq, skip them. We do this * because we've reassembled up to the ra_base_seq point already, * so we won't do anything with segments before it anyway. */ SCLogDebug("checking for pre ra_base_seq %"PRIu32" seg %p seq %"PRIu32"" - " len %"PRIu16", combined %"PRIu32" and stream->last_ack " + " len %"PRIu16", combined %"PRIu32" and right_edge " "%"PRIu32"", ra_base_seq, seg, seg->seq, - seg->payload_len, seg->seq+seg->payload_len, stream->last_ack); + seg->payload_len, seg->seq+seg->payload_len, right_edge); - /* Remove the segments which are either completely before the - ra_base_seq or if they are beyond ra_base_seq, but the segment offset - from which we need to copy in to smsg is beyond the stream->last_ack. - As we are copying until the stream->last_ack only */ - /** \todo we should probably not even insert them into the seglist */ + /* Remove the segments which are completely before the ra_base_seq */ + if (SEQ_LT((seg->seq + seg->payload_len), (ra_base_seq - stream_config.reassembly_inline_window))) + { + SCLogDebug("removing pre ra_base_seq %"PRIu32" seg %p seq %"PRIu32"" + " len %"PRIu16"", ra_base_seq, seg, seg->seq, + seg->payload_len); - if ((SEQ_LEQ((seg->seq + seg->payload_len), (ra_base_seq+1)) || - SEQ_LEQ(stream->last_ack, (ra_base_seq + (ra_base_seq - seg->seq)))) && + /* only remove if app layer reassembly is ready too */ + if (seg->flags & SEGMENTTCP_FLAG_APPLAYER_PROCESSED || + stream->flags & STREAMTCP_STREAM_FLAG_GAP) + { + TcpSegment *next_seg = seg->next; + StreamTcpRemoveSegmentFromStream(stream, seg); + StreamTcpSegmentReturntoPool(seg); + seg = next_seg; + /* otherwise, just flag it for removal */ + } else { + seg->flags |= SEGMENTTCP_FLAG_RAW_PROCESSED; + seg = seg->next; + } + continue; + } + + /* if app layer protocol has been detected, then remove all the segments + * which has been previously processed and reassembled + * + * If the stream is in GAP state the app layer flag won't be set */ + if ((ssn->flags & STREAMTCP_FLAG_APPPROTO_DETECTION_COMPLETED) && (seg->flags & SEGMENTTCP_FLAG_RAW_PROCESSED) && - (seg->flags & SEGMENTTCP_FLAG_APPLAYER_PROCESSED)) + (seg->flags & SEGMENTTCP_FLAG_APPLAYER_PROCESSED || + stream->flags & STREAMTCP_STREAM_FLAG_GAP)) { - SCLogDebug("removing pre ra_base_seq %"PRIu32" seg %p seq %"PRIu32 - " len %"PRIu16"", ra_base_seq, seg, seg->seq, seg->payload_len); - + SCLogDebug("segment(%p) of length %"PRIu16" has been processed," + " so return it to pool", seg, seg->payload_len); TcpSegment *next_seg = seg->next; StreamTcpRemoveSegmentFromStream(stream, seg); StreamTcpSegmentReturntoPool(seg); @@ -2337,120 +2345,44 @@ static int StreamTcpReassembleAppLayer (TcpReassemblyThreadCtx *ra_ctx, continue; } - /* we've run into a sequence gap */ + /* we've run into a sequence gap, wrap up any existing smsg and + * queue it so the next chunk (if any) is in a new smsg */ if (SEQ_GT(seg->seq, next_seq)) { + /* pass on pre existing smsg (if any) */ + if (smsg != NULL && smsg->data.data_len > 0) { + StreamMsgPutInQueue(ra_ctx->stream_q, smsg); + stream->ra_raw_base_seq = ra_base_seq; + smsg = NULL; + } + } - /* first, pass on data before the gap */ - if (data_len > 0) { - SCLogDebug("pre GAP data"); - - STREAM_SET_FLAGS(ssn, stream, p, flags); + /* if the segment ends beyond left_edge we need to consider it */ + if (SEQ_GT((seg->seq + seg->payload_len), left_edge)) { + SCLogDebug("seg->seq %" PRIu32 ", seg->payload_len %" PRIu32 ", " + "left_edge %" PRIu32 "", seg->seq, + seg->payload_len, left_edge); - /* if app layer protocol has not been detected till yet, - then check did we have sent message to app layer already - or not. If not then sent the message and set flag that first - message has been sent. No more data till proto has not - been detected */ - if (!(ssn->flags & STREAMTCP_FLAG_APPPROTO_DETECTION_COMPLETED)) { - /* process what we have so far */ - BUG_ON(data_len > sizeof(data)); - AppLayerHandleTCPData(&ra_ctx->dp_ctx, p->flow, ssn, - data, data_len, flags); + /* handle segments partly before ra_base_seq */ + if (SEQ_GT(left_edge, seg->seq)) { + payload_offset = left_edge - seg->seq; - stream->tmp_ra_app_base_seq = ra_base_seq; + if (SEQ_LT(right_edge, (seg->seq + seg->payload_len))) { + payload_len = (right_edge - seg->seq) - payload_offset; } else { - /* process what we have so far */ - AppLayerHandleTCPData(&ra_ctx->dp_ctx, p->flow, ssn, - data, data_len, flags); - - stream->ra_app_base_seq = ra_base_seq; + payload_len = seg->payload_len - payload_offset; } - } - - /* don't conclude it's a gap straight away. If ra_base_seq is lower - * than last_ack - the window, we consider it a gap. */ - if (SEQ_GT((stream->last_ack - stream->window), ra_base_seq) || - ssn->state > TCP_ESTABLISHED) - { - /* see what the length of the gap is, gap length is seg->seq - - * (ra_base_seq +1) */ -#ifdef DEBUG - uint32_t gap_len = seg->seq - next_seq; - SCLogDebug("expected next_seq %" PRIu32 ", got %" PRIu32 " , " - "stream->last_ack %" PRIu32 ". Seq gap %" PRIu32"", - next_seq, seg->seq, stream->last_ack, gap_len); -#endif - next_seq = seg->seq; + if (SCLogDebugEnabled()) { + BUG_ON(payload_offset > seg->payload_len); + BUG_ON((payload_len + payload_offset) > seg->payload_len); + } + } else { + payload_offset = 0; - /* we need to update the ra_base_seq, if app layer proto has - been detected and we are setting new stream message. Otherwise - every smsg will be with flag STREAM_START set, which we - don't want :-) */ - if (ssn->flags & STREAMTCP_FLAG_APPPROTO_DETECTION_COMPLETED) { - stream->ra_app_base_seq = ra_base_seq; + if (SEQ_LT(right_edge, (seg->seq + seg->payload_len))) { + payload_len = right_edge - seg->seq; } else { - stream->tmp_ra_app_base_seq = ra_base_seq; - } - - /* We have missed the packet and end host has ack'd it, so - * IDS should advance it's ra_base_seq and should not consider this - * packet any longer, even if it is retransmitted, as end host will - * drop it anyway */ - ra_base_seq = seg->seq - 1; - - /* send gap signal */ - STREAM_SET_FLAGS(ssn, stream, p, flags); - AppLayerHandleTCPData(&ra_ctx->dp_ctx, p->flow, ssn, - NULL, 0, flags|STREAM_GAP); - data_len = 0; - - /* set a GAP flag and make sure not bothering this stream anymore */ - stream->flags |= STREAMTCP_STREAM_FLAG_GAP; - /* flag reassembly as started, so the to_client part can start */ - ssn->flags |= STREAMTCP_FLAG_TOSERVER_REASSEMBLY_STARTED; - - dbg_app_layer_gap++; - break; - } else { - SCLogDebug("possible GAP, but waiting to see if out of order " - "packets might solve that"); - dbg_app_layer_gap_candidate++; - break; - } - } - - /* if the segment ends beyond ra_base_seq we need to consider it */ - if (SEQ_GT((seg->seq + seg->payload_len), ra_base_seq)) { - SCLogDebug("seg->seq %" PRIu32 ", seg->payload_len %" PRIu32 ", " - "ra_base_seq %" PRIu32 "", seg->seq, - seg->payload_len, ra_base_seq); - - /* handle segments partly before ra_base_seq */ - if (SEQ_GT(ra_base_seq, seg->seq)) { - payload_offset = ra_base_seq - seg->seq; - - if (SEQ_LT(stream->last_ack, (seg->seq + seg->payload_len))) { - if (SEQ_LT(stream->last_ack, ra_base_seq)) { - payload_len = (stream->last_ack - seg->seq); - } else { - payload_len = (stream->last_ack - seg->seq) - payload_offset; - } - } else { - payload_len = seg->payload_len - payload_offset; - } - - if (SCLogDebugEnabled()) { - BUG_ON(payload_offset > seg->payload_len); - BUG_ON((payload_len + payload_offset) > seg->payload_len); - } - } else { - payload_offset = 0; - - if (SEQ_LT(stream->last_ack, (seg->seq + seg->payload_len))) { - payload_len = stream->last_ack - seg->seq; - } else { - payload_len = seg->payload_len; + payload_len = seg->payload_len; } } SCLogDebug("payload_offset is %"PRIu16", payload_len is %"PRIu16"" @@ -2462,46 +2394,41 @@ static int StreamTcpReassembleAppLayer (TcpReassemblyThreadCtx *ra_ctx, break; } + if (smsg == NULL) { + smsg = StreamMsgGetFromPool(); + if (smsg == NULL) { + SCLogDebug("stream_msg_pool is empty"); + return -1; + } + + smsg_offset = 0; + + StreamTcpSetupMsg(ssn, stream, p, smsg); + } + smsg->data.seq = ra_base_seq; + /* copy the data into the smsg */ - uint16_t copy_size = sizeof(data) - data_len; + uint16_t copy_size = sizeof (smsg->data.data) - smsg_offset; if (copy_size > payload_len) { copy_size = payload_len; } if (SCLogDebugEnabled()) { - BUG_ON(copy_size > sizeof(data)); + BUG_ON(copy_size > sizeof(smsg->data.data)); } SCLogDebug("copy_size is %"PRIu16"", copy_size); - memcpy(data + data_len, seg->payload + payload_offset, copy_size); - data_len += copy_size; + memcpy(smsg->data.data + smsg_offset, seg->payload + payload_offset, + copy_size); + smsg_offset += copy_size; ra_base_seq += copy_size; - SCLogDebug("ra_base_seq %"PRIu32", data_len %"PRIu32, ra_base_seq, data_len); - - /* queue the smsg if it's full */ - if (data_len == sizeof(data)) { - /* if app layer protocol has not been detected till yet, - then check did we have sent message to app layer already - or not. If not then sent the message and set flag that first - message has been sent. No more data till proto has not - been detected */ - if (!(ssn->flags & STREAMTCP_FLAG_APPPROTO_DETECTION_COMPLETED)) { - /* process what we have so far */ - STREAM_SET_FLAGS(ssn, stream, p, flags); - BUG_ON(data_len > sizeof(data)); - AppLayerHandleTCPData(&ra_ctx->dp_ctx, p->flow, ssn, - data, data_len, flags); - data_len = 0; + SCLogDebug("ra_base_seq %"PRIu32, ra_base_seq); - stream->tmp_ra_app_base_seq = ra_base_seq; - } else { - /* process what we have so far */ - STREAM_SET_FLAGS(ssn, stream, p, flags); - BUG_ON(data_len > sizeof(data)); - AppLayerHandleTCPData(&ra_ctx->dp_ctx, p->flow, ssn, - data, data_len, flags); - data_len = 0; + smsg->data.data_len += copy_size; - stream->ra_app_base_seq = ra_base_seq; - } + /* queue the smsg if it's full */ + if (smsg->data.data_len == sizeof (smsg->data.data)) { + StreamMsgPutInQueue(ra_ctx->stream_q, smsg); + stream->ra_raw_base_seq = ra_base_seq; + smsg = NULL; } /* if the payload len is bigger than what we copied, we handle the @@ -2509,7 +2436,6 @@ static int StreamTcpReassembleAppLayer (TcpReassemblyThreadCtx *ra_ctx, if (copy_size < payload_len) { SCLogDebug("copy_size %" PRIu32 " < %" PRIu32 "", copy_size, payload_len); - payload_offset += copy_size; payload_len -= copy_size; SCLogDebug("payload_offset is %"PRIu16", seg->payload_len is " @@ -2525,53 +2451,43 @@ static int StreamTcpReassembleAppLayer (TcpReassemblyThreadCtx *ra_ctx, while (segment_done == FALSE) { SCLogDebug("new msg at offset %" PRIu32 ", payload_len " "%" PRIu32 "", payload_offset, payload_len); - data_len = 0; - copy_size = sizeof(data) - data_len; + /* get a new message + XXX we need a setup function */ + smsg = StreamMsgGetFromPool(); + if (smsg == NULL) { + SCLogDebug("stream_msg_pool is empty"); + SCReturnInt(-1); + } + smsg_offset = 0; + + StreamTcpSetupMsg(ssn, stream,p,smsg); + smsg->data.seq = ra_base_seq; + + copy_size = sizeof(smsg->data.data) - smsg_offset; if (copy_size > (seg->payload_len - payload_offset)) { copy_size = (seg->payload_len - payload_offset); } if (SCLogDebugEnabled()) { - BUG_ON(copy_size > sizeof(data)); + BUG_ON(copy_size > sizeof(smsg->data.data)); } - SCLogDebug("copy payload_offset %" PRIu32 ", data_len " + SCLogDebug("copy payload_offset %" PRIu32 ", smsg_offset " "%" PRIu32 ", copy_size %" PRIu32 "", - payload_offset, data_len, copy_size); - memcpy(data + data_len, seg->payload + + payload_offset, smsg_offset, copy_size); + memcpy(smsg->data.data + smsg_offset, seg->payload + payload_offset, copy_size); - data_len += copy_size; + smsg_offset += copy_size; ra_base_seq += copy_size; SCLogDebug("ra_base_seq %"PRIu32, ra_base_seq); + smsg->data.data_len += copy_size; SCLogDebug("copied payload_offset %" PRIu32 ", " - "data_len %" PRIu32 ", copy_size %" PRIu32 "", - payload_offset, data_len, copy_size); - - if (data_len == sizeof(data)) { - /* if app layer protocol has not been detected till yet, - then check did we have sent message to app layer already - or not. If not then sent the message and set flag that first - message has been sent. No more data till proto has not - been detected */ - if (!(ssn->flags & STREAMTCP_FLAG_APPPROTO_DETECTION_COMPLETED)) { - /* process what we have so far */ - STREAM_SET_FLAGS(ssn, stream, p, flags); - BUG_ON(data_len > sizeof(data)); - AppLayerHandleTCPData(&ra_ctx->dp_ctx, p->flow, ssn, - data, data_len, flags); - data_len = 0; - - stream->tmp_ra_app_base_seq = ra_base_seq; - } else { - /* process what we have so far */ - STREAM_SET_FLAGS(ssn, stream, p, flags); - BUG_ON(data_len > sizeof(data)); - AppLayerHandleTCPData(&ra_ctx->dp_ctx, p->flow, ssn, - data, data_len, flags); - data_len = 0; - - stream->ra_app_base_seq = ra_base_seq; - } + "smsg_offset %" PRIu32 ", copy_size %" PRIu32 "", + payload_offset, smsg_offset, copy_size); + if (smsg->data.data_len == sizeof (smsg->data.data)) { + StreamMsgPutInQueue(ra_ctx->stream_q, smsg); + stream->ra_raw_base_seq = ra_base_seq; + smsg = NULL; } /* see if we have segment payload left to process */ @@ -2595,36 +2511,24 @@ static int StreamTcpReassembleAppLayer (TcpReassemblyThreadCtx *ra_ctx, /* done with this segment, return it to the pool */ TcpSegment *next_seg = seg->next; next_seq = seg->seq + seg->payload_len; - seg->flags |= SEGMENTTCP_FLAG_APPLAYER_PROCESSED; - seg = next_seg; - } - - /* put the partly filled smsg in the queue to the l7 handler */ - if (data_len > 0) { - SCLogDebug("data_len > 0, %u", data_len); - if (!(ssn->flags & STREAMTCP_FLAG_APPPROTO_DETECTION_COMPLETED)) { - /* process what we have so far */ - STREAM_SET_FLAGS(ssn, stream, p, flags); - BUG_ON(data_len > sizeof(data)); - AppLayerHandleTCPData(&ra_ctx->dp_ctx, p->flow, ssn, - data, data_len, flags); - stream->tmp_ra_app_base_seq = ra_base_seq; - } else { - /* process what we have so far */ - STREAM_SET_FLAGS(ssn, stream, p, flags); - BUG_ON(data_len > sizeof(data)); - AppLayerHandleTCPData(&ra_ctx->dp_ctx, p->flow, ssn, - data, data_len, flags); - stream->ra_app_base_seq = ra_base_seq; + if (SEQ_LT((seg->seq + seg->payload_len), (ra_base_seq - stream_config.reassembly_inline_window))) { + if (seg->flags & SEGMENTTCP_FLAG_APPLAYER_PROCESSED) { + StreamTcpRemoveSegmentFromStream(stream, seg); + SCLogDebug("removing seg %p, seg->next %p", seg, seg->next); + StreamTcpSegmentReturntoPool(seg); + } else { + seg->flags |= SEGMENTTCP_FLAG_RAW_PROCESSED; + } } + seg = next_seg; } - if (ssn->flags & STREAMTCP_FLAG_APPPROTO_DETECTION_COMPLETED) { - SCLogDebug("proto detection already completed"); - ssn->flags |= STREAMTCP_FLAG_TOSERVER_REASSEMBLY_STARTED; - } else { - SCLogDebug("protocol detection not yet completed"); + /* put the partly filled smsg in the queue */ + if (smsg != NULL) { + StreamMsgPutInQueue(ra_ctx->stream_q, smsg); + smsg = NULL; + stream->ra_raw_base_seq = ra_base_seq; } SCReturnInt(0); @@ -2632,14 +2536,20 @@ static int StreamTcpReassembleAppLayer (TcpReassemblyThreadCtx *ra_ctx, /** * \brief Update the stream reassembly upon receiving an ACK packet. + * + * Stream is in the opposite direction of the packet, as the ACK-packet + * is ACK'ing the stream. + * * \todo this function is too long, we need to break it up. It needs it BAD */ -static int StreamTcpReassembleRaw (TcpReassemblyThreadCtx *ra_ctx, +static int StreamTcpReassembleAppLayer (TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p) { SCEnter(); - SCLogDebug("start p %p", p); + uint8_t flags = 0; + + /* check if toserver reassembly has started before reassembling toclient. */ if (PKT_IS_TOSERVER(p) && !(ssn->flags & STREAMTCP_FLAG_TOSERVER_REASSEMBLY_STARTED)) { @@ -2648,17 +2558,20 @@ static int StreamTcpReassembleRaw (TcpReassemblyThreadCtx *ra_ctx, SCReturnInt(0); } + SCLogDebug("stream->seg_list %p", stream->seg_list); +#ifdef DEBUG + PrintList(stream->seg_list); +#endif + if (stream->seg_list == NULL) { /* send an empty EOF msg if we have no segments but TCP state * is beyond ESTABLISHED */ if (ssn->state > TCP_ESTABLISHED) { - StreamMsg *smsg = StreamMsgGetFromPool(); - if (smsg == NULL) { - SCLogDebug("stream_msg_pool is empty"); - SCReturnInt(-1); - } - StreamTcpSetupMsg(ssn, stream, p, smsg); - StreamMsgPutInQueue(ra_ctx->stream_q,smsg); + SCLogDebug("sending empty eof message"); + /* send EOF to app layer */ + STREAM_SET_FLAGS(ssn, stream, p, flags); + AppLayerHandleTCPData(&ra_ctx->dp_ctx, p->flow, ssn, + NULL, 0, flags|STREAM_EOF); /* even if app layer detection failed, we will now move on to * release reassembly for both directions. */ @@ -2678,42 +2591,68 @@ static int StreamTcpReassembleRaw (TcpReassemblyThreadCtx *ra_ctx, SCReturnInt(0); } - if (ssn->state <= TCP_ESTABLISHED && - !(ssn->flags & STREAMTCP_FLAG_APPPROTO_DETECTION_COMPLETED)) { - SCLogDebug("only starting raw reassembly after app layer protocol " - "detection has completed."); - SCReturnInt(0); - } - - /* check if we have enough data */ - if (StreamTcpReassembleRawCheckLimit(ssn,stream,p) == 0) { - SCLogDebug("not yet reassembling"); + if (stream->flags & STREAMTCP_STREAM_FLAG_GAP) { SCReturnInt(0); } - uint32_t ra_base_seq = stream->ra_raw_base_seq; - StreamMsg *smsg = NULL; - uint16_t smsg_offset = 0; - uint16_t payload_offset = 0; - uint16_t payload_len = 0; - TcpSegment *seg = stream->seg_list; - uint32_t next_seq = ra_base_seq + 1; + uint32_t ra_base_seq; - if (SEQ_GEQ(seg->seq, stream->last_ack)) - SCLogDebug("seg is %"PRIu32" and st %"PRIu32"",seg->seq, stream->last_ack); + /* check if we have detected the app layer protocol or not. If it has been + detected then, process data normally, as we have sent one smsg from + toserver side already to the app layer */ + if (ssn->state <= TCP_ESTABLISHED) { + if (!(ssn->flags & STREAMTCP_FLAG_APPPROTO_DETECTION_COMPLETED)) { + /* Do not perform reassembling of data from server, until the app layer + proto has been detected and we have sent atleast one smsg from client + data to app layer */ + if (PKT_IS_TOSERVER(p)) { + SCLogDebug("we didn't detected the app layer protocol till " + "yet, so not doing toclient reassembling"); + SCReturnInt(0); + } + /* initialize the tmp_ra_base_seq for each new run */ + stream->tmp_ra_app_base_seq = stream->ra_app_base_seq; + ra_base_seq = stream->tmp_ra_app_base_seq; + /* if app layer protocol has been detected, then restore the reassembled + seq. to the value till reassembling has been done and unset the queue + init flag permanently for this tcp session */ + } else if (SEQ_GT(stream->tmp_ra_app_base_seq, stream->ra_app_base_seq)) { + stream->ra_app_base_seq = stream->tmp_ra_app_base_seq; + ra_base_seq = stream->ra_app_base_seq; + SCLogDebug("the app layer protocol has been detected"); + } else { + ra_base_seq = stream->ra_app_base_seq; + } + /* set the ra_bas_seq to stream->ra_base_seq as now app layer protocol + has been detected */ + } else { + if (SEQ_GT(stream->tmp_ra_app_base_seq, stream->ra_app_base_seq)) { + stream->ra_app_base_seq = stream->tmp_ra_app_base_seq; + ra_base_seq = stream->ra_app_base_seq; + } else { + ra_base_seq = stream->ra_app_base_seq; + } + } + + SCLogDebug("ra_base_seq %u", ra_base_seq); + + uint8_t data[4096]; + uint32_t data_len = 0; + uint16_t payload_offset = 0; + uint16_t payload_len = 0; + uint32_t next_seq = ra_base_seq + 1; /* loop through the segments and fill one or more msgs */ + TcpSegment *seg = stream->seg_list; + SCLogDebug("pre-loop seg %p", seg); for (; seg != NULL && SEQ_LT(seg->seq, stream->last_ack);) { SCLogDebug("seg %p", seg); /* if app layer protocol has been detected, then remove all the segments - * which has been previously processed and reassembled - * - * If the stream is in GAP state the app layer flag won't be set */ + which has been previously processed and reassembled */ if ((ssn->flags & STREAMTCP_FLAG_APPPROTO_DETECTION_COMPLETED) && (seg->flags & SEGMENTTCP_FLAG_RAW_PROCESSED) && - (seg->flags & SEGMENTTCP_FLAG_APPLAYER_PROCESSED || - stream->flags & STREAMTCP_STREAM_FLAG_GAP)) + (seg->flags & SEGMENTTCP_FLAG_APPLAYER_PROCESSED)) { SCLogDebug("segment(%p) of length %"PRIu16" has been processed," " so return it to pool", seg, seg->payload_len); @@ -2730,17 +2669,21 @@ static int StreamTcpReassembleRaw (TcpReassemblyThreadCtx *ra_ctx, SCLogDebug("checking for pre ra_base_seq %"PRIu32" seg %p seq %"PRIu32"" " len %"PRIu16", combined %"PRIu32" and stream->last_ack " "%"PRIu32"", ra_base_seq, seg, seg->seq, - seg->payload_len, seg->seq+seg->payload_len, stream->last_ack); + seg->payload_len, seg->seq+seg->payload_len, stream->last_ack); /* Remove the segments which are either completely before the ra_base_seq or if they are beyond ra_base_seq, but the segment offset from which we need to copy in to smsg is beyond the stream->last_ack. As we are copying until the stream->last_ack only */ - if (SEQ_LT((seg->seq + seg->payload_len), ra_base_seq)) + /** \todo we should probably not even insert them into the seglist */ + + if ((SEQ_LEQ((seg->seq + seg->payload_len), (ra_base_seq+1)) || + SEQ_LEQ(stream->last_ack, (ra_base_seq + (ra_base_seq - seg->seq)))) && + (seg->flags & SEGMENTTCP_FLAG_RAW_PROCESSED) && + (seg->flags & SEGMENTTCP_FLAG_APPLAYER_PROCESSED)) { - SCLogDebug("removing pre ra_base_seq %"PRIu32" seg %p seq %"PRIu32"" - " len %"PRIu16"", ra_base_seq, seg, seg->seq, - seg->payload_len); + SCLogDebug("removing pre ra_base_seq %"PRIu32" seg %p seq %"PRIu32 + " len %"PRIu16"", ra_base_seq, seg, seg->seq, seg->payload_len); TcpSegment *next_seg = seg->next; StreamTcpRemoveSegmentFromStream(stream, seg); @@ -2752,16 +2695,31 @@ static int StreamTcpReassembleRaw (TcpReassemblyThreadCtx *ra_ctx, /* we've run into a sequence gap */ if (SEQ_GT(seg->seq, next_seq)) { - /* pass on pre existing smsg (if any) */ - if (smsg != NULL && smsg->data.data_len > 0) { + /* first, pass on data before the gap */ + if (data_len > 0) { + SCLogDebug("pre GAP data"); + + STREAM_SET_FLAGS(ssn, stream, p, flags); + /* if app layer protocol has not been detected till yet, then check did we have sent message to app layer already or not. If not then sent the message and set flag that first message has been sent. No more data till proto has not been detected */ - StreamMsgPutInQueue(ra_ctx->stream_q, smsg); - stream->ra_raw_base_seq = ra_base_seq; - smsg = NULL; + if (!(ssn->flags & STREAMTCP_FLAG_APPPROTO_DETECTION_COMPLETED)) { + /* process what we have so far */ + BUG_ON(data_len > sizeof(data)); + AppLayerHandleTCPData(&ra_ctx->dp_ctx, p->flow, ssn, + data, data_len, flags); + + stream->tmp_ra_app_base_seq = ra_base_seq; + } else { + /* process what we have so far */ + AppLayerHandleTCPData(&ra_ctx->dp_ctx, p->flow, ssn, + data, data_len, flags); + + stream->ra_app_base_seq = ra_base_seq; + } } /* don't conclude it's a gap straight away. If ra_base_seq is lower @@ -2771,23 +2729,24 @@ static int StreamTcpReassembleRaw (TcpReassemblyThreadCtx *ra_ctx, { /* see what the length of the gap is, gap length is seg->seq - * (ra_base_seq +1) */ +#ifdef DEBUG uint32_t gap_len = seg->seq - next_seq; SCLogDebug("expected next_seq %" PRIu32 ", got %" PRIu32 " , " "stream->last_ack %" PRIu32 ". Seq gap %" PRIu32"", next_seq, seg->seq, stream->last_ack, gap_len); +#endif next_seq = seg->seq; - if (smsg == NULL) { - smsg = StreamMsgGetFromPool(); - if (smsg == NULL) { - SCLogDebug("stream_msg_pool is empty"); - return -1; - } + /* we need to update the ra_base_seq, if app layer proto has + been detected and we are setting new stream message. Otherwise + every smsg will be with flag STREAM_START set, which we + don't want :-) */ + if (ssn->flags & STREAMTCP_FLAG_APPPROTO_DETECTION_COMPLETED) { + stream->ra_app_base_seq = ra_base_seq; + } else { + stream->tmp_ra_app_base_seq = ra_base_seq; } - stream->ra_raw_base_seq = ra_base_seq; - - StreamTcpSetupMsg(ssn, stream, p, smsg); /* We have missed the packet and end host has ack'd it, so * IDS should advance it's ra_base_seq and should not consider this @@ -2795,16 +2754,23 @@ static int StreamTcpReassembleRaw (TcpReassemblyThreadCtx *ra_ctx, * drop it anyway */ ra_base_seq = seg->seq - 1; - SCLogDebug("setting STREAM_GAP"); - smsg->flags |= STREAM_GAP; - smsg->gap.gap_size = gap_len; + /* send gap signal */ + STREAM_SET_FLAGS(ssn, stream, p, flags); + AppLayerHandleTCPData(&ra_ctx->dp_ctx, p->flow, ssn, + NULL, 0, flags|STREAM_GAP); + data_len = 0; - StreamMsgPutInQueue(ra_ctx->stream_q,smsg); - smsg = NULL; - smsg_offset = 0; + /* set a GAP flag and make sure not bothering this stream anymore */ + stream->flags |= STREAMTCP_STREAM_FLAG_GAP; + /* flag reassembly as started, so the to_client part can start */ + ssn->flags |= STREAMTCP_FLAG_TOSERVER_REASSEMBLY_STARTED; + + dbg_app_layer_gap++; + break; } else { SCLogDebug("possible GAP, but waiting to see if out of order " "packets might solve that"); + dbg_app_layer_gap_candidate++; break; } } @@ -2812,15 +2778,14 @@ static int StreamTcpReassembleRaw (TcpReassemblyThreadCtx *ra_ctx, /* if the segment ends beyond ra_base_seq we need to consider it */ if (SEQ_GT((seg->seq + seg->payload_len), ra_base_seq)) { SCLogDebug("seg->seq %" PRIu32 ", seg->payload_len %" PRIu32 ", " - "ra_base_seq %" PRIu32 "", seg->seq, - seg->payload_len, ra_base_seq); + "ra_base_seq %" PRIu32 "", seg->seq, + seg->payload_len, ra_base_seq); /* handle segments partly before ra_base_seq */ if (SEQ_GT(ra_base_seq, seg->seq)) { payload_offset = ra_base_seq - seg->seq; if (SEQ_LT(stream->last_ack, (seg->seq + seg->payload_len))) { - if (SEQ_LT(stream->last_ack, ra_base_seq)) { payload_len = (stream->last_ack - seg->seq); } else { @@ -2852,42 +2817,46 @@ static int StreamTcpReassembleRaw (TcpReassemblyThreadCtx *ra_ctx, break; } - if (smsg == NULL) { - smsg = StreamMsgGetFromPool(); - if (smsg == NULL) { - SCLogDebug("stream_msg_pool is empty"); - return -1; - } - - smsg_offset = 0; - - StreamTcpSetupMsg(ssn, stream, p, smsg); - } - smsg->data.seq = ra_base_seq; - - /* copy the data into the smsg */ - uint16_t copy_size = sizeof (smsg->data.data) - smsg_offset; + uint16_t copy_size = sizeof(data) - data_len; if (copy_size > payload_len) { copy_size = payload_len; } if (SCLogDebugEnabled()) { - BUG_ON(copy_size > sizeof(smsg->data.data)); + BUG_ON(copy_size > sizeof(data)); } SCLogDebug("copy_size is %"PRIu16"", copy_size); - memcpy(smsg->data.data + smsg_offset, seg->payload + payload_offset, - copy_size); - smsg_offset += copy_size; + memcpy(data + data_len, seg->payload + payload_offset, copy_size); + data_len += copy_size; ra_base_seq += copy_size; - SCLogDebug("ra_base_seq %"PRIu32, ra_base_seq); - - smsg->data.data_len += copy_size; + SCLogDebug("ra_base_seq %"PRIu32", data_len %"PRIu32, ra_base_seq, data_len); /* queue the smsg if it's full */ - if (smsg->data.data_len == sizeof (smsg->data.data)) { - StreamMsgPutInQueue(ra_ctx->stream_q, smsg); - stream->ra_raw_base_seq = ra_base_seq; - smsg = NULL; + if (data_len == sizeof(data)) { + /* if app layer protocol has not been detected till yet, + then check did we have sent message to app layer already + or not. If not then sent the message and set flag that first + message has been sent. No more data till proto has not + been detected */ + if (!(ssn->flags & STREAMTCP_FLAG_APPPROTO_DETECTION_COMPLETED)) { + /* process what we have so far */ + STREAM_SET_FLAGS(ssn, stream, p, flags); + BUG_ON(data_len > sizeof(data)); + AppLayerHandleTCPData(&ra_ctx->dp_ctx, p->flow, ssn, + data, data_len, flags); + data_len = 0; + + stream->tmp_ra_app_base_seq = ra_base_seq; + } else { + /* process what we have so far */ + STREAM_SET_FLAGS(ssn, stream, p, flags); + BUG_ON(data_len > sizeof(data)); + AppLayerHandleTCPData(&ra_ctx->dp_ctx, p->flow, ssn, + data, data_len, flags); + data_len = 0; + + stream->ra_app_base_seq = ra_base_seq; + } } /* if the payload len is bigger than what we copied, we handle the @@ -2911,43 +2880,53 @@ static int StreamTcpReassembleRaw (TcpReassemblyThreadCtx *ra_ctx, while (segment_done == FALSE) { SCLogDebug("new msg at offset %" PRIu32 ", payload_len " "%" PRIu32 "", payload_offset, payload_len); + data_len = 0; - /* get a new message - XXX we need a setup function */ - smsg = StreamMsgGetFromPool(); - if (smsg == NULL) { - SCLogDebug("stream_msg_pool is empty"); - SCReturnInt(-1); - } - smsg_offset = 0; - - StreamTcpSetupMsg(ssn, stream,p,smsg); - smsg->data.seq = ra_base_seq; - - copy_size = sizeof(smsg->data.data) - smsg_offset; + copy_size = sizeof(data) - data_len; if (copy_size > (seg->payload_len - payload_offset)) { copy_size = (seg->payload_len - payload_offset); } if (SCLogDebugEnabled()) { - BUG_ON(copy_size > sizeof(smsg->data.data)); + BUG_ON(copy_size > sizeof(data)); } - SCLogDebug("copy payload_offset %" PRIu32 ", smsg_offset " + SCLogDebug("copy payload_offset %" PRIu32 ", data_len " "%" PRIu32 ", copy_size %" PRIu32 "", - payload_offset, smsg_offset, copy_size); - memcpy(smsg->data.data + smsg_offset, seg->payload + + payload_offset, data_len, copy_size); + memcpy(data + data_len, seg->payload + payload_offset, copy_size); - smsg_offset += copy_size; + data_len += copy_size; ra_base_seq += copy_size; SCLogDebug("ra_base_seq %"PRIu32, ra_base_seq); - smsg->data.data_len += copy_size; SCLogDebug("copied payload_offset %" PRIu32 ", " - "smsg_offset %" PRIu32 ", copy_size %" PRIu32 "", - payload_offset, smsg_offset, copy_size); - if (smsg->data.data_len == sizeof (smsg->data.data)) { - StreamMsgPutInQueue(ra_ctx->stream_q, smsg); - stream->ra_raw_base_seq = ra_base_seq; - smsg = NULL; + "data_len %" PRIu32 ", copy_size %" PRIu32 "", + payload_offset, data_len, copy_size); + + if (data_len == sizeof(data)) { + /* if app layer protocol has not been detected till yet, + then check did we have sent message to app layer already + or not. If not then sent the message and set flag that first + message has been sent. No more data till proto has not + been detected */ + if (!(ssn->flags & STREAMTCP_FLAG_APPPROTO_DETECTION_COMPLETED)) { + /* process what we have so far */ + STREAM_SET_FLAGS(ssn, stream, p, flags); + BUG_ON(data_len > sizeof(data)); + AppLayerHandleTCPData(&ra_ctx->dp_ctx, p->flow, ssn, + data, data_len, flags); + data_len = 0; + + stream->tmp_ra_app_base_seq = ra_base_seq; + } else { + /* process what we have so far */ + STREAM_SET_FLAGS(ssn, stream, p, flags); + BUG_ON(data_len > sizeof(data)); + AppLayerHandleTCPData(&ra_ctx->dp_ctx, p->flow, ssn, + data, data_len, flags); + data_len = 0; + + stream->ra_app_base_seq = ra_base_seq; + } } /* see if we have segment payload left to process */ @@ -2971,173 +2950,556 @@ static int StreamTcpReassembleRaw (TcpReassemblyThreadCtx *ra_ctx, /* done with this segment, return it to the pool */ TcpSegment *next_seg = seg->next; next_seq = seg->seq + seg->payload_len; + seg->flags |= SEGMENTTCP_FLAG_APPLAYER_PROCESSED; + seg = next_seg; + } - if (seg->flags & SEGMENTTCP_FLAG_APPLAYER_PROCESSED) { - StreamTcpRemoveSegmentFromStream(stream, seg); - SCLogDebug("removing seg %p, seg->next %p", seg, seg->next); - StreamTcpSegmentReturntoPool(seg); + /* put the partly filled smsg in the queue to the l7 handler */ + if (data_len > 0) { + SCLogDebug("data_len > 0, %u", data_len); + if (!(ssn->flags & STREAMTCP_FLAG_APPPROTO_DETECTION_COMPLETED)) { + /* process what we have so far */ + STREAM_SET_FLAGS(ssn, stream, p, flags); + BUG_ON(data_len > sizeof(data)); + AppLayerHandleTCPData(&ra_ctx->dp_ctx, p->flow, ssn, + data, data_len, flags); + stream->tmp_ra_app_base_seq = ra_base_seq; } else { - seg->flags |= SEGMENTTCP_FLAG_RAW_PROCESSED; + /* process what we have so far */ + STREAM_SET_FLAGS(ssn, stream, p, flags); + BUG_ON(data_len > sizeof(data)); + AppLayerHandleTCPData(&ra_ctx->dp_ctx, p->flow, ssn, + data, data_len, flags); + + stream->ra_app_base_seq = ra_base_seq; } - seg = next_seg; } - /* put the partly filled smsg in the queue to the l7 handler */ - if (smsg != NULL) { - StreamMsgPutInQueue(ra_ctx->stream_q, smsg); - smsg = NULL; - stream->ra_raw_base_seq = ra_base_seq; + if (ssn->flags & STREAMTCP_FLAG_APPPROTO_DETECTION_COMPLETED) { + SCLogDebug("proto detection already completed"); + ssn->flags |= STREAMTCP_FLAG_TOSERVER_REASSEMBLY_STARTED; + } else { + SCLogDebug("protocol detection not yet completed"); } SCReturnInt(0); } -/** \brief update app layer and raw reassembly - * - * \retval r 0 on success, -1 on error +/** + * \brief Update the stream reassembly upon receiving an ACK packet. + * \todo this function is too long, we need to break it up. It needs it BAD */ -int StreamTcpReassembleHandleSegmentUpdateACK (TcpReassemblyThreadCtx *ra_ctx, +static int StreamTcpReassembleRaw (TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, TcpStream *stream, Packet *p) { SCEnter(); + SCLogDebug("start p %p", p); - SCLogDebug("stream->seg_list %p", stream->seg_list); - - int r = 0; - if (!(StreamTcpInlineMode())) { - if (StreamTcpReassembleAppLayer(ra_ctx, ssn, stream, p) < 0) - r = -1; + if (PKT_IS_TOSERVER(p) && + !(ssn->flags & STREAMTCP_FLAG_TOSERVER_REASSEMBLY_STARTED)) + { + SCLogDebug("toserver reassembling is not done yet, so " + "skipping reassembling at the moment for to_client"); + SCReturnInt(0); } - if (StreamTcpReassembleRaw(ra_ctx, ssn, stream, p) < 0) - r = -1; - - SCLogDebug("stream->seg_list %p", stream->seg_list); - SCReturnInt(r); -} - -/** \brief Handle the queue'd smsgs containing reassembled app layer data when - * we're running the app layer handling as part of the stream threads. - * - * \param ra_ctx Reassembly thread ctx, contains the queue with stream msgs - * - * \todo Currently we process all msgs even if we encounter an error in one - * of them. We do this to make sure the thread ctx's queue is emptied. - * Maybe we should just clear & return the msgs in case of error. - * - * \retval 0 ok - * \retval -1 error - */ -int StreamTcpReassembleProcessAppLayer(TcpReassemblyThreadCtx *ra_ctx) -{ - SCEnter(); - - int r = 0; - if (ra_ctx != NULL && ra_ctx->stream_q && ra_ctx->stream_q->len > 0) { - StreamMsg *smsg = NULL; - do { - smsg = StreamMsgGetFromQueue(ra_ctx->stream_q); - if (smsg != NULL) { - SCLogDebug("smsg %p, next %p, prev %p, flow %p, q->len %u, " - "smsg->data.datalen %u, direction %s%s", - smsg, smsg->next, smsg->prev, smsg->flow, - ra_ctx->stream_q->len, smsg->data.data_len, - smsg->flags & STREAM_TOSERVER ? "toserver":"", - smsg->flags & STREAM_TOCLIENT ? "toclient":""); - BUG_ON(smsg->flow == NULL); + if (stream->seg_list == NULL) { + /* send an empty EOF msg if we have no segments but TCP state + * is beyond ESTABLISHED */ + if (ssn->state > TCP_ESTABLISHED) { + StreamMsg *smsg = StreamMsgGetFromPool(); + if (smsg == NULL) { + SCLogDebug("stream_msg_pool is empty"); + SCReturnInt(-1); + } + StreamTcpSetupMsg(ssn, stream, p, smsg); + StreamMsgPutInQueue(ra_ctx->stream_q,smsg); - //PrintRawDataFp(stderr, smsg->data.data, smsg->data.data_len); + /* even if app layer detection failed, we will now move on to + * release reassembly for both directions. */ + ssn->flags |= STREAMTCP_FLAG_TOSERVER_REASSEMBLY_STARTED; - /* Handle the stream msg. No need to use locking, flow is - * already locked at this point. Don't break out of the - * loop if we encounter an error. */ - if (AppLayerHandleTCPMsg(&ra_ctx->dp_ctx, smsg) != 0) - r = -1; - } + } else { + SCLogDebug("no segments in the list to reassemble"); + } - } while (ra_ctx->stream_q->len > 0); + SCReturnInt(0); } - SCReturnInt(r); -} - -int StreamTcpReassembleHandleSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, - TcpSession *ssn, TcpStream *stream, - Packet *p, PacketQueue *pq) -{ - SCEnter(); - SCLogDebug("ssn %p, stream %p, p %p, p->payload_len %"PRIu16"", - ssn, stream, p, p->payload_len); + /* check if reassembling has been paused for the moment or not */ + if (stream->flags & STREAMTCP_STREAM_FLAG_PAUSE_REASSEMBLY) { + SCLogDebug("reassembling has been paused for this stream, so no" + " reassembling at the moment"); + SCReturnInt(0); + } - /* we need to update the opposing stream in - * StreamTcpReassembleHandleSegmentUpdateACK */ - TcpStream *opposing_stream = NULL; - if (stream == &ssn->client) { - opposing_stream = &ssn->server; - } else { - opposing_stream = &ssn->client; + if (ssn->state <= TCP_ESTABLISHED && + !(ssn->flags & STREAMTCP_FLAG_APPPROTO_DETECTION_COMPLETED)) { + SCLogDebug("only starting raw reassembly after app layer protocol " + "detection has completed."); + SCReturnInt(0); } - /* handle ack received */ - if (StreamTcpReassembleHandleSegmentUpdateACK(ra_ctx, ssn, opposing_stream, p) != 0) - { - SCLogDebug("StreamTcpReassembleHandleSegmentUpdateACK error"); - SCReturnInt(-1); + /* check if we have enough data */ + if (StreamTcpReassembleRawCheckLimit(ssn,stream,p) == 0) { + SCLogDebug("not yet reassembling"); + SCReturnInt(0); } - /* If no stream reassembly/application layer protocol inspection, then - simple return */ - if (p->payload_len > 0 && !(stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY)) { - if (!(StreamTcpInlineMode())) { - SCLogDebug("calling StreamTcpReassembleHandleSegmentHandleData"); + uint32_t ra_base_seq = stream->ra_raw_base_seq; + StreamMsg *smsg = NULL; + uint16_t smsg_offset = 0; + uint16_t payload_offset = 0; + uint16_t payload_len = 0; + TcpSegment *seg = stream->seg_list; + uint32_t next_seq = ra_base_seq + 1; - if (StreamTcpReassembleHandleSegmentHandleData(tv, ra_ctx, ssn, stream, p) != 0) { - SCLogDebug("StreamTcpReassembleHandleSegmentHandleData error"); - SCReturnInt(-1); - } - } else { - SCLogDebug("calling StreamTcpReassembleHandleSegmentHandleData"); + if (SEQ_GEQ(seg->seq, stream->last_ack)) + SCLogDebug("seg is %"PRIu32" and st %"PRIu32"",seg->seq, stream->last_ack); - if (StreamTcpReassembleHandleSegmentHandleData(tv, ra_ctx, ssn, stream, p) != 0) { - SCLogDebug("StreamTcpReassembleHandleSegmentHandleData error"); - SCReturnInt(-1); - } + /* loop through the segments and fill one or more msgs */ + for (; seg != NULL && SEQ_LT(seg->seq, stream->last_ack);) { + SCLogDebug("seg %p", seg); - if (StreamTcpReassembleInlineAppLayer(ra_ctx, ssn, stream, p) < 0) - SCReturnInt(-1); + /* if app layer protocol has been detected, then remove all the segments + * which has been previously processed and reassembled + * + * If the stream is in GAP state the app layer flag won't be set */ + if ((ssn->flags & STREAMTCP_FLAG_APPPROTO_DETECTION_COMPLETED) && + (seg->flags & SEGMENTTCP_FLAG_RAW_PROCESSED) && + (seg->flags & SEGMENTTCP_FLAG_APPLAYER_PROCESSED || + stream->flags & STREAMTCP_STREAM_FLAG_GAP)) + { + SCLogDebug("segment(%p) of length %"PRIu16" has been processed," + " so return it to pool", seg, seg->payload_len); + TcpSegment *next_seg = seg->next; + StreamTcpRemoveSegmentFromStream(stream, seg); + StreamTcpSegmentReturntoPool(seg); + seg = next_seg; + continue; } - p->flags |= PKT_STREAM_ADD; - } + /* If packets are fully before ra_base_seq, skip them. We do this + * because we've reassembled up to the ra_base_seq point already, + * so we won't do anything with segments before it anyway. */ + SCLogDebug("checking for pre ra_base_seq %"PRIu32" seg %p seq %"PRIu32"" + " len %"PRIu16", combined %"PRIu32" and stream->last_ack " + "%"PRIu32"", ra_base_seq, seg, seg->seq, + seg->payload_len, seg->seq+seg->payload_len, stream->last_ack); - SCReturnInt(0); -} + /* Remove the segments which are either completely before the + ra_base_seq or if they are beyond ra_base_seq, but the segment offset + from which we need to copy in to smsg is beyond the stream->last_ack. + As we are copying until the stream->last_ack only */ + if (SEQ_LT((seg->seq + seg->payload_len), ra_base_seq)) + { + SCLogDebug("removing pre ra_base_seq %"PRIu32" seg %p seq %"PRIu32"" + " len %"PRIu16"", ra_base_seq, seg, seg->seq, + seg->payload_len); -/** - * \brief Function to replace the data from a specific point up to given length. - * - * \param dst_seg Destination segment to replace the data - * \param src_seg Source segment of which data is to be written to destination - * \param start_point Starting point to replace the data onwards - * \param len Length up to which data is need to be replaced - * - * \todo VJ We can remove the abort()s later. - * \todo VJ Why not memcpy? - */ -void StreamTcpSegmentDataReplace(TcpSegment *dst_seg, TcpSegment *src_seg, - uint32_t start_point, uint16_t len) -{ - uint32_t seq; - uint16_t src_pos = 0; - uint16_t dst_pos = 0; + TcpSegment *next_seg = seg->next; + StreamTcpRemoveSegmentFromStream(stream, seg); + StreamTcpSegmentReturntoPool(seg); + seg = next_seg; + continue; + } - SCLogDebug("start_point %u", start_point); + /* we've run into a sequence gap */ + if (SEQ_GT(seg->seq, next_seq)) { - if (SEQ_GT(start_point, dst_seg->seq)) { - dst_pos = start_point - dst_seg->seq; - } else if (SEQ_LT(dst_seg->seq, start_point)) { - dst_pos = dst_seg->seq - start_point; - } + /* pass on pre existing smsg (if any) */ + if (smsg != NULL && smsg->data.data_len > 0) { + /* if app layer protocol has not been detected till yet, + then check did we have sent message to app layer already + or not. If not then sent the message and set flag that first + message has been sent. No more data till proto has not + been detected */ + StreamMsgPutInQueue(ra_ctx->stream_q, smsg); + stream->ra_raw_base_seq = ra_base_seq; + smsg = NULL; + } + + /* don't conclude it's a gap straight away. If ra_base_seq is lower + * than last_ack - the window, we consider it a gap. */ + if (SEQ_GT((stream->last_ack - stream->window), ra_base_seq) || + ssn->state > TCP_ESTABLISHED) + { + /* see what the length of the gap is, gap length is seg->seq - + * (ra_base_seq +1) */ + uint32_t gap_len = seg->seq - next_seq; + SCLogDebug("expected next_seq %" PRIu32 ", got %" PRIu32 " , " + "stream->last_ack %" PRIu32 ". Seq gap %" PRIu32"", + next_seq, seg->seq, stream->last_ack, gap_len); + + next_seq = seg->seq; + + if (smsg == NULL) { + smsg = StreamMsgGetFromPool(); + if (smsg == NULL) { + SCLogDebug("stream_msg_pool is empty"); + return -1; + } + } + stream->ra_raw_base_seq = ra_base_seq; + + StreamTcpSetupMsg(ssn, stream, p, smsg); + + /* We have missed the packet and end host has ack'd it, so + * IDS should advance it's ra_base_seq and should not consider this + * packet any longer, even if it is retransmitted, as end host will + * drop it anyway */ + ra_base_seq = seg->seq - 1; + + SCLogDebug("setting STREAM_GAP"); + smsg->flags |= STREAM_GAP; + smsg->gap.gap_size = gap_len; + + StreamMsgPutInQueue(ra_ctx->stream_q,smsg); + smsg = NULL; + smsg_offset = 0; + } else { + SCLogDebug("possible GAP, but waiting to see if out of order " + "packets might solve that"); + break; + } + } + + /* if the segment ends beyond ra_base_seq we need to consider it */ + if (SEQ_GT((seg->seq + seg->payload_len), ra_base_seq)) { + SCLogDebug("seg->seq %" PRIu32 ", seg->payload_len %" PRIu32 ", " + "ra_base_seq %" PRIu32 "", seg->seq, + seg->payload_len, ra_base_seq); + + /* handle segments partly before ra_base_seq */ + if (SEQ_GT(ra_base_seq, seg->seq)) { + payload_offset = ra_base_seq - seg->seq; + + if (SEQ_LT(stream->last_ack, (seg->seq + seg->payload_len))) { + + if (SEQ_LT(stream->last_ack, ra_base_seq)) { + payload_len = (stream->last_ack - seg->seq); + } else { + payload_len = (stream->last_ack - seg->seq) - payload_offset; + } + } else { + payload_len = seg->payload_len - payload_offset; + } + + if (SCLogDebugEnabled()) { + BUG_ON(payload_offset > seg->payload_len); + BUG_ON((payload_len + payload_offset) > seg->payload_len); + } + } else { + payload_offset = 0; + + if (SEQ_LT(stream->last_ack, (seg->seq + seg->payload_len))) { + payload_len = stream->last_ack - seg->seq; + } else { + payload_len = seg->payload_len; + } + } + SCLogDebug("payload_offset is %"PRIu16", payload_len is %"PRIu16"" + " and stream->last_ack is %"PRIu32"", payload_offset, + payload_len, stream->last_ack); + + if (payload_len == 0) { + SCLogDebug("no payload_len, so bail out"); + break; + } + + if (smsg == NULL) { + smsg = StreamMsgGetFromPool(); + if (smsg == NULL) { + SCLogDebug("stream_msg_pool is empty"); + return -1; + } + + smsg_offset = 0; + + StreamTcpSetupMsg(ssn, stream, p, smsg); + } + smsg->data.seq = ra_base_seq; + + + /* copy the data into the smsg */ + uint16_t copy_size = sizeof (smsg->data.data) - smsg_offset; + if (copy_size > payload_len) { + copy_size = payload_len; + } + if (SCLogDebugEnabled()) { + BUG_ON(copy_size > sizeof(smsg->data.data)); + } + SCLogDebug("copy_size is %"PRIu16"", copy_size); + memcpy(smsg->data.data + smsg_offset, seg->payload + payload_offset, + copy_size); + smsg_offset += copy_size; + ra_base_seq += copy_size; + SCLogDebug("ra_base_seq %"PRIu32, ra_base_seq); + + smsg->data.data_len += copy_size; + + /* queue the smsg if it's full */ + if (smsg->data.data_len == sizeof (smsg->data.data)) { + StreamMsgPutInQueue(ra_ctx->stream_q, smsg); + stream->ra_raw_base_seq = ra_base_seq; + smsg = NULL; + } + + /* if the payload len is bigger than what we copied, we handle the + * rest of the payload next... */ + if (copy_size < payload_len) { + SCLogDebug("copy_size %" PRIu32 " < %" PRIu32 "", copy_size, + payload_len); + + payload_offset += copy_size; + payload_len -= copy_size; + SCLogDebug("payload_offset is %"PRIu16", seg->payload_len is " + "%"PRIu16" and stream->last_ack is %"PRIu32"", + payload_offset, seg->payload_len, stream->last_ack); + if (SCLogDebugEnabled()) { + BUG_ON(payload_offset > seg->payload_len); + } + + /* we need a while loop here as the packets theoretically can be + * 64k */ + char segment_done = FALSE; + while (segment_done == FALSE) { + SCLogDebug("new msg at offset %" PRIu32 ", payload_len " + "%" PRIu32 "", payload_offset, payload_len); + + /* get a new message + XXX we need a setup function */ + smsg = StreamMsgGetFromPool(); + if (smsg == NULL) { + SCLogDebug("stream_msg_pool is empty"); + SCReturnInt(-1); + } + smsg_offset = 0; + + StreamTcpSetupMsg(ssn, stream,p,smsg); + smsg->data.seq = ra_base_seq; + + copy_size = sizeof(smsg->data.data) - smsg_offset; + if (copy_size > (seg->payload_len - payload_offset)) { + copy_size = (seg->payload_len - payload_offset); + } + if (SCLogDebugEnabled()) { + BUG_ON(copy_size > sizeof(smsg->data.data)); + } + + SCLogDebug("copy payload_offset %" PRIu32 ", smsg_offset " + "%" PRIu32 ", copy_size %" PRIu32 "", + payload_offset, smsg_offset, copy_size); + memcpy(smsg->data.data + smsg_offset, seg->payload + + payload_offset, copy_size); + smsg_offset += copy_size; + ra_base_seq += copy_size; + SCLogDebug("ra_base_seq %"PRIu32, ra_base_seq); + smsg->data.data_len += copy_size; + SCLogDebug("copied payload_offset %" PRIu32 ", " + "smsg_offset %" PRIu32 ", copy_size %" PRIu32 "", + payload_offset, smsg_offset, copy_size); + if (smsg->data.data_len == sizeof (smsg->data.data)) { + StreamMsgPutInQueue(ra_ctx->stream_q, smsg); + stream->ra_raw_base_seq = ra_base_seq; + smsg = NULL; + } + + /* see if we have segment payload left to process */ + if ((copy_size + payload_offset) < seg->payload_len) { + payload_offset += copy_size; + payload_len -= copy_size; + + if (SCLogDebugEnabled()) { + BUG_ON(payload_offset > seg->payload_len); + } + } else { + payload_offset = 0; + segment_done = TRUE; + } + } + } else { + payload_offset = 0; + } + } + + /* done with this segment, return it to the pool */ + TcpSegment *next_seg = seg->next; + next_seq = seg->seq + seg->payload_len; + + if (seg->flags & SEGMENTTCP_FLAG_APPLAYER_PROCESSED) { + StreamTcpRemoveSegmentFromStream(stream, seg); + SCLogDebug("removing seg %p, seg->next %p", seg, seg->next); + StreamTcpSegmentReturntoPool(seg); + } else { + seg->flags |= SEGMENTTCP_FLAG_RAW_PROCESSED; + } + seg = next_seg; + } + + /* put the partly filled smsg in the queue to the l7 handler */ + if (smsg != NULL) { + StreamMsgPutInQueue(ra_ctx->stream_q, smsg); + smsg = NULL; + stream->ra_raw_base_seq = ra_base_seq; + } + + SCReturnInt(0); +} + +/** \brief update app layer and raw reassembly + * + * \retval r 0 on success, -1 on error + */ +int StreamTcpReassembleHandleSegmentUpdateACK (TcpReassemblyThreadCtx *ra_ctx, + TcpSession *ssn, TcpStream *stream, Packet *p) +{ + SCEnter(); + + SCLogDebug("stream->seg_list %p", stream->seg_list); + + int r = 0; + if (!(StreamTcpInlineMode())) { + if (StreamTcpReassembleAppLayer(ra_ctx, ssn, stream, p) < 0) + r = -1; + if (StreamTcpReassembleRaw(ra_ctx, ssn, stream, p) < 0) + r = -1; + } + + SCLogDebug("stream->seg_list %p", stream->seg_list); + SCReturnInt(r); +} + +/** \brief Handle the queue'd smsgs containing reassembled app layer data when + * we're running the app layer handling as part of the stream threads. + * + * \param ra_ctx Reassembly thread ctx, contains the queue with stream msgs + * + * \todo Currently we process all msgs even if we encounter an error in one + * of them. We do this to make sure the thread ctx's queue is emptied. + * Maybe we should just clear & return the msgs in case of error. + * + * \retval 0 ok + * \retval -1 error + */ +int StreamTcpReassembleProcessAppLayer(TcpReassemblyThreadCtx *ra_ctx) +{ + SCEnter(); + + int r = 0; + if (ra_ctx != NULL && ra_ctx->stream_q && ra_ctx->stream_q->len > 0) { + StreamMsg *smsg = NULL; + do { + smsg = StreamMsgGetFromQueue(ra_ctx->stream_q); + if (smsg != NULL) { + SCLogDebug("smsg %p, next %p, prev %p, flow %p, q->len %u, " + "smsg->data.datalen %u, direction %s%s", + smsg, smsg->next, smsg->prev, smsg->flow, + ra_ctx->stream_q->len, smsg->data.data_len, + smsg->flags & STREAM_TOSERVER ? "toserver":"", + smsg->flags & STREAM_TOCLIENT ? "toclient":""); + + BUG_ON(smsg->flow == NULL); + + //PrintRawDataFp(stderr, smsg->data.data, smsg->data.data_len); + + /* Handle the stream msg. No need to use locking, flow is + * already locked at this point. Don't break out of the + * loop if we encounter an error. */ + if (AppLayerHandleTCPMsg(&ra_ctx->dp_ctx, smsg) != 0) + r = -1; + } + + } while (ra_ctx->stream_q->len > 0); + } + + SCReturnInt(r); +} + +int StreamTcpReassembleHandleSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, + TcpSession *ssn, TcpStream *stream, + Packet *p, PacketQueue *pq) +{ + SCEnter(); + SCLogDebug("ssn %p, stream %p, p %p, p->payload_len %"PRIu16"", + ssn, stream, p, p->payload_len); + + /* we need to update the opposing stream in + * StreamTcpReassembleHandleSegmentUpdateACK */ + TcpStream *opposing_stream = NULL; + if (stream == &ssn->client) { + opposing_stream = &ssn->server; + } else { + opposing_stream = &ssn->client; + } + + /* handle ack received */ + if (StreamTcpReassembleHandleSegmentUpdateACK(ra_ctx, ssn, opposing_stream, p) != 0) + { + SCLogDebug("StreamTcpReassembleHandleSegmentUpdateACK error"); + SCReturnInt(-1); + } + + /* If no stream reassembly/application layer protocol inspection, then + simple return */ + if (p->payload_len > 0 && !(stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY)) { + if (!(StreamTcpInlineMode())) { + SCLogDebug("calling StreamTcpReassembleHandleSegmentHandleData"); + + if (StreamTcpReassembleHandleSegmentHandleData(tv, ra_ctx, ssn, stream, p) != 0) { + SCLogDebug("StreamTcpReassembleHandleSegmentHandleData error"); + SCReturnInt(-1); + } + } else { + SCLogDebug("calling StreamTcpReassembleHandleSegmentHandleData"); + + if (StreamTcpReassembleHandleSegmentHandleData(tv, ra_ctx, ssn, stream, p) != 0) { + SCLogDebug("StreamTcpReassembleHandleSegmentHandleData error"); + SCReturnInt(-1); + } + + int r = 0; + if (StreamTcpReassembleInlineAppLayer(ra_ctx, ssn, stream, p) < 0) + r = -1; + if (StreamTcpReassembleInlineAppLayer(ra_ctx, ssn, stream, p) < 0) + r = -1; + + if (r < 0) { + SCReturnInt(-1); + } + } + + p->flags |= PKT_STREAM_ADD; + } + + SCReturnInt(0); +} + +/** + * \brief Function to replace the data from a specific point up to given length. + * + * \param dst_seg Destination segment to replace the data + * \param src_seg Source segment of which data is to be written to destination + * \param start_point Starting point to replace the data onwards + * \param len Length up to which data is need to be replaced + * + * \todo VJ We can remove the abort()s later. + * \todo VJ Why not memcpy? + */ +void StreamTcpSegmentDataReplace(TcpSegment *dst_seg, TcpSegment *src_seg, + uint32_t start_point, uint16_t len) +{ + uint32_t seq; + uint16_t src_pos = 0; + uint16_t dst_pos = 0; + + SCLogDebug("start_point %u", start_point); + + if (SEQ_GT(start_point, dst_seg->seq)) { + dst_pos = start_point - dst_seg->seq; + } else if (SEQ_LT(dst_seg->seq, start_point)) { + dst_pos = dst_seg->seq - start_point; + } if (SCLogDebugEnabled()) { BUG_ON(((len + dst_pos) - 1) > dst_seg->payload_len); @@ -3225,6 +3587,10 @@ TcpSegment* StreamTcpGetSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, /* Increment the counter to show that we are not able to serve the segment request due to memcap limit */ SCPerfCounterIncr(ra_ctx->counter_tcp_segment_memcap, tv->sc_perf_pca); + } else { + seg->flags = 0; + seg->next = NULL; + seg->prev = NULL; } #ifdef DEBUG @@ -3233,37 +3599,9 @@ TcpSegment* StreamTcpGetSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, SCMutexUnlock(&segment_pool_cnt_mutex); #endif - seg->flags = 0; - seg->next = NULL; - seg->prev = NULL; return seg; } -/** - * \brief Function to return the segment back to the pool. - * - * \param seg Segment which will be returned back to the pool. - */ - -void StreamTcpSegmentReturntoPool(TcpSegment *seg) -{ - seg->next = NULL; - seg->prev = NULL; - - uint16_t idx = segment_pool_idx[seg->pool_size]; - SCMutexLock(&segment_pool_mutex[idx]); - PoolReturn(segment_pool[idx], (void *) seg); - SCLogDebug("segment_pool[%"PRIu16"]->empty_list_size %"PRIu32"", - idx,segment_pool[idx]->empty_list_size); - SCMutexUnlock(&segment_pool_mutex[idx]); - -#ifdef DEBUG - SCMutexLock(&segment_pool_cnt_mutex); - segment_pool_cnt--; - SCMutexUnlock(&segment_pool_cnt_mutex); -#endif -} - #ifdef UNITTESTS /** unit tests and it's support functions below */ @@ -5172,33 +5510,208 @@ static int StreamTcpReassembleTest31 (void) { goto end; } - flowflags = FLOW_PKT_TOSERVER; - StreamTcpCreateTestPacket(payload, 0x42, 1, 4); /*B*/ - seq = 16; - ack = 20; - if (StreamTcpTestMissedPacket (ra_ctx, &ssn, seq, ack, payload, 1, th_flag, flowflags, TCP_ESTABLISHED) == -1) { - printf("failed in segments reassembly: "); - goto end; - } + flowflags = FLOW_PKT_TOSERVER; + StreamTcpCreateTestPacket(payload, 0x42, 1, 4); /*B*/ + seq = 16; + ack = 20; + if (StreamTcpTestMissedPacket (ra_ctx, &ssn, seq, ack, payload, 1, th_flag, flowflags, TCP_ESTABLISHED) == -1) { + printf("failed in segments reassembly: "); + goto end; + } + + if (StreamTcpCheckStreamContents(check_contents, 5, &ssn.server) == 0) { + printf("failed in stream matching: "); + goto end; + } + + if (ssn.server.seg_list_tail->seq != 16) { + printf("failed in fast track handling: "); + goto end; + } + + ret = 1; +end: + StreamTcpFreeConfig(TRUE); + StreamTcpReassembleFreeThreadCtx(ra_ctx); + return ret; +} + +static int StreamTcpReassembleTest32(void) { + TcpSession ssn; + Packet *p = SCMalloc(SIZE_OF_PACKET); + if (p == NULL) + return 0; + Flow f; + TCPHdr tcph; + TcpReassemblyThreadCtx *ra_ctx = StreamTcpReassembleInitThreadCtx(); + TcpStream stream; + uint8_t ret = 0; + uint8_t check_contents[35] = {0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, + 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, + 0x41, 0x41, 0x41, 0x41, 0x42, 0x42, 0x42, 0x42, + 0x42, 0x43, 0x43, 0x43, 0x43, 0x43, 0x43, 0x43, + 0x43, 0x43, 0x43}; + memset(&stream, 0, sizeof (TcpStream)); + stream.os_policy = OS_POLICY_BSD; + uint8_t payload[20] = ""; + + StreamTcpInitConfig(TRUE); + + /* prevent L7 from kicking in */ + StreamMsgQueueSetMinInitChunkLen(FLOW_PKT_TOSERVER, 4096); + StreamMsgQueueSetMinInitChunkLen(FLOW_PKT_TOCLIENT, 4096); + StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 4096); + StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 4096); + + PacketQueue pq; + memset(&pq,0,sizeof(PacketQueue)); + memset(&ssn, 0, sizeof (TcpSession)); + memset(p, 0, SIZE_OF_PACKET); + p->pkt = (uint8_t *)(p + 1); + memset(&f, 0, sizeof (Flow)); + memset(&tcph, 0, sizeof (TCPHdr)); + ThreadVars tv; + memset(&tv, 0, sizeof (ThreadVars)); + FLOW_INITIALIZE(&f); + f.protoctx = &ssn; + p->src.family = AF_INET; + p->dst.family = AF_INET; + p->proto = IPPROTO_TCP; + p->flow = &f; + tcph.th_win = 5480; + tcph.th_flags = TH_PUSH | TH_ACK; + p->tcph = &tcph; + p->flowflags = FLOW_PKT_TOSERVER; + + p->tcph->th_seq = htonl(10); + p->tcph->th_ack = htonl(31); + p->payload_len = 10; + StreamTcpCreateTestPacket(payload, 0x41, 10, 20); /*AA*/ + p->payload = payload; + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) == -1) + goto end; + + p->tcph->th_seq = htonl(20); + p->tcph->th_ack = htonl(31); + p->payload_len = 10; + StreamTcpCreateTestPacket(payload, 0x42, 10, 20); /*BB*/ + p->payload = payload; + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) == -1) + goto end; + + p->tcph->th_seq = htonl(40); + p->tcph->th_ack = htonl(31); + p->payload_len = 10; + StreamTcpCreateTestPacket(payload, 0x43, 10, 20); /*CC*/ + p->payload = payload; + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) == -1) + goto end; + + p->tcph->th_seq = htonl(5); + p->tcph->th_ack = htonl(31); + p->payload_len = 20; + StreamTcpCreateTestPacket(payload, 0x41, 20, 20); /*AA*/ + p->payload = payload; + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) == -1) + goto end; + + if (StreamTcpCheckStreamContents(check_contents, 35, &stream) != 0) { + ret = 1; + } else { + printf("failed in stream matching: "); + } + + +end: + StreamTcpFreeConfig(TRUE); + SCFree(p); + return ret; +} + +static int StreamTcpReassembleTest33(void) { + TcpSession ssn; + Packet *p = SCMalloc(SIZE_OF_PACKET); + if (p == NULL) + return 0; + Flow f; + TCPHdr tcph; + TcpReassemblyThreadCtx *ra_ctx = StreamTcpReassembleInitThreadCtx(); + TcpStream stream; + memset(&stream, 0, sizeof (TcpStream)); + stream.os_policy = OS_POLICY_BSD; + uint8_t packet[1460] = ""; + + StreamTcpInitConfig(TRUE); + + /* prevent L7 from kicking in */ + StreamMsgQueueSetMinInitChunkLen(FLOW_PKT_TOSERVER, 4096); + StreamMsgQueueSetMinInitChunkLen(FLOW_PKT_TOCLIENT, 4096); + StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 4096); + StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 4096); + + PacketQueue pq; + memset(&pq,0,sizeof(PacketQueue)); + memset(&ssn, 0, sizeof (TcpSession)); + memset(p, 0, SIZE_OF_PACKET); + p->pkt = (uint8_t *)(p + 1); + memset(&f, 0, sizeof (Flow)); + memset(&tcph, 0, sizeof (TCPHdr)); + ThreadVars tv; + memset(&tv, 0, sizeof (ThreadVars)); + FLOW_INITIALIZE(&f); + f.protoctx = &ssn; + p->src.family = AF_INET; + p->dst.family = AF_INET; + p->proto = IPPROTO_TCP; + p->flow = &f; + tcph.th_win = 5480; + tcph.th_flags = TH_PUSH | TH_ACK; + p->tcph = &tcph; + p->flowflags = FLOW_PKT_TOSERVER; + p->payload = packet; + + p->tcph->th_seq = htonl(10); + p->tcph->th_ack = htonl(31); + p->payload_len = 10; + + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) == -1) { + SCFree(p); + return 0; + } + + p->tcph->th_seq = htonl(20); + p->tcph->th_ack = htonl(31); + p->payload_len = 10; + + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) == -1) { + SCFree(p); + return 0; + } + + p->tcph->th_seq = htonl(40); + p->tcph->th_ack = htonl(31); + p->payload_len = 10; - if (StreamTcpCheckStreamContents(check_contents, 5, &ssn.server) == 0) { - printf("failed in stream matching: "); - goto end; + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) == -1) { + SCFree(p); + return 0; } - if (ssn.server.seg_list_tail->seq != 16) { - printf("failed in fast track handling: "); - goto end; + p->tcph->th_seq = htonl(5); + p->tcph->th_ack = htonl(31); + p->payload_len = 30; + + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) == -1) { + SCFree(p); + return 0; } - ret = 1; -end: StreamTcpFreeConfig(TRUE); - StreamTcpReassembleFreeThreadCtx(ra_ctx); - return ret; + SCFree(p); + return 1; } -static int StreamTcpReassembleTest32(void) { +static int StreamTcpReassembleTest34(void) { TcpSession ssn; Packet *p = SCMalloc(SIZE_OF_PACKET); if (p == NULL) @@ -5207,15 +5720,9 @@ static int StreamTcpReassembleTest32(void) { TCPHdr tcph; TcpReassemblyThreadCtx *ra_ctx = StreamTcpReassembleInitThreadCtx(); TcpStream stream; - uint8_t ret = 0; - uint8_t check_contents[35] = {0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, - 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, 0x41, - 0x41, 0x41, 0x41, 0x41, 0x42, 0x42, 0x42, 0x42, - 0x42, 0x43, 0x43, 0x43, 0x43, 0x43, 0x43, 0x43, - 0x43, 0x43, 0x43}; memset(&stream, 0, sizeof (TcpStream)); stream.os_policy = OS_POLICY_BSD; - uint8_t payload[20] = ""; + uint8_t packet[1460] = ""; StreamTcpInitConfig(TRUE); @@ -5244,53 +5751,122 @@ static int StreamTcpReassembleTest32(void) { tcph.th_flags = TH_PUSH | TH_ACK; p->tcph = &tcph; p->flowflags = FLOW_PKT_TOSERVER; + p->payload = packet; - p->tcph->th_seq = htonl(10); + p->tcph->th_seq = htonl(857961230); p->tcph->th_ack = htonl(31); - p->payload_len = 10; - StreamTcpCreateTestPacket(payload, 0x41, 10, 20); /*AA*/ - p->payload = payload; - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) == -1) - goto end; + p->payload_len = 304; - p->tcph->th_seq = htonl(20); + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) == -1) { + SCFree(p); + return 0; + } + + p->tcph->th_seq = htonl(857961534); p->tcph->th_ack = htonl(31); - p->payload_len = 10; - StreamTcpCreateTestPacket(payload, 0x42, 10, 20); /*BB*/ - p->payload = payload; - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) == -1) - goto end; + p->payload_len = 1460; - p->tcph->th_seq = htonl(40); + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) == -1) { + SCFree(p); + return 0; + } + + p->tcph->th_seq = htonl(857963582); p->tcph->th_ack = htonl(31); - p->payload_len = 10; - StreamTcpCreateTestPacket(payload, 0x43, 10, 20); /*CC*/ - p->payload = payload; - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) == -1) - goto end; + p->payload_len = 1460; - p->tcph->th_seq = htonl(5); + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) == -1) { + SCFree(p); + return 0; + } + + p->tcph->th_seq = htonl(857960946); p->tcph->th_ack = htonl(31); - p->payload_len = 20; - StreamTcpCreateTestPacket(payload, 0x41, 20, 20); /*AA*/ - p->payload = payload; - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) == -1) - goto end; + p->payload_len = 1460; - if (StreamTcpCheckStreamContents(check_contents, 35, &stream) != 0) { - ret = 1; - } else { - printf("failed in stream matching: "); + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) == -1) { + SCFree(p); + return 0; } + StreamTcpFreeConfig(TRUE); + SCFree(p); + return 1; +} + +/** \test Test the bug 56 condition */ +static int StreamTcpReassembleTest35(void) { + TcpSession ssn; + Packet *p = SCMalloc(SIZE_OF_PACKET); + if (p == NULL) + return 0; + Flow f; + TCPHdr tcph; + TcpReassemblyThreadCtx *ra_ctx = StreamTcpReassembleInitThreadCtx(); + TcpStream stream; + memset(&stream, 0, sizeof (TcpStream)); + stream.os_policy = OS_POLICY_BSD; + uint8_t packet[1460] = ""; + + StreamTcpInitConfig(TRUE); + + /* prevent L7 from kicking in */ + StreamMsgQueueSetMinInitChunkLen(FLOW_PKT_TOSERVER, 10); + StreamMsgQueueSetMinInitChunkLen(FLOW_PKT_TOCLIENT, 10); + StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 10); + StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 10); + + PacketQueue pq; + memset(&pq,0,sizeof(PacketQueue)); + memset(&ssn, 0, sizeof (TcpSession)); + memset(p, 0, SIZE_OF_PACKET); + p->pkt = (uint8_t *)(p + 1); + memset(&f, 0, sizeof (Flow)); + memset(&tcph, 0, sizeof (TCPHdr)); + ThreadVars tv; + memset(&tv, 0, sizeof (ThreadVars)); + FLOW_INITIALIZE(&f); + f.protoctx = &ssn; + p->src.family = AF_INET; + p->dst.family = AF_INET; + p->proto = IPPROTO_TCP; + p->flow = &f; + tcph.th_win = 5480; + tcph.th_flags = TH_PUSH | TH_ACK; + p->tcph = &tcph; + p->flowflags = FLOW_PKT_TOSERVER; + p->payload = packet; + + p->tcph->th_seq = htonl(2257022155UL); + p->tcph->th_ack = htonl(1374943142); + p->payload_len = 142; + stream.last_ack = 2257022285UL; + stream.ra_raw_base_seq = 2257022172UL; + stream.ra_app_base_seq = 2257022172UL; + + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) == -1) { + SCFree(p); + return 0; + } + + p->tcph->th_seq = htonl(2257022285UL); + p->tcph->th_ack = htonl(1374943142); + p->payload_len = 34; + stream.last_ack = 2257022285UL; + stream.ra_raw_base_seq = 2257022172UL; + + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) == -1) { + SCFree(p); + return 0; + } -end: StreamTcpFreeConfig(TRUE); SCFree(p); - return ret; + return 1; } -static int StreamTcpReassembleTest33(void) { +/** \test Test the bug 57 condition */ +static int StreamTcpReassembleTest36(void) { TcpSession ssn; Packet *p = SCMalloc(SIZE_OF_PACKET); if (p == NULL) @@ -5306,20 +5882,92 @@ static int StreamTcpReassembleTest33(void) { StreamTcpInitConfig(TRUE); /* prevent L7 from kicking in */ - StreamMsgQueueSetMinInitChunkLen(FLOW_PKT_TOSERVER, 4096); - StreamMsgQueueSetMinInitChunkLen(FLOW_PKT_TOCLIENT, 4096); - StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 4096); - StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 4096); + StreamMsgQueueSetMinInitChunkLen(FLOW_PKT_TOSERVER, 10); + StreamMsgQueueSetMinInitChunkLen(FLOW_PKT_TOCLIENT, 10); + StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 10); + StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 10); + + PacketQueue pq; + memset(&pq,0,sizeof(PacketQueue)); + memset(&ssn, 0, sizeof (TcpSession)); + memset(p, 0, SIZE_OF_PACKET); + p->pkt = (uint8_t *)(p + 1); + memset(&f, 0, sizeof (Flow)); + memset(&tcph, 0, sizeof (TCPHdr)); + ThreadVars tv; + memset(&tv, 0, sizeof (ThreadVars)); + FLOW_INITIALIZE(&f); + f.protoctx = &ssn; + p->src.family = AF_INET; + p->dst.family = AF_INET; + p->proto = IPPROTO_TCP; + p->flow = &f; + tcph.th_win = 5480; + tcph.th_flags = TH_PUSH | TH_ACK; + p->tcph = &tcph; + p->flowflags = FLOW_PKT_TOSERVER; + p->payload = packet; + + p->tcph->th_seq = htonl(1549588966); + p->tcph->th_ack = htonl(4162241372UL); + p->payload_len = 204; + stream.last_ack = 1549589007; + stream.ra_raw_base_seq = 1549589101; + + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) == -1) { + SCFree(p); + return 0; + } + + p->tcph->th_seq = htonl(1549589007); + p->tcph->th_ack = htonl(4162241372UL); + p->payload_len = 23; + stream.last_ack = 1549589007; + stream.ra_raw_base_seq = 1549589101; + + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) == -1) { + SCFree(p); + return 0; + } + + StreamTcpFreeConfig(TRUE); + SCFree(p); + return 1; +} + +/** \test Test the bug 76 condition */ +static int StreamTcpReassembleTest37(void) { + TcpSession ssn; + Flow f; + TCPHdr tcph; + TcpReassemblyThreadCtx *ra_ctx = StreamTcpReassembleInitThreadCtx(); + TcpStream stream; + uint8_t packet[1460] = ""; + PacketQueue pq; + ThreadVars tv; + + Packet *p = SCMalloc(SIZE_OF_PACKET); + if (p == NULL) + return 0; + + StreamTcpInitConfig(TRUE); + + /* prevent L7 from kicking in */ + StreamMsgQueueSetMinInitChunkLen(FLOW_PKT_TOSERVER, 10); + StreamMsgQueueSetMinInitChunkLen(FLOW_PKT_TOCLIENT, 10); + StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 10); + StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 10); + + memset(p, 0, SIZE_OF_PACKET); + p->pkt = (uint8_t *)(p + 1); - PacketQueue pq; + memset(&stream, 0, sizeof (TcpStream)); memset(&pq,0,sizeof(PacketQueue)); memset(&ssn, 0, sizeof (TcpSession)); - memset(p, 0, SIZE_OF_PACKET); - p->pkt = (uint8_t *)(p + 1); memset(&f, 0, sizeof (Flow)); memset(&tcph, 0, sizeof (TCPHdr)); - ThreadVars tv; memset(&tv, 0, sizeof (ThreadVars)); + FLOW_INITIALIZE(&f); f.protoctx = &ssn; p->src.family = AF_INET; @@ -5331,37 +5979,39 @@ static int StreamTcpReassembleTest33(void) { p->tcph = &tcph; p->flowflags = FLOW_PKT_TOSERVER; p->payload = packet; + stream.os_policy = OS_POLICY_BSD; - p->tcph->th_seq = htonl(10); - p->tcph->th_ack = htonl(31); - p->payload_len = 10; - - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) == -1) { - SCFree(p); - return 0; - } - - p->tcph->th_seq = htonl(20); - p->tcph->th_ack = htonl(31); - p->payload_len = 10; + p->tcph->th_seq = htonl(3061088537UL); + p->tcph->th_ack = htonl(1729548549UL); + p->payload_len = 1391; + stream.last_ack = 3061091137UL; + stream.ra_raw_base_seq = 3061091309UL; + stream.ra_app_base_seq = 3061091309UL; - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) == -1) { + /* pre base_seq, so should be rejected */ + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) != -1) { SCFree(p); return 0; } - p->tcph->th_seq = htonl(40); - p->tcph->th_ack = htonl(31); - p->payload_len = 10; + p->tcph->th_seq = htonl(3061089928UL); + p->tcph->th_ack = htonl(1729548549UL); + p->payload_len = 1391; + stream.last_ack = 3061091137UL; + stream.ra_raw_base_seq = 3061091309UL; + stream.ra_app_base_seq = 3061091309UL; if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) == -1) { SCFree(p); return 0; } - p->tcph->th_seq = htonl(5); - p->tcph->th_ack = htonl(31); - p->payload_len = 30; + p->tcph->th_seq = htonl(3061091319UL); + p->tcph->th_ack = htonl(1729548549UL); + p->payload_len = 1391; + stream.last_ack = 3061091137UL; + stream.ra_raw_base_seq = 3061091309UL; + stream.ra_app_base_seq = 3061091309UL; if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) == -1) { SCFree(p); @@ -5373,327 +6023,673 @@ static int StreamTcpReassembleTest33(void) { return 1; } -static int StreamTcpReassembleTest34(void) { - TcpSession ssn; +/** + * \test Test to make sure we don't send the smsg from toclient to app layer + * until the app layer protocol has been detected and one smsg from + * toserver side has been sent to app layer. + * + * \retval On success it returns 1 and on failure 0. + */ + +static int StreamTcpReassembleTest38 (void) { + int ret = 0; Packet *p = SCMalloc(SIZE_OF_PACKET); if (p == NULL) return 0; Flow f; TCPHdr tcph; - TcpReassemblyThreadCtx *ra_ctx = StreamTcpReassembleInitThreadCtx(); - TcpStream stream; - memset(&stream, 0, sizeof (TcpStream)); - stream.os_policy = OS_POLICY_BSD; - uint8_t packet[1460] = ""; + Port sp; + Port dp; + Address src; + Address dst; + struct in_addr in; + TcpSession ssn; - StreamTcpInitConfig(TRUE); + memset(p, 0, SIZE_OF_PACKET); + p->pkt = (uint8_t *)(p + 1); + PacketQueue pq; + memset(&pq,0,sizeof(PacketQueue)); + memset(&f, 0, sizeof (Flow)); + memset(&tcph, 0, sizeof (TCPHdr)); + memset(&src, 0, sizeof(Address)); + memset(&dst, 0, sizeof(Address)); + memset(&ssn, 0, sizeof(TcpSession)); + ThreadVars tv; + memset(&tv, 0, sizeof (ThreadVars)); /* prevent L7 from kicking in */ - StreamMsgQueueSetMinInitChunkLen(FLOW_PKT_TOSERVER, 4096); - StreamMsgQueueSetMinInitChunkLen(FLOW_PKT_TOCLIENT, 4096); - StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 4096); - StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 4096); + StreamMsgQueueSetMinInitChunkLen(FLOW_PKT_TOSERVER, 0); + StreamMsgQueueSetMinInitChunkLen(FLOW_PKT_TOCLIENT, 0); + StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 0); + StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 0); + + FLOW_INITIALIZE(&f); + StreamTcpInitConfig(TRUE); + TcpReassemblyThreadCtx *ra_ctx = StreamTcpReassembleInitThreadCtx(); + AppLayerDetectProtoThreadInit(); + + uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\n"; + uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */ + + uint8_t httpbuf2[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n"; + uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */ + + inet_pton(AF_INET, "1.2.3.4", &in); + src.family = AF_INET; + src.addr_data32[0] = in.s_addr; + inet_pton(AF_INET, "1.2.3.5", &in); + dst.family = AF_INET; + dst.addr_data32[0] = in.s_addr; + sp = 200; + dp = 220; + + ssn.server.ra_raw_base_seq = ssn.server.ra_app_base_seq = 9; + ssn.server.isn = 9; + ssn.server.last_ack = 60; + ssn.client.ra_raw_base_seq = ssn.client.ra_app_base_seq = 9; + ssn.client.isn = 9; + ssn.client.last_ack = 60; + f.alproto = ALPROTO_UNKNOWN; + + f.src = src; + f.dst = dst; + f.sp = sp; + f.dp = dp; + f.protoctx = &ssn; + p->flow = &f; + + tcph.th_win = htons(5480); + tcph.th_seq = htonl(10); + tcph.th_ack = htonl(20); + tcph.th_flags = TH_ACK|TH_PUSH; + p->tcph = &tcph; + p->flowflags = FLOW_PKT_TOCLIENT; + + p->payload = httpbuf2; + p->payload_len = httplen2; + ssn.state = TCP_ESTABLISHED; + + TcpStream *s = NULL; + s = &ssn.server; + + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { + printf("failed in segments reassembly, while processing toserver packet\n"); + goto end; + } + + /* Check if we have stream smsgs in queue */ + if (ra_ctx->stream_q->len > 0) { + printf("there shouldn't be any stream smsgs in the queue\n"); + goto end; + } + + p->flowflags = FLOW_PKT_TOSERVER; + p->payload = httpbuf1; + p->payload_len = httplen1; + tcph.th_seq = htonl(10); + tcph.th_ack = htonl(55); + s = &ssn.client; + + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { + printf("failed in segments reassembly, while processing toserver packet\n"); + goto end; + } + + /* Check if we have stream smsgs in queue */ + if (ra_ctx->stream_q->len > 0) { + printf("there shouldn't be any stream smsgs in the queue, as we didn't" + " processed any smsg from toserver side till yet\n"); + goto end; + } + + p->flowflags = FLOW_PKT_TOCLIENT; + p->payload = httpbuf2; + p->payload_len = httplen2; + tcph.th_seq = htonl(55); + tcph.th_ack = htonl(53); + s = &ssn.server; + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { + printf("failed in segments reassembly, while processing toserver packet\n"); + goto end; + } + + /* Check if we have stream smsgs in queue */ + if (ra_ctx->stream_q->len == 0) { + printf("there should be a stream smsgs in the queue\n"); + goto end; + /* Process stream smsgs we may have in queue */ + } else if (StreamTcpReassembleProcessAppLayer(ra_ctx) < 0) { + printf("failed in processing stream smsgs\n"); + goto end; + } + + p->flowflags = FLOW_PKT_TOSERVER; + p->payload = httpbuf1; + p->payload_len = httplen1; + tcph.th_seq = htonl(53); + tcph.th_ack = htonl(100); + s = &ssn.client; + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { + printf("failed in segments reassembly, while processing toserver packet\n"); + goto end; + } + + p->flowflags = FLOW_PKT_TOCLIENT; + p->payload = NULL; + p->payload_len = 0; + tcph.th_seq = htonl(100); + tcph.th_ack = htonl(53); + s = &ssn.server; + + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { + printf("failed in segments reassembly, while processing toserver packet\n"); + goto end; + } +#if 0 + /* Check if we have stream smsgs in queue */ + if (ra_ctx->stream_q->len == 0) { + printf("there should be a stream smsgs in the queue, as we have detected" + " the app layer protocol and one smsg from toserver side has " + "been sent\n"); + goto end; + /* Process stream smsgs we may have in queue */ + } else if (StreamTcpReassembleProcessAppLayer(ra_ctx) < 0) { + printf("failed in processing stream smsgs\n"); + goto end; + } +#endif + ret = 1; +end: + StreamTcpFreeConfig(TRUE); + StreamTcpReassembleFreeThreadCtx(ra_ctx); + SCFree(p); + return ret; +} + +/** + * \test Test to make sure that we don't return the segments until the app + * layer proto has been detected and after that remove the processed + * segments. + * + * \retval On success it returns 1 and on failure 0. + */ + +static int StreamTcpReassembleTest39 (void) { + SCEnter(); + + int ret = 0; + Packet *p = SCMalloc(SIZE_OF_PACKET); + if (p == NULL) + return 0; + Flow f; + TCPHdr tcph; + Port sp; + Port dp; + Address src; + Address dst; + struct in_addr in; + TcpSession ssn; - PacketQueue pq; - memset(&pq,0,sizeof(PacketQueue)); - memset(&ssn, 0, sizeof (TcpSession)); memset(p, 0, SIZE_OF_PACKET); p->pkt = (uint8_t *)(p + 1); + PacketQueue pq; + memset(&pq,0,sizeof(PacketQueue)); memset(&f, 0, sizeof (Flow)); memset(&tcph, 0, sizeof (TCPHdr)); + memset(&src, 0, sizeof(Address)); + memset(&dst, 0, sizeof(Address)); + memset(&ssn, 0, sizeof(TcpSession)); ThreadVars tv; memset(&tv, 0, sizeof (ThreadVars)); + FLOW_INITIALIZE(&f); + StreamTcpInitConfig(TRUE); + TcpReassemblyThreadCtx *ra_ctx = StreamTcpReassembleInitThreadCtx(); + AppLayerDetectProtoThreadInit(); + StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 7); + StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 7); + + uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\n"; + uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */ + + uint8_t httpbuf2[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n"; + uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */ + + ssn.server.ra_raw_base_seq = ssn.server.ra_app_base_seq = 9; + ssn.server.isn = 9; + ssn.server.last_ack = 60; + ssn.client.ra_raw_base_seq = ssn.client.ra_app_base_seq= 9; + ssn.client.isn = 9; + ssn.client.last_ack = 60; + f.alproto = ALPROTO_UNKNOWN; + + inet_pton(AF_INET, "1.2.3.4", &in); + src.family = AF_INET; + src.addr_data32[0] = in.s_addr; + inet_pton(AF_INET, "1.2.3.5", &in); + dst.family = AF_INET; + dst.addr_data32[0] = in.s_addr; + sp = 200; + dp = 220; + + f.src = src; + f.dst = dst; + f.sp = sp; + f.dp = dp; f.protoctx = &ssn; - p->src.family = AF_INET; - p->dst.family = AF_INET; - p->proto = IPPROTO_TCP; p->flow = &f; - tcph.th_win = 5480; - tcph.th_flags = TH_PUSH | TH_ACK; + + SCLogDebug("check client seg list %p", ssn.client.seg_list); + tcph.th_win = htons(5480); + tcph.th_seq = htonl(10); + tcph.th_ack = htonl(20); + tcph.th_flags = TH_ACK|TH_PUSH; p->tcph = &tcph; - p->flowflags = FLOW_PKT_TOSERVER; - p->payload = packet; + p->flowflags = FLOW_PKT_TOCLIENT; - p->tcph->th_seq = htonl(857961230); - p->tcph->th_ack = htonl(31); - p->payload_len = 304; + p->payload = httpbuf2; + p->payload_len = httplen2; + ssn.state = TCP_ESTABLISHED; - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) == -1) { - SCFree(p); - return 0; - } + TcpStream *s = NULL; + s = &ssn.server; - p->tcph->th_seq = htonl(857961534); - p->tcph->th_ack = htonl(31); - p->payload_len = 1460; + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { + printf("failed in segments reassembly, while processing toserver packet (1): "); + goto end; + } - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) == -1) { - SCFree(p); - return 0; + /* Check if we have stream smsgs in queue */ + if (ra_ctx->stream_q->len > 0) { + printf("there shouldn't be any stream smsgs in the queue: (2): "); + goto end; } - p->tcph->th_seq = htonl(857963582); - p->tcph->th_ack = htonl(31); - p->payload_len = 1460; + SCLogDebug("check client seg list %p", ssn.client.seg_list); + p->flowflags = FLOW_PKT_TOSERVER; + p->payload = httpbuf1; + p->payload_len = httplen1; + tcph.th_seq = htonl(10); + tcph.th_ack = htonl(55); + s = &ssn.client; - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) == -1) { - SCFree(p); - return 0; + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { + printf("failed in segments reassembly, while processing toserver packet (3): "); + goto end; } - p->tcph->th_seq = htonl(857960946); - p->tcph->th_ack = htonl(31); - p->payload_len = 1460; + /* Check if we have stream smsgs in queue */ + if (ra_ctx->stream_q->len > 0) { + printf("there shouldn't be any stream smsgs in the queue, as we didn't" + " processed any smsg from toserver side till yet (4): "); + goto end; + } + SCLogDebug("check client seg list %p", ssn.client.seg_list); - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) == -1) { - SCFree(p); - return 0; + + p->flowflags = FLOW_PKT_TOCLIENT; + p->payload = httpbuf2; + p->payload_len = httplen2; + tcph.th_seq = htonl(55); + tcph.th_ack = htonl(53); + s = &ssn.server; + + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { + printf("failed in segments reassembly, while processing toserver packet (5): "); + goto end; } + SCLogDebug("check client seg list %p", ssn.client.seg_list); - StreamTcpFreeConfig(TRUE); - SCFree(p); - return 1; -} -/** \test Test the bug 56 condition */ -static int StreamTcpReassembleTest35(void) { - TcpSession ssn; - Packet *p = SCMalloc(SIZE_OF_PACKET); - if (p == NULL) - return 0; - Flow f; - TCPHdr tcph; - TcpReassemblyThreadCtx *ra_ctx = StreamTcpReassembleInitThreadCtx(); - TcpStream stream; - memset(&stream, 0, sizeof (TcpStream)); - stream.os_policy = OS_POLICY_BSD; - uint8_t packet[1460] = ""; + /* Check if we have stream smsgs in queue */ + SCLogDebug("check if we have stream smsgs in queue"); + if (ra_ctx->stream_q->len == 0) { + printf("there should be a stream smsgs in the queue (6): "); + goto end; + } - StreamTcpInitConfig(TRUE); + SCLogDebug("check client seg list %p", ssn.client.seg_list); + /* Process stream smsgs we may have in queue */ + SCLogDebug("process stream smsgs we may have in queue"); + if (StreamTcpReassembleProcessAppLayer(ra_ctx) < 0) { + printf("failed in processing stream smsgs (7): "); + goto end; + } - /* prevent L7 from kicking in */ - StreamMsgQueueSetMinInitChunkLen(FLOW_PKT_TOSERVER, 10); - StreamMsgQueueSetMinInitChunkLen(FLOW_PKT_TOCLIENT, 10); - StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 10); - StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 10); + SCLogDebug("check client seg list %p", ssn.client.seg_list); - PacketQueue pq; - memset(&pq,0,sizeof(PacketQueue)); - memset(&ssn, 0, sizeof (TcpSession)); - memset(p, 0, SIZE_OF_PACKET); - p->pkt = (uint8_t *)(p + 1); - memset(&f, 0, sizeof (Flow)); - memset(&tcph, 0, sizeof (TCPHdr)); - ThreadVars tv; - memset(&tv, 0, sizeof (ThreadVars)); - FLOW_INITIALIZE(&f); - f.protoctx = &ssn; - p->src.family = AF_INET; - p->dst.family = AF_INET; - p->proto = IPPROTO_TCP; - p->flow = &f; - tcph.th_win = 5480; - tcph.th_flags = TH_PUSH | TH_ACK; - p->tcph = &tcph; + /* check is have the segment in the list and flagged or not */ +/* + if (ssn.client.seg_list == NULL || + !(ssn.client.seg_list->flags & SEGMENTTCP_FLAG_RAW_PROCESSED)) + { + printf("the list is NULL or the processed segment has not been flaged (8), seg %p, flags %02X: ", + ssn.client.seg_list, ssn.client.seg_list? ssn.client.seg_list->flags:0); +//abort(); + goto end; + } +*/ p->flowflags = FLOW_PKT_TOSERVER; - p->payload = packet; + p->payload = httpbuf1; + p->payload_len = httplen1; + tcph.th_seq = htonl(53); + tcph.th_ack = htonl(100); + s = &ssn.client; - p->tcph->th_seq = htonl(2257022155UL); - p->tcph->th_ack = htonl(1374943142); - p->payload_len = 142; - stream.last_ack = 2257022285UL; - stream.ra_raw_base_seq = 2257022172UL; - stream.ra_app_base_seq = 2257022172UL; + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { + printf("failed in segments reassembly, while processing toserver packet (9): "); + goto end; + } - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) == -1) { - SCFree(p); - return 0; + /* Check if we have stream smsgs in queue */ + if (ra_ctx->stream_q->len == 0 && + !(ssn.flags & STREAMTCP_FLAG_APPPROTO_DETECTION_COMPLETED)) { + printf("there should be a stream smsgs in the queue, as we have detected" + " the app layer protocol and one smsg from toserver side has " + "been sent (10): "); + goto end; + /* Process stream smsgs we may have in queue */ + } else if (StreamTcpReassembleProcessAppLayer(ra_ctx) < 0) { + printf("failed in processing stream smsgs (11): "); + goto end; } - p->tcph->th_seq = htonl(2257022285UL); - p->tcph->th_ack = htonl(1374943142); - p->payload_len = 34; - stream.last_ack = 2257022285UL; - stream.ra_raw_base_seq = 2257022172UL; + p->flowflags = FLOW_PKT_TOCLIENT; + p->payload = httpbuf2; + p->payload_len = httplen2; + tcph.th_seq = htonl(100); + tcph.th_ack = htonl(96); + s = &ssn.server; - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) == -1) { - SCFree(p); - return 0; + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { + printf("failed in segments reassembly, while processing toserver packet (12): "); + goto end; + } + + SCLogDebug("final check"); + + /* check if the segment in the list is flagged or not */ + if ((ssn.client.seg_list != NULL) && + (ssn.flags & STREAMTCP_FLAG_APPPROTO_DETECTION_COMPLETED)) { + printf("the segments should have been removed and there should be " + "no more segments in the list as they have been sent to app layer (13): "); + goto end; } + ret = 1; +end: StreamTcpFreeConfig(TRUE); + StreamTcpReassembleFreeThreadCtx(ra_ctx); SCFree(p); - return 1; + return ret; } -/** \test Test the bug 57 condition */ -static int StreamTcpReassembleTest36(void) { - TcpSession ssn; +/** + * \test Test to make sure that we sent all the segments from the initial + * segments to app layer until we have detected the app layer proto. + * + * \retval On success it returns 1 and on failure 0. + */ + +static int StreamTcpReassembleTest40 (void) { + int ret = 0; Packet *p = SCMalloc(SIZE_OF_PACKET); if (p == NULL) return 0; Flow f; TCPHdr tcph; - TcpReassemblyThreadCtx *ra_ctx = StreamTcpReassembleInitThreadCtx(); - TcpStream stream; - memset(&stream, 0, sizeof (TcpStream)); - stream.os_policy = OS_POLICY_BSD; - uint8_t packet[1460] = ""; - - StreamTcpInitConfig(TRUE); - - /* prevent L7 from kicking in */ - StreamMsgQueueSetMinInitChunkLen(FLOW_PKT_TOSERVER, 10); - StreamMsgQueueSetMinInitChunkLen(FLOW_PKT_TOCLIENT, 10); - StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 10); - StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 10); + Port sp; + Port dp; + Address src; + Address dst; + struct in_addr in; + TcpSession ssn; - PacketQueue pq; - memset(&pq,0,sizeof(PacketQueue)); - memset(&ssn, 0, sizeof (TcpSession)); memset(p, 0, SIZE_OF_PACKET); p->pkt = (uint8_t *)(p + 1); + PacketQueue pq; + memset(&pq,0,sizeof(PacketQueue)); memset(&f, 0, sizeof (Flow)); memset(&tcph, 0, sizeof (TCPHdr)); + memset(&src, 0, sizeof(Address)); + memset(&dst, 0, sizeof(Address)); + memset(&ssn, 0, sizeof(TcpSession)); ThreadVars tv; memset(&tv, 0, sizeof (ThreadVars)); + FLOW_INITIALIZE(&f); + StreamTcpInitConfig(TRUE); + TcpReassemblyThreadCtx *ra_ctx = StreamTcpReassembleInitThreadCtx(); + AppLayerDetectProtoThreadInit(); + + uint8_t httpbuf1[] = "P"; + uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */ + uint8_t httpbuf3[] = "O"; + uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */ + uint8_t httpbuf4[] = "S"; + uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */ + uint8_t httpbuf5[] = "T \r\n"; + uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */ + + uint8_t httpbuf2[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n"; + uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */ + + ssn.server.ra_raw_base_seq = ssn.server.ra_app_base_seq = 9; + ssn.server.isn = 9; + ssn.server.last_ack = 10; + ssn.client.ra_raw_base_seq = ssn.client.ra_app_base_seq = 9; + ssn.client.isn = 9; + ssn.client.last_ack = 10; + f.alproto = ALPROTO_UNKNOWN; + + inet_pton(AF_INET, "1.2.3.4", &in); + src.family = AF_INET; + src.addr_data32[0] = in.s_addr; + inet_pton(AF_INET, "1.2.3.5", &in); + dst.family = AF_INET; + dst.addr_data32[0] = in.s_addr; + sp = 200; + dp = 220; + + f.src = src; + f.dst = dst; + f.sp = sp; + f.dp = dp; f.protoctx = &ssn; - p->src.family = AF_INET; - p->dst.family = AF_INET; - p->proto = IPPROTO_TCP; p->flow = &f; - tcph.th_win = 5480; - tcph.th_flags = TH_PUSH | TH_ACK; + + tcph.th_win = htons(5480); + tcph.th_seq = htonl(10); + tcph.th_ack = htonl(10); + tcph.th_flags = TH_ACK|TH_PUSH; p->tcph = &tcph; p->flowflags = FLOW_PKT_TOSERVER; - p->payload = packet; - p->tcph->th_seq = htonl(1549588966); - p->tcph->th_ack = htonl(4162241372UL); - p->payload_len = 204; - stream.last_ack = 1549589007; - stream.ra_raw_base_seq = 1549589101; + p->payload = httpbuf1; + p->payload_len = httplen1; + ssn.state = TCP_ESTABLISHED; - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) == -1) { - SCFree(p); - return 0; - } + TcpStream *s = NULL; + s = &ssn.client; - p->tcph->th_seq = htonl(1549589007); - p->tcph->th_ack = htonl(4162241372UL); - p->payload_len = 23; - stream.last_ack = 1549589007; - stream.ra_raw_base_seq = 1549589101; + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { + printf("failed in segments reassembly, while processing toserver packet (1): "); + goto end; + } - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) == -1) { - SCFree(p); - return 0; + /* Check if we have stream smsgs in queue */ + if (ra_ctx->stream_q->len > 0) { + printf("there shouldn't be any stream smsgs in the queue, as we didn't" + " processed any smsg from toserver side till yet (2): "); + goto end; } - StreamTcpFreeConfig(TRUE); - SCFree(p); - return 1; -} + p->flowflags = FLOW_PKT_TOCLIENT; + p->payload = httpbuf2; + p->payload_len = httplen2; + tcph.th_seq = htonl(10); + tcph.th_ack = htonl(11); + s = &ssn.server; + ssn.server.last_ack = 11; -/** \test Test the bug 76 condition */ -static int StreamTcpReassembleTest37(void) { - TcpSession ssn; - Flow f; - TCPHdr tcph; - TcpReassemblyThreadCtx *ra_ctx = StreamTcpReassembleInitThreadCtx(); - TcpStream stream; - uint8_t packet[1460] = ""; - PacketQueue pq; - ThreadVars tv; + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { + printf("failed in segments reassembly, while processing toserver packet (3): "); + goto end; + } - Packet *p = SCMalloc(SIZE_OF_PACKET); - if (p == NULL) - return 0; + /* Process stream smsgs we may have in queue */ + if (StreamTcpReassembleProcessAppLayer(ra_ctx) < 0) { + printf("failed in processing stream smsgs (4): "); + goto end; + } - StreamTcpInitConfig(TRUE); + p->flowflags = FLOW_PKT_TOSERVER; + p->payload = httpbuf3; + p->payload_len = httplen3; + tcph.th_seq = htonl(11); + tcph.th_ack = htonl(55); + s = &ssn.client; + ssn.client.last_ack = 55; - /* prevent L7 from kicking in */ - StreamMsgQueueSetMinInitChunkLen(FLOW_PKT_TOSERVER, 10); - StreamMsgQueueSetMinInitChunkLen(FLOW_PKT_TOCLIENT, 10); - StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 10); - StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 10); + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { + printf("failed in segments reassembly, while processing toserver packet (5): "); + goto end; + } - memset(p, 0, SIZE_OF_PACKET); - p->pkt = (uint8_t *)(p + 1); + p->flowflags = FLOW_PKT_TOCLIENT; + p->payload = httpbuf2; + p->payload_len = httplen2; + tcph.th_seq = htonl(55); + tcph.th_ack = htonl(12); + s = &ssn.server; + ssn.server.last_ack = 12; - memset(&stream, 0, sizeof (TcpStream)); - memset(&pq,0,sizeof(PacketQueue)); - memset(&ssn, 0, sizeof (TcpSession)); - memset(&f, 0, sizeof (Flow)); - memset(&tcph, 0, sizeof (TCPHdr)); - memset(&tv, 0, sizeof (ThreadVars)); + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { + printf("failed in segments reassembly, while processing toserver packet (6): "); + goto end; + } - FLOW_INITIALIZE(&f); - f.protoctx = &ssn; - p->src.family = AF_INET; - p->dst.family = AF_INET; - p->proto = IPPROTO_TCP; - p->flow = &f; - tcph.th_win = 5480; - tcph.th_flags = TH_PUSH | TH_ACK; - p->tcph = &tcph; + /* check is have the segment in the list and flagged or not */ + if (ssn.client.seg_list == NULL || + !(ssn.client.seg_list->flags & SEGMENTTCP_FLAG_APPLAYER_PROCESSED)) + { + printf("the list is NULL or the processed segment has not been flaged (7): "); + goto end; + } + + /* Check if we have stream smsgs in queue */ +#if 0 + if (ra_ctx->stream_q->len == 0) { + printf("there should be a stream smsgs in the queue, as we have detected" + " the app layer protocol and one smsg from toserver side has " + "been sent (8): "); + goto end; + /* Process stream smsgs we may have in queue */ + } else if (StreamTcpReassembleProcessAppLayer(ra_ctx) < 0) { + printf("failed in processing stream smsgs (9): "); + goto end; + } +#endif p->flowflags = FLOW_PKT_TOSERVER; - p->payload = packet; - stream.os_policy = OS_POLICY_BSD; + p->payload = httpbuf4; + p->payload_len = httplen4; + tcph.th_seq = htonl(12); + tcph.th_ack = htonl(100); + s = &ssn.client; + ssn.client.last_ack = 100; - p->tcph->th_seq = htonl(3061088537UL); - p->tcph->th_ack = htonl(1729548549UL); - p->payload_len = 1391; - stream.last_ack = 3061091137UL; - stream.ra_raw_base_seq = 3061091309UL; - stream.ra_app_base_seq = 3061091309UL; + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { + printf("failed in segments reassembly, while processing toserver packet (10): "); + goto end; + } + + p->flowflags = FLOW_PKT_TOCLIENT; + p->payload = httpbuf2; + p->payload_len = httplen2; + tcph.th_seq = htonl(100); + tcph.th_ack = htonl(13); + s = &ssn.server; + ssn.server.last_ack = 13; + + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { + printf("failed in segments reassembly, while processing toserver packet (11): "); + goto end; + } + + /* Check if we have stream smsgs in queue */ +#if 0 + if (ra_ctx->stream_q->len == 0) { + printf("there should be a stream smsgs in the queue, as we have detected" + " the app layer protocol and one smsg from toserver side has " + "been sent (12): "); + goto end; + /* Process stream smsgs we may have in queue */ + } else if (StreamTcpReassembleProcessAppLayer(ra_ctx) < 0) { + printf("failed in processing stream smsgs (13): "); + goto end; + } +#endif + p->flowflags = FLOW_PKT_TOSERVER; + p->payload = httpbuf5; + p->payload_len = httplen5; + tcph.th_seq = htonl(13); + tcph.th_ack = htonl(145); + s = &ssn.client; + ssn.client.last_ack = 145; - /* pre base_seq, so should be rejected */ - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) != -1) { - SCFree(p); - return 0; + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { + printf("failed in segments reassembly, while processing toserver packet (14): "); + goto end; } - p->tcph->th_seq = htonl(3061089928UL); - p->tcph->th_ack = htonl(1729548549UL); - p->payload_len = 1391; - stream.last_ack = 3061091137UL; - stream.ra_raw_base_seq = 3061091309UL; - stream.ra_app_base_seq = 3061091309UL; + p->flowflags = FLOW_PKT_TOCLIENT; + p->payload = httpbuf2; + p->payload_len = httplen2; + tcph.th_seq = htonl(145); + tcph.th_ack = htonl(16); + s = &ssn.server; + ssn.server.last_ack = 16; - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) == -1) { - SCFree(p); - return 0; + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { + printf("failed in segments reassembly, while processing toserver packet (15): "); + goto end; } - p->tcph->th_seq = htonl(3061091319UL); - p->tcph->th_ack = htonl(1729548549UL); - p->payload_len = 1391; - stream.last_ack = 3061091137UL; - stream.ra_raw_base_seq = 3061091309UL; - stream.ra_app_base_seq = 3061091309UL; + /* Check if we have stream smsgs in queue */ + if (ra_ctx->stream_q->len == 0) { + printf("there should be a stream smsgs in the queue, as we have detected" + " the app layer protocol and one smsg from toserver side has " + "been sent (16): "); + goto end; + /* Process stream smsgs we may have in queue */ + } else if (StreamTcpReassembleProcessAppLayer(ra_ctx) < 0) { + printf("failed in processing stream smsgs (17): "); + goto end; + } - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx,&ssn, &stream, p, &pq) == -1) { - SCFree(p); - return 0; + if (f.alproto != ALPROTO_HTTP) { + printf("app layer proto has not been detected (18): "); + goto end; } + ret = 1; +end: StreamTcpFreeConfig(TRUE); + StreamTcpReassembleFreeThreadCtx(ra_ctx); SCFree(p); - return 1; + return ret; } /** - * \test Test to make sure we don't send the smsg from toclient to app layer - * until the app layer protocol has been detected and one smsg from - * toserver side has been sent to app layer. + * \test Test to make sure we don't send more than one smsg from toserver to + * app layer until the app layer protocol has not been detected. After + * protocol has been detected the processed segments should be returned + * to pool. * * \retval On success it returns 1 and on failure 0. */ -static int StreamTcpReassembleTest38 (void) { +static int StreamTcpReassembleTest41 (void) { int ret = 0; Packet *p = SCMalloc(SIZE_OF_PACKET); if (p == NULL) @@ -5719,23 +6715,47 @@ static int StreamTcpReassembleTest38 (void) { ThreadVars tv; memset(&tv, 0, sizeof (ThreadVars)); - /* prevent L7 from kicking in */ - StreamMsgQueueSetMinInitChunkLen(FLOW_PKT_TOSERVER, 0); - StreamMsgQueueSetMinInitChunkLen(FLOW_PKT_TOCLIENT, 0); - StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 0); - StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 0); - FLOW_INITIALIZE(&f); StreamTcpInitConfig(TRUE); TcpReassemblyThreadCtx *ra_ctx = StreamTcpReassembleInitThreadCtx(); AppLayerDetectProtoThreadInit(); - uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\n"; + uint8_t httpbuf1[] = "GET / HTTP/1.0\r\nUser-Agent: Victor/1.0" + "W2dyb3VwMV0NCnBob25lMT1wMDB3ODgyMTMxMzAyMTINCmxvZ2lu" + "MT0NCnBhc3N3b3JkMT0NCnBob25lMj1wMDB3ODgyMTMxMzAyMTIN" + "CmxvZ2luMj0NCnBhc3N3b3JkMj0NCnBob25lMz0NCmxvZ2luMz0N" + "CnBhc3N3b3JkMz0NCnBob25lND0NCmxvZ2luND0NCnBhc3N3b3Jk" + "ND0NCnBob25lNT0NCmxvZ2luNT0NCnBhc3N3b3JkNT0NCnBob25l" + "Nj0NCmxvZ2luNj0NCnBhc3N3b3JkNj0NCmNhbGxfdGltZTE9MzIN" + "CmNhbGxfdGltZTI9MjMyDQpkYXlfbGltaXQ9NQ0KbW9udGhfbGlt" + "aXQ9MTUNCltncm91cDJdDQpwaG9uZTE9DQpsb2dpbjE9DQpwYXNz" + "d29yZDE9DQpwaG9uZTI9DQpsb2dpbjI9DQpwYXNzd29yZDI9DQpw" + "aG9uZT"; uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */ + uint8_t httpbuf3[] = "psb2dpbjM9DQpwYXNzd29yZDM9DQpwaG9uZTQ9DQps" + "b2dpbjQ9DQpwYXNzd29yZDQ9DQpwaG9uZTU9DQpsb2dpbjU9DQpw" + "YXNzd29yZDU9DQpwaG9uZTY9DQpsb2dpbjY9DQpwYXNzd29yZDY9" + "DQpjYWxsX3RpbWUxPQ0KY2FsbF90aW1lMj0NCmRheV9saW1pdD0N" + "\r\n\r\n"; + uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */ + uint8_t httpbuf2[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n"; uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */ + StreamMsgQueueSetMinInitChunkLen(FLOW_PKT_TOSERVER, 0); + StreamMsgQueueSetMinInitChunkLen(FLOW_PKT_TOCLIENT, 0); + StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 0); + StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 0); + + ssn.server.ra_raw_base_seq = ssn.server.ra_app_base_seq = 9; + ssn.server.isn = 9; + ssn.server.last_ack = 600; + ssn.client.ra_raw_base_seq = ssn.client.ra_app_base_seq = 9; + ssn.client.isn = 9; + ssn.client.last_ack = 600; + f.alproto = ALPROTO_UNKNOWN; + inet_pton(AF_INET, "1.2.3.4", &in); src.family = AF_INET; src.addr_data32[0] = in.s_addr; @@ -5745,14 +6765,6 @@ static int StreamTcpReassembleTest38 (void) { sp = 200; dp = 220; - ssn.server.ra_raw_base_seq = ssn.server.ra_app_base_seq = 9; - ssn.server.isn = 9; - ssn.server.last_ack = 60; - ssn.client.ra_raw_base_seq = ssn.client.ra_app_base_seq = 9; - ssn.client.isn = 9; - ssn.client.last_ack = 60; - f.alproto = ALPROTO_UNKNOWN; - f.src = src; f.dst = dst; f.sp = sp; @@ -5775,13 +6787,13 @@ static int StreamTcpReassembleTest38 (void) { s = &ssn.server; if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet\n"); + printf("failed in segments reassembly, while processing toserver packet: "); goto end; } /* Check if we have stream smsgs in queue */ if (ra_ctx->stream_q->len > 0) { - printf("there shouldn't be any stream smsgs in the queue\n"); + printf("there shouldn't be any stream smsgs in the queue: "); goto end; } @@ -5793,14 +6805,26 @@ static int StreamTcpReassembleTest38 (void) { s = &ssn.client; if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet\n"); + printf("failed in segments reassembly, while processing toserver packet: "); + goto end; + } + + p->flowflags = FLOW_PKT_TOSERVER; + p->payload = httpbuf3; + p->payload_len = httplen3; + tcph.th_seq = htonl(522); + tcph.th_ack = htonl(100); + s = &ssn.client; + + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { + printf("failed in segments reassembly, while processing toserver packet: "); goto end; } /* Check if we have stream smsgs in queue */ if (ra_ctx->stream_q->len > 0) { printf("there shouldn't be any stream smsgs in the queue, as we didn't" - " processed any smsg from toserver side till yet\n"); + " processed any smsg from toserver side till yet: "); goto end; } @@ -5808,58 +6832,44 @@ static int StreamTcpReassembleTest38 (void) { p->payload = httpbuf2; p->payload_len = httplen2; tcph.th_seq = htonl(55); - tcph.th_ack = htonl(53); + tcph.th_ack = htonl(522); s = &ssn.server; + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet\n"); + printf("failed in segments reassembly, while processing toserver packet: "); goto end; } /* Check if we have stream smsgs in queue */ if (ra_ctx->stream_q->len == 0) { - printf("there should be a stream smsgs in the queue\n"); + printf("there should be a stream smsgs in the queue: "); goto end; /* Process stream smsgs we may have in queue */ - } else if (StreamTcpReassembleProcessAppLayer(ra_ctx) < 0) { - printf("failed in processing stream smsgs\n"); + } else if (ra_ctx->stream_q->len > 1) { + printf("there should be only one stream smsgs in the queue: "); goto end; - } - - p->flowflags = FLOW_PKT_TOSERVER; - p->payload = httpbuf1; - p->payload_len = httplen1; - tcph.th_seq = htonl(53); - tcph.th_ack = htonl(100); - s = &ssn.client; - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet\n"); + } else if (StreamTcpReassembleProcessAppLayer(ra_ctx) < 0) { + printf("failed in processing stream smsgs: "); goto end; } p->flowflags = FLOW_PKT_TOCLIENT; - p->payload = NULL; - p->payload_len = 0; + p->payload = httpbuf2; + p->payload_len = httplen2; tcph.th_seq = htonl(100); - tcph.th_ack = htonl(53); + tcph.th_ack = htonl(522); s = &ssn.server; if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet\n"); + printf("failed in segments reassembly, while processing toserver packet: "); goto end; } -#if 0 - /* Check if we have stream smsgs in queue */ - if (ra_ctx->stream_q->len == 0) { - printf("there should be a stream smsgs in the queue, as we have detected" - " the app layer protocol and one smsg from toserver side has " - "been sent\n"); - goto end; - /* Process stream smsgs we may have in queue */ - } else if (StreamTcpReassembleProcessAppLayer(ra_ctx) < 0) { - printf("failed in processing stream smsgs\n"); + + if (ssn.client.seg_list != NULL) { + printf("seg_list should be null: "); goto end; } -#endif + ret = 1; end: StreamTcpFreeConfig(TRUE); @@ -5869,16 +6879,12 @@ end: } /** - * \test Test to make sure that we don't return the segments until the app - * layer proto has been detected and after that remove the processed - * segments. + * \test Test to make sure that Pause/Unpause API is working. * * \retval On success it returns 1 and on failure 0. */ -static int StreamTcpReassembleTest39 (void) { - SCEnter(); - +static int StreamTcpReassembleTest42 (void) { int ret = 0; Packet *p = SCMalloc(SIZE_OF_PACKET); if (p == NULL) @@ -5908,8 +6914,6 @@ static int StreamTcpReassembleTest39 (void) { StreamTcpInitConfig(TRUE); TcpReassemblyThreadCtx *ra_ctx = StreamTcpReassembleInitThreadCtx(); AppLayerDetectProtoThreadInit(); - StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 7); - StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 7); uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\n"; uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */ @@ -5920,7 +6924,7 @@ static int StreamTcpReassembleTest39 (void) { ssn.server.ra_raw_base_seq = ssn.server.ra_app_base_seq = 9; ssn.server.isn = 9; ssn.server.last_ack = 60; - ssn.client.ra_raw_base_seq = ssn.client.ra_app_base_seq= 9; + ssn.client.ra_raw_base_seq = ssn.client.ra_app_base_seq = 9; ssn.client.isn = 9; ssn.client.last_ack = 60; f.alproto = ALPROTO_UNKNOWN; @@ -5941,7 +6945,6 @@ static int StreamTcpReassembleTest39 (void) { f.protoctx = &ssn; p->flow = &f; - SCLogDebug("check client seg list %p", ssn.client.seg_list); tcph.th_win = htons(5480); tcph.th_seq = htonl(10); tcph.th_ack = htonl(20); @@ -5961,120 +6964,76 @@ static int StreamTcpReassembleTest39 (void) { goto end; } - /* Check if we have stream smsgs in queue */ - if (ra_ctx->stream_q->len > 0) { - printf("there shouldn't be any stream smsgs in the queue: (2): "); - goto end; - } - - SCLogDebug("check client seg list %p", ssn.client.seg_list); - p->flowflags = FLOW_PKT_TOSERVER; - p->payload = httpbuf1; - p->payload_len = httplen1; - tcph.th_seq = htonl(10); - tcph.th_ack = htonl(55); - s = &ssn.client; - - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet (3): "); - goto end; - } - - /* Check if we have stream smsgs in queue */ - if (ra_ctx->stream_q->len > 0) { - printf("there shouldn't be any stream smsgs in the queue, as we didn't" - " processed any smsg from toserver side till yet (4): "); - goto end; - } - SCLogDebug("check client seg list %p", ssn.client.seg_list); - - - p->flowflags = FLOW_PKT_TOCLIENT; - p->payload = httpbuf2; - p->payload_len = httplen2; - tcph.th_seq = htonl(55); - tcph.th_ack = htonl(53); - s = &ssn.server; - - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet (5): "); - goto end; - } - SCLogDebug("check client seg list %p", ssn.client.seg_list); - - - /* Check if we have stream smsgs in queue */ - SCLogDebug("check if we have stream smsgs in queue"); - if (ra_ctx->stream_q->len == 0) { - printf("there should be a stream smsgs in the queue (6): "); - goto end; - } - - SCLogDebug("check client seg list %p", ssn.client.seg_list); - /* Process stream smsgs we may have in queue */ - SCLogDebug("process stream smsgs we may have in queue"); - if (StreamTcpReassembleProcessAppLayer(ra_ctx) < 0) { - printf("failed in processing stream smsgs (7): "); - goto end; - } - - SCLogDebug("check client seg list %p", ssn.client.seg_list); - - /* check is have the segment in the list and flagged or not */ -/* - if (ssn.client.seg_list == NULL || - !(ssn.client.seg_list->flags & SEGMENTTCP_FLAG_RAW_PROCESSED)) - { - printf("the list is NULL or the processed segment has not been flaged (8), seg %p, flags %02X: ", - ssn.client.seg_list, ssn.client.seg_list? ssn.client.seg_list->flags:0); -//abort(); + /* Check if we have stream smsgs in queue */ + if (ra_ctx->stream_q->len > 0) { + printf("there shouldn't be any stream smsgs in the queue (2): "); goto end; } -*/ + p->flowflags = FLOW_PKT_TOSERVER; p->payload = httpbuf1; p->payload_len = httplen1; - tcph.th_seq = htonl(53); - tcph.th_ack = htonl(100); + tcph.th_seq = htonl(10); + tcph.th_ack = htonl(55); s = &ssn.client; if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet (9): "); + printf("failed in segments reassembly, while processing toserver packet (3): "); goto end; } /* Check if we have stream smsgs in queue */ - if (ra_ctx->stream_q->len == 0 && - !(ssn.flags & STREAMTCP_FLAG_APPPROTO_DETECTION_COMPLETED)) { - printf("there should be a stream smsgs in the queue, as we have detected" - " the app layer protocol and one smsg from toserver side has " - "been sent (10): "); + if (ra_ctx->stream_q->len > 0) { + printf("there shouldn't be any stream smsgs in the queue, as we didn't" + " processed any smsg from toserver side till yet (4): "); goto end; - /* Process stream smsgs we may have in queue */ - } else if (StreamTcpReassembleProcessAppLayer(ra_ctx) < 0) { - printf("failed in processing stream smsgs (11): "); + } + + /* pause the reassembling */ + StreamTcpReassemblePause(&ssn, 1); + + p->flowflags = FLOW_PKT_TOCLIENT; + p->payload = httpbuf2; + p->payload_len = httplen2; + tcph.th_seq = htonl(55); + tcph.th_ack = htonl(53); + s = &ssn.server; + + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { + printf("failed in segments reassembly, while processing toserver packet (5): "); + goto end; + } + + /* Check if we have stream smsgs in queue */ + if (ra_ctx->stream_q->len > 0) { + printf("there shouldn't be any stream smsgs in the queue, as we have" + " paused the reassembling (6): "); goto end; } + /* Unpause the reassembling */ + StreamTcpReassembleUnPause(&ssn, 1); + p->flowflags = FLOW_PKT_TOCLIENT; p->payload = httpbuf2; p->payload_len = httplen2; tcph.th_seq = htonl(100); - tcph.th_ack = htonl(96); + tcph.th_ack = htonl(53); s = &ssn.server; if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet (12): "); + printf("failed in segments reassembly, while processing toserver packet (7): "); goto end; } - SCLogDebug("final check"); - - /* check if the segment in the list is flagged or not */ - if ((ssn.client.seg_list != NULL) && - (ssn.flags & STREAMTCP_FLAG_APPPROTO_DETECTION_COMPLETED)) { - printf("the segments should have been removed and there should be " - "no more segments in the list as they have been sent to app layer (13): "); + /* Check if we have stream smsgs in queue */ + if (ra_ctx->stream_q->len == 0) { + printf("there should be a stream smsgs in the queue, as reassembling has" + " been unpaused now (8): "); + goto end; + /* Process stream smsgs we may have in queue */ + } else if (StreamTcpReassembleProcessAppLayer(ra_ctx) < 0) { + printf("failed in processing stream smsgs (9): "); goto end; } @@ -6087,13 +7046,12 @@ end: } /** - * \test Test to make sure that we sent all the segments from the initial - * segments to app layer until we have detected the app layer proto. + * \test Test to make sure that Pause/Unpause API is working. * * \retval On success it returns 1 and on failure 0. */ -static int StreamTcpReassembleTest40 (void) { +static int StreamTcpReassembleTest43 (void) { int ret = 0; Packet *p = SCMalloc(SIZE_OF_PACKET); if (p == NULL) @@ -6124,26 +7082,49 @@ static int StreamTcpReassembleTest40 (void) { TcpReassemblyThreadCtx *ra_ctx = StreamTcpReassembleInitThreadCtx(); AppLayerDetectProtoThreadInit(); - uint8_t httpbuf1[] = "P"; + uint8_t httpbuf1[] = "/ HTTP/1.0\r\nUser-Agent: Victor/1.0"; + uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */ - uint8_t httpbuf3[] = "O"; - uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */ - uint8_t httpbuf4[] = "S"; - uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */ - uint8_t httpbuf5[] = "T \r\n"; - uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */ uint8_t httpbuf2[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n"; uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */ + uint8_t httpbuf3[] = "W2dyb3VwMV0NCnBob25lMT1wMDB3ODgyMTMxMzAyMTINCmxvZ2lu" + "MT0NCnBhc3N3b3JkMT0NCnBob25lMj1wMDB3ODgyMTMxMzAyMTIN" + "CmxvZ2luMj0NCnBhc3N3b3JkMj0NCnBob25lMz0NCmxvZ2luMz0N" + "CnBhc3N3b3JkMz0NCnBob25lND0NCmxvZ2luND0NCnBhc3N3b3Jk" + "ND0NCnBob25lNT0NCmxvZ2luNT0NCnBhc3N3b3JkNT0NCnBob25l" + "Nj0NCmxvZ2luNj0NCnBhc3N3b3JkNj0NCmNhbGxfdGltZTE9MzIN" + "CmNhbGxfdGltZTI9MjMyDQpkYXlfbGltaXQ9NQ0KbW9udGhfbGlt" + "aXQ9MTUNCltncm91cDJdDQpwaG9uZTE9DQpsb2dpbjE9DQpwYXNz" + "d29yZDE9DQpwaG9uZTI9DQpsb2dpbjI9DQpwYXNzd29yZDI9DQpw" + "aG9uZT\r\n\r\n"; + uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */ + ssn.server.ra_raw_base_seq = ssn.server.ra_app_base_seq = 9; ssn.server.isn = 9; - ssn.server.last_ack = 10; + ssn.server.last_ack = 600; ssn.client.ra_raw_base_seq = ssn.client.ra_app_base_seq = 9; ssn.client.isn = 9; - ssn.client.last_ack = 10; + ssn.client.last_ack = 600; f.alproto = ALPROTO_UNKNOWN; + /* Check the minimum init smsg length. It should be equal to the min length + of given signature in toserver direction. */ + if (StreamMsgQueueGetMinInitChunkLen(FLOW_PKT_TOSERVER) != 2) { + printf("the min init length should be equal to 2, not %"PRIu16": ", + StreamMsgQueueGetMinInitChunkLen(FLOW_PKT_TOSERVER)); + goto end; + } + + /* Check the minimum init smsg length. It should be equal to the min length + of given signature in toclient direction. */ + if (StreamMsgQueueGetMinInitChunkLen(FLOW_PKT_TOCLIENT) != 2) { + printf("the min init length should be equal to 2, not %"PRIu16": ", + StreamMsgQueueGetMinInitChunkLen(FLOW_PKT_TOCLIENT)); + goto end; + } + inet_pton(AF_INET, "1.2.3.4", &in); src.family = AF_INET; src.addr_data32[0] = in.s_addr; @@ -6165,14 +7146,14 @@ static int StreamTcpReassembleTest40 (void) { tcph.th_ack = htonl(10); tcph.th_flags = TH_ACK|TH_PUSH; p->tcph = &tcph; - p->flowflags = FLOW_PKT_TOSERVER; + p->flowflags = FLOW_PKT_TOCLIENT; - p->payload = httpbuf1; - p->payload_len = httplen1; + p->payload = httpbuf2; + p->payload_len = httplen2; ssn.state = TCP_ESTABLISHED; TcpStream *s = NULL; - s = &ssn.client; + s = &ssn.server; if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { printf("failed in segments reassembly, while processing toserver packet (1): "); @@ -6181,40 +7162,26 @@ static int StreamTcpReassembleTest40 (void) { /* Check if we have stream smsgs in queue */ if (ra_ctx->stream_q->len > 0) { - printf("there shouldn't be any stream smsgs in the queue, as we didn't" - " processed any smsg from toserver side till yet (2): "); + printf("there shouldn't be any stream smsgs in the queue (2): "); goto end; } - p->flowflags = FLOW_PKT_TOCLIENT; - p->payload = httpbuf2; - p->payload_len = httplen2; + p->flowflags = FLOW_PKT_TOSERVER; + p->payload = httpbuf1; + p->payload_len = httplen1; tcph.th_seq = htonl(10); - tcph.th_ack = htonl(11); - s = &ssn.server; - ssn.server.last_ack = 11; + tcph.th_ack = htonl(55); + s = &ssn.client; if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { printf("failed in segments reassembly, while processing toserver packet (3): "); goto end; } - /* Process stream smsgs we may have in queue */ - if (StreamTcpReassembleProcessAppLayer(ra_ctx) < 0) { - printf("failed in processing stream smsgs (4): "); - goto end; - } - - p->flowflags = FLOW_PKT_TOSERVER; - p->payload = httpbuf3; - p->payload_len = httplen3; - tcph.th_seq = htonl(11); - tcph.th_ack = htonl(55); - s = &ssn.client; - ssn.client.last_ack = 55; - - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet (5): "); + /* Check if we have stream smsgs in queue */ + if (ra_ctx->stream_q->len > 0) { + printf("there shouldn't be any stream smsgs in the queue, as we didn't" + " processed any smsg from toserver side till yet (4): "); goto end; } @@ -6222,46 +7189,52 @@ static int StreamTcpReassembleTest40 (void) { p->payload = httpbuf2; p->payload_len = httplen2; tcph.th_seq = htonl(55); - tcph.th_ack = htonl(12); + tcph.th_ack = htonl(44); s = &ssn.server; - ssn.server.last_ack = 12; if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet (6): "); - goto end; - } - - /* check is have the segment in the list and flagged or not */ - if (ssn.client.seg_list == NULL || - !(ssn.client.seg_list->flags & SEGMENTTCP_FLAG_APPLAYER_PROCESSED)) - { - printf("the list is NULL or the processed segment has not been flaged (7): "); + printf("failed in segments reassembly, while processing toserver packet (5): "); goto end; } - - /* Check if we have stream smsgs in queue */ #if 0 + /* Check if we have stream smsgs in queue */ if (ra_ctx->stream_q->len == 0) { - printf("there should be a stream smsgs in the queue, as we have detected" - " the app layer protocol and one smsg from toserver side has " - "been sent (8): "); + printf("there should be a stream smsgs in the queue (6): "); goto end; /* Process stream smsgs we may have in queue */ } else if (StreamTcpReassembleProcessAppLayer(ra_ctx) < 0) { - printf("failed in processing stream smsgs (9): "); + printf("failed in processing stream smsgs (7): "); goto end; } #endif + if (ssn.flags & STREAMTCP_FLAG_APPPROTO_DETECTION_COMPLETED) { + printf("app layer detected flag is set, it shouldn't be (8): "); + goto end; + } + + /* This packets induces a packet gap and also shows why we need to + process the current segment completely, even if it results in sending more + than one smsg to the app layer. If we don't send more than one smsg in + this case, then the first segment of lentgh 34 bytes will be sent to + app layer and protocol can not be detected in that message and moreover + the segment lentgh is less than the max. signature size for protocol + detection, so this will keep looping !! */ p->flowflags = FLOW_PKT_TOSERVER; - p->payload = httpbuf4; - p->payload_len = httplen4; - tcph.th_seq = htonl(12); + p->payload = httpbuf3; + p->payload_len = httplen3; + tcph.th_seq = htonl(54); tcph.th_ack = htonl(100); s = &ssn.client; - ssn.client.last_ack = 100; if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet (10): "); + printf("failed in segments reassembly, while processing toserver packet (9): "); + goto end; + } + + /* Check if we have stream smsgs in queue */ + if (ra_ctx->stream_q->len > 0) { + printf("there shouldn't be any stream smsgs in the queue, as we didn't" + " detected the app layer protocol till yet (10): "); goto end; } @@ -6269,21 +7242,18 @@ static int StreamTcpReassembleTest40 (void) { p->payload = httpbuf2; p->payload_len = httplen2; tcph.th_seq = htonl(100); - tcph.th_ack = htonl(13); + tcph.th_ack = htonl(53); s = &ssn.server; - ssn.server.last_ack = 13; if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { printf("failed in segments reassembly, while processing toserver packet (11): "); goto end; } - - /* Check if we have stream smsgs in queue */ #if 0 + /* Check if we have stream smsgs in queue */ if (ra_ctx->stream_q->len == 0) { - printf("there should be a stream smsgs in the queue, as we have detected" - " the app layer protocol and one smsg from toserver side has " - "been sent (12): "); + printf("there should be a stream smsgs in the queue, as reassembling has" + " been unpaused now (12): "); goto end; /* Process stream smsgs we may have in queue */ } else if (StreamTcpReassembleProcessAppLayer(ra_ctx) < 0) { @@ -6291,67 +7261,71 @@ static int StreamTcpReassembleTest40 (void) { goto end; } #endif - p->flowflags = FLOW_PKT_TOSERVER; - p->payload = httpbuf5; - p->payload_len = httplen5; - tcph.th_seq = htonl(13); - tcph.th_ack = htonl(145); - s = &ssn.client; - ssn.client.last_ack = 145; + /* the flag should be set, as the smsg scanned size has crossed the max. + signature size for app proto detection */ + if (! (ssn.flags & STREAMTCP_FLAG_APPPROTO_DETECTION_COMPLETED)) { + printf("app layer detected flag is not set, it should be (14): "); + goto end; + } - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet (14): "); + ret = 1; +end: + StreamTcpFreeConfig(TRUE); + StreamTcpReassembleFreeThreadCtx(ra_ctx); + SCFree(p); + return ret; +} + +/** \test Test the memcap incrementing/decrementing and memcap check */ +static int StreamTcpReassembleTest44(void) +{ + uint8_t ret = 0; + StreamTcpInitConfig(TRUE); + uint32_t memuse = stream_reassembly_memuse; + + StreamTcpReassembleIncrMemuse(500); + if (stream_reassembly_memuse != (memuse+500)) { + printf("failed in incrementing the memory"); goto end; } - p->flowflags = FLOW_PKT_TOCLIENT; - p->payload = httpbuf2; - p->payload_len = httplen2; - tcph.th_seq = htonl(145); - tcph.th_ack = htonl(16); - s = &ssn.server; - ssn.server.last_ack = 16; - - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet (15): "); + StreamTcpReassembleDecrMemuse(500); + if (stream_reassembly_memuse != memuse) { + printf("failed in decrementing the memory"); goto end; } - /* Check if we have stream smsgs in queue */ - if (ra_ctx->stream_q->len == 0) { - printf("there should be a stream smsgs in the queue, as we have detected" - " the app layer protocol and one smsg from toserver side has " - "been sent (16): "); + if (StreamTcpReassembleCheckMemcap(500) != 1) { + printf("failed in validating the memcap"); goto end; - /* Process stream smsgs we may have in queue */ - } else if (StreamTcpReassembleProcessAppLayer(ra_ctx) < 0) { - printf("failed in processing stream smsgs (17): "); + } + + if (StreamTcpReassembleCheckMemcap((memuse + stream_config.reassembly_memcap)) != 0) { + printf("failed in validating the memcap"); goto end; } - if (f.alproto != ALPROTO_HTTP) { - printf("app layer proto has not been detected (18): "); + StreamTcpFreeConfig(TRUE); + + if (stream_reassembly_memuse != 0) { + printf("failed in clearing the memory"); goto end; } ret = 1; + return ret; end: StreamTcpFreeConfig(TRUE); - StreamTcpReassembleFreeThreadCtx(ra_ctx); - SCFree(p); return ret; } /** - * \test Test to make sure we don't send more than one smsg from toserver to - * app layer until the app layer protocol has not been detected. After - * protocol has been detected the processed segments should be returned - * to pool. + * \test Test to make sure that reassembly_depth is enforced. * * \retval On success it returns 1 and on failure 0. */ -static int StreamTcpReassembleTest41 (void) { +static int StreamTcpReassembleTest45 (void) { int ret = 0; Packet *p = SCMalloc(SIZE_OF_PACKET); if (p == NULL) @@ -6377,45 +7351,20 @@ static int StreamTcpReassembleTest41 (void) { ThreadVars tv; memset(&tv, 0, sizeof (ThreadVars)); - FLOW_INITIALIZE(&f); - StreamTcpInitConfig(TRUE); - TcpReassemblyThreadCtx *ra_ctx = StreamTcpReassembleInitThreadCtx(); - AppLayerDetectProtoThreadInit(); + uint8_t httpbuf1[] = "/ HTTP/1.0\r\nUser-Agent: Victor/1.0"; - uint8_t httpbuf1[] = "GET / HTTP/1.0\r\nUser-Agent: Victor/1.0" - "W2dyb3VwMV0NCnBob25lMT1wMDB3ODgyMTMxMzAyMTINCmxvZ2lu" - "MT0NCnBhc3N3b3JkMT0NCnBob25lMj1wMDB3ODgyMTMxMzAyMTIN" - "CmxvZ2luMj0NCnBhc3N3b3JkMj0NCnBob25lMz0NCmxvZ2luMz0N" - "CnBhc3N3b3JkMz0NCnBob25lND0NCmxvZ2luND0NCnBhc3N3b3Jk" - "ND0NCnBob25lNT0NCmxvZ2luNT0NCnBhc3N3b3JkNT0NCnBob25l" - "Nj0NCmxvZ2luNj0NCnBhc3N3b3JkNj0NCmNhbGxfdGltZTE9MzIN" - "CmNhbGxfdGltZTI9MjMyDQpkYXlfbGltaXQ9NQ0KbW9udGhfbGlt" - "aXQ9MTUNCltncm91cDJdDQpwaG9uZTE9DQpsb2dpbjE9DQpwYXNz" - "d29yZDE9DQpwaG9uZTI9DQpsb2dpbjI9DQpwYXNzd29yZDI9DQpw" - "aG9uZT"; uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */ - uint8_t httpbuf3[] = "psb2dpbjM9DQpwYXNzd29yZDM9DQpwaG9uZTQ9DQps" - "b2dpbjQ9DQpwYXNzd29yZDQ9DQpwaG9uZTU9DQpsb2dpbjU9DQpw" - "YXNzd29yZDU9DQpwaG9uZTY9DQpsb2dpbjY9DQpwYXNzd29yZDY9" - "DQpjYWxsX3RpbWUxPQ0KY2FsbF90aW1lMj0NCmRheV9saW1pdD0N" - "\r\n\r\n"; - uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */ - - uint8_t httpbuf2[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n"; - uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */ - - StreamMsgQueueSetMinInitChunkLen(FLOW_PKT_TOSERVER, 0); - StreamMsgQueueSetMinInitChunkLen(FLOW_PKT_TOCLIENT, 0); - StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 0); - StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 0); + FLOW_INITIALIZE(&f); + StreamTcpInitConfig(TRUE); + TcpReassemblyThreadCtx *ra_ctx = StreamTcpReassembleInitThreadCtx(); - ssn.server.ra_raw_base_seq = ssn.server.ra_app_base_seq = 9; + STREAMTCP_SET_RA_BASE_SEQ(&ssn.server, 9); ssn.server.isn = 9; - ssn.server.last_ack = 600; - ssn.client.ra_raw_base_seq = ssn.client.ra_app_base_seq = 9; + ssn.server.last_ack = 60; + STREAMTCP_SET_RA_BASE_SEQ(&ssn.client, 9); ssn.client.isn = 9; - ssn.client.last_ack = 600; + ssn.client.last_ack = 60; f.alproto = ALPROTO_UNKNOWN; inet_pton(AF_INET, "1.2.3.4", &in); @@ -6441,41 +7390,30 @@ static int StreamTcpReassembleTest41 (void) { p->tcph = &tcph; p->flowflags = FLOW_PKT_TOCLIENT; - p->payload = httpbuf2; - p->payload_len = httplen2; + p->payload = httpbuf1; + p->payload_len = httplen1; ssn.state = TCP_ESTABLISHED; + /* set the default value of reassembly depth, as there is no config file */ + stream_config.reassembly_depth = httplen1 + 1; + TcpStream *s = NULL; s = &ssn.server; if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet: "); + printf("failed in segments reassembly, while processing toclient packet: "); goto end; } - /* Check if we have stream smsgs in queue */ - if (ra_ctx->stream_q->len > 0) { - printf("there shouldn't be any stream smsgs in the queue: "); + /* Check if we have flags set or not */ + if (s->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) { + printf("there shouldn't be a noreassembly flag be set: "); goto end; } + STREAMTCP_SET_RA_BASE_SEQ(&ssn.server, ssn.server.isn + httplen1); p->flowflags = FLOW_PKT_TOSERVER; - p->payload = httpbuf1; p->payload_len = httplen1; - tcph.th_seq = htonl(10); - tcph.th_ack = htonl(55); - s = &ssn.client; - - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet: "); - goto end; - } - - p->flowflags = FLOW_PKT_TOSERVER; - p->payload = httpbuf3; - p->payload_len = httplen3; - tcph.th_seq = htonl(522); - tcph.th_ack = htonl(100); s = &ssn.client; if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { @@ -6483,43 +7421,15 @@ static int StreamTcpReassembleTest41 (void) { goto end; } - /* Check if we have stream smsgs in queue */ - if (ra_ctx->stream_q->len > 0) { - printf("there shouldn't be any stream smsgs in the queue, as we didn't" - " processed any smsg from toserver side till yet: "); - goto end; - } - - p->flowflags = FLOW_PKT_TOCLIENT; - p->payload = httpbuf2; - p->payload_len = httplen2; - tcph.th_seq = htonl(55); - tcph.th_ack = htonl(522); - s = &ssn.server; - - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet: "); - goto end; - } - - /* Check if we have stream smsgs in queue */ - if (ra_ctx->stream_q->len == 0) { - printf("there should be a stream smsgs in the queue: "); - goto end; - /* Process stream smsgs we may have in queue */ - } else if (ra_ctx->stream_q->len > 1) { - printf("there should be only one stream smsgs in the queue: "); - goto end; - } else if (StreamTcpReassembleProcessAppLayer(ra_ctx) < 0) { - printf("failed in processing stream smsgs: "); + /* Check if we have flags set or not */ + if (s->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) { + printf("there shouldn't be a noreassembly flag be set: "); goto end; } + STREAMTCP_SET_RA_BASE_SEQ(&ssn.client, ssn.client.isn + httplen1); p->flowflags = FLOW_PKT_TOCLIENT; - p->payload = httpbuf2; - p->payload_len = httplen2; - tcph.th_seq = htonl(100); - tcph.th_ack = htonl(522); + p->payload_len = httplen1; s = &ssn.server; if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { @@ -6527,8 +7437,12 @@ static int StreamTcpReassembleTest41 (void) { goto end; } - if (ssn.client.seg_list != NULL) { - printf("seg_list should be null: "); + /* Check if we have flags set or not */ + if (!(s->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY)) { + printf("the noreassembly flags should be set, " + "p.payload_len %"PRIu16" stream_config.reassembly_" + "depth %"PRIu32": ", p->payload_len, + stream_config.reassembly_depth); goto end; } @@ -6541,12 +7455,14 @@ end: } /** - * \test Test to make sure that Pause/Unpause API is working. + * \test Test the undefined config value of reassembly depth. + * the default value of 0 will be loaded and stream will be reassembled + * until the session ended * * \retval On success it returns 1 and on failure 0. */ -static int StreamTcpReassembleTest42 (void) { +static int StreamTcpReassembleTest46 (void) { int ret = 0; Packet *p = SCMalloc(SIZE_OF_PACKET); if (p == NULL) @@ -6559,6 +7475,7 @@ static int StreamTcpReassembleTest42 (void) { Address dst; struct in_addr in; TcpSession ssn; + ThreadVars tv; memset(p, 0, SIZE_OF_PACKET); p->pkt = (uint8_t *)(p + 1); @@ -6569,26 +7486,23 @@ static int StreamTcpReassembleTest42 (void) { memset(&src, 0, sizeof(Address)); memset(&dst, 0, sizeof(Address)); memset(&ssn, 0, sizeof(TcpSession)); - ThreadVars tv; memset(&tv, 0, sizeof (ThreadVars)); + uint8_t httpbuf1[] = "/ HTTP/1.0\r\nUser-Agent: Victor/1.0"; + uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */ + FLOW_INITIALIZE(&f); StreamTcpInitConfig(TRUE); TcpReassemblyThreadCtx *ra_ctx = StreamTcpReassembleInitThreadCtx(); - AppLayerDetectProtoThreadInit(); - - uint8_t httpbuf1[] = "POST / HTTP/1.0\r\nUser-Agent: Victor/1.0\r\n\r\n"; - uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */ - - uint8_t httpbuf2[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n"; - uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */ - ssn.server.ra_raw_base_seq = ssn.server.ra_app_base_seq = 9; + STREAMTCP_SET_RA_BASE_SEQ(&ssn.server, 9); ssn.server.isn = 9; ssn.server.last_ack = 60; - ssn.client.ra_raw_base_seq = ssn.client.ra_app_base_seq = 9; + ssn.server.next_seq = ssn.server.isn; + STREAMTCP_SET_RA_BASE_SEQ(&ssn.client, 9); ssn.client.isn = 9; ssn.client.last_ack = 60; + ssn.client.next_seq = ssn.client.isn; f.alproto = ALPROTO_UNKNOWN; inet_pton(AF_INET, "1.2.3.4", &in); @@ -6614,88 +7528,63 @@ static int StreamTcpReassembleTest42 (void) { p->tcph = &tcph; p->flowflags = FLOW_PKT_TOCLIENT; - p->payload = httpbuf2; - p->payload_len = httplen2; + p->payload = httpbuf1; + p->payload_len = httplen1; ssn.state = TCP_ESTABLISHED; + stream_config.reassembly_depth = 0; + TcpStream *s = NULL; s = &ssn.server; if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet (1): "); + printf("failed in segments reassembly, while processing toclient packet\n"); goto end; } - /* Check if we have stream smsgs in queue */ - if (ra_ctx->stream_q->len > 0) { - printf("there shouldn't be any stream smsgs in the queue (2): "); + /* Check if we have flags set or not */ + if ((ssn.client.flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) || + (ssn.server.flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY)) { + printf("there shouldn't be any no reassembly flag be set \n"); goto end; } + STREAMTCP_SET_RA_BASE_SEQ(&ssn.server, ssn.server.isn + httplen1); p->flowflags = FLOW_PKT_TOSERVER; - p->payload = httpbuf1; p->payload_len = httplen1; - tcph.th_seq = htonl(10); - tcph.th_ack = htonl(55); s = &ssn.client; if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet (3): "); - goto end; - } - - /* Check if we have stream smsgs in queue */ - if (ra_ctx->stream_q->len > 0) { - printf("there shouldn't be any stream smsgs in the queue, as we didn't" - " processed any smsg from toserver side till yet (4): "); - goto end; - } - - /* pause the reassembling */ - StreamTcpReassemblePause(&ssn, 1); - - p->flowflags = FLOW_PKT_TOCLIENT; - p->payload = httpbuf2; - p->payload_len = httplen2; - tcph.th_seq = htonl(55); - tcph.th_ack = htonl(53); - s = &ssn.server; - - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet (5): "); + printf("failed in segments reassembly, while processing toserver packet\n"); goto end; } - /* Check if we have stream smsgs in queue */ - if (ra_ctx->stream_q->len > 0) { - printf("there shouldn't be any stream smsgs in the queue, as we have" - " paused the reassembling (6): "); + /* Check if we have flags set or not */ + if ((ssn.client.flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) || + (ssn.server.flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY)) { + printf("there shouldn't be any no reassembly flag be set \n"); goto end; } - - /* Unpause the reassembling */ - StreamTcpReassembleUnPause(&ssn, 1); + STREAMTCP_SET_RA_BASE_SEQ(&ssn.client, ssn.client.isn + httplen1); p->flowflags = FLOW_PKT_TOCLIENT; - p->payload = httpbuf2; - p->payload_len = httplen2; - tcph.th_seq = htonl(100); - tcph.th_ack = htonl(53); + p->payload_len = httplen1; + tcph.th_seq = htonl(10 + httplen1); + tcph.th_ack = htonl(20 + httplen1); s = &ssn.server; if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet (7): "); - goto end; - } - - /* Check if we have stream smsgs in queue */ - if (ra_ctx->stream_q->len == 0) { - printf("there should be a stream smsgs in the queue, as reassembling has" - " been unpaused now (8): "); + printf("failed in segments reassembly, while processing toserver packet\n"); goto end; - /* Process stream smsgs we may have in queue */ - } else if (StreamTcpReassembleProcessAppLayer(ra_ctx) < 0) { - printf("failed in processing stream smsgs (9): "); + } + + /* Check if we have flags set or not */ + if ((ssn.client.flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) || + (ssn.server.flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY)) { + printf("the no_reassembly flags should not be set, " + "p->payload_len %"PRIu16" stream_config.reassembly_" + "depth %"PRIu32": ", p->payload_len, + stream_config.reassembly_depth); goto end; } @@ -6708,12 +7597,13 @@ end: } /** - * \test Test to make sure that Pause/Unpause API is working. + * \test Test to make sure we detect the sequence wrap around and continue + * stream reassembly properly. * * \retval On success it returns 1 and on failure 0. */ -static int StreamTcpReassembleTest43 (void) { +static int StreamTcpReassembleTest47 (void) { int ret = 0; Packet *p = SCMalloc(SIZE_OF_PACKET); if (p == NULL) @@ -6726,6 +7616,7 @@ static int StreamTcpReassembleTest43 (void) { Address dst; struct in_addr in; TcpSession ssn; + ThreadVars tv; memset(p, 0, SIZE_OF_PACKET); p->pkt = (uint8_t *)(p + 1); @@ -6736,57 +7627,22 @@ static int StreamTcpReassembleTest43 (void) { memset(&src, 0, sizeof(Address)); memset(&dst, 0, sizeof(Address)); memset(&ssn, 0, sizeof(TcpSession)); - ThreadVars tv; memset(&tv, 0, sizeof (ThreadVars)); + /* prevent L7 from kicking in */ + StreamMsgQueueSetMinInitChunkLen(FLOW_PKT_TOSERVER, 0); + StreamMsgQueueSetMinInitChunkLen(FLOW_PKT_TOCLIENT, 0); + StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 0); + StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 0); + FLOW_INITIALIZE(&f); StreamTcpInitConfig(TRUE); TcpReassemblyThreadCtx *ra_ctx = StreamTcpReassembleInitThreadCtx(); AppLayerDetectProtoThreadInit(); - uint8_t httpbuf1[] = "/ HTTP/1.0\r\nUser-Agent: Victor/1.0"; - + uint8_t httpbuf1[] = "GET /EVILSUFF HTTP/1.1\r\n\r\n"; uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */ - uint8_t httpbuf2[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n"; - uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */ - - uint8_t httpbuf3[] = "W2dyb3VwMV0NCnBob25lMT1wMDB3ODgyMTMxMzAyMTINCmxvZ2lu" - "MT0NCnBhc3N3b3JkMT0NCnBob25lMj1wMDB3ODgyMTMxMzAyMTIN" - "CmxvZ2luMj0NCnBhc3N3b3JkMj0NCnBob25lMz0NCmxvZ2luMz0N" - "CnBhc3N3b3JkMz0NCnBob25lND0NCmxvZ2luND0NCnBhc3N3b3Jk" - "ND0NCnBob25lNT0NCmxvZ2luNT0NCnBhc3N3b3JkNT0NCnBob25l" - "Nj0NCmxvZ2luNj0NCnBhc3N3b3JkNj0NCmNhbGxfdGltZTE9MzIN" - "CmNhbGxfdGltZTI9MjMyDQpkYXlfbGltaXQ9NQ0KbW9udGhfbGlt" - "aXQ9MTUNCltncm91cDJdDQpwaG9uZTE9DQpsb2dpbjE9DQpwYXNz" - "d29yZDE9DQpwaG9uZTI9DQpsb2dpbjI9DQpwYXNzd29yZDI9DQpw" - "aG9uZT\r\n\r\n"; - uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */ - - ssn.server.ra_raw_base_seq = ssn.server.ra_app_base_seq = 9; - ssn.server.isn = 9; - ssn.server.last_ack = 600; - ssn.client.ra_raw_base_seq = ssn.client.ra_app_base_seq = 9; - ssn.client.isn = 9; - ssn.client.last_ack = 600; - f.alproto = ALPROTO_UNKNOWN; - - /* Check the minimum init smsg length. It should be equal to the min length - of given signature in toserver direction. */ - if (StreamMsgQueueGetMinInitChunkLen(FLOW_PKT_TOSERVER) != 2) { - printf("the min init length should be equal to 2, not %"PRIu16": ", - StreamMsgQueueGetMinInitChunkLen(FLOW_PKT_TOSERVER)); - goto end; - } - - /* Check the minimum init smsg length. It should be equal to the min length - of given signature in toclient direction. */ - if (StreamMsgQueueGetMinInitChunkLen(FLOW_PKT_TOCLIENT) != 2) { - printf("the min init length should be equal to 2, not %"PRIu16": ", - StreamMsgQueueGetMinInitChunkLen(FLOW_PKT_TOCLIENT)); - goto end; - } - inet_pton(AF_INET, "1.2.3.4", &in); src.family = AF_INET; src.addr_data32[0] = in.s_addr; @@ -6796,593 +7652,824 @@ static int StreamTcpReassembleTest43 (void) { sp = 200; dp = 220; + ssn.server.ra_raw_base_seq = ssn.server.ra_app_base_seq = 572799781UL; + ssn.server.isn = 572799781UL; + ssn.server.last_ack = 572799782UL; + ssn.client.ra_raw_base_seq = ssn.client.ra_app_base_seq = 4294967289UL; + ssn.client.isn = 4294967289UL; + ssn.client.last_ack = 21; + f.alproto = ALPROTO_UNKNOWN; + f.src = src; f.dst = dst; f.sp = sp; f.dp = dp; f.protoctx = &ssn; p->flow = &f; - tcph.th_win = htons(5480); - tcph.th_seq = htonl(10); - tcph.th_ack = htonl(10); - tcph.th_flags = TH_ACK|TH_PUSH; - p->tcph = &tcph; - p->flowflags = FLOW_PKT_TOCLIENT; - - p->payload = httpbuf2; - p->payload_len = httplen2; ssn.state = TCP_ESTABLISHED; - TcpStream *s = NULL; - s = &ssn.server; + uint8_t cnt = 0; - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet (1): "); + for (cnt=0; cnt < httplen1; cnt++) { + tcph.th_seq = htonl(ssn.client.isn + 1 + cnt); + tcph.th_ack = htonl(572799782UL); + tcph.th_flags = TH_ACK|TH_PUSH; + p->tcph = &tcph; + p->flowflags = FLOW_PKT_TOSERVER; + p->payload = &httpbuf1[cnt]; + p->payload_len = 1; + s = &ssn.client; + + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { + printf("failed in segments reassembly, while processing toserver " + "packet\n"); + goto end; + } + + p->flowflags = FLOW_PKT_TOCLIENT; + p->payload = NULL; + p->payload_len = 0; + tcph.th_seq = htonl(572799782UL); + tcph.th_ack = htonl(ssn.client.isn + 1 + cnt); + tcph.th_flags = TH_ACK; + p->tcph = &tcph; + s = &ssn.server; + + if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { + printf("failed in segments reassembly, while processing toserver " + "packet\n"); + goto end; + } + + /* Process stream smsgs we may have in queue */ + if (StreamTcpReassembleProcessAppLayer(ra_ctx) < 0) { + printf("failed in processing stream smsgs\n"); + goto end; + } + } + + if (f.alproto != ALPROTO_HTTP) { + printf("App layer protocol (HTTP) should have been detected\n"); goto end; } - /* Check if we have stream smsgs in queue */ - if (ra_ctx->stream_q->len > 0) { - printf("there shouldn't be any stream smsgs in the queue (2): "); + ret = 1; +end: + StreamTcpFreeConfig(TRUE); + StreamTcpReassembleFreeThreadCtx(ra_ctx); + SCFree(p); + return ret; +} + +/** \test 3 in order segments in inline reassembly */ +static int StreamTcpReassembleInlineTest01(void) { + int ret = 0; + TcpReassemblyThreadCtx *ra_ctx = NULL; + ThreadVars tv; + TcpSession ssn; + Flow f; + + memset(&tv, 0x00, sizeof(tv)); + + StreamTcpUTInit(&ra_ctx); + StreamTcpUTSetupSession(&ssn); + StreamTcpUTSetupStream(&ssn.client, 1); + FLOW_INITIALIZE(&f); + + uint8_t stream_payload[] = "AAAAABBBBBCCCCC"; + uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' }; + Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); + if (p == NULL) { + printf("couldn't get a packet: "); goto end; } + p->tcph->th_seq = htonl(12); + p->flow = &f; - p->flowflags = FLOW_PKT_TOSERVER; - p->payload = httpbuf1; - p->payload_len = httplen1; - tcph.th_seq = htonl(10); - tcph.th_ack = htonl(55); - s = &ssn.client; + if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) { + printf("failed to add segment 1: "); + goto end; + } + if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) { + printf("failed to add segment 2: "); + goto end; + } + if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) { + printf("failed to add segment 3: "); + goto end; + } + ssn.client.next_seq = 17; - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet (3): "); + int r = StreamTcpReassembleInlineRaw(ra_ctx, &ssn, &ssn.client, p); + if (r < 0) { + printf("StreamTcpReassembleInlineRaw failed: "); goto end; } - /* Check if we have stream smsgs in queue */ - if (ra_ctx->stream_q->len > 0) { - printf("there shouldn't be any stream smsgs in the queue, as we didn't" - " processed any smsg from toserver side till yet (4): "); + if (ra_ctx->stream_q->len != 1) { + printf("expected a single stream message, got %u: ", ra_ctx->stream_q->len); goto end; } - p->flowflags = FLOW_PKT_TOCLIENT; - p->payload = httpbuf2; - p->payload_len = httplen2; - tcph.th_seq = htonl(55); - tcph.th_ack = htonl(44); - s = &ssn.server; + StreamMsg *smsg = ra_ctx->stream_q->top; + if (smsg->data.data_len != 15) { + printf("expected data length to be 15, got %u: ", smsg->data.data_len); + goto end; + } - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet (5): "); + if (!(memcmp(stream_payload, smsg->data.data, 15) == 0)) { + printf("data is not what we expected:\nExpected:\n"); + PrintRawDataFp(stdout, stream_payload, 15); + printf("Got:\n"); + PrintRawDataFp(stdout, smsg->data.data, smsg->data.data_len); + goto end; + } + + ret = 1; +end: + FLOW_DESTROY(&f); + UTHFreePacket(p); + StreamTcpUTClearSession(&ssn); + StreamTcpUTDeinit(ra_ctx); + return ret; +} + +/** \test 3 in order segments, then reassemble, add one more and reassemble again. + * test the sliding window reassembly. + */ +static int StreamTcpReassembleInlineTest02(void) { + int ret = 0; + TcpReassemblyThreadCtx *ra_ctx = NULL; + ThreadVars tv; + TcpSession ssn; + Flow f; + + memset(&tv, 0x00, sizeof(tv)); + + StreamTcpUTInit(&ra_ctx); + StreamTcpUTSetupSession(&ssn); + StreamTcpUTSetupStream(&ssn.client, 1); + FLOW_INITIALIZE(&f); + + uint8_t stream_payload1[] = "AAAAABBBBBCCCCC"; + uint8_t stream_payload2[] = "AAAAABBBBBCCCCCDDDDD"; + uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' }; + Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); + if (p == NULL) { + printf("couldn't get a packet: "); + goto end; + } + p->tcph->th_seq = htonl(12); + p->flow = &f; + + if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) { + printf("failed to add segment 1: "); + goto end; + } + if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) { + printf("failed to add segment 2: "); + goto end; + } + if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) { + printf("failed to add segment 3: "); + goto end; + } + ssn.client.next_seq = 17; + + int r = StreamTcpReassembleInlineRaw(ra_ctx, &ssn, &ssn.client, p); + if (r < 0) { + printf("StreamTcpReassembleInlineRaw failed: "); + goto end; + } + + if (ra_ctx->stream_q->len != 1) { + printf("expected a single stream message, got %u: ", ra_ctx->stream_q->len); + goto end; + } + + StreamMsg *smsg = ra_ctx->stream_q->top; + if (smsg->data.data_len != 15) { + printf("expected data length to be 15, got %u: ", smsg->data.data_len); + goto end; + } + + if (!(memcmp(stream_payload1, smsg->data.data, 15) == 0)) { + printf("data is not what we expected:\nExpected:\n"); + PrintRawDataFp(stdout, stream_payload1, 15); + printf("Got:\n"); + PrintRawDataFp(stdout, smsg->data.data, smsg->data.data_len); + goto end; + } + + if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) { + printf("failed to add segment 4: "); + goto end; + } + ssn.client.next_seq = 22; + + r = StreamTcpReassembleInlineRaw(ra_ctx, &ssn, &ssn.client, p); + if (r < 0) { + printf("StreamTcpReassembleInlineRaw failed 2: "); + goto end; + } + + if (ra_ctx->stream_q->len != 2) { + printf("expected a single stream message, got %u: ", ra_ctx->stream_q->len); + goto end; + } + + smsg = ra_ctx->stream_q->top; + if (smsg->data.data_len != 20) { + printf("expected data length to be 20, got %u: ", smsg->data.data_len); + goto end; + } + + if (!(memcmp(stream_payload2, smsg->data.data, 20) == 0)) { + printf("data is not what we expected:\nExpected:\n"); + PrintRawDataFp(stdout, stream_payload2, 20); + printf("Got:\n"); + PrintRawDataFp(stdout, smsg->data.data, smsg->data.data_len); + goto end; + } + + ret = 1; +end: + FLOW_DESTROY(&f); + UTHFreePacket(p); + StreamTcpUTClearSession(&ssn); + StreamTcpUTDeinit(ra_ctx); + return ret; +} + +/** \test 3 in order segments, then reassemble, add one more and reassemble again. + * test the sliding window reassembly with a small window size so that we + * cutting off at the start (left edge) + */ +static int StreamTcpReassembleInlineTest03(void) { + int ret = 0; + TcpReassemblyThreadCtx *ra_ctx = NULL; + ThreadVars tv; + TcpSession ssn; + Flow f; + + memset(&tv, 0x00, sizeof(tv)); + + StreamTcpUTInit(&ra_ctx); + StreamTcpUTSetupSession(&ssn); + StreamTcpUTSetupStream(&ssn.client, 1); + FLOW_INITIALIZE(&f); + + stream_config.reassembly_inline_window = 15; + + uint8_t stream_payload1[] = "AAAAABBBBBCCCCC"; + uint8_t stream_payload2[] = "BBBBBCCCCCDDDDD"; + uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' }; + Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); + if (p == NULL) { + printf("couldn't get a packet: "); + goto end; + } + p->tcph->th_seq = htonl(12); + p->flow = &f; + + if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) { + printf("failed to add segment 1: "); + goto end; + } + if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) { + printf("failed to add segment 2: "); goto end; } -#if 0 - /* Check if we have stream smsgs in queue */ - if (ra_ctx->stream_q->len == 0) { - printf("there should be a stream smsgs in the queue (6): "); + if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) { + printf("failed to add segment 3: "); goto end; - /* Process stream smsgs we may have in queue */ - } else if (StreamTcpReassembleProcessAppLayer(ra_ctx) < 0) { - printf("failed in processing stream smsgs (7): "); + } + ssn.client.next_seq = 17; + + int r = StreamTcpReassembleInlineRaw(ra_ctx, &ssn, &ssn.client, p); + if (r < 0) { + printf("StreamTcpReassembleInlineRaw failed: "); goto end; } -#endif - if (ssn.flags & STREAMTCP_FLAG_APPPROTO_DETECTION_COMPLETED) { - printf("app layer detected flag is set, it shouldn't be (8): "); + + if (ra_ctx->stream_q->len != 1) { + printf("expected a single stream message, got %u: ", ra_ctx->stream_q->len); goto end; } - /* This packets induces a packet gap and also shows why we need to - process the current segment completely, even if it results in sending more - than one smsg to the app layer. If we don't send more than one smsg in - this case, then the first segment of lentgh 34 bytes will be sent to - app layer and protocol can not be detected in that message and moreover - the segment lentgh is less than the max. signature size for protocol - detection, so this will keep looping !! */ - p->flowflags = FLOW_PKT_TOSERVER; - p->payload = httpbuf3; - p->payload_len = httplen3; - tcph.th_seq = htonl(54); - tcph.th_ack = htonl(100); - s = &ssn.client; + StreamMsg *smsg = ra_ctx->stream_q->top; + if (smsg->data.data_len != 15) { + printf("expected data length to be 15, got %u: ", smsg->data.data_len); + goto end; + } - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet (9): "); + if (!(memcmp(stream_payload1, smsg->data.data, 15) == 0)) { + printf("data is not what we expected:\nExpected:\n"); + PrintRawDataFp(stdout, stream_payload1, 15); + printf("Got:\n"); + PrintRawDataFp(stdout, smsg->data.data, smsg->data.data_len); goto end; } - /* Check if we have stream smsgs in queue */ - if (ra_ctx->stream_q->len > 0) { - printf("there shouldn't be any stream smsgs in the queue, as we didn't" - " detected the app layer protocol till yet (10): "); + if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) { + printf("failed to add segment 4: "); goto end; } + ssn.client.next_seq = 22; - p->flowflags = FLOW_PKT_TOCLIENT; - p->payload = httpbuf2; - p->payload_len = httplen2; - tcph.th_seq = htonl(100); - tcph.th_ack = htonl(53); - s = &ssn.server; + p->tcph->th_seq = htonl(17); - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet (11): "); + r = StreamTcpReassembleInlineRaw(ra_ctx, &ssn, &ssn.client, p); + if (r < 0) { + printf("StreamTcpReassembleInlineRaw failed 2: "); goto end; } -#if 0 - /* Check if we have stream smsgs in queue */ - if (ra_ctx->stream_q->len == 0) { - printf("there should be a stream smsgs in the queue, as reassembling has" - " been unpaused now (12): "); + + if (ra_ctx->stream_q->len != 2) { + printf("expected a single stream message, got %u: ", ra_ctx->stream_q->len); goto end; - /* Process stream smsgs we may have in queue */ - } else if (StreamTcpReassembleProcessAppLayer(ra_ctx) < 0) { - printf("failed in processing stream smsgs (13): "); + } + + smsg = ra_ctx->stream_q->top; + if (smsg->data.data_len != 15) { + printf("expected data length to be 15, got %u: ", smsg->data.data_len); goto end; } -#endif - /* the flag should be set, as the smsg scanned size has crossed the max. - signature size for app proto detection */ - if (! (ssn.flags & STREAMTCP_FLAG_APPPROTO_DETECTION_COMPLETED)) { - printf("app layer detected flag is not set, it should be (14): "); + + if (!(memcmp(stream_payload2, smsg->data.data, 15) == 0)) { + printf("data is not what we expected:\nExpected:\n"); + PrintRawDataFp(stdout, stream_payload2, 15); + printf("Got:\n"); + PrintRawDataFp(stdout, smsg->data.data, smsg->data.data_len); goto end; } ret = 1; end: - StreamTcpFreeConfig(TRUE); - StreamTcpReassembleFreeThreadCtx(ra_ctx); - SCFree(p); + FLOW_DESTROY(&f); + UTHFreePacket(p); + StreamTcpUTClearSession(&ssn); + StreamTcpUTDeinit(ra_ctx); return ret; } -/** \test Test the memcap incrementing/decrementing and memcap check */ -static int StreamTcpReassembleTest44(void) -{ - uint8_t ret = 0; - StreamTcpInitConfig(TRUE); - uint32_t memuse = stream_reassembly_memuse; +/** \test 3 in order segments, then reassemble, add one more and reassemble again. + * test the sliding window reassembly with a small window size so that we + * cutting off at the start (left edge) with small packet overlap. + */ +static int StreamTcpReassembleInlineTest04(void) { + int ret = 0; + TcpReassemblyThreadCtx *ra_ctx = NULL; + ThreadVars tv; + TcpSession ssn; + Flow f; - StreamTcpReassembleIncrMemuse(500); - if (stream_reassembly_memuse != (memuse+500)) { - printf("failed in incrementing the memory"); + memset(&tv, 0x00, sizeof(tv)); + + StreamTcpUTInit(&ra_ctx); + StreamTcpUTSetupSession(&ssn); + StreamTcpUTSetupStream(&ssn.client, 1); + FLOW_INITIALIZE(&f); + + stream_config.reassembly_inline_window = 16; + + uint8_t stream_payload1[] = "AAAAABBBBBCCCCC"; + uint8_t stream_payload2[] = "ABBBBBCCCCCDDDDD"; + uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' }; + Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); + if (p == NULL) { + printf("couldn't get a packet: "); goto end; } + p->tcph->th_seq = htonl(12); + p->flow = &f; - StreamTcpReassembleDecrMemuse(500); - if (stream_reassembly_memuse != memuse) { - printf("failed in decrementing the memory"); + if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) { + printf("failed to add segment 1: "); + goto end; + } + if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) { + printf("failed to add segment 2: "); goto end; } + if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) { + printf("failed to add segment 3: "); + goto end; + } + ssn.client.next_seq = 17; - if (StreamTcpReassembleCheckMemcap(500) != 1) { - printf("failed in validating the memcap"); + int r = StreamTcpReassembleInlineRaw(ra_ctx, &ssn, &ssn.client, p); + if (r < 0) { + printf("StreamTcpReassembleInlineRaw failed: "); goto end; } - if (StreamTcpReassembleCheckMemcap((memuse + stream_config.reassembly_memcap)) != 0) { - printf("failed in validating the memcap"); + if (ra_ctx->stream_q->len != 1) { + printf("expected a single stream message, got %u: ", ra_ctx->stream_q->len); goto end; } - StreamTcpFreeConfig(TRUE); + StreamMsg *smsg = ra_ctx->stream_q->top; + if (smsg->data.data_len != 15) { + printf("expected data length to be 15, got %u: ", smsg->data.data_len); + goto end; + } - if (stream_reassembly_memuse != 0) { - printf("failed in clearing the memory"); + if (!(memcmp(stream_payload1, smsg->data.data, 15) == 0)) { + printf("data is not what we expected:\nExpected:\n"); + PrintRawDataFp(stdout, stream_payload1, 15); + printf("Got:\n"); + PrintRawDataFp(stdout, smsg->data.data, smsg->data.data_len); + goto end; + } + + if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) { + printf("failed to add segment 4: "); + goto end; + } + ssn.client.next_seq = 22; + + p->tcph->th_seq = htonl(17); + + r = StreamTcpReassembleInlineRaw(ra_ctx, &ssn, &ssn.client, p); + if (r < 0) { + printf("StreamTcpReassembleInlineRaw failed 2: "); + goto end; + } + + if (ra_ctx->stream_q->len != 2) { + printf("expected a single stream message, got %u: ", ra_ctx->stream_q->len); + goto end; + } + + smsg = ra_ctx->stream_q->top; + if (smsg->data.data_len != 16) { + printf("expected data length to be 16, got %u: ", smsg->data.data_len); + goto end; + } + + if (!(memcmp(stream_payload2, smsg->data.data, 16) == 0)) { + printf("data is not what we expected:\nExpected:\n"); + PrintRawDataFp(stdout, stream_payload2, 16); + printf("Got:\n"); + PrintRawDataFp(stdout, smsg->data.data, smsg->data.data_len); goto end; } ret = 1; - return ret; end: - StreamTcpFreeConfig(TRUE); + FLOW_DESTROY(&f); + UTHFreePacket(p); + StreamTcpUTClearSession(&ssn); + StreamTcpUTDeinit(ra_ctx); return ret; } -/** - * \test Test to make sure that reassembly_depth is enforced. - * - * \retval On success it returns 1 and on failure 0. - */ - -static int StreamTcpReassembleTest45 (void) { +/** \test with a GAP we should have 2 smsgs */ +static int StreamTcpReassembleInlineTest05(void) { int ret = 0; - Packet *p = SCMalloc(SIZE_OF_PACKET); - if (p == NULL) - return 0; - Flow f; - TCPHdr tcph; - Port sp; - Port dp; - Address src; - Address dst; - struct in_addr in; - TcpSession ssn; - - memset(p, 0, SIZE_OF_PACKET); - p->pkt = (uint8_t *)(p + 1); - PacketQueue pq; - memset(&pq,0,sizeof(PacketQueue)); - memset(&f, 0, sizeof (Flow)); - memset(&tcph, 0, sizeof (TCPHdr)); - memset(&src, 0, sizeof(Address)); - memset(&dst, 0, sizeof(Address)); - memset(&ssn, 0, sizeof(TcpSession)); + TcpReassemblyThreadCtx *ra_ctx = NULL; ThreadVars tv; - memset(&tv, 0, sizeof (ThreadVars)); - - uint8_t httpbuf1[] = "/ HTTP/1.0\r\nUser-Agent: Victor/1.0"; + TcpSession ssn; + Flow f; - uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */ + memset(&tv, 0x00, sizeof(tv)); + StreamTcpUTInit(&ra_ctx); + StreamTcpUTSetupSession(&ssn); + StreamTcpUTSetupStream(&ssn.client, 1); FLOW_INITIALIZE(&f); - StreamTcpInitConfig(TRUE); - TcpReassemblyThreadCtx *ra_ctx = StreamTcpReassembleInitThreadCtx(); - - STREAMTCP_SET_RA_BASE_SEQ(&ssn.server, 9); - ssn.server.isn = 9; - ssn.server.last_ack = 60; - STREAMTCP_SET_RA_BASE_SEQ(&ssn.client, 9); - ssn.client.isn = 9; - ssn.client.last_ack = 60; - f.alproto = ALPROTO_UNKNOWN; - - inet_pton(AF_INET, "1.2.3.4", &in); - src.family = AF_INET; - src.addr_data32[0] = in.s_addr; - inet_pton(AF_INET, "1.2.3.5", &in); - dst.family = AF_INET; - dst.addr_data32[0] = in.s_addr; - sp = 200; - dp = 220; - f.src = src; - f.dst = dst; - f.sp = sp; - f.dp = dp; - f.protoctx = &ssn; + uint8_t stream_payload1[] = "AAAAABBBBB"; + uint8_t stream_payload2[] = "DDDDD"; + uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' }; + Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); + if (p == NULL) { + printf("couldn't get a packet: "); + goto end; + } + p->tcph->th_seq = htonl(12); p->flow = &f; - tcph.th_win = htons(5480); - tcph.th_seq = htonl(10); - tcph.th_ack = htonl(20); - tcph.th_flags = TH_ACK|TH_PUSH; - p->tcph = &tcph; - p->flowflags = FLOW_PKT_TOCLIENT; - - p->payload = httpbuf1; - p->payload_len = httplen1; - ssn.state = TCP_ESTABLISHED; - - /* set the default value of reassembly depth, as there is no config file */ - stream_config.reassembly_depth = httplen1 + 1; - - TcpStream *s = NULL; - s = &ssn.server; - - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toclient packet: "); + if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) { + printf("failed to add segment 1: "); + goto end; + } + if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) { + printf("failed to add segment 2: "); goto end; } + ssn.client.next_seq = 12; - /* Check if we have flags set or not */ - if (s->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) { - printf("there shouldn't be a noreassembly flag be set: "); + if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) { + printf("failed to add segment 4: "); goto end; } - STREAMTCP_SET_RA_BASE_SEQ(&ssn.server, ssn.server.isn + httplen1); - p->flowflags = FLOW_PKT_TOSERVER; - p->payload_len = httplen1; - s = &ssn.client; + p->tcph->th_seq = htonl(17); - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet: "); + int r = StreamTcpReassembleInlineRaw(ra_ctx, &ssn, &ssn.client, p); + if (r < 0) { + printf("StreamTcpReassembleInlineRaw failed: "); goto end; } - /* Check if we have flags set or not */ - if (s->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) { - printf("there shouldn't be a noreassembly flag be set: "); + if (ra_ctx->stream_q->len != 2) { + printf("expected a single stream message, got %u: ", ra_ctx->stream_q->len); goto end; } - STREAMTCP_SET_RA_BASE_SEQ(&ssn.client, ssn.client.isn + httplen1); - - p->flowflags = FLOW_PKT_TOCLIENT; - p->payload_len = httplen1; - s = &ssn.server; - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet: "); + StreamMsg *smsg = ra_ctx->stream_q->top->next; + if (smsg->data.data_len != 10) { + printf("expected data length to be 10, got %u: ", smsg->data.data_len); goto end; } - /* Check if we have flags set or not */ - if (!(s->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY)) { - printf("the noreassembly flags should be set, " - "p.payload_len %"PRIu16" stream_config.reassembly_" - "depth %"PRIu32": ", p->payload_len, - stream_config.reassembly_depth); + if (!(memcmp(stream_payload1, smsg->data.data, 10) == 0)) { + printf("data is not what we expected:\nExpected:\n"); + PrintRawDataFp(stdout, stream_payload2, 10); + printf("Got:\n"); + PrintRawDataFp(stdout, smsg->data.data, smsg->data.data_len); goto end; } - ret = 1; -end: - StreamTcpFreeConfig(TRUE); - StreamTcpReassembleFreeThreadCtx(ra_ctx); - SCFree(p); - return ret; -} + smsg = ra_ctx->stream_q->top; + if (smsg->data.data_len != 5) { + printf("expected data length to be 5, got %u: ", smsg->data.data_len); + goto end; + } -/** - * \test Test the undefined config value of reassembly depth. - * the default value of 0 will be loaded and stream will be reassembled - * until the session ended - * - * \retval On success it returns 1 and on failure 0. - */ + if (!(memcmp(stream_payload2, smsg->data.data, 5) == 0)) { + printf("data is not what we expected:\nExpected:\n"); + PrintRawDataFp(stdout, stream_payload2, 5); + printf("Got:\n"); + PrintRawDataFp(stdout, smsg->data.data, smsg->data.data_len); + goto end; + } -static int StreamTcpReassembleTest46 (void) { - int ret = 0; - Packet *p = SCMalloc(SIZE_OF_PACKET); - if (p == NULL) - return 0; - Flow f; - TCPHdr tcph; - Port sp; - Port dp; - Address src; - Address dst; - struct in_addr in; - TcpSession ssn; - ThreadVars tv; + ret = 1; +end: + FLOW_DESTROY(&f); + UTHFreePacket(p); + StreamTcpUTClearSession(&ssn); + StreamTcpUTDeinit(ra_ctx); + return ret; +} - memset(p, 0, SIZE_OF_PACKET); - p->pkt = (uint8_t *)(p + 1); - PacketQueue pq; - memset(&pq,0,sizeof(PacketQueue)); - memset(&f, 0, sizeof (Flow)); - memset(&tcph, 0, sizeof (TCPHdr)); - memset(&src, 0, sizeof(Address)); - memset(&dst, 0, sizeof(Address)); - memset(&ssn, 0, sizeof(TcpSession)); - memset(&tv, 0, sizeof (ThreadVars)); +/** \test with a GAP we should have 2 smsgs, with filling the GAP later */ +static int StreamTcpReassembleInlineTest06(void) { + int ret = 0; + TcpReassemblyThreadCtx *ra_ctx = NULL; + ThreadVars tv; + TcpSession ssn; + Flow f; - uint8_t httpbuf1[] = "/ HTTP/1.0\r\nUser-Agent: Victor/1.0"; - uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */ + memset(&tv, 0x00, sizeof(tv)); + StreamTcpUTInit(&ra_ctx); + StreamTcpUTSetupSession(&ssn); + StreamTcpUTSetupStream(&ssn.client, 1); FLOW_INITIALIZE(&f); - StreamTcpInitConfig(TRUE); - TcpReassemblyThreadCtx *ra_ctx = StreamTcpReassembleInitThreadCtx(); - STREAMTCP_SET_RA_BASE_SEQ(&ssn.server, 9); - ssn.server.isn = 9; - ssn.server.last_ack = 60; - ssn.server.next_seq = ssn.server.isn; - STREAMTCP_SET_RA_BASE_SEQ(&ssn.client, 9); - ssn.client.isn = 9; - ssn.client.last_ack = 60; - ssn.client.next_seq = ssn.client.isn; - f.alproto = ALPROTO_UNKNOWN; + uint8_t stream_payload1[] = "AAAAABBBBB"; + uint8_t stream_payload2[] = "DDDDD"; + uint8_t stream_payload3[] = "AAAAABBBBBCCCCCDDDDD"; + uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' }; + Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); + if (p == NULL) { + printf("couldn't get a packet: "); + goto end; + } + p->tcph->th_seq = htonl(12); + p->flow = &f; - inet_pton(AF_INET, "1.2.3.4", &in); - src.family = AF_INET; - src.addr_data32[0] = in.s_addr; - inet_pton(AF_INET, "1.2.3.5", &in); - dst.family = AF_INET; - dst.addr_data32[0] = in.s_addr; - sp = 200; - dp = 220; + if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) { + printf("failed to add segment 1: "); + goto end; + } + if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) { + printf("failed to add segment 2: "); + goto end; + } + ssn.client.next_seq = 12; - f.src = src; - f.dst = dst; - f.sp = sp; - f.dp = dp; - f.protoctx = &ssn; - p->flow = &f; + if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) { + printf("failed to add segment 4: "); + goto end; + } - tcph.th_win = htons(5480); - tcph.th_seq = htonl(10); - tcph.th_ack = htonl(20); - tcph.th_flags = TH_ACK|TH_PUSH; - p->tcph = &tcph; - p->flowflags = FLOW_PKT_TOCLIENT; + p->tcph->th_seq = htonl(17); - p->payload = httpbuf1; - p->payload_len = httplen1; - ssn.state = TCP_ESTABLISHED; + int r = StreamTcpReassembleInlineRaw(ra_ctx, &ssn, &ssn.client, p); + if (r < 0) { + printf("StreamTcpReassembleInlineRaw failed: "); + goto end; + } - stream_config.reassembly_depth = 0; + if (ra_ctx->stream_q->len != 2) { + printf("expected a single stream message, got %u: ", ra_ctx->stream_q->len); + goto end; + } - TcpStream *s = NULL; - s = &ssn.server; + StreamMsg *smsg = ra_ctx->stream_q->top->next; + if (smsg->data.data_len != 10) { + printf("expected data length to be 10, got %u: ", smsg->data.data_len); + goto end; + } - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toclient packet\n"); + if (!(memcmp(stream_payload1, smsg->data.data, 10) == 0)) { + printf("data is not what we expected:\nExpected:\n"); + PrintRawDataFp(stdout, stream_payload2, 10); + printf("Got:\n"); + PrintRawDataFp(stdout, smsg->data.data, smsg->data.data_len); goto end; } - /* Check if we have flags set or not */ - if ((ssn.client.flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) || - (ssn.server.flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY)) { - printf("there shouldn't be any no reassembly flag be set \n"); + smsg = ra_ctx->stream_q->top; + if (smsg->data.data_len != 5) { + printf("expected data length to be 5, got %u: ", smsg->data.data_len); goto end; } - STREAMTCP_SET_RA_BASE_SEQ(&ssn.server, ssn.server.isn + httplen1); - p->flowflags = FLOW_PKT_TOSERVER; - p->payload_len = httplen1; - s = &ssn.client; + if (!(memcmp(stream_payload2, smsg->data.data, 5) == 0)) { + printf("data is not what we expected:\nExpected:\n"); + PrintRawDataFp(stdout, stream_payload2, 5); + printf("Got:\n"); + PrintRawDataFp(stdout, smsg->data.data, smsg->data.data_len); + goto end; + } - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet\n"); + if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) { + printf("failed to add segment 3: "); goto end; } + ssn.client.next_seq = 22; - /* Check if we have flags set or not */ - if ((ssn.client.flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) || - (ssn.server.flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY)) { - printf("there shouldn't be any no reassembly flag be set \n"); + p->tcph->th_seq = htonl(12); + + r = StreamTcpReassembleInlineRaw(ra_ctx, &ssn, &ssn.client, p); + if (r < 0) { + printf("StreamTcpReassembleInlineRaw failed: "); goto end; } - STREAMTCP_SET_RA_BASE_SEQ(&ssn.client, ssn.client.isn + httplen1); - p->flowflags = FLOW_PKT_TOCLIENT; - p->payload_len = httplen1; - tcph.th_seq = htonl(10 + httplen1); - tcph.th_ack = htonl(20 + httplen1); - s = &ssn.server; + if (ra_ctx->stream_q->len != 3) { + printf("expected a single stream message, got %u: ", ra_ctx->stream_q->len); + goto end; + } - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet\n"); + smsg = ra_ctx->stream_q->top; + if (smsg->data.data_len != 20) { + printf("expected data length to be 20, got %u: ", smsg->data.data_len); goto end; } - /* Check if we have flags set or not */ - if ((ssn.client.flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY) || - (ssn.server.flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY)) { - printf("the no_reassembly flags should not be set, " - "p->payload_len %"PRIu16" stream_config.reassembly_" - "depth %"PRIu32": ", p->payload_len, - stream_config.reassembly_depth); + if (!(memcmp(stream_payload3, smsg->data.data, 20) == 0)) { + printf("data is not what we expected:\nExpected:\n"); + PrintRawDataFp(stdout, stream_payload3, 20); + printf("Got:\n"); + PrintRawDataFp(stdout, smsg->data.data, smsg->data.data_len); goto end; } ret = 1; end: - StreamTcpFreeConfig(TRUE); - StreamTcpReassembleFreeThreadCtx(ra_ctx); - SCFree(p); + FLOW_DESTROY(&f); + UTHFreePacket(p); + StreamTcpUTClearSession(&ssn); + StreamTcpUTDeinit(ra_ctx); return ret; } -/** - * \test Test to make sure we detect the sequence wrap around and continue - * stream reassembly properly. - * - * \retval On success it returns 1 and on failure 0. - */ - -static int StreamTcpReassembleTest47 (void) { +/** \test with a GAP we should have 2 smsgs, with filling the GAP later, small + * window */ +static int StreamTcpReassembleInlineTest07(void) { int ret = 0; - Packet *p = SCMalloc(SIZE_OF_PACKET); - if (p == NULL) - return 0; - Flow f; - TCPHdr tcph; - Port sp; - Port dp; - Address src; - Address dst; - struct in_addr in; - TcpSession ssn; + TcpReassemblyThreadCtx *ra_ctx = NULL; ThreadVars tv; + TcpSession ssn; + Flow f; - memset(p, 0, SIZE_OF_PACKET); - p->pkt = (uint8_t *)(p + 1); - PacketQueue pq; - memset(&pq,0,sizeof(PacketQueue)); - memset(&f, 0, sizeof (Flow)); - memset(&tcph, 0, sizeof (TCPHdr)); - memset(&src, 0, sizeof(Address)); - memset(&dst, 0, sizeof(Address)); - memset(&ssn, 0, sizeof(TcpSession)); - memset(&tv, 0, sizeof (ThreadVars)); - - /* prevent L7 from kicking in */ - StreamMsgQueueSetMinInitChunkLen(FLOW_PKT_TOSERVER, 0); - StreamMsgQueueSetMinInitChunkLen(FLOW_PKT_TOCLIENT, 0); - StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 0); - StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 0); + memset(&tv, 0x00, sizeof(tv)); + StreamTcpUTInit(&ra_ctx); + StreamTcpUTSetupSession(&ssn); + StreamTcpUTSetupStream(&ssn.client, 1); FLOW_INITIALIZE(&f); - StreamTcpInitConfig(TRUE); - TcpReassemblyThreadCtx *ra_ctx = StreamTcpReassembleInitThreadCtx(); - AppLayerDetectProtoThreadInit(); - uint8_t httpbuf1[] = "GET /EVILSUFF HTTP/1.1\r\n\r\n"; - uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */ + stream_config.reassembly_inline_window = 16; - inet_pton(AF_INET, "1.2.3.4", &in); - src.family = AF_INET; - src.addr_data32[0] = in.s_addr; - inet_pton(AF_INET, "1.2.3.5", &in); - dst.family = AF_INET; - dst.addr_data32[0] = in.s_addr; - sp = 200; - dp = 220; + uint8_t stream_payload1[] = "ABBBBB"; + uint8_t stream_payload2[] = "DDDDD"; + uint8_t stream_payload3[] = "AAAAABBBBBCCCCCD"; + uint8_t payload[] = { 'C', 'C', 'C', 'C', 'C' }; + Packet *p = UTHBuildPacketReal(payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); + if (p == NULL) { + printf("couldn't get a packet: "); + goto end; + } + p->tcph->th_seq = htonl(12); + p->flow = &f; - ssn.server.ra_raw_base_seq = ssn.server.ra_app_base_seq = 572799781UL; - ssn.server.isn = 572799781UL; - ssn.server.last_ack = 572799782UL; - ssn.client.ra_raw_base_seq = ssn.client.ra_app_base_seq = 4294967289UL; - ssn.client.isn = 4294967289UL; - ssn.client.last_ack = 21; - f.alproto = ALPROTO_UNKNOWN; + if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 2, 'A', 5) == -1) { + printf("failed to add segment 1: "); + goto end; + } + if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 7, 'B', 5) == -1) { + printf("failed to add segment 2: "); + goto end; + } + ssn.client.next_seq = 12; - f.src = src; - f.dst = dst; - f.sp = sp; - f.dp = dp; - f.protoctx = &ssn; - p->flow = &f; - tcph.th_win = htons(5480); - ssn.state = TCP_ESTABLISHED; - TcpStream *s = NULL; - uint8_t cnt = 0; + if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 17, 'D', 5) == -1) { + printf("failed to add segment 4: "); + goto end; + } - for (cnt=0; cnt < httplen1; cnt++) { - tcph.th_seq = htonl(ssn.client.isn + 1 + cnt); - tcph.th_ack = htonl(572799782UL); - tcph.th_flags = TH_ACK|TH_PUSH; - p->tcph = &tcph; - p->flowflags = FLOW_PKT_TOSERVER; - p->payload = &httpbuf1[cnt]; - p->payload_len = 1; - s = &ssn.client; + p->tcph->th_seq = htonl(17); - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver " - "packet\n"); - goto end; - } + int r = StreamTcpReassembleInlineRaw(ra_ctx, &ssn, &ssn.client, p); + if (r < 0) { + printf("StreamTcpReassembleInlineRaw failed: "); + goto end; + } - p->flowflags = FLOW_PKT_TOCLIENT; - p->payload = NULL; - p->payload_len = 0; - tcph.th_seq = htonl(572799782UL); - tcph.th_ack = htonl(ssn.client.isn + 1 + cnt); - tcph.th_flags = TH_ACK; - p->tcph = &tcph; - s = &ssn.server; + if (ra_ctx->stream_q->len != 2) { + printf("expected a single stream message, got %u: ", ra_ctx->stream_q->len); + goto end; + } - if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver " - "packet\n"); - goto end; - } + StreamMsg *smsg = ra_ctx->stream_q->top->next; + if (smsg->data.data_len != 6) { + printf("expected data length to be 6, got %u: ", smsg->data.data_len); + goto end; + } - /* Process stream smsgs we may have in queue */ - if (StreamTcpReassembleProcessAppLayer(ra_ctx) < 0) { - printf("failed in processing stream smsgs\n"); - goto end; - } + if (!(memcmp(stream_payload1, smsg->data.data, 6) == 0)) { + printf("data is not what we expected:\nExpected:\n"); + PrintRawDataFp(stdout, stream_payload1, 6); + printf("Got:\n"); + PrintRawDataFp(stdout, smsg->data.data, smsg->data.data_len); + goto end; } - if (f.alproto != ALPROTO_HTTP) { - printf("App layer protocol (HTTP) should have been detected\n"); + smsg = ra_ctx->stream_q->top; + if (smsg->data.data_len != 5) { + printf("expected data length to be 5, got %u: ", smsg->data.data_len); + goto end; + } + + if (!(memcmp(stream_payload2, smsg->data.data, 5) == 0)) { + printf("data is not what we expected:\nExpected:\n"); + PrintRawDataFp(stdout, stream_payload2, 5); + printf("Got:\n"); + PrintRawDataFp(stdout, smsg->data.data, smsg->data.data_len); + goto end; + } + + if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &ssn.client, 12, 'C', 5) == -1) { + printf("failed to add segment 3: "); + goto end; + } + ssn.client.next_seq = 22; + + p->tcph->th_seq = htonl(12); + + r = StreamTcpReassembleInlineRaw(ra_ctx, &ssn, &ssn.client, p); + if (r < 0) { + printf("StreamTcpReassembleInlineRaw failed: "); + goto end; + } + + if (ra_ctx->stream_q->len != 3) { + printf("expected a single stream message, got %u: ", ra_ctx->stream_q->len); + goto end; + } + + smsg = ra_ctx->stream_q->top; + if (smsg->data.data_len != 16) { + printf("expected data length to be 16, got %u: ", smsg->data.data_len); + goto end; + } + + if (!(memcmp(stream_payload3, smsg->data.data, 16) == 0)) { + printf("data is not what we expected:\nExpected:\n"); + PrintRawDataFp(stdout, stream_payload3, 16); + printf("Got:\n"); + PrintRawDataFp(stdout, smsg->data.data, smsg->data.data_len); goto end; } ret = 1; end: - StreamTcpFreeConfig(TRUE); - StreamTcpReassembleFreeThreadCtx(ra_ctx); - SCFree(p); + FLOW_DESTROY(&f); + UTHFreePacket(p); + StreamTcpUTClearSession(&ssn); + StreamTcpUTDeinit(ra_ctx); return ret; } + #endif /* UNITTESTS */ /** \brief The Function Register the Unit tests to test the reassembly engine @@ -7439,6 +8526,15 @@ void StreamTcpReassembleRegisterTests(void) { UtRegisterTest("StreamTcpReassembleTest46 -- Depth Test", StreamTcpReassembleTest46, 1); UtRegisterTest("StreamTcpReassembleTest47 -- TCP Sequence Wraparound Test", StreamTcpReassembleTest47, 1); + UtRegisterTest("StreamTcpReassembleInlineTest01 -- inline RAW reassembly", StreamTcpReassembleInlineTest01, 1); + UtRegisterTest("StreamTcpReassembleInlineTest02 -- inline RAW reassembly 2", StreamTcpReassembleInlineTest02, 1); + UtRegisterTest("StreamTcpReassembleInlineTest03 -- inline RAW reassembly 3", StreamTcpReassembleInlineTest03, 1); + UtRegisterTest("StreamTcpReassembleInlineTest04 -- inline RAW reassembly 4", StreamTcpReassembleInlineTest04, 1); + UtRegisterTest("StreamTcpReassembleInlineTest05 -- inline RAW reassembly 5 GAP", StreamTcpReassembleInlineTest05, 1); + UtRegisterTest("StreamTcpReassembleInlineTest06 -- inline RAW reassembly 6 GAP", StreamTcpReassembleInlineTest06, 1); + UtRegisterTest("StreamTcpReassembleInlineTest07 -- inline RAW reassembly 7 GAP", StreamTcpReassembleInlineTest07, 1); + StreamTcpInlineRegisterTests(); + StreamTcpUtilRegisterTests(); #endif /* UNITTESTS */ } diff --git a/src/stream-tcp-reassemble.h b/src/stream-tcp-reassemble.h index 438d4abff3..6e6cb70870 100644 --- a/src/stream-tcp-reassemble.h +++ b/src/stream-tcp-reassemble.h @@ -102,5 +102,11 @@ void StreamTcpReassemblePause (TcpSession *, char ); void StreamTcpReassembleUnPause (TcpSession *, char ); int StreamTcpCheckStreamContents(uint8_t *, uint16_t , TcpStream *); +int StreamTcpReassembleInsertSegment(ThreadVars *, TcpReassemblyThreadCtx *, TcpStream *, TcpSegment *, Packet *); +TcpSegment* StreamTcpGetSegment(ThreadVars *, TcpReassemblyThreadCtx *, uint16_t); + +void StreamTcpReturnStreamSegments(TcpStream *); +void StreamTcpSegmentReturntoPool(TcpSegment *); + #endif /* __STREAM_TCP_REASSEMBLE_H__ */ diff --git a/src/stream-tcp-util.c b/src/stream-tcp-util.c new file mode 100644 index 0000000000..0958e01195 --- /dev/null +++ b/src/stream-tcp-util.c @@ -0,0 +1,217 @@ +/* Copyright (C) 2007-2011 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + * + * Helper functions for the stream engine. + */ + +#include "suricata-common.h" + +#include "stream-tcp-reassemble.h" +#include "stream-tcp-inline.h" +#include "stream-tcp.h" +#include "stream-tcp-util.h" + +#include "util-memcmp.h" +#include "util-print.h" + +#include "util-unittest.h" +#include "util-unittest-helper.h" + +#ifdef UNITTESTS + +/* unittest helper functions */ + +void StreamTcpUTInit(TcpReassemblyThreadCtx **ra_ctx) { + StreamTcpInitConfig(TRUE); + *ra_ctx = StreamTcpReassembleInitThreadCtx(); +} + +void StreamTcpUTDeinit(TcpReassemblyThreadCtx *ra_ctx) { + StreamTcpReassembleFreeThreadCtx(ra_ctx); + StreamTcpFreeConfig(TRUE); +} + +void StreamTcpUTSetupSession(TcpSession *ssn) { + memset(ssn, 0x00, sizeof(TcpSession)); +} + +void StreamTcpUTClearSession(TcpSession *ssn) { + StreamTcpUTClearStream(&ssn->client); + StreamTcpUTClearStream(&ssn->server); +} + +void StreamTcpUTSetupStream(TcpStream *s, uint32_t isn) { + memset(s, 0x00, sizeof(TcpStream)); + + s->isn = isn; + STREAMTCP_SET_RA_BASE_SEQ(s, isn); +} + +void StreamTcpUTClearStream(TcpStream *s) { + StreamTcpReturnStreamSegments(s); +} + +int StreamTcpUTAddSegmentWithByte(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpStream *stream, uint32_t seq, uint8_t byte, uint16_t len) { + TcpSegment *s = StreamTcpGetSegment(tv, ra_ctx, len); + if (s == NULL) { + return -1; + } + + s->seq = seq; + s->payload_len = len; + memset(s->payload, byte, len); + + Packet *p = UTHBuildPacketReal(s->payload, 5, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); + if (StreamTcpReassembleInsertSegment(tv, ra_ctx, stream, s, p) < 0) + return -1; + UTHFreePacket(p); + return 0; +} + +/* tests */ + +int StreamTcpUtilTest01(void) { + int ret = 0; + TcpReassemblyThreadCtx *ra_ctx = NULL; + + StreamTcpUTInit(&ra_ctx); + + if (ra_ctx == NULL) { + printf("ra_ctx is NULL: "); + goto end; + } + + ret = 1; +end: + StreamTcpUTDeinit(ra_ctx); + return ret; +} + + +int StreamTcpUtilStreamTest01(void) { + int ret = 0; + TcpReassemblyThreadCtx *ra_ctx = NULL; + ThreadVars tv; + TcpStream stream; + + memset(&tv, 0x00, sizeof(tv)); + + StreamTcpUTInit(&ra_ctx); + StreamTcpUTSetupStream(&stream, 1); + + if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &stream, 2, 'A', 5) == -1) { + printf("failed to add segment 1: "); + goto end; + } + if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &stream, 7, 'B', 5) == -1) { + printf("failed to add segment 2: "); + goto end; + } + if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &stream, 12, 'C', 5) == -1) { + printf("failed to add segment 3: "); + goto end; + } + + TcpSegment *seg = stream.seg_list; + if (seg->seq != 2) { + printf("first seg in the list should have seq 2: "); + goto end; + } + + seg = seg->next; + if (seg->seq != 7) { + printf("first seg in the list should have seq 7: "); + goto end; + } + + seg = seg->next; + if (seg->seq != 12) { + printf("first seg in the list should have seq 12: "); + goto end; + } + + ret = 1; +end: + StreamTcpUTClearStream(&stream); + StreamTcpUTDeinit(ra_ctx); + return ret; +} + +int StreamTcpUtilStreamTest02(void) { + int ret = 0; + TcpReassemblyThreadCtx *ra_ctx = NULL; + ThreadVars tv; + TcpStream stream; + + memset(&tv, 0x00, sizeof(tv)); + + StreamTcpUTInit(&ra_ctx); + StreamTcpUTSetupStream(&stream, 1); + + if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &stream, 7, 'B', 5) == -1) { + printf("failed to add segment 2: "); + goto end; + } + if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &stream, 12, 'C', 5) == -1) { + printf("failed to add segment 3: "); + goto end; + } + if (StreamTcpUTAddSegmentWithByte(&tv, ra_ctx, &stream, 2, 'A', 5) == -1) { + printf("failed to add segment 1: "); + goto end; + } + + TcpSegment *seg = stream.seg_list; + if (seg->seq != 2) { + printf("first seg in the list should have seq 2: "); + goto end; + } + + seg = seg->next; + if (seg->seq != 7) { + printf("first seg in the list should have seq 7: "); + goto end; + } + + seg = seg->next; + if (seg->seq != 12) { + printf("first seg in the list should have seq 12: "); + goto end; + } + + ret = 1; +end: + StreamTcpUTClearStream(&stream); + StreamTcpUTDeinit(ra_ctx); + return ret; +} + +#endif + +void StreamTcpUtilRegisterTests(void) { +#ifdef UNITTESTS + UtRegisterTest("StreamTcpUtilTest01", StreamTcpUtilTest01, 1); + UtRegisterTest("StreamTcpUtilStreamTest01", StreamTcpUtilStreamTest01, 1); + UtRegisterTest("StreamTcpUtilStreamTest02", StreamTcpUtilStreamTest02, 1); +#endif /* UNITTESTS */ +} + diff --git a/src/stream-tcp-util.h b/src/stream-tcp-util.h new file mode 100644 index 0000000000..32dec5c79d --- /dev/null +++ b/src/stream-tcp-util.h @@ -0,0 +1,45 @@ +/* Copyright (C) 2007-2011 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + */ + +#ifndef __STREAM_TCP_UTIL_H__ +#define __STREAM_TCP_UTIL_H__ + +#include "stream-tcp-private.h" + +void StreamTcpUTInit(TcpReassemblyThreadCtx **); +void StreamTcpUTDeinit(TcpReassemblyThreadCtx *); + +void StreamTcpUTSetupSession(TcpSession *); +void StreamTcpUTClearSession(TcpSession *); + +void StreamTcpUTSetupStream(TcpStream *, uint32_t isn); +void StreamTcpUTClearStream(TcpStream *); + +int StreamTcpUTAddSegmentWithByte(ThreadVars *, TcpReassemblyThreadCtx *, TcpStream *, uint32_t, uint8_t, uint16_t); +int StreamTcpUTAddSegmentWithPayload(ThreadVars *, TcpReassemblyThreadCtx *, TcpStream *, uint32_t, uint8_t *, uint16_t); + + +void StreamTcpUtilRegisterTests(void); + +#endif /* __STREAM_TCP_UTIL_H__ */ + diff --git a/src/stream-tcp.c b/src/stream-tcp.c index 5450b5f4fc..f6a79985b4 100644 --- a/src/stream-tcp.c +++ b/src/stream-tcp.c @@ -49,8 +49,8 @@ #include "stream-tcp-private.h" #include "stream-tcp-reassemble.h" #include "stream-tcp.h" +#include "stream-tcp-util.h" #include "stream.h" -#include "stream-tcp.h" #include "app-layer-parser.h" #include "app-layer-protos.h" @@ -64,6 +64,7 @@ #define STREAMTCP_DEFAULT_PREALLOC 32768 #define STREAMTCP_DEFAULT_MEMCAP 32 * 1024 * 1024 /* 32mb */ #define STREAMTCP_DEFAULT_REASSEMBLY_MEMCAP 64 * 1024 * 1024 /* 64mb */ +#define STREAMTCP_DEFAULT_REASSEMBLY_WINDOW 3000 #define STREAMTCP_NEW_TIMEOUT 60 #define STREAMTCP_EST_TIMEOUT 3600 @@ -98,7 +99,6 @@ static int StreamTcpHandleFin(ThreadVars *tv, StreamTcpThread *, TcpSession *, P void StreamTcpRegisterTests (void); void StreamTcpReturnStreamSegments (TcpStream *); void StreamTcpInitConfig(char); -extern void StreamTcpSegmentReturntoPool(TcpSegment *); int StreamTcpGetFlowState(void *); void StreamTcpSetOSPolicy(TcpStream*, Packet*); void StreamTcpPseudoPacketCreateStreamEndPacket(Packet *, TcpSession *, PacketQueue *); @@ -169,24 +169,6 @@ int StreamTcpCheckMemcap(uint32_t size) { SCReturnInt(ret); } -void StreamTcpReturnStreamSegments (TcpStream *stream) -{ - TcpSegment *seg = stream->seg_list; - TcpSegment *next_seg; - - if (seg == NULL) - return; - - while (seg != NULL) { - next_seg = seg->next; - StreamTcpSegmentReturntoPool(seg); - seg = next_seg; - } - - stream->seg_list = NULL; - stream->seg_list_tail = NULL; -} - /** \brief Function to return the stream back to the pool. It returns the * segments in the stream to the segment pool. * @@ -438,6 +420,9 @@ void StreamTcpInitConfig(char quiet) SCLogInfo("stream.\"inline\": %s", stream_inline ? "enabled" : "disabled"); } + /** \todo yaml part */ + stream_config.reassembly_inline_window = STREAMTCP_DEFAULT_REASSEMBLY_WINDOW; + /* init the memcap and it's lock */ stream_memuse = 0; stream_memuse_max = 0; diff --git a/src/stream-tcp.h b/src/stream-tcp.h index cbe430386f..8e723fd492 100644 --- a/src/stream-tcp.h +++ b/src/stream-tcp.h @@ -41,13 +41,24 @@ /*global flow data*/ typedef struct TcpStreamCnf_ { - uint32_t memcap; /** max stream mem usage */ + /** stream tracking + * + * max stream mem usage + */ + uint32_t memcap; + uint32_t max_sessions; uint32_t prealloc_sessions; int midstream; int async_oneside; uint32_t reassembly_memcap; /**< max memory usage for stream reassembly */ uint32_t reassembly_depth; /**< Depth until when we reassemble the stream */ + + /** reassembly -- inline mode + * + * sliding window size for raw stream reassembly + */ + uint32_t reassembly_inline_window; uint8_t flags; } TcpStreamCnf;