diff --git a/src/Makefile.am b/src/Makefile.am index bcc2650b44..edc3a3803f 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -301,6 +301,9 @@ app-layer-ssh.c app-layer-ssh.h \ app-layer-tls-handshake.c app-layer-tls-handshake.h \ app-layer-smtp.c app-layer-smtp.h \ defrag.c defrag.h \ +defrag-hash.c defrag-hash.h \ +defrag-queue.c defrag-queue.h \ +defrag-timeout.c defrag-timeout.h \ output.c output.h \ win32-misc.c win32-misc.h \ win32-service.c win32-service.h \ diff --git a/src/decode-ipv4.c b/src/decode-ipv4.c index 199cc758b5..f96b09fc04 100644 --- a/src/decode-ipv4.c +++ b/src/decode-ipv4.c @@ -527,7 +527,7 @@ void DecodeIPV4(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p, uint8_t *pkt, /* If a fragment, pass off for re-assembly. */ if (unlikely(IPV4_GET_IPOFFSET(p) > 0 || IPV4_GET_MF(p) == 1)) { - Packet *rp = Defrag(tv, dtv, NULL, p); + Packet *rp = Defrag(tv, dtv, p); if (rp != NULL) { /* Got re-assembled packet, re-run through decoder. */ DecodeIPV4(tv, dtv, rp, (void *)rp->ip4h, IPV4_GET_IPLEN(rp), pq); @@ -1558,6 +1558,7 @@ int DecodeIPV4DefragTest01(void) PACKET_INITIALIZE(p); FlowInitConfig(FLOW_QUIET); + DefragInit(); PacketCopyData(p, pkt1, sizeof(pkt1)); DecodeIPV4(&tv, &dtv, p, GET_PKT_DATA(p) + ETHERNET_HEADER_LEN, @@ -1622,6 +1623,7 @@ int DecodeIPV4DefragTest01(void) SCFree(tp); end: + DefragDestroy(); FlowShutdown(); PACKET_CLEANUP(p); SCFree(p); @@ -1686,7 +1688,7 @@ int DecodeIPV4DefragTest02(void) ThreadVars tv; DecodeThreadVars dtv; PacketQueue pq; - int result = 1; + int result = 0; memset(&tv, 0, sizeof(ThreadVars)); memset(&dtv, 0, sizeof(DecodeThreadVars)); @@ -1694,13 +1696,13 @@ int DecodeIPV4DefragTest02(void) PACKET_INITIALIZE(p); FlowInitConfig(FLOW_QUIET); + DefragInit(); PacketCopyData(p, pkt1, sizeof(pkt1)); DecodeIPV4(&tv, &dtv, p, GET_PKT_DATA(p) + ETHERNET_HEADER_LEN, GET_PKT_LEN(p) - ETHERNET_HEADER_LEN, &pq); if (p->tcph != NULL) { printf("tcp header should be NULL for ip fragment, but it isn't\n"); - result = 0; goto end; } PACKET_DO_RECYCLE(p); @@ -1710,7 +1712,6 @@ int DecodeIPV4DefragTest02(void) GET_PKT_LEN(p) - ETHERNET_HEADER_LEN, &pq); if (p->tcph != NULL) { printf("tcp header should be NULL for ip fragment, but it isn't\n"); - result = 0; goto end; } PACKET_DO_RECYCLE(p); @@ -1721,45 +1722,41 @@ int DecodeIPV4DefragTest02(void) GET_PKT_LEN(p) - ETHERNET_HEADER_LEN, &pq); if (p->tcph != NULL) { printf("tcp header should be NULL for ip fragment, but it isn't\n"); - result = 0; goto end; } Packet *tp = PacketDequeue(&pq); if (tp == NULL) { printf("Failed to get defragged pseudo packet\n"); - result = 0; goto end; } if (tp->recursion_level != p->recursion_level) { printf("defragged pseudo packet's and parent packet's recursion " - "level don't match\n %d != %d", + "level don't match %d != %d: ", tp->recursion_level, p->recursion_level); - result = 0; goto end; } if (tp->ip4h == NULL || tp->tcph == NULL) { printf("pseudo packet's ip header and tcp header shouldn't be NULL, " "but it is\n"); - result = 0; goto end; } if (GET_PKT_LEN(tp) != sizeof(tunnel_pkt)) { printf("defragged pseudo packet's and parent packet's pkt lens " - "don't match\n %u != %"PRIuMAX, + "don't match %u != %"PRIuMAX": ", GET_PKT_LEN(tp), (uintmax_t)sizeof(tunnel_pkt)); - result = 0; goto end; } if (memcmp(GET_PKT_DATA(tp), tunnel_pkt, sizeof(tunnel_pkt)) != 0) { - result = 0; - goto end; + goto end; } + result = 1; PACKET_CLEANUP(p); SCFree(tp); end: + DefragDestroy(); FlowShutdown(); PACKET_CLEANUP(p); SCFree(p); @@ -1828,6 +1825,7 @@ int DecodeIPV4DefragTest03(void) PACKET_INITIALIZE(p); FlowInitConfig(FLOW_QUIET); + DefragInit(); PacketCopyData(p, pkt, sizeof(pkt)); DecodeIPV4(&tv, &dtv, p, GET_PKT_DATA(p) + ETHERNET_HEADER_LEN, @@ -1918,6 +1916,7 @@ int DecodeIPV4DefragTest03(void) SCFree(tp); end: + DefragDestroy(); FlowShutdown(); PACKET_CLEANUP(p); SCFree(p); diff --git a/src/decode-ipv6.c b/src/decode-ipv6.c index 03f5c99c5e..a3bebb7689 100644 --- a/src/decode-ipv6.c +++ b/src/decode-ipv6.c @@ -564,7 +564,7 @@ void DecodeIPV6(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p, uint8_t *pkt, /* Pass to defragger if a fragment. */ if (IPV6_EXTHDR_ISSET_FH(p)) { - Packet *rp = Defrag(tv, dtv, NULL, p); + Packet *rp = Defrag(tv, dtv, p); if (rp != NULL) { DecodeIPV6(tv, dtv, rp, (uint8_t *)rp->ip6h, IPV6_GET_PLEN(rp) + IPV6_HEADER_LEN, pq); PacketEnqueue(pq, rp); @@ -725,6 +725,7 @@ static int DecodeIPV6FragTest01 (void) { PacketQueue pq; FlowInitConfig(FLOW_QUIET); + DefragInit(); memset(&pq, 0, sizeof(PacketQueue)); memset(&tv, 0, sizeof(ThreadVars)); @@ -765,6 +766,7 @@ end: PACKET_CLEANUP(p2); SCFree(p1); SCFree(p2); + DefragDestroy(); FlowShutdown(); return result; } diff --git a/src/defrag-hash.c b/src/defrag-hash.c new file mode 100644 index 0000000000..e7dc32b2ff --- /dev/null +++ b/src/defrag-hash.c @@ -0,0 +1,699 @@ +/* Copyright (C) 2007-2012 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +#include "suricata-common.h" +#include "conf.h" +#include "defrag-hash.h" +#include "defrag-queue.h" +#include "util-random.h" +#include "util-byte.h" +#include "util-misc.h" +#include "util-hash-lookup3.h" + +static DefragTracker *DefragTrackerGetUsedDefragTracker(void); + +/** queue with spare tracker */ +static DefragTrackerQueue defragtracker_spare_q; + +uint32_t DefragTrackerSpareQueueGetSize(void) { + return DefragTrackerQueueLen(&defragtracker_spare_q); +} + +void DefragTrackerMoveToSpare(DefragTracker *h) { + DefragTrackerEnqueue(&defragtracker_spare_q, h); + (void) SC_ATOMIC_SUB(defragtracker_counter, 1); +} + +DefragTracker *DefragTrackerAlloc(void) { + if (!(DEFRAG_CHECK_MEMCAP(sizeof(DefragTracker)))) { + return NULL; + } + + (void) SC_ATOMIC_ADD(defrag_memuse, sizeof(DefragTracker)); + + DefragTracker *dt = SCMalloc(sizeof(DefragTracker)); + if (dt == NULL) + goto error; + + memset(dt, 0x00, sizeof(DefragTracker)); + + SCMutexInit(&dt->lock, NULL); + SC_ATOMIC_INIT(dt->use_cnt); + return dt; + +error: + return NULL; +} + +void DefragTrackerFree(DefragTracker *dt) { + if (dt != NULL) { + DefragTrackerClearMemory(dt); + + SCMutexDestroy(&dt->lock); + SCFree(dt); + (void) SC_ATOMIC_SUB(defrag_memuse, sizeof(DefragTracker)); + } +} + +#define DefragTrackerIncrUsecnt(dt) \ + SC_ATOMIC_ADD((dt)->use_cnt, 1) +#define DefragTrackerDecrUsecnt(dt) \ + SC_ATOMIC_SUB((dt)->use_cnt, 1) + +static void DefragTrackerInit(DefragTracker *dt, Packet *p) { + /* copy address */ + COPY_ADDRESS(&p->src, &dt->src_addr); + COPY_ADDRESS(&p->dst, &dt->dst_addr); + + if (PKT_IS_IPV4(p)) { + dt->id = (int32_t)IPV4_GET_IPID(p); + dt->af = AF_INET; + } else { + dt->id = (int32_t)IPV6_EXTHDR_GET_FH_ID(p); + dt->af = AF_INET6; + } + dt->policy = DefragGetOsPolicy(p); + TAILQ_INIT(&dt->frags); + (void) DefragTrackerIncrUsecnt(dt); +} + +static DefragTracker *DefragTrackerNew(Packet *p) { + DefragTracker *dt = DefragTrackerAlloc(); + if (dt == NULL) + goto error; + + DefragTrackerInit(dt, p); + return dt; + +error: + return NULL; +} + +void DefragTrackerRelease(DefragTracker *t) { + (void) DefragTrackerDecrUsecnt(t); + SCMutexUnlock(&t->lock); +} + +void DefragTrackerClearMemory(DefragTracker *dt) { + DefragTrackerFreeFrags(dt); + SC_ATOMIC_DESTROY(dt->use_cnt); +} + +#define DEFRAG_DEFAULT_HASHSIZE 4096 +#define DEFRAG_DEFAULT_MEMCAP 16777216 +#define DEFRAG_DEFAULT_PREALLOC 1000 + +/** \brief initialize the configuration + * \warning Not thread safe */ +void DefragInitConfig(char quiet) +{ + SCLogDebug("initializing defrag engine..."); + + memset(&defrag_config, 0, sizeof(defrag_config)); + //SC_ATOMIC_INIT(flow_flags); + SC_ATOMIC_INIT(defragtracker_counter); + SC_ATOMIC_INIT(defrag_memuse); + SC_ATOMIC_INIT(defragtracker_prune_idx); + DefragTrackerQueueInit(&defragtracker_spare_q); + + unsigned int seed = RandomTimePreseed(); + /* set defaults */ + defrag_config.hash_rand = (int)(DEFRAG_DEFAULT_HASHSIZE * (rand_r(&seed) / RAND_MAX + 1.0)); + + defrag_config.hash_size = DEFRAG_DEFAULT_HASHSIZE; + defrag_config.memcap = DEFRAG_DEFAULT_MEMCAP; + defrag_config.prealloc = DEFRAG_DEFAULT_PREALLOC; + + /* Check if we have memcap and hash_size defined at config */ + char *conf_val; + uint32_t configval = 0; + + /** set config values for memcap, prealloc and hash_size */ + if ((ConfGet("defrag.memcap", &conf_val)) == 1) + { + if (ParseSizeStringU64(conf_val, &defrag_config.memcap) < 0) { + SCLogError(SC_ERR_SIZE_PARSE, "Error parsing defrag.memcap " + "from conf file - %s. Killing engine", + conf_val); + exit(EXIT_FAILURE); + } + } + if ((ConfGet("defrag.hash-size", &conf_val)) == 1) + { + if (ByteExtractStringUint32(&configval, 10, strlen(conf_val), + conf_val) > 0) { + defrag_config.hash_size = configval; + } + } + + + if ((ConfGet("defrag.trackers", &conf_val)) == 1) + { + if (ByteExtractStringUint32(&configval, 10, strlen(conf_val), + conf_val) > 0) { + defrag_config.prealloc = configval; + } + } + SCLogDebug("DefragTracker config from suricata.yaml: memcap: %"PRIu64", hash-size: " + "%"PRIu32", prealloc: %"PRIu32, defrag_config.memcap, + defrag_config.hash_size, defrag_config.prealloc); + + /* alloc hash memory */ + uint64_t hash_size = defrag_config.hash_size * sizeof(DefragTrackerHashRow); + if (!(DEFRAG_CHECK_MEMCAP(hash_size))) { + SCLogError(SC_ERR_DEFRAG_INIT, "allocating defrag hash failed: " + "max defrag memcap is smaller than projected hash size. " + "Memcap: %"PRIu64", Hash table size %"PRIu64". Calculate " + "total hash size by multiplying \"defrag.hash-size\" with %"PRIuMAX", " + "which is the hash bucket size.", defrag_config.memcap, hash_size, + (uintmax_t)sizeof(DefragTrackerHashRow)); + exit(EXIT_FAILURE); + } + defragtracker_hash = SCCalloc(defrag_config.hash_size, sizeof(DefragTrackerHashRow)); + if (defragtracker_hash == NULL) { + SCLogError(SC_ERR_FATAL, "Fatal error encountered in DefragTrackerInitConfig. Exiting..."); + exit(EXIT_FAILURE); + } + memset(defragtracker_hash, 0, defrag_config.hash_size * sizeof(DefragTrackerHashRow)); + + uint32_t i = 0; + for (i = 0; i < defrag_config.hash_size; i++) { + DRLOCK_INIT(&defragtracker_hash[i]); + } + (void) SC_ATOMIC_ADD(defrag_memuse, (defrag_config.hash_size * sizeof(DefragTrackerHashRow))); + + if (quiet == FALSE) { + SCLogInfo("allocated %llu bytes of memory for the defrag hash... " + "%" PRIu32 " buckets of size %" PRIuMAX "", + SC_ATOMIC_GET(defrag_memuse), defrag_config.hash_size, + (uintmax_t)sizeof(DefragTrackerHashRow)); + } + + if ((ConfGet("defrag.prealloc", &conf_val)) == 1) + { + if (ConfValIsTrue(conf_val)) { + /* pre allocate defrag trackers */ + for (i = 0; i < defrag_config.prealloc; i++) { + if (!(DEFRAG_CHECK_MEMCAP(sizeof(DefragTracker)))) { + SCLogError(SC_ERR_DEFRAG_INIT, "preallocating defrag trackers failed: " + "max defrag memcap reached. Memcap %"PRIu64", " + "Memuse %"PRIu64".", defrag_config.memcap, + ((uint64_t)SC_ATOMIC_GET(defrag_memuse) + (uint64_t)sizeof(DefragTracker))); + exit(EXIT_FAILURE); + } + + DefragTracker *h = DefragTrackerAlloc(); + if (h == NULL) { + SCLogError(SC_ERR_DEFRAG_INIT, "preallocating defrag failed: %s", strerror(errno)); + exit(EXIT_FAILURE); + } + DefragTrackerEnqueue(&defragtracker_spare_q,h); + } + if (quiet == FALSE) { + SCLogInfo("preallocated %" PRIu32 " defrag trackers of size %" PRIuMAX "", + defragtracker_spare_q.len, (uintmax_t)sizeof(DefragTracker)); + } + } + } + + if (quiet == FALSE) { + SCLogInfo("defrag memory usage: %llu bytes, maximum: %"PRIu64, + SC_ATOMIC_GET(defrag_memuse), defrag_config.memcap); + } + + return; +} + +/** \brief print some defrag stats + * \warning Not thread safe */ +static void DefragTrackerPrintStats (void) +{ +} + +/** \brief shutdown the flow engine + * \warning Not thread safe */ +void DefragHashShutdown(void) +{ + DefragTracker *dt; + uint32_t u; + + DefragTrackerPrintStats(); + + /* free spare queue */ + while((dt = DefragTrackerDequeue(&defragtracker_spare_q))) { + BUG_ON(SC_ATOMIC_GET(dt->use_cnt) > 0); + DefragTrackerFree(dt); + } + + /* clear and free the hash */ + if (defragtracker_hash != NULL) { + for (u = 0; u < defrag_config.hash_size; u++) { + dt = defragtracker_hash[u].head; + while (dt) { + DefragTracker *n = dt->hnext; + DefragTrackerClearMemory(dt); + DefragTrackerFree(dt); + dt = n; + } + + DRLOCK_DESTROY(&defragtracker_hash[u]); + } + SCFree(defragtracker_hash); + defragtracker_hash = NULL; + } + (void) SC_ATOMIC_SUB(defrag_memuse, defrag_config.hash_size * sizeof(DefragTrackerHashRow)); + DefragTrackerQueueDestroy(&defragtracker_spare_q); + + SC_ATOMIC_DESTROY(defragtracker_prune_idx); + SC_ATOMIC_DESTROY(defrag_memuse); + SC_ATOMIC_DESTROY(defragtracker_counter); + //SC_ATOMIC_DESTROY(flow_flags); + return; +} + +/** \brief compare two raw ipv6 addrs + * + * \note we don't care about the real ipv6 ip's, this is just + * to consistently fill the DefragHashKey6 struct, without all + * the ntohl calls. + * + * \warning do not use elsewhere unless you know what you're doing. + * detect-engine-address-ipv6.c's AddressIPv6GtU32 is likely + * what you are looking for. + */ +static inline int DefragHashRawAddressIPv6GtU32(uint32_t *a, uint32_t *b) +{ + int i; + + for (i = 0; i < 4; i++) { + if (a[i] > b[i]) + return 1; + if (a[i] < b[i]) + break; + } + + return 0; +} + +typedef struct DefragHashKey4_ { + union { + struct { + uint32_t src, dst; + uint32_t id; + }; + uint32_t u32[3]; + }; +} DefragHashKey4; + +typedef struct DefragHashKey6_ { + union { + struct { + uint32_t src[4], dst[4]; + uint32_t id; + }; + uint32_t u32[9]; + }; +} DefragHashKey6; + +/* calculate the hash key for this packet + * + * we're using: + * hash_rand -- set at init time + * source address + * destination address + * id + */ +static inline uint32_t DefragHashGetKey(Packet *p) { + uint32_t key; + + if (p->ip4h != NULL) { + DefragHashKey4 dhk; + if (p->src.addr_data32[0] > p->dst.addr_data32[0]) { + dhk.src = p->src.addr_data32[0]; + dhk.dst = p->dst.addr_data32[0]; + } else { + dhk.src = p->dst.addr_data32[0]; + dhk.dst = p->src.addr_data32[0]; + } + dhk.id = (uint32_t)IPV4_GET_IPID(p); + + uint32_t hash = hashword(dhk.u32, 3, defrag_config.hash_rand); + key = hash % defrag_config.hash_size; + } else if (p->ip6h != NULL) { + DefragHashKey6 dhk; + if (DefragHashRawAddressIPv6GtU32(p->src.addr_data32, p->dst.addr_data32)) { + dhk.src[0] = p->src.addr_data32[0]; + dhk.src[1] = p->src.addr_data32[1]; + dhk.src[2] = p->src.addr_data32[2]; + dhk.src[3] = p->src.addr_data32[3]; + dhk.dst[0] = p->dst.addr_data32[0]; + dhk.dst[1] = p->dst.addr_data32[1]; + dhk.dst[2] = p->dst.addr_data32[2]; + dhk.dst[3] = p->dst.addr_data32[3]; + } else { + dhk.src[0] = p->dst.addr_data32[0]; + dhk.src[1] = p->dst.addr_data32[1]; + dhk.src[2] = p->dst.addr_data32[2]; + dhk.src[3] = p->dst.addr_data32[3]; + dhk.dst[0] = p->src.addr_data32[0]; + dhk.dst[1] = p->src.addr_data32[1]; + dhk.dst[2] = p->src.addr_data32[2]; + dhk.dst[3] = p->src.addr_data32[3]; + } + dhk.id = IPV6_EXTHDR_GET_FH_ID(p); + + uint32_t hash = hashword(dhk.u32, 9, defrag_config.hash_rand); + key = hash % defrag_config.hash_size; + } else + key = 0; + + return key; +} + +/* Since two or more trackers can have the same hash key, we need to compare + * the tracker with the current tracker key. */ +#define CMP_DEFRAGTRACKER(d1,d2,id) \ + (((CMP_ADDR(&(d1)->src_addr, &(d2)->src) && \ + CMP_ADDR(&(d1)->dst_addr, &(d2)->dst)) || \ + (CMP_ADDR(&(d1)->src_addr, &(d2)->dst) && \ + CMP_ADDR(&(d1)->dst_addr, &(d2)->src))) && \ + (d1)->id == (id)) + +static inline int DefragTrackerCompare(DefragTracker *t, Packet *p) { + uint32_t id; + if (PKT_IS_IPV4(p)) { + id = (uint32_t)IPV4_GET_IPID(p); + } else { + id = IPV6_EXTHDR_GET_FH_ID(p); + } + + return CMP_DEFRAGTRACKER(t, p, id); +} + +/** + * \brief Get a new defrag tracker + * + * Get a new defrag tracker. We're checking memcap first and will try to make room + * if the memcap is reached. + * + * \retval dt *LOCKED* tracker on succes, NULL on error. + */ +static DefragTracker *DefragTrackerGetNew(Packet *p) { + DefragTracker *dt = NULL; + + /* get a tracker from the spare queue */ + dt = DefragTrackerDequeue(&defragtracker_spare_q); + if (dt == NULL) { + /* If we reached the max memcap, we get a used tracker */ + if (!(DEFRAG_CHECK_MEMCAP(sizeof(DefragTracker)))) { + /* declare state of emergency */ + //if (!(SC_ATOMIC_GET(defragtracker_flags) & DEFRAG_EMERGENCY)) { + // SC_ATOMIC_OR(defragtracker_flags, DEFRAG_EMERGENCY); + + /* under high load, waking up the flow mgr each time leads + * to high cpu usage. Flows are not timed out much faster if + * we check a 1000 times a second. */ + // FlowWakeupFlowManagerThread(); + //} + + dt = DefragTrackerGetUsedDefragTracker(); + if (dt == NULL) { + return NULL; + } + + /* freed a tracker, but it's unlocked */ + } else { + /* now see if we can alloc a new tracker */ + dt = DefragTrackerNew(p); + if (dt == NULL) { + return NULL; + } + + /* tracker is initialized but *unlocked* */ + } + } else { + /* tracker has been recycled before it went into the spare queue */ + + /* tracker is initialized (recylced) but *unlocked* */ + } + + (void) SC_ATOMIC_ADD(defragtracker_counter, 1); + SCMutexLock(&dt->lock); + return dt; +} + +/* DefragGetTrackerFromHash + * + * Hash retrieval function for trackers. Looks up the hash bucket containing the + * tracker pointer. Then compares the packet with the found tracker to see if it is + * the tracker we need. If it isn't, walk the list until the right tracker is found. + * + * returns a *LOCKED* tracker or NULL + */ +DefragTracker *DefragGetTrackerFromHash (Packet *p) +{ + DefragTracker *dt = NULL; + + /* get the key to our bucket */ + uint32_t key = DefragHashGetKey(p); + /* get our hash bucket and lock it */ + DefragTrackerHashRow *hb = &defragtracker_hash[key]; + DRLOCK_LOCK(hb); + + /* see if the bucket already has a tracker */ + if (hb->head == NULL) { + dt = DefragTrackerGetNew(p); + if (dt == NULL) { + DRLOCK_UNLOCK(hb); + return NULL; + } + + /* tracker is locked */ + hb->head = dt; + hb->tail = dt; + + /* got one, now lock, initialize and return */ + DefragTrackerInit(dt,p); + + DRLOCK_UNLOCK(hb); + return dt; + } + + /* ok, we have a tracker in the bucket. Let's find out if it is our tracker */ + dt = hb->head; + + /* see if this is the tracker we are looking for */ + if (DefragTrackerCompare(dt, p) == 0) { + DefragTracker *pdt = NULL; /* previous tracker */ + + while (dt) { + pdt = dt; + dt = dt->hnext; + + if (dt == NULL) { + dt = pdt->hnext = DefragTrackerGetNew(p); + if (dt == NULL) { + DRLOCK_UNLOCK(hb); + return NULL; + } + hb->tail = dt; + + /* tracker is locked */ + + dt->hprev = pdt; + + /* initialize and return */ + DefragTrackerInit(dt,p); + + DRLOCK_UNLOCK(hb); + return dt; + } + + if (DefragTrackerCompare(dt, p) != 0) { + /* we found our tracker, lets put it on top of the + * hash list -- this rewards active trackers */ + if (dt->hnext) { + dt->hnext->hprev = dt->hprev; + } + if (dt->hprev) { + dt->hprev->hnext = dt->hnext; + } + if (dt == hb->tail) { + hb->tail = dt->hprev; + } + + dt->hnext = hb->head; + dt->hprev = NULL; + hb->head->hprev = dt; + hb->head = dt; + + /* found our tracker, lock & return */ + SCMutexLock(&dt->lock); + (void) DefragTrackerIncrUsecnt(dt); + DRLOCK_UNLOCK(hb); + return dt; + } + } + } + + /* lock & return */ + SCMutexLock(&dt->lock); + (void) DefragTrackerIncrUsecnt(dt); + DRLOCK_UNLOCK(hb); + return dt; +} + +/** \brief look up a tracker in the hash + * + * \param a address to look up + * + * \retval h *LOCKED* tracker or NULL + */ +DefragTracker *DefragLookupTrackerFromHash (Packet *p) +{ + DefragTracker *dt = NULL; + + /* get the key to our bucket */ + uint32_t key = DefragHashGetKey(p); + /* get our hash bucket and lock it */ + DefragTrackerHashRow *hb = &defragtracker_hash[key]; + DRLOCK_LOCK(hb); + + /* see if the bucket already has a tracker */ + if (hb->head == NULL) { + DRLOCK_UNLOCK(hb); + return dt; + } + + /* ok, we have a tracker in the bucket. Let's find out if it is our tracker */ + dt = hb->head; + + /* see if this is the tracker we are looking for */ + if (DefragTrackerCompare(dt, p) == 0) { + while (dt) { + dt = dt->hnext; + + if (dt == NULL) { + DRLOCK_UNLOCK(hb); + return dt; + } + + if (DefragTrackerCompare(dt, p) != 0) { + /* we found our tracker, lets put it on top of the + * hash list -- this rewards active tracker */ + if (dt->hnext) { + dt->hnext->hprev = dt->hprev; + } + if (dt->hprev) { + dt->hprev->hnext = dt->hnext; + } + if (dt == hb->tail) { + hb->tail = dt->hprev; + } + + dt->hnext = hb->head; + dt->hprev = NULL; + hb->head->hprev = dt; + hb->head = dt; + + /* found our tracker, lock & return */ + SCMutexLock(&dt->lock); + (void) DefragTrackerIncrUsecnt(dt); + DRLOCK_UNLOCK(hb); + return dt; + } + } + } + + /* lock & return */ + SCMutexLock(&dt->lock); + (void) DefragTrackerIncrUsecnt(dt); + DRLOCK_UNLOCK(hb); + return dt; +} + +/** \internal + * \brief Get a tracker from the hash directly. + * + * Called in conditions where the spare queue is empty and memcap is reached. + * + * Walks the hash until a tracker can be freed. "defragtracker_prune_idx" atomic int makes + * sure we don't start at the top each time since that would clear the top of + * the hash leading to longer and longer search times under high pressure (observed). + * + * \retval dt tracker or NULL + */ +static DefragTracker *DefragTrackerGetUsedDefragTracker(void) { + uint32_t idx = SC_ATOMIC_GET(defragtracker_prune_idx) % defrag_config.hash_size; + uint32_t cnt = defrag_config.hash_size; + + while (cnt--) { + if (idx++ >= defrag_config.hash_size) + idx = 0; + + DefragTrackerHashRow *hb = &defragtracker_hash[idx]; + if (hb == NULL) + continue; + + if (DRLOCK_TRYLOCK(hb) != 0) + continue; + + DefragTracker *dt = hb->tail; + if (dt == NULL) { + DRLOCK_UNLOCK(hb); + continue; + } + + if (SCMutexTrylock(&dt->lock) != 0) { + DRLOCK_UNLOCK(hb); + continue; + } + + /** never prune a tracker that is used by a packets + * we are currently processing in one of the threads */ + if (SC_ATOMIC_GET(dt->use_cnt) > 0) { + DRLOCK_UNLOCK(hb); + SCMutexUnlock(&dt->lock); + continue; + } + + /* remove from the hash */ + if (dt->hprev != NULL) + dt->hprev->hnext = dt->hnext; + if (dt->hnext != NULL) + dt->hnext->hprev = dt->hprev; + if (hb->head == dt) + hb->head = dt->hnext; + if (hb->tail == dt) + hb->tail = dt->hprev; + + dt->hnext = NULL; + dt->hprev = NULL; + DRLOCK_UNLOCK(hb); + + DefragTrackerClearMemory(dt); + + SCMutexUnlock(&dt->lock); + + (void) SC_ATOMIC_ADD(defragtracker_prune_idx, (defrag_config.hash_size - cnt)); + return dt; + } + + return NULL; +} + + diff --git a/src/defrag-hash.h b/src/defrag-hash.h new file mode 100644 index 0000000000..2d48393a77 --- /dev/null +++ b/src/defrag-hash.h @@ -0,0 +1,103 @@ +/* Copyright (C) 2007-2012 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + */ + +#ifndef __DEFRAG_HASH_H__ +#define __DEFRAG_HASH_H__ + +#include "decode.h" +#include "defrag.h" + +/** Spinlocks or Mutex for the flow buckets. */ +//#define DRLOCK_SPIN +#define DRLOCK_MUTEX + +#ifdef DRLOCK_SPIN + #ifdef DRLOCK_MUTEX + #error Cannot enable both DRLOCK_SPIN and DRLOCK_MUTEX + #endif +#endif + +#ifdef DRLOCK_SPIN + #define DRLOCK_TYPE SCSpinlock + #define DRLOCK_INIT(fb) SCSpinInit(&(fb)->lock, 0) + #define DRLOCK_DESTROY(fb) SCSpinDestroy(&(fb)->lock) + #define DRLOCK_LOCK(fb) SCSpinLock(&(fb)->lock) + #define DRLOCK_TRYLOCK(fb) SCSpinTrylock(&(fb)->lock) + #define DRLOCK_UNLOCK(fb) SCSpinUnlock(&(fb)->lock) +#elif defined DRLOCK_MUTEX + #define DRLOCK_TYPE SCMutex + #define DRLOCK_INIT(fb) SCMutexInit(&(fb)->lock, NULL) + #define DRLOCK_DESTROY(fb) SCMutexDestroy(&(fb)->lock) + #define DRLOCK_LOCK(fb) SCMutexLock(&(fb)->lock) + #define DRLOCK_TRYLOCK(fb) SCMutexTrylock(&(fb)->lock) + #define DRLOCK_UNLOCK(fb) SCMutexUnlock(&(fb)->lock) +#else + #error Enable DRLOCK_SPIN or DRLOCK_MUTEX +#endif + +typedef struct DefragTrackerHashRow_ { + DRLOCK_TYPE lock; + DefragTracker *head; + DefragTracker *tail; +} DefragTrackerHashRow; + +/** defrag tracker hash table */ +DefragTrackerHashRow *defragtracker_hash; + +#define DEFRAG_VERBOSE 0 +#define DEFRAG_QUIET 1 + +typedef struct DefragConfig_ { + uint64_t memcap; + uint32_t hash_rand; + uint32_t hash_size; + uint32_t prealloc; +} DefragConfig; + +/** \brief check if a memory alloc would fit in the memcap + * + * \param size memory allocation size to check + * + * \retval 1 it fits + * \retval 0 no fit + */ +#define DEFRAG_CHECK_MEMCAP(size) \ + ((((uint64_t)SC_ATOMIC_GET(defrag_memuse) + (uint64_t)(size)) <= defrag_config.memcap)) + +DefragConfig defrag_config; +SC_ATOMIC_DECLARE(unsigned long long int,defrag_memuse); +SC_ATOMIC_DECLARE(unsigned int,defragtracker_counter); +SC_ATOMIC_DECLARE(unsigned int,defragtracker_prune_idx); + +void DefragInitConfig(char quiet); +void DefragHashShutdown(void); + +DefragTracker *DefragLookupTrackerFromHash (Packet *); +DefragTracker *DefragGetTrackerFromHash (Packet *); +void DefragTrackerRelease(DefragTracker *); +void DefragTrackerClearMemory(DefragTracker *); +void DefragTrackerMoveToSpare(DefragTracker *); +uint32_t DefragTrackerSpareQueueGetSize(void); + +#endif /* __DEFRAG_HASH_H__ */ + diff --git a/src/defrag-queue.c b/src/defrag-queue.c new file mode 100644 index 0000000000..9862e90889 --- /dev/null +++ b/src/defrag-queue.c @@ -0,0 +1,138 @@ +/* Copyright (C) 2007-2012 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + * + * Defrag tracker queue handler functions + */ + +#include "suricata-common.h" +#include "threads.h" +#include "debug.h" +#include "defrag-queue.h" +#include "util-error.h" +#include "util-debug.h" +#include "util-print.h" + +DefragTrackerQueue *DefragTrackerQueueInit (DefragTrackerQueue *q) { + if (q != NULL) { + memset(q, 0, sizeof(DefragTrackerQueue)); + DQLOCK_INIT(q); + } + return q; +} + +DefragTrackerQueue *DefragTrackerQueueNew() { + DefragTrackerQueue *q = (DefragTrackerQueue *)SCMalloc(sizeof(DefragTrackerQueue)); + if (q == NULL) { + SCLogError(SC_ERR_FATAL, "Fatal error encountered in DefragTrackerQueueNew. Exiting..."); + exit(EXIT_SUCCESS); + } + q = DefragTrackerQueueInit(q); + return q; +} + +/** + * \brief Destroy a tracker queue + * + * \param q the tracker queue to destroy + */ +void DefragTrackerQueueDestroy (DefragTrackerQueue *q) { + DQLOCK_DESTROY(q); +} + +/** + * \brief add a tracker to a queue + * + * \param q queue + * \param dt tracker + */ +void DefragTrackerEnqueue (DefragTrackerQueue *q, DefragTracker *dt) { +#ifdef DEBUG + BUG_ON(q == NULL || dt == NULL); +#endif + + DQLOCK_LOCK(q); + + /* more trackers in queue */ + if (q->top != NULL) { + dt->lnext = q->top; + q->top->lprev = dt; + q->top = dt; + /* only tracker */ + } else { + q->top = dt; + q->bot = dt; + } + q->len++; +#ifdef DBG_PERF + if (q->len > q->dbg_maxlen) + q->dbg_maxlen = q->len; +#endif /* DBG_PERF */ + DQLOCK_UNLOCK(q); +} + +/** + * \brief remove a tracker from the queue + * + * \param q queue + * + * \retval dt tracker or NULL if empty list. + */ +DefragTracker *DefragTrackerDequeue (DefragTrackerQueue *q) { + DQLOCK_LOCK(q); + + DefragTracker *dt = q->bot; + if (dt == NULL) { + DQLOCK_UNLOCK(q); + return NULL; + } + + /* more packets in queue */ + if (q->bot->lprev != NULL) { + q->bot = q->bot->lprev; + q->bot->lnext = NULL; + /* just the one we remove, so now empty */ + } else { + q->top = NULL; + q->bot = NULL; + } + +#ifdef DEBUG + BUG_ON(q->len == 0); +#endif + if (q->len > 0) + q->len--; + + dt->lnext = NULL; + dt->lprev = NULL; + + DQLOCK_UNLOCK(q); + return dt; +} + +uint32_t DefragTrackerQueueLen(DefragTrackerQueue *q) { + uint32_t len; + DQLOCK_LOCK(q); + len = q->len; + DQLOCK_UNLOCK(q); + return len; +} + diff --git a/src/defrag-queue.h b/src/defrag-queue.h new file mode 100644 index 0000000000..87b5f4d18b --- /dev/null +++ b/src/defrag-queue.h @@ -0,0 +1,84 @@ +/* Copyright (C) 2007-2012 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + */ + +#ifndef __DEFRAG_QUEUE_H__ +#define __DEFRAG_QUEUE_H__ + +#include "suricata-common.h" +#include "defrag.h" + +/** Spinlocks or Mutex for the defrag tracker queues. */ +//#define DQLOCK_SPIN +#define DQLOCK_MUTEX + +#ifdef DQLOCK_SPIN + #ifdef DQLOCK_MUTEX + #error Cannot enable both DQLOCK_SPIN and DQLOCK_MUTEX + #endif +#endif + +/* Define a queue for storing defrag trackers */ +typedef struct DefragTrackerQueue_ +{ + DefragTracker *top; + DefragTracker *bot; + uint32_t len; +#ifdef DBG_PERF + uint32_t dbg_maxlen; +#endif /* DBG_PERF */ +#ifdef DQLOCK_MUTEX + SCMutex m; +#elif defined DQLOCK_SPIN + SCSpinlock s; +#else + #error Enable DQLOCK_SPIN or DQLOCK_MUTEX +#endif +} DefragTrackerQueue; + +#ifdef DQLOCK_SPIN + #define DQLOCK_INIT(q) SCSpinInit(&(q)->s, 0) + #define DQLOCK_DESTROY(q) SCSpinDestroy(&(q)->s) + #define DQLOCK_LOCK(q) SCSpinLock(&(q)->s) + #define DQLOCK_TRYLOCK(q) SCSpinTrylock(&(q)->s) + #define DQLOCK_UNLOCK(q) SCSpinUnlock(&(q)->s) +#elif defined DQLOCK_MUTEX + #define DQLOCK_INIT(q) SCMutexInit(&(q)->m, NULL) + #define DQLOCK_DESTROY(q) SCMutexDestroy(&(q)->m) + #define DQLOCK_LOCK(q) SCMutexLock(&(q)->m) + #define DQLOCK_TRYLOCK(q) SCMutexTrylock(&(q)->m) + #define DQLOCK_UNLOCK(q) SCMutexUnlock(&(q)->m) +#else + #error Enable DQLOCK_SPIN or DQLOCK_MUTEX +#endif + +/* prototypes */ +DefragTrackerQueue *DefragTrackerQueueNew(); +DefragTrackerQueue *DefragTrackerQueueInit(DefragTrackerQueue *); +void DefragTrackerQueueDestroy (DefragTrackerQueue *); + +void DefragTrackerEnqueue (DefragTrackerQueue *, DefragTracker *); +DefragTracker *DefragTrackerDequeue (DefragTrackerQueue *); +uint32_t DefragTrackerQueueLen(DefragTrackerQueue *); + +#endif /* __DEFRAG_QUEUE_H__ */ + diff --git a/src/defrag-timeout.c b/src/defrag-timeout.c new file mode 100644 index 0000000000..d8cc8cc59b --- /dev/null +++ b/src/defrag-timeout.c @@ -0,0 +1,150 @@ +/* Copyright (C) 2007-2012 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + */ + +#include "suricata-common.h" +#include "defrag.h" +#include "defrag-hash.h" + +uint32_t DefragTrackerGetSpareCount(void) { + return DefragTrackerSpareQueueGetSize(); +} + +uint32_t DefragTrackerGetActiveCount(void) { + return SC_ATOMIC_GET(defragtracker_counter); +} + +/** \internal + * \brief See if we can really discard this tracker. Check use_cnt reference. + * + * \param dt tracker + * \param ts timestamp + * + * \retval 0 not timed out just yet + * \retval 1 fully timed out, lets kill it + */ +static int DefragTrackerTimedOut(DefragTracker *dt, struct timeval *ts) { + /** never prune a trackers that is used by a packet + * we are currently processing in one of the threads */ + if (SC_ATOMIC_GET(dt->use_cnt) > 0) { + return 0; + } + + /* retain if remove is not set and not timed out */ + if (!dt->remove && dt->timeout > ts->tv_sec) + return 0; + + return 1; +} + +/** + * \internal + * + * \brief check all trackers in a hash row for timing out + * + * \param hb tracker hash row *LOCKED* + * \param dt last tracker in the hash row + * \param ts timestamp + * + * \retval cnt timed out tracker + */ +static uint32_t DefragTrackerHashRowTimeout(DefragTrackerHashRow *hb, DefragTracker *dt, struct timeval *ts) +{ + uint32_t cnt = 0; + + do { + if (SCMutexTrylock(&dt->lock) != 0) { + dt = dt->hprev; + continue; + } + + DefragTracker *next_dt = dt->hprev; + + /* check if the tracker is fully timed out and + * ready to be discarded. */ + if (DefragTrackerTimedOut(dt, ts) == 1) { + /* remove from the hash */ + if (dt->hprev != NULL) + dt->hprev->hnext = dt->hnext; + if (dt->hnext != NULL) + dt->hnext->hprev = dt->hprev; + if (hb->head == dt) + hb->head = dt->hnext; + if (hb->tail == dt) + hb->tail = dt->hprev; + + dt->hnext = NULL; + dt->hprev = NULL; + + DefragTrackerClearMemory(dt); + + /* no one is referring to this tracker, use_cnt 0, removed from hash + * so we can unlock it and move it back to the spare queue. */ + SCMutexUnlock(&dt->lock); + + /* move to spare list */ + DefragTrackerMoveToSpare(dt); + + cnt++; + } else { + SCMutexUnlock(&dt->lock); + } + + dt = next_dt; + } while (dt != NULL); + + return cnt; +} + +/** + * \brief time out tracker from the hash + * + * \param ts timestamp + * + * \retval cnt number of timed out tracker + */ +uint32_t DefragTimeoutHash(struct timeval *ts) { + uint32_t idx = 0; + uint32_t cnt = 0; + + for (idx = 0; idx < defrag_config.hash_size; idx++) { + DefragTrackerHashRow *hb = &defragtracker_hash[idx]; + if (hb == NULL) + continue; + if (DRLOCK_TRYLOCK(hb) != 0) + continue; + + /* defrag hash bucket is now locked */ + + if (hb->tail == NULL) { + DRLOCK_UNLOCK(hb); + continue; + } + + /* we have a tracker, or more than one */ + cnt += DefragTrackerHashRowTimeout(hb, hb->tail, ts); + DRLOCK_UNLOCK(hb); + } + + return cnt; +} + diff --git a/src/defrag-timeout.h b/src/defrag-timeout.h new file mode 100644 index 0000000000..77de4572f1 --- /dev/null +++ b/src/defrag-timeout.h @@ -0,0 +1,33 @@ +/* Copyright (C) 2007-2012 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + */ + +#ifndef __DEFRAG_TIMEOUT_H__ +#define __DEFRAG_TIMEOUT_H__ + +uint32_t DefragTimeoutHash(struct timeval *ts); + +uint32_t DefragGetSpareCount(void); +uint32_t DefragGetActiveCount(void); + +#endif + diff --git a/src/defrag.c b/src/defrag.c index 6cd548003a..44886c6048 100644 --- a/src/defrag.c +++ b/src/defrag.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2010 Open Information Security Foundation +/* Copyright (C) 2007-2012 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -50,6 +50,10 @@ #include "stream-tcp-reassemble.h" #include "util-host-os-info.h" +#include "defrag.h" +#include "defrag-hash.h" +#include "defrag-queue.h" + #ifdef UNITTESTS #include "util-unittest.h" #endif @@ -88,114 +92,6 @@ enum defrag_policies { static int default_policy = DEFRAG_POLICY_BSD; -/** - * A context for an instance of a fragmentation re-assembler, in case - * we ever need more than one. - */ -typedef struct DefragContext_ { - uint64_t ip4_frags; /**< Number of IPv4 fragments seen. */ - uint64_t ip6_frags; /**< Number of IPv6 fragments seen. */ - - HashListTable *frag_table; /**< Hash (list) table of fragment trackers. */ - SCMutex frag_table_lock; - - Pool *tracker_pool; /**< Pool of trackers. */ - SCMutex tracker_pool_lock; - - Pool *frag_pool; /**< Pool of fragments. */ - SCMutex frag_pool_lock; - - time_t timeout; /**< Default timeout. */ - time_t last_timeouted; /**< time of last cleaning */ -} DefragContext; - -/** - * Storage for an individual fragment. - */ -typedef struct Frag_ { - DefragContext *dc; /**< The defragmentation context this frag was - * allocated under. */ - - uint16_t offset; /**< The offset of this fragment, already - * multiplied by 8. */ - - uint16_t len; /**< The length of this fragment. */ - - uint8_t hlen; /**< The length of this fragments IP header. */ - - uint8_t more_frags; /**< More frags? */ - - uint16_t ip_hdr_offset; /**< Offset in the packet where the IP - * header starts. */ - uint16_t frag_hdr_offset; /**< Offset in the packet where the frag - * header starts. */ - - uint16_t data_offset; /**< Offset to the packet data. */ - uint16_t data_len; /**< Length of data. */ - - uint8_t *pkt; /**< The actual packet. */ - - uint16_t ltrim; /**< Number of leading bytes to trim when - * re-assembling the packet. */ - - int8_t skip; /**< Skip this fragment during re-assembly. */ - -#ifdef DEBUG - uint64_t pcap_cnt; /* pcap_cnt of original packet */ -#endif - - TAILQ_ENTRY(Frag_) next; /**< Pointer to next fragment for tailq. */ -} Frag; - -/** \brief Reset tracker fields except "dc" and "lock" */ -#define DEFRAG_TRACKER_RESET(t) { \ - (t)->timeout = 0; \ - (t)->id = 0; \ - (t)->policy = 0; \ - (t)->af = 0; \ - (t)->seen_last = 0; \ - CLEAR_ADDR(&(t)->src_addr); \ - CLEAR_ADDR(&(t)->dst_addr); \ - (t)->frags.tqh_first = NULL; \ - (t)->frags.tqh_last = NULL; \ -} - -/** - * A defragmentation tracker. Used to track fragments that make up a - * single packet. - */ -typedef struct DefragTracker_ { - SCMutex lock; /**< Mutex for locking list operations on - * this tracker. */ - - DefragContext *dc; /**< The defragmentation context this tracker - * was allocated under. */ - - uint32_t timeout; /**< When this tracker will timeout. */ - - uint32_t id; /**< IP ID for this tracker. 32 bits for IPv6, 16 - * for IPv4. */ - - uint8_t policy; /**< Reassembly policy this tracker will use. */ - - uint8_t af; /**< Address family for this tracker, AF_INET or - * AF_INET6. */ - - uint8_t seen_last; /**< Has this tracker seen the last fragment? */ - - Address src_addr; /**< Source address for this tracker. */ - Address dst_addr; /**< Destination address for this tracker. */ - - TAILQ_HEAD(frag_tailq, Frag_) frags; /**< Head of list of fragments. */ -} DefragTracker; - -/** A random value used for hash key generation. */ -static int defrag_hash_rand; - -/** Hash table size, and also the maximum number of trackers that will - * be allocated. */ -static int defrag_hash_size; - /** The global DefragContext so all threads operate from the same * context. */ static DefragContext *defrag_context; @@ -220,85 +116,15 @@ DumpFrags(DefragTracker *tracker) #endif /* UNITTESTS */ #endif -/** - * Generate a key for looking of a fragtracker in a hash - * table. Adapted from the hash function in flow-hash.c. - * - * \todo Test performance and distribution. - */ -static uint32_t -DefragHashFunc(HashListTable *ht, void *data, uint16_t datalen) -{ - DefragTracker *p = (DefragTracker *)data; - uint32_t key; - - if (p->af == AF_INET) { - key = (defrag_hash_rand + p->id + - p->src_addr.addr_data32[0] + p->dst_addr.addr_data32[0]) % - defrag_hash_size; - } - else if (p->af == AF_INET6) { - key = (defrag_hash_rand + p->id + - p->src_addr.addr_data32[0] + p->src_addr.addr_data32[1] + - p->src_addr.addr_data32[2] + p->src_addr.addr_data32[3] + - p->dst_addr.addr_data32[0] + p->dst_addr.addr_data32[1] + - p->dst_addr.addr_data32[2] + p->dst_addr.addr_data32[3]) % - defrag_hash_size; - } - else - key = 0; - - return key; -} - -/** - * \brief Compare 2 DefragTracker nodes in case of hash conflict. - * - * \retval 1 if a and b match, otherwise 0. - */ -static char -DefragHashCompare(void *a, uint16_t a_len, void *b, uint16_t b_len) -{ - DefragTracker *dta = (DefragTracker *)a; - DefragTracker *dtb = (DefragTracker *)b; - - if (dta->af != dtb->af) - return 0; - else if (dta->id != dtb->id) - return 0; - else if (!CMP_ADDR(&dta->src_addr, &dtb->src_addr)) - return 0; - else if (!CMP_ADDR(&dta->dst_addr, &dtb->dst_addr)) - return 0; - - /* Match. */ - return 1; -} - -/** - * \brief Called by the hash table when a tracker is removed from the - * hash table. - * - * We don't actually do anything here. The tracker will be reset and - * put back into a memory pool. - */ -static void -DefragHashFree(void *data) -{ -} - /** * \brief Reset a frag for reuse in a pool. */ static void DefragFragReset(Frag *frag) { - DefragContext *dc = frag->dc; - if (frag->pkt != NULL) SCFree(frag->pkt); memset(frag, 0, sizeof(*frag)); - frag->dc = dc; } /** @@ -307,84 +133,32 @@ DefragFragReset(Frag *frag) static int DefragFragInit(void *data, void *initdata) { - DefragContext *dc = initdata; Frag *frag = data; memset(frag, 0, sizeof(*frag)); - frag->dc = dc; - return 1; } /** * \brief Free all frags associated with a tracker. */ -static void +void DefragTrackerFreeFrags(DefragTracker *tracker) { Frag *frag; /* Lock the frag pool as we'll be return items to it. */ - SCMutexLock(&tracker->dc->frag_pool_lock); + SCMutexLock(&defrag_context->frag_pool_lock); while ((frag = TAILQ_FIRST(&tracker->frags)) != NULL) { TAILQ_REMOVE(&tracker->frags, frag, next); /* Don't SCFree the frag, just give it back to its pool. */ DefragFragReset(frag); - PoolReturn(frag->dc->frag_pool, frag); + PoolReturn(defrag_context->frag_pool, frag); } - SCMutexUnlock(&tracker->dc->frag_pool_lock); -} - -/** - * \brief Reset a tracker for reuse. - */ -static void -DefragTrackerReset(DefragTracker *tracker) -{ - DefragTrackerFreeFrags(tracker); - DEFRAG_TRACKER_RESET(tracker); - TAILQ_INIT(&tracker->frags); -} - -/** - * \brief Allocates a new defragmentation tracker for use in the pool - * for trackers. - * - * \arg Pointer to DefragContext this new tracker will be associated - * with. - * - * \retval A new DefragTracker if successfull, NULL on failure. - */ -static int -DefragTrackerInit(void *data, void *initdata) -{ - DefragContext *dc = initdata; - DefragTracker *tracker = data; - - memset(tracker, 0, sizeof(*tracker)); - if (SCMutexInit(&tracker->lock, NULL) != 0) { - return 0; - } - tracker->dc = dc; - TAILQ_INIT(&tracker->frags); - - return 1; -} - -/** - * \brief Free a defragmentation tracker that is being removed from - * the pool. - */ -static void -DefragTrackerCleanup(void *arg) -{ - DefragTracker *tracker = arg; - - SCMutexDestroy(&tracker->lock); - DefragTrackerFreeFrags(tracker); + SCMutexUnlock(&defrag_context->frag_pool_lock); } /** @@ -402,38 +176,11 @@ DefragContextNew(void) if (dc == NULL) return NULL; - /* Initialize the hash table. */ - dc->frag_table = HashListTableInit(defrag_hash_size, DefragHashFunc, - DefragHashCompare, DefragHashFree); - if (dc->frag_table == NULL) { - SCLogError(SC_ERR_MEM_ALLOC, - "Defrag: Failed to initialize hash table."); - exit(EXIT_FAILURE); - } - if (SCMutexInit(&dc->frag_table_lock, NULL) != 0) { - SCLogError(SC_ERR_MEM_ALLOC, - "Defrag: Failed to initialize hash table mutex."); - exit(EXIT_FAILURE); - } - /* Initialize the pool of trackers. */ intmax_t tracker_pool_size; if (!ConfGetInt("defrag.trackers", &tracker_pool_size) || tracker_pool_size == 0) { tracker_pool_size = DEFAULT_DEFRAG_HASH_SIZE; } - dc->tracker_pool = PoolInit(tracker_pool_size, tracker_pool_size, - sizeof(DefragTracker), - NULL, DefragTrackerInit, dc, DefragTrackerCleanup); - if (dc->tracker_pool == NULL) { - SCLogError(SC_ERR_MEM_ALLOC, - "Defrag: Failed to initialize tracker pool."); - exit(EXIT_FAILURE); - } - if (SCMutexInit(&dc->tracker_pool_lock, NULL) != 0) { - SCLogError(SC_ERR_MUTEX, - "Defrag: Failed to initialize tracker pool mutex."); - exit(EXIT_FAILURE); - } /* Initialize the pool of frags. */ intmax_t frag_pool_size; @@ -474,8 +221,6 @@ DefragContextNew(void) dc->timeout = timeout; } - dc->last_timeouted = 0; - SCLogDebug("Defrag Initialized:"); SCLogDebug("\tTimeout: %"PRIuMAX, (uintmax_t)dc->timeout); SCLogDebug("\tMaximum defrag trackers: %"PRIuMAX, tracker_pool_size); @@ -492,9 +237,7 @@ DefragContextDestroy(DefragContext *dc) if (dc == NULL) return; - HashListTableFree(dc->frag_table); PoolFree(dc->frag_pool); - PoolFree(dc->tracker_pool); SCFree(dc); } @@ -504,8 +247,7 @@ DefragContextDestroy(DefragContext *dc) * \param tracker The defragmentation tracker to reassemble from. */ static Packet * -Defrag4Reassemble(ThreadVars *tv, DefragContext *dc, DefragTracker *tracker, - Packet *p) +Defrag4Reassemble(ThreadVars *tv, DefragTracker *tracker, Packet *p) { Packet *rp = NULL; @@ -547,7 +289,6 @@ Defrag4Reassemble(ThreadVars *tv, DefragContext *dc, DefragTracker *tracker, "fragmentation re-assembly, dumping fragments."); goto remove_tracker; } - SCLogDebug("Packet rp %p, p %p, rp->root %p", rp, p, rp->root); rp->recursion_level = p->recursion_level; int fragmentable_offset = 0; @@ -605,15 +346,9 @@ Defrag4Reassemble(ThreadVars *tv, DefragContext *dc, DefragTracker *tracker, SET_PKT_LEN(rp, ip_hdr_offset + hlen + fragmentable_len); remove_tracker: - /* Remove the frag tracker. */ - SCMutexLock(&dc->frag_table_lock); - HashListTableRemove(dc->frag_table, tracker, HASHLIST_NO_SIZE); - SCMutexUnlock(&dc->frag_table_lock); - DefragTrackerReset(tracker); - SCMutexLock(&dc->tracker_pool_lock); - PoolReturn(dc->tracker_pool, tracker); - SCMutexUnlock(&dc->tracker_pool_lock); - + /** \todo check locking */ + tracker->remove = 1; + DefragTrackerFreeFrags(tracker); done: return rp; } @@ -624,8 +359,7 @@ done: * \param tracker The defragmentation tracker to reassemble from. */ static Packet * -Defrag6Reassemble(ThreadVars *tv, DefragContext *dc, DefragTracker *tracker, - Packet *p) +Defrag6Reassemble(ThreadVars *tv, DefragTracker *tracker, Packet *p) { Packet *rp = NULL; @@ -716,15 +450,9 @@ Defrag6Reassemble(ThreadVars *tv, DefragContext *dc, DefragTracker *tracker, SET_PKT_LEN(rp, ip_hdr_offset + sizeof(IPV6Hdr) + fragmentable_len); remove_tracker: - /* Remove the frag tracker. */ - SCMutexLock(&dc->frag_table_lock); - HashListTableRemove(dc->frag_table, tracker, HASHLIST_NO_SIZE); - SCMutexUnlock(&dc->frag_table_lock); - DefragTrackerReset(tracker); - SCMutexLock(&dc->tracker_pool_lock); - PoolReturn(dc->tracker_pool, tracker); - SCMutexUnlock(&dc->tracker_pool_lock); - + /** \todo check locking */ + tracker->remove = 1; + DefragTrackerFreeFrags(tracker); done: return rp; } @@ -735,8 +463,7 @@ done: * \todo Allocate packet buffers from a pool. */ static Packet * -DefragInsertFrag(ThreadVars *tv, DecodeThreadVars *dtv, DefragContext *dc, - DefragTracker *tracker, Packet *p) +DefragInsertFrag(ThreadVars *tv, DecodeThreadVars *dtv, DefragTracker *tracker, Packet *p) { Packet *r = NULL; int ltrim = 0; @@ -811,11 +538,8 @@ DefragInsertFrag(ThreadVars *tv, DecodeThreadVars *dtv, DefragContext *dc, return NULL; } - /* Lock this tracker as we'll be doing list operations on it. */ - SCMutexLock(&tracker->lock); - /* Update timeout. */ - tracker->timeout = p->ts.tv_sec + dc->timeout; + tracker->timeout = p->ts.tv_sec + defrag_context->timeout; Frag *prev = NULL, *next; int overlap = 0; @@ -930,9 +654,9 @@ insert: } /* Allocate fragment and insert. */ - SCMutexLock(&dc->frag_pool_lock); - Frag *new = PoolGet(dc->frag_pool); - SCMutexUnlock(&dc->frag_pool_lock); + SCMutexLock(&defrag_context->frag_pool_lock); + Frag *new = PoolGet(defrag_context->frag_pool); + SCMutexUnlock(&defrag_context->frag_pool_lock); if (new == NULL) { if (af == AF_INET) { ENGINE_SET_EVENT(p, IPV4_FRAG_IGNORED); @@ -943,9 +667,9 @@ insert: } new->pkt = SCMalloc(GET_PKT_LEN(p)); if (new->pkt == NULL) { - SCMutexLock(&dc->frag_pool_lock); - PoolReturn(dc->frag_pool, new); - SCMutexUnlock(&dc->frag_pool_lock); + SCMutexLock(&defrag_context->frag_pool_lock); + PoolReturn(defrag_context->frag_pool, new); + SCMutexUnlock(&defrag_context->frag_pool_lock); if (af == AF_INET) { ENGINE_SET_EVENT(p, IPV4_FRAG_IGNORED); } else { @@ -983,14 +707,14 @@ insert: if (tracker->seen_last) { if (tracker->af == AF_INET) { - r = Defrag4Reassemble(tv, dc, tracker, p); + r = Defrag4Reassemble(tv, tracker, p); if (r != NULL && tv != NULL && dtv != NULL) { SCPerfCounterIncr(dtv->counter_defrag_ipv4_reassembled, tv->sc_perf_pca); } } else if (tracker->af == AF_INET6) { - r = Defrag6Reassemble(tv, dc, tracker, p); + r = Defrag6Reassemble(tv, tracker, p); if (r != NULL && tv != NULL && dtv != NULL) { SCPerfCounterIncr(dtv->counter_defrag_ipv6_reassembled, tv->sc_perf_pca); @@ -1007,61 +731,9 @@ done: ENGINE_SET_EVENT(p, IPV6_FRAG_OVERLAP); } } - SCMutexUnlock(&tracker->lock); return r; } -/** - * \brief Timeout trackers. - * - * Called when we fail to get a tracker from the pool. The trackers - * that has expired will be released back to the pool then the - * function will exit. - * - * Intended to be called with the tracker pool already locked. - * - * \param dc Current DefragContext. - * \param p Packet that triggered this timeout run, used for timestamp. - */ -static void -DefragTimeoutTracker(ThreadVars *tv, DecodeThreadVars *dtv, DefragContext *dc, - Packet *p) -{ - HashListTableBucket *next = HashListTableGetListHead(dc->frag_table); - DefragTracker *tracker; - struct timeval ts; - - TimeGet(&ts); - /* If last cleaning was made in the current second, we leave */ - if (dc->last_timeouted >= ts.tv_sec) { - return; - } else { - dc->last_timeouted = ts.tv_sec; - } - while (next != NULL) { - tracker = HashListTableGetListData(next); - next = HashListTableGetListNext(next); - - if (tracker->timeout < (unsigned int)p->ts.tv_sec) { - int af_family = tracker->af; - /* Tracker has timeout out. */ - HashListTableRemove(dc->frag_table, tracker, HASHLIST_NO_SIZE); - DefragTrackerReset(tracker); - PoolReturn(dc->tracker_pool, tracker); - if (tv != NULL && dtv != NULL) { - if (af_family == AF_INET) { - SCPerfCounterIncr(dtv->counter_defrag_ipv4_timeouts, - tv->sc_perf_pca); - } - else if (af_family == AF_INET6) { - SCPerfCounterIncr(dtv->counter_defrag_ipv6_timeouts, - tv->sc_perf_pca); - } - } - } - } -} - /** * \brief Get the defrag policy based on the destination address of * the packet. @@ -1070,7 +742,7 @@ DefragTimeoutTracker(ThreadVars *tv, DecodeThreadVars *dtv, DefragContext *dc, * * \retval The defrag policy to use. */ -static uint8_t +uint8_t DefragGetOsPolicy(Packet *p) { int policy = -1; @@ -1130,58 +802,19 @@ DefragGetOsPolicy(Packet *p) } } +/** \internal + * + * \retval NULL or a *LOCKED* tracker */ static DefragTracker * -DefragGetTracker(ThreadVars *tv, DecodeThreadVars *dtv, DefragContext *dc, - DefragTracker *lookup_key, Packet *p) +DefragGetTracker(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p) { - DefragTracker *tracker; - - SCMutexLock(&dc->frag_table_lock); - tracker = HashListTableLookup(dc->frag_table, lookup_key, - sizeof(*lookup_key)); - if (tracker == NULL) { - SCMutexLock(&dc->tracker_pool_lock); - tracker = PoolGet(dc->tracker_pool); - if (tracker == NULL) { - /* Timeout trackers and try again. */ - DefragTimeoutTracker(tv, dtv, dc, p); - tracker = PoolGet(dc->tracker_pool); - } - SCMutexUnlock(&dc->tracker_pool_lock); - if (tracker == NULL) { - SCPerfCounterIncr(dtv->counter_defrag_max_hit, - tv->sc_perf_pca); - goto done; - } - DefragTrackerReset(tracker); - tracker->af = lookup_key->af; - tracker->id = lookup_key->id; - tracker->src_addr = lookup_key->src_addr; - tracker->dst_addr = lookup_key->dst_addr; - tracker->policy = DefragGetOsPolicy(p); - - if (HashListTableAdd(dc->frag_table, tracker, HASHLIST_NO_SIZE) != 0) { - /* Failed to add new tracker. */ - SCLogError(SC_ERR_MEM_ALLOC, - "Defrag: Failed to add new tracker to hash table."); - SCMutexLock(&dc->tracker_pool_lock); - PoolReturn(dc->tracker_pool, tracker); - SCMutexUnlock(&dc->tracker_pool_lock); - tracker = NULL; - goto done; - } - } - -done: - SCMutexUnlock(&dc->frag_table_lock); - return tracker; + return DefragGetTrackerFromHash(p); } /** * \brief Entry point for IPv4 and IPv6 fragments. * * \param tv ThreadVars for the calling decoder. - * \param dc A DefragContext to use, may be NULL for the default. * \param p The packet fragment. * * \retval A new Packet resembling the re-assembled packet if the most @@ -1189,30 +822,22 @@ done: * NULL is returned. */ Packet * -Defrag(ThreadVars *tv, DecodeThreadVars *dtv, DefragContext *dc, Packet *p) +Defrag(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p) { uint16_t frag_offset; uint8_t more_frags; - DefragTracker *tracker, lookup; - uint32_t id; + DefragTracker *tracker; int af; - /* If no DefragContext was passed in, use the global one. Passing - * one in is primarily useful for unit tests. */ - if (dc == NULL) - dc = defrag_context; - if (PKT_IS_IPV4(p)) { af = AF_INET; more_frags = IPV4_GET_MF(p); frag_offset = IPV4_GET_IPOFFSET(p); - id = IPV4_GET_IPID(p); } else if (PKT_IS_IPV6(p)) { af = AF_INET6; frag_offset = IPV6_EXTHDR_GET_FH_OFFSET(p); more_frags = IPV6_EXTHDR_GET_FH_FLAG(p); - id = IPV6_EXTHDR_GET_FH_ID(p); } else { return NULL; @@ -1233,33 +858,25 @@ Defrag(ThreadVars *tv, DecodeThreadVars *dtv, DefragContext *dc, Packet *p) } } - /* Create a lookup key. */ - lookup.af = af; - lookup.id = id; - lookup.src_addr = p->src; - lookup.dst_addr = p->dst; - - tracker = DefragGetTracker(tv, dtv, dc, &lookup, p); + /* return a locked tracker or NULL */ + tracker = DefragGetTracker(tv, dtv, p); if (tracker == NULL) return NULL; - return DefragInsertFrag(tv, dtv, dc, tracker, p); + Packet *rp = DefragInsertFrag(tv, dtv, tracker, p); + DefragTrackerRelease(tracker); + + return rp; } void DefragInit(void) { - /* Initialize random value for hashing and hash table size. */ - unsigned int seed = RandomTimePreseed(); intmax_t tracker_pool_size; if (!ConfGetInt("defrag.trackers", &tracker_pool_size)) { tracker_pool_size = DEFAULT_DEFRAG_HASH_SIZE; } - /* set defaults */ - defrag_hash_rand = (int)(tracker_pool_size * (rand_r(&seed) / RAND_MAX + 1.0)); - defrag_hash_size = tracker_pool_size; - /* Allocate the DefragContext. */ defrag_context = DefragContextNew(); if (defrag_context == NULL) { @@ -1267,9 +884,12 @@ DefragInit(void) "Failed to allocate memory for the Defrag module."); exit(EXIT_FAILURE); } + + DefragInitConfig(FALSE); } void DefragDestroy(void) { + DefragHashShutdown(); DefragContextDestroy(defrag_context); defrag_context = NULL; } @@ -1431,7 +1051,6 @@ error: static int DefragInOrderSimpleTest(void) { - DefragContext *dc = NULL; Packet *p1 = NULL, *p2 = NULL, *p3 = NULL; Packet *reassembled = NULL; int id = 12; @@ -1440,10 +1059,6 @@ DefragInOrderSimpleTest(void) DefragInit(); - dc = DefragContextNew(); - if (dc == NULL) - goto end; - p1 = BuildTestPacket(id, 0, 1, 'A', 8); if (p1 == NULL) goto end; @@ -1454,30 +1069,35 @@ DefragInOrderSimpleTest(void) if (p3 == NULL) goto end; - if (Defrag(NULL, NULL, dc, p1) != NULL) + if (Defrag(NULL, NULL, p1) != NULL) goto end; - if (Defrag(NULL, NULL, dc, p2) != NULL) + if (Defrag(NULL, NULL, p2) != NULL) goto end; - reassembled = Defrag(NULL, NULL, dc, p3); - if (reassembled == NULL) + reassembled = Defrag(NULL, NULL, p3); + if (reassembled == NULL) { goto end; + } - if (IPV4_GET_HLEN(reassembled) != 20) + if (IPV4_GET_HLEN(reassembled) != 20) { goto end; - if (IPV4_GET_IPLEN(reassembled) != 39) + } + if (IPV4_GET_IPLEN(reassembled) != 39) { goto end; + } /* 20 bytes in we should find 8 bytes of A. */ for (i = 20; i < 20 + 8; i++) { - if (GET_PKT_DATA(reassembled)[i] != 'A') + if (GET_PKT_DATA(reassembled)[i] != 'A') { goto end; + } } /* 28 bytes in we should find 8 bytes of B. */ for (i = 28; i < 28 + 8; i++) { - if (GET_PKT_DATA(reassembled)[i] != 'B') + if (GET_PKT_DATA(reassembled)[i] != 'B') { goto end; + } } /* And 36 bytes in we should find 3 bytes of C. */ @@ -1487,10 +1107,8 @@ DefragInOrderSimpleTest(void) } ret = 1; -end: - if (dc != NULL) - DefragContextDestroy(dc); +end: if (p1 != NULL) SCFree(p1); if (p2 != NULL) @@ -1510,7 +1128,6 @@ end: static int DefragReverseSimpleTest(void) { - DefragContext *dc = NULL; Packet *p1 = NULL, *p2 = NULL, *p3 = NULL; Packet *reassembled = NULL; int id = 12; @@ -1519,10 +1136,6 @@ DefragReverseSimpleTest(void) DefragInit(); - dc = DefragContextNew(); - if (dc == NULL) - goto end; - p1 = BuildTestPacket(id, 0, 1, 'A', 8); if (p1 == NULL) goto end; @@ -1533,12 +1146,12 @@ DefragReverseSimpleTest(void) if (p3 == NULL) goto end; - if (Defrag(NULL, NULL, dc, p3) != NULL) + if (Defrag(NULL, NULL, p3) != NULL) goto end; - if (Defrag(NULL, NULL, dc, p2) != NULL) + if (Defrag(NULL, NULL, p2) != NULL) goto end; - reassembled = Defrag(NULL, NULL, dc, p1); + reassembled = Defrag(NULL, NULL, p1); if (reassembled == NULL) goto end; @@ -1567,9 +1180,6 @@ DefragReverseSimpleTest(void) ret = 1; end: - - if (dc != NULL) - DefragContextDestroy(dc); if (p1 != NULL) SCFree(p1); if (p2 != NULL) @@ -1590,7 +1200,6 @@ end: static int IPV6DefragInOrderSimpleTest(void) { - DefragContext *dc = NULL; Packet *p1 = NULL, *p2 = NULL, *p3 = NULL; Packet *reassembled = NULL; int id = 12; @@ -1599,10 +1208,6 @@ IPV6DefragInOrderSimpleTest(void) DefragInit(); - dc = DefragContextNew(); - if (dc == NULL) - goto end; - p1 = IPV6BuildTestPacket(id, 0, 1, 'A', 8); if (p1 == NULL) goto end; @@ -1613,11 +1218,11 @@ IPV6DefragInOrderSimpleTest(void) if (p3 == NULL) goto end; - if (Defrag(NULL, NULL, dc, p1) != NULL) + if (Defrag(NULL, NULL, p1) != NULL) goto end; - if (Defrag(NULL, NULL, dc, p2) != NULL) + if (Defrag(NULL, NULL, p2) != NULL) goto end; - reassembled = Defrag(NULL, NULL, dc, p3); + reassembled = Defrag(NULL, NULL, p3); if (reassembled == NULL) goto end; @@ -1644,8 +1249,6 @@ IPV6DefragInOrderSimpleTest(void) ret = 1; end: - if (dc != NULL) - DefragContextDestroy(dc); if (p1 != NULL) SCFree(p1); if (p2 != NULL) @@ -1685,11 +1288,11 @@ IPV6DefragReverseSimpleTest(void) if (p3 == NULL) goto end; - if (Defrag(NULL, NULL, dc, p3) != NULL) + if (Defrag(NULL, NULL, p3) != NULL) goto end; - if (Defrag(NULL, NULL, dc, p2) != NULL) + if (Defrag(NULL, NULL, p2) != NULL) goto end; - reassembled = Defrag(NULL, NULL, dc, p1); + reassembled = Defrag(NULL, NULL, p1); if (reassembled == NULL) goto end; @@ -1733,7 +1336,6 @@ DefragDoSturgesNovakTest(int policy, u_char *expected, size_t expected_len) { int i; int ret = 0; - DefragContext *dc = NULL; DefragInit(); @@ -1804,14 +1406,11 @@ DefragDoSturgesNovakTest(int policy, u_char *expected, size_t expected_len) /* Q*16 at 176. */ packets[16] = BuildTestPacket(id, 176 >> 3, 0, 'Q', 16); - dc = DefragContextNew(); - if (dc == NULL) - goto end; default_policy = policy; /* Send all but the last. */ for (i = 0; i < 9; i++) { - Packet *tp = Defrag(NULL, NULL, dc, packets[i]); + Packet *tp = Defrag(NULL, NULL, packets[i]); if (tp != NULL) { SCFree(tp); goto end; @@ -1822,7 +1421,7 @@ DefragDoSturgesNovakTest(int policy, u_char *expected, size_t expected_len) } int overlap = 0; for (; i < 16; i++) { - Packet *tp = Defrag(NULL, NULL, dc, packets[i]); + Packet *tp = Defrag(NULL, NULL, packets[i]); if (tp != NULL) { SCFree(tp); goto end; @@ -1831,35 +1430,35 @@ DefragDoSturgesNovakTest(int policy, u_char *expected, size_t expected_len) overlap++; } } - if (!overlap) + if (!overlap) { goto end; + } /* And now the last one. */ - Packet *reassembled = Defrag(NULL, NULL, dc, packets[16]); - if (reassembled == NULL) + Packet *reassembled = Defrag(NULL, NULL, packets[16]); + if (reassembled == NULL) { goto end; + } - if (IPV4_GET_HLEN(reassembled) != 20) + if (IPV4_GET_HLEN(reassembled) != 20) { goto end; - if (IPV4_GET_IPLEN(reassembled) != 20 + 192) + } + if (IPV4_GET_IPLEN(reassembled) != 20 + 192) { goto end; + } - if (memcmp(GET_PKT_DATA(reassembled) + 20, expected, expected_len) != 0) + if (memcmp(GET_PKT_DATA(reassembled) + 20, expected, expected_len) != 0) { goto end; + } SCFree(reassembled); - /* Make sure the tracker was released back to the pool. */ - if (dc->tracker_pool->outstanding != 0) - return 0; - /* Make sure all frags were returned back to the pool. */ - if (dc->frag_pool->outstanding != 0) - return 0; + if (defrag_context->frag_pool->outstanding != 0) { + goto end; + } ret = 1; end: - if (dc != NULL) - DefragContextDestroy(dc); for (i = 0; i < 17; i++) { SCFree(packets[i]); } @@ -1872,7 +1471,6 @@ IPV6DefragDoSturgesNovakTest(int policy, u_char *expected, size_t expected_len) { int i; int ret = 0; - DefragContext *dc = NULL; DefragInit(); @@ -1943,14 +1541,11 @@ IPV6DefragDoSturgesNovakTest(int policy, u_char *expected, size_t expected_len) /* Q*16 at 176. */ packets[16] = IPV6BuildTestPacket(id, 176 >> 3, 0, 'Q', 16); - dc = DefragContextNew(); - if (dc == NULL) - goto end; default_policy = policy; /* Send all but the last. */ for (i = 0; i < 9; i++) { - Packet *tp = Defrag(NULL, NULL, dc, packets[i]); + Packet *tp = Defrag(NULL, NULL, packets[i]); if (tp != NULL) { SCFree(tp); goto end; @@ -1961,7 +1556,7 @@ IPV6DefragDoSturgesNovakTest(int policy, u_char *expected, size_t expected_len) } int overlap = 0; for (; i < 16; i++) { - Packet *tp = Defrag(NULL, NULL, dc, packets[i]); + Packet *tp = Defrag(NULL, NULL, packets[i]); if (tp != NULL) { SCFree(tp); goto end; @@ -1974,7 +1569,7 @@ IPV6DefragDoSturgesNovakTest(int policy, u_char *expected, size_t expected_len) goto end; /* And now the last one. */ - Packet *reassembled = Defrag(NULL, NULL, dc, packets[16]); + Packet *reassembled = Defrag(NULL, NULL, packets[16]); if (reassembled == NULL) goto end; if (memcmp(GET_PKT_DATA(reassembled) + 40, expected, expected_len) != 0) @@ -1985,18 +1580,15 @@ IPV6DefragDoSturgesNovakTest(int policy, u_char *expected, size_t expected_len) SCFree(reassembled); - /* Make sure the tracker was released back to the pool. */ - if (dc->tracker_pool->outstanding != 0) - return 0; - /* Make sure all frags were returned to the pool. */ - if (dc->frag_pool->outstanding != 0) - return 0; + if (defrag_context->frag_pool->outstanding != 0) { + printf("defrag_context->frag_pool->outstanding %u: ", defrag_context->frag_pool->outstanding); + goto end; + } ret = 1; + end: - if (dc != NULL) - DefragContextDestroy(dc); for (i = 0; i < 17; i++) { SCFree(packets[i]); } @@ -2422,7 +2014,6 @@ DefragTimeoutTest(void) { int i; int ret = 0; - DefragContext *dc = NULL; /* Setup a small numberr of trackers. */ if (ConfSet("defrag.trackers", "16", 1) != 1) { @@ -2432,17 +2023,13 @@ DefragTimeoutTest(void) DefragInit(); - dc = DefragContextNew(); - if (dc == NULL) - goto end; - /* Load in 16 packets. */ for (i = 0; i < 16; i++) { Packet *p = BuildTestPacket(i, 0, 1, 'A' + i, 16); if (p == NULL) goto end; - Packet *tp = Defrag(NULL, NULL, dc, p); + Packet *tp = Defrag(NULL, NULL, p); SCFree(p); @@ -2458,39 +2045,25 @@ DefragTimeoutTest(void) if (p == NULL) goto end; - p->ts.tv_sec += (dc->timeout + 1); - Packet *tp = Defrag(NULL, NULL, dc, p); - - SCFree(p); + p->ts.tv_sec += (defrag_context->timeout + 1); + Packet *tp = Defrag(NULL, NULL, p); if (tp != NULL) { SCFree(tp); goto end; } - /* Iterate our HashList and look for the trackerr with id 99. */ - int found = 0; - HashListTableBucket *next = HashListTableGetListHead(dc->frag_table); - if (next == NULL) + DefragTracker *tracker = DefragLookupTrackerFromHash(p); + if (tracker == NULL) goto end; - for (;;) { - if (next == NULL) - break; - DefragTracker *tracker = HashListTableGetListData(next); - if (tracker->id == 99) { - found = 1; - break; - } - next = HashListTableGetListNext(next); - } - if (found == 0) + if (tracker->id != 99) goto end; + SCFree(p); + ret = 1; end: - if (dc != NULL) - DefragContextDestroy(dc); DefragDestroy(); return ret; } @@ -2521,7 +2094,7 @@ DefragIPv4NoDataTest(void) goto end; /* We do not expect a packet returned. */ - if (Defrag(NULL, NULL, dc, p) != NULL) + if (Defrag(NULL, NULL, p) != NULL) goto end; /* The fragment should have been ignored so no fragments should @@ -2560,7 +2133,7 @@ DefragIPv4TooLargeTest(void) goto end; /* We do not expect a packet returned. */ - if (Defrag(NULL, NULL, dc, p) != NULL) + if (Defrag(NULL, NULL, p) != NULL) goto end; if (!ENGINE_ISSET_EVENT(p, IPV4_FRAG_PKT_TOO_LARGE)) goto end; diff --git a/src/defrag.h b/src/defrag.h index 5881c2597a..b5ba29c728 100644 --- a/src/defrag.h +++ b/src/defrag.h @@ -24,12 +24,113 @@ #ifndef __DEFRAG_H__ #define __DEFRAG_H__ -typedef struct _DefragContext DefragContext; +#include "util-pool.h" + +/** + * A context for an instance of a fragmentation re-assembler, in case + * we ever need more than one. + */ +typedef struct DefragContext_ { + Pool *frag_pool; /**< Pool of fragments. */ + SCMutex frag_pool_lock; + + time_t timeout; /**< Default timeout. */ +} DefragContext; + +/** + * Storage for an individual fragment. + */ +typedef struct Frag_ { + uint16_t offset; /**< The offset of this fragment, already + * multiplied by 8. */ + + uint16_t len; /**< The length of this fragment. */ + + uint8_t hlen; /**< The length of this fragments IP header. */ + + uint8_t more_frags:4; /**< More frags? */ + uint8_t skip:4; /**< Skip this fragment during re-assembly. */ + + uint16_t ip_hdr_offset; /**< Offset in the packet where the IP + * header starts. */ + uint16_t frag_hdr_offset; /**< Offset in the packet where the frag + * header starts. */ + + uint16_t data_offset; /**< Offset to the packet data. */ + uint16_t data_len; /**< Length of data. */ + + uint16_t ltrim; /**< Number of leading bytes to trim when + * re-assembling the packet. */ + + uint8_t *pkt; /**< The actual packet. */ + +#ifdef DEBUG + uint64_t pcap_cnt; /**< pcap_cnt of original packet */ +#endif + + TAILQ_ENTRY(Frag_) next; /**< Pointer to next fragment for tailq. */ +} Frag; + +/** \brief Reset tracker fields except "lock" */ +#define DEFRAG_TRACKER_RESET(t) { \ + (t)->timeout = 0; \ + (t)->id = 0; \ + (t)->policy = 0; \ + (t)->af = 0; \ + (t)->seen_last = 0; \ + (t)->remove = 0; \ + CLEAR_ADDR(&(t)->src_addr); \ + CLEAR_ADDR(&(t)->dst_addr); \ + (t)->frags.tqh_first = NULL; \ + (t)->frags.tqh_last = NULL; \ +} + +/** + * A defragmentation tracker. Used to track fragments that make up a + * single packet. + */ +typedef struct DefragTracker_ { + SCMutex lock; /**< Mutex for locking list operations on + * this tracker. */ + + uint32_t id; /**< IP ID for this tracker. 32 bits for IPv6, 16 + * for IPv4. */ + + uint8_t policy; /**< Reassembly policy this tracker will use. */ + + uint8_t af; /**< Address family for this tracker, AF_INET or + * AF_INET6. */ + + uint8_t seen_last; /**< Has this tracker seen the last fragment? */ + + uint8_t remove; /**< remove */ + + Address src_addr; /**< Source address for this tracker. */ + Address dst_addr; /**< Destination address for this tracker. */ + + uint32_t timeout; /**< When this tracker will timeout. */ + + /** use cnt, reference counter */ + SC_ATOMIC_DECLARE(unsigned short, use_cnt); + + TAILQ_HEAD(frag_tailq, Frag_) frags; /**< Head of list of fragments. */ + + /** hash pointers, protected by hash row mutex/spin */ + struct DefragTracker_ *hnext; + struct DefragTracker_ *hprev; + + /** list pointers, protected by tracker-queue mutex/spin */ + struct DefragTracker_ *lnext; + struct DefragTracker_ *lprev; +} DefragTracker; void DefragInit(void); void DefragDestroy(void); +void DefragReload(void); /**< use only in unittests */ -Packet *Defrag(ThreadVars *, DecodeThreadVars *, DefragContext *, Packet *); +uint8_t DefragGetOsPolicy(Packet *); +void DefragTrackerFreeFrags(DefragTracker *); +Packet *Defrag(ThreadVars *, DecodeThreadVars *, Packet *); void DefragRegisterTests(void); #endif /* __DEFRAG_H__ */ diff --git a/src/flow-manager.c b/src/flow-manager.c index 95ef705a2d..c419598c7f 100644 --- a/src/flow-manager.c +++ b/src/flow-manager.c @@ -476,6 +476,7 @@ void *FlowManagerThread(void *td) FlowTimeoutHash(&ts, 0 /* check all */, &counters); + DefragTimeoutHash(&ts); //uint32_t hosts_pruned = HostTimeoutHash(&ts); /* diff --git a/src/util-error.c b/src/util-error.c index 1a3dc5a8d0..221c9e0744 100644 --- a/src/util-error.c +++ b/src/util-error.c @@ -233,6 +233,7 @@ const char * SCErrorToString(SCError err) CASE_CODE (SC_ERR_EVENT_ENGINE); CASE_CODE (SC_ERR_NO_LUAJIT_SUPPORT); CASE_CODE (SC_ERR_LUAJIT_ERROR); + CASE_CODE (SC_ERR_DEFRAG_INIT); default: return "UNKNOWN_ERROR"; } diff --git a/src/util-error.h b/src/util-error.h index 4a98533aa1..9ec4ecca6a 100644 --- a/src/util-error.h +++ b/src/util-error.h @@ -248,6 +248,7 @@ typedef enum { SC_ERR_EVENT_ENGINE, SC_ERR_NO_LUAJIT_SUPPORT, SC_ERR_LUAJIT_ERROR, + SC_ERR_DEFRAG_INIT, } SCError; const char *SCErrorToString(SCError); diff --git a/suricata.yaml.in b/suricata.yaml.in index 1cff85e228..5c546aec62 100644 --- a/suricata.yaml.in +++ b/suricata.yaml.in @@ -458,6 +458,8 @@ pattern-matcher: # Defrag settings: defrag: + memcap: 32mb + hash-size: 65536 trackers: 65535 # number of defragmented flows to follow max-frags: 65535 # number of fragments to keep (higher than trackers) prealloc: yes