Modify Packet structure and prepare accessor.

This patch modifies decode.c and decode.h to avoid the usage
by default of a bigger than 65535 bytes array in Packet structure.
The idea is that the packet are mainly under 1514 bytes size and
a bigger size must be supported but should not be the default.

If the packet length is bigger than DFLT_PACKET_SIZE then the
data are stored in a dynamically allocated part of the memory.

To ease the modification of the rest of the code, functions to
access and set the payload/length in a Packet have been introduced.

The default packet size can be set at runtime via the default-packet-size
configuration variable.

src/decode.c

@@ -27,6 +27,7 @@
 #include "suricata.h"
 #include "decode.h"
 #include "util-debug.h"
+#include "util-mem.h"
 #include "app-layer-detect-proto.h"
 #include "tm-modules.h"
 #include "util-error.h"
@@ -66,7 +67,7 @@ Packet *PacketGetFromQueueOrAlloc(void) {
 
     if (p == NULL) {
         /* non fatal, we're just not processing a packet then */
-        p = SCMalloc(sizeof(Packet));
+        p = SCMalloc(SIZE_OF_PACKET);
         if (p == NULL) {
             SCLogError(SC_ERR_MEM_ALLOC, "SCMalloc failed: %s", strerror(errno));
             return NULL;
@@ -107,8 +108,7 @@ Packet *PacketPseudoPktSetup(Packet *parent, uint8_t *pkt, uint16_t len, uint8_t
 
     /* copy packet and set lenght, proto */
     p->tunnel_proto = proto;
-    p->pktlen = len;
-    memcpy(&p->pkt, pkt, len);
+    PacketCopyData(p, pkt, len);
     p->recursion_level = parent->recursion_level + 1;
     p->ts.tv_sec = parent->ts.tv_sec;
     p->ts.tv_usec = parent->ts.tv_usec;
@@ -243,3 +243,55 @@ DecodeThreadVars *DecodeThreadVarsAlloc() {
 
     return dtv;
 }
+
+/**
+ *  \brief Copy data to Packet payload at given offset
+ *
+ *  \param Pointer to the Packet to modify
+ *  \param Offset of the copy relatively to payload of Packet
+ *  \param Pointer to the data to copy
+ *  \param Length of the data to copy
+ */
+inline int PacketCopyDataOffset(Packet *p, int offset, uint8_t *data, int datalen)
+{
+    if (offset + datalen > MAX_PAYLOAD_SIZE) {
+        /* too big */
+        return -1;
+    }
+
+    if (! p->ext_pkt) {
+        if (offset + datalen <= default_packet_size) {
+            memcpy(p->pkt + offset, data, datalen);
+        } else {
+            /* here we need a dynamic allocation. This case should rarely
+             * occur as there is a high probability the first frag has
+             * reveal the packet size*/
+            p->ext_pkt = SCMalloc(MAX_PAYLOAD_SIZE);
+            if (p->ext_pkt == NULL) {
+                SCLogError(SC_ERR_MEM_ALLOC, "SCMalloc failed: %s", strerror(errno));
+                SET_PKT_LEN(p, 0);
+                return -1;
+            }
+            /* copy initial data */
+            memcpy(p->ext_pkt, &p->pkt, p->pktlen);
+            /* copy data as asked */
+            memcpy(p->ext_pkt + offset, data, datalen);
+        }
+    } else {
+        memcpy(p->ext_pkt + offset, data, datalen);
+    }
+    return 0;
+}
+
+/**
+ *  \brief Copy data to Packet payload and set packet length
+ *
+ *  \param Pointer to the Packet to modify
+ *  \param Pointer to the data to copy
+ *  \param Length of the data to copy
+ */
+inline int PacketCopyData(Packet *p, uint8_t *pktdata, int pktlen)
+{
+    SET_PKT_LEN(p, (size_t)pktlen);
+    return PacketCopyDataOffset(p, 0, pktdata, pktlen);
+}

src/decode.h

@@ -156,6 +156,14 @@ typedef struct Address_ {
 #define GET_TCP_SRC_PORT(p)  ((p)->sp)
 #define GET_TCP_DST_PORT(p)  ((p)->dp)
 
+#define GET_PKT_LEN(p) ((p)->pktlen)
+#define GET_PKT_DATA(p) ((((p)->ext_pkt) == NULL ) ? (p)->pkt : (p)->ext_pkt)
+
+#define SET_PKT_LEN(p, len) do { \
+    (p)->pktlen = len;		 \
+    } while (0)
+
+
 /* Port is just a uint16_t */
 typedef uint16_t Port;
 #define SET_PORT(v, p) ((p) = (v))
@@ -340,8 +348,9 @@ typedef struct Packet_
     uint8_t *payload;
     uint16_t payload_len;
 
-    /* storage: maximum ip packet size + link header */
-    uint8_t pkt[IPV6_HEADER_LEN + 65536 + 28];
+    /* storage: set to pointer to heap and extended via allocation if necessary */
+    uint8_t *pkt;
+    uint8_t *ext_pkt;
     uint32_t pktlen;
 
     PacketAlerts alerts;
@@ -395,6 +404,12 @@ typedef struct Packet_
 #endif
 } Packet;
 
+#define DEFAULT_PACKET_SIZE 1500 + ETHERNET_HEADER_LEN
+/* storage: maximum ip packet size + link header */
+#define MAX_PAYLOAD_SIZE IPV6_HEADER_LEN + 65536 + 28
+intmax_t default_packet_size;
+#define SIZE_OF_PACKET default_packet_size + sizeof(Packet)
+
 typedef struct PacketQueue_ {
     Packet *top;
     Packet *bot;
@@ -471,17 +486,19 @@ typedef struct DecodeThreadVars_
  */
 #ifndef __SC_CUDA_SUPPORT__
 #define PACKET_INITIALIZE(p) { \
-    memset((p), 0x00, sizeof(Packet)); \
+    memset((p), 0x00, SIZE_OF_PACKET); \
     SCMutexInit(&(p)->mutex_rtv_cnt, NULL); \
     PACKET_RESET_CHECKSUMS((p)); \
+    (p)->pkt = ((uint8_t *)(p)) + sizeof(Packet); \
 }
 #else
 #define PACKET_INITIALIZE(p) { \
-    memset((p), 0x00, sizeof(Packet)); \
+    memset((p), 0x00, SIZE_OF_PACKET); \
     SCMutexInit(&(p)->mutex_rtv_cnt, NULL); \
     PACKET_RESET_CHECKSUMS((p)); \
     SCMutexInit(&(p)->cuda_mutex, NULL); \
     SCCondInit(&(p)->cuda_cond, NULL); \
+    (p)->pkt = ((uint8_t *)(p)) + sizeof(Packet); \
 }
 #endif
 
@@ -627,6 +644,8 @@ typedef struct DecodeThreadVars_
 void DecodeRegisterPerfCounters(DecodeThreadVars *, ThreadVars *);
 Packet *PacketPseudoPktSetup(Packet *parent, uint8_t *pkt, uint16_t len, uint8_t proto);
 Packet *PacketGetFromQueueOrAlloc(void);
+int PacketCopyData(Packet *p, uint8_t *pktdata, int pktlen);
+int PacketCopyDataOffset(Packet *p, int offset, uint8_t *data, int datalen);
 
 DecodeThreadVars *DecodeThreadVarsAlloc();
 

src/suricata.c

@@ -812,6 +812,12 @@ int main(int argc, char **argv)
         max_pending_packets = DEFAULT_MAX_PENDING_PACKETS;
     SCLogDebug("Max pending packets set to %"PRIiMAX, max_pending_packets);
 
+    /* Pull the default packet size from the config, if not found fall
+     * back on a sane default. */
+    if (ConfGetInt("default-packet-size", &default_packet_size) != 1)
+        default_packet_size = DEFAULT_PACKET_SIZE;
+    SCLogDebug("Default packet size set to %"PRIiMAX, default_packet_size);
+
     /* Since our config is now loaded we can finish configurating the
      * logging module. */
     SCLogLoadConfig();

src/tmqh-packetpool.c

@@ -192,6 +192,11 @@ void TmqhOutputPacketpool(ThreadVars *t, Packet *p)
     /* we're done with the tunnel root now as well */
     if (proot == 1) {
         SCLogDebug("getting rid of root pkt... alloc'd %s", p->root->flags & PKT_ALLOC ? "true" : "false");
+        /* if p->root uses extended data, free them */
+        if (p->root->ext_pkt) {
+            SCFree(p->root->ext_pkt);
+            p->root->ext_pkt = NULL;
+        }
         if (p->root->flags & PKT_ALLOC) {
             PACKET_CLEANUP(p->root);
             SCFree(p->root);
@@ -202,6 +207,12 @@ void TmqhOutputPacketpool(ThreadVars *t, Packet *p)
         }
     }
 
+    /* if p uses extended data, free them */
+    if (p->ext_pkt) {
+        SCFree(p->ext_pkt);
+        p->ext_pkt = NULL;
+    }
+
     SCLogDebug("getting rid of tunnel pkt... alloc'd %s (root %p)", p->flags & PKT_ALLOC ? "true" : "false", p->root);
     if (p->flags & PKT_ALLOC) {
         PACKET_CLEANUP(p);

suricata.yaml

@@ -15,6 +15,11 @@
 # pattern matcher scans many packets in parallel.
 #max-pending-packets: 50
 
+# Preallocated size for packet. Default is 1514 which is the classical
+# size for pcap on ethernet. You should adjust this value to the highest
+# packet size (MTU + hardware header) on your system.
+#default-packet-size: 1514
+
 # Set the order of alerts bassed on actions
 # The default order is pass, drop, reject, alert
 action-order: