af-packet: Implement zero copy

This patch adds support for zero copy to AF_PACKET running mode.
This requires to use the 'worker' mode which is the only one where
the threading architecture is simple enough to permit this without
heavy modification.

src/decode.h

@@ -895,6 +895,7 @@ void AddressDebugPrint(Address *);
 #define PKT_TUNNEL_VERDICTED            0x2000
 
 #define PKT_IGNORE_CHECKSUM             0x4000    /**< Packet checksum is not computed (TX packet for example) */
+#define PKT_ZERO_COPY                   0x8000    /**< Packet comes from zero copy (ext_pkt must not be freed) */
 
 /** \brief return 1 if the packet is a pseudo packet */
 #define PKT_IS_PSEUDOPKT(p) ((p)->flags & PKT_PSEUDO_STREAM_END)

src/source-af-packet.c

@@ -65,6 +65,7 @@
 #include "util-checksum.h"
 #include "tmqh-packetpool.h"
 #include "source-af-packet.h"
+#include "runmodes.h"
 
 #ifdef HAVE_AF_PACKET
 #include <sys/ioctl.h>
@@ -390,11 +391,16 @@ int AFPReadFromRing(AFPThreadVars *ptv)
     }
 
     p->datalink = ptv->datalink;
-    /* FIXME switch to no copy */
-    SET_PKT_LEN(p, h.h2->tp_len);
-    if (PacketCopyData(p, (unsigned char*)h.raw + h.h2->tp_mac, GET_PKT_LEN(p)) == -1) {
-        TmqhOutputPacketpool(ptv->tv, p);
-        SCReturnInt(AFP_FAILURE);
+    if (ptv->flags & AFP_ZERO_COPY) {
+        if (PacketSetData(p, (unsigned char*)h.raw + h.h2->tp_mac, h.h2->tp_len) == -1) {
+            TmqhOutputPacketpool(ptv->tv, p);
+            SCReturnInt(AFP_FAILURE);
+        }
+    } else {
+        if (PacketCopyData(p, (unsigned char*)h.raw + h.h2->tp_mac, h.h2->tp_len) == -1) {
+            TmqhOutputPacketpool(ptv->tv, p);
+            SCReturnInt(AFP_FAILURE);
+        }
     }
     /* Timestamp */
     p->ts.tv_sec = h.h2->tp_sec;
@@ -419,18 +425,21 @@ int AFPReadFromRing(AFPThreadVars *ptv)
                 p->flags |= PKT_IGNORE_CHECKSUM;
         }
     }
-    /* tell kernel it can use the buffer */
-    h.h2->tp_status = TP_STATUS_KERNEL;
-    if (++ptv->frame_offset >= ptv->req.tp_frame_nr) {
-        ptv->frame_offset = 0;
-    }
-
 
     if (TmThreadsSlotProcessPkt(ptv->tv, ptv->slot, p) != TM_ECODE_OK) {
+        h.h2->tp_status = TP_STATUS_KERNEL;
+        if (++ptv->frame_offset >= ptv->req.tp_frame_nr) {
+            ptv->frame_offset = 0;
+        }
         TmqhOutputPacketpool(ptv->tv, p);
         SCReturnInt(AFP_FAILURE);
     }
 
+    h.h2->tp_status = TP_STATUS_KERNEL;
+    if (++ptv->frame_offset >= ptv->req.tp_frame_nr) {
+        ptv->frame_offset = 0;
+    }
+
     SCReturnInt(AFP_READ_OK);
 }
 
@@ -890,6 +899,13 @@ TmEcode ReceiveAFPThreadInit(ThreadVars *tv, void *initdata, void **data) {
 #endif
     ptv->flags = afpconfig->flags;
 
+    char *active_runmode = RunmodeGetActive();
+
+    if (active_runmode && !strcmp("workers", active_runmode)) {
+        ptv->flags |= AFP_ZERO_COPY;
+        SCLogInfo("Enabling zero copy mode");
+    }
+
     r = AFPCreateSocket(ptv, ptv->iface, 1);
     if (r < 0) {
         SCLogError(SC_ERR_AFP_CREATE, "Couldn't init AF_PACKET socket");

src/source-af-packet.h

@@ -40,6 +40,7 @@
 
 /* value for flags */
 #define AFP_RING_MODE (1<<0)
+#define AFP_ZERO_COPY (1<<1)
 
 #define AFP_FILE_MAX_PKTS 256
 #define AFP_IFACE_NAME_LENGTH 48

src/tmqh-packetpool.c

@@ -229,7 +229,9 @@ void TmqhOutputPacketpool(ThreadVars *t, Packet *p)
 
     /* if p uses extended data, free them */
     if (p->ext_pkt) {
-        SCFree(p->ext_pkt);
+        if (!(p->flags & PKT_ZERO_COPY)) {
+            SCFree(p->ext_pkt);
+        }
         p->ext_pkt = NULL;
     }