From d1d09ecfa8d96059e1b4a3c52c8dca87a2f48619 Mon Sep 17 00:00:00 2001 From: Phil Young Date: Mon, 1 Jun 2020 11:01:06 -0400 Subject: [PATCH] Napatech: Change to use separate FlowStream handle for each thread Previously a single handle to the FlowStream (which is used to program flows to the card) was shared between the threads. This resulted in contention between the threads where sometimes programming the flow would silently fail. --- src/runmode-napatech.c | 2 +- src/source-napatech.c | 69 ++++++++++++++++++++++-------- src/util-napatech.c | 96 ++++++++++++++---------------------------- src/util-napatech.h | 7 +-- 4 files changed, 89 insertions(+), 85 deletions(-) diff --git a/src/runmode-napatech.c b/src/runmode-napatech.c index 1b23cdb2a0..bb5fd61726 100644 --- a/src/runmode-napatech.c +++ b/src/runmode-napatech.c @@ -252,7 +252,7 @@ static int NapatechInit(int runmode) if (use_hw_bypass) { #ifdef NAPATECH_ENABLE_BYPASS - if (NapatechInitFlowStreams()) { + if (NapatechVerifyBypassSupport()) { SCLogInfo("Napatech Hardware Bypass is supported and enabled."); } else { SCLogError(SC_ERR_NAPATECH_PARSE_CONFIG, diff --git a/src/source-napatech.c b/src/source-napatech.c index a574f24177..e3416cbb74 100644 --- a/src/source-napatech.c +++ b/src/source-napatech.c @@ -359,6 +359,37 @@ static int CompareIPv6Addr(uint8_t addr_a[16], uint8_t addr_b[16]) { return 0; } +/** + * \brief Initializes the FlowStreams used to program flow data. + * + * Opens a FlowStream on the adapter associated with the rx port. This + * FlowStream is subsequently used to program the adapter with + * flows to bypass. + * + * \return the flow stream handle, NULL if failure. + */ +static NtFlowStream_t InitFlowStream(int adapter, int stream_id) +{ + int status; + NtFlowStream_t hFlowStream; + + NtFlowAttr_t attr; + char flow_name[80]; + + NT_FlowOpenAttrInit(&attr); + NT_FlowOpenAttrSetAdapterNo(&attr, adapter); + + snprintf(flow_name, sizeof(flow_name), "Flow_stream_%d", stream_id ); + SCLogDebug("Opening flow programming stream: %s", flow_name); + if ((status = NT_FlowOpen_Attr(&hFlowStream, flow_name, &attr)) != NT_SUCCESS) { + SCLogWarning(SC_WARN_COMPATIBILITY, + "Napatech bypass functionality not supported by the FPGA version on adapter %d - disabling support.", + adapter); + return NULL; + } + return hFlowStream; +} + /** * \brief Callback function to process Bypass events on Napatech Adapter. * @@ -373,17 +404,11 @@ static int CompareIPv6Addr(uint8_t addr_a[16], uint8_t addr_b[16]) { */ static int ProgramFlow(Packet *p, int is_inline) { - int status; NtFlow_t flow_match; memset(&flow_match, 0, sizeof(flow_match)); NapatechPacketVars *ntpv = &(p->ntpv); - int adapter = NapatechGetAdapter(ntpv->dyn3->rxPort); - - NtFlowStream_t *phFlowStream = NapatechGetFlowStreamPtr(adapter); - - /* * The hardware decoder will "color" the packets according to the protocols * in the packet and the port the packet arrived on. packet_type gets @@ -428,7 +453,6 @@ static int ProgramFlow(Packet *p, int is_inline) case RTE_PTYPE_L3_IPV4: { pIPv4_hdr = (struct ipv4_hdr *) (packet + ntpv->dyn3->offset0); - if (!is_span) { v4Tuple.sa = pIPv4_hdr->src_addr; v4Tuple.da = pIPv4_hdr->dst_addr; @@ -571,12 +595,11 @@ static int ProgramFlow(Packet *p, int is_inline) } } - status = NT_FlowWrite(*phFlowStream, &flow_match, -1); - if (status == NT_STATUS_TIMEOUT) { - SCLogInfo("NT_FlowWrite returned NT_STATUS_TIMEOUT"); - } else if (status != NT_SUCCESS) { - SCLogError(SC_ERR_NAPATECH_OPEN_FAILED,"NT_FlowWrite failed!."); - exit(EXIT_FAILURE); + if (NT_FlowWrite(ntpv->flow_stream, &flow_match, -1) != NT_SUCCESS) { + if (!(suricata_ctl_flags & SURICATA_STOP)) { + SCLogError(SC_ERR_NAPATECH_OPEN_FAILED,"NT_FlowWrite failed!."); + exit(EXIT_FAILURE); + } } return 1; @@ -781,6 +804,18 @@ TmEcode NapatechPacketLoop(ThreadVars *tv, void *data, void *slot) /* This just keeps the startup output more orderly. */ usleep(200000 * ntv->stream_id); +#ifdef NAPATECH_ENABLE_BYPASS + NtFlowStream_t flow_stream[MAX_ADAPTERS] = { 0 }; + + /* Get a FlowStream handle for each adapter so we can efficiently find the + * correct handle corresponding to the port on which a packet is received. + */ + int adapter = 0; + for (adapter = 0; adapter < NapatechGetNumAdapters(); ++adapter) { + flow_stream[adapter] = InitFlowStream(adapter, ntv->stream_id); + } +#endif + if (ConfGetBool("napatech.auto-config", &is_autoconfig) == 0) { is_autoconfig = 0; } @@ -820,7 +855,7 @@ TmEcode NapatechPacketLoop(ThreadVars *tv, void *data, void *slot) is_inline = 0; } - #ifdef NAPATECH_ENABLE_BYPASS +#ifdef NAPATECH_ENABLE_BYPASS /* Initialize the port map before we setup traffic filters */ for (int i = 0; i < MAX_PORTS; ++i) { inline_port_map[i] = -1; @@ -961,7 +996,10 @@ TmEcode NapatechPacketLoop(ThreadVars *tv, void *data, void *slot) p->ntpv.dyn3 = _NT_NET_GET_PKT_DESCR_PTR_DYN3(packet_buffer); p->BypassPacketsFlow = (NapatechIsBypassSupported() ? NapatechBypassCallback : NULL); NT_NET_SET_PKT_TXPORT(packet_buffer, inline_port_map[p->ntpv.dyn3->rxPort]); + p->ntpv.flow_stream = flow_stream[NapatechGetAdapter(p->ntpv.dyn3->rxPort)]; + #endif + p->ReleasePacket = NapatechReleasePacket; p->ntpv.nt_packet_buf = packet_buffer; p->ntpv.stream_id = ntv->stream_id; @@ -987,9 +1025,6 @@ TmEcode NapatechPacketLoop(ThreadVars *tv, void *data, void *slot) } // while if (closer) { -#ifdef NAPATECH_ENABLE_BYPASS - NapatechCloseFlowStreams(); -#endif NapatechDeleteFilters(); } diff --git a/src/util-napatech.c b/src/util-napatech.c index ddd5774aba..710131b14c 100644 --- a/src/util-napatech.c +++ b/src/util-napatech.c @@ -46,7 +46,6 @@ typedef struct FlowStatsCounters_ uint16_t total_bypass_flows; } FlowStatsCounters; -static NtFlowStream_t hFlowStream[MAX_ADAPTERS]; static int bypass_supported; int NapatechIsBypassSupported(void) @@ -59,45 +58,48 @@ int NapatechIsBypassSupported(void) * * \return count of the Napatech adapters present in the system. */ -static int GetNumAdapters(void) +int NapatechGetNumAdapters(void) { NtInfoStream_t hInfo; NtInfo_t hInfoSys; int status; + static int num_adapters = -1; - if ((status = NT_InfoOpen(&hInfo, "InfoStream")) != NT_SUCCESS) { - NAPATECH_ERROR(SC_ERR_NAPATECH_OPEN_FAILED, status); - exit(EXIT_FAILURE); - } + if (num_adapters == -1) { + if ((status = NT_InfoOpen(&hInfo, "InfoStream")) != NT_SUCCESS) { + NAPATECH_ERROR(SC_ERR_NAPATECH_OPEN_FAILED, status); + exit(EXIT_FAILURE); + } - hInfoSys.cmd = NT_INFO_CMD_READ_SYSTEM; - if ((status = NT_InfoRead(hInfo, &hInfoSys)) != NT_SUCCESS) { - NAPATECH_ERROR(SC_ERR_NAPATECH_OPEN_FAILED, status); - exit(EXIT_FAILURE); - } + hInfoSys.cmd = NT_INFO_CMD_READ_SYSTEM; + if ((status = NT_InfoRead(hInfo, &hInfoSys)) != NT_SUCCESS) { + NAPATECH_ERROR(SC_ERR_NAPATECH_OPEN_FAILED, status); + exit(EXIT_FAILURE); + } - int num_adapters = hInfoSys.u.system.data.numAdapters; + num_adapters = hInfoSys.u.system.data.numAdapters; + + NT_InfoClose(hInfo); + } - NT_InfoClose(hInfo); return num_adapters; } /** - * \brief Initializes the FlowStreams used to program flow data. + * \brief Verifies that the Napatech adapters support bypass. * - * Opens a FlowStream on each adapter present in the system. This - * FlowStream is subsequently used to program the adapter with - * flows to bypass. + * Attempts to opens a FlowStream on each adapter present in the system. + * If successful then bypass is supported * * \return 1 if Bypass functionality is supported; zero otherwise. */ -int NapatechInitFlowStreams(void) +int NapatechVerifyBypassSupport(void) { int status; int adapter = 0; - int num_adapters = GetNumAdapters(); - SCLogInfo("Found %d Napatech adapters.\n", num_adapters); - memset(&hFlowStream, 0, sizeof(hFlowStream)); + int num_adapters = NapatechGetNumAdapters(); + SCLogInfo("Found %d Napatech adapters.", num_adapters); + NtFlowStream_t hFlowStream; if (!NapatechUseHWBypass()) { /* HW Bypass is disabled in the conf file */ @@ -113,50 +115,18 @@ int NapatechInitFlowStreams(void) snprintf(flow_name, sizeof(flow_name), "Flow stream %d", adapter ); SCLogInfo("Opening flow programming stream: %s\n", flow_name); - if ((status = NT_FlowOpen_Attr(&hFlowStream[adapter], flow_name, &attr)) != NT_SUCCESS) { + if ((status = NT_FlowOpen_Attr(&hFlowStream, flow_name, &attr)) != NT_SUCCESS) { SCLogWarning(SC_WARN_COMPATIBILITY, "Napatech bypass functionality not supported by the FPGA version on adapter %d - disabling support.", adapter); bypass_supported = 0; return 0; } + NT_FlowClose(hFlowStream); } bypass_supported = 1; return bypass_supported; } -/** - * \brief Returns a pointer to the FlowStream associated with this adapter. - * - * \return count of the Napatech adapters present in the system. - */ -NtFlowStream_t *NapatechGetFlowStreamPtr(int device) -{ - return &hFlowStream[device]; -} - -/** - * \brief Closes all open FlowStreams - * - * \return Success of the operation. - */ -int NapatechCloseFlowStreams(void) -{ - int status = 0; - int adapter = 0; - int num_adapters = GetNumAdapters(); - - for (adapter = 0; adapter < num_adapters; ++adapter) { - if (hFlowStream[adapter]) { - SCLogInfo("Closing Napatech Flow Stream on adapter %d.", adapter); - if ((status = NT_FlowClose(hFlowStream[adapter])) != NT_SUCCESS) { - NAPATECH_ERROR(SC_ERR_SHUTDOWN, status); - } - hFlowStream[adapter] = NULL; - } - } - return (status == NT_SUCCESS); -} - /** * \brief Updates statistic counters for Napatech FlowStats @@ -183,7 +153,7 @@ static void UpdateFlowStats( uint64_t removed = 0; int adapter = 0; - for (adapter = 0; adapter < GetNumAdapters(); ++adapter) { + for (adapter = 0; adapter < NapatechGetNumAdapters(); ++adapter) { hStat.cmd = NT_STATISTICS_READ_CMD_FLOW_V0; hStat.u.flowData_v0.clear = clear_stats; hStat.u.flowData_v0.adapterNo = adapter; @@ -309,8 +279,7 @@ static uint32_t UpdateStreamStats(ThreadVars *tv, int is_inline, int enable_stream_stats, PacketCounters stream_counters[] - ) -{ + ) { static uint64_t rxPktsStart[MAX_STREAMS] = {0}; static uint64_t rxByteStart[MAX_STREAMS] = {0}; static uint64_t dropPktStart[MAX_STREAMS] = {0}; @@ -383,7 +352,7 @@ static uint32_t UpdateStreamStats(ThreadVars *tv, current_stats[stream_id].current_bytes = rx_byte_total - rxByteStart[stream_id]; current_stats[stream_id].current_drop_packets = drop_pkts_total - dropPktStart[stream_id]; current_stats[stream_id].current_drop_bytes = drop_byte_total - dropByteStart[stream_id]; - } + } if (enable_stream_stats) { StatsSetUI64(tv, stream_counters[inst_id].pkts, current_stats[stream_id].current_packets); @@ -420,7 +389,6 @@ static uint32_t UpdateStreamStats(ThreadVars *tv, total_stats.current_drop_packets = 0; total_stats.current_drop_bytes = 0; - /* Read usage data for the chosen stream ID */ memset(&hStat, 0, sizeof (NtStatistics_t)); @@ -453,7 +421,7 @@ static uint32_t UpdateStreamStats(ThreadVars *tv, uint64_t total_dispatch_fwd_pkts = 0; uint64_t total_dispatch_fwd_byte = 0; - for (adapter = 0; adapter < GetNumAdapters(); ++adapter) { + for (adapter = 0; adapter < NapatechGetNumAdapters(); ++adapter) { total_dispatch_host_pkts += hStat.u.query_v3.data.adapter.aAdapters[adapter].color.aColor[0].pkts; total_dispatch_host_byte += hStat.u.query_v3.data.adapter.aAdapters[adapter].color.aColor[0].octets; @@ -468,9 +436,9 @@ static uint32_t UpdateStreamStats(ThreadVars *tv, + hStat.u.query_v3.data.adapter.aAdapters[adapter].color.aColor[4].octets; total_stats.current_packets += hStat.u.query_v3.data.adapter.aAdapters[adapter].color.aColor[0].pkts - + hStat.u.query_v3.data.adapter.aAdapters[adapter].color.aColor[1].pkts - + hStat.u.query_v3.data.adapter.aAdapters[adapter].color.aColor[2].pkts - + hStat.u.query_v3.data.adapter.aAdapters[adapter].color.aColor[3].pkts; + + hStat.u.query_v3.data.adapter.aAdapters[adapter].color.aColor[1].pkts + + hStat.u.query_v3.data.adapter.aAdapters[adapter].color.aColor[2].pkts + + hStat.u.query_v3.data.adapter.aAdapters[adapter].color.aColor[3].pkts; total_stats.current_bytes = hStat.u.query_v3.data.adapter.aAdapters[adapter].color.aColor[0].octets + hStat.u.query_v3.data.adapter.aAdapters[adapter].color.aColor[1].octets diff --git a/src/util-napatech.h b/src/util-napatech.h index 4f518b81cc..c51f13582e 100644 --- a/src/util-napatech.h +++ b/src/util-napatech.h @@ -31,6 +31,7 @@ typedef struct NapatechPacketVars_ uint64_t stream_id; NtNetBuf_t nt_packet_buf; NtNetStreamRx_t rx_stream; + NtFlowStream_t flow_stream; ThreadVars *tv; #ifdef NAPATECH_ENABLE_BYPASS NtDyn3Descr_t *dyn3; @@ -112,9 +113,9 @@ uint32_t NapatechDeleteFilters(void); #define NAPATECH_FLOWTYPE_DROP 7 #define NAPATECH_FLOWTYPE_PASS 8 -int NapatechInitFlowStreams(void); -NtFlowStream_t *NapatechGetFlowStreamPtr(int device); -int NapatechCloseFlowStreams(void); +int NapatechVerifyBypassSupport(void); +int NapatechGetNumAdapters(void); + int NapatechIsBypassSupported(void);