diff --git a/src/flow.c b/src/flow.c index 16e6160ceb..695f00a732 100644 --- a/src/flow.c +++ b/src/flow.c @@ -9,7 +9,9 @@ */ #include "suricata-common.h" +#include "suricata.h" #include "decode.h" +#include "conf.h" #include "threadvars.h" #include "tm-modules.h" #include "tm-threads.h" @@ -24,6 +26,7 @@ #include "flow-var.h" #include "flow-private.h" #include "util-unittest.h" +#include "util-byte.h" #include "util-debug.h" @@ -249,7 +252,9 @@ static int FlowUpdateSpareFlows(void) { uint32_t toalloc = 0, tofree = 0, len; SCMutexLock(&flow_spare_q.mutex_q); + len = flow_spare_q.len; + SCMutexUnlock(&flow_spare_q.mutex_q); if (len < flow_config.prealloc) { @@ -443,11 +448,36 @@ void FlowInitConfig (char quiet) flow_config.hash_size = FLOW_DEFAULT_HASHSIZE; flow_config.memcap = FLOW_DEFAULT_MEMCAP; flow_config.prealloc = FLOW_DEFAULT_PREALLOC; - /* init timeouts */ - flow_config.timeout_new = FLOW_DEFAULT_NEW_TIMEOUT; - flow_config.timeout_est = FLOW_DEFAULT_EST_TIMEOUT; - flow_config.emerg_timeout_new = FLOW_DEFAULT_EMERG_NEW_TIMEOUT; - flow_config.emerg_timeout_est = FLOW_DEFAULT_EMERG_EST_TIMEOUT; + + /* If we have specific config, overwrite the defaults with them, + * otherwise, leave the default values */ + + /* Check if we have memcap and hash_size defined at config */ + char *conf_val; + uint32_t configval = 0; + + /** set config values for memcap, prealloc and hash_size */ + if ((ConfGet("flow.memcap", &conf_val)) == 1) + { + if (ByteExtractStringUint32(&configval, 10, strlen(conf_val), + conf_val) > 0) + flow_config.memcap = configval; + } + if ((ConfGet("flow.hash_size", &conf_val)) == 1) + { + if (ByteExtractStringUint32(&configval, 10, strlen(conf_val), + conf_val) > 0) + flow_config.hash_size = configval; + } + if ((ConfGet("flow.prealloc", &conf_val)) == 1) + { + if (ByteExtractStringUint32(&configval, 10, strlen(conf_val), + conf_val) > 0) + flow_config.prealloc = configval; + } + SCLogDebug("Flow config from suricata.yaml: memcap: %"PRIu32", hash_size: " + "%"PRIu32", prealloc: %"PRIu32, flow_config.memcap, + flow_config.hash_size, flow_config.prealloc); /* alloc hash memory */ flow_hash = calloc(flow_config.hash_size, sizeof(FlowBucket)); @@ -463,8 +493,10 @@ void FlowInitConfig (char quiet) flow_config.memuse += (flow_config.hash_size * sizeof(FlowBucket)); if (quiet == FALSE) - SCLogInfo("allocated %" PRIu32 " bytes of memory for the flow hash... %" PRIu32 " buckets of size %" PRIuMAX "", - flow_config.memuse, flow_config.hash_size, (uintmax_t)sizeof(FlowBucket)); + SCLogInfo("allocated %" PRIu32 " bytes of memory for the flow hash... " + "%" PRIu32 " buckets of size %" PRIuMAX "", + flow_config.memuse, flow_config.hash_size, + (uintmax_t)sizeof(FlowBucket)); /* pre allocate flows */ for (i = 0; i < flow_config.prealloc; i++) { @@ -588,10 +620,13 @@ void *FlowManagerThread(void *td) { ThreadVars *th_v = (ThreadVars *)td; struct timeval ts; + struct timeval tsdiff; uint32_t established_cnt = 0, new_cnt = 0, closing_cnt = 0, nowcnt; uint32_t sleeping = 0; uint8_t emerg = FALSE; + memset(&tsdiff, 0, sizeof(tsdiff)); + SCLogDebug("%s started...", th_v->name); TmThreadsSetFlag(th_v, THV_INIT_DONE); @@ -603,7 +638,8 @@ void *FlowManagerThread(void *td) { /*uint32_t timeout_new = flow_config.timeout_new; uint32_t timeout_est = flow_config.timeout_est; - printf("The Timeout values are %" PRIu32" and %" PRIu32"\n", timeout_est, timeout_new);*/ + printf("The Timeout values are %" PRIu32" and %" + PRIu32"\n", timeout_est, timeout_new);*/ if (flow_flags & FLOW_EMERGENCY) { emerg = TRUE; printf("Flow emergency mode entered...\n"); @@ -657,11 +693,34 @@ void *FlowManagerThread(void *td) break; } - usleep(10); - sleeping += 10; + if (run_mode != MODE_PCAP_FILE) { + usleep(10); + sleeping += 10; + } else { + /* If we are reading a pcap, how long the pcap timestamps + * says that has passed */ + TimeGet(&tsdiff); + if (tsdiff.tv_sec == ts.tv_sec && tsdiff.tv_usec - ts.tv_usec < 10 + && tsdiff.tv_usec != ts.tv_usec) { + /* if it has passed less than 10 usec, sleep that usecs */ + sleeping += tsdiff.tv_usec - ts.tv_usec; + usleep(tsdiff.tv_usec - ts.tv_usec); + } else { + /* Else update the sleeping var but don't sleep so long */ + if (tsdiff.tv_sec == ts.tv_sec && tsdiff.tv_usec != ts.tv_usec) + sleeping += tsdiff.tv_usec - ts.tv_usec; + else if (tsdiff.tv_sec == ts.tv_sec + 1) + sleeping += tsdiff.tv_usec + (1000000 - ts.tv_usec); + else + sleeping += 100; + usleep(1); + } + } } - SCLogInfo("%" PRIu32 " new flows, %" PRIu32 " established flows were timed out, %"PRIu32" flows in closed state", new_cnt, established_cnt, closing_cnt); + SCLogInfo("%" PRIu32 " new flows, %" PRIu32 " established flows were " + "timed out, %"PRIu32" flows in closed state", new_cnt, + established_cnt, closing_cnt); pthread_exit((void *) 0); } @@ -726,6 +785,176 @@ void FlowInitFlowProto(void) { flow_proto[FLOW_PROTO_ICMP].emerg_closed_timeout = FLOW_DEFAULT_EMERG_CLOSED_TIMEOUT; flow_proto[FLOW_PROTO_ICMP].Freefunc = NULL; flow_proto[FLOW_PROTO_ICMP].GetProtoState = NULL; + + /* Let's see if we have custom timeouts defined from config */ + const char *new = NULL; + const char *established = NULL; + const char *closed = NULL; + const char *emergency_new = NULL; + const char *emergency_established = NULL; + const char *emergency_closed = NULL; + + ConfNode *flow_timeouts = ConfGetNode("flow-timeouts"); + if (flow_timeouts != NULL) { + ConfNode *proto = NULL; + uint32_t configval = 0; + + TAILQ_FOREACH(proto, &flow_timeouts->head, next) { + if (strncmp("default", proto->val, 7)) { + new = ConfNodeLookupChildValue(proto->head.tqh_first, "new"); + established = ConfNodeLookupChildValue(proto->head.tqh_first, + "established"); + closed = ConfNodeLookupChildValue(proto->head.tqh_first, + "closed"); + emergency_new = ConfNodeLookupChildValue(proto->head.tqh_first, + "emergency_new"); + emergency_established = ConfNodeLookupChildValue( + proto->head.tqh_first, + "emergency_established"); + emergency_closed = ConfNodeLookupChildValue( + proto->head.tqh_first, + "emergency_closed"); + + if (new != NULL && ByteExtractStringUint32(&configval, 10, + strlen(new), new) > 0) { + + flow_proto[FLOW_PROTO_DEFAULT].new_timeout = configval; + } + if (established != NULL && ByteExtractStringUint32(&configval, + 10, strlen(established), established) > 0) { + + flow_proto[FLOW_PROTO_DEFAULT].est_timeout = configval; + } + if (closed != NULL && ByteExtractStringUint32(&configval, 10, + strlen(closed), closed) > 0) { + + flow_proto[FLOW_PROTO_DEFAULT].closed_timeout = configval; + } + if (emergency_new != NULL && ByteExtractStringUint32(&configval, + 10, strlen(emergency_new), emergency_new) > 0) { + + flow_proto[FLOW_PROTO_DEFAULT].emerg_new_timeout = configval; + } + if (emergency_established != NULL && + ByteExtractStringUint32(&configval, 10, + strlen(emergency_established), emergency_established) > 0) { + + flow_proto[FLOW_PROTO_DEFAULT].emerg_est_timeout= configval; + } + if (emergency_closed != NULL && + ByteExtractStringUint32(&configval, 10, + strlen(emergency_closed), emergency_closed) > 0) { + + flow_proto[FLOW_PROTO_DEFAULT].emerg_closed_timeout = configval; + } + } else if (strncmp("tcp", proto->val, 3)) { + + new = ConfNodeLookupChildValue(proto->head.tqh_first, "new"); + + established = ConfNodeLookupChildValue(proto->head.tqh_first, + "established"); + closed = ConfNodeLookupChildValue(proto->head.tqh_first, + "closed"); + emergency_new = ConfNodeLookupChildValue(proto->head.tqh_first, + "emergency_new"); + emergency_established = ConfNodeLookupChildValue( + proto->head.tqh_first, "emergency_established"); + emergency_closed = ConfNodeLookupChildValue( + proto->head.tqh_first, "emergency_closed"); + + if (new != NULL && ByteExtractStringUint32(&configval, 10, + strlen(new), new) > 0) { + + flow_proto[FLOW_PROTO_TCP].new_timeout = configval; + } + if (established != NULL && ByteExtractStringUint32(&configval, + 10, strlen(established), established) > 0) { + + flow_proto[FLOW_PROTO_TCP].est_timeout = configval; + } + if (closed != NULL && ByteExtractStringUint32(&configval, 10, + strlen(closed), closed) > 0) { + + flow_proto[FLOW_PROTO_TCP].closed_timeout = configval; + } + if (emergency_new != NULL && ByteExtractStringUint32(&configval, + 10, strlen(emergency_new), emergency_new) > 0) { + flow_proto[FLOW_PROTO_TCP].emerg_new_timeout = configval; + } + if (emergency_established != NULL && + ByteExtractStringUint32(&configval, 10, + strlen(emergency_established), emergency_established) > 0) { + + flow_proto[FLOW_PROTO_TCP].emerg_est_timeout = configval; + } + if (emergency_closed != NULL && + ByteExtractStringUint32(&configval, 10, + strlen(emergency_closed), emergency_closed) > 0) { + + flow_proto[FLOW_PROTO_TCP].emerg_closed_timeout = configval; + } + } else if (strncmp("udp", proto->val, 3)) { + + new = ConfNodeLookupChildValue(proto->head.tqh_first, "new"); + established = ConfNodeLookupChildValue(proto->head.tqh_first, + "established"); + emergency_new = ConfNodeLookupChildValue(proto->head.tqh_first, + "emergency_new"); + emergency_established = ConfNodeLookupChildValue( + proto->head.tqh_first, "emergency_established"); + if (new != NULL && ByteExtractStringUint32(&configval, 10, + strlen(new), new) > 0) { + flow_proto[FLOW_PROTO_TCP].new_timeout = configval; + } + if (established != NULL && ByteExtractStringUint32(&configval, + 10, strlen(established), established) > 0) { + flow_proto[FLOW_PROTO_TCP].est_timeout = configval; + } + if (emergency_new != NULL && ByteExtractStringUint32(&configval, + 10, strlen(emergency_new), emergency_new) > 0) { + flow_proto[FLOW_PROTO_TCP].emerg_new_timeout = configval; + } + if (emergency_established != NULL && + ByteExtractStringUint32(&configval, 10, + strlen(emergency_established), emergency_established) > 0) { + flow_proto[FLOW_PROTO_TCP].emerg_est_timeout = configval; + } + } else if (strncmp("icmp", proto->val, 4)) { + new = ConfNodeLookupChildValue(proto->head.tqh_first, "new"); + established = ConfNodeLookupChildValue(proto->head.tqh_first, + "established"); + emergency_new = ConfNodeLookupChildValue(proto->head.tqh_first, + "emergency_new"); + emergency_established = ConfNodeLookupChildValue( + proto->head.tqh_first, "emergency_established"); + + if (new != NULL && ByteExtractStringUint32(&configval, 10, + strlen(new), new) > 0) { + + flow_proto[FLOW_PROTO_TCP].new_timeout = configval; + } + if (established != NULL && ByteExtractStringUint32(&configval, + 10, strlen(established), established) > 0) { + + flow_proto[FLOW_PROTO_TCP].est_timeout = configval; + } + if (emergency_new != NULL && ByteExtractStringUint32(&configval, + 10, strlen(emergency_new), emergency_new) > 0) { + + flow_proto[FLOW_PROTO_TCP].emerg_new_timeout = configval; + } + if (emergency_established != NULL && + ByteExtractStringUint32(&configval, 10, + strlen(emergency_established), emergency_established) > 0) { + + flow_proto[FLOW_PROTO_TCP].emerg_est_timeout = configval; + } + } else { + SCLogError(SC_ERR_UNKNOWN_PROTOCOL, "Unknown protocol for flow" + "timeouts. Please, review your config"); + } + } + } } /** diff --git a/src/suricata.c b/src/suricata.c index 046d32dd02..da04b65c9f 100644 --- a/src/suricata.c +++ b/src/suricata.c @@ -300,7 +300,7 @@ void usage(const char *progname) int main(int argc, char **argv) { int opt; - int mode = MODE_UNKNOWN; + int run_mode = MODE_UNKNOWN; char *pcap_file = NULL; char *pcap_dev = NULL; char *pfring_dev = NULL; @@ -343,7 +343,7 @@ int main(int argc, char **argv) switch (opt) { case 0: if(strcmp((long_opts[option_index]).name , "pfring-int") == 0){ - mode = MODE_PFRING; + run_mode = MODE_PFRING; if (ConfSet("pfring.interface", optarg, 0) != 1) { fprintf(stderr, "ERROR: Failed to set pfring interface.\n"); exit(EXIT_FAILURE); @@ -364,8 +364,8 @@ int main(int argc, char **argv) } else if(strcmp((long_opts[option_index]).name, "list-unittests") == 0) { #ifdef UNITTESTS - /* Set mode to unit tests. */ - mode = MODE_UNITTEST; + /* Set run_mode to unit tests. */ + run_mode = MODE_UNITTEST; #else fprintf(stderr, "ERROR: Unit tests not enabled. Make sure to pass --enable-unittests to configure when building.\n"); exit(EXIT_FAILURE); @@ -394,7 +394,7 @@ int main(int argc, char **argv) exit(EXIT_SUCCESS); break; case 'i': - mode = MODE_PCAP_DEV; + run_mode = MODE_PCAP_DEV; pcap_dev = optarg; break; case 'l': @@ -410,11 +410,11 @@ int main(int argc, char **argv) } break; case 'q': - mode = MODE_NFQ; + run_mode = MODE_NFQ; nfq_id = atoi(optarg); /* strtol? */ break; case 'r': - mode = MODE_PCAP_FILE; + run_mode = MODE_PCAP_FILE; pcap_file = optarg; break; case 's': @@ -422,7 +422,7 @@ int main(int argc, char **argv) break; case 'u': #ifdef UNITTESTS - mode = MODE_UNITTEST; + run_mode = MODE_UNITTEST; #else fprintf(stderr, "ERROR: Unit tests not enabled. Make sure to pass --enable-unittests to configure when building.\n"); exit(EXIT_FAILURE); @@ -448,7 +448,7 @@ int main(int argc, char **argv) SCLogInfo("This is %s version %s", PROG_NAME, PROG_VER); UtilCpuPrintSummary(); - if (!CheckValidDaemonModes(daemon, mode)) { + if (!CheckValidDaemonModes(daemon, run_mode)) { exit(EXIT_FAILURE); } @@ -458,7 +458,7 @@ int main(int argc, char **argv) /* Load yaml configuration file if provided. */ if (conf_filename != NULL) { ConfYamlLoadFile(conf_filename); - } else if (mode != MODE_UNITTEST){ + } else if (run_mode != MODE_UNITTEST){ SCLogError(SC_ERR_OPENING_FILE, "Configuration file has not been provided"); usage(argv[0]); exit(EXIT_FAILURE); @@ -484,7 +484,7 @@ int main(int argc, char **argv) * logging module. */ SCLogLoadConfig(); - if (mode == MODE_UNKNOWN) { + if (run_mode == MODE_UNKNOWN) { usage(argv[0]); exit(EXIT_FAILURE); } @@ -543,7 +543,7 @@ int main(int argc, char **argv) TmModuleDebugList(); #ifdef UNITTESTS - if (mode == MODE_UNITTEST) { + if (run_mode == MODE_UNITTEST) { /* test and initialize the unittesting subsystem */ if(regex_arg == NULL){ regex_arg = ".*"; @@ -654,26 +654,26 @@ int main(int argc, char **argv) gettimeofday(&start_time, NULL); RunModeInitializeOutputs(); - if (mode == MODE_PCAP_DEV) { + if (run_mode == MODE_PCAP_DEV) { //RunModeIdsPcap3(de_ctx, pcap_dev); RunModeIdsPcap2(de_ctx, pcap_dev); //RunModeIdsPcap(de_ctx, pcap_dev); } - else if (mode == MODE_PCAP_FILE) { + else if (run_mode == MODE_PCAP_FILE) { RunModeFilePcap(de_ctx, pcap_file); //RunModeFilePcap2(de_ctx, pcap_file); } - else if (mode == MODE_PFRING) { + else if (run_mode == MODE_PFRING) { //RunModeIdsPfring3(de_ctx, pfring_dev); //RunModeIdsPfring2(de_ctx, pfring_dev); //RunModeIdsPfring(de_ctx, pfring_dev); RunModeIdsPfring4(de_ctx, pfring_dev); } - else if (mode == MODE_NFQ) { + else if (run_mode == MODE_NFQ) { RunModeIpsNFQ(de_ctx); } else { - printf("ERROR: Unknown runtime mode.\n"); + SCLogError(SC_ERR_UNKNOWN_RUN_MODE, "Unknown runtime mode. Aborting"); exit(EXIT_FAILURE); } @@ -694,7 +694,8 @@ int main(int argc, char **argv) /* Wait till all the threads have been initialized */ if (TmThreadWaitOnThreadInit() == TM_ECODE_FAILED) { - printf("ERROR: Engine initialization failed, aborting...\n"); + SCLogError(SC_INITIALIZATION_ERROR, "Engine initialization failed, " + "aborting..."); exit(EXIT_FAILURE); } diff --git a/src/suricata.h b/src/suricata.h index 00cc8ddec8..6d0e27ea8e 100644 --- a/src/suricata.h +++ b/src/suricata.h @@ -41,6 +41,8 @@ enum { MODE_NFQ, MODE_UNITTEST }; +/* Run mode selected */ +int run_mode; /* preallocated packet structures here * XXX move to the packetpool queue handler code diff --git a/src/util-error.c b/src/util-error.c index ca948e5738..bec4056fcf 100644 --- a/src/util-error.c +++ b/src/util-error.c @@ -75,6 +75,9 @@ const char * SCErrorToString(SCError err) CASE_CODE (SC_ERR_LOGDIR_CONFIG); CASE_CODE (SC_ERR_LOGDIR_CMDLINE); CASE_CODE (SC_ERR_MISSING_QUOTE); + CASE_CODE (SC_ERR_UNKNOWN_PROTOCOL); + CASE_CODE (SC_ERR_UNKNOWN_RUN_MODE); + default: return "UNKNOWN_ERROR"; } diff --git a/src/util-error.h b/src/util-error.h index adbab75b5d..53d2a9e617 100644 --- a/src/util-error.h +++ b/src/util-error.h @@ -96,6 +96,8 @@ typedef enum { SC_ERR_MUTEX, SC_REPUTATION_INVALID_OPERATION, SC_REPUTATION_INVALID_TYPE + SC_ERR_UNKNOWN_PROTOCOL, + SC_ERR_UNKNOWN_RUN_MODE } SCError; const char *SCErrorToString(SCError); diff --git a/suricata.yaml b/suricata.yaml index 0f6d68ea8d..9180bda5f5 100644 --- a/suricata.yaml +++ b/suricata.yaml @@ -37,6 +37,63 @@ defrag: prealloc: yes timeout: 60 +# Flow settings: +# By default, the reserved memory (memcap) for flows is 32MB. This is the limit +# for flow allocation inside the engine. You can change this value to allow +# more memory usage for flows. +# The hash_size determine the size of the hash used to identify flows inside +# the engine, and by default the value is 65536. +# At the startup, the engine can preallocate a number of flows, to get a better +# performance. The number of flows preallocated is 10000 by default. + +flow: + memcap: 33554432 + hash_size: 65536 + prealloc: 10000 + +# Specific timeouts for flows. Here you can specify the timeouts that the +# active flows will wait to transit from the current state to another, on each +# protocol. The value of "new" determine the seconds to wait after a hanshake or +# stream startup before the engine free the data of that flow it doesn't +# change the state to established (usually if we don't receive more packets +# of that flow). The value of "established" is the amount of +# seconds that the engine will wait to free the flow if it spend that amount +# without receiving new packets or closing the connection. "closed" is the +# amount of time to wait after a flow is closed (usually zero). +# +# There's an emergency mode that will become active under attack circumstances, +# making the engine to check flow status faster. This configuration variables +# use the prefix "emergency_" and work similar as the normal ones. +# Some timeouts doesn't apply to all the protocols, like "closed", for udp and +# icmp. + +flow-timeouts: + + - default: + new: 30 + established: 300 + closed: 0 + emergency_new: 10 + emergency_established: 100 + emergency_closed: 0 + - tcp: + new: 30 + established: 300 + closed: 0 + emergency_new: 10 + emergency_established: 100 + emergency_closed: 0 + - udp: + new: 30 + established: 300 + emergency_new: 10 + emergency_established: 100 + - icmp: + new: 30 + established: 300 + emergency_new: 10 + emergency_established: 100 + # Logging configuration. This is not about logging IDS alerts, but # IDS output about what its doing, errors, etc. logging: