From e30b1bfe64c94ddca9297a549fa02083940c2e60 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Fri, 2 Nov 2012 17:44:21 +0100 Subject: [PATCH] Simple IP reputation implementation --- src/Makefile.am | 1 + src/decode-ipv4.c | 1 + src/decode.h | 51 ++-- src/detect-engine-iponly.c | 9 +- src/detect-engine.c | 7 + src/detect-iprep.c | 388 +++++++++++++++++++++++++++++ src/detect-iprep.h | 46 ++++ src/detect.c | 2 + src/detect.h | 4 + src/flow-timeout.c | 1 + src/host-timeout.c | 9 + src/host.c | 13 +- src/host.h | 21 ++ src/reputation.c | 496 ++++++++++++++++++++++++++++++++++++- src/reputation.h | 14 ++ src/tmqh-packetpool.c | 1 + src/util-error.c | 1 + src/util-error.h | 1 + suricata.yaml.in | 5 + 19 files changed, 1037 insertions(+), 34 deletions(-) create mode 100644 src/detect-iprep.c create mode 100644 src/detect-iprep.h diff --git a/src/Makefile.am b/src/Makefile.am index fe48c72f29..66185b3e0e 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -137,6 +137,7 @@ detect-icode.c detect-icode.h \ detect-id.c detect-id.h \ detect-ipopts.c detect-ipopts.h \ detect-ipproto.c detect-ipproto.h \ +detect-iprep.c detect-iprep.h \ detect-isdataat.c detect-isdataat.h \ detect-itype.c detect-itype.h \ detect-l3proto.c detect-l3proto.h \ diff --git a/src/decode-ipv4.c b/src/decode-ipv4.c index 7c60bb974a..6477e2f05a 100644 --- a/src/decode-ipv4.c +++ b/src/decode-ipv4.c @@ -38,6 +38,7 @@ #include "decode-events.h" #include "defrag.h" #include "pkt-var.h" +#include "host.h" #include "util-unittest.h" #include "util-debug.h" diff --git a/src/decode.h b/src/decode.h index 68333e4605..04113c59cc 100644 --- a/src/decode.h +++ b/src/decode.h @@ -368,7 +368,8 @@ typedef struct Packet_ uint8_t recursion_level; /* Pkt Flags */ - uint16_t flags; + uint32_t flags; + /* flow */ uint8_t flowflags; @@ -457,6 +458,9 @@ typedef struct Packet_ PacketAlerts alerts; + struct Host_ *host_src; + struct Host_ *host_dst; + /** packet number in the pcap file, matches wireshark */ uint64_t pcap_cnt; @@ -662,8 +666,8 @@ typedef struct DecodeThreadVars_ if ((p)->udph != NULL) { \ CLEAR_UDP_PACKET((p)); \ } \ - if ((p)->sctph != NULL) { \ - CLEAR_SCTP_PACKET((p)); \ + if ((p)->sctph != NULL) { \ + CLEAR_SCTP_PACKET((p)); \ } \ if ((p)->icmpv4h != NULL) { \ CLEAR_ICMPV4_PACKET((p)); \ @@ -680,6 +684,8 @@ typedef struct DecodeThreadVars_ (p)->payload_len = 0; \ (p)->pktlen = 0; \ (p)->alerts.cnt = 0; \ + HostDeReference(&((p)->host_src)); \ + HostDeReference(&((p)->host_dst)); \ (p)->pcap_cnt = 0; \ (p)->tunnel_rtv_cnt = 0; \ (p)->tunnel_tpr_cnt = 0; \ @@ -915,24 +921,27 @@ void AddressDebugPrint(Address *); #define VLAN_OVER_GRE 13 /*Packet Flags*/ -#define PKT_NOPACKET_INSPECTION 0x0001 /**< Flag to indicate that packet header or contents should not be inspected*/ -#define PKT_NOPAYLOAD_INSPECTION 0x0002 /**< Flag to indicate that packet contents should not be inspected*/ -#define PKT_ALLOC 0x0004 /**< Packet was alloc'd this run, needs to be freed */ -#define PKT_HAS_TAG 0x0008 /**< Packet has matched a tag */ -#define PKT_STREAM_ADD 0x0010 /**< Packet payload was added to reassembled stream */ -#define PKT_STREAM_EST 0x0020 /**< Packet is part of establised stream */ -#define PKT_STREAM_EOF 0x0040 /**< Stream is in eof state */ -#define PKT_HAS_FLOW 0x0080 -#define PKT_PSEUDO_STREAM_END 0x0100 /**< Pseudo packet to end the stream */ -#define PKT_STREAM_MODIFIED 0x0200 /**< Packet is modified by the stream engine, we need to recalc the csum and reinject/replace */ -#define PKT_MARK_MODIFIED 0x0400 /**< Packet mark is modified */ -#define PKT_STREAM_NOPCAPLOG 0x0800 /**< Exclude packet from pcap logging as it's part of a stream that has reassembly depth reached. */ - -#define PKT_TUNNEL 0x1000 -#define PKT_TUNNEL_VERDICTED 0x2000 - -#define PKT_IGNORE_CHECKSUM 0x4000 /**< Packet checksum is not computed (TX packet for example) */ -#define PKT_ZERO_COPY 0x8000 /**< Packet comes from zero copy (ext_pkt must not be freed) */ +#define PKT_NOPACKET_INSPECTION (1) /**< Flag to indicate that packet header or contents should not be inspected*/ +#define PKT_NOPAYLOAD_INSPECTION (1<<2) /**< Flag to indicate that packet contents should not be inspected*/ +#define PKT_ALLOC (1<<3) /**< Packet was alloc'd this run, needs to be freed */ +#define PKT_HAS_TAG (1<<4) /**< Packet has matched a tag */ +#define PKT_STREAM_ADD (1<<5) /**< Packet payload was added to reassembled stream */ +#define PKT_STREAM_EST (1<<6) /**< Packet is part of establised stream */ +#define PKT_STREAM_EOF (1<<7) /**< Stream is in eof state */ +#define PKT_HAS_FLOW (1<<8) +#define PKT_PSEUDO_STREAM_END (1<<9) /**< Pseudo packet to end the stream */ +#define PKT_STREAM_MODIFIED (1<<10) /**< Packet is modified by the stream engine, we need to recalc the csum and reinject/replace */ +#define PKT_MARK_MODIFIED (1<<11) /**< Packet mark is modified */ +#define PKT_STREAM_NOPCAPLOG (1<<12) /**< Exclude packet from pcap logging as it's part of a stream that has reassembly depth reached. */ + +#define PKT_TUNNEL (1<<13) +#define PKT_TUNNEL_VERDICTED (1<<14) + +#define PKT_IGNORE_CHECKSUM (1<<15) /**< Packet checksum is not computed (TX packet for example) */ +#define PKT_ZERO_COPY (1<<16) /**< Packet comes from zero copy (ext_pkt must not be freed) */ + +#define PKT_HOST_SRC_LOOKED_UP (1<<17) +#define PKT_HOST_DST_LOOKED_UP (1<<18) /** \brief return 1 if the packet is a pseudo packet */ #define PKT_IS_PSEUDOPKT(p) ((p)->flags & PKT_PSEUDO_STREAM_END) diff --git a/src/detect-engine-iponly.c b/src/detect-engine-iponly.c index 903edc504a..093c954827 100644 --- a/src/detect-engine-iponly.c +++ b/src/detect-engine-iponly.c @@ -927,16 +927,11 @@ int IPOnlyMatchCompatSMs(ThreadVars *tv, Signature *s, Packet *p) { SigMatch *sm = s->sm_lists[DETECT_SM_LIST_MATCH]; - int match; while (sm != NULL) { - if (sm->type != DETECT_FLOWBITS) { - sm = sm->next; - continue; - } + BUG_ON(!(sigmatch_table[sm->type].flags & SIGMATCH_IPONLY_COMPAT)); - match = sigmatch_table[sm->type].Match(tv, det_ctx, p, s, sm); - if (match > 0) { + if (sigmatch_table[sm->type].Match(tv, det_ctx, p, s, sm) > 0) { sm = sm->next; continue; } diff --git a/src/detect-engine.c b/src/detect-engine.c index ec80abc9e0..03ea33ae95 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -84,6 +84,8 @@ #include "util-profiling.h" #endif +#include "reputation.h" + #define DETECT_ENGINE_DEFAULT_INSPECTION_RECURSION_LIMIT 3000 static uint32_t detect_engine_ctx_id = 1; @@ -541,6 +543,8 @@ static void *DetectEngineLiveRuleSwap(void *arg) } DetectEngineCtxFree(old_de_ctx); + SRepReloadComplete(); + /* reset the handler */ UtilSignalHandlerSetup(SIGUSR2, SignalHandlerSigusr2); @@ -678,6 +682,9 @@ DetectEngineCtx *DetectEngineCtxInit(void) { de_ctx->id = detect_engine_ctx_id++; + /* init iprep... ignore errors for now */ + (void)SRepInit(de_ctx); + return de_ctx; error: return NULL; diff --git a/src/detect-iprep.c b/src/detect-iprep.c new file mode 100644 index 0000000000..9e537b5a31 --- /dev/null +++ b/src/detect-iprep.c @@ -0,0 +1,388 @@ +/* Copyright (C) 2012 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + * + * Implements the iprep keyword + */ + +#include "suricata-common.h" +#include "decode.h" +#include "detect.h" +#include "threads.h" +#include "flow.h" +#include "flow-bit.h" +#include "flow-util.h" +#include "detect-iprep.h" +#include "util-spm.h" + +#include "app-layer-parser.h" + +#include "detect-parse.h" +#include "detect-engine.h" +#include "detect-engine-mpm.h" +#include "detect-engine-state.h" + +#include "util-debug.h" + +#include "reputation.h" +#include "host.h" + +#define PARSE_REGEX "\\s*(any|src|dst|both)\\s*,\\s*([A-Za-z0-9\\-\\_]+)\\s*,\\s*(\\<|\\>|\\=)\\s*,\\s*([0-9]+)\\s*" +static pcre *parse_regex; +static pcre_extra *parse_regex_study; + +int DetectIPRepMatch (ThreadVars *, DetectEngineThreadCtx *, Packet *, Signature *, SigMatch *); +static int DetectIPRepSetup (DetectEngineCtx *, Signature *, char *); +void DetectIPRepFree (void *); +void IPRepRegisterTests(void); + +void DetectIPRepRegister (void) { + sigmatch_table[DETECT_IPREP].name = "iprep"; + sigmatch_table[DETECT_IPREP].Match = DetectIPRepMatch; + sigmatch_table[DETECT_IPREP].Setup = DetectIPRepSetup; + sigmatch_table[DETECT_IPREP].Free = DetectIPRepFree; + sigmatch_table[DETECT_IPREP].RegisterTests = IPRepRegisterTests; + /* this is compatible to ip-only signatures */ + sigmatch_table[DETECT_IPREP].flags |= SIGMATCH_IPONLY_COMPAT; + + const char *eb; + int eo; + int opts = 0; + + parse_regex = pcre_compile(PARSE_REGEX, opts, &eb, &eo, NULL); + if(parse_regex == NULL) + { + SCLogError(SC_ERR_PCRE_COMPILE, "pcre compile of \"%s\" failed at offset %" PRId32 ": %s", PARSE_REGEX, eo, eb); + goto error; + } + + parse_regex_study = pcre_study(parse_regex, 0, &eb); + if(eb != NULL) + { + SCLogError(SC_ERR_PCRE_STUDY, "pcre study failed: %s", eb); + goto error; + } + + return; + +error: + return; +} + +static uint8_t GetHostRepSrc(Packet *p, uint8_t cat, uint32_t version) { + uint8_t val = 0; + Host *h = NULL; + + if (p->flags & PKT_HOST_SRC_LOOKED_UP && p->host_src == NULL) { + return 0; + } else if (p->host_src != NULL) { + h = (Host *)p->host_src; + HostLock(h); + } else { + h = HostLookupHostFromHash(&(p->src)); + + p->flags |= PKT_HOST_SRC_LOOKED_UP; + + if (h == NULL) + return 0; + + HostReference(&p->host_src, h); + } + + if (h->iprep == NULL) { + HostRelease(h); + return 0; + } + + SReputation *r = (SReputation *)h->iprep; + + /* allow higher versions as this happens during + * rule reload */ + if (r->version >= version) + val = r->rep[cat]; + else + SCLogDebug("version mismatch %u != %u", r->version, version); + + HostRelease(h); + return val; +} + +static uint8_t GetHostRepDst(Packet *p, uint8_t cat, uint32_t version) { + uint8_t val = 0; + Host *h = NULL; + + if (p->flags & PKT_HOST_DST_LOOKED_UP && p->host_dst == NULL) { + return 0; + } else if (p->host_dst != NULL) { + h = (Host *)p->host_dst; + HostLock(h); + } else { + h = HostLookupHostFromHash(&(p->dst)); + + p->flags |= PKT_HOST_DST_LOOKED_UP; + + if (h == NULL) { + return 0; + } + + HostReference(&p->host_dst, h); + } + + if (h->iprep == NULL) { + HostRelease(h); + return 0; + } + + SReputation *r = (SReputation *)h->iprep; + + /* allow higher versions as this happens during + * rule reload */ + if (r->version >= version) + val = r->rep[cat]; + else + SCLogDebug("version mismatch %u != %u", r->version, version); + + HostRelease(h); + return val; +} + +static inline int RepMatch(uint8_t op, uint8_t val1, uint8_t val2) { + if (op == DETECT_IPREP_OP_GT && val1 > val2) { + return 1; + } else if (op == DETECT_IPREP_OP_LT && val1 < val2) { + return 1; + } else if (op == DETECT_IPREP_OP_EQ && val1 == val2) { + return 1; + } + return 0; +} + +/* + * returns 0: no match + * 1: match + * -1: error + */ +int DetectIPRepMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, Packet *p, Signature *s, SigMatch *m) +{ + DetectIPRepData *rd = (DetectIPRepData *)m->ctx; + if (rd == NULL) + return 0; + + uint32_t version = det_ctx->de_ctx->srep_version; + uint8_t val = 0; + + SCLogDebug("rd->cmd %u", rd->cmd); + switch(rd->cmd) { + case DETECT_IPREP_CMD_ANY: + val = GetHostRepSrc(p, rd->cat, version); + if (val > 0) { + if (RepMatch(rd->op, val, rd->val) == 1) + return 1; + } + val = GetHostRepDst(p, rd->cat, version); + if (val > 0) { + return RepMatch(rd->op, val, rd->val); + } + break; + + case DETECT_IPREP_CMD_SRC: + SCLogDebug("checking src"); + val = GetHostRepSrc(p, rd->cat, version); + if (val > 0) { + return RepMatch(rd->op, val, rd->val); + } + break; + + case DETECT_IPREP_CMD_DST: + SCLogDebug("checking dst"); + val = GetHostRepDst(p, rd->cat, version); + if (val > 0) { + return RepMatch(rd->op, val, rd->val); + } + break; + + case DETECT_IPREP_CMD_BOTH: + val = GetHostRepSrc(p, rd->cat, version); + if (val == 0 || RepMatch(rd->op, val, rd->val) == 0) + return 0; + val = GetHostRepDst(p, rd->cat, version); + if (val > 0) { + return RepMatch(rd->op, val, rd->val); + } + break; + } + + return 0; +} + +int DetectIPRepSetup (DetectEngineCtx *de_ctx, Signature *s, char *rawstr) +{ + DetectIPRepData *cd = NULL; + SigMatch *sm = NULL; + char *cmd_str = NULL, *name = NULL, *op_str = NULL, *value = NULL; + uint8_t cmd = 0; +#define MAX_SUBSTRINGS 30 + int ret = 0, res = 0; + int ov[MAX_SUBSTRINGS]; + + ret = pcre_exec(parse_regex, parse_regex_study, rawstr, strlen(rawstr), 0, 0, ov, MAX_SUBSTRINGS); + if (ret != 5) { + SCLogError(SC_ERR_PCRE_MATCH, "\"%s\" is not a valid setting for iprep", rawstr); + return -1; + } + + const char *str_ptr; + res = pcre_get_substring((char *)rawstr, ov, MAX_SUBSTRINGS, 1, &str_ptr); + if (res < 0) { + SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed"); + return -1; + } + cmd_str = (char *)str_ptr; + + res = pcre_get_substring((char *)rawstr, ov, MAX_SUBSTRINGS, 2, &str_ptr); + if (res < 0) { + SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed"); + goto error; + } + name = (char *)str_ptr; + + res = pcre_get_substring((char *)rawstr, ov, MAX_SUBSTRINGS, 3, &str_ptr); + if (res < 0) { + SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed"); + goto error; + } + op_str = (char *)str_ptr; + + res = pcre_get_substring((char *)rawstr, ov, MAX_SUBSTRINGS, 4, &str_ptr); + if (res < 0) { + SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed"); + goto error; + } + value = (char *)str_ptr; + + if (strcmp(cmd_str,"any") == 0) { + cmd = DETECT_IPREP_CMD_ANY; + } else if (strcmp(cmd_str,"both") == 0) { + cmd = DETECT_IPREP_CMD_BOTH; + } else if (strcmp(cmd_str,"src") == 0) { + cmd = DETECT_IPREP_CMD_SRC; + } else if (strcmp(cmd_str,"dst") == 0) { + cmd = DETECT_IPREP_CMD_DST; + } else { + SCLogError(SC_ERR_UNKNOWN_VALUE, "ERROR: iprep \"%s\" is not supported.", cmd_str); + goto error; + } + + //SCLogInfo("category %s", name); + uint8_t cat = SRepCatGetByShortname(name); + if (cat == 0) { + SCLogError(SC_ERR_UNKNOWN_VALUE, "unknown iprep category \"%s\"", name); + goto error; + } + + uint8_t op = 0; + uint8_t val = 0; + + if (op_str == NULL || strlen(op_str) != 1) { + goto error; + } + + switch(op_str[0]) { + case '<': + op = DETECT_IPREP_OP_LT; + break; + case '>': + op = DETECT_IPREP_OP_GT; + break; + case '=': + op = DETECT_IPREP_OP_EQ; + break; + default: + goto error; + break; + } + + if (value != NULL && strlen(value) > 0) { + int ival = atoi(value); + if (ival < 0 || ival > 127) + goto error; + val = (uint8_t)ival; + } + + cd = SCMalloc(sizeof(DetectIPRepData)); + if (unlikely(cd == NULL)) + goto error; + + cd->cmd = cmd; + cd->cat = cat; + cd->op = op; + cd->val = val; + //SCLogInfo("cmd %u, cat %u, op %u, val %u", cd->cmd, cd->cat, cd->op, cd->val); + + pcre_free_substring(name); + name = NULL; + pcre_free_substring(cmd_str); + cmd_str = NULL; + + /* Okay so far so good, lets get this into a SigMatch + * and put it in the Signature. */ + sm = SigMatchAlloc(); + if (sm == NULL) + goto error; + + sm->type = DETECT_IPREP; + sm->ctx = (void *)cd; + + SigMatchAppendSMToList(s, sm, DETECT_SM_LIST_MATCH); + + return 0; + +error: + if (name != NULL) + pcre_free_substring(name); + if (cmd_str != NULL) + pcre_free_substring(cmd_str); + if (cd != NULL) + SCFree(cd); + if (sm != NULL) + SCFree(sm); + return -1; +} + +void DetectIPRepFree (void *ptr) { + DetectIPRepData *fd = (DetectIPRepData *)ptr; + + if (fd == NULL) + return; + + SCFree(fd); +} + +#ifdef UNITTESTS +#endif /* UNITTESTS */ + +/** + * \brief this function registers unit tests for IPRep + */ +void IPRepRegisterTests(void) { +#ifdef UNITTESTS +#endif /* UNITTESTS */ +} diff --git a/src/detect-iprep.h b/src/detect-iprep.h new file mode 100644 index 0000000000..129851b900 --- /dev/null +++ b/src/detect-iprep.h @@ -0,0 +1,46 @@ +/* Copyright (C) 2012 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + */ + +#ifndef __DETECT_IPREP_H__ +#define __DETECT_IPREP_H__ + +#define DETECT_IPREP_CMD_ANY 0 +#define DETECT_IPREP_CMD_BOTH 1 +#define DETECT_IPREP_CMD_SRC 2 +#define DETECT_IPREP_CMD_DST 3 + +#define DETECT_IPREP_OP_LT 0 +#define DETECT_IPREP_OP_GT 1 +#define DETECT_IPREP_OP_EQ 2 + +typedef struct DetectIPRepData_ { + uint8_t cmd; + int8_t cat; + int8_t op; + uint8_t val; +} DetectIPRepData; + +/* prototypes */ +void DetectIPRepRegister (void); + +#endif /* __DETECT_IPREP_H__ */ diff --git a/src/detect.c b/src/detect.c index 1833e7432e..928bc31f6d 100644 --- a/src/detect.c +++ b/src/detect.c @@ -143,6 +143,7 @@ #include "detect-tos.h" #include "detect-app-layer-event.h" #include "detect-luajit.h" +#include "detect-iprep.h" #include "util-rule-vars.h" @@ -4666,6 +4667,7 @@ void SigTableSetup(void) { DetectAppLayerEventRegister(); DetectHttpUARegister(); DetectLuajitRegister(); + DetectIPRepRegister(); uint8_t i = 0; for (i = 0; i < DETECT_TBLSIZE; i++) { diff --git a/src/detect.h b/src/detect.h index 9b32954e8f..8af730fde9 100644 --- a/src/detect.h +++ b/src/detect.h @@ -546,6 +546,9 @@ typedef struct DetectEngineCtx_ { Signature *sig_list; uint32_t sig_cnt; + /* version of the srep data */ + uint32_t srep_version; + Signature **sig_array; uint32_t sig_array_size; /* size in bytes */ uint32_t sig_array_len; /* size in array members */ @@ -1087,6 +1090,7 @@ enum { DETECT_L3PROTO, DETECT_LUAJIT, + DETECT_IPREP, /* make sure this stays last */ DETECT_TBLSIZE, diff --git a/src/flow-timeout.c b/src/flow-timeout.c index ed4b59ecca..7b735e96e2 100644 --- a/src/flow-timeout.c +++ b/src/flow-timeout.c @@ -40,6 +40,7 @@ #include "flow-private.h" #include "flow-manager.h" #include "pkt-var.h" +#include "host.h" #include "stream-tcp-private.h" #include "stream-tcp-reassemble.h" diff --git a/src/host-timeout.c b/src/host-timeout.c index a544f10b68..3cedb80beb 100644 --- a/src/host-timeout.c +++ b/src/host-timeout.c @@ -26,6 +26,7 @@ #include "detect-engine-tag.h" #include "detect-engine-threshold.h" +#include "reputation.h" uint32_t HostGetSpareCount(void) { return HostSpareQueueGetSize(); @@ -54,6 +55,13 @@ static int HostHostTimedOut(Host *h, struct timeval *ts) { return 0; } + if (h->iprep) { + if (SRepHostTimedOut(h) == 0) + return 0; + + SCLogDebug("host %p reputation timed out", h); + } + if (h->tag && TagTimeoutCheck(h, ts) == 0) { tags = 1; } @@ -64,6 +72,7 @@ static int HostHostTimedOut(Host *h, struct timeval *ts) { if (tags || thresholds) return 0; + SCLogDebug("host %p timed out", h); return 1; } diff --git a/src/host.c b/src/host.c index df210d4b7a..76854660f7 100644 --- a/src/host.c +++ b/src/host.c @@ -108,6 +108,10 @@ void HostClearMemory(Host *h) { ThresholdListFree(h->threshold); h->threshold = NULL; } + if (h->iprep != NULL) { + SCFree(h->iprep); + h->iprep = NULL; + } SC_ATOMIC_DESTROY(h->use_cnt); } @@ -362,11 +366,6 @@ static Host *HostGetNew(Address *a) { return h; } -#define HostIncrUsecnt(h) \ - SC_ATOMIC_ADD((h)->use_cnt, 1) -#define HostDecrUsecnt(h) \ - SC_ATOMIC_SUB((h)->use_cnt, 1) - void HostInit(Host *h, Address *a) { COPY_ADDRESS(a, &h->a); (void) HostIncrUsecnt(h); @@ -377,6 +376,10 @@ void HostRelease(Host *h) { SCMutexUnlock(&h->m); } +void HostLock(Host *h) { + SCMutexLock(&h->m); +} + /* HostGetHostFromHash * * Hash retrieval function for hosts. Looks up the hash bucket containing the diff --git a/src/host.h b/src/host.h index a4634bf499..c25a42e96a 100644 --- a/src/host.h +++ b/src/host.h @@ -67,6 +67,7 @@ typedef struct Host_ { /** pointers to tag and threshold storage */ void *tag; void *threshold; + void *iprep; /** hash pointers, protected by hash row mutex/spin */ struct Host_ *hnext; @@ -106,6 +107,25 @@ typedef struct HostConfig_ { #define HOST_CHECK_MEMCAP(size) \ ((((uint64_t)SC_ATOMIC_GET(host_memuse) + (uint64_t)(size)) <= host_config.memcap)) +#define HostIncrUsecnt(h) \ + SC_ATOMIC_ADD((h)->use_cnt, 1) +#define HostDecrUsecnt(h) \ + SC_ATOMIC_SUB((h)->use_cnt, 1) + +#define HostReference(dst_h_ptr, h) do { \ + if ((h) != NULL) { \ + HostIncrUsecnt((h)); \ + *(dst_h_ptr) = h; \ + } \ + } while (0) + +#define HostDeReference(src_h_ptr) do { \ + if (*(src_h_ptr) != NULL) { \ + HostDecrUsecnt(*(src_h_ptr)); \ + *(src_h_ptr) = NULL; \ + } \ + } while (0) + HostConfig host_config; SC_ATOMIC_DECLARE(unsigned long long int,host_memuse); SC_ATOMIC_DECLARE(unsigned int,host_counter); @@ -117,6 +137,7 @@ void HostShutdown(void); Host *HostLookupHostFromHash (Address *); Host *HostGetHostFromHash (Address *); void HostRelease(Host *); +void HostLock(Host *); void HostClearMemory(Host *); void HostMoveToSpare(Host *); uint32_t HostSpareQueueGetSize(void); diff --git a/src/reputation.c b/src/reputation.c index 7b1ca0d099..dd51d6c2ba 100644 --- a/src/reputation.c +++ b/src/reputation.c @@ -28,11 +28,501 @@ #include "util-error.h" #include "util-debug.h" #include "util-radix-tree.h" -#include "reputation.h" #include "util-host-os-info.h" #include "util-unittest.h" #include "suricata-common.h" #include "threads.h" +#include "util-print.h" +#include "host.h" +#include "conf.h" +#include "detect.h" +#include "reputation.h" + +/** effective reputation version, atomic as the host + * time out code will use it to check if a host's + * reputation info is outdated. */ +SC_ATOMIC_DECL_AND_INIT(uint32_t, srep_eversion); +/** reputation version set to the host's reputation */ +static uint32_t srep_version = 0; + +static uint32_t SRepIncrVersion(void) { + return ++srep_version; +} + +static uint32_t SRepGetVersion(void) { + return srep_version; +} + +static uint32_t SRepGetEffectiveVersion(void) { + return SC_ATOMIC_GET(srep_eversion); +} + +/** \brief Increment effective reputation version after + * a rule/reputatio reload is complete. */ +void SRepReloadComplete(void) { + (void) SC_ATOMIC_ADD(srep_eversion, 1); + SCLogDebug("effective Reputation version %u", SRepGetEffectiveVersion()); +} + +/** \brief Set effective reputation version after + * reputation initialization is complete. */ +void SRepInitComplete(void) { + SC_ATOMIC_SET(srep_eversion, 1); + SCLogDebug("effective Reputation version %u", SRepGetEffectiveVersion()); +} + +/** \brief Check if a Host is timed out wrt ip rep, meaning a new + * version is in place. + * + * We clean up the old version here. + * + * \param h host + * + * \retval 0 not timed out + * \retval 1 timed out + */ +int SRepHostTimedOut(Host *h) { + BUG_ON(h == NULL); + + if (h->iprep == NULL) + return 1; + + uint32_t eversion = SRepGetEffectiveVersion(); + SReputation *r = h->iprep; + if (r->version < eversion) { + SCLogDebug("host %p has reputation version %u, " + "effective version is %u", h, r->version, eversion); + + SCFree(h->iprep); + h->iprep = NULL; + return 1; + } + + return 0; +} + +static int SRepCatSplitLine(char *line, uint8_t *cat, char *shortname, size_t shortname_len) { + size_t line_len = strlen(line); + char *ptrs[2] = {NULL,NULL}; + int i = 0; + int idx = 0; + + while (i < (int)line_len) { + if (line[i] == ',' || line[i] == '\n' || line[i] == '\0' || i == (int)(line_len - 1)) { + line[i] = '\0'; + + ptrs[idx] = line; + idx++; + + line += (i+1); + i = 0; + + if (strlen(line) == 0) + break; + if (idx == 2) + break; + } else { + i++; + } + } + + if (idx != 2) { + return -1; + } + + SCLogDebug("%s, %s", ptrs[0], ptrs[1]); + + int c = atoi(ptrs[0]); + if (c < 0 || c >= SREP_MAX_CATS) { + return -1; + } + + *cat = (uint8_t)c; + strlcpy(shortname, ptrs[1], shortname_len); + return 0; + +} + +/** + * \retval 0 valid + * \retval 1 header + * \retval -1 boo + */ +static int SRepSplitLine(char *line, uint32_t *ip, uint8_t *cat, uint8_t *value) { + size_t line_len = strlen(line); + char *ptrs[3] = {NULL,NULL,NULL}; + int i = 0; + int idx = 0; + + while (i < (int)line_len) { + if (line[i] == ',' || line[i] == '\n' || line[i] == '\0' || i == (int)(line_len - 1)) { + line[i] = '\0'; + + ptrs[idx] = line; + idx++; + + line += (i+1); + i = 0; + + if (strlen(line) == 0) + break; + if (idx == 3) + break; + } else { + i++; + } + } + + if (idx != 3) { + return -1; + } + + //SCLogInfo("%s, %s, %s", ptrs[0], ptrs[1], ptrs[2]); + + if (strcmp(ptrs[0], "ip") == 0) + return 1; + + uint32_t addr; + if (inet_pton(AF_INET, ptrs[0], &addr) <= 0) { + return -1; + } + + int c = atoi(ptrs[1]); + if (c < 0 || c >= SREP_MAX_CATS) { + return -1; + } + + int v = atoi(ptrs[2]); + if (v < 0 || v > 127) { + return -1; + } + + *ip = addr; + *cat = c; + *value = v; + return 0; +} + +#define SREP_SHORTNAME_LEN 32 +static char srep_cat_table[SREP_MAX_CATS][SREP_SHORTNAME_LEN]; + +int SRepCatValid(uint8_t cat) { + if (cat > SREP_MAX_CATS) + return 0; + + if (strlen(srep_cat_table[cat]) == 0) + return 0; + + return 1; +} + +uint8_t SRepCatGetByShortname(char *shortname) { + uint8_t cat; + for (cat = 0; cat < SREP_MAX_CATS; cat++) { + if (strcmp(srep_cat_table[cat], shortname) == 0) + return cat; + } + + return 0; +} + +int SRepLoadCatFile(char *filename) { + char line[8192] = ""; + Address a; + memset(&a, 0x00, sizeof(a)); + a.family = AF_INET; + memset(&srep_cat_table, 0x00, sizeof(srep_cat_table)); + + BUG_ON(SRepGetVersion() > 0); + + FILE *fp = fopen(filename, "r"); + if (fp == NULL) { + SCLogError(SC_ERR_OPENING_RULE_FILE, "opening ip rep file %s: %s", filename, strerror(errno)); + return -1; + } + + while(fgets(line, (int)sizeof(line), fp) != NULL) { + size_t len = strlen(line); + + /* ignore comments and empty lines */ + if (line[0] == '\n' || line [0] == '\r' || line[0] == ' ' || line[0] == '#' || line[0] == '\t') + continue; + + while (isspace(line[--len])); + + /* Check if we have a trailing newline, and remove it */ + len = strlen(line); + if (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r')) { + line[len - 1] = '\0'; + } + + uint8_t cat = 0; + char shortname[SREP_SHORTNAME_LEN]; + if (SRepCatSplitLine(line, &cat, shortname, sizeof(shortname)) == 0) { + strlcpy(srep_cat_table[cat], shortname, SREP_SHORTNAME_LEN); + } else { + SCLogError(SC_ERR_NO_REPUTATION, "bad line \"%s\"", line); + } + } + fclose(fp); + fp = NULL; + + SCLogDebug("IP Rep categories:"); + int i; + for (i = 0; i < SREP_MAX_CATS; i++) { + if (strlen(srep_cat_table[i]) == 0) + continue; + SCLogDebug("CAT %d, name %s", i, srep_cat_table[i]); + } + return 0; +} + +static int SRepLoadFile(char *filename) { + char line[8192] = ""; + Address a; + memset(&a, 0x00, sizeof(a)); + a.family = AF_INET; + + FILE *fp = fopen(filename, "r"); + if (fp == NULL) { + SCLogError(SC_ERR_OPENING_RULE_FILE, "opening ip rep file \"%s\": %s", filename, strerror(errno)); + return -1; + } + + while(fgets(line, (int)sizeof(line), fp) != NULL) { + size_t len = strlen(line); + + /* ignore comments and empty lines */ + if (line[0] == '\n' || line [0] == '\r' || line[0] == ' ' || line[0] == '#' || line[0] == '\t') + continue; + + while (isspace(line[--len])); + + /* Check if we have a trailing newline, and remove it */ + len = strlen(line); + if (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r')) { + line[len - 1] = '\0'; + } + + uint32_t ip = 0; + uint8_t cat = 0, value = 0; + int r = SRepSplitLine(line, &ip, &cat, &value); + if (r < 0) { + SCLogError(SC_ERR_NO_REPUTATION, "bad line \"%s\"", line); + } else if (r == 0) { + char ipstr[16]; + PrintInet(AF_INET, (const void *)&ip, ipstr, sizeof(ipstr)); + SCLogDebug("%s %u %u", ipstr, cat, value); + + a.addr_data32[0] = ip; + Host *h = HostGetHostFromHash(&a); + if (h) { + //SCLogInfo("host %p", h); + + if (h->iprep == NULL) { + h->iprep = SCMalloc(sizeof(SReputation)); + if (h->iprep != NULL) + memset(h->iprep, 0x00, sizeof(SReputation)); + } + if (h->iprep != NULL) { + SReputation *rep = h->iprep; + + /* if version is 0, it has been used before, so + * clear it */ + if (rep->version != 0) { + memset(rep, 0x00, sizeof(SReputation)); + } + + rep->version = SRepGetVersion(); + rep->rep[cat] = value; + } + + HostRelease(h); + } + } + } + fclose(fp); + fp = NULL; + + return 0; +} + +/** + * \brief Create the path if default-rule-path was specified + * \param sig_file The name of the file + * \retval str Pointer to the string path + sig_file + */ +static char *SRepCompleteFilePath(char *file) +{ + char *defaultpath = NULL; + char *path = NULL; + + /* Path not specified */ + if (PathIsRelative(file)) { + if (ConfGet("default-reputation-path", &defaultpath) == 1) { + SCLogDebug("Default path: %s", defaultpath); + size_t path_len = sizeof(char) * (strlen(defaultpath) + + strlen(file) + 2); + path = SCMalloc(path_len); + if (unlikely(path == NULL)) + return NULL; + strlcpy(path, defaultpath, path_len); +#if defined OS_WIN32 || defined __CYGWIN__ + if (path[strlen(path) - 1] != '\\') + strlcat(path, "\\\\", path_len); +#else + if (path[strlen(path) - 1] != '/') + strlcat(path, "/", path_len); +#endif + strlcat(path, file, path_len); + } else { + path = SCStrdup(file); + if (unlikely(path == NULL)) + return NULL; + } + } else { + path = SCStrdup(file); + if (unlikely(path == NULL)) + return NULL; + } + return path; +} + +/** \brief init reputation + * + * \param de_ctx detection engine ctx for tracking iprep version + * + * \retval 0 ok + * \retval -1 error + * + * If this function is called more than once, the category file + * is not reloaded. + */ +int SRepInit(DetectEngineCtx *de_ctx) { + ConfNode *files; + ConfNode *file = NULL; + int r = 0; + char *sfile = NULL; + char *filename = NULL; + int init = 0; + + if (SRepGetVersion() == 0) { + init = 1; + } + + if (init) { + if (ConfGet("reputation-categories-file", &filename) != 1) { + SCLogError(SC_ERR_NO_REPUTATION, "\"reputation-categories-file\" not set"); + return -1; + } + } + + files = ConfGetNode("reputation-files"); + if (files == NULL) { + SCLogError(SC_ERR_NO_REPUTATION, "\"reputation-files\" not set"); + return -1; + } + + if (init) { + /* init even if we have reputation files, so that when we + * have a live reload, we have inited the cats */ + if (SRepLoadCatFile(filename) < 0) { + SCLogError(SC_ERR_NO_REPUTATION, "failed to load reputation " + "categories file %s", filename); + return -1; + } + } + + de_ctx->srep_version = SRepIncrVersion(); + SCLogDebug("Reputation version %u", de_ctx->srep_version); + + /* ok, let's load signature files from the general config */ + if (files != NULL) { + TAILQ_FOREACH(file, &files->head, next) { + sfile = SRepCompleteFilePath(file->val); + SCLogInfo("Loading reputation file: %s", sfile); + + r = SRepLoadFile(sfile); + if (r < 0){ + SCLogWarning(SC_ERR_NO_REPUTATION, "no reputation loaded from \"%s\"", sfile); + if (de_ctx->failure_fatal == 1) { + exit(EXIT_FAILURE); + } + } + SCFree(sfile); + } + } + + /* Set effective rep version. + * On live reload we will handle this after de_ctx has been swapped */ + if (init) { + SRepInitComplete(); + } + return 0; +} + +#ifdef UNITTESTS +static int SRepTest01(void) { + char str[] = "1.2.3.4,1,2"; + + uint32_t ip = 0; + uint8_t cat = 0, value = 0; + if (SRepSplitLine(str, &ip, &cat, &value) != 0) { + return 0; + } + + char ipstr[16]; + PrintInet(AF_INET, (const void *)&ip, ipstr, sizeof(ipstr)); + + if (strcmp(ipstr, "1.2.3.4") != 0) + return 0; + + if (cat != 1) + return 0; + + if (value != 2) + return 0; + + return 1; +} + +static int SRepTest02(void) { + char str[] = "1.1.1.1,"; + + uint32_t ip = 0; + uint8_t cat = 0, value = 0; + if (SRepSplitLine(str, &ip, &cat, &value) == 0) { + return 0; + } + return 1; +} + +static int SRepTest03(void) { + char str[] = "1,Shortname,Long Name"; + + uint8_t cat = 0; + char shortname[SREP_SHORTNAME_LEN]; + + if (SRepCatSplitLine(str, &cat, shortname, sizeof(shortname)) != 0) { + printf("split failed: "); + return 0; + } + + if (strcmp(shortname, "Shortname") != 0) { + printf("%s != Shortname: ", shortname); + return 0; + } + + if (cat != 1) { + printf("cat 1 != %u: ", cat); + return 0; + } + + return 1; +} + + +#endif /** Global trees that hold host reputation for IPV4 and IPV6 hosts */ IPReputationCtx *rep_ctx; @@ -1452,6 +1942,10 @@ void SCReputationRegisterTests(void) SCReputationTestIPV4Update01, 1); UtRegisterTest("SCReputationTestIPV6Update01", SCReputationTestIPV6Update01, 1); + + UtRegisterTest("SRepTest01", SRepTest01, 1); + UtRegisterTest("SRepTest02", SRepTest02, 1); + UtRegisterTest("SRepTest03", SRepTest03, 1); #endif /* UNITTESTS */ } diff --git a/src/reputation.h b/src/reputation.h index e5cbd296f5..1266bf500a 100644 --- a/src/reputation.h +++ b/src/reputation.h @@ -26,6 +26,20 @@ #ifndef __REPUTATION_H__ #define __REPUTATION_H__ +#include "detect.h" +#include "host.h" + +#define SREP_MAX_CATS 60 +typedef struct SReputation_ { + uint32_t version; + uint8_t rep[SREP_MAX_CATS]; +} SReputation; + +uint8_t SRepCatGetByShortname(char *shortname); +int SRepInit(DetectEngineCtx *de_ctx); +void SRepReloadComplete(void); +int SRepHostTimedOut(Host *); + /** Reputation numbers (types) that we can use to lookup/update, etc * Please, dont convert this to a enum since we want the same reputation * codes always. */ diff --git a/src/tmqh-packetpool.c b/src/tmqh-packetpool.c index 9310f88075..7bd7a59153 100644 --- a/src/tmqh-packetpool.c +++ b/src/tmqh-packetpool.c @@ -37,6 +37,7 @@ #include "threadvars.h" #include "flow.h" #include "flow-util.h" +#include "host.h" #include "stream.h" #include "stream-tcp-reassemble.h" diff --git a/src/util-error.c b/src/util-error.c index e08eabef71..005e34ece2 100644 --- a/src/util-error.c +++ b/src/util-error.c @@ -239,6 +239,7 @@ const char * SCErrorToString(SCError err) CASE_CODE (SC_ERR_NO_LUAJIT_SUPPORT); CASE_CODE (SC_ERR_LUAJIT_ERROR); CASE_CODE (SC_ERR_DEFRAG_INIT); + CASE_CODE (SC_ERR_NO_REPUTATION); default: return "UNKNOWN_ERROR"; } diff --git a/src/util-error.h b/src/util-error.h index 8edf2dfdd1..3b3f8b1475 100644 --- a/src/util-error.h +++ b/src/util-error.h @@ -253,6 +253,7 @@ typedef enum { SC_ERR_NAPATECH_STREAMS_REGISTER_FAILED, SC_ERR_NAPATECH_STAT_DROPS_FAILED, SC_ERR_NAPATECH_PARSE_CONFIG, + SC_ERR_NO_REPUTATION, } SCError; const char *SCErrorToString(SCError); diff --git a/suricata.yaml.in b/suricata.yaml.in index 3df6384983..dabf9de46d 100644 --- a/suricata.yaml.in +++ b/suricata.yaml.in @@ -828,6 +828,11 @@ action-order: - reject - alert +# IP Reputation +#reputation-categories-file: @e_sysconfdir@iprep/categories.txt +#default-reputation-path: @e_sysconfdir@iprep +#reputation-files: +# - reputation.list # Host specific policies for defragmentation and TCP stream # reassembly. The host OS lookup is done using a radix tree, just