Simple IP reputation implementation

src/Makefile.am

@@ -137,6 +137,7 @@ detect-icode.c detect-icode.h \
 detect-id.c detect-id.h \
 detect-ipopts.c detect-ipopts.h \
 detect-ipproto.c detect-ipproto.h \
+detect-iprep.c detect-iprep.h \
 detect-isdataat.c detect-isdataat.h \
 detect-itype.c detect-itype.h \
 detect-l3proto.c detect-l3proto.h \

src/decode-ipv4.c

@@ -38,6 +38,7 @@
 #include "decode-events.h"
 #include "defrag.h"
 #include "pkt-var.h"
+#include "host.h"
 
 #include "util-unittest.h"
 #include "util-debug.h"

src/decode.h

@@ -368,7 +368,8 @@ typedef struct Packet_
     uint8_t recursion_level;
 
     /* Pkt Flags */
-    uint16_t flags;
+    uint32_t flags;
+
     /* flow */
     uint8_t flowflags;
 
@@ -457,6 +458,9 @@ typedef struct Packet_
 
     PacketAlerts alerts;
 
+    struct Host_ *host_src;
+    struct Host_ *host_dst;
+
     /** packet number in the pcap file, matches wireshark */
     uint64_t pcap_cnt;
 
@@ -662,8 +666,8 @@ typedef struct DecodeThreadVars_
         if ((p)->udph != NULL) {                \
             CLEAR_UDP_PACKET((p));              \
         }                                       \
-        if ((p)->sctph != NULL) {                \
+        if ((p)->sctph != NULL) {               \
-            CLEAR_SCTP_PACKET((p));              \
+            CLEAR_SCTP_PACKET((p));             \
         }                                       \
         if ((p)->icmpv4h != NULL) {             \
             CLEAR_ICMPV4_PACKET((p));           \
@@ -680,6 +684,8 @@ typedef struct DecodeThreadVars_
         (p)->payload_len = 0;                   \
         (p)->pktlen = 0;                        \
         (p)->alerts.cnt = 0;                    \
+        HostDeReference(&((p)->host_src));      \
+        HostDeReference(&((p)->host_dst));      \
         (p)->pcap_cnt = 0;                      \
         (p)->tunnel_rtv_cnt = 0;                \
         (p)->tunnel_tpr_cnt = 0;                \
@@ -915,24 +921,27 @@ void AddressDebugPrint(Address *);
 #define VLAN_OVER_GRE       13
 
 /*Packet Flags*/
-#define PKT_NOPACKET_INSPECTION         0x0001    /**< Flag to indicate that packet header or contents should not be inspected*/
+#define PKT_NOPACKET_INSPECTION         (1)         /**< Flag to indicate that packet header or contents should not be inspected*/
-#define PKT_NOPAYLOAD_INSPECTION        0x0002    /**< Flag to indicate that packet contents should not be inspected*/
+#define PKT_NOPAYLOAD_INSPECTION        (1<<2)      /**< Flag to indicate that packet contents should not be inspected*/
-#define PKT_ALLOC                       0x0004    /**< Packet was alloc'd this run, needs to be freed */
+#define PKT_ALLOC                       (1<<3)      /**< Packet was alloc'd this run, needs to be freed */
-#define PKT_HAS_TAG                     0x0008    /**< Packet has matched a tag */
+#define PKT_HAS_TAG                     (1<<4)      /**< Packet has matched a tag */
-#define PKT_STREAM_ADD                  0x0010    /**< Packet payload was added to reassembled stream */
+#define PKT_STREAM_ADD                  (1<<5)      /**< Packet payload was added to reassembled stream */
-#define PKT_STREAM_EST                  0x0020    /**< Packet is part of establised stream */
+#define PKT_STREAM_EST                  (1<<6)      /**< Packet is part of establised stream */
-#define PKT_STREAM_EOF                  0x0040    /**< Stream is in eof state */
+#define PKT_STREAM_EOF                  (1<<7)      /**< Stream is in eof state */
-#define PKT_HAS_FLOW                    0x0080
+#define PKT_HAS_FLOW                    (1<<8)
-#define PKT_PSEUDO_STREAM_END           0x0100    /**< Pseudo packet to end the stream */
+#define PKT_PSEUDO_STREAM_END           (1<<9)      /**< Pseudo packet to end the stream */
-#define PKT_STREAM_MODIFIED             0x0200    /**< Packet is modified by the stream engine, we need to recalc the csum and reinject/replace */
+#define PKT_STREAM_MODIFIED             (1<<10)     /**< Packet is modified by the stream engine, we need to recalc the csum and reinject/replace */
-#define PKT_MARK_MODIFIED               0x0400    /**< Packet mark is modified */
+#define PKT_MARK_MODIFIED               (1<<11)     /**< Packet mark is modified */
-#define PKT_STREAM_NOPCAPLOG            0x0800    /**< Exclude packet from pcap logging as it's part of a stream that has reassembly depth reached. */
+#define PKT_STREAM_NOPCAPLOG            (1<<12)     /**< Exclude packet from pcap logging as it's part of a stream that has reassembly depth reached. */
-
+
-#define PKT_TUNNEL                      0x1000
+#define PKT_TUNNEL                      (1<<13)
-#define PKT_TUNNEL_VERDICTED            0x2000
+#define PKT_TUNNEL_VERDICTED            (1<<14)
-
+
-#define PKT_IGNORE_CHECKSUM             0x4000    /**< Packet checksum is not computed (TX packet for example) */
+#define PKT_IGNORE_CHECKSUM             (1<<15)     /**< Packet checksum is not computed (TX packet for example) */
-#define PKT_ZERO_COPY                   0x8000    /**< Packet comes from zero copy (ext_pkt must not be freed) */
+#define PKT_ZERO_COPY                   (1<<16)     /**< Packet comes from zero copy (ext_pkt must not be freed) */
+
+#define PKT_HOST_SRC_LOOKED_UP          (1<<17)
+#define PKT_HOST_DST_LOOKED_UP          (1<<18)
 
 /** \brief return 1 if the packet is a pseudo packet */
 #define PKT_IS_PSEUDOPKT(p) ((p)->flags & PKT_PSEUDO_STREAM_END)

src/detect-engine-iponly.c

@@ -927,16 +927,11 @@ int IPOnlyMatchCompatSMs(ThreadVars *tv,
                          Signature *s, Packet *p)
 {
     SigMatch *sm = s->sm_lists[DETECT_SM_LIST_MATCH];
-    int match;
 
     while (sm != NULL) {
-        if (sm->type != DETECT_FLOWBITS) {
+        BUG_ON(!(sigmatch_table[sm->type].flags & SIGMATCH_IPONLY_COMPAT));
-            sm = sm->next;
-            continue;
-        }
 
-        match = sigmatch_table[sm->type].Match(tv, det_ctx, p, s, sm);
+        if (sigmatch_table[sm->type].Match(tv, det_ctx, p, s, sm) > 0) {
-        if (match > 0) {
             sm = sm->next;
             continue;
         }

src/detect-engine.c

@@ -84,6 +84,8 @@
 #include "util-profiling.h"
 #endif
 
+#include "reputation.h"
+
 #define DETECT_ENGINE_DEFAULT_INSPECTION_RECURSION_LIMIT 3000
 
 static uint32_t detect_engine_ctx_id = 1;
@@ -541,6 +543,8 @@ static void *DetectEngineLiveRuleSwap(void *arg)
     }
     DetectEngineCtxFree(old_de_ctx);
 
+    SRepReloadComplete();
+
     /* reset the handler */
     UtilSignalHandlerSetup(SIGUSR2, SignalHandlerSigusr2);
 
@@ -678,6 +682,9 @@ DetectEngineCtx *DetectEngineCtxInit(void) {
 
     de_ctx->id = detect_engine_ctx_id++;
 
+    /* init iprep... ignore errors for now */
+    (void)SRepInit(de_ctx);
+
     return de_ctx;
 error:
     return NULL;

src/detect-iprep.c

@@ -0,0 +1,388 @@
+/* Copyright (C) 2012 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ *  \author Victor Julien <victor@inliniac.net>
+ *
+ * Implements the iprep keyword
+ */
+
+#include "suricata-common.h"
+#include "decode.h"
+#include "detect.h"
+#include "threads.h"
+#include "flow.h"
+#include "flow-bit.h"
+#include "flow-util.h"
+#include "detect-iprep.h"
+#include "util-spm.h"
+
+#include "app-layer-parser.h"
+
+#include "detect-parse.h"
+#include "detect-engine.h"
+#include "detect-engine-mpm.h"
+#include "detect-engine-state.h"
+
+#include "util-debug.h"
+
+#include "reputation.h"
+#include "host.h"
+
+#define PARSE_REGEX         "\\s*(any|src|dst|both)\\s*,\\s*([A-Za-z0-9\\-\\_]+)\\s*,\\s*(\\<|\\>|\\=)\\s*,\\s*([0-9]+)\\s*"
+static pcre *parse_regex;
+static pcre_extra *parse_regex_study;
+
+int DetectIPRepMatch (ThreadVars *, DetectEngineThreadCtx *, Packet *, Signature *, SigMatch *);
+static int DetectIPRepSetup (DetectEngineCtx *, Signature *, char *);
+void DetectIPRepFree (void *);
+void IPRepRegisterTests(void);
+
+void DetectIPRepRegister (void) {
+    sigmatch_table[DETECT_IPREP].name = "iprep";
+    sigmatch_table[DETECT_IPREP].Match = DetectIPRepMatch;
+    sigmatch_table[DETECT_IPREP].Setup = DetectIPRepSetup;
+    sigmatch_table[DETECT_IPREP].Free  = DetectIPRepFree;
+    sigmatch_table[DETECT_IPREP].RegisterTests = IPRepRegisterTests;
+    /* this is compatible to ip-only signatures */
+    sigmatch_table[DETECT_IPREP].flags |= SIGMATCH_IPONLY_COMPAT;
+
+    const char *eb;
+    int eo;
+    int opts = 0;
+
+    parse_regex = pcre_compile(PARSE_REGEX, opts, &eb, &eo, NULL);
+    if(parse_regex == NULL)
+    {
+        SCLogError(SC_ERR_PCRE_COMPILE, "pcre compile of \"%s\" failed at offset %" PRId32 ": %s", PARSE_REGEX, eo, eb);
+        goto error;
+    }
+
+    parse_regex_study = pcre_study(parse_regex, 0, &eb);
+    if(eb != NULL)
+    {
+        SCLogError(SC_ERR_PCRE_STUDY, "pcre study failed: %s", eb);
+        goto error;
+    }
+
+    return;
+
+error:
+    return;
+}
+
+static uint8_t GetHostRepSrc(Packet *p, uint8_t cat, uint32_t version) {
+    uint8_t val = 0;
+    Host *h = NULL;
+
+    if (p->flags & PKT_HOST_SRC_LOOKED_UP && p->host_src == NULL) {
+        return 0;
+    } else if (p->host_src != NULL) {
+        h = (Host *)p->host_src;
+        HostLock(h);
+    } else {
+        h = HostLookupHostFromHash(&(p->src));
+
+        p->flags |= PKT_HOST_SRC_LOOKED_UP;
+
+        if (h == NULL)
+            return 0;
+
+        HostReference(&p->host_src, h);
+    }
+
+    if (h->iprep == NULL) {
+        HostRelease(h);
+        return 0;
+    }
+
+    SReputation *r = (SReputation *)h->iprep;
+
+    /* allow higher versions as this happens during
+     * rule reload */
+    if (r->version >= version)
+        val = r->rep[cat];
+    else
+        SCLogDebug("version mismatch %u != %u", r->version, version);
+
+    HostRelease(h);
+    return val;
+}
+
+static uint8_t GetHostRepDst(Packet *p, uint8_t cat, uint32_t version) {
+    uint8_t val = 0;
+    Host *h = NULL;
+
+    if (p->flags & PKT_HOST_DST_LOOKED_UP && p->host_dst == NULL) {
+        return 0;
+    } else if (p->host_dst != NULL) {
+        h = (Host *)p->host_dst;
+        HostLock(h);
+    } else {
+        h = HostLookupHostFromHash(&(p->dst));
+
+        p->flags |= PKT_HOST_DST_LOOKED_UP;
+
+        if (h == NULL) {
+            return 0;
+        }
+
+        HostReference(&p->host_dst, h);
+    }
+
+    if (h->iprep == NULL) {
+        HostRelease(h);
+        return 0;
+    }
+
+    SReputation *r = (SReputation *)h->iprep;
+
+    /* allow higher versions as this happens during
+     * rule reload */
+    if (r->version >= version)
+        val = r->rep[cat];
+    else
+        SCLogDebug("version mismatch %u != %u", r->version, version);
+
+    HostRelease(h);
+    return val;
+}
+
+static inline int RepMatch(uint8_t op, uint8_t val1, uint8_t val2) {
+    if (op == DETECT_IPREP_OP_GT && val1 > val2) {
+        return 1;
+    } else if (op == DETECT_IPREP_OP_LT && val1 < val2) {
+        return 1;
+    } else if (op == DETECT_IPREP_OP_EQ && val1 == val2) {
+        return 1;
+    }
+    return 0;
+}
+
+/*
+ * returns 0: no match
+ *         1: match
+ *        -1: error
+ */
+int DetectIPRepMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, Packet *p, Signature *s, SigMatch *m)
+{
+    DetectIPRepData *rd = (DetectIPRepData *)m->ctx;
+    if (rd == NULL)
+        return 0;
+
+    uint32_t version = det_ctx->de_ctx->srep_version;
+    uint8_t val = 0;
+
+    SCLogDebug("rd->cmd %u", rd->cmd);
+    switch(rd->cmd) {
+        case DETECT_IPREP_CMD_ANY:
+            val = GetHostRepSrc(p, rd->cat, version);
+            if (val > 0) {
+                if (RepMatch(rd->op, val, rd->val) == 1)
+                    return 1;
+            }
+            val = GetHostRepDst(p, rd->cat, version);
+            if (val > 0) {
+                return RepMatch(rd->op, val, rd->val);
+            }
+            break;
+
+        case DETECT_IPREP_CMD_SRC:
+            SCLogDebug("checking src");
+            val = GetHostRepSrc(p, rd->cat, version);
+            if (val > 0) {
+                return RepMatch(rd->op, val, rd->val);
+            }
+            break;
+
+        case DETECT_IPREP_CMD_DST:
+            SCLogDebug("checking dst");
+            val = GetHostRepDst(p, rd->cat, version);
+            if (val > 0) {
+                return RepMatch(rd->op, val, rd->val);
+            }
+            break;
+
+        case DETECT_IPREP_CMD_BOTH:
+            val = GetHostRepSrc(p, rd->cat, version);
+            if (val == 0 || RepMatch(rd->op, val, rd->val) == 0)
+                return 0;
+            val = GetHostRepDst(p, rd->cat, version);
+            if (val > 0) {
+                return RepMatch(rd->op, val, rd->val);
+            }
+            break;
+    }
+
+    return 0;
+}
+
+int DetectIPRepSetup (DetectEngineCtx *de_ctx, Signature *s, char *rawstr)
+{
+    DetectIPRepData *cd = NULL;
+    SigMatch *sm = NULL;
+    char *cmd_str = NULL, *name = NULL, *op_str = NULL, *value = NULL;
+    uint8_t cmd = 0;
+#define MAX_SUBSTRINGS 30
+    int ret = 0, res = 0;
+    int ov[MAX_SUBSTRINGS];
+
+    ret = pcre_exec(parse_regex, parse_regex_study, rawstr, strlen(rawstr), 0, 0, ov, MAX_SUBSTRINGS);
+    if (ret != 5) {
+        SCLogError(SC_ERR_PCRE_MATCH, "\"%s\" is not a valid setting for iprep", rawstr);
+        return -1;
+    }
+
+    const char *str_ptr;
+    res = pcre_get_substring((char *)rawstr, ov, MAX_SUBSTRINGS, 1, &str_ptr);
+    if (res < 0) {
+        SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed");
+        return -1;
+    }
+    cmd_str = (char *)str_ptr;
+
+    res = pcre_get_substring((char *)rawstr, ov, MAX_SUBSTRINGS, 2, &str_ptr);
+    if (res < 0) {
+        SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed");
+        goto error;
+    }
+    name = (char *)str_ptr;
+
+    res = pcre_get_substring((char *)rawstr, ov, MAX_SUBSTRINGS, 3, &str_ptr);
+    if (res < 0) {
+        SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed");
+        goto error;
+    }
+    op_str = (char *)str_ptr;
+
+    res = pcre_get_substring((char *)rawstr, ov, MAX_SUBSTRINGS, 4, &str_ptr);
+    if (res < 0) {
+        SCLogError(SC_ERR_PCRE_GET_SUBSTRING, "pcre_get_substring failed");
+        goto error;
+    }
+    value = (char *)str_ptr;
+
+    if (strcmp(cmd_str,"any") == 0) {
+        cmd = DETECT_IPREP_CMD_ANY;
+    } else if (strcmp(cmd_str,"both") == 0) {
+        cmd = DETECT_IPREP_CMD_BOTH;
+    } else if (strcmp(cmd_str,"src") == 0) {
+        cmd = DETECT_IPREP_CMD_SRC;
+    } else if (strcmp(cmd_str,"dst") == 0) {
+        cmd = DETECT_IPREP_CMD_DST;
+    } else {
+        SCLogError(SC_ERR_UNKNOWN_VALUE, "ERROR: iprep \"%s\" is not supported.", cmd_str);
+        goto error;
+    }
+
+    //SCLogInfo("category %s", name);
+    uint8_t cat = SRepCatGetByShortname(name);
+    if (cat == 0) {
+        SCLogError(SC_ERR_UNKNOWN_VALUE, "unknown iprep category \"%s\"", name);
+        goto error;
+    }
+
+    uint8_t op = 0;
+    uint8_t val = 0;
+
+    if (op_str == NULL || strlen(op_str) != 1) {
+        goto error;
+    }
+
+    switch(op_str[0]) {
+        case '<':
+            op = DETECT_IPREP_OP_LT;
+            break;
+        case '>':
+            op = DETECT_IPREP_OP_GT;
+            break;
+        case '=':
+            op = DETECT_IPREP_OP_EQ;
+            break;
+        default:
+            goto error;
+            break;
+    }
+
+    if (value != NULL && strlen(value) > 0) {
+        int ival = atoi(value);
+        if (ival < 0 || ival > 127)
+            goto error;
+        val = (uint8_t)ival;
+    }
+
+    cd = SCMalloc(sizeof(DetectIPRepData));
+    if (unlikely(cd == NULL))
+        goto error;
+
+    cd->cmd = cmd;
+    cd->cat = cat;
+    cd->op = op;
+    cd->val = val;
+    //SCLogInfo("cmd %u, cat %u, op %u, val %u", cd->cmd, cd->cat, cd->op, cd->val);
+
+    pcre_free_substring(name);
+    name = NULL;
+    pcre_free_substring(cmd_str);
+    cmd_str = NULL;
+
+    /* Okay so far so good, lets get this into a SigMatch
+     * and put it in the Signature. */
+    sm = SigMatchAlloc();
+    if (sm == NULL)
+        goto error;
+
+    sm->type = DETECT_IPREP;
+    sm->ctx = (void *)cd;
+
+    SigMatchAppendSMToList(s, sm, DETECT_SM_LIST_MATCH);
+
+    return 0;
+
+error:
+    if (name != NULL)
+        pcre_free_substring(name);
+    if (cmd_str != NULL)
+        pcre_free_substring(cmd_str);
+    if (cd != NULL)
+        SCFree(cd);
+    if (sm != NULL)
+        SCFree(sm);
+    return -1;
+}
+
+void DetectIPRepFree (void *ptr) {
+    DetectIPRepData *fd = (DetectIPRepData *)ptr;
+
+    if (fd == NULL)
+        return;
+
+    SCFree(fd);
+}
+
+#ifdef UNITTESTS
+#endif /* UNITTESTS */
+
+/**
+ * \brief this function registers unit tests for IPRep
+ */
+void IPRepRegisterTests(void) {
+#ifdef UNITTESTS
+#endif /* UNITTESTS */
+}

src/detect-iprep.h

@@ -0,0 +1,46 @@
+/* Copyright (C) 2012 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ *  \author Victor Julien <victor@inliniac.net>
+ */
+
+#ifndef __DETECT_IPREP_H__
+#define __DETECT_IPREP_H__
+
+#define DETECT_IPREP_CMD_ANY      0
+#define DETECT_IPREP_CMD_BOTH     1
+#define DETECT_IPREP_CMD_SRC      2
+#define DETECT_IPREP_CMD_DST      3
+
+#define DETECT_IPREP_OP_LT        0
+#define DETECT_IPREP_OP_GT        1
+#define DETECT_IPREP_OP_EQ        2
+
+typedef struct DetectIPRepData_ {
+    uint8_t cmd;
+    int8_t cat;
+    int8_t op;
+    uint8_t val;
+} DetectIPRepData;
+
+/* prototypes */
+void DetectIPRepRegister (void);
+
+#endif /* __DETECT_IPREP_H__ */

src/detect.c

@@ -143,6 +143,7 @@
 #include "detect-tos.h"
 #include "detect-app-layer-event.h"
 #include "detect-luajit.h"
+#include "detect-iprep.h"
 
 #include "util-rule-vars.h"
 
@@ -4666,6 +4667,7 @@ void SigTableSetup(void) {
     DetectAppLayerEventRegister();
     DetectHttpUARegister();
     DetectLuajitRegister();
+    DetectIPRepRegister();
 
     uint8_t i = 0;
     for (i = 0; i < DETECT_TBLSIZE; i++) {

src/detect.h

@@ -546,6 +546,9 @@ typedef struct DetectEngineCtx_ {
     Signature *sig_list;
     uint32_t sig_cnt;
 
+    /* version of the srep data */
+    uint32_t srep_version;
+
     Signature **sig_array;
     uint32_t sig_array_size; /* size in bytes */
     uint32_t sig_array_len;  /* size in array members */
@@ -1087,6 +1090,7 @@ enum {
 
     DETECT_L3PROTO,
     DETECT_LUAJIT,
+    DETECT_IPREP,
 
     /* make sure this stays last */
     DETECT_TBLSIZE,

src/flow-timeout.c

@@ -40,6 +40,7 @@
 #include "flow-private.h"
 #include "flow-manager.h"
 #include "pkt-var.h"
+#include "host.h"
 
 #include "stream-tcp-private.h"
 #include "stream-tcp-reassemble.h"

src/host-timeout.c

@@ -26,6 +26,7 @@
 
 #include "detect-engine-tag.h"
 #include "detect-engine-threshold.h"
+#include "reputation.h"
 
 uint32_t HostGetSpareCount(void) {
     return HostSpareQueueGetSize();
@@ -54,6 +55,13 @@ static int HostHostTimedOut(Host *h, struct timeval *ts) {
         return 0;
     }
 
+    if (h->iprep) {
+        if (SRepHostTimedOut(h) == 0)
+            return 0;
+
+        SCLogDebug("host %p reputation timed out", h);
+    }
+
     if (h->tag && TagTimeoutCheck(h, ts) == 0) {
         tags = 1;
     }
@@ -64,6 +72,7 @@ static int HostHostTimedOut(Host *h, struct timeval *ts) {
     if (tags || thresholds)
         return 0;
 
+    SCLogDebug("host %p timed out", h);
     return 1;
 }
 

src/host.c

@@ -108,6 +108,10 @@ void HostClearMemory(Host *h) {
         ThresholdListFree(h->threshold);
         h->threshold = NULL;
     }
+    if (h->iprep != NULL) {
+        SCFree(h->iprep);
+        h->iprep = NULL;
+    }
     SC_ATOMIC_DESTROY(h->use_cnt);
 }
 
@@ -362,11 +366,6 @@ static Host *HostGetNew(Address *a) {
     return h;
 }
 
-#define HostIncrUsecnt(h) \
-    SC_ATOMIC_ADD((h)->use_cnt, 1)
-#define HostDecrUsecnt(h) \
-    SC_ATOMIC_SUB((h)->use_cnt, 1)
-
 void HostInit(Host *h, Address *a) {
     COPY_ADDRESS(a, &h->a);
     (void) HostIncrUsecnt(h);
@@ -377,6 +376,10 @@ void HostRelease(Host *h) {
     SCMutexUnlock(&h->m);
 }
 
+void HostLock(Host *h) {
+    SCMutexLock(&h->m);
+}
+
 /* HostGetHostFromHash
  *
  * Hash retrieval function for hosts. Looks up the hash bucket containing the

src/host.h

@@ -67,6 +67,7 @@ typedef struct Host_ {
     /** pointers to tag and threshold storage */
     void *tag;
     void *threshold;
+    void *iprep;
 
     /** hash pointers, protected by hash row mutex/spin */
     struct Host_ *hnext;
@@ -106,6 +107,25 @@ typedef struct HostConfig_ {
 #define HOST_CHECK_MEMCAP(size) \
     ((((uint64_t)SC_ATOMIC_GET(host_memuse) + (uint64_t)(size)) <= host_config.memcap))
 
+#define HostIncrUsecnt(h) \
+    SC_ATOMIC_ADD((h)->use_cnt, 1)
+#define HostDecrUsecnt(h) \
+    SC_ATOMIC_SUB((h)->use_cnt, 1)
+
+#define HostReference(dst_h_ptr, h) do {            \
+        if ((h) != NULL) {                          \
+            HostIncrUsecnt((h));                    \
+            *(dst_h_ptr) = h;                       \
+        }                                           \
+    } while (0)
+
+#define HostDeReference(src_h_ptr) do {               \
+        if (*(src_h_ptr) != NULL) {                   \
+            HostDecrUsecnt(*(src_h_ptr));             \
+            *(src_h_ptr) = NULL;                      \
+        }                                             \
+    } while (0)
+
 HostConfig host_config;
 SC_ATOMIC_DECLARE(unsigned long long int,host_memuse);
 SC_ATOMIC_DECLARE(unsigned int,host_counter);
@@ -117,6 +137,7 @@ void HostShutdown(void);
 Host *HostLookupHostFromHash (Address *);
 Host *HostGetHostFromHash (Address *);
 void HostRelease(Host *);
+void HostLock(Host *);
 void HostClearMemory(Host *);
 void HostMoveToSpare(Host *);
 uint32_t HostSpareQueueGetSize(void);

src/reputation.c

@@ -28,11 +28,501 @@
 #include "util-error.h"
 #include "util-debug.h"
 #include "util-radix-tree.h"
-#include "reputation.h"
 #include "util-host-os-info.h"
 #include "util-unittest.h"
 #include "suricata-common.h"
 #include "threads.h"
+#include "util-print.h"
+#include "host.h"
+#include "conf.h"
+#include "detect.h"
+#include "reputation.h"
+
+/** effective reputation version, atomic as the host
+ *  time out code will use it to check if a host's
+ *  reputation info is outdated. */
+SC_ATOMIC_DECL_AND_INIT(uint32_t, srep_eversion);
+/** reputation version set to the host's reputation */
+static uint32_t srep_version = 0;
+
+static uint32_t SRepIncrVersion(void) {
+    return ++srep_version;
+}
+
+static uint32_t SRepGetVersion(void) {
+    return srep_version;
+}
+
+static uint32_t SRepGetEffectiveVersion(void) {
+    return SC_ATOMIC_GET(srep_eversion);
+}
+
+/** \brief Increment effective reputation version after
+ *         a rule/reputatio reload is complete. */
+void SRepReloadComplete(void) {
+    (void) SC_ATOMIC_ADD(srep_eversion, 1);
+    SCLogDebug("effective Reputation version %u", SRepGetEffectiveVersion());
+}
+
+/** \brief Set effective reputation version after
+ *         reputation initialization is complete. */
+void SRepInitComplete(void) {
+    SC_ATOMIC_SET(srep_eversion, 1);
+    SCLogDebug("effective Reputation version %u", SRepGetEffectiveVersion());
+}
+
+/** \brief Check if a Host is timed out wrt ip rep, meaning a new
+ *         version is in place.
+ *
+ *  We clean up the old version here.
+ *
+ *  \param h host
+ *
+ *  \retval 0 not timed out
+ *  \retval 1 timed out
+ */
+int SRepHostTimedOut(Host *h) {
+    BUG_ON(h == NULL);
+
+    if (h->iprep == NULL)
+        return 1;
+
+    uint32_t eversion = SRepGetEffectiveVersion();
+    SReputation *r = h->iprep;
+    if (r->version < eversion) {
+        SCLogDebug("host %p has reputation version %u, "
+                "effective version is %u", h, r->version, eversion);
+
+        SCFree(h->iprep);
+        h->iprep = NULL;
+        return 1;
+    }
+
+    return 0;
+}
+
+static int SRepCatSplitLine(char *line, uint8_t *cat, char *shortname, size_t shortname_len) {
+    size_t line_len = strlen(line);
+    char *ptrs[2] = {NULL,NULL};
+    int i = 0;
+    int idx = 0;
+
+    while (i < (int)line_len) {
+        if (line[i] == ',' || line[i] == '\n' || line[i] == '\0' || i == (int)(line_len - 1)) {
+            line[i] = '\0';
+
+            ptrs[idx] = line;
+            idx++;
+
+            line += (i+1);
+            i = 0;
+
+            if (strlen(line) == 0)
+                break;
+            if (idx == 2)
+                break;
+        } else {
+            i++;
+        }
+    }
+
+    if (idx != 2) {
+        return -1;
+    }
+
+    SCLogDebug("%s, %s", ptrs[0], ptrs[1]);
+
+    int c = atoi(ptrs[0]);
+    if (c < 0 || c >= SREP_MAX_CATS) {
+        return -1;
+    }
+
+    *cat = (uint8_t)c;
+    strlcpy(shortname, ptrs[1], shortname_len);
+    return 0;
+
+}
+
+/**
+ *  \retval 0 valid
+ *  \retval 1 header
+ *  \retval -1 boo
+ */
+static int SRepSplitLine(char *line, uint32_t *ip, uint8_t *cat, uint8_t *value) {
+    size_t line_len = strlen(line);
+    char *ptrs[3] = {NULL,NULL,NULL};
+    int i = 0;
+    int idx = 0;
+
+    while (i < (int)line_len) {
+        if (line[i] == ',' || line[i] == '\n' || line[i] == '\0' || i == (int)(line_len - 1)) {
+            line[i] = '\0';
+
+            ptrs[idx] = line;
+            idx++;
+
+            line += (i+1);
+            i = 0;
+
+            if (strlen(line) == 0)
+                break;
+            if (idx == 3)
+                break;
+        } else {
+            i++;
+        }
+    }
+
+    if (idx != 3) {
+        return -1;
+    }
+
+    //SCLogInfo("%s, %s, %s", ptrs[0], ptrs[1], ptrs[2]);
+
+    if (strcmp(ptrs[0], "ip") == 0)
+        return 1;
+
+    uint32_t addr;
+    if (inet_pton(AF_INET, ptrs[0], &addr) <= 0) {
+        return -1;
+    }
+
+    int c = atoi(ptrs[1]);
+    if (c < 0 || c >= SREP_MAX_CATS) {
+        return -1;
+    }
+
+    int v = atoi(ptrs[2]);
+    if (v < 0 || v > 127) {
+        return -1;
+    }
+
+    *ip = addr;
+    *cat = c;
+    *value = v;
+    return 0;
+}
+
+#define SREP_SHORTNAME_LEN 32
+static char srep_cat_table[SREP_MAX_CATS][SREP_SHORTNAME_LEN];
+
+int SRepCatValid(uint8_t cat) {
+    if (cat > SREP_MAX_CATS)
+        return 0;
+
+    if (strlen(srep_cat_table[cat]) == 0)
+        return 0;
+
+    return 1;
+}
+
+uint8_t SRepCatGetByShortname(char *shortname) {
+    uint8_t cat;
+    for (cat = 0; cat < SREP_MAX_CATS; cat++) {
+        if (strcmp(srep_cat_table[cat], shortname) == 0)
+            return cat;
+    }
+
+    return 0;
+}
+
+int SRepLoadCatFile(char *filename) {
+    char line[8192] = "";
+    Address a;
+    memset(&a, 0x00, sizeof(a));
+    a.family = AF_INET;
+    memset(&srep_cat_table, 0x00, sizeof(srep_cat_table));
+
+    BUG_ON(SRepGetVersion() > 0);
+
+    FILE *fp = fopen(filename, "r");
+    if (fp == NULL) {
+        SCLogError(SC_ERR_OPENING_RULE_FILE, "opening ip rep file %s: %s", filename, strerror(errno));
+        return -1;
+    }
+
+    while(fgets(line, (int)sizeof(line), fp) != NULL) {
+        size_t len = strlen(line);
+
+        /* ignore comments and empty lines */
+        if (line[0] == '\n' || line [0] == '\r' || line[0] == ' ' || line[0] == '#' || line[0] == '\t')
+            continue;
+
+        while (isspace(line[--len]));
+
+        /* Check if we have a trailing newline, and remove it */
+        len = strlen(line);
+        if (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r')) {
+            line[len - 1] = '\0';
+        }
+
+        uint8_t cat = 0;
+        char shortname[SREP_SHORTNAME_LEN];
+        if (SRepCatSplitLine(line, &cat, shortname, sizeof(shortname)) == 0) {
+            strlcpy(srep_cat_table[cat], shortname, SREP_SHORTNAME_LEN);
+        } else {
+            SCLogError(SC_ERR_NO_REPUTATION, "bad line \"%s\"", line);
+        }
+    }
+    fclose(fp);
+    fp = NULL;
+
+    SCLogDebug("IP Rep categories:");
+    int i;
+    for (i = 0; i < SREP_MAX_CATS; i++) {
+        if (strlen(srep_cat_table[i]) == 0)
+            continue;
+        SCLogDebug("CAT %d, name %s", i, srep_cat_table[i]);
+    }
+    return 0;
+}
+
+static int SRepLoadFile(char *filename) {
+    char line[8192] = "";
+    Address a;
+    memset(&a, 0x00, sizeof(a));
+    a.family = AF_INET;
+
+    FILE *fp = fopen(filename, "r");
+    if (fp == NULL) {
+        SCLogError(SC_ERR_OPENING_RULE_FILE, "opening ip rep file \"%s\": %s", filename, strerror(errno));
+        return -1;
+    }
+
+    while(fgets(line, (int)sizeof(line), fp) != NULL) {
+        size_t len = strlen(line);
+
+        /* ignore comments and empty lines */
+        if (line[0] == '\n' || line [0] == '\r' || line[0] == ' ' || line[0] == '#' || line[0] == '\t')
+            continue;
+
+        while (isspace(line[--len]));
+
+        /* Check if we have a trailing newline, and remove it */
+        len = strlen(line);
+        if (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r')) {
+            line[len - 1] = '\0';
+        }
+
+        uint32_t ip = 0;
+        uint8_t cat = 0, value = 0;
+        int r = SRepSplitLine(line, &ip, &cat, &value);
+        if (r < 0) {
+            SCLogError(SC_ERR_NO_REPUTATION, "bad line \"%s\"", line);
+        } else if (r == 0) {
+            char ipstr[16];
+            PrintInet(AF_INET, (const void *)&ip, ipstr, sizeof(ipstr));
+            SCLogDebug("%s %u %u", ipstr, cat, value);
+
+            a.addr_data32[0] = ip;
+            Host *h = HostGetHostFromHash(&a);
+            if (h) {
+                //SCLogInfo("host %p", h);
+
+                if (h->iprep == NULL) {
+                    h->iprep = SCMalloc(sizeof(SReputation));
+                    if (h->iprep != NULL)
+                        memset(h->iprep, 0x00, sizeof(SReputation));
+                }
+                if (h->iprep != NULL) {
+                    SReputation *rep = h->iprep;
+
+                    /* if version is 0, it has been used before, so
+                     * clear it */
+                    if (rep->version != 0) {
+                        memset(rep, 0x00, sizeof(SReputation));
+                    }
+
+                    rep->version = SRepGetVersion();
+                    rep->rep[cat] = value;
+                }
+
+                HostRelease(h);
+            }
+        }
+    }
+    fclose(fp);
+    fp = NULL;
+
+    return 0;
+}
+
+/**
+ *  \brief Create the path if default-rule-path was specified
+ *  \param sig_file The name of the file
+ *  \retval str Pointer to the string path + sig_file
+ */
+static char *SRepCompleteFilePath(char *file)
+{
+    char *defaultpath = NULL;
+    char *path = NULL;
+
+    /* Path not specified */
+    if (PathIsRelative(file)) {
+        if (ConfGet("default-reputation-path", &defaultpath) == 1) {
+            SCLogDebug("Default path: %s", defaultpath);
+            size_t path_len = sizeof(char) * (strlen(defaultpath) +
+                          strlen(file) + 2);
+            path = SCMalloc(path_len);
+            if (unlikely(path == NULL))
+                return NULL;
+            strlcpy(path, defaultpath, path_len);
+#if defined OS_WIN32 || defined __CYGWIN__
+            if (path[strlen(path) - 1] != '\\')
+                strlcat(path, "\\\\", path_len);
+#else
+            if (path[strlen(path) - 1] != '/')
+                strlcat(path, "/", path_len);
+#endif
+            strlcat(path, file, path_len);
+       } else {
+            path = SCStrdup(file);
+            if (unlikely(path == NULL))
+                return NULL;
+        }
+    } else {
+        path = SCStrdup(file);
+        if (unlikely(path == NULL))
+            return NULL;
+    }
+    return path;
+}
+
+/** \brief init reputation
+ *
+ *  \param de_ctx detection engine ctx for tracking iprep version
+ *
+ *  \retval 0 ok
+ *  \retval -1 error
+ *
+ *  If this function is called more than once, the category file
+ *  is not reloaded.
+ */
+int SRepInit(DetectEngineCtx *de_ctx) {
+    ConfNode *files;
+    ConfNode *file = NULL;
+    int r = 0;
+    char *sfile = NULL;
+    char *filename = NULL;
+    int init = 0;
+
+    if (SRepGetVersion() == 0) {
+        init = 1;
+    }
+
+    if (init) {
+        if (ConfGet("reputation-categories-file", &filename) != 1) {
+            SCLogError(SC_ERR_NO_REPUTATION, "\"reputation-categories-file\" not set");
+            return -1;
+        }
+    }
+
+    files = ConfGetNode("reputation-files");
+    if (files == NULL) {
+        SCLogError(SC_ERR_NO_REPUTATION, "\"reputation-files\" not set");
+        return -1;
+    }
+
+    if (init) {
+        /* init even if we have reputation files, so that when we
+         * have a live reload, we have inited the cats */
+        if (SRepLoadCatFile(filename) < 0) {
+            SCLogError(SC_ERR_NO_REPUTATION, "failed to load reputation "
+                    "categories file %s", filename);
+            return -1;
+        }
+    }
+
+    de_ctx->srep_version = SRepIncrVersion();
+    SCLogDebug("Reputation version %u", de_ctx->srep_version);
+
+    /* ok, let's load signature files from the general config */
+    if (files != NULL) {
+        TAILQ_FOREACH(file, &files->head, next) {
+            sfile = SRepCompleteFilePath(file->val);
+            SCLogInfo("Loading reputation file: %s", sfile);
+
+            r = SRepLoadFile(sfile);
+            if (r < 0){
+                SCLogWarning(SC_ERR_NO_REPUTATION, "no reputation loaded from \"%s\"", sfile);
+                if (de_ctx->failure_fatal == 1) {
+                    exit(EXIT_FAILURE);
+                }
+            }
+            SCFree(sfile);
+        }
+    }
+
+    /* Set effective rep version.
+     * On live reload we will handle this after de_ctx has been swapped */
+    if (init) {
+        SRepInitComplete();
+    }
+    return 0;
+}
+
+#ifdef UNITTESTS
+static int SRepTest01(void) {
+    char str[] = "1.2.3.4,1,2";
+
+    uint32_t ip = 0;
+    uint8_t cat = 0, value = 0;
+    if (SRepSplitLine(str, &ip, &cat, &value) != 0) {
+        return 0;
+    }
+
+    char ipstr[16];
+    PrintInet(AF_INET, (const void *)&ip, ipstr, sizeof(ipstr));
+
+    if (strcmp(ipstr, "1.2.3.4") != 0)
+        return 0;
+
+    if (cat != 1)
+        return 0;
+
+    if (value != 2)
+        return 0;
+
+    return 1;
+}
+
+static int SRepTest02(void) {
+    char str[] = "1.1.1.1,";
+
+    uint32_t ip = 0;
+    uint8_t cat = 0, value = 0;
+    if (SRepSplitLine(str, &ip, &cat, &value) == 0) {
+        return 0;
+    }
+    return 1;
+}
+
+static int SRepTest03(void) {
+    char str[] = "1,Shortname,Long Name";
+
+    uint8_t cat = 0;
+    char shortname[SREP_SHORTNAME_LEN];
+
+    if (SRepCatSplitLine(str, &cat, shortname, sizeof(shortname)) != 0) {
+        printf("split failed: ");
+        return 0;
+    }
+
+    if (strcmp(shortname, "Shortname") != 0) {
+        printf("%s != Shortname: ", shortname);
+        return 0;
+    }
+
+    if (cat != 1) {
+        printf("cat 1 != %u: ", cat);
+        return 0;
+    }
+
+    return 1;
+}
+
+
+#endif
 
 /** Global trees that hold host reputation for IPV4 and IPV6 hosts */
 IPReputationCtx *rep_ctx;
@@ -1452,6 +1942,10 @@ void SCReputationRegisterTests(void)
                    SCReputationTestIPV4Update01, 1);
     UtRegisterTest("SCReputationTestIPV6Update01",
                    SCReputationTestIPV6Update01, 1);
+
+    UtRegisterTest("SRepTest01", SRepTest01, 1);
+    UtRegisterTest("SRepTest02", SRepTest02, 1);
+    UtRegisterTest("SRepTest03", SRepTest03, 1);
 #endif /* UNITTESTS */
 }
 

src/reputation.h

@@ -26,6 +26,20 @@
 #ifndef __REPUTATION_H__
 #define __REPUTATION_H__
 
+#include "detect.h"
+#include "host.h"
+
+#define SREP_MAX_CATS 60
+typedef struct SReputation_ {
+    uint32_t version;
+    uint8_t rep[SREP_MAX_CATS];
+} SReputation;
+
+uint8_t SRepCatGetByShortname(char *shortname);
+int SRepInit(DetectEngineCtx *de_ctx);
+void SRepReloadComplete(void);
+int SRepHostTimedOut(Host *);
+
 /** Reputation numbers (types) that we can use to lookup/update, etc
  *  Please, dont convert this to a enum since we want the same reputation
  *  codes always. */

src/tmqh-packetpool.c

@@ -37,6 +37,7 @@
 #include "threadvars.h"
 #include "flow.h"
 #include "flow-util.h"
+#include "host.h"
 
 #include "stream.h"
 #include "stream-tcp-reassemble.h"

src/util-error.c

@@ -239,6 +239,7 @@ const char * SCErrorToString(SCError err)
         CASE_CODE (SC_ERR_NO_LUAJIT_SUPPORT);
         CASE_CODE (SC_ERR_LUAJIT_ERROR);
         CASE_CODE (SC_ERR_DEFRAG_INIT);
+        CASE_CODE (SC_ERR_NO_REPUTATION);
         default:
             return "UNKNOWN_ERROR";
     }

src/util-error.h

@@ -253,6 +253,7 @@ typedef enum {
     SC_ERR_NAPATECH_STREAMS_REGISTER_FAILED,
     SC_ERR_NAPATECH_STAT_DROPS_FAILED,
     SC_ERR_NAPATECH_PARSE_CONFIG,
+    SC_ERR_NO_REPUTATION,
 } SCError;
 
 const char *SCErrorToString(SCError);

suricata.yaml.in

@@ -828,6 +828,11 @@ action-order:
   - reject
   - alert
 
+# IP Reputation
+#reputation-categories-file: @e_sysconfdir@iprep/categories.txt
+#default-reputation-path: @e_sysconfdir@iprep
+#reputation-files:
+# - reputation.list
 
 # Host specific policies for defragmentation and TCP stream
 # reassembly.  The host OS lookup is done using a radix tree, just