suricatactl: a new python script for misc. tasks

Use a new directory, Python to host the Suricata python modules.
One entry point is suricatactl, a control script for
miscalleneous tasks. Currently onl filestore pruning
is implemented.

Makefile.am

@@ -5,7 +5,7 @@ ACLOCAL_AMFLAGS = -I m4
 EXTRA_DIST = ChangeLog COPYING LICENSE suricata.yaml.in \
              classification.config threshold.config \
              reference.config
-SUBDIRS = $(HTP_DIR) rust src qa rules doc contrib scripts etc
+SUBDIRS = $(HTP_DIR) rust src qa rules doc contrib scripts etc python
 
 CLEANFILES = stamp-h[0-9]*
 

configure.ac

@@ -2125,7 +2125,7 @@ AC_SUBST(CONFIGURE_SYSCONDIR)
 AC_SUBST(CONFIGURE_LOCALSTATEDIR)
 AC_SUBST(PACKAGE_VERSION)
 
-AC_OUTPUT(Makefile src/Makefile rust/Makefile rust/Cargo.toml rust/.cargo/config qa/Makefile qa/coccinelle/Makefile rules/Makefile doc/Makefile doc/userguide/Makefile contrib/Makefile contrib/file_processor/Makefile contrib/file_processor/Action/Makefile contrib/file_processor/Processor/Makefile contrib/tile_pcie_logd/Makefile suricata.yaml scripts/Makefile scripts/suricatasc/Makefile scripts/suricatasc/suricatasc etc/Makefile etc/suricata.logrotate etc/suricata.service)
+AC_OUTPUT(Makefile src/Makefile rust/Makefile rust/Cargo.toml rust/.cargo/config qa/Makefile qa/coccinelle/Makefile rules/Makefile doc/Makefile doc/userguide/Makefile contrib/Makefile contrib/file_processor/Makefile contrib/file_processor/Action/Makefile contrib/file_processor/Processor/Makefile contrib/tile_pcie_logd/Makefile suricata.yaml scripts/Makefile scripts/suricatasc/Makefile scripts/suricatasc/suricatasc etc/Makefile etc/suricata.logrotate etc/suricata.service python/Makefile)
 
 SURICATA_BUILD_CONF="Suricata Configuration:
   AF_PACKET support:                       ${enable_af_packet}

python/.gitignore

@@ -0,0 +1,3 @@
+*.pyc
+.cache
+build

python/Makefile.am

@@ -0,0 +1,29 @@
+EXTRA_DIST =	setup.py \
+		bin \
+		suricata
+
+if HAVE_PYTHON
+all-local:
+	cd $(srcdir) && \
+		$(HAVE_PYTHON) setup.py build --build-base $(abs_builddir)
+
+install-exec-local:
+	cd $(srcdir) && \
+		$(HAVE_PYTHON) setup.py build --build-base $(abs_builddir) \
+		install --prefix $(DESTDIR)$(prefix)
+
+uninstall-local:
+	rm -f $(DESTDIR)$(bindir)/suricatactl
+	rm -rf $(DESTDIR)$(prefix)/lib*/python*/site-packages/suricata
+	rm -rf $(DESTDIR)$(prefix)/lib*/python*/site-packages/suricata-[0-9]*.egg-info
+
+clean-local:
+	cd $(srcdir) && \
+		$(HAVE_PYTHON) setup.py clean \
+		--build-base $(abs_builddir)
+	rm -rf scripts-* lib* build
+	find . -name \*.pyc -print0 | xargs -0 rm -f
+
+distclean-local:
+	rm -f version
+endif

python/bin/suricatactl

@@ -0,0 +1,40 @@
+#! /usr/bin/env python
+#
+# Copyright (C) 2017 Open Information Security Foundation
+#
+# You can copy, redistribute or modify this Program under the terms of
+# the GNU General Public License version 2 as published by the Free
+# Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# version 2 along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+import sys
+import os
+import site
+
+exec_dir = os.path.dirname(__file__)
+
+if os.path.exists(os.path.join(exec_dir, "..", "suricata", "ctl", "main.py")):
+    # Looks like we're running from the development directory.
+    sys.path.insert(0, ".")
+else:
+    # This is to find the suricata module in the case of being installed
+    # to a non-standard prefix.
+    version_info = sys.version_info
+    pyver = "%d.%d" % (version_info.major, version_info.minor)
+    path = os.path.join(
+        exec_dir, "..", "lib", "python%s" % (pyver), "site-packages",
+        "suricata")
+    if os.path.exists(path):
+        sys.path.insert(0, os.path.dirname(path))
+
+from suricata.ctl.main import main
+sys.exit(main())

python/setup.py

@@ -0,0 +1,32 @@
+from __future__ import print_function
+
+import os
+import re
+import sys
+
+from distutils.core import setup
+
+version = None
+if os.path.exists("../configure.ac"):
+    with open("../configure.ac", "r") as conf:
+        for line in conf:
+            m = re.search("AC_INIT\(suricata,\s+(\d.+)\)", line)
+            if m:
+                version = m.group(1)
+                break
+if version is None:
+    print("error: failed to parse Suricata version, will use 0.0.0",
+          file=sys.stderr)
+    version = "0.0.0"
+    
+setup(
+    name="suricata",
+    version=version,
+    packages=[
+        "suricata",
+        "suricata.ctl",
+    ],
+    scripts=[
+        "bin/suricatactl",
+    ]
+)

python/suricata/ctl/filestore.py

@@ -0,0 +1,118 @@
+# Copyright (C) 2018 Open Information Security Foundation
+#
+# You can copy, redistribute or modify this Program under the terms of
+# the GNU General Public License version 2 as published by the Free
+# Software Foundation.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# version 2 along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+from __future__ import print_function
+
+import sys
+import os
+import os.path
+import time
+import re
+import glob
+import logging
+
+logger = logging.getLogger("filestore")
+
+class InvalidAgeFormatError(Exception):
+    pass
+
+def register_args(parser):
+
+    parsers = parser.add_subparsers()
+
+    prune_parser = parsers.add_parser("prune")
+    prune_parser.add_argument("-d", "--directory", help="filestore directory")
+    prune_parser.add_argument("--age", help="prune files older than age")
+    prune_parser.add_argument(
+        "-n", "--dry-run", action="store_true", default=False,
+        help="only print what would happen");
+    prune_parser.add_argument(
+        "-v", "--verbose", action="store_true",
+        default=False, help="increase verbosity")
+    prune_parser.add_argument(
+        "-q", "--quiet", action="store_true", default=False,
+        help="be quiet, log warnings and errors only")
+    prune_parser.set_defaults(func=prune)
+
+def is_fileinfo(path):
+    return path.endswith(".json")
+
+def parse_age(age):
+    m = re.match("(\d+)\s*(\w+)", age)
+    if not m:
+        raise InvalidAgeFormatError(age)
+    val = int(m.group(1))
+    unit = m.group(2)
+
+    if unit == "s":
+        return val
+    elif unit == "m":
+        return val * 60
+    elif unit == "h":
+        return val * 60 * 60
+    elif unit == "d":
+        return val * 60 * 60 * 24
+    else:
+        raise InvalidAgeFormatError("bad unit: %s" % (unit))
+
+def get_filesize(path):
+    return os.stat(path).st_size
+
+def remove_file(path, dry_run):
+    size = 0
+    size += get_filesize(path)
+    if not dry_run:
+        os.unlink(path)
+    return size
+
+def prune(args):
+
+    if args.verbose:
+        logger.setLevel(logging.DEBUG)
+    if args.quiet:
+        logger.setLevel(logging.WARNING)
+
+    if not args.directory:
+        print(
+            "error: the filestore directory must be provided with --directory",
+            file=sys.stderr)
+        return 1
+    
+    if not args.age:
+        print("error: no age provided, nothing to do", file=sys.stderr)
+        return 1
+
+    age = parse_age(args.age)
+    now = time.time()
+    size = 0
+    count = 0
+
+    for dirpath, dirnames, filenames in os.walk(args.directory, topdown=True):
+
+        # Do not go into the tmp directory.
+        if "tmp" in dirnames:
+            dirnames.remove("tmp")
+
+        for filename in filenames:
+            path = os.path.join(dirpath, filename)
+            mtime = os.path.getmtime(path)
+            this_age = now - mtime
+            if this_age > age:
+                logger.debug("Deleting %s; age=%ds" % (path, this_age))
+                size += remove_file(path, args.dry_run)
+                count += 1
+
+    logger.info("Removed %d files; %d bytes." % (count, size))

python/suricata/ctl/loghandler.py

@@ -0,0 +1,79 @@
+# Copyright (C) 2017 Open Information Security Foundation
+# Copyright (c) 2016 Jason Ish
+#
+# You can copy, redistribute or modify this Program under the terms of
+# the GNU General Public License version 2 as published by the Free
+# Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# version 2 along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+import logging
+import time
+
+GREEN = "\x1b[32m"
+BLUE = "\x1b[34m"
+REDB = "\x1b[1;31m"
+YELLOW = "\x1b[33m"
+RED = "\x1b[31m"
+YELLOWB = "\x1b[1;33m"
+ORANGE = "\x1b[38;5;208m"
+RESET = "\x1b[0m"
+
+# A list of secrets that will be replaced in the log output.
+secrets = {}
+
+def add_secret(secret, replacement):
+    """Register a secret to be masked. The secret will be replaced with:
+           <replacement>
+    """
+    secrets[str(secret)] = str(replacement)
+
+class SuriColourLogHandler(logging.StreamHandler):
+    """An alternative stream log handler that logs with Suricata inspired
+    log colours."""
+
+    def formatTime(self, record):
+        lt = time.localtime(record.created)
+        t = "%d/%d/%d -- %02d:%02d:%02d" % (lt.tm_mday,
+                                            lt.tm_mon,
+                                            lt.tm_year,
+                                            lt.tm_hour,
+                                            lt.tm_min,
+                                            lt.tm_sec)
+        return "%s" % (t)
+
+    def emit(self, record):
+
+        if record.levelname == "ERROR":
+            level_prefix = REDB
+            message_prefix = REDB
+        elif record.levelname == "WARNING":
+            level_prefix = ORANGE
+            message_prefix = ORANGE
+        else:
+            level_prefix = YELLOW
+            message_prefix = ""
+
+        self.stream.write("%s%s%s - <%s%s%s> -- %s%s%s\n" % (
+            GREEN,
+            self.formatTime(record),
+            RESET,
+            level_prefix,
+            record.levelname.title(),
+            RESET,
+            message_prefix,
+            self.mask_secrets(record.getMessage()),
+            RESET))
+
+    def mask_secrets(self, msg):
+        for secret in secrets:
+            msg = msg.replace(secret, "<%s>" % secrets[secret])
+        return msg

python/suricata/ctl/main.py

@@ -0,0 +1,50 @@
+# Copyright (C) 2018 Open Information Security Foundation
+#
+# You can copy, redistribute or modify this Program under the terms of
+# the GNU General Public License version 2 as published by the Free
+# Software Foundation.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# version 2 along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.
+
+import sys
+import os
+import argparse
+import logging
+
+from suricata.ctl import filestore
+from suricata.ctl import loghandler
+
+def init_logger():
+    """ Initialize logging, use colour if on a tty. """
+    if os.isatty(sys.stderr.fileno()):
+        logger = logging.getLogger()
+        logger.setLevel(level=logging.INFO)
+        logger.addHandler(loghandler.SuriColourLogHandler())
+    else:
+        logging.basicConfig(
+            level=logging.INFO,
+            format="%(asctime)s - <%(levelname)s> - %(message)s")
+
+def main():
+
+    init_logger()
+
+    parser = argparse.ArgumentParser(description="Suricata Control Tool")
+
+    subparsers = parser.add_subparsers(
+        title="subcommands",
+        description="Commands")
+
+    filestore.register_args(subparsers.add_parser("filestore"))
+
+    args = parser.parse_args()
+
+    args.func(args)

python/suricata/ctl/test_filestore.py

@@ -0,0 +1,18 @@
+from __future__ import print_function
+
+import unittest
+
+import filestore
+
+class PruneTestCase(unittest.TestCase):
+
+    def test_parse_age(self):
+        self.assertEqual(filestore.parse_age("1s"), 1)
+        self.assertEqual(filestore.parse_age("1m"), 60)
+        self.assertEqual(filestore.parse_age("1h"), 3600)
+        self.assertEqual(filestore.parse_age("1d"), 86400)
+
+        with self.assertRaises(filestore.InvalidAgeFormatError) as err:
+            filestore.parse_age("1")
+        with self.assertRaises(filestore.InvalidAgeFormatError) as err:
+            filestore.parse_age("1y")