From 50b5a3a56d3b4623d2cf193c2e796fb345385dac Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Tue, 9 Jan 2018 07:51:26 -0600 Subject: [PATCH] suricatactl: a new python script for misc. tasks Use a new directory, Python to host the Suricata python modules. One entry point is suricatactl, a control script for miscalleneous tasks. Currently onl filestore pruning is implemented. --- Makefile.am | 2 +- configure.ac | 2 +- python/.gitignore | 3 + python/Makefile.am | 29 +++++++ python/bin/suricatactl | 40 +++++++++ python/setup.py | 32 +++++++ python/suricata/__init__.py | 0 python/suricata/ctl/__init__.py | 0 python/suricata/ctl/filestore.py | 118 ++++++++++++++++++++++++++ python/suricata/ctl/loghandler.py | 79 +++++++++++++++++ python/suricata/ctl/main.py | 50 +++++++++++ python/suricata/ctl/test_filestore.py | 18 ++++ 12 files changed, 371 insertions(+), 2 deletions(-) create mode 100644 python/.gitignore create mode 100644 python/Makefile.am create mode 100755 python/bin/suricatactl create mode 100644 python/setup.py create mode 100644 python/suricata/__init__.py create mode 100644 python/suricata/ctl/__init__.py create mode 100644 python/suricata/ctl/filestore.py create mode 100644 python/suricata/ctl/loghandler.py create mode 100644 python/suricata/ctl/main.py create mode 100644 python/suricata/ctl/test_filestore.py diff --git a/Makefile.am b/Makefile.am index baaeb17445..4823047296 100644 --- a/Makefile.am +++ b/Makefile.am @@ -5,7 +5,7 @@ ACLOCAL_AMFLAGS = -I m4 EXTRA_DIST = ChangeLog COPYING LICENSE suricata.yaml.in \ classification.config threshold.config \ reference.config -SUBDIRS = $(HTP_DIR) rust src qa rules doc contrib scripts etc +SUBDIRS = $(HTP_DIR) rust src qa rules doc contrib scripts etc python CLEANFILES = stamp-h[0-9]* diff --git a/configure.ac b/configure.ac index 85ce982440..3c94f9a006 100644 --- a/configure.ac +++ b/configure.ac @@ -2125,7 +2125,7 @@ AC_SUBST(CONFIGURE_SYSCONDIR) AC_SUBST(CONFIGURE_LOCALSTATEDIR) AC_SUBST(PACKAGE_VERSION) -AC_OUTPUT(Makefile src/Makefile rust/Makefile rust/Cargo.toml rust/.cargo/config qa/Makefile qa/coccinelle/Makefile rules/Makefile doc/Makefile doc/userguide/Makefile contrib/Makefile contrib/file_processor/Makefile contrib/file_processor/Action/Makefile contrib/file_processor/Processor/Makefile contrib/tile_pcie_logd/Makefile suricata.yaml scripts/Makefile scripts/suricatasc/Makefile scripts/suricatasc/suricatasc etc/Makefile etc/suricata.logrotate etc/suricata.service) +AC_OUTPUT(Makefile src/Makefile rust/Makefile rust/Cargo.toml rust/.cargo/config qa/Makefile qa/coccinelle/Makefile rules/Makefile doc/Makefile doc/userguide/Makefile contrib/Makefile contrib/file_processor/Makefile contrib/file_processor/Action/Makefile contrib/file_processor/Processor/Makefile contrib/tile_pcie_logd/Makefile suricata.yaml scripts/Makefile scripts/suricatasc/Makefile scripts/suricatasc/suricatasc etc/Makefile etc/suricata.logrotate etc/suricata.service python/Makefile) SURICATA_BUILD_CONF="Suricata Configuration: AF_PACKET support: ${enable_af_packet} diff --git a/python/.gitignore b/python/.gitignore new file mode 100644 index 0000000000..05b2dbafde --- /dev/null +++ b/python/.gitignore @@ -0,0 +1,3 @@ +*.pyc +.cache +build diff --git a/python/Makefile.am b/python/Makefile.am new file mode 100644 index 0000000000..e9b6bb63dc --- /dev/null +++ b/python/Makefile.am @@ -0,0 +1,29 @@ +EXTRA_DIST = setup.py \ + bin \ + suricata + +if HAVE_PYTHON +all-local: + cd $(srcdir) && \ + $(HAVE_PYTHON) setup.py build --build-base $(abs_builddir) + +install-exec-local: + cd $(srcdir) && \ + $(HAVE_PYTHON) setup.py build --build-base $(abs_builddir) \ + install --prefix $(DESTDIR)$(prefix) + +uninstall-local: + rm -f $(DESTDIR)$(bindir)/suricatactl + rm -rf $(DESTDIR)$(prefix)/lib*/python*/site-packages/suricata + rm -rf $(DESTDIR)$(prefix)/lib*/python*/site-packages/suricata-[0-9]*.egg-info + +clean-local: + cd $(srcdir) && \ + $(HAVE_PYTHON) setup.py clean \ + --build-base $(abs_builddir) + rm -rf scripts-* lib* build + find . -name \*.pyc -print0 | xargs -0 rm -f + +distclean-local: + rm -f version +endif diff --git a/python/bin/suricatactl b/python/bin/suricatactl new file mode 100755 index 0000000000..12e55272fe --- /dev/null +++ b/python/bin/suricatactl @@ -0,0 +1,40 @@ +#! /usr/bin/env python +# +# Copyright (C) 2017 Open Information Security Foundation +# +# You can copy, redistribute or modify this Program under the terms of +# the GNU General Public License version 2 as published by the Free +# Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# version 2 along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. + +import sys +import os +import site + +exec_dir = os.path.dirname(__file__) + +if os.path.exists(os.path.join(exec_dir, "..", "suricata", "ctl", "main.py")): + # Looks like we're running from the development directory. + sys.path.insert(0, ".") +else: + # This is to find the suricata module in the case of being installed + # to a non-standard prefix. + version_info = sys.version_info + pyver = "%d.%d" % (version_info.major, version_info.minor) + path = os.path.join( + exec_dir, "..", "lib", "python%s" % (pyver), "site-packages", + "suricata") + if os.path.exists(path): + sys.path.insert(0, os.path.dirname(path)) + +from suricata.ctl.main import main +sys.exit(main()) diff --git a/python/setup.py b/python/setup.py new file mode 100644 index 0000000000..eca9a92483 --- /dev/null +++ b/python/setup.py @@ -0,0 +1,32 @@ +from __future__ import print_function + +import os +import re +import sys + +from distutils.core import setup + +version = None +if os.path.exists("../configure.ac"): + with open("../configure.ac", "r") as conf: + for line in conf: + m = re.search("AC_INIT\(suricata,\s+(\d.+)\)", line) + if m: + version = m.group(1) + break +if version is None: + print("error: failed to parse Suricata version, will use 0.0.0", + file=sys.stderr) + version = "0.0.0" + +setup( + name="suricata", + version=version, + packages=[ + "suricata", + "suricata.ctl", + ], + scripts=[ + "bin/suricatactl", + ] +) diff --git a/python/suricata/__init__.py b/python/suricata/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/python/suricata/ctl/__init__.py b/python/suricata/ctl/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/python/suricata/ctl/filestore.py b/python/suricata/ctl/filestore.py new file mode 100644 index 0000000000..f9f804d343 --- /dev/null +++ b/python/suricata/ctl/filestore.py @@ -0,0 +1,118 @@ +# Copyright (C) 2018 Open Information Security Foundation +# +# You can copy, redistribute or modify this Program under the terms of +# the GNU General Public License version 2 as published by the Free +# Software Foundation. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# version 2 along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. + +from __future__ import print_function + +import sys +import os +import os.path +import time +import re +import glob +import logging + +logger = logging.getLogger("filestore") + +class InvalidAgeFormatError(Exception): + pass + +def register_args(parser): + + parsers = parser.add_subparsers() + + prune_parser = parsers.add_parser("prune") + prune_parser.add_argument("-d", "--directory", help="filestore directory") + prune_parser.add_argument("--age", help="prune files older than age") + prune_parser.add_argument( + "-n", "--dry-run", action="store_true", default=False, + help="only print what would happen"); + prune_parser.add_argument( + "-v", "--verbose", action="store_true", + default=False, help="increase verbosity") + prune_parser.add_argument( + "-q", "--quiet", action="store_true", default=False, + help="be quiet, log warnings and errors only") + prune_parser.set_defaults(func=prune) + +def is_fileinfo(path): + return path.endswith(".json") + +def parse_age(age): + m = re.match("(\d+)\s*(\w+)", age) + if not m: + raise InvalidAgeFormatError(age) + val = int(m.group(1)) + unit = m.group(2) + + if unit == "s": + return val + elif unit == "m": + return val * 60 + elif unit == "h": + return val * 60 * 60 + elif unit == "d": + return val * 60 * 60 * 24 + else: + raise InvalidAgeFormatError("bad unit: %s" % (unit)) + +def get_filesize(path): + return os.stat(path).st_size + +def remove_file(path, dry_run): + size = 0 + size += get_filesize(path) + if not dry_run: + os.unlink(path) + return size + +def prune(args): + + if args.verbose: + logger.setLevel(logging.DEBUG) + if args.quiet: + logger.setLevel(logging.WARNING) + + if not args.directory: + print( + "error: the filestore directory must be provided with --directory", + file=sys.stderr) + return 1 + + if not args.age: + print("error: no age provided, nothing to do", file=sys.stderr) + return 1 + + age = parse_age(args.age) + now = time.time() + size = 0 + count = 0 + + for dirpath, dirnames, filenames in os.walk(args.directory, topdown=True): + + # Do not go into the tmp directory. + if "tmp" in dirnames: + dirnames.remove("tmp") + + for filename in filenames: + path = os.path.join(dirpath, filename) + mtime = os.path.getmtime(path) + this_age = now - mtime + if this_age > age: + logger.debug("Deleting %s; age=%ds" % (path, this_age)) + size += remove_file(path, args.dry_run) + count += 1 + + logger.info("Removed %d files; %d bytes." % (count, size)) diff --git a/python/suricata/ctl/loghandler.py b/python/suricata/ctl/loghandler.py new file mode 100644 index 0000000000..f417eca8a7 --- /dev/null +++ b/python/suricata/ctl/loghandler.py @@ -0,0 +1,79 @@ +# Copyright (C) 2017 Open Information Security Foundation +# Copyright (c) 2016 Jason Ish +# +# You can copy, redistribute or modify this Program under the terms of +# the GNU General Public License version 2 as published by the Free +# Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# version 2 along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. + +import logging +import time + +GREEN = "\x1b[32m" +BLUE = "\x1b[34m" +REDB = "\x1b[1;31m" +YELLOW = "\x1b[33m" +RED = "\x1b[31m" +YELLOWB = "\x1b[1;33m" +ORANGE = "\x1b[38;5;208m" +RESET = "\x1b[0m" + +# A list of secrets that will be replaced in the log output. +secrets = {} + +def add_secret(secret, replacement): + """Register a secret to be masked. The secret will be replaced with: + + """ + secrets[str(secret)] = str(replacement) + +class SuriColourLogHandler(logging.StreamHandler): + """An alternative stream log handler that logs with Suricata inspired + log colours.""" + + def formatTime(self, record): + lt = time.localtime(record.created) + t = "%d/%d/%d -- %02d:%02d:%02d" % (lt.tm_mday, + lt.tm_mon, + lt.tm_year, + lt.tm_hour, + lt.tm_min, + lt.tm_sec) + return "%s" % (t) + + def emit(self, record): + + if record.levelname == "ERROR": + level_prefix = REDB + message_prefix = REDB + elif record.levelname == "WARNING": + level_prefix = ORANGE + message_prefix = ORANGE + else: + level_prefix = YELLOW + message_prefix = "" + + self.stream.write("%s%s%s - <%s%s%s> -- %s%s%s\n" % ( + GREEN, + self.formatTime(record), + RESET, + level_prefix, + record.levelname.title(), + RESET, + message_prefix, + self.mask_secrets(record.getMessage()), + RESET)) + + def mask_secrets(self, msg): + for secret in secrets: + msg = msg.replace(secret, "<%s>" % secrets[secret]) + return msg diff --git a/python/suricata/ctl/main.py b/python/suricata/ctl/main.py new file mode 100644 index 0000000000..6a742adaf4 --- /dev/null +++ b/python/suricata/ctl/main.py @@ -0,0 +1,50 @@ +# Copyright (C) 2018 Open Information Security Foundation +# +# You can copy, redistribute or modify this Program under the terms of +# the GNU General Public License version 2 as published by the Free +# Software Foundation. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# version 2 along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. + +import sys +import os +import argparse +import logging + +from suricata.ctl import filestore +from suricata.ctl import loghandler + +def init_logger(): + """ Initialize logging, use colour if on a tty. """ + if os.isatty(sys.stderr.fileno()): + logger = logging.getLogger() + logger.setLevel(level=logging.INFO) + logger.addHandler(loghandler.SuriColourLogHandler()) + else: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - <%(levelname)s> - %(message)s") + +def main(): + + init_logger() + + parser = argparse.ArgumentParser(description="Suricata Control Tool") + + subparsers = parser.add_subparsers( + title="subcommands", + description="Commands") + + filestore.register_args(subparsers.add_parser("filestore")) + + args = parser.parse_args() + + args.func(args) diff --git a/python/suricata/ctl/test_filestore.py b/python/suricata/ctl/test_filestore.py new file mode 100644 index 0000000000..26b107fd23 --- /dev/null +++ b/python/suricata/ctl/test_filestore.py @@ -0,0 +1,18 @@ +from __future__ import print_function + +import unittest + +import filestore + +class PruneTestCase(unittest.TestCase): + + def test_parse_age(self): + self.assertEqual(filestore.parse_age("1s"), 1) + self.assertEqual(filestore.parse_age("1m"), 60) + self.assertEqual(filestore.parse_age("1h"), 3600) + self.assertEqual(filestore.parse_age("1d"), 86400) + + with self.assertRaises(filestore.InvalidAgeFormatError) as err: + filestore.parse_age("1") + with self.assertRaises(filestore.InvalidAgeFormatError) as err: + filestore.parse_age("1y")