Support for configuration include files.

src/conf-yaml-loader.c

@@ -26,6 +26,7 @@
 #include <yaml.h>
 #include "suricata-common.h"
 #include "conf.h"
+#include "util-path.h"
 #include "util-debug.h"
 #include "util-unittest.h"
 
@@ -40,10 +41,15 @@
 #define MANGLE_ERRORS_MAX 10
 static int mangle_errors = 0;
 
+static char *conf_dirname = NULL;
+
+static int ConfYamlParse(yaml_parser_t *parser, ConfNode *parent, int inseq);
+
 /* Configuration processing states. */
 enum conf_state {
     CONF_KEY = 0,
     CONF_VAL,
+    CONF_INCLUDE,
 };
 
 /**
@@ -64,6 +70,90 @@ Mangle(char *string)
     return;
 }
 
+/**
+ * \brief Set the directory name of the configuration file.
+ *
+ * \param filename The configuration filename.
+ */
+static void
+ConfYamlSetConfDirname(const char *filename)
+{
+    char *ep;
+
+    ep = strrchr(filename, '\\');
+    if (ep == NULL)
+        ep = strrchr(filename, '/');
+
+    if (ep == NULL) {
+        conf_dirname = SCStrdup(".");
+        if (conf_dirname == NULL) {
+            SCLogError(SC_ERR_MEM_ALLOC,
+               "ERROR: Failed to allocate memory while loading configuration.");
+            exit(EXIT_FAILURE);
+        }
+    }
+    else {
+        conf_dirname = SCStrdup(filename);
+        if (conf_dirname == NULL) {
+            SCLogError(SC_ERR_MEM_ALLOC,
+               "ERROR: Failed to allocate memory while loading configuration.");
+            exit(EXIT_FAILURE);
+        }
+        conf_dirname[ep - filename] = '\0';
+    }
+}
+
+/**
+ * \brief Include a file in the configuration.
+ *
+ * \param parent The configuration node the included configuration will be
+ *          placed at.
+ * \param filename The filename to include.
+ *
+ * \retval 0 on success, -1 on failure.
+ */
+static int
+ConfYamlHandleInclude(ConfNode *parent, const char *filename)
+{
+    yaml_parser_t parser;
+    char include_filename[PATH_MAX];
+    FILE *file;
+
+    if (yaml_parser_initialize(&parser) != 1) {
+        SCLogError(SC_ERR_CONF_YAML_ERROR, "Failed to initialize YAML parser");
+        return -1;
+    }
+
+    if (PathIsAbsolute(filename)) {
+        strlcpy(include_filename, filename, sizeof(include_filename));
+    }
+    else {
+        snprintf(include_filename, sizeof(include_filename), "%s/%s",
+            conf_dirname, filename);
+    }
+
+    file = fopen(include_filename, "r");
+    if (file == NULL) {
+        SCLogError(SC_ERR_FOPEN,
+            "Failed to open configuration include file %s: %s",
+            include_filename, strerror(errno));
+        return -1;
+    }
+
+    yaml_parser_set_input_file(&parser, file);
+
+    if (ConfYamlParse(&parser, parent, 0) != 0) {
+        SCLogError(SC_ERR_CONF_YAML_ERROR,
+            "Failed to include configuration file %s", filename);
+        return -1;
+    }
+
+    yaml_parser_delete(&parser);
+    fclose(file);
+
+    return 0;
+}
+
 /**
  * \brief Parse a YAML layer.
  *
@@ -83,13 +173,14 @@ ConfYamlParse(yaml_parser_t *parser, ConfNode *parent, int inseq)
 
     while (!done) {
         if (!yaml_parser_parse(parser, &event)) {
-            fprintf(stderr,
+            SCLogError(SC_ERR_CONF_YAML_ERROR,
                 "Failed to parse configuration file at line %" PRIuMAX ": %s\n",
                 (uintmax_t)parser->problem_mark.line, parser->problem);
             return -1;
         }
 
         if (event.type == YAML_DOCUMENT_START_EVENT) {
+            SCLogDebug("event.type=YAML_DOCUMENT_START_EVENT; state=%d", state);
             /* Verify YAML version - its more likely to be a valid
              * Suricata configuration file if the version is
              * correct. */
@@ -110,8 +201,9 @@ ConfYamlParse(yaml_parser_t *parser, ConfNode *parent, int inseq)
         }
         else if (event.type == YAML_SCALAR_EVENT) {
             char *value = (char *)event.data.scalar.value;
-            SCLogDebug("event.type = YAML_SCALAR_EVENT (%s) inseq=%d",
+            char *tag = (char *)event.data.scalar.tag;
-                value, inseq);
+            SCLogDebug("event.type=YAML_SCALAR_EVENT; state=%d; value=%s; "
+                "tag=%s; inseq=%d", state, value, tag, inseq);
             if (inseq) {
                 ConfNode *seq_node = ConfNodeNew();
                 seq_node->name = SCCalloc(1, DEFAULT_NAME_LEN);
@@ -122,7 +214,22 @@ ConfYamlParse(yaml_parser_t *parser, ConfNode *parent, int inseq)
                 TAILQ_INSERT_TAIL(&parent->head, seq_node, next);
             }
             else {
-                if (state == CONF_KEY) {
+                if (state == CONF_INCLUDE) {
+                    SCLogInfo("Including configuration file %s.", value);
+                    if (ConfYamlHandleInclude(parent, value) != 0) {
+                        goto fail;
+                    }
+                    state = CONF_KEY;
+                }
+                else if (state == CONF_KEY) {
+
+                    /* Top level include statements. */
+                    if ((strcmp(value, "include") == 0) &&
+                        (parent == ConfGetRootNode())) {
+                        state = CONF_INCLUDE;
+                        goto next;
+                    }
+
                     if (parent->is_seq) {
                         if (parent->val == NULL) {
                             parent->val = SCStrdup(value);
@@ -158,7 +265,13 @@ ConfYamlParse(yaml_parser_t *parser, ConfNode *parent, int inseq)
                     state = CONF_VAL;
                 }
                 else {
-                    if (node->allow_override) {
+                    if ((tag != NULL) && (strcmp(tag, "!include") == 0)) {
+                        SCLogInfo("Including configuration file %s at "
+                            "parent node %s.", value, node->name);
+                        if (ConfYamlHandleInclude(node, value) != 0)
+                            goto fail;
+                    }
+                    else if (node->allow_override) {
                         if (node->val != NULL)
                             SCFree(node->val);
                         node->val = SCStrdup(value);
@@ -168,17 +281,17 @@ ConfYamlParse(yaml_parser_t *parser, ConfNode *parent, int inseq)
             }
         }
         else if (event.type == YAML_SEQUENCE_START_EVENT) {
-            SCLogDebug("event.type = YAML_SEQUENCE_START_EVENT");
+            SCLogDebug("event.type=YAML_SEQUENCE_START_EVENT; state=%d", state);
             if (ConfYamlParse(parser, node, 1) != 0)
                 goto fail;
             state = CONF_KEY;
         }
         else if (event.type == YAML_SEQUENCE_END_EVENT) {
-            SCLogDebug("event.type = YAML_SEQUENCE_END_EVENT");
+            SCLogDebug("event.type=YAML_SEQUENCE_END_EVENT; state=%d", state);
             return 0;
         }
         else if (event.type == YAML_MAPPING_START_EVENT) {
-            SCLogDebug("event.type = YAML_MAPPING_START_EVENT");
+            SCLogDebug("event.type=YAML_MAPPING_START_EVENT; state=%d", state);
             if (inseq) {
                 ConfNode *seq_node = ConfNodeNew();
                 seq_node->is_seq = 1;
@@ -197,13 +310,15 @@ ConfYamlParse(yaml_parser_t *parser, ConfNode *parent, int inseq)
             state = CONF_KEY;
         }
         else if (event.type == YAML_MAPPING_END_EVENT) {
-            SCLogDebug("event.type = YAML_MAPPING_END_EVENT");
+            SCLogDebug("event.type=YAML_MAPPING_END_EVENT; state=%d", state);
             done = 1;
         }
         else if (event.type == YAML_STREAM_END_EVENT) {
+            SCLogDebug("event.type=YAML_STREAM_END_EVENT; state=%d", state);
             done = 1;
         }
 
+    next:
         yaml_event_delete(&event);
         continue;
 
@@ -256,6 +371,11 @@ ConfYamlLoadFile(const char *filename)
         yaml_parser_delete(&parser);
         return -1;
     }
+
+    if (conf_dirname == NULL) {
+        ConfYamlSetConfDirname(filename);
+    }
+
     yaml_parser_set_input_file(&parser, infile);
     ret = ConfYamlParse(&parser, root, 0);
     yaml_parser_delete(&parser);
@@ -519,6 +639,92 @@ libhtp:\n\
     return 1;
 }
 
+/**
+ * Test file inclusion support.
+ */
+static int
+ConfYamlFileIncludeTest(void)
+{
+    int ret = 0;
+    FILE *config_file;
+
+    const char config_filename[] = "ConfYamlFileIncludeTest-config.yaml";
+    const char config_file_contents[] =
+        "%YAML 1.1\n"
+        "---\n"
+        "# Include something at the root level.\n"
+        "include: ConfYamlFileIncludeTest-include.yaml\n"
+        "# Test including under a mapping.\n"
+        "mapping: !include ConfYamlFileIncludeTest-include.yaml\n";
+
+    const char include_filename[] = "ConfYamlFileIncludeTest-include.yaml";
+    const char include_file_contents[] =
+        "%YAML 1.1\n"
+        "---\n"
+        "host-mode: auto\n"
+        "unix-command:\n"
+        "  enabled: no\n";
+
+    ConfCreateContextBackup();
+    ConfInit();
+
+    /* Write out the test files. */
+    if ((config_file = fopen(config_filename, "w")) == NULL) {
+        goto cleanup;
+    }
+    if (fwrite(config_file_contents, strlen(config_file_contents), 1,
+            config_file) != 1) {
+        goto cleanup;
+    }
+    fclose(config_file);
+    if ((config_file = fopen(include_filename, "w")) == NULL) {
+        goto cleanup;
+    }
+    if (fwrite(include_file_contents, strlen(include_file_contents), 1,
+            config_file) != 1) {
+        goto cleanup;
+    }
+    fclose(config_file);
+
+    /* Reset conf_dirname. */
+    if (conf_dirname != NULL) {
+        SCFree(conf_dirname);
+        conf_dirname = NULL;
+    }
+
+    if (ConfYamlLoadFile("ConfYamlFileIncludeTest-config.yaml") != 0)
+        goto cleanup;
+
+    /* Check values that should have been loaded into the root of the
+     * configuration. */
+    ConfNode *node;
+    node = ConfGetNode("host-mode");
+    if (node == NULL) goto cleanup;
+    if (strcmp(node->val, "auto") != 0) goto cleanup;
+    node = ConfGetNode("unix-command.enabled");
+    if (node == NULL) goto cleanup;
+    if (strcmp(node->val, "no") != 0) goto cleanup;
+
+    /* Check for values that were included under a mapping. */
+    node = ConfGetNode("mapping.host-mode");
+    if (node == NULL) goto cleanup;
+    if (strcmp(node->val, "auto") != 0) goto cleanup;
+    node = ConfGetNode("mapping.unix-command.enabled");
+    if (node == NULL) goto cleanup;
+    if (strcmp(node->val, "no") != 0) goto cleanup;
+
+    ConfDeInit();
+    ConfRestoreContextBackup();
+
+    ret = 1;
+
+cleanup:
+    unlink(config_filename);
+    unlink(include_filename);
+
+    return ret;
+}
+
 #endif /* UNITTESTS */
 
 void
@@ -531,5 +737,6 @@ ConfYamlRegisterTests(void)
     UtRegisterTest("ConfYamlBadYamlVersionTest", ConfYamlBadYamlVersionTest, 1);
     UtRegisterTest("ConfYamlSecondLevelSequenceTest",
         ConfYamlSecondLevelSequenceTest, 1);
+    UtRegisterTest("ConfYamlFileIncludeTest", ConfYamlFileIncludeTest, 1);
 #endif /* UNITTESTS */
 }

src/util-error.c

@@ -280,6 +280,7 @@ const char * SCErrorToString(SCError err)
         CASE_CODE (SC_WARN_XFF_INVALID_HEADER);
         CASE_CODE (SC_ERR_THRESHOLD_SETUP);
         CASE_CODE (SC_ERR_DNS_CONFIG);
+        CASE_CODE (SC_ERR_CONF_YAML_ERROR);
     }
 
     return "UNKNOWN_ERROR";

src/util-error.h

@@ -269,6 +269,7 @@ typedef enum {
     SC_WARN_XFF_INVALID_HEADER,
     SC_ERR_THRESHOLD_SETUP,
     SC_ERR_DNS_CONFIG,
+    SC_ERR_CONF_YAML_ERROR,
 } SCError;
 
 const char *SCErrorToString(SCError);

suricata.yaml.in

@@ -1203,3 +1203,8 @@ napatech:
 
     # The streams to listen on
     streams: [1, 2, 3]
+
+# Includes.  Files included here will be handled as if they were
+# inlined in this configuration file.
+#include: include1.yaml
+#include: include2.yaml