output: Support buffer-size value

Issue: 3449

src/util-logopenfile.c

@@ -31,6 +31,7 @@
 #include "util-byte.h"
 #include "util-conf.h"
 #include "util-path.h"
+#include "util-misc.h"
 #include "util-time.h"
 
 #if defined(HAVE_SYS_UN_H) && defined(HAVE_SYS_SOCKET_H) && defined(HAVE_SYS_TYPES_H)
@@ -223,7 +224,7 @@ static int SCLogFileWriteNoLock(const char *buffer, int buffer_len, LogFileCtx *
                         log_ctx->filename);
             }
             log_ctx->output_errors++;
-        } else {
+        } else if (log_ctx->buffer_size) {
             SCFflushUnlocked(log_ctx->fp);
         }
     }
@@ -307,8 +308,11 @@ static char *SCLogFilenameFromPattern(const char *pattern)
 static void SCLogFileCloseNoLock(LogFileCtx *log_ctx)
 {
     SCLogDebug("Closing %s", log_ctx->filename);
-    if (log_ctx->fp)
+    if (log_ctx->fp) {
+        if (log_ctx->buffer_size)
+            SCFflushUnlocked(log_ctx->fp);
         fclose(log_ctx->fp);
+    }
 
     if (log_ctx->output_errors) {
         SCLogError("There were %" PRIu64 " output errors to %s", log_ctx->output_errors,
@@ -402,8 +406,8 @@ error_exit:
  *  \retval FILE* on success
  *  \retval NULL on error
  */
-static FILE *
-SCLogOpenFileFp(const char *path, const char *append_setting, uint32_t mode)
+static FILE *SCLogOpenFileFp(
+        const char *path, const char *append_setting, uint32_t mode, const uint32_t buffer_size)
 {
     FILE *ret = NULL;
 
@@ -426,6 +430,7 @@ SCLogOpenFileFp(const char *path, const char *append_setting, uint32_t mode)
 
     if (ret == NULL) {
         SCLogError("Error opening file: \"%s\": %s", filename, strerror(errno));
+        goto error_exit;
     } else {
         if (mode != 0) {
 #ifdef OS_WIN32
@@ -439,7 +444,19 @@ SCLogOpenFileFp(const char *path, const char *append_setting, uint32_t mode)
         }
     }
 
+    /* Set buffering behavior */
+    if (buffer_size == 0) {
+        setbuf(ret, NULL);
+        SCLogConfig("Setting output to %s non-buffered", filename);
+    } else {
+        if (setvbuf(ret, NULL, _IOFBF, buffer_size) < 0)
+            FatalError("unable to set %s to buffered: %d", filename, buffer_size);
+        SCLogConfig("Setting output to %s buffered [limit %d bytes]", filename, buffer_size);
+    }
+
+error_exit:
     SCFree(filename);
+
     return ret;
 }
 
@@ -519,6 +536,22 @@ SCConfLogOpenGeneric(ConfNode *conf,
     if (filetype == NULL)
         filetype = DEFAULT_LOG_FILETYPE;
 
+    /* Determine the buffering for this output device; a value of 0 means to not buffer;
+     * any other value must be a multiple of 4096
+     */
+    uint32_t buffer_size = LOGFILE_EVE_BUFFER_SIZE;
+    const char *buffer_size_value = ConfNodeLookupChildValue(conf, "buffer-size");
+    if (buffer_size_value != NULL) {
+        uint32_t value;
+        if (ParseSizeStringU32(buffer_size_value, &value) < 0) {
+            FatalError("Error parsing "
+                       "buffer-size - %s. Killing engine",
+                    buffer_size_value);
+        }
+        buffer_size = value;
+    }
+
+    SCLogDebug("buffering: %s -> %d", buffer_size_value, buffer_size);
     const char *filemode = ConfNodeLookupChildValue(conf, "filemode");
     uint32_t mode = 0;
     if (filemode != NULL && StringParseUint32(&mode, 8, (uint16_t)strlen(filemode), filemode) > 0) {
@@ -563,6 +596,10 @@ SCConfLogOpenGeneric(ConfNode *conf,
         }
     }
 #endif
+    if (!(strcasecmp(filetype, DEFAULT_LOG_FILETYPE) == 0 || strcasecmp(filetype, "file") == 0)) {
+        SCLogConfig("buffering setting ignored for %s output types", filetype);
+    }
+
     // Now, what have we been asked to open?
     if (strcasecmp(filetype, "unix_stream") == 0) {
 #ifdef BUILD_WITH_UNIXSOCKET
@@ -585,8 +622,10 @@ SCConfLogOpenGeneric(ConfNode *conf,
     } else if (strcasecmp(filetype, DEFAULT_LOG_FILETYPE) == 0 ||
                strcasecmp(filetype, "file") == 0) {
         log_ctx->is_regular = 1;
+        log_ctx->buffer_size = buffer_size;
         if (!log_ctx->threaded) {
-            log_ctx->fp = SCLogOpenFileFp(log_path, append, log_ctx->filemode);
+            log_ctx->fp =
+                    SCLogOpenFileFp(log_path, append, log_ctx->filemode, log_ctx->buffer_size);
             if (log_ctx->fp == NULL)
                 return -1; // Error already logged by Open...Fp routine
         } else {
@@ -648,7 +687,8 @@ int SCConfLogReopen(LogFileCtx *log_ctx)
     /* Reopen the file. Append is forced in case the file was not
      * moved as part of a rotation process. */
     SCLogDebug("Reopening log file %s.", log_ctx->filename);
-    log_ctx->fp = SCLogOpenFileFp(log_ctx->filename, "yes", log_ctx->filemode);
+    log_ctx->fp =
+            SCLogOpenFileFp(log_ctx->filename, "yes", log_ctx->filemode, log_ctx->buffer_size);
     if (log_ctx->fp == NULL) {
         return -1; // Already logged by Open..Fp routine.
     }
@@ -827,7 +867,7 @@ static bool LogFileNewThreadedCtx(LogFileCtx *parent_ctx, const char *log_path,
         }
         SCLogDebug("%s: thread open -- using name %s [replaces %s] - thread %d [slot %d]",
                 t_thread_name, fname, log_path, entry->internal_thread_id, entry->slot_number);
-        thread->fp = SCLogOpenFileFp(fname, append, thread->filemode);
+        thread->fp = SCLogOpenFileFp(fname, append, thread->filemode, parent_ctx->buffer_size);
         if (thread->fp == NULL) {
             goto error;
         }

src/util-logopenfile.h

@@ -107,6 +107,9 @@ typedef struct LogFileCtx_ {
     /** File permissions */
     uint32_t filemode;
 
+    /** File buffering */
+    uint32_t buffer_size;
+
     /** Suricata sensor name */
     char *sensor_name;
 
@@ -164,6 +167,9 @@ typedef struct LogFileCtx_ {
 /* flags for LogFileCtx */
 #define LOGFILE_ROTATE_INTERVAL 0x04
 
+/* Default EVE output buffering size */
+#define LOGFILE_EVE_BUFFER_SIZE (8 * 1024)
+
 LogFileCtx *LogFileNewCtx(void);
 int LogFileFreeCtx(LogFileCtx *);
 int LogFileWrite(LogFileCtx *file_ctx, MemBuffer *buffer);