file: optionally use detect tracking in pruning

When the file API is used to do content inspection (currently only
smtp does this), the detection should be considered while pruning
the file chunks.

This patch introduces a new flag for the file API: FILE_USE_DETECT

When it is used, 'FilePrune' will not remove chunks that are (partly)
beyond the File::content_inspected tracker.

When using this flag, it's important to realize that when the detect
engine is disabled or rules are not matching, content_inspected
might not get updated.

src/util-file.c

@@ -173,6 +173,14 @@ static int FilePruneFile(File *file)
         SCLogDebug("fd %p", fd);
 
         if (file->flags & FILE_NOSTORE || fd->stored == 1) {
+            /* keep chunks in memory as long as we still need to
+             * inspect them or parts of them */
+            if (file->flags & FILE_USE_DETECT) {
+                uint64_t right_edge = fd->stream_offset + fd->len;
+                if (file->content_inspected < right_edge)
+                    break;
+            }
+
             file->chunks_head = fd->next;
             if (file->chunks_tail == fd)
                 file->chunks_tail = fd->next;
@@ -563,6 +571,10 @@ File *FileOpenFile(FileContainer *ffc, const uint8_t *name, uint16_t name_len,
         SCLogDebug("not doing md5 for this file");
         ff->flags |= FILE_NOMD5;
     }
+    if (flags & FILE_USE_DETECT) {
+        SCLogDebug("considering content_inspect tracker when pruning");
+        ff->flags |= FILE_USE_DETECT;
+    }
 
 #ifdef HAVE_NSS
     if (!(ff->flags & FILE_NOMD5) || g_file_force_md5) {

src/util-file.h

@@ -38,6 +38,7 @@
 #define FILE_STORE      0x0040
 #define FILE_STORED     0x0080
 #define FILE_NOTRACK    0x0100 /**< track size of file */
+#define FILE_USE_DETECT 0x0200 /**< use content_inspected tracker */
 
 typedef enum FileState_ {
     FILE_STATE_NONE = 0,    /**< no state */
@@ -80,7 +81,8 @@ typedef struct File_ {
     uint64_t chunks_cnt_max;
 #endif
     uint64_t content_len_so_far;
-    uint64_t content_inspected;
+    uint64_t content_inspected;     /**< used in pruning if FILE_USE_DETECT
+                                     *   flag is set */
 } File;
 
 typedef struct FileContainer_ {