hs: prune stale MPM cache files

Hyperscan MPM can cache the compiled contexts to files.
This however grows as rulesets change and leads to bloating
the system. This addition prunes the stale cache files based
on their modified file timestamp.

Part of this work incorporates new model for MPM cache stats
to split it out from the cache save function and aggregate
cache-related stats in one place (newly added pruning).

Ticket: 7830

doc/userguide/performance/hyperscan.rst

@@ -83,6 +83,8 @@ if it is present on the system in case of the "auto" setting.
 If the current suricata installation does not have hyperscan
 support, refer to :ref:`installation`
 
+.. _hyperscan-cache-configuration:
+
 Hyperscan caching
 ~~~~~~~~~~~~~~~~~
 
@@ -104,6 +106,24 @@ To enable this function, in `suricata.yaml` configure:
     sgh-mpm-caching-path: /var/lib/suricata/cache/hs
 
 
+To avoid cache files growing indefinitely, Suricata supports pruning of old
+cache files. Suricata removes cache files older than the specified age
+on startup/rule reloads, where age is determined by delta of the file
+modification time and the current time.
+Cache files that are actively being used will have their modification time
+updated when loaded, so they won't be deleted.
+
+In `suricata.yaml` configure:
+
+::
+
+  detect:
+    sgh-mpm-caching-max-age: 7d
+
+The setting accepts a combination of time units (s,m,h,d,w,y),
+e.g. `1w 3d 12h` for 1 week, 3 days and 12 hours. Setting the value to `0`
+disables pruning.
+
 **Note**:
 You might need to create and adjust permissions to the default caching folder
 path, especially if you are running Suricata as a non-root user.

doc/userguide/upgrade.rst

@@ -68,6 +68,10 @@ Other Changes
   from unbounded to 2048. Configuration options, ``max-tx``,
   ``max-points``, and ``max-objects`` have been added for users who
   may need to change these defaults.
+- Hyperscan caching (`detect.sgh-mpm-caching`), when enabled, prunes
+  cache files that have not been used in the last 7 days by default.
+  See :ref:`Hyperscan caching configuration
+  <hyperscan-cache-configuration>` for more information.
 
 Upgrading to 8.0.1
 ------------------

src/detect-engine-loader.c

@@ -502,10 +502,6 @@ skip_regular_rules:
 
     ret = 0;
 
-    if (mpm_table[de_ctx->mpm_matcher].CacheRuleset != NULL) {
-        mpm_table[de_ctx->mpm_matcher].CacheRuleset(de_ctx->mpm_cfg);
-    }
-
  end:
     gettimeofday(&de_ctx->last_reload, NULL);
     if (SCRunmodeGet() == RUNMODE_ENGINE_ANALYSIS) {

src/detect-engine.c

@@ -2481,6 +2481,49 @@ const char *DetectEngineMpmCachingGetPath(void)
     return SGH_CACHE_DIR;
 }
 
+void DetectEngineMpmCacheService(uint32_t op_flags)
+{
+    DetectEngineCtx *de_ctx = DetectEngineGetCurrent();
+    if (!de_ctx) {
+        return;
+    }
+
+    if (!de_ctx->mpm_cfg || !de_ctx->mpm_cfg->cache_dir_path) {
+        goto error;
+    }
+
+    if (mpm_table[de_ctx->mpm_matcher].CacheStatsInit != NULL) {
+        de_ctx->mpm_cfg->cache_stats = mpm_table[de_ctx->mpm_matcher].CacheStatsInit();
+        if (de_ctx->mpm_cfg->cache_stats == NULL) {
+            goto error;
+        }
+    }
+
+    if (op_flags & DETECT_ENGINE_MPM_CACHE_OP_SAVE) {
+        if (mpm_table[de_ctx->mpm_matcher].CacheRuleset != NULL) {
+            mpm_table[de_ctx->mpm_matcher].CacheRuleset(de_ctx->mpm_cfg);
+        }
+    }
+
+    if (op_flags & DETECT_ENGINE_MPM_CACHE_OP_PRUNE) {
+        if (mpm_table[de_ctx->mpm_matcher].CachePrune != NULL) {
+            mpm_table[de_ctx->mpm_matcher].CachePrune(de_ctx->mpm_cfg);
+        }
+    }
+
+    if (mpm_table[de_ctx->mpm_matcher].CacheStatsPrint != NULL) {
+        mpm_table[de_ctx->mpm_matcher].CacheStatsPrint(de_ctx->mpm_cfg->cache_stats);
+    }
+
+    if (mpm_table[de_ctx->mpm_matcher].CacheStatsDeinit != NULL) {
+        mpm_table[de_ctx->mpm_matcher].CacheStatsDeinit(de_ctx->mpm_cfg->cache_stats);
+        de_ctx->mpm_cfg->cache_stats = NULL;
+    }
+
+error:
+    DetectEngineDeReference(&de_ctx);
+}
+
 static DetectEngineCtx *DetectEngineCtxInitReal(
         enum DetectEngineType type, const char *prefix, uint32_t tenant_id)
 {
@@ -2503,10 +2546,18 @@ static DetectEngineCtx *DetectEngineCtxInitReal(
         if (de_ctx->mpm_cfg == NULL) {
             goto error;
         }
-    }
-    if (DetectEngineMpmCachingEnabled() && mpm_table[de_ctx->mpm_matcher].ConfigCacheDirSet) {
-        mpm_table[de_ctx->mpm_matcher].ConfigCacheDirSet(
-                de_ctx->mpm_cfg, DetectEngineMpmCachingGetPath());
+
+        if (DetectEngineMpmCachingEnabled() && mpm_table[de_ctx->mpm_matcher].ConfigCacheDirSet) {
+            mpm_table[de_ctx->mpm_matcher].ConfigCacheDirSet(
+                    de_ctx->mpm_cfg, DetectEngineMpmCachingGetPath());
+
+            if (mpm_table[de_ctx->mpm_matcher].CachePrune) {
+                if (SCConfGetTime("detect.sgh-mpm-caching-max-age",
+                            &de_ctx->mpm_cfg->cache_max_age_seconds) != 1) {
+                    de_ctx->mpm_cfg->cache_max_age_seconds = 7ULL * 24ULL * 60ULL * 60ULL;
+                }
+            }
+        }
     }
 
     if (type == DETECT_ENGINE_TYPE_DD_STUB || type == DETECT_ENGINE_TYPE_MT_STUB) {
@@ -4885,6 +4936,8 @@ int DetectEngineReload(const SCInstance *suri)
 
     SCLogDebug("old_de_ctx should have been freed");
 
+    DetectEngineMpmCacheService(DETECT_ENGINE_MPM_CACHE_OP_SAVE | DETECT_ENGINE_MPM_CACHE_OP_PRUNE);
+
     SCLogNotice("rule reload complete");
 
 #ifdef HAVE_MALLOC_TRIM

src/detect-engine.h

@@ -88,6 +88,7 @@ TmEcode DetectEngineThreadCtxInit(ThreadVars *, void *, void **);
 TmEcode DetectEngineThreadCtxDeinit(ThreadVars *, void *);
 bool DetectEngineMpmCachingEnabled(void);
 const char *DetectEngineMpmCachingGetPath(void);
+void DetectEngineMpmCacheService(uint32_t op_flags);
 /* faster as a macro than a inline function on my box -- VJ */
 #define DetectEngineGetMaxSigId(de_ctx) ((de_ctx)->signum)
 void DetectEngineResetMaxSigId(DetectEngineCtx *);

src/detect.h

@@ -1750,6 +1750,9 @@ extern SigTableElmt *sigmatch_table;
 
 /** Remember to add the options in SignatureIsIPOnly() at detect.c otherwise it wont be part of a signature group */
 
+#define DETECT_ENGINE_MPM_CACHE_OP_PRUNE BIT_U32(0)
+#define DETECT_ENGINE_MPM_CACHE_OP_SAVE  BIT_U32(1)
+
 /* detection api */
 TmEcode Detect(ThreadVars *tv, Packet *p, void *data);
 uint8_t DetectPreFlow(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Packet *p);

src/runmode-unix-socket.c

@@ -967,6 +967,8 @@ TmEcode UnixSocketRegisterTenantHandler(json_t *cmd, json_t* answer, void *data)
         return TM_ECODE_FAILED;
     }
 
+    DetectEngineMpmCacheService(DETECT_ENGINE_MPM_CACHE_OP_SAVE);
+
     json_object_set_new(answer, "message", json_string("handler added"));
     return TM_ECODE_OK;
 }
@@ -1054,6 +1056,8 @@ TmEcode UnixSocketUnregisterTenantHandler(json_t *cmd, json_t* answer, void *dat
         return TM_ECODE_FAILED;
     }
 
+    DetectEngineMpmCacheService(DETECT_ENGINE_MPM_CACHE_OP_PRUNE);
+
     json_object_set_new(answer, "message", json_string("handler removed"));
     return TM_ECODE_OK;
 }
@@ -1126,6 +1130,8 @@ TmEcode UnixSocketRegisterTenant(json_t *cmd, json_t* answer, void *data)
         return TM_ECODE_FAILED;
     }
 
+    DetectEngineMpmCacheService(DETECT_ENGINE_MPM_CACHE_OP_SAVE);
+
     json_object_set_new(answer, "message", json_string("adding tenant succeeded"));
     return TM_ECODE_OK;
 }
@@ -1193,6 +1199,8 @@ TmEcode UnixSocketReloadTenant(json_t *cmd, json_t* answer, void *data)
         return TM_ECODE_FAILED;
     }
 
+    DetectEngineMpmCacheService(DETECT_ENGINE_MPM_CACHE_OP_SAVE | DETECT_ENGINE_MPM_CACHE_OP_PRUNE);
+
     json_object_set_new(answer, "message", json_string("reloading tenant succeeded"));
     return TM_ECODE_OK;
 }
@@ -1226,6 +1234,7 @@ TmEcode UnixSocketReloadTenants(json_t *cmd, json_t *answer, void *data)
         return TM_ECODE_FAILED;
     }
 
+    DetectEngineMpmCacheService(DETECT_ENGINE_MPM_CACHE_OP_SAVE | DETECT_ENGINE_MPM_CACHE_OP_PRUNE);
     SCLogNotice("reload-tenants complete");
 
     json_object_set_new(answer, "message", json_string("reloading tenants succeeded"));
@@ -1284,6 +1293,8 @@ TmEcode UnixSocketUnregisterTenant(json_t *cmd, json_t* answer, void *data)
         return TM_ECODE_FAILED;
     }
 
+    DetectEngineMpmCacheService(DETECT_ENGINE_MPM_CACHE_OP_PRUNE);
+
     /* walk free list, freeing the removed de_ctx */
     DetectEnginePruneFreeList();
 

src/suricata.c

@@ -2688,6 +2688,8 @@ void PostConfLoadedDetectSetup(SCInstance *suri)
         gettimeofday(&de_ctx->last_reload, NULL);
         DetectEngineAddToMaster(de_ctx);
         DetectEngineBumpVersion();
+        DetectEngineMpmCacheService(
+                DETECT_ENGINE_MPM_CACHE_OP_SAVE | DETECT_ENGINE_MPM_CACHE_OP_PRUNE);
     }
 }
 

src/util-mpm-hs-cache.c

@@ -37,21 +37,22 @@
 #include "rust.h"
 #include <hs.h>
 
-static const char *HSCacheConstructFPath(const char *folder_path, const char *hs_db_hash)
-{
-    static char hash_file_path[PATH_MAX];
+#define HS_CACHE_FILE_VERSION "2"
+#define HS_CACHE_FILE_SUFFIX  "_v" HS_CACHE_FILE_VERSION ".hs"
 
-    char hash_file_path_suffix[] = "_v1.hs";
+static int16_t HSCacheConstructFPath(
+        const char *dir_path, const char *db_hash, char *out_path, uint16_t out_path_size)
+{
     char filename[NAME_MAX];
-    uint64_t r = snprintf(filename, sizeof(filename), "%s%s", hs_db_hash, hash_file_path_suffix);
-    if (r != (uint64_t)(strlen(hs_db_hash) + strlen(hash_file_path_suffix)))
-        return NULL;
+    uint64_t r = snprintf(filename, sizeof(filename), "%s" HS_CACHE_FILE_SUFFIX, db_hash);
+    if (r != (uint64_t)(strlen(db_hash) + strlen(HS_CACHE_FILE_SUFFIX)))
+        return -1;
 
-    r = PathMerge(hash_file_path, sizeof(hash_file_path), folder_path, filename);
+    r = PathMerge(out_path, out_path_size, dir_path, filename);
     if (r)
-        return NULL;
+        return -1;
 
-    return hash_file_path;
+    return 0;
 }
 
 static char *HSReadStream(const char *file_path, size_t *buffer_sz)
@@ -121,8 +122,11 @@ static void SCHSCachePatternHash(const SCHSPattern *p, SCSha256 *sha256)
 
 int HSLoadCache(hs_database_t **hs_db, const char *hs_db_hash, const char *dirpath)
 {
-    const char *hash_file_static = HSCacheConstructFPath(dirpath, hs_db_hash);
-    if (hash_file_static == NULL)
+    char hash_file_static[PATH_MAX];
+    int ret = (int)HSCacheConstructFPath(
+            dirpath, hs_db_hash, hash_file_static, sizeof(hash_file_static));
+
+    if (ret != 0)
         return -1;
 
     SCLogDebug("Loading the cached HS DB from %s", hash_file_static);
@@ -131,7 +135,6 @@ int HSLoadCache(hs_database_t **hs_db, const char *hs_db_hash, const char *dirpa
 
     FILE *db_cache = fopen(hash_file_static, "r");
     char *buffer = NULL;
-    int ret = 0;
     if (db_cache) {
         size_t buffer_size;
         buffer = HSReadStream(hash_file_static, &buffer_size);
@@ -170,15 +173,20 @@ static int HSSaveCache(hs_database_t *hs_db, const char *hs_db_hash, const char
     static bool notified = false;
     char *db_stream = NULL;
     size_t db_size;
-    int ret = -1;
+    int ret;
 
     hs_error_t err = hs_serialize_database(hs_db, &db_stream, &db_size);
     if (err != HS_SUCCESS) {
         SCLogWarning("Failed to serialize Hyperscan database: %s", HSErrorToStr(err));
+        ret = -1;
         goto cleanup;
     }
 
-    const char *hash_file_static = HSCacheConstructFPath(dstpath, hs_db_hash);
+    char hash_file_static[PATH_MAX];
+    ret = (int)HSCacheConstructFPath(
+            dstpath, hs_db_hash, hash_file_static, sizeof(hash_file_static));
+    if (ret != 0)
+        goto cleanup;
     SCLogDebug("Caching the compiled HS at %s", hash_file_static);
     if (SCPathExists(hash_file_static)) {
         // potentially signs that it might not work as expected as we got into
@@ -198,6 +206,7 @@ static int HSSaveCache(hs_database_t *hs_db, const char *hs_db_hash, const char
                     hash_file_static);
             notified = true;
         }
+        ret = -1;
         goto cleanup;
     }
     size_t r = fwrite(db_stream, sizeof(db_stream[0]), db_size, db_cache_out);
@@ -217,7 +226,6 @@ static int HSSaveCache(hs_database_t *hs_db, const char *hs_db_hash, const char
         goto cleanup;
     }
 
-    ret = 0;
 cleanup:
     if (db_stream)
         SCFree(db_stream);
@@ -270,4 +278,187 @@ void HSSaveCacheIterator(void *data, void *aux)
     }
 }
 
+void HSCacheFilenameUsedIterator(void *data, void *aux)
+{
+    PatternDatabase *pd = (PatternDatabase *)data;
+    struct HsInUseCacheFilesIteratorData *iter_data = (struct HsInUseCacheFilesIteratorData *)aux;
+    if (pd->no_cache || !pd->cached)
+        return;
+
+    char hs_db_hash[SC_SHA256_LEN * 2 + 1]; // * 2 for hex +1 for nul terminator
+    if (HSHashDb(pd, hs_db_hash, ARRAY_SIZE(hs_db_hash)) != 0) {
+        return;
+    }
+
+    char *fpath = SCCalloc(PATH_MAX, sizeof(char));
+    if (fpath == NULL) {
+        SCLogWarning("Failed to allocate memory for cache file path");
+        return;
+    }
+    if (HSCacheConstructFPath(iter_data->cache_path, hs_db_hash, fpath, PATH_MAX)) {
+        SCFree(fpath);
+        return;
+    }
+
+    int r = HashTableAdd(iter_data->tbl, (void *)fpath, (uint16_t)strlen(fpath));
+    if (r < 0) {
+        SCLogWarning("Failed to add used cache file path %s to hash table", fpath);
+        SCFree(fpath);
+    }
+}
+
+/**
+ * \brief Check if HS cache file is stale by age.
+ *
+ * \param mtime   File modification time.
+ * \param cutoff  Time cutoff (files older than this will be removed).
+ *
+ * \retval true if file should be pruned, false otherwise.
+ */
+static bool HSPruneFileByAge(time_t mtime, time_t cutoff)
+{
+    return mtime < cutoff;
+}
+
+/**
+ * \brief Check if HS cache file is version-compatible.
+ *
+ * \param filename  Cache file name.
+ *
+ * \retval true if file should be pruned, false otherwise.
+ */
+static bool HSPruneFileByVersion(const char *filename)
+{
+    if (strlen(filename) < strlen(HS_CACHE_FILE_SUFFIX)) {
+        return true;
+    }
+
+    const char *underscore = strrchr(filename, '_');
+    if (underscore == NULL || strcmp(underscore, HS_CACHE_FILE_SUFFIX) != 0) {
+        return true;
+    }
+
+    return false;
+}
+
+int SCHSCachePruneEvaluate(MpmConfig *mpm_conf, HashTable *inuse_caches)
+{
+    if (mpm_conf == NULL || mpm_conf->cache_dir_path == NULL)
+        return -1;
+    if (mpm_conf->cache_max_age_seconds == 0)
+        return 0; // disabled
+
+    const time_t now = time(NULL);
+    if (now == (time_t)-1) {
+        return -1;
+    } else if (mpm_conf->cache_max_age_seconds >= (uint64_t)now) {
+        return 0;
+    }
+
+    DIR *dir = opendir(mpm_conf->cache_dir_path);
+    if (dir == NULL) {
+        return -1;
+    }
+
+    struct dirent *ent;
+    char path[PATH_MAX];
+    uint32_t considered = 0, removed = 0;
+    const time_t cutoff = now - (time_t)mpm_conf->cache_max_age_seconds;
+    while ((ent = readdir(dir)) != NULL) {
+        const char *name = ent->d_name;
+        size_t namelen = strlen(name);
+        if (namelen < 3 || strcmp(name + namelen - 3, ".hs") != 0)
+            continue;
+
+        if (PathMerge(path, ARRAY_SIZE(path), mpm_conf->cache_dir_path, name) != 0)
+            continue;
+
+        struct stat st;
+        if (stat(path, &st) != 0 || !S_ISREG(st.st_mode))
+            continue;
+
+        considered++;
+
+        const bool prune_by_age = HSPruneFileByAge(st.st_mtime, cutoff);
+        const bool prune_by_version = HSPruneFileByVersion(name);
+        if (!prune_by_age && !prune_by_version)
+            continue;
+
+        void *cache_inuse = HashTableLookup(inuse_caches, path, (uint16_t)strlen(path));
+        if (cache_inuse != NULL)
+            continue; // in use
+
+        if (unlink(path) == 0) {
+            removed++;
+            SCLogDebug("File %s removed because of %s%s%s", path, prune_by_age ? "age" : "",
+                    prune_by_age && prune_by_version ? " and " : "",
+                    prune_by_version ? "incompatible version" : "");
+        } else {
+            SCLogWarning("Failed to prune \"%s\": %s", path, strerror(errno));
+        }
+    }
+    closedir(dir);
+
+    PatternDatabaseCache *pd_cache_stats = mpm_conf->cache_stats;
+    if (pd_cache_stats) {
+        pd_cache_stats->hs_dbs_cache_pruned_cnt = removed;
+        pd_cache_stats->hs_dbs_cache_pruned_considered_cnt = considered;
+        pd_cache_stats->hs_dbs_cache_pruned_cutoff = cutoff;
+        pd_cache_stats->cache_max_age_seconds = mpm_conf->cache_max_age_seconds;
+    }
+    return 0;
+}
+
+void *SCHSCacheStatsInit(void)
+{
+    PatternDatabaseCache *pd_cache_stats = SCCalloc(1, sizeof(PatternDatabaseCache));
+    if (pd_cache_stats == NULL) {
+        SCLogError("Failed to allocate memory for Hyperscan cache stats");
+        return NULL;
+    }
+    return pd_cache_stats;
+}
+
+void SCHSCacheStatsPrint(void *data)
+{
+    if (data == NULL) {
+        return;
+    }
+
+    PatternDatabaseCache *pd_cache_stats = (PatternDatabaseCache *)data;
+
+    char time_str[64];
+    struct tm tm_s;
+    struct tm *tm_info = SCLocalTime(pd_cache_stats->hs_dbs_cache_pruned_cutoff, &tm_s);
+    if (tm_info != NULL) {
+        strftime(time_str, ARRAY_SIZE(time_str), "%Y-%m-%d %H:%M:%S", tm_info);
+    } else {
+        snprintf(time_str, ARRAY_SIZE(time_str), "%" PRIu64 " seconds",
+                pd_cache_stats->cache_max_age_seconds);
+    }
+
+    if (pd_cache_stats->hs_cacheable_dbs_cnt) {
+        SCLogInfo("Rule group caching - loaded: %u newly cached: %u total cacheable: %u",
+                pd_cache_stats->hs_dbs_cache_loaded_cnt, pd_cache_stats->hs_dbs_cache_saved_cnt,
+                pd_cache_stats->hs_cacheable_dbs_cnt);
+    }
+    if (pd_cache_stats->hs_dbs_cache_pruned_considered_cnt) {
+        SCLogInfo("Rule group cache pruning removed %u/%u of HS caches due to "
+                  "version-incompatibility (not v%s) or "
+                  "age (older than %s)",
+                pd_cache_stats->hs_dbs_cache_pruned_cnt,
+                pd_cache_stats->hs_dbs_cache_pruned_considered_cnt, HS_CACHE_FILE_VERSION,
+                time_str);
+    }
+}
+
+void SCHSCacheStatsDeinit(void *data)
+{
+    if (data == NULL) {
+        return;
+    }
+    PatternDatabaseCache *pd_cache_stats = (PatternDatabaseCache *)data;
+    SCFree(pd_cache_stats);
+}
+
 #endif /* BUILD_HYPERSCAN */

src/util-mpm-hs-cache.h

@@ -35,9 +35,24 @@ struct HsIteratorData {
     const char *cache_path;
 };
 
+/**
+ * \brief Data structure to store in-use cache files.
+ * Used in cache pruning to avoid deleting files that are still in use.
+ */
+struct HsInUseCacheFilesIteratorData {
+    HashTable *tbl; // stores file paths of in-use cache files
+    const char *cache_path;
+};
+
 int HSLoadCache(hs_database_t **hs_db, const char *hs_db_hash, const char *dirpath);
 int HSHashDb(const PatternDatabase *pd, char *hash, size_t hash_len);
 void HSSaveCacheIterator(void *data, void *aux);
+void HSCacheFilenameUsedIterator(void *data, void *aux);
+int SCHSCachePruneEvaluate(MpmConfig *mpm_conf, HashTable *inuse_caches);
+
+void *SCHSCacheStatsInit(void);
+void SCHSCacheStatsPrint(void *data);
+void SCHSCacheStatsDeinit(void *data);
 #endif /* BUILD_HYPERSCAN */
 
 #endif /* SURICATA_UTIL_MPM_HS_CACHE__H */

src/util-mpm-hs-core.h

@@ -93,6 +93,10 @@ typedef struct PatternDatabaseCache_ {
     uint32_t hs_cacheable_dbs_cnt;
     uint32_t hs_dbs_cache_loaded_cnt;
     uint32_t hs_dbs_cache_saved_cnt;
+    uint32_t hs_dbs_cache_pruned_cnt;
+    uint32_t hs_dbs_cache_pruned_considered_cnt;
+    time_t hs_dbs_cache_pruned_cutoff;
+    uint64_t cache_max_age_seconds;
 } PatternDatabaseCache;
 
 const char *HSErrorToStr(hs_error_t error_code);

src/util-mpm-hs.c

@@ -835,18 +835,53 @@ static int SCHSCacheRuleset(MpmConfig *mpm_conf)
                 mpm_conf->cache_dir_path);
         return -1;
     }
-    PatternDatabaseCache pd_stats = { 0 };
-    struct HsIteratorData iter_data = { .pd_stats = &pd_stats,
+    PatternDatabaseCache *pd_stats = mpm_conf->cache_stats;
+    struct HsIteratorData iter_data = { .pd_stats = pd_stats,
         .cache_path = mpm_conf->cache_dir_path };
     SCMutexLock(&g_db_table_mutex);
     HashTableIterate(g_db_table, HSSaveCacheIterator, &iter_data);
     SCMutexUnlock(&g_db_table_mutex);
-    SCLogNotice("Rule group caching - loaded: %u newly cached: %u total cacheable: %u",
-            pd_stats.hs_dbs_cache_loaded_cnt, pd_stats.hs_dbs_cache_saved_cnt,
-            pd_stats.hs_cacheable_dbs_cnt);
     return 0;
 }
 
+static uint32_t FilenameTableHash(HashTable *ht, void *data, uint16_t len)
+{
+    const char *fname = data;
+    uint32_t hash = hashlittle_safe(data, strlen(fname), 0);
+    hash %= ht->array_size;
+    return hash;
+}
+
+static void FilenameTableFree(void *data)
+{
+    SCFree(data);
+}
+
+static int SCHSCachePrune(MpmConfig *mpm_conf)
+{
+    if (!mpm_conf || !mpm_conf->cache_dir_path) {
+        return -1;
+    }
+
+    SCLogDebug("Pruning the Hyperscan cache folder %s", mpm_conf->cache_dir_path);
+    // we need to initialize hash map of in-use cache files
+    HashTable *inuse_caches =
+            HashTableInit(INIT_DB_HASH_SIZE, FilenameTableHash, NULL, FilenameTableFree);
+    if (inuse_caches == NULL) {
+        return -1;
+    }
+    struct HsInUseCacheFilesIteratorData iter_data = { .tbl = inuse_caches,
+        .cache_path = mpm_conf->cache_dir_path };
+
+    SCMutexLock(&g_db_table_mutex);
+    HashTableIterate(g_db_table, HSCacheFilenameUsedIterator, &iter_data);
+    SCMutexUnlock(&g_db_table_mutex);
+
+    int r = SCHSCachePruneEvaluate(mpm_conf, inuse_caches);
+    HashTableFree(inuse_caches);
+    return r;
+}
+
 /**
  * \brief Init the mpm thread context.
  *
@@ -1178,7 +1213,11 @@ void MpmHSRegister(void)
     mpm_table[MPM_HS].AddPattern = SCHSAddPatternCS;
     mpm_table[MPM_HS].AddPatternNocase = SCHSAddPatternCI;
     mpm_table[MPM_HS].Prepare = SCHSPreparePatterns;
+    mpm_table[MPM_HS].CacheStatsInit = SCHSCacheStatsInit;
+    mpm_table[MPM_HS].CacheStatsPrint = SCHSCacheStatsPrint;
+    mpm_table[MPM_HS].CacheStatsDeinit = SCHSCacheStatsDeinit;
     mpm_table[MPM_HS].CacheRuleset = SCHSCacheRuleset;
+    mpm_table[MPM_HS].CachePrune = SCHSCachePrune;
     mpm_table[MPM_HS].Search = SCHSSearch;
     mpm_table[MPM_HS].PrintCtx = SCHSPrintInfo;
     mpm_table[MPM_HS].PrintThreadCtx = SCHSPrintSearchStats;

src/util-mpm.h

@@ -90,6 +90,8 @@ typedef struct MpmPattern_ {
 
 typedef struct MpmConfig_ {
     const char *cache_dir_path;
+    uint64_t cache_max_age_seconds; /* 0 means disabled/no pruning policy */
+    void *cache_stats;
 } MpmConfig;
 
 typedef struct MpmCtx_ {
@@ -175,7 +177,11 @@ typedef struct MpmTableElmt_ {
     int (*AddPatternNocase)(struct MpmCtx_ *, const uint8_t *, uint16_t, uint16_t, uint16_t,
             uint32_t, SigIntId, uint8_t);
     int (*Prepare)(MpmConfig *, struct MpmCtx_ *);
+    void *(*CacheStatsInit)(void);
+    void (*CacheStatsPrint)(void *data);
+    void (*CacheStatsDeinit)(void *data);
     int (*CacheRuleset)(MpmConfig *);
+    int (*CachePrune)(MpmConfig *);
     /** \retval cnt number of patterns that matches: once per pattern max. */
     uint32_t (*Search)(const struct MpmCtx_ *, struct MpmThreadCtx_ *, PrefilterRuleStore *, const uint8_t *, uint32_t);
     void (*PrintCtx)(struct MpmCtx_ *);

suricata.yaml.in

@@ -1810,6 +1810,10 @@ detect:
   # Cache files are created in the standard library directory.
   sgh-mpm-caching: yes
   sgh-mpm-caching-path: @e_sghcachedir@
+  # Maximum age for cached MPM databases before they are pruned.
+  # Accepts a combination of time units (s,m,h,d,w,y).
+  # Omit to use the default, 0 to disable.
+  # sgh-mpm-caching-max-age: 7d
   # inspection-recursion-limit: 3000
   # maximum number of times a tx will get logged for rules without app-layer keywords
   # stream-tx-log-limit: 4