From 15c83be61ac3f47bf198fe24eb908db5a84b7ccd Mon Sep 17 00:00:00 2001 From: Lukas Sismis Date: Mon, 15 Sep 2025 11:24:23 +0200 Subject: [PATCH] hs: prune stale MPM cache files Hyperscan MPM can cache the compiled contexts to files. This however grows as rulesets change and leads to bloating the system. This addition prunes the stale cache files based on their modified file timestamp. Part of this work incorporates new model for MPM cache stats to split it out from the cache save function and aggregate cache-related stats in one place (newly added pruning). Ticket: 7830 --- doc/userguide/performance/hyperscan.rst | 20 +++ doc/userguide/upgrade.rst | 4 + src/detect-engine-loader.c | 4 - src/detect-engine.c | 61 ++++++- src/detect-engine.h | 1 + src/detect.h | 3 + src/runmode-unix-socket.c | 11 ++ src/suricata.c | 2 + src/util-mpm-hs-cache.c | 223 ++++++++++++++++++++++-- src/util-mpm-hs-cache.h | 15 ++ src/util-mpm-hs-core.h | 4 + src/util-mpm-hs.c | 49 +++++- src/util-mpm.h | 6 + suricata.yaml.in | 4 + 14 files changed, 378 insertions(+), 29 deletions(-) diff --git a/doc/userguide/performance/hyperscan.rst b/doc/userguide/performance/hyperscan.rst index 065163110c..1060d3aefb 100644 --- a/doc/userguide/performance/hyperscan.rst +++ b/doc/userguide/performance/hyperscan.rst @@ -83,6 +83,8 @@ if it is present on the system in case of the "auto" setting. If the current suricata installation does not have hyperscan support, refer to :ref:`installation` +.. _hyperscan-cache-configuration: + Hyperscan caching ~~~~~~~~~~~~~~~~~ @@ -104,6 +106,24 @@ To enable this function, in `suricata.yaml` configure: sgh-mpm-caching-path: /var/lib/suricata/cache/hs +To avoid cache files growing indefinitely, Suricata supports pruning of old +cache files. Suricata removes cache files older than the specified age +on startup/rule reloads, where age is determined by delta of the file +modification time and the current time. +Cache files that are actively being used will have their modification time +updated when loaded, so they won't be deleted. + +In `suricata.yaml` configure: + +:: + + detect: + sgh-mpm-caching-max-age: 7d + +The setting accepts a combination of time units (s,m,h,d,w,y), +e.g. `1w 3d 12h` for 1 week, 3 days and 12 hours. Setting the value to `0` +disables pruning. + **Note**: You might need to create and adjust permissions to the default caching folder path, especially if you are running Suricata as a non-root user. diff --git a/doc/userguide/upgrade.rst b/doc/userguide/upgrade.rst index ef8d1e369d..054e3eb386 100644 --- a/doc/userguide/upgrade.rst +++ b/doc/userguide/upgrade.rst @@ -68,6 +68,10 @@ Other Changes from unbounded to 2048. Configuration options, ``max-tx``, ``max-points``, and ``max-objects`` have been added for users who may need to change these defaults. +- Hyperscan caching (`detect.sgh-mpm-caching`), when enabled, prunes + cache files that have not been used in the last 7 days by default. + See :ref:`Hyperscan caching configuration + ` for more information. Upgrading to 8.0.1 ------------------ diff --git a/src/detect-engine-loader.c b/src/detect-engine-loader.c index ef0e8ef13d..a97ebd6d27 100644 --- a/src/detect-engine-loader.c +++ b/src/detect-engine-loader.c @@ -502,10 +502,6 @@ skip_regular_rules: ret = 0; - if (mpm_table[de_ctx->mpm_matcher].CacheRuleset != NULL) { - mpm_table[de_ctx->mpm_matcher].CacheRuleset(de_ctx->mpm_cfg); - } - end: gettimeofday(&de_ctx->last_reload, NULL); if (SCRunmodeGet() == RUNMODE_ENGINE_ANALYSIS) { diff --git a/src/detect-engine.c b/src/detect-engine.c index 12b1683c59..28e0bc14a3 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -2481,6 +2481,49 @@ const char *DetectEngineMpmCachingGetPath(void) return SGH_CACHE_DIR; } +void DetectEngineMpmCacheService(uint32_t op_flags) +{ + DetectEngineCtx *de_ctx = DetectEngineGetCurrent(); + if (!de_ctx) { + return; + } + + if (!de_ctx->mpm_cfg || !de_ctx->mpm_cfg->cache_dir_path) { + goto error; + } + + if (mpm_table[de_ctx->mpm_matcher].CacheStatsInit != NULL) { + de_ctx->mpm_cfg->cache_stats = mpm_table[de_ctx->mpm_matcher].CacheStatsInit(); + if (de_ctx->mpm_cfg->cache_stats == NULL) { + goto error; + } + } + + if (op_flags & DETECT_ENGINE_MPM_CACHE_OP_SAVE) { + if (mpm_table[de_ctx->mpm_matcher].CacheRuleset != NULL) { + mpm_table[de_ctx->mpm_matcher].CacheRuleset(de_ctx->mpm_cfg); + } + } + + if (op_flags & DETECT_ENGINE_MPM_CACHE_OP_PRUNE) { + if (mpm_table[de_ctx->mpm_matcher].CachePrune != NULL) { + mpm_table[de_ctx->mpm_matcher].CachePrune(de_ctx->mpm_cfg); + } + } + + if (mpm_table[de_ctx->mpm_matcher].CacheStatsPrint != NULL) { + mpm_table[de_ctx->mpm_matcher].CacheStatsPrint(de_ctx->mpm_cfg->cache_stats); + } + + if (mpm_table[de_ctx->mpm_matcher].CacheStatsDeinit != NULL) { + mpm_table[de_ctx->mpm_matcher].CacheStatsDeinit(de_ctx->mpm_cfg->cache_stats); + de_ctx->mpm_cfg->cache_stats = NULL; + } + +error: + DetectEngineDeReference(&de_ctx); +} + static DetectEngineCtx *DetectEngineCtxInitReal( enum DetectEngineType type, const char *prefix, uint32_t tenant_id) { @@ -2503,10 +2546,18 @@ static DetectEngineCtx *DetectEngineCtxInitReal( if (de_ctx->mpm_cfg == NULL) { goto error; } - } - if (DetectEngineMpmCachingEnabled() && mpm_table[de_ctx->mpm_matcher].ConfigCacheDirSet) { - mpm_table[de_ctx->mpm_matcher].ConfigCacheDirSet( - de_ctx->mpm_cfg, DetectEngineMpmCachingGetPath()); + + if (DetectEngineMpmCachingEnabled() && mpm_table[de_ctx->mpm_matcher].ConfigCacheDirSet) { + mpm_table[de_ctx->mpm_matcher].ConfigCacheDirSet( + de_ctx->mpm_cfg, DetectEngineMpmCachingGetPath()); + + if (mpm_table[de_ctx->mpm_matcher].CachePrune) { + if (SCConfGetTime("detect.sgh-mpm-caching-max-age", + &de_ctx->mpm_cfg->cache_max_age_seconds) != 1) { + de_ctx->mpm_cfg->cache_max_age_seconds = 7ULL * 24ULL * 60ULL * 60ULL; + } + } + } } if (type == DETECT_ENGINE_TYPE_DD_STUB || type == DETECT_ENGINE_TYPE_MT_STUB) { @@ -4885,6 +4936,8 @@ int DetectEngineReload(const SCInstance *suri) SCLogDebug("old_de_ctx should have been freed"); + DetectEngineMpmCacheService(DETECT_ENGINE_MPM_CACHE_OP_SAVE | DETECT_ENGINE_MPM_CACHE_OP_PRUNE); + SCLogNotice("rule reload complete"); #ifdef HAVE_MALLOC_TRIM diff --git a/src/detect-engine.h b/src/detect-engine.h index 2c56475f62..2d45d32535 100644 --- a/src/detect-engine.h +++ b/src/detect-engine.h @@ -88,6 +88,7 @@ TmEcode DetectEngineThreadCtxInit(ThreadVars *, void *, void **); TmEcode DetectEngineThreadCtxDeinit(ThreadVars *, void *); bool DetectEngineMpmCachingEnabled(void); const char *DetectEngineMpmCachingGetPath(void); +void DetectEngineMpmCacheService(uint32_t op_flags); /* faster as a macro than a inline function on my box -- VJ */ #define DetectEngineGetMaxSigId(de_ctx) ((de_ctx)->signum) void DetectEngineResetMaxSigId(DetectEngineCtx *); diff --git a/src/detect.h b/src/detect.h index 62c888e6a7..49fbfe3eb6 100644 --- a/src/detect.h +++ b/src/detect.h @@ -1750,6 +1750,9 @@ extern SigTableElmt *sigmatch_table; /** Remember to add the options in SignatureIsIPOnly() at detect.c otherwise it wont be part of a signature group */ +#define DETECT_ENGINE_MPM_CACHE_OP_PRUNE BIT_U32(0) +#define DETECT_ENGINE_MPM_CACHE_OP_SAVE BIT_U32(1) + /* detection api */ TmEcode Detect(ThreadVars *tv, Packet *p, void *data); uint8_t DetectPreFlow(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Packet *p); diff --git a/src/runmode-unix-socket.c b/src/runmode-unix-socket.c index c2405f0571..706a35b7e6 100644 --- a/src/runmode-unix-socket.c +++ b/src/runmode-unix-socket.c @@ -967,6 +967,8 @@ TmEcode UnixSocketRegisterTenantHandler(json_t *cmd, json_t* answer, void *data) return TM_ECODE_FAILED; } + DetectEngineMpmCacheService(DETECT_ENGINE_MPM_CACHE_OP_SAVE); + json_object_set_new(answer, "message", json_string("handler added")); return TM_ECODE_OK; } @@ -1054,6 +1056,8 @@ TmEcode UnixSocketUnregisterTenantHandler(json_t *cmd, json_t* answer, void *dat return TM_ECODE_FAILED; } + DetectEngineMpmCacheService(DETECT_ENGINE_MPM_CACHE_OP_PRUNE); + json_object_set_new(answer, "message", json_string("handler removed")); return TM_ECODE_OK; } @@ -1126,6 +1130,8 @@ TmEcode UnixSocketRegisterTenant(json_t *cmd, json_t* answer, void *data) return TM_ECODE_FAILED; } + DetectEngineMpmCacheService(DETECT_ENGINE_MPM_CACHE_OP_SAVE); + json_object_set_new(answer, "message", json_string("adding tenant succeeded")); return TM_ECODE_OK; } @@ -1193,6 +1199,8 @@ TmEcode UnixSocketReloadTenant(json_t *cmd, json_t* answer, void *data) return TM_ECODE_FAILED; } + DetectEngineMpmCacheService(DETECT_ENGINE_MPM_CACHE_OP_SAVE | DETECT_ENGINE_MPM_CACHE_OP_PRUNE); + json_object_set_new(answer, "message", json_string("reloading tenant succeeded")); return TM_ECODE_OK; } @@ -1226,6 +1234,7 @@ TmEcode UnixSocketReloadTenants(json_t *cmd, json_t *answer, void *data) return TM_ECODE_FAILED; } + DetectEngineMpmCacheService(DETECT_ENGINE_MPM_CACHE_OP_SAVE | DETECT_ENGINE_MPM_CACHE_OP_PRUNE); SCLogNotice("reload-tenants complete"); json_object_set_new(answer, "message", json_string("reloading tenants succeeded")); @@ -1284,6 +1293,8 @@ TmEcode UnixSocketUnregisterTenant(json_t *cmd, json_t* answer, void *data) return TM_ECODE_FAILED; } + DetectEngineMpmCacheService(DETECT_ENGINE_MPM_CACHE_OP_PRUNE); + /* walk free list, freeing the removed de_ctx */ DetectEnginePruneFreeList(); diff --git a/src/suricata.c b/src/suricata.c index c6f94c3ce8..a106c56f76 100644 --- a/src/suricata.c +++ b/src/suricata.c @@ -2688,6 +2688,8 @@ void PostConfLoadedDetectSetup(SCInstance *suri) gettimeofday(&de_ctx->last_reload, NULL); DetectEngineAddToMaster(de_ctx); DetectEngineBumpVersion(); + DetectEngineMpmCacheService( + DETECT_ENGINE_MPM_CACHE_OP_SAVE | DETECT_ENGINE_MPM_CACHE_OP_PRUNE); } } diff --git a/src/util-mpm-hs-cache.c b/src/util-mpm-hs-cache.c index 41b3081713..58a2aa6ab4 100644 --- a/src/util-mpm-hs-cache.c +++ b/src/util-mpm-hs-cache.c @@ -37,21 +37,22 @@ #include "rust.h" #include -static const char *HSCacheConstructFPath(const char *folder_path, const char *hs_db_hash) -{ - static char hash_file_path[PATH_MAX]; +#define HS_CACHE_FILE_VERSION "2" +#define HS_CACHE_FILE_SUFFIX "_v" HS_CACHE_FILE_VERSION ".hs" - char hash_file_path_suffix[] = "_v1.hs"; +static int16_t HSCacheConstructFPath( + const char *dir_path, const char *db_hash, char *out_path, uint16_t out_path_size) +{ char filename[NAME_MAX]; - uint64_t r = snprintf(filename, sizeof(filename), "%s%s", hs_db_hash, hash_file_path_suffix); - if (r != (uint64_t)(strlen(hs_db_hash) + strlen(hash_file_path_suffix))) - return NULL; + uint64_t r = snprintf(filename, sizeof(filename), "%s" HS_CACHE_FILE_SUFFIX, db_hash); + if (r != (uint64_t)(strlen(db_hash) + strlen(HS_CACHE_FILE_SUFFIX))) + return -1; - r = PathMerge(hash_file_path, sizeof(hash_file_path), folder_path, filename); + r = PathMerge(out_path, out_path_size, dir_path, filename); if (r) - return NULL; + return -1; - return hash_file_path; + return 0; } static char *HSReadStream(const char *file_path, size_t *buffer_sz) @@ -121,8 +122,11 @@ static void SCHSCachePatternHash(const SCHSPattern *p, SCSha256 *sha256) int HSLoadCache(hs_database_t **hs_db, const char *hs_db_hash, const char *dirpath) { - const char *hash_file_static = HSCacheConstructFPath(dirpath, hs_db_hash); - if (hash_file_static == NULL) + char hash_file_static[PATH_MAX]; + int ret = (int)HSCacheConstructFPath( + dirpath, hs_db_hash, hash_file_static, sizeof(hash_file_static)); + + if (ret != 0) return -1; SCLogDebug("Loading the cached HS DB from %s", hash_file_static); @@ -131,7 +135,6 @@ int HSLoadCache(hs_database_t **hs_db, const char *hs_db_hash, const char *dirpa FILE *db_cache = fopen(hash_file_static, "r"); char *buffer = NULL; - int ret = 0; if (db_cache) { size_t buffer_size; buffer = HSReadStream(hash_file_static, &buffer_size); @@ -170,15 +173,20 @@ static int HSSaveCache(hs_database_t *hs_db, const char *hs_db_hash, const char static bool notified = false; char *db_stream = NULL; size_t db_size; - int ret = -1; + int ret; hs_error_t err = hs_serialize_database(hs_db, &db_stream, &db_size); if (err != HS_SUCCESS) { SCLogWarning("Failed to serialize Hyperscan database: %s", HSErrorToStr(err)); + ret = -1; goto cleanup; } - const char *hash_file_static = HSCacheConstructFPath(dstpath, hs_db_hash); + char hash_file_static[PATH_MAX]; + ret = (int)HSCacheConstructFPath( + dstpath, hs_db_hash, hash_file_static, sizeof(hash_file_static)); + if (ret != 0) + goto cleanup; SCLogDebug("Caching the compiled HS at %s", hash_file_static); if (SCPathExists(hash_file_static)) { // potentially signs that it might not work as expected as we got into @@ -198,6 +206,7 @@ static int HSSaveCache(hs_database_t *hs_db, const char *hs_db_hash, const char hash_file_static); notified = true; } + ret = -1; goto cleanup; } size_t r = fwrite(db_stream, sizeof(db_stream[0]), db_size, db_cache_out); @@ -217,7 +226,6 @@ static int HSSaveCache(hs_database_t *hs_db, const char *hs_db_hash, const char goto cleanup; } - ret = 0; cleanup: if (db_stream) SCFree(db_stream); @@ -270,4 +278,187 @@ void HSSaveCacheIterator(void *data, void *aux) } } +void HSCacheFilenameUsedIterator(void *data, void *aux) +{ + PatternDatabase *pd = (PatternDatabase *)data; + struct HsInUseCacheFilesIteratorData *iter_data = (struct HsInUseCacheFilesIteratorData *)aux; + if (pd->no_cache || !pd->cached) + return; + + char hs_db_hash[SC_SHA256_LEN * 2 + 1]; // * 2 for hex +1 for nul terminator + if (HSHashDb(pd, hs_db_hash, ARRAY_SIZE(hs_db_hash)) != 0) { + return; + } + + char *fpath = SCCalloc(PATH_MAX, sizeof(char)); + if (fpath == NULL) { + SCLogWarning("Failed to allocate memory for cache file path"); + return; + } + if (HSCacheConstructFPath(iter_data->cache_path, hs_db_hash, fpath, PATH_MAX)) { + SCFree(fpath); + return; + } + + int r = HashTableAdd(iter_data->tbl, (void *)fpath, (uint16_t)strlen(fpath)); + if (r < 0) { + SCLogWarning("Failed to add used cache file path %s to hash table", fpath); + SCFree(fpath); + } +} + +/** + * \brief Check if HS cache file is stale by age. + * + * \param mtime File modification time. + * \param cutoff Time cutoff (files older than this will be removed). + * + * \retval true if file should be pruned, false otherwise. + */ +static bool HSPruneFileByAge(time_t mtime, time_t cutoff) +{ + return mtime < cutoff; +} + +/** + * \brief Check if HS cache file is version-compatible. + * + * \param filename Cache file name. + * + * \retval true if file should be pruned, false otherwise. + */ +static bool HSPruneFileByVersion(const char *filename) +{ + if (strlen(filename) < strlen(HS_CACHE_FILE_SUFFIX)) { + return true; + } + + const char *underscore = strrchr(filename, '_'); + if (underscore == NULL || strcmp(underscore, HS_CACHE_FILE_SUFFIX) != 0) { + return true; + } + + return false; +} + +int SCHSCachePruneEvaluate(MpmConfig *mpm_conf, HashTable *inuse_caches) +{ + if (mpm_conf == NULL || mpm_conf->cache_dir_path == NULL) + return -1; + if (mpm_conf->cache_max_age_seconds == 0) + return 0; // disabled + + const time_t now = time(NULL); + if (now == (time_t)-1) { + return -1; + } else if (mpm_conf->cache_max_age_seconds >= (uint64_t)now) { + return 0; + } + + DIR *dir = opendir(mpm_conf->cache_dir_path); + if (dir == NULL) { + return -1; + } + + struct dirent *ent; + char path[PATH_MAX]; + uint32_t considered = 0, removed = 0; + const time_t cutoff = now - (time_t)mpm_conf->cache_max_age_seconds; + while ((ent = readdir(dir)) != NULL) { + const char *name = ent->d_name; + size_t namelen = strlen(name); + if (namelen < 3 || strcmp(name + namelen - 3, ".hs") != 0) + continue; + + if (PathMerge(path, ARRAY_SIZE(path), mpm_conf->cache_dir_path, name) != 0) + continue; + + struct stat st; + if (stat(path, &st) != 0 || !S_ISREG(st.st_mode)) + continue; + + considered++; + + const bool prune_by_age = HSPruneFileByAge(st.st_mtime, cutoff); + const bool prune_by_version = HSPruneFileByVersion(name); + if (!prune_by_age && !prune_by_version) + continue; + + void *cache_inuse = HashTableLookup(inuse_caches, path, (uint16_t)strlen(path)); + if (cache_inuse != NULL) + continue; // in use + + if (unlink(path) == 0) { + removed++; + SCLogDebug("File %s removed because of %s%s%s", path, prune_by_age ? "age" : "", + prune_by_age && prune_by_version ? " and " : "", + prune_by_version ? "incompatible version" : ""); + } else { + SCLogWarning("Failed to prune \"%s\": %s", path, strerror(errno)); + } + } + closedir(dir); + + PatternDatabaseCache *pd_cache_stats = mpm_conf->cache_stats; + if (pd_cache_stats) { + pd_cache_stats->hs_dbs_cache_pruned_cnt = removed; + pd_cache_stats->hs_dbs_cache_pruned_considered_cnt = considered; + pd_cache_stats->hs_dbs_cache_pruned_cutoff = cutoff; + pd_cache_stats->cache_max_age_seconds = mpm_conf->cache_max_age_seconds; + } + return 0; +} + +void *SCHSCacheStatsInit(void) +{ + PatternDatabaseCache *pd_cache_stats = SCCalloc(1, sizeof(PatternDatabaseCache)); + if (pd_cache_stats == NULL) { + SCLogError("Failed to allocate memory for Hyperscan cache stats"); + return NULL; + } + return pd_cache_stats; +} + +void SCHSCacheStatsPrint(void *data) +{ + if (data == NULL) { + return; + } + + PatternDatabaseCache *pd_cache_stats = (PatternDatabaseCache *)data; + + char time_str[64]; + struct tm tm_s; + struct tm *tm_info = SCLocalTime(pd_cache_stats->hs_dbs_cache_pruned_cutoff, &tm_s); + if (tm_info != NULL) { + strftime(time_str, ARRAY_SIZE(time_str), "%Y-%m-%d %H:%M:%S", tm_info); + } else { + snprintf(time_str, ARRAY_SIZE(time_str), "%" PRIu64 " seconds", + pd_cache_stats->cache_max_age_seconds); + } + + if (pd_cache_stats->hs_cacheable_dbs_cnt) { + SCLogInfo("Rule group caching - loaded: %u newly cached: %u total cacheable: %u", + pd_cache_stats->hs_dbs_cache_loaded_cnt, pd_cache_stats->hs_dbs_cache_saved_cnt, + pd_cache_stats->hs_cacheable_dbs_cnt); + } + if (pd_cache_stats->hs_dbs_cache_pruned_considered_cnt) { + SCLogInfo("Rule group cache pruning removed %u/%u of HS caches due to " + "version-incompatibility (not v%s) or " + "age (older than %s)", + pd_cache_stats->hs_dbs_cache_pruned_cnt, + pd_cache_stats->hs_dbs_cache_pruned_considered_cnt, HS_CACHE_FILE_VERSION, + time_str); + } +} + +void SCHSCacheStatsDeinit(void *data) +{ + if (data == NULL) { + return; + } + PatternDatabaseCache *pd_cache_stats = (PatternDatabaseCache *)data; + SCFree(pd_cache_stats); +} + #endif /* BUILD_HYPERSCAN */ diff --git a/src/util-mpm-hs-cache.h b/src/util-mpm-hs-cache.h index 225c5001a9..24b4eece05 100644 --- a/src/util-mpm-hs-cache.h +++ b/src/util-mpm-hs-cache.h @@ -35,9 +35,24 @@ struct HsIteratorData { const char *cache_path; }; +/** + * \brief Data structure to store in-use cache files. + * Used in cache pruning to avoid deleting files that are still in use. + */ +struct HsInUseCacheFilesIteratorData { + HashTable *tbl; // stores file paths of in-use cache files + const char *cache_path; +}; + int HSLoadCache(hs_database_t **hs_db, const char *hs_db_hash, const char *dirpath); int HSHashDb(const PatternDatabase *pd, char *hash, size_t hash_len); void HSSaveCacheIterator(void *data, void *aux); +void HSCacheFilenameUsedIterator(void *data, void *aux); +int SCHSCachePruneEvaluate(MpmConfig *mpm_conf, HashTable *inuse_caches); + +void *SCHSCacheStatsInit(void); +void SCHSCacheStatsPrint(void *data); +void SCHSCacheStatsDeinit(void *data); #endif /* BUILD_HYPERSCAN */ #endif /* SURICATA_UTIL_MPM_HS_CACHE__H */ diff --git a/src/util-mpm-hs-core.h b/src/util-mpm-hs-core.h index 699dd69568..8392127cf3 100644 --- a/src/util-mpm-hs-core.h +++ b/src/util-mpm-hs-core.h @@ -93,6 +93,10 @@ typedef struct PatternDatabaseCache_ { uint32_t hs_cacheable_dbs_cnt; uint32_t hs_dbs_cache_loaded_cnt; uint32_t hs_dbs_cache_saved_cnt; + uint32_t hs_dbs_cache_pruned_cnt; + uint32_t hs_dbs_cache_pruned_considered_cnt; + time_t hs_dbs_cache_pruned_cutoff; + uint64_t cache_max_age_seconds; } PatternDatabaseCache; const char *HSErrorToStr(hs_error_t error_code); diff --git a/src/util-mpm-hs.c b/src/util-mpm-hs.c index ad7178eb8b..df4a66b2ef 100644 --- a/src/util-mpm-hs.c +++ b/src/util-mpm-hs.c @@ -835,18 +835,53 @@ static int SCHSCacheRuleset(MpmConfig *mpm_conf) mpm_conf->cache_dir_path); return -1; } - PatternDatabaseCache pd_stats = { 0 }; - struct HsIteratorData iter_data = { .pd_stats = &pd_stats, + PatternDatabaseCache *pd_stats = mpm_conf->cache_stats; + struct HsIteratorData iter_data = { .pd_stats = pd_stats, .cache_path = mpm_conf->cache_dir_path }; SCMutexLock(&g_db_table_mutex); HashTableIterate(g_db_table, HSSaveCacheIterator, &iter_data); SCMutexUnlock(&g_db_table_mutex); - SCLogNotice("Rule group caching - loaded: %u newly cached: %u total cacheable: %u", - pd_stats.hs_dbs_cache_loaded_cnt, pd_stats.hs_dbs_cache_saved_cnt, - pd_stats.hs_cacheable_dbs_cnt); return 0; } +static uint32_t FilenameTableHash(HashTable *ht, void *data, uint16_t len) +{ + const char *fname = data; + uint32_t hash = hashlittle_safe(data, strlen(fname), 0); + hash %= ht->array_size; + return hash; +} + +static void FilenameTableFree(void *data) +{ + SCFree(data); +} + +static int SCHSCachePrune(MpmConfig *mpm_conf) +{ + if (!mpm_conf || !mpm_conf->cache_dir_path) { + return -1; + } + + SCLogDebug("Pruning the Hyperscan cache folder %s", mpm_conf->cache_dir_path); + // we need to initialize hash map of in-use cache files + HashTable *inuse_caches = + HashTableInit(INIT_DB_HASH_SIZE, FilenameTableHash, NULL, FilenameTableFree); + if (inuse_caches == NULL) { + return -1; + } + struct HsInUseCacheFilesIteratorData iter_data = { .tbl = inuse_caches, + .cache_path = mpm_conf->cache_dir_path }; + + SCMutexLock(&g_db_table_mutex); + HashTableIterate(g_db_table, HSCacheFilenameUsedIterator, &iter_data); + SCMutexUnlock(&g_db_table_mutex); + + int r = SCHSCachePruneEvaluate(mpm_conf, inuse_caches); + HashTableFree(inuse_caches); + return r; +} + /** * \brief Init the mpm thread context. * @@ -1178,7 +1213,11 @@ void MpmHSRegister(void) mpm_table[MPM_HS].AddPattern = SCHSAddPatternCS; mpm_table[MPM_HS].AddPatternNocase = SCHSAddPatternCI; mpm_table[MPM_HS].Prepare = SCHSPreparePatterns; + mpm_table[MPM_HS].CacheStatsInit = SCHSCacheStatsInit; + mpm_table[MPM_HS].CacheStatsPrint = SCHSCacheStatsPrint; + mpm_table[MPM_HS].CacheStatsDeinit = SCHSCacheStatsDeinit; mpm_table[MPM_HS].CacheRuleset = SCHSCacheRuleset; + mpm_table[MPM_HS].CachePrune = SCHSCachePrune; mpm_table[MPM_HS].Search = SCHSSearch; mpm_table[MPM_HS].PrintCtx = SCHSPrintInfo; mpm_table[MPM_HS].PrintThreadCtx = SCHSPrintSearchStats; diff --git a/src/util-mpm.h b/src/util-mpm.h index c2c4341523..859ceae126 100644 --- a/src/util-mpm.h +++ b/src/util-mpm.h @@ -90,6 +90,8 @@ typedef struct MpmPattern_ { typedef struct MpmConfig_ { const char *cache_dir_path; + uint64_t cache_max_age_seconds; /* 0 means disabled/no pruning policy */ + void *cache_stats; } MpmConfig; typedef struct MpmCtx_ { @@ -175,7 +177,11 @@ typedef struct MpmTableElmt_ { int (*AddPatternNocase)(struct MpmCtx_ *, const uint8_t *, uint16_t, uint16_t, uint16_t, uint32_t, SigIntId, uint8_t); int (*Prepare)(MpmConfig *, struct MpmCtx_ *); + void *(*CacheStatsInit)(void); + void (*CacheStatsPrint)(void *data); + void (*CacheStatsDeinit)(void *data); int (*CacheRuleset)(MpmConfig *); + int (*CachePrune)(MpmConfig *); /** \retval cnt number of patterns that matches: once per pattern max. */ uint32_t (*Search)(const struct MpmCtx_ *, struct MpmThreadCtx_ *, PrefilterRuleStore *, const uint8_t *, uint32_t); void (*PrintCtx)(struct MpmCtx_ *); diff --git a/suricata.yaml.in b/suricata.yaml.in index a0ab5a0665..d7ce7c2cc8 100644 --- a/suricata.yaml.in +++ b/suricata.yaml.in @@ -1810,6 +1810,10 @@ detect: # Cache files are created in the standard library directory. sgh-mpm-caching: yes sgh-mpm-caching-path: @e_sghcachedir@ + # Maximum age for cached MPM databases before they are pruned. + # Accepts a combination of time units (s,m,h,d,w,y). + # Omit to use the default, 0 to disable. + # sgh-mpm-caching-max-age: 7d # inspection-recursion-limit: 3000 # maximum number of times a tx will get logged for rules without app-layer keywords # stream-tx-log-limit: 4