diff --git a/doc/userguide/performance/hyperscan.rst b/doc/userguide/performance/hyperscan.rst index 065163110c..1060d3aefb 100644 --- a/doc/userguide/performance/hyperscan.rst +++ b/doc/userguide/performance/hyperscan.rst @@ -83,6 +83,8 @@ if it is present on the system in case of the "auto" setting. If the current suricata installation does not have hyperscan support, refer to :ref:`installation` +.. _hyperscan-cache-configuration: + Hyperscan caching ~~~~~~~~~~~~~~~~~ @@ -104,6 +106,24 @@ To enable this function, in `suricata.yaml` configure: sgh-mpm-caching-path: /var/lib/suricata/cache/hs +To avoid cache files growing indefinitely, Suricata supports pruning of old +cache files. Suricata removes cache files older than the specified age +on startup/rule reloads, where age is determined by delta of the file +modification time and the current time. +Cache files that are actively being used will have their modification time +updated when loaded, so they won't be deleted. + +In `suricata.yaml` configure: + +:: + + detect: + sgh-mpm-caching-max-age: 7d + +The setting accepts a combination of time units (s,m,h,d,w,y), +e.g. `1w 3d 12h` for 1 week, 3 days and 12 hours. Setting the value to `0` +disables pruning. + **Note**: You might need to create and adjust permissions to the default caching folder path, especially if you are running Suricata as a non-root user. diff --git a/doc/userguide/upgrade.rst b/doc/userguide/upgrade.rst index ef8d1e369d..054e3eb386 100644 --- a/doc/userguide/upgrade.rst +++ b/doc/userguide/upgrade.rst @@ -68,6 +68,10 @@ Other Changes from unbounded to 2048. Configuration options, ``max-tx``, ``max-points``, and ``max-objects`` have been added for users who may need to change these defaults. +- Hyperscan caching (`detect.sgh-mpm-caching`), when enabled, prunes + cache files that have not been used in the last 7 days by default. + See :ref:`Hyperscan caching configuration + ` for more information. Upgrading to 8.0.1 ------------------ diff --git a/src/detect-engine-loader.c b/src/detect-engine-loader.c index ef0e8ef13d..a97ebd6d27 100644 --- a/src/detect-engine-loader.c +++ b/src/detect-engine-loader.c @@ -502,10 +502,6 @@ skip_regular_rules: ret = 0; - if (mpm_table[de_ctx->mpm_matcher].CacheRuleset != NULL) { - mpm_table[de_ctx->mpm_matcher].CacheRuleset(de_ctx->mpm_cfg); - } - end: gettimeofday(&de_ctx->last_reload, NULL); if (SCRunmodeGet() == RUNMODE_ENGINE_ANALYSIS) { diff --git a/src/detect-engine.c b/src/detect-engine.c index 12b1683c59..28e0bc14a3 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -2481,6 +2481,49 @@ const char *DetectEngineMpmCachingGetPath(void) return SGH_CACHE_DIR; } +void DetectEngineMpmCacheService(uint32_t op_flags) +{ + DetectEngineCtx *de_ctx = DetectEngineGetCurrent(); + if (!de_ctx) { + return; + } + + if (!de_ctx->mpm_cfg || !de_ctx->mpm_cfg->cache_dir_path) { + goto error; + } + + if (mpm_table[de_ctx->mpm_matcher].CacheStatsInit != NULL) { + de_ctx->mpm_cfg->cache_stats = mpm_table[de_ctx->mpm_matcher].CacheStatsInit(); + if (de_ctx->mpm_cfg->cache_stats == NULL) { + goto error; + } + } + + if (op_flags & DETECT_ENGINE_MPM_CACHE_OP_SAVE) { + if (mpm_table[de_ctx->mpm_matcher].CacheRuleset != NULL) { + mpm_table[de_ctx->mpm_matcher].CacheRuleset(de_ctx->mpm_cfg); + } + } + + if (op_flags & DETECT_ENGINE_MPM_CACHE_OP_PRUNE) { + if (mpm_table[de_ctx->mpm_matcher].CachePrune != NULL) { + mpm_table[de_ctx->mpm_matcher].CachePrune(de_ctx->mpm_cfg); + } + } + + if (mpm_table[de_ctx->mpm_matcher].CacheStatsPrint != NULL) { + mpm_table[de_ctx->mpm_matcher].CacheStatsPrint(de_ctx->mpm_cfg->cache_stats); + } + + if (mpm_table[de_ctx->mpm_matcher].CacheStatsDeinit != NULL) { + mpm_table[de_ctx->mpm_matcher].CacheStatsDeinit(de_ctx->mpm_cfg->cache_stats); + de_ctx->mpm_cfg->cache_stats = NULL; + } + +error: + DetectEngineDeReference(&de_ctx); +} + static DetectEngineCtx *DetectEngineCtxInitReal( enum DetectEngineType type, const char *prefix, uint32_t tenant_id) { @@ -2503,10 +2546,18 @@ static DetectEngineCtx *DetectEngineCtxInitReal( if (de_ctx->mpm_cfg == NULL) { goto error; } - } - if (DetectEngineMpmCachingEnabled() && mpm_table[de_ctx->mpm_matcher].ConfigCacheDirSet) { - mpm_table[de_ctx->mpm_matcher].ConfigCacheDirSet( - de_ctx->mpm_cfg, DetectEngineMpmCachingGetPath()); + + if (DetectEngineMpmCachingEnabled() && mpm_table[de_ctx->mpm_matcher].ConfigCacheDirSet) { + mpm_table[de_ctx->mpm_matcher].ConfigCacheDirSet( + de_ctx->mpm_cfg, DetectEngineMpmCachingGetPath()); + + if (mpm_table[de_ctx->mpm_matcher].CachePrune) { + if (SCConfGetTime("detect.sgh-mpm-caching-max-age", + &de_ctx->mpm_cfg->cache_max_age_seconds) != 1) { + de_ctx->mpm_cfg->cache_max_age_seconds = 7ULL * 24ULL * 60ULL * 60ULL; + } + } + } } if (type == DETECT_ENGINE_TYPE_DD_STUB || type == DETECT_ENGINE_TYPE_MT_STUB) { @@ -4885,6 +4936,8 @@ int DetectEngineReload(const SCInstance *suri) SCLogDebug("old_de_ctx should have been freed"); + DetectEngineMpmCacheService(DETECT_ENGINE_MPM_CACHE_OP_SAVE | DETECT_ENGINE_MPM_CACHE_OP_PRUNE); + SCLogNotice("rule reload complete"); #ifdef HAVE_MALLOC_TRIM diff --git a/src/detect-engine.h b/src/detect-engine.h index 2c56475f62..2d45d32535 100644 --- a/src/detect-engine.h +++ b/src/detect-engine.h @@ -88,6 +88,7 @@ TmEcode DetectEngineThreadCtxInit(ThreadVars *, void *, void **); TmEcode DetectEngineThreadCtxDeinit(ThreadVars *, void *); bool DetectEngineMpmCachingEnabled(void); const char *DetectEngineMpmCachingGetPath(void); +void DetectEngineMpmCacheService(uint32_t op_flags); /* faster as a macro than a inline function on my box -- VJ */ #define DetectEngineGetMaxSigId(de_ctx) ((de_ctx)->signum) void DetectEngineResetMaxSigId(DetectEngineCtx *); diff --git a/src/detect.h b/src/detect.h index 62c888e6a7..49fbfe3eb6 100644 --- a/src/detect.h +++ b/src/detect.h @@ -1750,6 +1750,9 @@ extern SigTableElmt *sigmatch_table; /** Remember to add the options in SignatureIsIPOnly() at detect.c otherwise it wont be part of a signature group */ +#define DETECT_ENGINE_MPM_CACHE_OP_PRUNE BIT_U32(0) +#define DETECT_ENGINE_MPM_CACHE_OP_SAVE BIT_U32(1) + /* detection api */ TmEcode Detect(ThreadVars *tv, Packet *p, void *data); uint8_t DetectPreFlow(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Packet *p); diff --git a/src/runmode-unix-socket.c b/src/runmode-unix-socket.c index c2405f0571..706a35b7e6 100644 --- a/src/runmode-unix-socket.c +++ b/src/runmode-unix-socket.c @@ -967,6 +967,8 @@ TmEcode UnixSocketRegisterTenantHandler(json_t *cmd, json_t* answer, void *data) return TM_ECODE_FAILED; } + DetectEngineMpmCacheService(DETECT_ENGINE_MPM_CACHE_OP_SAVE); + json_object_set_new(answer, "message", json_string("handler added")); return TM_ECODE_OK; } @@ -1054,6 +1056,8 @@ TmEcode UnixSocketUnregisterTenantHandler(json_t *cmd, json_t* answer, void *dat return TM_ECODE_FAILED; } + DetectEngineMpmCacheService(DETECT_ENGINE_MPM_CACHE_OP_PRUNE); + json_object_set_new(answer, "message", json_string("handler removed")); return TM_ECODE_OK; } @@ -1126,6 +1130,8 @@ TmEcode UnixSocketRegisterTenant(json_t *cmd, json_t* answer, void *data) return TM_ECODE_FAILED; } + DetectEngineMpmCacheService(DETECT_ENGINE_MPM_CACHE_OP_SAVE); + json_object_set_new(answer, "message", json_string("adding tenant succeeded")); return TM_ECODE_OK; } @@ -1193,6 +1199,8 @@ TmEcode UnixSocketReloadTenant(json_t *cmd, json_t* answer, void *data) return TM_ECODE_FAILED; } + DetectEngineMpmCacheService(DETECT_ENGINE_MPM_CACHE_OP_SAVE | DETECT_ENGINE_MPM_CACHE_OP_PRUNE); + json_object_set_new(answer, "message", json_string("reloading tenant succeeded")); return TM_ECODE_OK; } @@ -1226,6 +1234,7 @@ TmEcode UnixSocketReloadTenants(json_t *cmd, json_t *answer, void *data) return TM_ECODE_FAILED; } + DetectEngineMpmCacheService(DETECT_ENGINE_MPM_CACHE_OP_SAVE | DETECT_ENGINE_MPM_CACHE_OP_PRUNE); SCLogNotice("reload-tenants complete"); json_object_set_new(answer, "message", json_string("reloading tenants succeeded")); @@ -1284,6 +1293,8 @@ TmEcode UnixSocketUnregisterTenant(json_t *cmd, json_t* answer, void *data) return TM_ECODE_FAILED; } + DetectEngineMpmCacheService(DETECT_ENGINE_MPM_CACHE_OP_PRUNE); + /* walk free list, freeing the removed de_ctx */ DetectEnginePruneFreeList(); diff --git a/src/suricata.c b/src/suricata.c index c6f94c3ce8..a106c56f76 100644 --- a/src/suricata.c +++ b/src/suricata.c @@ -2688,6 +2688,8 @@ void PostConfLoadedDetectSetup(SCInstance *suri) gettimeofday(&de_ctx->last_reload, NULL); DetectEngineAddToMaster(de_ctx); DetectEngineBumpVersion(); + DetectEngineMpmCacheService( + DETECT_ENGINE_MPM_CACHE_OP_SAVE | DETECT_ENGINE_MPM_CACHE_OP_PRUNE); } } diff --git a/src/util-mpm-hs-cache.c b/src/util-mpm-hs-cache.c index 41b3081713..58a2aa6ab4 100644 --- a/src/util-mpm-hs-cache.c +++ b/src/util-mpm-hs-cache.c @@ -37,21 +37,22 @@ #include "rust.h" #include -static const char *HSCacheConstructFPath(const char *folder_path, const char *hs_db_hash) -{ - static char hash_file_path[PATH_MAX]; +#define HS_CACHE_FILE_VERSION "2" +#define HS_CACHE_FILE_SUFFIX "_v" HS_CACHE_FILE_VERSION ".hs" - char hash_file_path_suffix[] = "_v1.hs"; +static int16_t HSCacheConstructFPath( + const char *dir_path, const char *db_hash, char *out_path, uint16_t out_path_size) +{ char filename[NAME_MAX]; - uint64_t r = snprintf(filename, sizeof(filename), "%s%s", hs_db_hash, hash_file_path_suffix); - if (r != (uint64_t)(strlen(hs_db_hash) + strlen(hash_file_path_suffix))) - return NULL; + uint64_t r = snprintf(filename, sizeof(filename), "%s" HS_CACHE_FILE_SUFFIX, db_hash); + if (r != (uint64_t)(strlen(db_hash) + strlen(HS_CACHE_FILE_SUFFIX))) + return -1; - r = PathMerge(hash_file_path, sizeof(hash_file_path), folder_path, filename); + r = PathMerge(out_path, out_path_size, dir_path, filename); if (r) - return NULL; + return -1; - return hash_file_path; + return 0; } static char *HSReadStream(const char *file_path, size_t *buffer_sz) @@ -121,8 +122,11 @@ static void SCHSCachePatternHash(const SCHSPattern *p, SCSha256 *sha256) int HSLoadCache(hs_database_t **hs_db, const char *hs_db_hash, const char *dirpath) { - const char *hash_file_static = HSCacheConstructFPath(dirpath, hs_db_hash); - if (hash_file_static == NULL) + char hash_file_static[PATH_MAX]; + int ret = (int)HSCacheConstructFPath( + dirpath, hs_db_hash, hash_file_static, sizeof(hash_file_static)); + + if (ret != 0) return -1; SCLogDebug("Loading the cached HS DB from %s", hash_file_static); @@ -131,7 +135,6 @@ int HSLoadCache(hs_database_t **hs_db, const char *hs_db_hash, const char *dirpa FILE *db_cache = fopen(hash_file_static, "r"); char *buffer = NULL; - int ret = 0; if (db_cache) { size_t buffer_size; buffer = HSReadStream(hash_file_static, &buffer_size); @@ -170,15 +173,20 @@ static int HSSaveCache(hs_database_t *hs_db, const char *hs_db_hash, const char static bool notified = false; char *db_stream = NULL; size_t db_size; - int ret = -1; + int ret; hs_error_t err = hs_serialize_database(hs_db, &db_stream, &db_size); if (err != HS_SUCCESS) { SCLogWarning("Failed to serialize Hyperscan database: %s", HSErrorToStr(err)); + ret = -1; goto cleanup; } - const char *hash_file_static = HSCacheConstructFPath(dstpath, hs_db_hash); + char hash_file_static[PATH_MAX]; + ret = (int)HSCacheConstructFPath( + dstpath, hs_db_hash, hash_file_static, sizeof(hash_file_static)); + if (ret != 0) + goto cleanup; SCLogDebug("Caching the compiled HS at %s", hash_file_static); if (SCPathExists(hash_file_static)) { // potentially signs that it might not work as expected as we got into @@ -198,6 +206,7 @@ static int HSSaveCache(hs_database_t *hs_db, const char *hs_db_hash, const char hash_file_static); notified = true; } + ret = -1; goto cleanup; } size_t r = fwrite(db_stream, sizeof(db_stream[0]), db_size, db_cache_out); @@ -217,7 +226,6 @@ static int HSSaveCache(hs_database_t *hs_db, const char *hs_db_hash, const char goto cleanup; } - ret = 0; cleanup: if (db_stream) SCFree(db_stream); @@ -270,4 +278,187 @@ void HSSaveCacheIterator(void *data, void *aux) } } +void HSCacheFilenameUsedIterator(void *data, void *aux) +{ + PatternDatabase *pd = (PatternDatabase *)data; + struct HsInUseCacheFilesIteratorData *iter_data = (struct HsInUseCacheFilesIteratorData *)aux; + if (pd->no_cache || !pd->cached) + return; + + char hs_db_hash[SC_SHA256_LEN * 2 + 1]; // * 2 for hex +1 for nul terminator + if (HSHashDb(pd, hs_db_hash, ARRAY_SIZE(hs_db_hash)) != 0) { + return; + } + + char *fpath = SCCalloc(PATH_MAX, sizeof(char)); + if (fpath == NULL) { + SCLogWarning("Failed to allocate memory for cache file path"); + return; + } + if (HSCacheConstructFPath(iter_data->cache_path, hs_db_hash, fpath, PATH_MAX)) { + SCFree(fpath); + return; + } + + int r = HashTableAdd(iter_data->tbl, (void *)fpath, (uint16_t)strlen(fpath)); + if (r < 0) { + SCLogWarning("Failed to add used cache file path %s to hash table", fpath); + SCFree(fpath); + } +} + +/** + * \brief Check if HS cache file is stale by age. + * + * \param mtime File modification time. + * \param cutoff Time cutoff (files older than this will be removed). + * + * \retval true if file should be pruned, false otherwise. + */ +static bool HSPruneFileByAge(time_t mtime, time_t cutoff) +{ + return mtime < cutoff; +} + +/** + * \brief Check if HS cache file is version-compatible. + * + * \param filename Cache file name. + * + * \retval true if file should be pruned, false otherwise. + */ +static bool HSPruneFileByVersion(const char *filename) +{ + if (strlen(filename) < strlen(HS_CACHE_FILE_SUFFIX)) { + return true; + } + + const char *underscore = strrchr(filename, '_'); + if (underscore == NULL || strcmp(underscore, HS_CACHE_FILE_SUFFIX) != 0) { + return true; + } + + return false; +} + +int SCHSCachePruneEvaluate(MpmConfig *mpm_conf, HashTable *inuse_caches) +{ + if (mpm_conf == NULL || mpm_conf->cache_dir_path == NULL) + return -1; + if (mpm_conf->cache_max_age_seconds == 0) + return 0; // disabled + + const time_t now = time(NULL); + if (now == (time_t)-1) { + return -1; + } else if (mpm_conf->cache_max_age_seconds >= (uint64_t)now) { + return 0; + } + + DIR *dir = opendir(mpm_conf->cache_dir_path); + if (dir == NULL) { + return -1; + } + + struct dirent *ent; + char path[PATH_MAX]; + uint32_t considered = 0, removed = 0; + const time_t cutoff = now - (time_t)mpm_conf->cache_max_age_seconds; + while ((ent = readdir(dir)) != NULL) { + const char *name = ent->d_name; + size_t namelen = strlen(name); + if (namelen < 3 || strcmp(name + namelen - 3, ".hs") != 0) + continue; + + if (PathMerge(path, ARRAY_SIZE(path), mpm_conf->cache_dir_path, name) != 0) + continue; + + struct stat st; + if (stat(path, &st) != 0 || !S_ISREG(st.st_mode)) + continue; + + considered++; + + const bool prune_by_age = HSPruneFileByAge(st.st_mtime, cutoff); + const bool prune_by_version = HSPruneFileByVersion(name); + if (!prune_by_age && !prune_by_version) + continue; + + void *cache_inuse = HashTableLookup(inuse_caches, path, (uint16_t)strlen(path)); + if (cache_inuse != NULL) + continue; // in use + + if (unlink(path) == 0) { + removed++; + SCLogDebug("File %s removed because of %s%s%s", path, prune_by_age ? "age" : "", + prune_by_age && prune_by_version ? " and " : "", + prune_by_version ? "incompatible version" : ""); + } else { + SCLogWarning("Failed to prune \"%s\": %s", path, strerror(errno)); + } + } + closedir(dir); + + PatternDatabaseCache *pd_cache_stats = mpm_conf->cache_stats; + if (pd_cache_stats) { + pd_cache_stats->hs_dbs_cache_pruned_cnt = removed; + pd_cache_stats->hs_dbs_cache_pruned_considered_cnt = considered; + pd_cache_stats->hs_dbs_cache_pruned_cutoff = cutoff; + pd_cache_stats->cache_max_age_seconds = mpm_conf->cache_max_age_seconds; + } + return 0; +} + +void *SCHSCacheStatsInit(void) +{ + PatternDatabaseCache *pd_cache_stats = SCCalloc(1, sizeof(PatternDatabaseCache)); + if (pd_cache_stats == NULL) { + SCLogError("Failed to allocate memory for Hyperscan cache stats"); + return NULL; + } + return pd_cache_stats; +} + +void SCHSCacheStatsPrint(void *data) +{ + if (data == NULL) { + return; + } + + PatternDatabaseCache *pd_cache_stats = (PatternDatabaseCache *)data; + + char time_str[64]; + struct tm tm_s; + struct tm *tm_info = SCLocalTime(pd_cache_stats->hs_dbs_cache_pruned_cutoff, &tm_s); + if (tm_info != NULL) { + strftime(time_str, ARRAY_SIZE(time_str), "%Y-%m-%d %H:%M:%S", tm_info); + } else { + snprintf(time_str, ARRAY_SIZE(time_str), "%" PRIu64 " seconds", + pd_cache_stats->cache_max_age_seconds); + } + + if (pd_cache_stats->hs_cacheable_dbs_cnt) { + SCLogInfo("Rule group caching - loaded: %u newly cached: %u total cacheable: %u", + pd_cache_stats->hs_dbs_cache_loaded_cnt, pd_cache_stats->hs_dbs_cache_saved_cnt, + pd_cache_stats->hs_cacheable_dbs_cnt); + } + if (pd_cache_stats->hs_dbs_cache_pruned_considered_cnt) { + SCLogInfo("Rule group cache pruning removed %u/%u of HS caches due to " + "version-incompatibility (not v%s) or " + "age (older than %s)", + pd_cache_stats->hs_dbs_cache_pruned_cnt, + pd_cache_stats->hs_dbs_cache_pruned_considered_cnt, HS_CACHE_FILE_VERSION, + time_str); + } +} + +void SCHSCacheStatsDeinit(void *data) +{ + if (data == NULL) { + return; + } + PatternDatabaseCache *pd_cache_stats = (PatternDatabaseCache *)data; + SCFree(pd_cache_stats); +} + #endif /* BUILD_HYPERSCAN */ diff --git a/src/util-mpm-hs-cache.h b/src/util-mpm-hs-cache.h index 225c5001a9..24b4eece05 100644 --- a/src/util-mpm-hs-cache.h +++ b/src/util-mpm-hs-cache.h @@ -35,9 +35,24 @@ struct HsIteratorData { const char *cache_path; }; +/** + * \brief Data structure to store in-use cache files. + * Used in cache pruning to avoid deleting files that are still in use. + */ +struct HsInUseCacheFilesIteratorData { + HashTable *tbl; // stores file paths of in-use cache files + const char *cache_path; +}; + int HSLoadCache(hs_database_t **hs_db, const char *hs_db_hash, const char *dirpath); int HSHashDb(const PatternDatabase *pd, char *hash, size_t hash_len); void HSSaveCacheIterator(void *data, void *aux); +void HSCacheFilenameUsedIterator(void *data, void *aux); +int SCHSCachePruneEvaluate(MpmConfig *mpm_conf, HashTable *inuse_caches); + +void *SCHSCacheStatsInit(void); +void SCHSCacheStatsPrint(void *data); +void SCHSCacheStatsDeinit(void *data); #endif /* BUILD_HYPERSCAN */ #endif /* SURICATA_UTIL_MPM_HS_CACHE__H */ diff --git a/src/util-mpm-hs-core.h b/src/util-mpm-hs-core.h index 699dd69568..8392127cf3 100644 --- a/src/util-mpm-hs-core.h +++ b/src/util-mpm-hs-core.h @@ -93,6 +93,10 @@ typedef struct PatternDatabaseCache_ { uint32_t hs_cacheable_dbs_cnt; uint32_t hs_dbs_cache_loaded_cnt; uint32_t hs_dbs_cache_saved_cnt; + uint32_t hs_dbs_cache_pruned_cnt; + uint32_t hs_dbs_cache_pruned_considered_cnt; + time_t hs_dbs_cache_pruned_cutoff; + uint64_t cache_max_age_seconds; } PatternDatabaseCache; const char *HSErrorToStr(hs_error_t error_code); diff --git a/src/util-mpm-hs.c b/src/util-mpm-hs.c index ad7178eb8b..df4a66b2ef 100644 --- a/src/util-mpm-hs.c +++ b/src/util-mpm-hs.c @@ -835,18 +835,53 @@ static int SCHSCacheRuleset(MpmConfig *mpm_conf) mpm_conf->cache_dir_path); return -1; } - PatternDatabaseCache pd_stats = { 0 }; - struct HsIteratorData iter_data = { .pd_stats = &pd_stats, + PatternDatabaseCache *pd_stats = mpm_conf->cache_stats; + struct HsIteratorData iter_data = { .pd_stats = pd_stats, .cache_path = mpm_conf->cache_dir_path }; SCMutexLock(&g_db_table_mutex); HashTableIterate(g_db_table, HSSaveCacheIterator, &iter_data); SCMutexUnlock(&g_db_table_mutex); - SCLogNotice("Rule group caching - loaded: %u newly cached: %u total cacheable: %u", - pd_stats.hs_dbs_cache_loaded_cnt, pd_stats.hs_dbs_cache_saved_cnt, - pd_stats.hs_cacheable_dbs_cnt); return 0; } +static uint32_t FilenameTableHash(HashTable *ht, void *data, uint16_t len) +{ + const char *fname = data; + uint32_t hash = hashlittle_safe(data, strlen(fname), 0); + hash %= ht->array_size; + return hash; +} + +static void FilenameTableFree(void *data) +{ + SCFree(data); +} + +static int SCHSCachePrune(MpmConfig *mpm_conf) +{ + if (!mpm_conf || !mpm_conf->cache_dir_path) { + return -1; + } + + SCLogDebug("Pruning the Hyperscan cache folder %s", mpm_conf->cache_dir_path); + // we need to initialize hash map of in-use cache files + HashTable *inuse_caches = + HashTableInit(INIT_DB_HASH_SIZE, FilenameTableHash, NULL, FilenameTableFree); + if (inuse_caches == NULL) { + return -1; + } + struct HsInUseCacheFilesIteratorData iter_data = { .tbl = inuse_caches, + .cache_path = mpm_conf->cache_dir_path }; + + SCMutexLock(&g_db_table_mutex); + HashTableIterate(g_db_table, HSCacheFilenameUsedIterator, &iter_data); + SCMutexUnlock(&g_db_table_mutex); + + int r = SCHSCachePruneEvaluate(mpm_conf, inuse_caches); + HashTableFree(inuse_caches); + return r; +} + /** * \brief Init the mpm thread context. * @@ -1178,7 +1213,11 @@ void MpmHSRegister(void) mpm_table[MPM_HS].AddPattern = SCHSAddPatternCS; mpm_table[MPM_HS].AddPatternNocase = SCHSAddPatternCI; mpm_table[MPM_HS].Prepare = SCHSPreparePatterns; + mpm_table[MPM_HS].CacheStatsInit = SCHSCacheStatsInit; + mpm_table[MPM_HS].CacheStatsPrint = SCHSCacheStatsPrint; + mpm_table[MPM_HS].CacheStatsDeinit = SCHSCacheStatsDeinit; mpm_table[MPM_HS].CacheRuleset = SCHSCacheRuleset; + mpm_table[MPM_HS].CachePrune = SCHSCachePrune; mpm_table[MPM_HS].Search = SCHSSearch; mpm_table[MPM_HS].PrintCtx = SCHSPrintInfo; mpm_table[MPM_HS].PrintThreadCtx = SCHSPrintSearchStats; diff --git a/src/util-mpm.h b/src/util-mpm.h index c2c4341523..859ceae126 100644 --- a/src/util-mpm.h +++ b/src/util-mpm.h @@ -90,6 +90,8 @@ typedef struct MpmPattern_ { typedef struct MpmConfig_ { const char *cache_dir_path; + uint64_t cache_max_age_seconds; /* 0 means disabled/no pruning policy */ + void *cache_stats; } MpmConfig; typedef struct MpmCtx_ { @@ -175,7 +177,11 @@ typedef struct MpmTableElmt_ { int (*AddPatternNocase)(struct MpmCtx_ *, const uint8_t *, uint16_t, uint16_t, uint16_t, uint32_t, SigIntId, uint8_t); int (*Prepare)(MpmConfig *, struct MpmCtx_ *); + void *(*CacheStatsInit)(void); + void (*CacheStatsPrint)(void *data); + void (*CacheStatsDeinit)(void *data); int (*CacheRuleset)(MpmConfig *); + int (*CachePrune)(MpmConfig *); /** \retval cnt number of patterns that matches: once per pattern max. */ uint32_t (*Search)(const struct MpmCtx_ *, struct MpmThreadCtx_ *, PrefilterRuleStore *, const uint8_t *, uint32_t); void (*PrintCtx)(struct MpmCtx_ *); diff --git a/suricata.yaml.in b/suricata.yaml.in index a0ab5a0665..d7ce7c2cc8 100644 --- a/suricata.yaml.in +++ b/suricata.yaml.in @@ -1810,6 +1810,10 @@ detect: # Cache files are created in the standard library directory. sgh-mpm-caching: yes sgh-mpm-caching-path: @e_sghcachedir@ + # Maximum age for cached MPM databases before they are pruned. + # Accepts a combination of time units (s,m,h,d,w,y). + # Omit to use the default, 0 to disable. + # sgh-mpm-caching-max-age: 7d # inspection-recursion-limit: 3000 # maximum number of times a tx will get logged for rules without app-layer keywords # stream-tx-log-limit: 4