lua: add bounds checks for flowints and flowvars

Ticket: 8556
(cherry picked from commit 0b239c4848)

src/util-lua-flowintlib.c

@@ -56,6 +56,9 @@ static int LuaFlowintRegister(lua_State *L)
     if (*flowvar_id == 0) {
         return luaL_error(L, "failed to register flowvar");
     }
+    if (ld->flowints >= DETECT_LUA_MAX_FLOWINTS) {
+        return luaL_error(L, "too many flowints registered");
+    }
     ld->flowint[ld->flowints++] = *flowvar_id;
 
     luaL_getmetatable(L, suricata_flowint_mt);

src/util-lua-flowvarlib.c

@@ -56,6 +56,9 @@ static int LuaFlowvarRegister(lua_State *L)
     if (*flowvar_id == 0) {
         return luaL_error(L, "failed to register flowvar");
     }
+    if (ld->flowvars >= DETECT_LUA_MAX_FLOWVARS) {
+        return luaL_error(L, "too many flowvars registered");
+    }
     ld->flowvar[ld->flowvars++] = *flowvar_id;
 
     luaL_getmetatable(L, suricata_flowvar_mt);