detect/alert: check alert queue capacity before expanding

So far, the alert queue was expanded by doubling in size w/o any
boundary checks in place. This led to situations where doubling
the alert_queue_capacity meant overflow of the very same value
stored in det_ctx.
This led to heap-use-after-free in some conditions where
det_ctx->alert_queue_capacity overflowed.

Fix this by capping the max of alert_queue_capacity by checking if its
expansion could result in an overflow.

Security 8190

(cherry picked from commit ac1eb39418)

src/detect-engine-alert.c

@@ -238,6 +238,22 @@ void AlertQueueFree(DetectEngineThreadCtx *det_ctx)
     det_ctx->alert_queue_capacity = 0;
 }
 
+static inline uint16_t AlertQueueExpandDo(DetectEngineThreadCtx *det_ctx, uint16_t new_cap)
+{
+    DEBUG_VALIDATE_BUG_ON(det_ctx->alert_queue_capacity >= new_cap);
+
+    void *tmp_queue = SCRealloc(det_ctx->alert_queue, new_cap * sizeof(PacketAlert));
+    if (unlikely(tmp_queue == NULL)) {
+        /* queue capacity didn't change */
+        return det_ctx->alert_queue_capacity;
+    }
+    det_ctx->alert_queue = tmp_queue;
+    det_ctx->alert_queue_capacity = new_cap;
+    SCLogDebug("Alert queue size expanded: %u elements, bytes: %" PRIuMAX "",
+            det_ctx->alert_queue_capacity, (uintmax_t)(new_cap * sizeof(PacketAlert)));
+    return new_cap;
+}
+
 /** \internal
  * \retval the new capacity
  */
@@ -247,18 +263,17 @@ static uint16_t AlertQueueExpand(DetectEngineThreadCtx *det_ctx)
     if (unlikely(g_eps_is_alert_queue_fail_mode))
         return det_ctx->alert_queue_capacity;
 #endif
-    uint16_t new_cap = det_ctx->alert_queue_capacity * 2;
-    void *tmp_queue = SCRealloc(det_ctx->alert_queue, (size_t)(sizeof(PacketAlert) * new_cap));
-    if (unlikely(tmp_queue == NULL)) {
-        /* queue capacity didn't change */
+    if (det_ctx->alert_queue_capacity == UINT16_MAX) {
         return det_ctx->alert_queue_capacity;
     }
-    det_ctx->alert_queue = tmp_queue;
-    det_ctx->alert_queue_capacity = new_cap;
-    SCLogDebug("Alert queue size doubled: %u elements, bytes: %" PRIuMAX "",
-            det_ctx->alert_queue_capacity,
-            (uintmax_t)(sizeof(PacketAlert) * det_ctx->alert_queue_capacity));
-    return new_cap;
+
+    uint16_t new_cap;
+    if (det_ctx->alert_queue_capacity > (UINT16_MAX / 2)) {
+        new_cap = UINT16_MAX;
+    } else {
+        new_cap = det_ctx->alert_queue_capacity * 2;
+    }
+    return AlertQueueExpandDo(det_ctx, new_cap);
 }
 
 /** \internal