threads: optimize start up check

When starting a large amount of threads, the loop was inefficient. It
would loop over the threads and if one wasn't yet ready it would sleep a
bit and then reevaluate all the threads. This reevaluation of threads
already checked was inefficient, and could lead to the time budget
running out.

This patch splits the check, and keeps track of the threads that have
already passed. This avoids the rescanning of already checked threads.

src/tm-threads.c

@@ -1797,58 +1797,72 @@ void TmThreadContinue(ThreadVars *tv)
     return;
 }
 
-/**
- * \brief Waits for all threads to be in a running state
- *
- * \retval TM_ECODE_OK if all are running or error if a thread failed
- */
-TmEcode TmThreadWaitOnThreadRunning(void)
+static TmEcode WaitOnThreadsRunningByType(const int t)
 {
-    uint16_t RX_num = 0;
-    uint16_t W_num = 0;
-    uint16_t FM_num = 0;
-    uint16_t FR_num = 0;
-    uint16_t TX_num = 0;
-
     struct timeval start_ts;
     struct timeval cur_ts;
     gettimeofday(&start_ts, NULL);
 
+    /* on retries, this will init to the last thread that started up already */
+    ThreadVars *tv_start = tv_root[t];
 again:
     SCMutexLock(&tv_root_lock);
-    for (int i = 0; i < TVT_MAX; i++) {
-        ThreadVars *tv = tv_root[i];
-        while (tv != NULL) {
-            if (TmThreadsCheckFlag(tv, (THV_FAILED | THV_CLOSED | THV_DEAD))) {
-                SCMutexUnlock(&tv_root_lock);
+    ThreadVars *tv = tv_start;
+    while (tv != NULL) {
+        if (TmThreadsCheckFlag(tv, (THV_FAILED | THV_CLOSED | THV_DEAD))) {
+            SCMutexUnlock(&tv_root_lock);
+
+            SCLogError("thread \"%s\" failed to "
+                       "start: flags %04x",
+                    tv->name, SC_ATOMIC_GET(tv->flags));
+            return TM_ECODE_FAILED;
+        }
+
+        if (!(TmThreadsCheckFlag(tv, THV_RUNNING | THV_RUNNING_DONE))) {
+            SCMutexUnlock(&tv_root_lock);
 
+            /* 60 seconds provided for the thread to transition from
+             * THV_INIT_DONE to THV_RUNNING */
+            gettimeofday(&cur_ts, NULL);
+            if ((cur_ts.tv_sec - start_ts.tv_sec) > 60) {
                 SCLogError("thread \"%s\" failed to "
-                           "start: flags %04x",
+                           "start in time: flags %04x",
                         tv->name, SC_ATOMIC_GET(tv->flags));
                 return TM_ECODE_FAILED;
             }
 
-            if (!(TmThreadsCheckFlag(tv, THV_RUNNING | THV_RUNNING_DONE))) {
-                SCMutexUnlock(&tv_root_lock);
+            /* sleep a little to give the thread some
+             * time to start running */
+            SleepUsec(100);
+            goto again;
+        }
+        tv_start = tv;
 
-                /* 60 seconds provided for the thread to transition from
-                 * THV_INIT_DONE to THV_RUNNING */
-                gettimeofday(&cur_ts, NULL);
-                if ((cur_ts.tv_sec - start_ts.tv_sec) > 60) {
-                    SCLogError("thread \"%s\" failed to "
-                               "start in time: flags %04x",
-                            tv->name, SC_ATOMIC_GET(tv->flags));
-                    return TM_ECODE_FAILED;
-                }
+        tv = tv->next;
+    }
+    SCMutexUnlock(&tv_root_lock);
+    return TM_ECODE_OK;
+}
 
-                /* sleep a little to give the thread some
-                 * time to start running */
-                SleepUsec(100);
-                goto again;
-            }
-            tv = tv->next;
-        }
+/**
+ * \brief Waits for all threads to be in a running state
+ *
+ * \retval TM_ECODE_OK if all are running or error if a thread failed
+ */
+TmEcode TmThreadWaitOnThreadRunning(void)
+{
+    uint16_t RX_num = 0;
+    uint16_t W_num = 0;
+    uint16_t FM_num = 0;
+    uint16_t FR_num = 0;
+    uint16_t TX_num = 0;
+
+    for (int i = 0; i < TVT_MAX; i++) {
+        if (WaitOnThreadsRunningByType(i) != TM_ECODE_OK)
+            return TM_ECODE_FAILED;
     }
+
+    SCMutexLock(&tv_root_lock);
     for (int i = 0; i < TVT_MAX; i++) {
         for (ThreadVars *tv = tv_root[i]; tv != NULL; tv = tv->next) {
             if (strncmp(thread_name_autofp, tv->name, strlen(thread_name_autofp)) == 0)