Add --unittests-coverage option to list how many code modules have tests

src/app-layer-dcerpc-udp.c

@@ -729,6 +729,9 @@ void RegisterDCERPCUDPParsers(void) {
 			DCERPCUDPParse);
 	AppLayerRegisterStateFuncs(ALPROTO_DCERPC_UDP, DCERPCUDPStateAlloc,
 			DCERPCUDPStateFree);
+#ifdef UNITTESTS
+    AppLayerRegisterUnittests(ALPROTO_DCERPC_UDP, DCERPCUDPParserRegisterTests);
+#endif
 }
 
 /* UNITTESTS */

src/app-layer-dcerpc.c

@@ -1878,6 +1878,9 @@ void RegisterDCERPCParsers(void) {
             DCERPCParseResponse);
     AppLayerRegisterStateFuncs(ALPROTO_DCERPC, DCERPCStateAlloc,
             DCERPCStateFree);
+#ifdef UNITTESTS
+    AppLayerRegisterUnittests(ALPROTO_DCERPC, DCERPCParserRegisterTests);
+#endif
 }
 
 /* UNITTESTS */

src/app-layer-ftp.c

@@ -279,6 +279,9 @@ void RegisterFTPParsers(void) {
                            FTP_FIELD_REQUEST_LINE, FTPParseRequestCommandLine,
                            "ftp");
     AppLayerRegisterStateFuncs(ALPROTO_FTP, FTPStateAlloc, FTPStateFree);
+#ifdef UNITTESTS
+    AppLayerRegisterUnittests(ALPROTO_FTP, FTPParserRegisterTests);
+#endif
 }
 
 void FTPAtExitPrintStats(void) {

src/app-layer-htp.c

@@ -2425,6 +2425,9 @@ void RegisterHTPParsers(void)
                           HTPHandleRequestData);
     AppLayerRegisterProto(proto_name, ALPROTO_HTTP, STREAM_TOCLIENT,
                           HTPHandleResponseData);
+#ifdef UNITTESTS
+    AppLayerRegisterUnittests(ALPROTO_HTTP, HTPParserRegisterTests);
+#endif
 
     SC_ATOMIC_INIT(htp_config_flags);
     HTPConfigure();

src/app-layer-parser.c

@@ -45,6 +45,7 @@
 #include "app-layer-protos.h"
 #include "app-layer-parser.h"
 #include "app-layer-smb.h"
+#include "app-layer-smb2.h"
 #include "app-layer-dcerpc.h"
 #include "app-layer-dcerpc-udp.h"
 #include "app-layer-htp.h"
@@ -727,6 +728,12 @@ int AppLayerRegisterProto(char *name, uint8_t proto, uint8_t flags,
     return 0;
 }
 
+#ifdef UNITTESTS
+void AppLayerRegisterUnittests(uint16_t proto, void (*RegisterUnittests)(void)) {
+    al_proto_table[proto].RegisterUnittests = RegisterUnittests;
+}
+#endif
+
 void AppLayerRegisterStateFuncs(uint16_t proto, void *(*StateAlloc)(void),
                                 void (*StateFree)(void *))
 {
@@ -1337,6 +1344,8 @@ void RegisterAppLayerParsers(void)
     RegisterHTPParsers();
     RegisterSSLParsers();
     RegisterSMBParsers();
+    /** \todo bug 719 */
+    //RegisterSMB2Parsers();
     RegisterDCERPCParsers();
     RegisterDCERPCUDPParsers();
     RegisterFTPParsers();
@@ -5869,6 +5878,25 @@ static int AppLayerProbingParserTest15(void)
 void AppLayerParserRegisterTests(void)
 {
 #ifdef UNITTESTS
+    int i;
+    for (i = 0; i < ALPROTO_MAX; i++) {
+        AppLayerProto *p = &al_proto_table[i];
+
+        if (p->name == NULL)
+            continue;
+
+        g_ut_modules++;
+
+        if (p->RegisterUnittests != NULL) {
+            p->RegisterUnittests();
+            g_ut_covered++;
+        } else {
+            if (coverage_unittests)
+                SCLogWarning(SC_WARN_NO_UNITTESTS, "app layer module %s has no "
+                        "unittests", p->name);
+        }
+    }
+
     UtRegisterTest("AppLayerParserTest01", AppLayerParserTest01, 1);
     UtRegisterTest("AppLayerParserTest02", AppLayerParserTest02, 1);
     UtRegisterTest("AppLayerProbingParserTest01", AppLayerProbingParserTest01, 1);

src/app-layer-parser.h

@@ -65,6 +65,10 @@ typedef struct AppLayerProto_ {
     uint64_t (*StateGetTxCnt)(void *alstate);
     void *(*StateGetTx)(void *alstate, uint64_t tx_id);
     int (*StateGetAlstateProgressCompletionStatus)(uint8_t direction);
+
+#ifdef UNITTESTS
+    void (*RegisterUnittests)(void);
+#endif
 } AppLayerProto;
 
 /** flags for the result elmts */
@@ -249,6 +253,9 @@ void AppLayerRegisterProbingParser(struct AlpProtoDetectCtx_ *, uint16_t, uint16
                                    uint16_t, uint16_t, uint8_t, uint8_t,
                                    uint8_t,
                                    uint16_t (*ProbingParser)(uint8_t *, uint32_t));
+#ifdef UNITTESTS
+void AppLayerRegisterUnittests(uint16_t proto, void (*RegisterUnittests)(void));
+#endif
 void AppLayerRegisterStateFuncs(uint16_t proto, void *(*StateAlloc)(void),
                                 void (*StateFree)(void *));
 void AppLayerRegisterLocalStorageFunc(uint16_t proto,

src/app-layer-smb.c

@@ -1419,7 +1419,9 @@ void RegisterSMBParsers(void) {
                                   STREAM_TOSERVER,
                                   APP_LAYER_PROBING_PARSER_PRIORITY_HIGH, 1,
                                   SMBProbingParser);
-
+#ifdef UNITTESTS
+    AppLayerRegisterUnittests(ALPROTO_SMB, SMBParserRegisterTests);
+#endif
     return;
 }
 

src/app-layer-smb2.c

@@ -521,6 +521,9 @@ void RegisterSMB2Parsers(void) {
     AppLayerRegisterProto("smb", ALPROTO_SMB2, STREAM_TOSERVER, SMB2Parse);
     AppLayerRegisterProto("smb", ALPROTO_SMB2, STREAM_TOCLIENT, SMB2Parse);
     AppLayerRegisterStateFuncs(ALPROTO_SMB2, SMB2StateAlloc, SMB2StateFree);
+#ifdef UNITTESTS
+    AppLayerRegisterUnittests(ALPROTO_SMB2, SMB2ParserRegisterTests);
+#endif
 }
 
 /* UNITTESTS */
@@ -585,7 +588,6 @@ end:
 }
 
 void SMB2ParserRegisterTests(void) {
-    printf("SMB2ParserRegisterTests\n");
     UtRegisterTest("SMB2ParserTest01", SMB2ParserTest01, 1);
 }
 #endif

src/app-layer-smtp.c

@@ -865,6 +865,9 @@ void RegisterSMTPParsers(void)
 
     SMTPSetMpmState();
 
+#ifdef UNITTESTS
+    AppLayerRegisterUnittests(ALPROTO_SMTP, SMTPParserRegisterTests);
+#endif
     return;
 }
 

src/app-layer-ssh.c

@@ -747,7 +747,9 @@ void RegisterSSHParsers(void)
                             SSHParseClientRecord);
 
     AppLayerRegisterStateFuncs(ALPROTO_SSH, SSHStateAlloc, SSHStateFree);
-
+#ifdef UNITTESTS
+    AppLayerRegisterUnittests(ALPROTO_SSH, SSHParserRegisterTests);
+#endif
 }
 
 /* UNITTESTS */

src/app-layer-ssl.c

@@ -989,6 +989,9 @@ void RegisterSSLParsers(void)
                                   STREAM_TOSERVER,
                                   APP_LAYER_PROBING_PARSER_PRIORITY_HIGH, 1,
                                   SSLProbingParser);
+#ifdef UNITTESTS
+    AppLayerRegisterUnittests(ALPROTO_TLS, SSLParserRegisterTests);
+#endif
 
     /* Get the value of no reassembly option from the config file */
     if (ConfGetBool("tls.no-reassemble", &ssl_config.no_reassemble) != 1)

src/detect.c

@@ -4903,22 +4903,23 @@ void SigTableSetup(void) {
     DetectLuajitRegister();
     DetectIPRepRegister();
     DetectDnsQueryRegister();
-
-    uint8_t i = 0;
-    for (i = 0; i < DETECT_TBLSIZE; i++) {
-        if (sigmatch_table[i].RegisterTests == NULL) {
-            SCLogDebug("detection plugin %s has no unittest "
-                   "registration function.", sigmatch_table[i].name);
-        }
-    }
 }
 
 void SigTableRegisterTests(void) {
     /* register the tests */
-    uint8_t i = 0;
+    int i = 0;
     for (i = 0; i < DETECT_TBLSIZE; i++) {
+        g_ut_modules++;
         if (sigmatch_table[i].RegisterTests != NULL) {
             sigmatch_table[i].RegisterTests();
+            g_ut_covered++;
+        } else {
+            SCLogDebug("detection plugin %s has no unittest "
+                   "registration function.", sigmatch_table[i].name);
+
+            if (coverage_unittests)
+                SCLogWarning(SC_WARN_NO_UNITTESTS, "detection plugin %s has no unittest "
+                        "registration function.", sigmatch_table[i].name);
         }
     }
 }

src/suricata-common.h

@@ -313,5 +313,8 @@ typedef enum PacketProfileDetectId_ {
 size_t strlcat(char *, const char *src, size_t siz);
 size_t strlcpy(char *dst, const char *src, size_t siz);
 
+extern int coverage_unittests;
+extern int g_ut_modules;
+extern int g_ut_covered;
 #endif /* __SURICATA_COMMON_H__ */
 

src/suricata.c

@@ -514,6 +514,7 @@ void usage(const char *progname)
     printf("\t-U, --unittest-filter=REGEX          : filter unittests with a regex\n");
     printf("\t--list-unittests                     : list unit tests\n");
     printf("\t--fatal-unittests                    : enable fatal failure on unittest error\n");
+    printf("\t--unittests-coverage                 : display unittest coverage report\n");
 #endif /* UNITTESTS */
     printf("\t--list-app-layer-protos              : list supported app layer protocols\n");
     printf("\t--list-keywords[=all|csv|<kword>]    : list keywords implemented by the engine\n");
@@ -703,6 +704,10 @@ void SCPrintBuildInfo(void) {
 #include "build-info.h"
 }
 
+int coverage_unittests;
+int g_ut_modules;
+int g_ut_covered;
+
 int main(int argc, char **argv)
 {
     int opt;
@@ -789,6 +794,12 @@ int main(int argc, char **argv)
     /* Initialize the configuration module. */
     ConfInit();
 
+#ifdef UNITTESTS
+    coverage_unittests = 0;
+    g_ut_modules = 0;
+    g_ut_covered = 0;
+#endif
+
     struct option long_opts[] = {
         {"dump-config", 0, &dump_config, 1},
         {"pfring", optional_argument, 0, 0},
@@ -817,6 +828,7 @@ int main(int argc, char **argv)
         {"pidfile", required_argument, 0, 0},
         {"init-errors-fatal", 0, 0, 0},
         {"fatal-unittests", 0, 0, 0},
+        {"unittests-coverage", 0, &coverage_unittests, 1},
         {"user", required_argument, 0, 0},
         {"group", required_argument, 0, 0},
         {"erf-in", required_argument, 0, 0},
@@ -1615,7 +1627,7 @@ int main(int argc, char **argv)
         SCLogInfo("Memory used at startup: %"PRIdMAX, (intmax_t)global_mem);
 #endif
         /* test and initialize the unittesting subsystem */
-        if(regex_arg == NULL){
+        if(regex_arg == NULL && !coverage_unittests){
             regex_arg = ".*";
             UtRunSelftest(regex_arg); /* inits and cleans up again */
         }
@@ -1638,15 +1650,9 @@ int main(int argc, char **argv)
         FlowBitRegisterTests();
         FlowAlertSidRegisterTests();
         SCPerfRegisterTests();
+
         DecodePPPRegisterTests();
         DecodeVLANRegisterTests();
-        HTPParserRegisterTests();
-        SSLParserRegisterTests();
-        SSHParserRegisterTests();
-        SMBParserRegisterTests();
-        DCERPCParserRegisterTests();
-        DCERPCUDPParserRegisterTests();
-        FTPParserRegisterTests();
         DecodeRawRegisterTests();
         DecodePPPOERegisterTests();
         DecodeICMPV4RegisterTests();
@@ -1657,6 +1663,7 @@ int main(int argc, char **argv)
         DecodeUDPV4RegisterTests();
         DecodeGRERegisterTests();
         DecodeAsn1RegisterTests();
+
         AlpDetectRegisterTests();
         ConfRegisterTests();
         ConfYamlRegisterTests();
@@ -1701,7 +1708,6 @@ int main(int argc, char **argv)
         DetectEngineHttpHRHRegisterTests();
         DetectEngineRegisterTests();
         SCLogRegisterTests();
-        SMTPParserRegisterTests();
         MagicRegisterTests();
         UtilMiscRegisterTests();
         DetectAddressTests();
@@ -1713,8 +1719,11 @@ int main(int argc, char **argv)
 #endif
         if (list_unittests) {
             UtListTests(regex_arg);
-        }
+        } else if (coverage_unittests) {
-        else {
+            /* nothing */
+            SCLogInfo("%d out of %d code modules have unittests (%.0f%%)",
+                    g_ut_covered, g_ut_modules, (float)((float)g_ut_covered/(float)g_ut_modules)*100);
+        } else {
             uint32_t failed = UtRunTests(regex_arg);
             UtCleanup();
             if (failed) {

src/tm-modules.c

@@ -202,11 +202,16 @@ void TmModuleRegisterTests(void) {
         if (t->name == NULL)
             continue;
 
+        g_ut_modules++;
+
+
         if (t->RegisterTests == NULL) {
-            SCLogDebug("threading module %s has no unittest "
+            if (coverage_unittests)
+                SCLogWarning(SC_WARN_NO_UNITTESTS, "threading module %s has no unittest "
                         "registration function.", t->name);
         } else {
             t->RegisterTests();
+            g_ut_covered++;
         }
     }
 #endif /* UNITTESTS */

src/util-error.c

@@ -274,6 +274,7 @@ const char * SCErrorToString(SCError err)
         CASE_CODE (SC_ERR_CUDA_BUFFER_ERROR);
         CASE_CODE (SC_ERR_DNS_LOG_GENERIC);
         CASE_CODE (SC_WARN_OPTION_OBSOLETE);
+        CASE_CODE (SC_WARN_NO_UNITTESTS);
     }
 
     return "UNKNOWN_ERROR";

src/util-error.h

@@ -263,6 +263,7 @@ typedef enum {
     SC_ERR_CUDA_BUFFER_ERROR,
     SC_ERR_DNS_LOG_GENERIC,
     SC_WARN_OPTION_OBSOLETE,
+    SC_WARN_NO_UNITTESTS,
 } SCError;
 
 const char *SCErrorToString(SCError);

src/util-mpm.c

@@ -663,10 +663,15 @@ void MpmRegisterTests(void) {
         if (i == MPM_NOTSET)
             continue;
 
+        g_ut_modules++;
+
         if (mpm_table[i].RegisterUnittests != NULL) {
+            g_ut_covered++;
             mpm_table[i].RegisterUnittests();
         } else {
-            printf("Warning: mpm %s has no unittest registration function...", mpm_table[i].name);
+            if (coverage_unittests)
+                SCLogWarning(SC_WARN_NO_UNITTESTS, "mpm module %s has no "
+                        "unittest registration function.", mpm_table[i].name);
         }
     }