der: fix recursion depth not being handled correctly

In a mix of sequences the 'depth reached' error would not
be fully propagated.

Found with AFL.

src/app-layer-tls-handshake.c

@@ -58,6 +58,7 @@ static void TLSCertificateErrCodeToWarning(SSLState *ssl_state,
     switch (errcode) {
         case ERR_DER_ELEMENT_SIZE_TOO_BIG:
         case ERR_DER_INVALID_SIZE:
+        case ERR_DER_RECURSION_LIMIT:
             SSLSetEvent(ssl_state,
                     TLS_DECODER_EVENT_CERTIFICATE_INVALID_LENGTH);
             break;

src/util-decode-der.c

@@ -144,6 +144,7 @@ static Asn1Generic * DecodeAsn1DerGeneric(const unsigned char *buffer,
 
     /* refuse excessive recursion */
     if (unlikely(depth == 255)) {
+        *errcode = ERR_DER_RECURSION_LIMIT;
         return NULL;
     }
 

src/util-decode-der.h

@@ -90,6 +90,8 @@ typedef struct Asn1Generic_ {
 #define ERR_DER_UNSUPPORTED_STRING    0x05
 /* Missing field or element */
 #define ERR_DER_MISSING_ELEMENT       0x06
+/* Generic error */
+#define ERR_DER_RECURSION_LIMIT       0x07
 
 Asn1Generic * DecodeDer(const unsigned char *buffer, uint32_t size, uint32_t *errcode) __attribute__((nonnull));
 void DerFree(Asn1Generic *a);