ssl: first pass limit when allocating buffer for certificates

With this check, on the first packet of a certificate presenting
a length of 16Mbytes, we only allocate up to 65Kb

When we get to the point where need more than 65Kb, we realloc
to the true size.

With this check, it makes it more expensive for an attacket to use
this allocation as a way to trigger ressource exhaustion...

(cherry picked from commit 862e84877f)

src/app-layer-ssl.c

@@ -1412,6 +1412,12 @@ static uint32_t GetCertsLen(SSLStateConnp *curr_connp, const uint8_t *input,
     }
 }
 
+// For certificates whose size is bigger than this,
+// we do not allocate all the required memory straight away,
+// to avoid DOS by RAM exhaustion, but we will allocate
+// this memory once a consequent part of the certificate has been seen.
+#define SSL_CERT_MAX_FIRST_ALLOC 65536 // 0x10000
+
 /** \internal
  *  \brief setup or grow the `trec` space in the connp
  */
@@ -1425,6 +1431,10 @@ static int EnsureRecordSpace(SSLStateConnp *curr_connp, const uint8_t * const in
         SCLogDebug("cert_len unknown still, create small buffer to start");
         certs_len = 256;
     }
+    // Limit in a first time allocation for very large certificates
+    if (certs_len > SSL_CERT_MAX_FIRST_ALLOC && certs_len > curr_connp->trec_pos + input_len) {
+        certs_len = SSL_CERT_MAX_FIRST_ALLOC;
+    }
 
     if (curr_connp->trec == NULL) {
         curr_connp->trec_len = certs_len;