tls/serial: use byte array instead of string

Bug 7887

(cherry picked from commit 24f5b7dab2)

rust/src/x509/mod.rs

@@ -144,15 +144,18 @@ pub unsafe extern "C" fn SCX509GetIssuer(ptr: *const X509, issuer_name: *mut *mu
 }
 
 #[no_mangle]
-pub unsafe extern "C" fn SCX509GetSerial(ptr: *const X509) -> *mut c_char {
+pub unsafe extern "C" fn SCX509GetSerial(ptr: *const X509, serial_num: *mut *mut u8, serial_len: *mut u32) {
     if ptr.is_null() {
-        return std::ptr::null_mut();
+        *serial_len = 0;
+        *serial_num = std::ptr::null_mut();
+        return;
     }
     let x509 = cast_pointer! {ptr, X509};
     let raw_serial = x509.0.tbs_certificate.raw_serial();
     let v: Vec<_> = raw_serial.iter().map(|x| format!("{:02X}", x)).collect();
     let serial = v.join(":");
-    rust_string_to_c(serial)
+    *serial_len = serial.len() as u32;
+    *serial_num = Box::into_raw(serial.into_bytes().into_boxed_slice()) as *mut u8;
 }
 
 /// Extract validity from input X.509 object

src/app-layer-ssl.c

@@ -511,12 +511,12 @@ static int TlsDecodeHSCertificate(SSLState *ssl_state, SSLStateConnp *connp,
             sans[i] = SCX509GetSubjectAltNameAt(x509, i);
         }
         connp->cert0_sans = sans;
-        char *str = SCX509GetSerial(x509);
-        if (str == NULL) {
+
+        SCX509GetSerial(x509, &connp->cert0_serial, &connp->cert0_serial_len);
+        if (connp->cert0_serial == NULL) {
             err_code = ERR_INVALID_SERIAL;
             goto error;
         }
-        connp->cert0_serial = str;
 
         rc = SCX509GetValidity(x509, &connp->cert0_not_before, &connp->cert0_not_after);
         if (rc != 0) {
@@ -2859,7 +2859,8 @@ static void SSLStateFree(void *p)
         SCX509ArrayFree(
                 ssl_state->client_connp.cert0_issuerdn, ssl_state->client_connp.cert0_issuerdn_len);
     if (ssl_state->client_connp.cert0_serial)
-        SCRustCStringFree(ssl_state->client_connp.cert0_serial);
+        SCX509ArrayFree(
+                ssl_state->client_connp.cert0_serial, ssl_state->client_connp.cert0_serial_len);
     if (ssl_state->client_connp.cert0_fingerprint)
         SCFree(ssl_state->client_connp.cert0_fingerprint);
     if (ssl_state->client_connp.sni)
@@ -2876,7 +2877,8 @@ static void SSLStateFree(void *p)
         SCX509ArrayFree(
                 ssl_state->server_connp.cert0_issuerdn, ssl_state->server_connp.cert0_issuerdn_len);
     if (ssl_state->server_connp.cert0_serial)
-        SCRustCStringFree(ssl_state->server_connp.cert0_serial);
+        SCX509ArrayFree(
+                ssl_state->server_connp.cert0_serial, ssl_state->server_connp.cert0_serial_len);
     if (ssl_state->server_connp.cert0_fingerprint)
         SCFree(ssl_state->server_connp.cert0_fingerprint);
     if (ssl_state->server_connp.sni)

src/app-layer-ssl.h

@@ -184,11 +184,12 @@ typedef struct SSLStateConnp_ {
     uint16_t session_id_length;
 
     uint8_t random[TLS_RANDOM_LEN];
-    char *cert0_serial;
     uint8_t *cert0_subject;
     uint32_t cert0_subject_len;
     uint8_t *cert0_issuerdn;
     uint32_t cert0_issuerdn_len;
+    uint8_t *cert0_serial;
+    uint32_t cert0_serial_len;
     int64_t cert0_not_before;
     int64_t cert0_not_after;
     char *cert0_fingerprint;

src/detect-tls-cert-serial.c

@@ -148,8 +148,8 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
             return NULL;
         }
 
-        const uint32_t data_len = (uint32_t)strlen(connp->cert0_serial);
-        const uint8_t *data = (uint8_t *)connp->cert0_serial;
+        const uint32_t data_len = connp->cert0_serial_len;
+        const uint8_t *data = connp->cert0_serial;
 
         InspectionBufferSetupAndApplyTransforms(
                 det_ctx, list_id, buffer, data, data_len, transforms);

src/log-tlslog.c

@@ -339,9 +339,12 @@ static void LogTlsLogExtended(LogTlsLogThread *aft, SSLState *ssl_state, const S
         LogTlsLogString(aft->buffer, "SNI", ssl_state->client_connp.sni);
     }
     if (ssl_state->server_connp.cert0_serial != NULL) {
+        char *serial = CreateStringFromByteArray(
+                ssl_state->client_connp.cert0_serial, ssl_state->client_connp.cert0_serial_len);
         LOG_CF_WRITE_SPACE_SEPARATOR(aft->buffer);
-        LogTlsLogString(aft->buffer, "SERIAL",
-                        ssl_state->server_connp.cert0_serial);
+        LogTlsLogString(aft->buffer, "SERIAL", serial ? serial : "<ERROR>");
+
+        SCFree(serial);
     }
 
     LOG_CF_WRITE_SPACE_SEPARATOR(aft->buffer);

src/output-json-tls.c

@@ -195,7 +195,8 @@ static void JsonTlsLogSni(SCJsonBuilder *js, SSLState *ssl_state)
 static void JsonTlsLogSerial(SCJsonBuilder *js, SSLState *ssl_state)
 {
     if (ssl_state->server_connp.cert0_serial) {
-        SCJbSetString(js, "serial", ssl_state->server_connp.cert0_serial);
+        SCJbSetStringFromBytes(js, "serial", ssl_state->server_connp.cert0_serial,
+                ssl_state->server_connp.cert0_serial_len);
     }
 }
 
@@ -366,7 +367,7 @@ static void JsonTlsLogClientCert(
         SCJbSetString(js, "fingerprint", connp->cert0_fingerprint);
     }
     if (connp->cert0_serial) {
-        SCJbSetString(js, "serial", connp->cert0_serial);
+        SCJbSetStringFromBytes(js, "serial", connp->cert0_serial, connp->cert0_serial_len);
     }
     if (connp->cert0_not_before != 0) {
         char timebuf[64];

src/util-lua-tls.c

@@ -291,8 +291,7 @@ static int GetCertSerial(lua_State *luastate, bool client)
     if (connp->cert0_serial == NULL)
         return LuaCallbackError(luastate, "error: no certificate serial");
 
-    return LuaPushStringBuffer(
-            luastate, (uint8_t *)connp->cert0_serial, strlen(connp->cert0_serial));
+    return LuaPushStringBuffer(luastate, connp->cert0_serial, connp->cert0_serial_len);
 }
 
 static int LuaTlsGetServerCertSerial(lua_State *luastate)