app/ftp: Use Rust FTP response line handling

Use the Rust logic to parse FTP response lines with the goal to support
multi-buffer matches better.

A side effect is that the completion codes are no longer strings; the
schema update reflects this.

Issue: 4082

src/app-layer-ftp.c

@@ -20,7 +20,7 @@
  *
  * \author Pablo Rincon Crespo <pablo.rincon.crespo@gmail.com>
  * \author Eric Leblond <eric@regit.org>
- * \author Jeff Lucovsky <jeff@lucovsky.org>
+ * \author Jeff Lucovsky <jlucovsky@oisf.net>
  *
  * App Layer Parser for FTP
  */
@@ -130,14 +130,13 @@ static void *FTPCalloc(size_t n, size_t size)
 
 static void *FTPRealloc(void *ptr, size_t orig_size, size_t size)
 {
-    void *rptr = NULL;
+    DEBUG_VALIDATE_BUG_ON(size == 0);
-
     if (FTPCheckMemcap((uint32_t)(size - orig_size)) == 0) {
         sc_errno = SC_ELIMIT;
         return NULL;
     }
 
-    rptr = SCRealloc(ptr, size);
+    void *rptr = SCRealloc(ptr, size);
     if (rptr == NULL) {
         sc_errno = SC_ENOMEM;
         return NULL;
@@ -159,18 +158,24 @@ static void FTPFree(void *ptr, size_t size)
     FTPDecrMemuse((uint64_t)size);
 }
 
-static FTPString *FTPStringAlloc(void)
+static FTPResponseWrapper *FTPResponseWrapperAlloc(FTPResponseLine *response)
 {
-    return FTPCalloc(1, sizeof(FTPString));
+    FTPResponseWrapper *wrapper = FTPCalloc(1, sizeof(FTPResponseWrapper));
+    if (likely(wrapper)) {
+        FTPIncrMemuse(response->total_size);
+        wrapper->response = response;
+    }
+    return wrapper;
 }
 
-static void FTPStringFree(FTPString *str)
+static void FTPResponseWrapperFree(FTPResponseWrapper *wrapper)
 {
-    if (str->str) {
+    if (wrapper->response) {
-        FTPFree(str->str, str->len);
+        FTPDecrMemuse(wrapper->response->total_size);
+        SCFTPFreeResponseLine(wrapper->response);
     }
 
-    FTPFree(str, sizeof(FTPString));
+    FTPFree(wrapper, sizeof(FTPResponseWrapper));
 }
 
 static void *FTPLocalStorageAlloc(void)
@@ -244,10 +249,10 @@ static void FTPTransactionFree(FTPTransaction *tx)
         FTPFree(tx->request, tx->request_length);
     }
 
-    FTPString *str = NULL;
+    FTPResponseWrapper *wrapper;
-    while ((str = TAILQ_FIRST(&tx->response_list))) {
+    while ((wrapper = TAILQ_FIRST(&tx->response_list))) {
-        TAILQ_REMOVE(&tx->response_list, str, next);
+        TAILQ_REMOVE(&tx->response_list, wrapper, next);
-        FTPStringFree(str);
+        FTPResponseWrapperFree(wrapper);
     }
 
     FTPFree(tx, sizeof(*tx));
@@ -693,18 +698,24 @@ static AppLayerResult FTPParseResponse(Flow *f, void *ftp_state, AppLayerParserS
         }
 
         if (likely(line.len)) {
-            FTPString *response = FTPStringAlloc();
+            FTPResponseLine *response = SCFTPParseResponseLine((const char *)line.buf, line.len);
             if (likely(response)) {
-                response->len = CopyCommandLine(&response->str, &line);
+                FTPResponseWrapper *wrapper = FTPResponseWrapperAlloc(response);
-                response->truncated = state->current_line_truncated_tc;
+                if (likely(wrapper)) {
-                if (response->truncated) {
+                    response->truncated = state->current_line_truncated_tc;
-                    AppLayerDecoderEventsSetEventRaw(
+                    if (response->truncated) {
-                            &tx->tx_data.events, FtpEventResponseCommandTooLong);
+                        AppLayerDecoderEventsSetEventRaw(
-                }
+                                &tx->tx_data.events, FtpEventResponseCommandTooLong);
-                if (line.lf_found) {
+                    }
-                    state->current_line_truncated_tc = false;
+                    if (line.lf_found) {
+                        state->current_line_truncated_tc = false;
+                    }
+                    TAILQ_INSERT_TAIL(&tx->response_list, wrapper, next);
+                } else {
+                    SCFTPFreeResponseLine(response);
                 }
-                TAILQ_INSERT_TAIL(&tx->response_list, response, next);
+            } else {
+                SCLogDebug("unable to parse FTP response line \"%s\"", line.buf);
             }
         }
 

src/app-layer-ftp.h

@@ -41,12 +41,10 @@ typedef struct FtpLineState_ {
     bool lf_found;
 } FtpLineState;
 
-typedef struct FTPString_ {
+typedef struct FTPResponseWrapper_ {
-    uint8_t *str;
+    FTPResponseLine *response;
-    uint32_t len;
+    TAILQ_ENTRY(FTPResponseWrapper_) next;
-    bool truncated;
+} FTPResponseWrapper;
-    TAILQ_ENTRY(FTPString_) next;
-} FTPString;
 
 /*
  * These are the values for the table index value and the FTP command
@@ -80,7 +78,7 @@ typedef struct FTPTransaction_  {
     uint8_t direction;
 
     /* Handle multiple responses */
-    TAILQ_HEAD(, FTPString_) response_list;
+    TAILQ_HEAD(, FTPResponseWrapper_) response_list;
 
     TAILQ_ENTRY(FTPTransaction_) next;
 } FTPTransaction;

src/output-json-ftp.c

@@ -87,44 +87,30 @@ bool EveFTPLogCommand(void *vtx, SCJsonBuilder *jb)
 
     if (!TAILQ_EMPTY(&tx->response_list)) {
         int resp_cnt = 0;
-        FTPString *response;
+        FTPResponseWrapper *wrapper;
         bool is_cc_array_open = false;
-        TAILQ_FOREACH(response, &tx->response_list, next) {
+        TAILQ_FOREACH (wrapper, &tx->response_list, next) {
             /* handle multiple lines within the response, \r\n delimited */
-            uint8_t *where = response->str;
+            if (!wrapper->response) {
-            uint16_t length = 0;
+                continue;
-            uint16_t pos;
-            if (response->len > 0 && response->len <= UINT16_MAX) {
-                length = (uint16_t)response->len - 1;
-            } else if (response->len > UINT16_MAX) {
-                length = UINT16_MAX;
             }
+            FTPResponseLine *response = wrapper->response;
+
             if (!reply_truncated && response->truncated) {
                 reply_truncated = true;
             }
-            while ((pos = JsonGetNextLineFromBuffer((const char *)where, length)) != UINT16_MAX) {
+            int code_len = strlen((const char *)response->code);
-                uint16_t offset = 0;
+            if (code_len > 0) {
-                /* Try to find a completion code for this line */
+                if (!is_cc_array_open) {
-                if (pos >= 3)  {
+                    SCJbOpenArray(jb, "completion_code");
-                    /* Gather the completion code if present */
+                    is_cc_array_open = true;
-                    if (isdigit(where[0]) && isdigit(where[1]) && isdigit(where[2])) {
-                        if (!is_cc_array_open) {
-                            SCJbOpenArray(jb, "completion_code");
-                            is_cc_array_open = true;
-                        }
-                        SCJbAppendStringFromBytes(jb, (const uint8_t *)where, 3);
-                        offset = 4;
-                    }
-                }
-                /* move past 3 character completion code */
-                if (pos >= offset) {
-                    SCJbAppendStringFromBytes(
-                            js_resplist, (const uint8_t *)where + offset, pos - offset);
-                    resp_cnt++;
                 }
-
+                SCJbAppendStringFromBytes(jb, (const uint8_t *)response->code, code_len);
-                where += pos;
+            }
-                length -= pos;
+            if (response->length) {
+                SCJbAppendStringFromBytes(
+                        js_resplist, (const uint8_t *)response->response, response->length);
+                resp_cnt++;
             }
         }