app-layer: protocol change API

Add API calls to upgrade to TLS or to request a protocol change
without a specific protocol expectation.

If the HTTP CONNECT session includes a port on the url, use that to
look up the probing parser during protocol detection. Solves a
missed detection of a SSLv2 session that upgrades to TLSv1. SSLv2
relies on the probing parser which is limited to certain ports.

In case of STARTTLS in SMTP and FTP, the port is hardcoded to 443.

A new event APPLAYER_UNEXPECTED_PROTOCOL is set if there was a
mismatch.

rules/app-layer-events.rules

@@ -10,6 +10,9 @@ alert ip any any -> any any (msg:"SURICATA Applayer Mismatch protocol both direc
 alert ip any any -> any any (msg:"SURICATA Applayer Wrong direction first Data"; flow:established; app-layer-event:applayer_wrong_direction_first_data; flowint:applayer.anomaly.count,+,1; classtype:protocol-command-decode; sid:2260001; rev:1;)
 alert ip any any -> any any (msg:"SURICATA Applayer Detect protocol only one direction"; flow:established; app-layer-event:applayer_detect_protocol_only_one_direction; flowint:applayer.anomaly.count,+,1; classtype:protocol-command-decode; sid:2260002; rev:1;)
 alert ip any any -> any any (msg:"SURICATA Applayer Protocol detection skipped"; flow:established; app-layer-event:applayer_proto_detection_skipped; flowint:applayer.anomaly.count,+,1; classtype:protocol-command-decode; sid:2260003; rev:1;)
-alert ip any any -> any any (msg:"SURICATA Applayer No TLS after STARTTLS"; flow:established; app-layer-event:applayer_no_tls_after_starttls; flowint:applayer.anomaly.count,+,1; classtype:protocol-command-decode; sid:226004; rev:1;)
+# alert if STARTTLS was not followed by actual SSL/TLS
+alert tcp any any -> any any (msg:"SURICATA Applayer No TLS after STARTTLS"; flow:established; app-layer-event:applayer_no_tls_after_starttls; flowint:applayer.anomaly.count,+,1; classtype:protocol-command-decode; sid:2260004; rev:2;)
+# unexpected protocol in protocol upgrade
+alert tcp any any -> any any (msg:"SURICATA Applayer Unexpected protocol"; flow:established; app-layer-event:applayer_unexpected_protocol; flowint:applayer.anomaly.count,+,1; classtype:protocol-command-decode; sid:2260005; rev:1;)
 
-#next sid is 2260005
+#next sid is 2260006

src/app-layer-detect-proto.c

@@ -334,52 +334,51 @@ static AppProto AppLayerProtoDetectPPGetProto(Flow *f,
     uint32_t *alproto_masks;
     uint32_t mask = 0;
 
+    const uint16_t dp = f->protodetect_dp ? f->protodetect_dp : f->dp;
+    const uint16_t sp = f->sp;
+
     if (direction & STREAM_TOSERVER) {
         /* first try the destination port */
-        pp_port_dp = AppLayerProtoDetectGetProbingParsers(alpd_ctx.ctx_pp, ipproto, f->dp);
+        pp_port_dp = AppLayerProtoDetectGetProbingParsers(alpd_ctx.ctx_pp, ipproto, dp);
         alproto_masks = &f->probing_parser_toserver_alproto_masks;
         if (pp_port_dp != NULL) {
-            SCLogDebug("toserver - Probing parser found for destination port %"PRIu16, f->dp);
+            SCLogDebug("toserver - Probing parser found for destination port %"PRIu16, dp);
 
             /* found based on destination port, so use dp registration */
             pe1 = pp_port_dp->dp;
         } else {
-            SCLogDebug("toserver - No probing parser registered for dest port %"PRIu16,
-                       f->dp);
+            SCLogDebug("toserver - No probing parser registered for dest port %"PRIu16, dp);
         }
 
-        pp_port_sp = AppLayerProtoDetectGetProbingParsers(alpd_ctx.ctx_pp, ipproto, f->sp);
+        pp_port_sp = AppLayerProtoDetectGetProbingParsers(alpd_ctx.ctx_pp, ipproto, sp);
         if (pp_port_sp != NULL) {
-            SCLogDebug("toserver - Probing parser found for source port %"PRIu16, f->sp);
+            SCLogDebug("toserver - Probing parser found for source port %"PRIu16, sp);
 
             /* found based on source port, so use sp registration */
             pe2 = pp_port_sp->sp;
         } else {
-            SCLogDebug("toserver - No probing parser registered for source port %"PRIu16,
-                    f->sp);
+            SCLogDebug("toserver - No probing parser registered for source port %"PRIu16, sp);
         }
     } else {
         /* first try the destination port */
-        pp_port_dp = AppLayerProtoDetectGetProbingParsers(alpd_ctx.ctx_pp, ipproto, f->dp);
+        pp_port_dp = AppLayerProtoDetectGetProbingParsers(alpd_ctx.ctx_pp, ipproto, dp);
         alproto_masks = &f->probing_parser_toclient_alproto_masks;
         if (pp_port_dp != NULL) {
-            SCLogDebug("toclient - Probing parser found for destination port %"PRIu16, f->dp);
+            SCLogDebug("toclient - Probing parser found for destination port %"PRIu16, dp);
 
             /* found based on destination port, so use dp registration */
             pe1 = pp_port_dp->dp;
         } else {
-            SCLogDebug("toclient - No probing parser registered for dest port %"PRIu16,
-                       f->dp);
+            SCLogDebug("toclient - No probing parser registered for dest port %"PRIu16, dp);
         }
 
-        pp_port_sp = AppLayerProtoDetectGetProbingParsers(alpd_ctx.ctx_pp, ipproto, f->sp);
+        pp_port_sp = AppLayerProtoDetectGetProbingParsers(alpd_ctx.ctx_pp, ipproto, sp);
         if (pp_port_sp != NULL) {
-            SCLogDebug("toclient - Probing parser found for source port %"PRIu16, f->sp);
+            SCLogDebug("toclient - Probing parser found for source port %"PRIu16, sp);
 
             pe2 = pp_port_sp->sp;
         } else {
-            SCLogDebug("toclient - No probing parser registered for source port %"PRIu16,
-                        f->sp);
+            SCLogDebug("toclient - No probing parser registered for source port %"PRIu16, sp);
         }
     }
 
@@ -1614,6 +1613,40 @@ void AppLayerProtoDetectRegisterProtocol(AppProto alproto, const char *alproto_n
     SCReturn;
 }
 
+/** \brief request applayer to wrap up this protocol and rerun protocol
+ *         detection.
+ *
+ *  When this is called, the old session is reset unconditionally. A
+ *  'detect/log' flush packet is generated for both direction before
+ *  the reset, so allow for final detection and logging.
+ *
+ *  \param f flow to act on
+ *  \param dp destination port to use in protocol detection. Set to 443
+ *            for start tls, set to the HTTP uri port for CONNECT and
+ *            set to 0 to not use it.
+ *  \param expect_proto expected protocol. AppLayer event will be set if
+ *                      detected protocol differs from this.
+ */
+void AppLayerRequestProtocolChange(Flow *f, uint16_t dp, AppProto expect_proto)
+{
+    FlowSetChangeProtoFlag(f);
+    f->protodetect_dp = dp;
+    f->alproto_expect = expect_proto;
+}
+
+/** \brief request applayer to wrap up this protocol and rerun protocol
+ *         detection with expectation of TLS. Used by STARTTLS.
+ *
+ *  Sets detection port to 443 to make port based TLS detection work for
+ *  SMTP, FTP etc as well.
+ *
+ *  \param f flow to act on
+ */
+void AppLayerRequestProtocolTLSUpgrade(Flow *f)
+{
+    AppLayerRequestProtocolChange(f, 443, ALPROTO_TLS);
+}
+
 void AppLayerProtoDetectReset(Flow *f)
 {
     FlowUnsetChangeProtoFlag(f);

src/app-layer-detect-proto.h

@@ -111,6 +111,9 @@ int AppLayerProtoDetectSetup(void);
  */
 void AppLayerProtoDetectReset(Flow *);
 
+void AppLayerRequestProtocolChange(Flow *f, uint16_t dp, AppProto expect_proto);
+void AppLayerRequestProtocolTLSUpgrade(Flow *f);
+
 /**
  * \brief Cleans up the app layer protocol detection phase.
  */

src/app-layer-events.c

@@ -42,6 +42,8 @@ SCEnumCharMap app_layer_event_pkt_table[ ] = {
       APPLAYER_PROTO_DETECTION_SKIPPED },
     { "APPLAYER_NO_TLS_AFTER_STARTTLS",
       APPLAYER_NO_TLS_AFTER_STARTTLS },
+    { "APPLAYER_UNEXPECTED_PROTOCOL",
+      APPLAYER_UNEXPECTED_PROTOCOL },
     { NULL,
       -1 },
 };

src/app-layer-events.h

@@ -47,6 +47,7 @@ enum {
     APPLAYER_DETECT_PROTOCOL_ONLY_ONE_DIRECTION,
     APPLAYER_PROTO_DETECTION_SKIPPED,
     APPLAYER_NO_TLS_AFTER_STARTTLS,
+    APPLAYER_UNEXPECTED_PROTOCOL,
 };
 
 /* the event types for app events */

src/app-layer-ftp.c

@@ -270,7 +270,7 @@ static int FTPParseResponse(Flow *f, void *ftp_state, AppLayerParserState *pstat
 
     if (state->command == FTP_COMMAND_AUTH_TLS) {
         if (input_len >= 4 && SCMemcmp("234 ", input, 4) == 0) {
-            FlowSetChangeProtoFlag(f);
+            AppLayerRequestProtocolTLSUpgrade(f);
         }
     }
 

src/app-layer-htp.c

@@ -2019,7 +2019,12 @@ static int HTPCallbackResponse(htp_tx_t *tx)
         if ((tx->response_status_number >= 200) &&
                 (tx->response_status_number < 300) &&
                 (hstate->transaction_cnt == 1)) {
-            FlowSetChangeProtoFlag(hstate->f);
+            uint16_t dp = 0;
+            if (tx->request_port_number != -1) {
+                dp = (uint16_t)tx->request_port_number;
+            }
+            // both ALPROTO_HTTP and ALPROTO_TLS are normal options
+            AppLayerRequestProtocolChange(hstate->f, dp, ALPROTO_UNKNOWN);
             tx->request_progress = HTP_REQUEST_COMPLETE;
             tx->response_progress = HTP_RESPONSE_COMPLETE;
         }

src/app-layer-smtp.c

@@ -954,7 +954,7 @@ static int SMTPProcessReply(SMTPState *state, Flow *f,
         if (reply_code == SMTP_REPLY_220) {
             /* we are entering STARRTTLS data mode */
             state->parser_state |= SMTP_PARSER_STATE_COMMAND_DATA_MODE;
-            FlowSetChangeProtoFlag(f);
+            AppLayerRequestProtocolTLSUpgrade(f);
             state->curr_tx->done = 1;
         } else {
             /* decoder event */

src/app-layer.c

@@ -574,9 +574,18 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
         }
         SCLogDebug("protocol change, old %s, new %s",
                 AppProtoToString(f->alproto_orig), AppProtoToString(f->alproto));
-        if (f->alproto != ALPROTO_TLS) {
+
+        if (f->alproto_expect != ALPROTO_UNKNOWN &&
+                f->alproto != f->alproto_expect)
+        {
             AppLayerDecoderEventsSetEventRaw(&p->app_layer_events,
-                                             APPLAYER_NO_TLS_AFTER_STARTTLS);
+                                             APPLAYER_UNEXPECTED_PROTOCOL);
+
+            if (f->alproto_expect == ALPROTO_TLS && f->alproto != ALPROTO_TLS) {
+                AppLayerDecoderEventsSetEventRaw(&p->app_layer_events,
+                        APPLAYER_NO_TLS_AFTER_STARTTLS);
+
+            }
         }
     } else {
         SCLogDebug("stream data (len %" PRIu32 " alproto "

src/flow-util.h

@@ -47,6 +47,7 @@
         (f)->probing_parser_toclient_alproto_masks = 0; \
         (f)->flags = 0; \
         (f)->file_flags = 0; \
+        (f)->protodetect_dp = 0; \
         (f)->lastts.tv_sec = 0; \
         (f)->lastts.tv_usec = 0; \
         FLOWLOCK_INIT((f)); \
@@ -55,6 +56,8 @@
         (f)->alproto = 0; \
         (f)->alproto_ts = 0; \
         (f)->alproto_tc = 0; \
+        (f)->alproto_orig = 0; \
+        (f)->alproto_expect = 0; \
         (f)->de_ctx_version = 0; \
         (f)->thread_id = 0; \
         (f)->alparser = NULL; \
@@ -86,6 +89,7 @@
         (f)->probing_parser_toclient_alproto_masks = 0; \
         (f)->flags = 0; \
         (f)->file_flags = 0; \
+        (f)->protodetect_dp = 0; \
         (f)->lastts.tv_sec = 0; \
         (f)->lastts.tv_usec = 0; \
         (f)->protoctx = NULL; \
@@ -95,6 +99,8 @@
         (f)->alproto = 0; \
         (f)->alproto_ts = 0; \
         (f)->alproto_tc = 0; \
+        (f)->alproto_orig = 0; \
+        (f)->alproto_expect = 0; \
         (f)->de_ctx_version = 0; \
         (f)->thread_id = 0; \
         (f)->sgh_toserver = NULL; \

src/flow.h

@@ -364,6 +364,10 @@ typedef struct Flow_
     uint16_t file_flags;    /**< file tracking/extraction flags */
     /* coccinelle: Flow:file_flags:FLOWFILE_ */
 
+    /** destination port to be used in protocol detection. This is meant
+     *  for use with STARTTLS and HTTP CONNECT detection */
+    uint16_t protodetect_dp; /**< 0 if not used */
+
 #ifdef FLOWLOCK_RWLOCK
     SCRWLock r;
 #elif defined FLOWLOCK_MUTEX
@@ -389,6 +393,9 @@ typedef struct Flow_
     /** original application level protocol. Used to indicate the previous
        protocol when changing to another protocol , e.g. with STARTTLS. */
     AppProto alproto_orig;
+    /** expected app protocol: used in protocol change/upgrade like in
+     *  STARTTLS. */
+    AppProto alproto_expect;
 
     /** detection engine ctx version used to inspect this flow. Set at initial
      *  inspection. If it doesn't match the currently in use de_ctx, the