dcerpc: return error on invalid header

DCERPC/TCP tends to return the same values for invalid and incomplete
headers. As a result of this, invalid headers and any traffic following
it is buffered and processed later on assumed to be valid DCERPC traffic.
Fix this by clearly defining error and incomplete data and taking
appropriate actions.

Bug 7230

rust/src/dcerpc/dcerpc.rs

@@ -1,4 +1,4 @@
-/* Copyright (C) 2020-2022 Open Information Security Foundation
+/* Copyright (C) 2020-2024 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -612,6 +612,7 @@ impl DCERPCState {
     /// * Success: Number of bytes successfully parsed.
     /// * Failure: -1 in case of Incomplete data or Eof.
     ///            -2 in case of Error while parsing.
+    ///            -3 in case of invalid DCERPC header.
     pub fn process_header(&mut self, input: &[u8]) -> i32 {
         match parser::parse_dcerpc_header(input) {
             Ok((leftover_bytes, header)) => {
@@ -623,7 +624,7 @@ impl DCERPCState {
                         header.rpc_vers,
                         header.rpc_vers_minor
                     );
-                    return -1;
+                    return -3;
                 }
                 self.header = Some(header);
                 (input.len() - leftover_bytes.len()) as i32
@@ -986,7 +987,7 @@ impl DCERPCState {
                 self.extend_buffer(buffer, direction);
                 return AppLayerResult::ok();
             }
-            if parsed == -2 {
+            if parsed < 0 {
                 return AppLayerResult::err();
             }
             self.bytes_consumed += parsed;