dcerpc: maximum number of live transactions also for UDP

Ticket: #6129

Avoids that quadratic complexity gets too bad

rust/src/dcerpc/dcerpc.rs

@@ -111,7 +111,7 @@ pub const DCERPC_TYPE_ORPHANED: u8 = 19;
 pub const DCERPC_TYPE_RTS: u8 = 20;
 pub const DCERPC_TYPE_UNKNOWN: u8 = 99;
 
-static mut DCERPC_MAX_TX: usize = 1024;
+pub(super) static mut DCERPC_MAX_TX: usize = 1024;
 
 pub static mut ALPROTO_DCERPC: AppProto = ALPROTO_UNKNOWN;
 

rust/src/dcerpc/dcerpc_udp.rs

@@ -18,7 +18,7 @@
 use crate::applayer::{self, *};
 use crate::core::{self, Direction, DIR_BOTH};
 use crate::dcerpc::dcerpc::{
-    DCERPCTransaction, DCERPC_TYPE_REQUEST, DCERPC_TYPE_RESPONSE, PFCL1_FRAG, PFCL1_LASTFRAG,
+    DCERPCTransaction, DCERPC_MAX_TX, DCERPC_TYPE_REQUEST, DCERPC_TYPE_RESPONSE, PFCL1_FRAG, PFCL1_LASTFRAG,
     rs_dcerpc_get_alstate_progress, ALPROTO_DCERPC, PARSER_NAME,
 };
 use nom7::Err;
@@ -58,6 +58,7 @@ pub struct DCERPCUDPState {
     state_data: AppLayerStateData,
     pub tx_id: u64,
     pub transactions: VecDeque<DCERPCTransaction>,
+    tx_index_completed: usize,
 }
 
 impl State<DCERPCTransaction> for DCERPCUDPState {
@@ -82,6 +83,18 @@ impl DCERPCUDPState {
         tx.activityuuid = hdr.activityuuid.to_vec();
         tx.seqnum = hdr.seqnum;
         self.tx_id += 1;
+        if self.transactions.len() > unsafe { DCERPC_MAX_TX } {
+            let mut index = self.tx_index_completed;
+            for tx_old in &mut self.transactions.range_mut(self.tx_index_completed..) {
+                index += 1;
+                if !tx_old.req_done || !tx_old.resp_done {
+                    tx_old.req_done = true;
+                    tx_old.resp_done = true;
+                    break;
+                }
+            }
+            self.tx_index_completed = index;
+        }
         tx
     }
 
@@ -102,6 +115,7 @@ impl DCERPCUDPState {
         if found {
             SCLogDebug!("freeing TX with ID {} TX.ID {} at index {} left: {} max id: {}",
                             tx_id, tx_id+1, index, self.transactions.len(), self.tx_id);
+            self.tx_index_completed = 0;
             self.transactions.remove(index);
         }
     }