detect/integers: generalize support for bitflags modifier

Ticket: 6724

Allows sugar syntax for bitflags keywords.
While the expressivity does not increase, because we could already
use numerial values with all generic integer modes, this modifier
prefix is used with the strings, and follows the syntax
that is already used for fragbits and tcp.flags keyword.

doc/userguide/rules/dnp3-keywords.rst

@@ -68,12 +68,14 @@ dnp3_ind
 This keyword matches on the DNP3 internal indicator flags in the
 response application header.
 
+dnp3_ind uses :ref:`unsigned 16-bit integer <rules-integer-keywords>` with bitmasks.
+
 Syntax
 ~~~~~~
 
 ::
 
-  dnp3_ind:<flag>{,<flag>...}
+  dnp3_ind:[*+-=]<flag>{,<flag>...}
 
 
 Where flag is the name of the internal indicator:
@@ -96,8 +98,7 @@ Where flag is the name of the internal indicator:
 * reserved_1
 
 This keyword will match of any of the flags listed are set. To match
-on multiple flags (AND type match), use dnp3_ind for each flag that
-must be set.
+on multiple flags use prefix ``+``.
 
 Examples
 ~~~~~~~~

doc/userguide/rules/integer-keywords.rst

@@ -88,6 +88,16 @@ Some integers on the wire represent multiple bits.
 Some of these bits have a string/meaning associated to it.
 Rules can be written using a list (comma-separated) of these strings,
 where each item can be negated.
+This list can be prefixed by a modifier:
+
+========  ===================================
+Modifier  Description
+========  ===================================
+``+``     match on all the bits, plus any others
+``*``     match if any of the bits are set
+``-``     match if not all the bits are set
+``=``     match on all the bits, and only them
+========  ===================================
 
 There is no right shift for trailing zeros applied here (even if there is one
 for ``byte_test`` and ``byte_math``). That means a rule with

doc/userguide/rules/mqtt-keywords.rst

@@ -62,7 +62,7 @@ mqtt.flags
 
 Match on a combination of MQTT header flags, separated by commas (``,``). Flags may be prefixed by ``!`` to indicate negation, i.e. a flag prefixed by ``!`` must `not` be set to match.
 
-mqtt.flags uses an :ref:`unsigned 8-bits integer <rules-integer-keywords>`
+mqtt.flags uses an :ref:`unsigned 8-bits integer <rules-integer-keywords>` with bitmasks.
 
 Valid flags are:
 
@@ -146,7 +146,7 @@ mqtt.connect.flags
 
 Match on a combination of MQTT CONNECT flags, separated by commas (``,``). Flags may be prefixed by ``!`` to indicate negation, i.e. a flag prefixed by ``!`` must `not` be set to match.
 
-mqtt.connect.flags uses an :ref:`unsigned 8-bits integer <rules-integer-keywords>`
+mqtt.connect.flags uses an :ref:`unsigned 8-bits integer <rules-integer-keywords>` with bitmasks.
 
 Valid flags are:
 

doc/userguide/rules/websocket-keywords.rst

@@ -25,7 +25,7 @@ The value can also be a list of strings (comma-separated),
 where each string is the name of a specific bit like `fin` and `comp`,
 and can be prefixed by `!` for negation.
 
-websocket.flags uses an :ref:`unsigned 8-bits integer <rules-integer-keywords>`
+websocket.flags uses an :ref:`unsigned 8-bits integer <rules-integer-keywords>` with bitmasks.
 
 Examples::
 

rust/src/detect/uint.rs

@@ -16,11 +16,10 @@
  */
 
 use nom7::branch::alt;
-use nom7::bytes::complete::{is_a, tag, tag_no_case, take_while};
-use nom7::character::complete::{char, digit1, hex_digit1, i32 as nom_i32};
+use nom7::bytes::complete::{is_a, tag, tag_no_case, take, take_while};
+use nom7::character::complete::{anychar, char, digit1, hex_digit1, i32 as nom_i32};
 use nom7::combinator::{all_consuming, map_opt, opt, value, verify};
 use nom7::error::{make_error, Error, ErrorKind};
-use nom7::multi::many1;
 use nom7::Err;
 use nom7::IResult;
 
@@ -348,37 +347,47 @@ struct FlagItem<T> {
     neg: bool,
 }
 
-fn parse_flag_list_item<T1: DetectIntType, T2: EnumString<T1>>(
-    s: &str,
-) -> IResult<&str, FlagItem<T1>> {
-    let (s, _) = opt(is_a(" "))(s)?;
-    let (s, neg) = opt(tag("!"))(s)?;
-    let neg = neg.is_some();
-    let (s, vals) = take_while(|c| c != ' ' && c != ',')(s)?;
-    let value = T2::from_str(vals);
-    if value.is_none() {
-        SCLogError!("Bitflag unexpected value {}", vals);
-        return Err(Err::Error(make_error(s, ErrorKind::Switch)));
-    }
-    let value = value.unwrap().into_u();
-    let (s, _) = opt(is_a(" ,"))(s)?;
-    Ok((s, FlagItem { neg, value }))
-}
-
 fn parse_flag_list<T1: DetectIntType, T2: EnumString<T1>>(
-    s: &str,
+    s: &str, singlechar: bool,
 ) -> IResult<&str, Vec<FlagItem<T1>>> {
-    return many1(parse_flag_list_item::<T1, T2>)(s);
+    let mut r = Vec::new();
+    let mut s2 = s;
+    while !s2.is_empty() {
+        let (s, _) = opt(is_a(" "))(s2)?;
+        let (s, neg) = opt(tag("!"))(s)?;
+        let neg = neg.is_some();
+        let (s, vals) = if singlechar {
+            take(1usize)(s)
+        } else {
+            take_while(|c| c != ' ' && c != ',')(s)
+        }?;
+        let value = T2::from_str(vals);
+        if value.is_none() {
+            SCLogError!("Bitflag unexpected value {}", vals);
+            return Err(Err::Error(make_error(s, ErrorKind::Switch)));
+        }
+        let value = value.unwrap().into_u();
+        let (s, _) = if singlechar {
+            Ok((s, None))
+        } else {
+            opt(is_a(" ,"))(s)
+        }?;
+        r.push(FlagItem { neg, value });
+        s2 = s;
+    }
+    return Ok((s2, r));
 }
 
 pub fn detect_parse_uint_bitflags<T1: DetectIntType, T2: EnumString<T1>>(
-    s: &str,
+    s: &str, defmod: DetectBitflagModifier, singlechar: bool,
 ) -> Option<DetectUintData<T1>> {
     if let Ok((_, ctx)) = detect_parse_uint::<T1>(s) {
         return Some(ctx);
     }
     // otherwise, try strings for bitmask
-    if let Ok((rem, l)) = parse_flag_list::<T1, T2>(s) {
+    let (s, modifier) = parse_bitchars_modifier(s, defmod).ok()?;
+    let (s, _) = take_while::<_, &str, Error<_>>(|c| c == ' ' || c == '\t')(s).ok()?;
+    if let Ok((rem, l)) = parse_flag_list::<T1, T2>(s, singlechar) {
         if !rem.is_empty() {
             SCLogError!("junk at the end of bitflags");
             return None;
@@ -398,16 +407,64 @@ pub fn detect_parse_uint_bitflags<T1: DetectIntType, T2: EnumString<T1>>(
                 arg2 |= elem.value;
             }
         }
-        let ctx = DetectUintData::<T1> {
-            arg1,
-            arg2,
-            mode: DetectUintMode::DetectUintModeBitmask,
+        let ctx = match modifier {
+            DetectBitflagModifier::Equal => DetectUintData::<T1> {
+                arg1,
+                arg2: T1::min_value(),
+                mode: DetectUintMode::DetectUintModeEqual,
+            },
+            DetectBitflagModifier::Plus => DetectUintData::<T1> {
+                arg1,
+                arg2,
+                mode: DetectUintMode::DetectUintModeBitmask,
+            },
+            DetectBitflagModifier::Any => DetectUintData::<T1> {
+                arg1,
+                arg2: T1::min_value(),
+                mode: DetectUintMode::DetectUintModeNegBitmask,
+            },
+            DetectBitflagModifier::Not => DetectUintData::<T1> {
+                arg1,
+                arg2,
+                mode: DetectUintMode::DetectUintModeNegBitmask,
+            },
         };
         return Some(ctx);
     }
     return None;
 }
 
+#[derive(Clone, Debug, PartialEq)]
+pub enum DetectBitflagModifier {
+    Equal,
+    Plus,
+    Any,
+    Not,
+}
+
+fn parse_bitchars_modifier(
+    s: &str, default: DetectBitflagModifier,
+) -> IResult<&str, DetectBitflagModifier> {
+    let (s1, m) = anychar(s)?;
+    match m {
+        '!' => {
+            // exclamation mark is only accepted for legacy keywords
+            // excluded for newer to avoid ambiguity with negating single flag
+            if default == DetectBitflagModifier::Equal {
+                Ok((s1, DetectBitflagModifier::Not))
+            } else {
+                Ok((s, default))
+            }
+        }
+        '-' => Ok((s1, DetectBitflagModifier::Not)),
+        '+' => Ok((s1, DetectBitflagModifier::Plus)),
+        '*' => Ok((s1, DetectBitflagModifier::Any)),
+        '=' => Ok((s1, DetectBitflagModifier::Equal)),
+        // do not consume if not a known modifier: use default equal
+        _ => Ok((s, default)),
+    }
+}
+
 /// Parses a string for detection with integers, using enumeration strings
 ///
 /// Needs to specify T1 the integer type (like u8)

rust/src/dnp3/detect.rs

@@ -15,7 +15,7 @@
  * 02110-1301, USA.
  */
 
-use crate::detect::uint::{detect_parse_uint_bitflags, DetectUintData};
+use crate::detect::uint::{detect_parse_uint_bitflags, DetectBitflagModifier, DetectUintData};
 
 use std::ffi::CStr;
 
@@ -41,13 +41,17 @@ pub enum Dnp3IndFlag {
     No_func_code_support = 0x0001,
 }
 
+fn dnp3_detect_ind_parse(s: &str) -> Option<DetectUintData<u16>> {
+    detect_parse_uint_bitflags::<u16, Dnp3IndFlag>(s, DetectBitflagModifier::Any, false)
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn SCDnp3DetectIndParse(
     ustr: *const std::os::raw::c_char,
 ) -> *mut DetectUintData<u16> {
     let ft_name: &CStr = CStr::from_ptr(ustr); //unsafe
     if let Ok(s) = ft_name.to_str() {
-        if let Some(ctx) = detect_parse_uint_bitflags::<u16, Dnp3IndFlag>(s) {
+        if let Some(ctx) = dnp3_detect_ind_parse(s) {
             let boxed = Box::new(ctx);
             return Box::into_raw(boxed) as *mut _;
         }
@@ -61,24 +65,23 @@ mod test {
 
     #[test]
     fn dnp3_ind_parse() {
-        let ctx = detect_parse_uint_bitflags::<u16, Dnp3IndFlag>("0").unwrap();
+        let ctx = dnp3_detect_ind_parse("0").unwrap();
         assert_eq!(ctx.arg1, 0);
-        let ctx = detect_parse_uint_bitflags::<u16, Dnp3IndFlag>("1").unwrap();
+        let ctx = dnp3_detect_ind_parse("1").unwrap();
         assert_eq!(ctx.arg1, 1);
-        let ctx = detect_parse_uint_bitflags::<u16, Dnp3IndFlag>("0x0").unwrap();
+        let ctx = dnp3_detect_ind_parse("0x0").unwrap();
         assert_eq!(ctx.arg1, 0);
-        let ctx = detect_parse_uint_bitflags::<u16, Dnp3IndFlag>("0x0000").unwrap();
+        let ctx = dnp3_detect_ind_parse("0x0000").unwrap();
         assert_eq!(ctx.arg1, 0);
-        let ctx = detect_parse_uint_bitflags::<u16, Dnp3IndFlag>("0x0001").unwrap();
+        let ctx = dnp3_detect_ind_parse("0x0001").unwrap();
         assert_eq!(ctx.arg1, 1);
-        let ctx = detect_parse_uint_bitflags::<u16, Dnp3IndFlag>("0x8421").unwrap();
+        let ctx = dnp3_detect_ind_parse("0x8421").unwrap();
         assert_eq!(ctx.arg1, 0x8421);
-        assert!(detect_parse_uint_bitflags::<u16, Dnp3IndFlag>("a").is_none());
-        let ctx = detect_parse_uint_bitflags::<u16, Dnp3IndFlag>("all_stations").unwrap();
+        assert!(dnp3_detect_ind_parse("a").is_none());
+        let ctx = dnp3_detect_ind_parse("all_stations").unwrap();
         assert_eq!(ctx.arg1, 0x0100);
-        let ctx = detect_parse_uint_bitflags::<u16, Dnp3IndFlag>("class_1_events , class_2_events")
-            .unwrap();
+        let ctx = dnp3_detect_ind_parse("class_1_events , class_2_events").unwrap();
         assert_eq!(ctx.arg1, 0x600);
-        assert!(detect_parse_uint_bitflags::<u16, Dnp3IndFlag>("something").is_none());
+        assert!(dnp3_detect_ind_parse("something",).is_none());
     }
 }

rust/src/mqtt/detect.rs

@@ -20,8 +20,8 @@
 use crate::core::{STREAM_TOCLIENT, STREAM_TOSERVER};
 use crate::detect::uint::{
     detect_match_uint, detect_parse_array_uint_enum, detect_parse_uint_bitflags,
-    detect_uint_match_at_index, DetectUintArrayData, DetectUintData, SCDetectU8Free,
-    SCDetectU8Parse,
+    detect_uint_match_at_index, DetectBitflagModifier, DetectUintArrayData, DetectUintData,
+    SCDetectU8Free, SCDetectU8Parse,
 };
 use crate::detect::{
     helper_keyword_register_multi_buffer, helper_keyword_register_sticky_buffer,
@@ -674,12 +674,16 @@ pub enum MqttFlag {
     Retain = 0x1,
 }
 
+fn mqtt_flags_parse(s: &str) -> Option<DetectUintData<u8>> {
+    detect_parse_uint_bitflags::<u8, MqttFlag>(s, DetectBitflagModifier::Plus, false)
+}
+
 unsafe extern "C" fn mqtt_parse_flags(
     ustr: *const std::os::raw::c_char,
 ) -> *mut DetectUintData<u8> {
     let ft_name: &CStr = CStr::from_ptr(ustr); //unsafe
     if let Ok(s) = ft_name.to_str() {
-        if let Some(ctx) = detect_parse_uint_bitflags::<u8, MqttFlag>(s) {
+        if let Some(ctx) = mqtt_flags_parse(s) {
             let boxed = Box::new(ctx);
             return Box::into_raw(boxed) as *mut _;
         }
@@ -753,12 +757,16 @@ pub enum MqttConnFlag {
     Clean_session = 0x2,
 }
 
+fn mqtt_connflags_parse(s: &str) -> Option<DetectUintData<u8>> {
+    detect_parse_uint_bitflags::<u8, MqttConnFlag>(s, DetectBitflagModifier::Plus, false)
+}
+
 unsafe extern "C" fn mqtt_parse_conn_flags(
     ustr: *const std::os::raw::c_char,
 ) -> *mut DetectUintData<u8> {
     let ft_name: &CStr = CStr::from_ptr(ustr); //unsafe
     if let Ok(s) = ft_name.to_str() {
-        if let Some(ctx) = detect_parse_uint_bitflags::<u8, MqttConnFlag>(s) {
+        if let Some(ctx) = mqtt_connflags_parse(s) {
             let boxed = Box::new(ctx);
             return Box::into_raw(boxed) as *mut _;
         }
@@ -1191,60 +1199,49 @@ mod test {
 
     #[test]
     fn mqtt_parse_flags() {
-        let ctx = detect_parse_uint_bitflags::<u8, MqttFlag>("retain").unwrap();
+        let ctx = mqtt_flags_parse("retain").unwrap();
         assert_eq!(ctx.arg1, 1);
         assert_eq!(ctx.arg2, 1);
-        let ctx = detect_parse_uint_bitflags::<u8, MqttFlag>("dup").unwrap();
+        let ctx = mqtt_flags_parse("dup").unwrap();
         assert_eq!(ctx.arg1, 8);
         assert_eq!(ctx.arg2, 8);
-        let ctx = detect_parse_uint_bitflags::<u8, MqttFlag>("retain,dup").unwrap();
+        let ctx = mqtt_flags_parse("retain,dup").unwrap();
         assert_eq!(ctx.arg1, 8 | 1);
         assert_eq!(ctx.arg2, 8 | 1);
-        let ctx = detect_parse_uint_bitflags::<u8, MqttFlag>("dup, retain").unwrap();
+        let ctx = mqtt_flags_parse("dup, retain").unwrap();
         assert_eq!(ctx.arg1, 8 | 1);
         assert_eq!(ctx.arg2, 8 | 1);
-        let ctx = detect_parse_uint_bitflags::<u8, MqttFlag>("retain,!dup").unwrap();
+        let ctx = mqtt_flags_parse("retain,!dup").unwrap();
         assert_eq!(ctx.arg1, 1 | 8);
         assert_eq!(ctx.arg2, 1);
-        assert!(detect_parse_uint_bitflags::<u8, MqttFlag>("ref").is_none());
-        assert!(detect_parse_uint_bitflags::<u8, MqttFlag>("dup,!").is_none());
-        assert!(detect_parse_uint_bitflags::<u8, MqttFlag>("dup,!dup").is_none());
-        assert!(detect_parse_uint_bitflags::<u8, MqttFlag>("!retain,retain").is_none());
+        assert!(mqtt_flags_parse("ref").is_none());
+        assert!(mqtt_flags_parse("dup,!").is_none());
+        assert!(mqtt_flags_parse("dup,!dup",).is_none());
+        assert!(mqtt_flags_parse("!retain,retain",).is_none());
     }
 
     #[test]
     fn mqtt_parse_conn_flags() {
-        let ctx = detect_parse_uint_bitflags::<u8, MqttConnFlag>("username").unwrap();
+        let ctx = mqtt_connflags_parse("username").unwrap();
         assert_eq!(ctx.arg1, 0x80);
         assert_eq!(ctx.arg2, 0x80);
-        let ctx = detect_parse_uint_bitflags::<u8, MqttConnFlag>(
-            "username,password,will,will_retain,clean_session",
-        )
-        .unwrap();
+        let ctx = mqtt_connflags_parse("username,password,will,will_retain,clean_session").unwrap();
         assert_eq!(ctx.arg1, 0xE6);
         assert_eq!(ctx.arg2, 0xE6);
-        let ctx = detect_parse_uint_bitflags::<u8, MqttConnFlag>(
-            "!username,!password,!will,!will_retain,!clean_session",
-        )
-        .unwrap();
+        let ctx =
+            mqtt_connflags_parse("!username,!password,!will,!will_retain,!clean_session").unwrap();
         assert_eq!(ctx.arg1, 0xE6);
         assert_eq!(ctx.arg2, 0);
-        let ctx = detect_parse_uint_bitflags::<u8, MqttConnFlag>("   username,password").unwrap();
+        let ctx = mqtt_connflags_parse("   username,password").unwrap();
         assert_eq!(ctx.arg1, 0xC0);
         assert_eq!(ctx.arg2, 0xC0);
-        assert!(detect_parse_uint_bitflags::<u8, MqttConnFlag>("foobar").is_none());
-        assert!(detect_parse_uint_bitflags::<u8, MqttConnFlag>("will,!").is_none());
-        assert!(detect_parse_uint_bitflags::<u8, MqttConnFlag>("").is_none());
-        assert!(detect_parse_uint_bitflags::<u8, MqttConnFlag>("username, username").is_none());
-        assert!(detect_parse_uint_bitflags::<u8, MqttConnFlag>("!username, username").is_none());
-        assert!(
-            detect_parse_uint_bitflags::<u8, MqttConnFlag>("!username,password,!password")
-                .is_none()
-        );
-        assert!(detect_parse_uint_bitflags::<u8, MqttConnFlag>(
-            "will, username,password,   !will, will"
-        )
-        .is_none());
+        assert!(mqtt_connflags_parse("foobar").is_none());
+        assert!(mqtt_connflags_parse("will,!").is_none());
+        assert!(mqtt_connflags_parse("").is_none());
+        assert!(mqtt_connflags_parse("username, username").is_none());
+        assert!(mqtt_connflags_parse("!username, username").is_none());
+        assert!(mqtt_connflags_parse("!username,password,!password").is_none());
+        assert!(mqtt_connflags_parse("will, username,password,   !will, will",).is_none());
     }
 
     #[test]

rust/src/websocket/detect.rs

@@ -18,8 +18,8 @@
 use super::websocket::{WebSocketTransaction, ALPROTO_WEBSOCKET};
 use crate::core::{STREAM_TOCLIENT, STREAM_TOSERVER};
 use crate::detect::uint::{
-    detect_parse_uint_bitflags, detect_parse_uint_enum, DetectUintData, SCDetectU32Free,
-    SCDetectU32Match, SCDetectU32Parse, SCDetectU8Free, SCDetectU8Match,
+    detect_parse_uint_bitflags, detect_parse_uint_enum, DetectBitflagModifier, DetectUintData,
+    SCDetectU32Free, SCDetectU32Match, SCDetectU32Parse, SCDetectU8Free, SCDetectU8Match,
 };
 use crate::detect::{
     helper_keyword_register_sticky_buffer, SigTableElmtStickyBuffer, SIGMATCH_INFO_BITFLAGS_UINT,
@@ -61,7 +61,9 @@ unsafe extern "C" fn websocket_parse_flags(
 ) -> *mut DetectUintData<u8> {
     let ft_name: &CStr = CStr::from_ptr(ustr); //unsafe
     if let Ok(s) = ft_name.to_str() {
-        if let Some(ctx) = detect_parse_uint_bitflags::<u8, WebSocketFlag>(s) {
+        if let Some(ctx) =
+            detect_parse_uint_bitflags::<u8, WebSocketFlag>(s, DetectBitflagModifier::Plus, false)
+        {
             let boxed = Box::new(ctx);
             return Box::into_raw(boxed) as *mut _;
         }