regex: accept literal [ in bracket expressions

src/sed/compiler.rs

@@ -655,6 +655,82 @@ fn bre_to_ere(pattern: &[u8]) -> Vec<u8> {
     result
 }
 
+/// Escape literal `[` characters that appear inside a bracket expression.
+///
+/// Within a bracket expression a `[` only begins a sub-construct when followed
+/// by `:`, `.`, or `=` (e.g. `[:alpha:]`); elsewhere it is an ordinary
+/// character. The `regex` crate rejects such a bare `[`, so we escape those
+/// occurrences (`[` becomes `\[`) before handing the pattern to the engine. See
+/// POSIX 9.3.5 RE Bracket Expression:
+/// <https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_03_05>
+fn escape_literal_open_brackets_in_classes(pattern: &[u8]) -> Vec<u8> {
+    let mut result = Vec::with_capacity(pattern.len());
+    let mut bytes = pattern.iter().copied().peekable();
+
+    while let Some(c) = bytes.next() {
+        match c {
+            b'\\' => {
+                result.push(b'\\');
+                if let Some(escaped) = bytes.next() {
+                    result.push(escaped);
+                }
+                continue;
+            }
+            b'[' => result.push(b'['),
+            _ => {
+                result.push(c);
+                continue;
+            }
+        }
+
+        if bytes.peek() == Some(&b'^') {
+            result.push(b'^');
+            bytes.next();
+        }
+
+        if bytes.peek() == Some(&b']') {
+            result.push(b']');
+            bytes.next();
+        }
+
+        while let Some(class_byte) = bytes.next() {
+            match class_byte {
+                b']' => {
+                    result.push(b']');
+                    break;
+                }
+                b'\\' => {
+                    result.push(b'\\');
+                    if let Some(escaped) = bytes.next() {
+                        result.push(escaped);
+                    }
+                }
+                b'[' => {
+                    if let Some(&marker @ (b':' | b'.' | b'=')) = bytes.peek() {
+                        bytes.next();
+                        result.push(b'[');
+                        result.push(marker);
+
+                        while let Some(posix_byte) = bytes.next() {
+                            result.push(posix_byte);
+                            if posix_byte == marker && bytes.peek() == Some(&b']') {
+                                result.push(b']');
+                                bytes.next();
+                                break;
+                            }
+                        }
+                    } else {
+                        result.extend_from_slice(br"\[");
+                    }
+                }
+                _ => result.push(class_byte),
+            }
+        }
+    }
+
+    result
+}
+
 /// Compile the provided regular expression string into a corresponding engine.
 /// An empty pattern results in None, which means that the last RE employed
 /// at runtime will be used.
@@ -677,6 +753,7 @@ fn compile_regex(
     } else {
         bre_to_ere(pattern)
     };
+    let pattern = escape_literal_open_brackets_in_classes(&pattern);
 
     // Add any required modifiers.
     let mut modifiers = Vec::new();
@@ -1952,6 +2029,65 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_compile_re_literal_open_bracket_in_classes() {
+        let (lines, chars) = dummy_providers();
+        let mut context = ctx();
+        context.regex_extended = true;
+
+        for (pattern, matching, non_matching) in [
+            ("[[]", "[", "x"),
+            ("[^[]", "x", "["),
+            ("[a[b]", "[", "x"),
+            ("[^a[b]", "x", "["),
+        ] {
+            let regex = compile_regex(&lines, &chars, pattern, &context, false, false)
+                .unwrap()
+                .expect("regex should be present");
+            assert!(
+                regex
+                    .is_match(&mut IOChunk::new_from_str(matching))
+                    .unwrap(),
+                "{pattern:?} should match {matching:?}"
+            );
+            assert!(
+                !regex
+                    .is_match(&mut IOChunk::new_from_str(non_matching))
+                    .unwrap(),
+                "{pattern:?} should not match {non_matching:?}"
+            );
+        }
+    }
+
+    #[test]
+    fn test_compile_re_escaped_open_bracket_before_class() {
+        let (lines, chars) = dummy_providers();
+        let mut context = ctx();
+        context.regex_extended = true;
+
+        let regex = compile_regex(&lines, &chars, r"\[[a]", &context, false, false)
+            .unwrap()
+            .expect("regex should be present");
+        assert!(regex.is_match(&mut IOChunk::new_from_str("[a")).unwrap());
+        assert!(!regex.is_match(&mut IOChunk::new_from_str("[b")).unwrap());
+    }
+
+    #[test]
+    fn test_escape_literal_open_brackets_preserves_class_syntax() {
+        for (pattern, expected) in [
+            (r"[a\]b]", r"[a\]b]"),
+            (r"[[:alpha:][x]", r"[[:alpha:]\[x]"),
+            (r"[[=a=][x]", r"[[=a=]\[x]"),
+            (r"[[.ch.][x]", r"[[.ch.]\[x]"),
+        ] {
+            assert_eq!(
+                escape_literal_open_brackets_in_classes(pattern.as_bytes()),
+                expected.as_bytes(),
+                "{pattern:?}"
+            );
+        }
+    }
+
     // compile_address
     #[test]
     fn test_compile_addr_line_number() {

src/sed/delimited_parser.rs

@@ -304,10 +304,9 @@ fn parse_character_class(
 
                 continue;
             }
-            // Not a POSIX construct — treat as literal
+            // Not a POSIX construct: '[' is literal, and the next character
+            // may still terminate or otherwise participate in the class.
             result.push(b'[');
-            result.push(line.current_byte());
-            line.advance();
             continue;
         }
 
@@ -965,6 +964,38 @@ mod tests {
         assert_eq!(result, b"[^]abc]");
     }
 
+    #[test]
+    fn test_literal_open_bracket() {
+        let mut line = char_provider_from("[[]");
+        let lines = test_lines();
+        let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8).unwrap();
+        assert_eq!(result, b"[[]");
+    }
+
+    #[test]
+    fn test_negated_literal_open_bracket() {
+        let mut line = char_provider_from("[^[]");
+        let lines = test_lines();
+        let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8).unwrap();
+        assert_eq!(result, b"[^[]");
+    }
+
+    #[test]
+    fn test_literal_open_bracket_in_class() {
+        let mut line = char_provider_from("[a[b]");
+        let lines = test_lines();
+        let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8).unwrap();
+        assert_eq!(result, b"[a[b]");
+    }
+
+    #[test]
+    fn test_negated_literal_open_bracket_in_class() {
+        let mut line = char_provider_from("[^a[b]");
+        let lines = test_lines();
+        let result = parse_character_class(&lines, &mut line, CharacterMode::Utf8).unwrap();
+        assert_eq!(result, b"[^a[b]");
+    }
+
     #[test]
     fn test_escaped_character_begin() {
         let mut line = char_provider_from("[\\nabc]");

tests/by-util/test_sed.rs

@@ -568,6 +568,23 @@ fn subst_multiline_flag_matches_embedded_line_end() {
         .stdout_is("foX\nbaX\n");
 }
 
+#[test]
+fn test_subst_literal_open_bracket_in_character_classes() {
+    for (script, input, expected) in [
+        (r"s/[[]/X/", "x\n", "x\n"),
+        (r"s/[^[]/X/", "x\n", "X\n"),
+        (r"s/[a[b]/X/", "x\n", "x\n"),
+        (r"s/[^a[b]/X/", "x\n", "X\n"),
+        (r"s/\[[a]/X/", "[a\n", "X\n"),
+    ] {
+        new_ucmd!()
+            .args(&["-E", script])
+            .pipe_in(input)
+            .succeeds()
+            .stdout_is(expected);
+    }
+}
+
 // Check appropriate selection and behavior of fast_Regex matcher
 // Literal matcher
 check_output!(subst_literal_start, ["-e", r"s/^l1/L1/", LINES1]);