Unified: Allow NULs in source code

These are apparently valid in Swift (there's a test for it in
`swiftlang/swift` that is parsed -- with a warning -- by the Swift
compiler).

To allow these, the SwiftSyntaxFFI now passes a string-with-length
rather than a NUL-terminated string.

Author: tausbn

unified/swift-syntax-rs/src/lib.rs

@@ -10,15 +10,15 @@
 //! by the extractor's own pure-Rust adapter module, keeping the Swift toolchain
 //! out of the extractor's build.
 
-use std::ffi::{CStr, CString};
+use std::ffi::CStr;
 use std::os::raw::c_char;
 
 // C ABI exported by the `SwiftSyntaxFFI` dynamic library.
 unsafe extern "C" {
-    /// Parse a NUL-terminated Swift source string, returning a heap-allocated
+    /// Parse a UTF-8 Swift source buffer, returning a heap-allocated
     /// NUL-terminated JSON string (or null on failure). The caller owns the
     /// returned pointer and must release it with `ssr_string_free`.
-    fn ssr_parse_json(source: *const c_char) -> *mut c_char;
+    fn ssr_parse_json(source: *const u8, source_len: usize) -> *mut c_char;
 
     /// Free a string previously returned by `ssr_parse_json`.
     fn ssr_string_free(ptr: *mut c_char);
@@ -27,8 +27,6 @@ unsafe extern "C" {
 /// Errors that can occur while parsing Swift source.
 #[derive(Debug)]
 pub enum ParseError {
-    /// The provided source contained an interior NUL byte.
-    NulByte,
     /// The Swift shim returned no result. `SwiftParser` recovers from invalid
     /// syntax (it always produces a tree, possibly with error nodes), so this
     /// does *not* indicate a syntax error in the source — it means the shim
@@ -39,7 +37,6 @@ pub enum ParseError {
 impl std::fmt::Display for ParseError {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
-            ParseError::NulByte => write!(f, "source contained an interior NUL byte"),
             ParseError::SwiftFailure => {
                 write!(f, "the swift-syntax shim failed to produce a JSON result")
             }
@@ -58,13 +55,11 @@ impl std::error::Error for ParseError {}
 /// println!("{json}");
 /// ```
 pub fn parse_to_json(source: &str) -> Result<String, ParseError> {
-    let c_source = CString::new(source).map_err(|_| ParseError::NulByte)?;
-
-    // SAFETY: `c_source` is a valid NUL-terminated string for the duration of
-    // the call. The returned pointer, if non-null, is owned by us and freed via
-    // `ssr_string_free` before returning.
+    // SAFETY: `source` is valid UTF-8 and its buffer remains alive for the
+    // duration of the call. The returned pointer, if non-null, is owned by us
+    // and freed via `ssr_string_free` before returning.
     unsafe {
-        let ptr = ssr_parse_json(c_source.as_ptr());
+        let ptr = ssr_parse_json(source.as_ptr(), source.len());
         if ptr.is_null() {
             return Err(ParseError::SwiftFailure);
         }
@@ -132,6 +127,15 @@ mod tests {
         );
     }
 
+    #[test]
+    fn parses_source_with_interior_nul() {
+        let json = parse_to_json("let x =\0 1").expect("parsing interior NUL should succeed");
+        assert!(
+            json.contains(r#"\u0000"#),
+            "interior NUL should be preserved in the JSON tree: {json}"
+        );
+    }
+
     #[test]
     fn serializes_json_strings_and_keys_deterministically() {
         let source = "/* quote \" slash / backslash \\ tab \t newline\n emoji 😀 combining e\u{301} control \u{1} */\nlet x = 1";

unified/swift-syntax-rs/swift/Sources/SwiftSyntaxFFI/SwiftSyntaxFFI.swift

@@ -292,15 +292,19 @@ private func appendJSON(_ value: Any, to output: inout [UInt8]) throws {
     }
 }
 
-/// Parse the given NUL-terminated Swift source string and return a
-/// heap-allocated, NUL-terminated JSON representation of the syntax tree.
+/// Parse the given UTF-8 Swift source buffer and return a heap-allocated,
+/// NUL-terminated JSON representation of the syntax tree.
 ///
 /// The returned pointer is owned by the caller and MUST be released with
 /// `ssr_string_free`. Returns `nil` on failure.
 @_cdecl("ssr_parse_json")
-public func ssr_parse_json(_ source: UnsafePointer<CChar>?) -> UnsafeMutablePointer<CChar>? {
-    guard let source = source else { return nil }
-    let code = String(cString: source)
+public func ssr_parse_json(
+    _ source: UnsafePointer<UInt8>?,
+    _ sourceLength: Int
+) -> UnsafeMutablePointer<CChar>? {
+    guard sourceLength >= 0, source != nil || sourceLength == 0 else { return nil }
+    let sourceBytes = UnsafeBufferPointer(start: source, count: sourceLength)
+    let code = String(decoding: sourceBytes, as: UTF8.self)
     let tree = Parser.parse(source: code)
     // Fold operator sequences before serializing. Source positions are
     // preserved by folding (the same tokens, in the same places), so a