ctest: Add translation of Rust types.

Author: mbyx

ctest-next/Cargo.toml

@@ -8,5 +8,13 @@ repository = "https://github.com/rust-lang/libc"
 publish = false
 
 [dependencies]
+askama = "0.14.0"
 cc = "1.2.25"
+proc-macro2 = { version = "1.0.95", features = ["span-locations"] }
+quote = "1.0.40"
 syn = { version = "2.0.101", features = ["full", "visit", "extra-traits"] }
+thiserror = "2.0.12"
+
+[dev-dependencies]
+pretty_assertions = "1.4.1"
+tempfile = "3.20.0"

ctest-next/askama.toml

@@ -0,0 +1,3 @@
+[[escaper]]
+path = "askama::filters::Text"
+extensions = ["rs", "c", "cpp"]

ctest-next/build.rs

@@ -0,0 +1,69 @@
+use std::env;
+
+// When we call `cargo test` for a cross compiled target, the following is required:
+// - CARGO_TARGET_{}_LINKER: To link the integration tests.
+// - CARGO_TARGET_{}_RUNNER: To run the integration tests.
+//
+// This is already set by the CI for all platforms, so there is no problem up till here.
+//
+// The integration tests (which are run in qemu, but use host rustc and cc) require the
+// following:
+// - TARGET_PLATFORM or target set manually. (We forward TARGET in build.rs for this.)
+// - HOST_PLATFORM or host set manually. (We forward HOST in build.rs for this.)
+// - LINKER: To link the C headers. (We forward CARGO_TARGET_{}_LINKER for this.)
+// - FLAGS: Any flags to pass when compiling the test binary for the cross compiled platform.
+//   (Forwarded from CARGO_TARGET_{}_RUSTFLAGS)
+// - RUNNER: To run the test binary with. (Forward the same runner as CARGO_TARGET_{}_RUNNER)
+//
+// The TARGET_PLATFORM and HOST_PLATFORM variables are not an issue, cargo will automatically set
+// TARGET and PLATFORM and we will forward them.
+//
+// Similarly FLAGS and RUNNER are also not an issue, if CARGO_TARGET_{}_RUSTFLAGS are present
+// they're forwarded. And RUSTFLAGS works by default anyway. Similarly the test binary doesn't
+// require any external applications so just the RUNNER is enough to run it.
+//
+// However since rustc and cc are the host versions, they will only work if we specify the
+// correct variables for them. Because we only use them to compile, not run things. For CC we
+// MUST specify CC or CC_target otherwise it will fail. (Other flags like AR etc. work without
+// forwarding because it is run in the host.) For rustc we MUST specify the correct linker.
+// Usually this is the same as CC or CC_target.
+//
+// In the CI, the CARGO_TARGET_{} variables are always set.
+
+fn main() {
+    let host = env::var("HOST").unwrap();
+    let target = env::var("TARGET").unwrap();
+    let target_key = target.replace('-', "_").to_uppercase();
+
+    println!("cargo:rustc-env=HOST_PLATFORM={host}");
+    println!("cargo:rerun-if-changed-env=HOST");
+
+    println!("cargo:rustc-env=TARGET_PLATFORM={target}");
+    println!("cargo:rerun-if-changed-env=TARGET");
+
+    let link_var = format!("CARGO_TARGET_{target_key}_LINKER");
+    println!("cargo:rerun-if-changed-env={link_var}");
+    if let Ok(linker) = env::var(link_var) {
+        println!("cargo:rustc-env=LINKER={linker}");
+    }
+
+    let run_var = format!("CARGO_TARGET_{target_key}_RUNNER");
+    println!("cargo:rerun-if-changed-env={run_var}");
+    if let Ok(runner) = env::var(run_var) {
+        println!("cargo:rustc-env=RUNNER={runner}");
+    }
+
+    // As we invoke rustc directly this does not get passed to it, although RUSTFLAGS does.
+    let flag_var = format!("CARGO_TARGET_{target_key}_RUSTFLAGS");
+    println!("cargo:rerun-if-changed-env={flag_var}");
+    if let Ok(flags) = env::var(flag_var) {
+        println!("cargo:rustc-env=FLAGS={flags}");
+    }
+
+    // Rerun this build script if any of these environment variables change.
+    println!("cargo:rerun-if-changed-env=CC");
+    println!(
+        "cargo:rerun-if-changed-env=CC_{}",
+        target_key.to_lowercase()
+    );
+}

ctest-next/src/ast/constant.rs

@@ -6,7 +6,6 @@ pub struct Const {
     #[expect(unused)]
     pub(crate) public: bool,
     pub(crate) ident: BoxStr,
-    #[expect(unused)]
     pub(crate) ty: syn::Type,
     #[expect(unused)]
     pub(crate) expr: syn::Expr,

ctest-next/src/ffi_items.rs

@@ -54,7 +54,6 @@ impl FfiItems {
     }
 
     /// Return a list of all constants found.
-    #[cfg_attr(not(test), expect(unused))]
     pub(crate) fn constants(&self) -> &Vec<Const> {
         &self.constants
     }

ctest-next/src/generator.rs

@@ -1,28 +1,100 @@
-use std::path::Path;
+use std::{
+    env,
+    fs::File,
+    io::Write,
+    path::{Path, PathBuf},
+};
 
+use askama::Template;
 use syn::visit::Visit;
 
-use crate::{expand, ffi_items::FfiItems, Result};
+use crate::{
+    expand,
+    ffi_items::FfiItems,
+    template::{CTestTemplate, RustTestTemplate},
+    Result,
+};
 
 /// A builder used to generate a test suite.
-#[non_exhaustive]
 #[derive(Default, Debug, Clone)]
-pub struct TestGenerator {}
+pub struct TestGenerator {
+    headers: Vec<String>,
+    pub(crate) target: Option<String>,
+    pub(crate) includes: Vec<PathBuf>,
+    out_dir: Option<PathBuf>,
+}
 
 impl TestGenerator {
     /// Creates a new blank test generator.
     pub fn new() -> Self {
         Self::default()
     }
 
-    /// Generate all tests for the given crate and output the Rust side to a file.
-    pub fn generate<P: AsRef<Path>>(&mut self, crate_path: P, _output_file_path: P) -> Result<()> {
+    /// Add a header to be included as part of the generated C file.
+    ///
+    /// The generate C test will be compiled by a C compiler, and this can be
+    /// used to ensure that all the necessary header files are included to test
+    /// all FFI definitions.
+    pub fn header(&mut self, header: &str) -> &mut Self {
+        self.headers.push(header.to_string());
+        self
+    }
+
+    /// Configures the target to compile C code for.
+    ///
+    /// Note that for Cargo builds this defaults to `$TARGET` and it's not
+    /// necessary to call.
+    pub fn target(&mut self, target: &str) -> &mut Self {
+        self.target = Some(target.to_string());
+        self
+    }
+
+    /// Add a path to the C compiler header lookup path.
+    ///
+    /// This is useful for if the C library is installed to a nonstandard
+    /// location to ensure that compiling the C file succeeds.
+    pub fn include<P: AsRef<Path>>(&mut self, p: P) -> &mut Self {
+        self.includes.push(p.as_ref().to_owned());
+        self
+    }
+
+    /// Configures the output directory of the generated Rust and C code.
+    pub fn out_dir<P: AsRef<Path>>(&mut self, p: P) -> &mut Self {
+        self.out_dir = Some(p.as_ref().to_owned());
+        self
+    }
+
+    /// Generate the Rust and C testing files.
+    ///
+    /// Returns the path to t generated file.
+    pub fn generate_files(
+        &mut self,
+        crate_path: impl AsRef<Path>,
+        output_file_path: impl AsRef<Path>,
+    ) -> Result<PathBuf> {
         let expanded = expand(crate_path)?;
         let ast = syn::parse_file(&expanded)?;
 
         let mut ffi_items = FfiItems::new();
         ffi_items.visit_file(&ast);
 
-        Ok(())
+        let output_directory = self
+            .out_dir
+            .clone()
+            .unwrap_or_else(|| env::var("OUT_DIR").unwrap().into());
+        let output_file_path = output_directory.join(output_file_path);
+
+        // Generate the Rust side of the tests.
+        File::create(output_file_path.with_extension("rs"))?
+            .write_all(RustTestTemplate::new(&ffi_items)?.render()?.as_bytes())?;
+
+        // Generate the C side of the tests.
+        // FIXME(ctest): Cpp not supported yet.
+        let c_output_path = output_file_path.with_extension("c");
+        let headers = self.headers.iter().map(|h| h.as_str()).collect();
+        File::create(&c_output_path)?
+            .write_all(CTestTemplate::new(headers, &ffi_items).render()?.as_bytes())?;
+
+        Ok(output_file_path)
     }
 }

ctest-next/src/lib.rs

@@ -15,10 +15,15 @@ mod ast;
 mod ffi_items;
 mod generator;
 mod macro_expansion;
+mod runner;
+mod template;
+mod translator;
 
 pub use ast::{Abi, Const, Field, Fn, Parameter, Static, Struct, Type, Union};
 pub use generator::TestGenerator;
 pub use macro_expansion::expand;
+pub use runner::{__compile_test, __run_test, generate_test};
+pub use translator::TranslationError;
 
 /// A possible error that can be encountered in our library.
 pub type Error = Box<dyn std::error::Error>;

ctest-next/src/macro_expansion.rs

@@ -9,6 +9,8 @@ pub fn expand<P: AsRef<Path>>(crate_path: P) -> Result<String> {
     let output = Command::new(rustc)
         .env("RUSTC_BOOTSTRAP", "1")
         .arg("-Zunpretty=expanded")
+        .arg("--edition")
+        .arg("2024") // By default, -Zunpretty=expanded uses 2015 edition.
         .arg(canonicalize(crate_path)?)
         .output()?;
 

ctest-next/src/runner.rs

@@ -0,0 +1,162 @@
+use crate::{Result, TestGenerator};
+use std::env;
+use std::fs::{canonicalize, File};
+use std::io::Write;
+use std::path::{Path, PathBuf};
+use std::process::Command;
+
+/// Generate all tests for the given crate and output the Rust side to a file.
+#[doc(hidden)]
+pub fn generate_test(
+    generator: &mut TestGenerator,
+    crate_path: impl AsRef<Path>,
+    output_file_path: impl AsRef<Path>,
+) -> Result<PathBuf> {
+    let output_file_path = generator.generate_files(crate_path, output_file_path)?;
+
+    // Search for the target and host to build for if specified manually
+    // (generator.target, generator.host),
+    // via build script (TARGET, HOST), or for internal testing (TARGET_PLATFORM, HOST_PLATFORM).
+    let target = generator.target.clone().unwrap_or_else(|| {
+        env::var("TARGET").unwrap_or_else(|_| env::var("TARGET_PLATFORM").unwrap())
+    });
+    let host = env::var("HOST").unwrap_or_else(|_| env::var("HOST_PLATFORM").unwrap());
+
+    let mut cfg = cc::Build::new();
+    // FIXME(ctest): Cpp not supported.
+    cfg.file(output_file_path.with_extension("c"));
+    cfg.host(&host);
+
+    if target.contains("msvc") {
+        cfg.flag("/W3")
+            .flag("/Wall")
+            .flag("/WX")
+            // ignored warnings
+            .flag("/wd4820") // warning about adding padding?
+            .flag("/wd4100") // unused parameters
+            .flag("/wd4996") // deprecated functions
+            .flag("/wd4296") // '<' being always false
+            .flag("/wd4255") // converting () to (void)
+            .flag("/wd4668") // using an undefined thing in preprocessor?
+            .flag("/wd4366") // taking ref to packed struct field might be unaligned
+            .flag("/wd4189") // local variable initialized but not referenced
+            .flag("/wd4710") // function not inlined
+            .flag("/wd5045") // compiler will insert Spectre mitigation
+            .flag("/wd4514") // unreferenced inline function removed
+            .flag("/wd4711"); // function selected for automatic inline
+    } else {
+        cfg.flag("-Wall")
+            .flag("-Wextra")
+            .flag("-Werror")
+            .flag("-Wno-unused-parameter")
+            .flag("-Wno-type-limits")
+            // allow taking address of packed struct members:
+            .flag("-Wno-address-of-packed-member")
+            .flag("-Wno-unknown-warning-option")
+            .flag("-Wno-deprecated-declarations"); // allow deprecated items
+    }
+
+    for p in &generator.includes {
+        cfg.include(p);
+    }
+
+    let stem: &str = output_file_path.file_stem().unwrap().to_str().unwrap();
+    cfg.target(&target)
+        .out_dir(output_file_path.parent().unwrap())
+        .compile(stem);
+
+    Ok(output_file_path)
+}
+
+/// Compiles a Rust source file and links it against a static library.
+///
+/// Returns the path to the generated binary.
+#[doc(hidden)]
+pub fn __compile_test(
+    output_dir: impl AsRef<Path>,
+    crate_path: impl AsRef<Path>,
+    library_file: impl AsRef<Path>,
+) -> Result<PathBuf> {
+    let rustc = env::var("RUSTC").unwrap_or_else(|_| "rustc".into());
+    let output_dir = output_dir.as_ref();
+    let crate_path = crate_path.as_ref();
+    let library_file = library_file.as_ref().file_stem().unwrap();
+
+    let rust_file = output_dir
+        .join(crate_path.file_stem().unwrap())
+        .with_extension("rs");
+    let binary_path = output_dir.join(rust_file.file_stem().unwrap());
+
+    // Create a file that contains the Rust 'bindings' as well as the generated test code.
+    File::create(&rust_file)?.write_all(
+        format!(
+            "include!(r#\"{}\"#);\ninclude!(r#\"{}.rs\"#);",
+            canonicalize(crate_path)?.display(),
+            library_file.to_str().unwrap()
+        )
+        .as_bytes(),
+    )?;
+
+    // Compile the test file with the compiled C library file found in `output_dir`
+    // into a binary file, ignoring all warnings about unused items. (not all items
+    // are currently tested)
+
+    let mut cmd = Command::new(rustc);
+    cmd.arg(&rust_file)
+        .arg(format!("-Lnative={}", output_dir.display()))
+        .arg(format!("-lstatic={}", library_file.to_str().unwrap()))
+        .arg("--edition")
+        .arg("2021") // Defaults to 2015.
+        .arg("-o")
+        .arg(&binary_path)
+        .arg("-Aunused");
+
+    // Pass in a different target, linker or flags if set, useful for cross compilation.
+
+    let target = env::var("TARGET_PLATFORM").unwrap_or_default();
+    if !target.is_empty() {
+        cmd.arg("--target").arg(target);
+    }
+
+    let linker = env::var("LINKER").unwrap_or_default();
+    if !linker.is_empty() {
+        cmd.arg(format!("-Clinker={linker}"));
+    }
+
+    let flags = env::var("FLAGS").unwrap_or_default();
+    if !flags.is_empty() {
+        cmd.args(flags.split_whitespace());
+    }
+
+    let output = cmd.output()?;
+    if !output.status.success() {
+        return Err(std::str::from_utf8(&output.stderr)?.into());
+    }
+
+    Ok(binary_path)
+}
+
+/// Executes the compiled test binary and returns its output.
+///
+/// If a RUNNER environment variable is present, it will use that to run the binary.
+#[doc(hidden)]
+pub fn __run_test<P: AsRef<Path>>(test_binary: P) -> Result<String> {
+    let runner = env::var("RUNNER").unwrap_or_default();
+    let mut cmd;
+    if runner.is_empty() {
+        cmd = Command::new(test_binary.as_ref());
+    } else {
+        let mut args = runner.split_whitespace();
+        cmd = Command::new(args.next().unwrap());
+        cmd.args(args);
+    };
+
+    cmd.arg(test_binary.as_ref());
+    let output = cmd.output()?;
+
+    if !output.status.success() {
+        return Err(std::str::from_utf8(&output.stderr)?.into());
+    }
+
+    Ok(std::str::from_utf8(&output.stdout)?.to_string())
+}