pyo3-build-config: fix windows "cross-compile" panic

Author: davidhewitt

CHANGELOG.md

@@ -8,6 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+
+- Panic during compilation when `PYO3_CROSS_LIB_DIR` is set for some host/target combinations. [#2232](https://github.com/PyO3/pyo3/pull/2232)
+
 ## [0.16.2] - 2022-03-15
 
 ### Packaging

pyo3-build-config/src/impl_.rs

@@ -568,35 +568,54 @@ pub struct CrossCompileConfig {
     os: String,
 }
 
-#[allow(unused)]
-pub fn any_cross_compiling_env_vars_set() -> bool {
-    env::var_os("PYO3_CROSS").is_some()
-        || env::var_os("PYO3_CROSS_LIB_DIR").is_some()
-        || env::var_os("PYO3_CROSS_PYTHON_VERSION").is_some()
+pub(crate) struct CrossCompileEnvVars {
+    pyo3_cross: Option<OsString>,
+    pyo3_cross_lib_dir: Option<OsString>,
+    pyo3_cross_python_version: Option<OsString>,
 }
 
-fn cross_compiling_from_cargo_env() -> Result<Option<CrossCompileConfig>> {
-    let host = cargo_env_var("HOST").ok_or("expected HOST env var")?;
-    let target = cargo_env_var("TARGET").ok_or("expected TARGET env var")?;
-
-    if host == target {
-        // Definitely not cross compiling if the host matches the target
-        return Ok(None);
+impl CrossCompileEnvVars {
+    pub fn any(&self) -> bool {
+        self.pyo3_cross.is_some()
+            || self.pyo3_cross_lib_dir.is_some()
+            || self.pyo3_cross_python_version.is_some()
     }
+}
 
-    if target == "i686-pc-windows-msvc" && host == "x86_64-pc-windows-msvc" {
-        // Not cross-compiling to compile for 32-bit Python from windows 64-bit
-        return Ok(None);
+pub(crate) fn cross_compile_env_vars() -> CrossCompileEnvVars {
+    CrossCompileEnvVars {
+        pyo3_cross: env::var_os("PYO3_CROSS"),
+        pyo3_cross_lib_dir: env::var_os("PYO3_CROSS_LIB_DIR"),
+        pyo3_cross_python_version: env::var_os("PYO3_CROSS_PYTHON_VERSION"),
     }
+}
 
-    let target_arch =
-        cargo_env_var("CARGO_CFG_TARGET_ARCH").ok_or("expected CARGO_CFG_TARGET_ARCH env var")?;
-    let target_vendor = cargo_env_var("CARGO_CFG_TARGET_VENDOR")
-        .ok_or("expected CARGO_CFG_TARGET_VENDOR env var")?;
-    let target_os =
-        cargo_env_var("CARGO_CFG_TARGET_OS").ok_or("expected CARGO_CFG_TARGET_OS env var")?;
-
-    cross_compiling(&host, &target_arch, &target_vendor, &target_os)
+fn cross_compiling_from_vars(
+    target_arch: &str,
+    target_os: &str,
+    target_vendor: &str,
+    cross_compile_vars: CrossCompileEnvVars,
+) -> Result<CrossCompileConfig> {
+    Ok(CrossCompileConfig {
+        lib_dir: cross_compile_vars
+            .pyo3_cross_lib_dir
+            .ok_or("The PYO3_CROSS_LIB_DIR environment variable must be set when cross-compiling")?
+            .into(),
+        arch: target_arch.into(),
+        vendor: target_vendor.into(),
+        os: target_os.into(),
+        version: cross_compile_vars
+            .pyo3_cross_python_version
+            .map(|os_string| {
+                let utf8_str = os_string
+                    .to_str()
+                    .ok_or("PYO3_CROSS_PYTHON_VERSION is not valid utf-8.")?;
+                utf8_str
+                    .parse()
+                    .context("failed to parse PYO3_CROSS_PYTHON_VERSION")
+            })
+            .transpose()?,
+    })
 }
 
 /// Detect whether we are cross compiling and return an assembled CrossCompileConfig if so.
@@ -619,52 +638,24 @@ pub fn cross_compiling(
     target_vendor: &str,
     target_os: &str,
 ) -> Result<Option<CrossCompileConfig>> {
-    let cross = env_var("PYO3_CROSS");
-    let cross_lib_dir = env_var("PYO3_CROSS_LIB_DIR");
-    let cross_python_version = env_var("PYO3_CROSS_PYTHON_VERSION");
-
-    let target_triple = format!(
-        "{}-{}-{}",
-        target_arch,
-        target_vendor,
-        if target_os == "macos" {
-            "darwin"
-        } else {
-            target_os
-        }
-    );
-
-    if cross.is_none() && cross_lib_dir.is_none() && cross_python_version.is_none() {
-        // No cross-compiling environment variables set; try to determine if this is a known case
-        // which is not cross-compilation.
+    let env_vars = cross_compile_env_vars();
 
-        if target_triple == "x86_64-apple-darwin" && host == "aarch64-apple-darwin" {
-            // Not cross-compiling to compile for x86-64 Python from macOS arm64
-            return Ok(None);
-        }
-
-        if target_triple == "aarch64-apple-darwin" && host == "x86_64-apple-darwin" {
-            // Not cross-compiling to compile for arm64 Python from macOS x86_64
-            return Ok(None);
-        }
-
-        if host.starts_with(&target_triple) {
-            // Not cross-compiling if arch-vendor-os is all the same
-            // e.g. x86_64-unknown-linux-musl on x86_64-unknown-linux-gnu host
-            return Ok(None);
-        }
+    if !env_vars.any() && is_not_cross_compiling(host, target_arch, target_vendor, target_os) {
+        return Ok(None);
     }
 
     // At this point we assume that we are cross compiling.
 
     Ok(Some(CrossCompileConfig {
-        lib_dir: cross_lib_dir
+        lib_dir: env_vars
+            .pyo3_cross_lib_dir
             .ok_or("The PYO3_CROSS_LIB_DIR environment variable must be set when cross-compiling")?
             .into(),
         arch: target_arch.into(),
         vendor: target_vendor.into(),
         os: target_os.into(),
-        version: cross_python_version
+        version: env_vars
+            .pyo3_cross_python_version
             .map(|os_string| {
                 let utf8_str = os_string
                     .to_str()
@@ -677,6 +668,39 @@ pub fn cross_compiling(
     }))
 }
 
+fn is_not_cross_compiling(
+    host: &str,
+    target_arch: &str,
+    target_vendor: &str,
+    target_os: &str,
+) -> bool {
+    let target_triple = format!(
+        "{}-{}-{}",
+        if target_arch == "x86" {
+            "i686"
+        } else {
+            target_arch
+        },
+        target_vendor,
+        if target_os == "macos" {
+            "darwin"
+        } else {
+            target_os
+        }
+    );
+
+    // Not cross-compiling if arch-vendor-os is all the same
+    // e.g. x86_64-unknown-linux-musl on x86_64-unknown-linux-gnu host
+    //      x86_64-pc-windows-gnu on x86_64-pc-windows-msvc host
+    host.starts_with(&target_triple)
+        // Not cross-compiling to compile for 32-bit Python from windows 64-bit
+        || (target_triple == "i686-pc-windows" && host.starts_with("x86_64-pc-windows"))
+        // Not cross-compiling to compile for x86-64 Python from macOS arm64
+        || (target_triple == "x86_64-apple-darwin" && host == "aarch64-apple-darwin")
+        // Not cross-compiling to compile for arm64 Python from macOS x86_64
+        || (target_triple == "aarch64-apple-darwin" && host == "x86_64-apple-darwin")
+}
+
 #[allow(non_camel_case_types)]
 #[derive(Debug, Clone, Hash, PartialEq, Eq)]
 pub enum BuildFlag {
@@ -1280,13 +1304,33 @@ fn fixup_config_for_abi3(
 /// This must be called from PyO3's build script, because it relies on environment variables such as
 /// CARGO_CFG_TARGET_OS which aren't available at any other time.
 pub fn make_cross_compile_config() -> Result<Option<InterpreterConfig>> {
-    let mut interpreter_config = if let Some(paths) = cross_compiling_from_cargo_env()? {
-        load_cross_compile_config(paths)?
-    } else {
-        return Ok(None);
-    };
-    fixup_config_for_abi3(&mut interpreter_config, get_abi3_version())?;
-    Ok(Some(interpreter_config))
+    let env_vars = cross_compile_env_vars();
+
+    let host = cargo_env_var("HOST").ok_or("expected HOST env var")?;
+    let target = cargo_env_var("TARGET").ok_or("expected TARGET env var")?;
+
+    let target_arch =
+        cargo_env_var("CARGO_CFG_TARGET_ARCH").ok_or("expected CARGO_CFG_TARGET_ARCH env var")?;
+    let target_vendor = cargo_env_var("CARGO_CFG_TARGET_VENDOR")
+        .ok_or("expected CARGO_CFG_TARGET_VENDOR env var")?;
+    let target_os =
+        cargo_env_var("CARGO_CFG_TARGET_OS").ok_or("expected CARGO_CFG_TARGET_OS env var")?;
+
+    if env_vars.any() {
+        let cross_config =
+            cross_compiling_from_vars(&target_arch, &target_os, &target_vendor, env_vars)?;
+        let mut interpreter_config = load_cross_compile_config(cross_config)?;
+        fixup_config_for_abi3(&mut interpreter_config, get_abi3_version())?;
+        return Ok(Some(interpreter_config));
+    }
+
+    ensure!(
+        host == target || is_not_cross_compiling(&host, &target_arch, &target_os, &target_vendor),
+        "PyO3 detected compile host {host} and build target {target}, but none of PYO3_CROSS, PYO3_CROSS_LIB_DIR \
+         or PYO3_CROSS_PYTHON_VERSION environment variables are set."
+    );
+
+    Ok(None)
 }
 
 /// Generates an interpreter config which will be hard-coded into the pyo3-build-config crate.

pyo3-build-config/src/lib.rs

@@ -70,7 +70,7 @@ pub fn get() -> &'static InterpreterConfig {
             InterpreterConfig::from_reader(Cursor::new(CONFIG_FILE))
         } else if !ABI3_CONFIG.is_empty() {
             Ok(abi3_config())
-        } else if impl_::any_cross_compiling_env_vars_set() {
+        } else if impl_::cross_compile_env_vars().any() {
             InterpreterConfig::from_path(DEFAULT_CROSS_COMPILE_CONFIG_PATH)
         } else {
             InterpreterConfig::from_reader(Cursor::new(HOST_CONFIG))