Merge pull request #2241 from ravenexp/cross-compile

pyo3-build-config: Make `PYO3_CROSS_LIB_DIR` optional

Architecture.md

@@ -188,8 +188,10 @@ Some of the functionality of `pyo3-build-config`:
     Currently we use the `extension-module` feature for this purpose. This may change in the future.
     See [#1123](https://github.com/PyO3/pyo3/pull/1123).
 - Cross-compiling configuration
-  - If `TARGET` architecture and `HOST` architecture differ, we find cross compile information
+  - If `TARGET` architecture and `HOST` architecture differ, we can find cross compile information
     from environment variables (`PYO3_CROSS_LIB_DIR` and `PYO3_CROSS_PYTHON_VERSION`) or system files.
+    When cross compiling extension modules it is often possible to make it work without any
+    additional user input.
 
 <!-- External Links -->
 

CHANGELOG.md

@@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+- Make `PYO3_CROSS_LIB_DIR` environment variable optional when cross compiling. [#2241](https://github.com/PyO3/pyo3/pull/2241)
 - Allow `#[pyo3(crate = "...", text_signature = "...")]` options to be used directly in `#[pyclass(crate = "...", text_signature = "...")]`. [#2234](https://github.com/PyO3/pyo3/pull/2234)
 - Mark `METH_FASTCALL` calling convention as limited API on Python 3.10. [#2250](https://github.com/PyO3/pyo3/pull/2250)
 

guide/src/building_and_distribution.md

@@ -226,8 +226,8 @@ After you've obtained the above, you can build a cross-compiled PyO3 module by u
 When cross-compiling, PyO3's build script cannot execute the target Python interpreter to query the configuration, so there are a few additional environment variables you may need to set:
 
 * `PYO3_CROSS`: If present this variable forces PyO3 to configure as a cross-compilation.
-* `PYO3_CROSS_LIB_DIR`: This variable must be set to the directory containing the target's libpython DSO and the associated `_sysconfigdata*.py` file for Unix-like targets, or the Python DLL import libraries for the Windows target.
+* `PYO3_CROSS_LIB_DIR`: This variable can be set to the directory containing the target's libpython DSO and the associated `_sysconfigdata*.py` file for Unix-like targets, or the Python DLL import libraries for the Windows target. This variable is only needed when the output binary must link to libpython explicitly (e.g. when targeting Windows and Android or embedding a Python interpreter), or when it is absolutely required to get the interpreter configuration from `_sysconfigdata*.py`.
-* `PYO3_CROSS_PYTHON_VERSION`: Major and minor version (e.g. 3.9) of the target Python installation. This variable is only needed if PyO3 cannot determine the version to target from `abi3-py3*` features, or if there are multiple versions of Python present in `PYO3_CROSS_LIB_DIR`.
+* `PYO3_CROSS_PYTHON_VERSION`: Major and minor version (e.g. 3.9) of the target Python installation. This variable is only needed if PyO3 cannot determine the version to target from `abi3-py3*` features, or if `PYO3_CROSS_LIB_DIR` is not set, or if there are multiple versions of Python present in `PYO3_CROSS_LIB_DIR`.
 
 An example might look like the following (assuming your target's sysroot is at `/home/pyo3/cross/sysroot` and that your target is `armv7`):
 
@@ -254,6 +254,7 @@ cargo build --target x86_64-pc-windows-gnu
 ```
 
 Any of the `abi3-py3*` features can be enabled instead of setting `PYO3_CROSS_PYTHON_VERSION` in the above examples.
+`PYO3_CROSS_LIB_DIR` can often be omitted when cross compiling extension modules for Unix and macOS targets.
 
 The following resources may also be useful for cross-compiling:
  - [github.com/japaric/rust-cross](https://github.com/japaric/rust-cross) is a primer on cross compiling Rust.

pyo3-build-config/src/impl_.rs

@@ -685,7 +685,7 @@ fn is_linking_libpython_for_target(target: &Triple) -> bool {
 #[derive(Debug, PartialEq)]
 pub struct CrossCompileConfig {
     /// The directory containing the Python library to link against.
-    pub lib_dir: PathBuf,
+    pub lib_dir: Option<PathBuf>,
 
     /// The version of the Python library to link against.
     version: Option<PythonVersion>,
@@ -745,8 +745,10 @@ impl CrossCompileConfig {
     ///
     /// The conversion can not fail because `PYO3_CROSS_LIB_DIR` variable
     /// is ensured contain a valid UTF-8 string.
-    fn lib_dir_string(&self) -> String {
+    fn lib_dir_string(&self) -> Option<String> {
-        self.lib_dir.to_str().unwrap().to_owned()
+        self.lib_dir
+            .as_ref()
+            .map(|s| s.to_str().unwrap().to_owned())
     }
 }
 
@@ -757,7 +759,7 @@ pub(crate) struct CrossCompileEnvVars {
 }
 
 impl CrossCompileEnvVars {
-    pub fn any(&self) -> bool {
+    fn any(&self) -> bool {
         self.pyo3_cross.is_some()
             || self.pyo3_cross_lib_dir.is_some()
             || self.pyo3_cross_python_version.is_some()
@@ -786,7 +788,7 @@ impl CrossCompileEnvVars {
     /// into a `PathBuf` instance.
     ///
     /// Ensures that the path is a valid UTF-8 string.
-    fn lib_dir_path(&self) -> Result<PathBuf> {
+    fn lib_dir_path(&self) -> Result<Option<PathBuf>> {
         let lib_dir = self.pyo3_cross_lib_dir.as_ref().map(PathBuf::from);
 
         if let Some(dir) = lib_dir.as_ref() {
@@ -794,11 +796,9 @@ impl CrossCompileEnvVars {
                 dir.to_str().is_some(),
                 "PYO3_CROSS_LIB_DIR variable value is not a valid UTF-8 string"
             );
-            Ok(dir.clone())
-        } else {
-            // FIXME: Relax this restriction in the future.
-            bail!("The PYO3_CROSS_LIB_DIR environment variable must be set when cross-compiling")
         }
+
+        Ok(lib_dir)
     }
 }
 
@@ -815,9 +815,9 @@ pub(crate) fn cross_compile_env_vars() -> CrossCompileEnvVars {
 /// This function relies on PyO3 cross-compiling environment variables:
 ///
 ///   * `PYO3_CROSS`: If present, forces PyO3 to configure as a cross-compilation.
-///   * `PYO3_CROSS_LIB_DIR`: Must be set to the directory containing the target's libpython DSO and
+///   * `PYO3_CROSS_LIB_DIR`: If present, must be set to the directory containing
-///   the associated `_sysconfigdata*.py` file for Unix-like targets, or the Python DLL import
+///   the target's libpython DSO and the associated `_sysconfigdata*.py` file for
-///   libraries for the Windows target.
+///   Unix-like targets, or the Python DLL import libraries for the Windows target.
 ///   * `PYO3_CROSS_PYTHON_VERSION`: Major and minor version (e.g. 3.9) of the target Python
 ///   installation. This variable is only needed if PyO3 cannnot determine the version to target
 ///   from `abi3-py3*` features, or if there are multiple versions of Python present in
@@ -1109,13 +1109,22 @@ fn ends_with(entry: &DirEntry, pat: &str) -> bool {
     name.to_string_lossy().ends_with(pat)
 }
 
-fn find_sysconfigdata(cross: &CrossCompileConfig) -> Result<PathBuf> {
+/// Finds the sysconfigdata file when the target Python library directory is set.
+///
+/// Returns `None` if the library directory is not available, and a runtime error
+/// when no or multiple sysconfigdata files are found.
+fn find_sysconfigdata(cross: &CrossCompileConfig) -> Result<Option<PathBuf>> {
     let mut sysconfig_paths = find_all_sysconfigdata(cross);
     if sysconfig_paths.is_empty() {
+        if let Some(lib_dir) = cross.lib_dir.as_ref() {
             bail!(
                 "Could not find either libpython.so or _sysconfigdata*.py in {}",
-            cross.lib_dir.display()
+                lib_dir.display()
             );
+        } else {
+            // Continue with the default configuration when PYO3_CROSS_LIB_DIR is not set.
+            return Ok(None);
+        }
     } else if sysconfig_paths.len() > 1 {
         let mut error_msg = String::from(
             "Detected multiple possible Python versions. Please set either the \
@@ -1129,7 +1138,7 @@ fn find_sysconfigdata(cross: &CrossCompileConfig) -> Result<PathBuf> {
         bail!("{}\n", error_msg);
     }
 
-    Ok(sysconfig_paths.remove(0))
+    Ok(Some(sysconfig_paths.remove(0)))
 }
 
 /// Finds `_sysconfigdata*.py` files for detected Python interpreters.
@@ -1167,8 +1176,16 @@ fn find_sysconfigdata(cross: &CrossCompileConfig) -> Result<PathBuf> {
 /// ```
 ///
 /// [1]: https://github.com/python/cpython/blob/3.5/Lib/sysconfig.py#L389
+///
+/// Returns an empty vector when the target Python library directory
+/// is not set via `PYO3_CROSS_LIB_DIR`.
 pub fn find_all_sysconfigdata(cross: &CrossCompileConfig) -> Vec<PathBuf> {
-    let sysconfig_paths = search_lib_dir(&cross.lib_dir, cross);
+    let sysconfig_paths = if let Some(lib_dir) = cross.lib_dir.as_ref() {
+        search_lib_dir(lib_dir, cross)
+    } else {
+        return Vec::new();
+    };
+
     let sysconfig_name = env_var("_PYTHON_SYSCONFIGDATA_NAME");
     let mut sysconfig_paths = sysconfig_paths
         .iter()
@@ -1267,16 +1284,28 @@ fn search_lib_dir(path: impl AsRef<Path>, cross: &CrossCompileConfig) -> Vec<Pat
 /// first find sysconfigdata file which follows the pattern [`_sysconfigdata_{abi}_{platform}_{multiarch}`][1]
 ///
 /// [1]: https://github.com/python/cpython/blob/3.8/Lib/sysconfig.py#L348
+///
+/// Returns `None` when the target Python library directory is not set.
 fn cross_compile_from_sysconfigdata(
-    cross_compile_config: CrossCompileConfig,
+    cross_compile_config: &CrossCompileConfig,
-) -> Result<InterpreterConfig> {
+) -> Result<Option<InterpreterConfig>> {
-    let sysconfigdata_path = find_sysconfigdata(&cross_compile_config)?;
+    if let Some(path) = find_sysconfigdata(cross_compile_config)? {
-    InterpreterConfig::from_sysconfigdata(&parse_sysconfigdata(sysconfigdata_path)?)
+        let data = parse_sysconfigdata(path)?;
+        let config = InterpreterConfig::from_sysconfigdata(&data)?;
+
+        Ok(Some(config))
+    } else {
+        Ok(None)
+    }
 }
 
-fn windows_hardcoded_cross_compile(
+/// Generates "default" cross compilation information for the target.
-    cross_compile_config: CrossCompileConfig,
+///
-) -> Result<InterpreterConfig> {
+/// This should work for most CPython extension modules when targeting
+/// Windows, MacOS and Linux.
+///
+/// Must be called from a PyO3 crate build script.
+fn default_cross_compile(cross_compile_config: &CrossCompileConfig) -> Result<InterpreterConfig> {
     let version = cross_compile_config
         .version
         .or_else(get_abi3_version)
@@ -1287,21 +1316,30 @@ fn windows_hardcoded_cross_compile(
 
     let abi3 = is_abi3();
     let implementation = PythonImplementation::CPython;
+
+    let lib_name = if cross_compile_config.target.operating_system == OperatingSystem::Windows {
         let mingw = cross_compile_config.target.environment == Environment::Gnu;
 
-    let lib_dir = Some(cross_compile_config.lib_dir_string());
+        Some(default_lib_name_windows(
+            version,
+            implementation,
+            abi3,
+            mingw,
+        ))
+    } else if is_linking_libpython_for_target(&cross_compile_config.target) {
+        Some(default_lib_name_unix(version, implementation, None))
+    } else {
+        None
+    };
+
+    let lib_dir = cross_compile_config.lib_dir_string();
 
     Ok(InterpreterConfig {
         implementation,
         version,
         shared: true,
         abi3,
-        lib_name: Some(default_lib_name_windows(
+        lib_name,
-            version,
-            PythonImplementation::CPython,
-            abi3,
-            mingw,
-        )),
         lib_dir,
         executable: None,
         pointer_width: None,
@@ -1311,26 +1349,39 @@ fn windows_hardcoded_cross_compile(
     })
 }
 
+/// Detects the cross compilation target interpreter configuration from all
+/// available sources (PyO3 environment variables, Python sysconfigdata, etc.).
+///
+/// Returns the "default" target interpreter configuration for Windows and
+/// when no target Python interpreter is found.
+///
+/// Must be called from a PyO3 crate build script.
 fn load_cross_compile_config(
     cross_compile_config: CrossCompileConfig,
 ) -> Result<InterpreterConfig> {
-    match cargo_env_var("CARGO_CFG_TARGET_FAMILY") {
+    // Load the defaults for Windows even when `PYO3_CROSS_LIB_DIR` is set
-        // Configure for unix platforms using the sysconfigdata file
+    // since it has no sysconfigdata files in it.
-        Some(os) if os == "unix" => cross_compile_from_sysconfigdata(cross_compile_config),
+    if cross_compile_config.target.operating_system == OperatingSystem::Windows {
-        // Use hardcoded interpreter config when targeting Windows
+        return default_cross_compile(&cross_compile_config);
-        Some(os) if os == "windows" => windows_hardcoded_cross_compile(cross_compile_config),
+    }
-        // sysconfigdata works fine on wasm/wasi
+
-        Some(os) if os == "wasm" => cross_compile_from_sysconfigdata(cross_compile_config),
+    // Try to find and parse sysconfigdata files on other targets
-        // Waiting for users to tell us what they expect on their target platform
+    // and fall back to the defaults when none are found.
-        Some(os) => bail!(
+    if let Some(config) = cross_compile_from_sysconfigdata(&cross_compile_config)? {
-            "Unknown target OS family for cross-compilation: {:?}.\n\
+        Ok(config)
-            \n\
+    } else {
-            Please set the PYO3_CONFIG_FILE environment variable to a config suitable for your \
+        let config = default_cross_compile(&cross_compile_config)?;
-            target interpreter.",
+
-            os
+        if config.lib_name.is_some() && config.lib_dir.is_none() {
-        ),
+            warn!(
-        // Unknown os family - try to do something useful
+                "The output binary will link to libpython, \
-        None => cross_compile_from_sysconfigdata(cross_compile_config),
+                but PYO3_CROSS_LIB_DIR environment variable is not set. \
+                Ensure that the target Python library directory is \
+                in the rustc native library search path."
+            );
+        }
+
+        Ok(config)
     }
 }
 
@@ -1758,13 +1809,13 @@ mod tests {
     #[test]
     fn windows_hardcoded_cross_compile() {
         let cross_config = CrossCompileConfig {
-            lib_dir: "C:\\some\\path".into(),
+            lib_dir: Some("C:\\some\\path".into()),
             version: Some(PythonVersion { major: 3, minor: 7 }),
             target: triple!("i686-pc-windows-msvc"),
         };
 
         assert_eq!(
-            super::windows_hardcoded_cross_compile(cross_config).unwrap(),
+            default_cross_compile(&cross_config).unwrap(),
             InterpreterConfig {
                 implementation: PythonImplementation::CPython,
                 version: PythonVersion { major: 3, minor: 7 },
@@ -1784,13 +1835,13 @@ mod tests {
     #[test]
     fn mingw_hardcoded_cross_compile() {
         let cross_config = CrossCompileConfig {
-            lib_dir: "/usr/lib/mingw".into(),
+            lib_dir: Some("/usr/lib/mingw".into()),
             version: Some(PythonVersion { major: 3, minor: 8 }),
             target: triple!("i686-pc-windows-gnu"),
         };
 
         assert_eq!(
-            super::windows_hardcoded_cross_compile(cross_config).unwrap(),
+            default_cross_compile(&cross_config).unwrap(),
             InterpreterConfig {
                 implementation: PythonImplementation::CPython,
                 version: PythonVersion { major: 3, minor: 8 },
@@ -1807,6 +1858,32 @@ mod tests {
         );
     }
 
+    #[test]
+    fn unix_hardcoded_cross_compile() {
+        let cross_config = CrossCompileConfig {
+            lib_dir: Some("/usr/arm64/lib".into()),
+            version: Some(PythonVersion { major: 3, minor: 9 }),
+            target: triple!("aarch64-unknown-linux-gnu"),
+        };
+
+        assert_eq!(
+            default_cross_compile(&cross_config).unwrap(),
+            InterpreterConfig {
+                implementation: PythonImplementation::CPython,
+                version: PythonVersion { major: 3, minor: 9 },
+                shared: true,
+                abi3: false,
+                lib_name: Some("python3.9".into()),
+                lib_dir: Some("/usr/arm64/lib".into()),
+                executable: None,
+                pointer_width: None,
+                build_flags: BuildFlags::default(),
+                suppress_build_script_link_lines: false,
+                extra_build_script_lines: vec![],
+            }
+        );
+    }
+
     #[test]
     fn default_lib_name_windows() {
         use PythonImplementation::*;
@@ -1989,15 +2066,15 @@ mod tests {
         };
 
         let cross = CrossCompileConfig {
-            lib_dir: lib_dir.into(),
+            lib_dir: Some(lib_dir.into()),
             version: Some(interpreter_config.version),
             target: triple!("x86_64-unknown-linux-gnu"),
         };
 
         let sysconfigdata_path = match find_sysconfigdata(&cross) {
-            Ok(path) => path,
+            Ok(Some(path)) => path,
             // Couldn't find a matching sysconfigdata; never mind!
-            Err(_) => return,
+            _ => return,
         };
         let sysconfigdata = super::parse_sysconfigdata(sysconfigdata_path).unwrap();
         let parsed_config = InterpreterConfig::from_sysconfigdata(&sysconfigdata).unwrap();

pyo3-build-config/src/lib.rs

@@ -11,7 +11,11 @@ mod errors;
 mod impl_;
 
 #[cfg(feature = "resolve-config")]
-use std::io::Cursor;
+use std::{
+    io::Cursor,
+    path::{Path, PathBuf},
+};
+
 use std::{env, process::Command};
 
 #[cfg(feature = "resolve-config")]
@@ -69,14 +73,21 @@ fn _add_extension_module_link_args(target_os: &str, mut writer: impl std::io::Wr
 pub fn get() -> &'static InterpreterConfig {
     static CONFIG: OnceCell<InterpreterConfig> = OnceCell::new();
     CONFIG.get_or_init(|| {
+        // Check if we are in a build script and cross compiling to a different target.
+        let cross_compile_config_path = resolve_cross_compile_config_path();
+        let cross_compiling = cross_compile_config_path
+            .as_ref()
+            .map(|path| path.exists())
+            .unwrap_or(false);
+
         if let Some(interpreter_config) = InterpreterConfig::from_cargo_dep_env() {
             interpreter_config
         } else if !CONFIG_FILE.is_empty() {
             InterpreterConfig::from_reader(Cursor::new(CONFIG_FILE))
         } else if !ABI3_CONFIG.is_empty() {
             Ok(abi3_config())
-        } else if impl_::cross_compile_env_vars().any() {
+        } else if cross_compiling {
-            InterpreterConfig::from_path(DEFAULT_CROSS_COMPILE_CONFIG_PATH)
+            InterpreterConfig::from_path(cross_compile_config_path.as_ref().unwrap())
         } else {
             InterpreterConfig::from_reader(Cursor::new(HOST_CONFIG))
         }
@@ -84,12 +95,6 @@ pub fn get() -> &'static InterpreterConfig {
     })
 }
 
-/// Path where PyO3's build.rs will write configuration by default.
-#[doc(hidden)]
-#[cfg(feature = "resolve-config")]
-const DEFAULT_CROSS_COMPILE_CONFIG_PATH: &str =
-    concat!(env!("OUT_DIR"), "/pyo3-cross-compile-config.txt");
-
 /// Build configuration provided by `PYO3_CONFIG_FILE`. May be empty if env var not set.
 #[doc(hidden)]
 #[cfg(feature = "resolve-config")]
@@ -107,6 +112,22 @@ const ABI3_CONFIG: &str = include_str!(concat!(env!("OUT_DIR"), "/pyo3-build-con
 #[cfg(feature = "resolve-config")]
 const HOST_CONFIG: &str = include_str!(concat!(env!("OUT_DIR"), "/pyo3-build-config.txt"));
 
+/// Returns the path where PyO3's build.rs writes its cross compile configuration.
+///
+/// The config file will be named `$OUT_DIR/<triple>/pyo3-build-config.txt`.
+///
+/// Must be called from a build script, returns `None` if not.
+#[doc(hidden)]
+#[cfg(feature = "resolve-config")]
+fn resolve_cross_compile_config_path() -> Option<PathBuf> {
+    env::var_os("TARGET").map(|target| {
+        let mut path = PathBuf::from(env!("OUT_DIR"));
+        path.push(Path::new(&target));
+        path.push("pyo3-build-config.txt");
+        path
+    })
+}
+
 #[cfg(feature = "resolve-config")]
 fn abi3_config() -> InterpreterConfig {
     let mut interpreter_config = InterpreterConfig::from_reader(Cursor::new(ABI3_CONFIG))
@@ -158,8 +179,6 @@ pub fn print_feature_cfgs() {
 pub mod pyo3_build_script_impl {
     #[cfg(feature = "resolve-config")]
     use crate::errors::{Context, Result};
-    #[cfg(feature = "resolve-config")]
-    use std::path::Path;
 
     #[cfg(feature = "resolve-config")]
     use super::*;
@@ -185,7 +204,8 @@ pub mod pyo3_build_script_impl {
             Ok(abi3_config())
         } else if let Some(interpreter_config) = make_cross_compile_config()? {
             // This is a cross compile and need to write the config file.
-            let path = Path::new(DEFAULT_CROSS_COMPILE_CONFIG_PATH);
+            let path = resolve_cross_compile_config_path()
+                .expect("resolve_interpreter_config() must be called from a build script");
             let parent_dir = path.parent().ok_or_else(|| {
                 format!(
                     "failed to resolve parent directory of config file {}",