Merge pull request #2282 from ravenexp/python3-dll-a

pyo3-build-config: Add `python3-dll-a` crate support

.github/workflows/ci.yml

@@ -271,6 +271,23 @@ jobs:
           target: aarch64-apple-darwin
           args: --release -i python3.9 --no-sdist -m examples/maturin-starter/Cargo.toml
 
+      - name: Test cross compile to Windows
+        if: ${{ matrix.platform.os == 'ubuntu-latest' && matrix.python-version == '3.8' }}
+        env:
+          XWIN_ARCH: x86_64
+        run: |
+          sudo apt-get install -y mingw-w64 llvm
+          rustup target add x86_64-pc-windows-gnu x86_64-pc-windows-msvc
+          which cargo-xwin > /dev/null || cargo install cargo-xwin
+          cargo build --manifest-path examples/maturin-starter/Cargo.toml --features abi3 --target x86_64-pc-windows-gnu
+          cargo xwin build --manifest-path examples/maturin-starter/Cargo.toml --features abi3 --target x86_64-pc-windows-msvc
+      - name: Test cross compile to Windows with maturin
+        if: ${{ matrix.platform.os == 'ubuntu-latest' && matrix.python-version == '3.8' }}
+        uses: messense/maturin-action@v1
+        with:
+          target: x86_64-pc-windows-gnu
+          args: -i python3.8 --no-sdist -m examples/maturin-starter/Cargo.toml --cargo-extra-args="--features abi3"
+
     env:
       CARGO_TERM_VERBOSE: true
       CARGO_BUILD_TARGET: ${{ matrix.platform.rust-target }}

Architecture.md

@@ -189,13 +189,19 @@ Some of the functionality of `pyo3-build-config`:
     See [#1123](https://github.com/PyO3/pyo3/pull/1123).
 - Cross-compiling configuration
   - If `TARGET` architecture and `HOST` architecture differ, we can find cross compile information
-    from environment variables (`PYO3_CROSS_LIB_DIR` and `PYO3_CROSS_PYTHON_VERSION`) or system files.
+    from environment variables (`PYO3_CROSS_LIB_DIR`, `PYO3_CROSS_PYTHON_VERSION` and
+    `PYO3_CROSS_PYTHON_IMPLEMENTATION`) or system files.
     When cross compiling extension modules it is often possible to make it work without any
     additional user input.
+  - When an experimental feature `generate-abi3-import-lib` is enabled, the `pyo3-ffi` build script can
+    generate `python3.dll` import libraries for Windows targets automatically via an external
+    [`python3-dll-a`] crate. This enables the users to cross compile abi3 extensions for Windows without
+    having to install any Windows Python libraries.
 
 <!-- External Links -->
 
 [python/c api]: https://docs.python.org/3/c-api/
+[`python3-dll-a`]: https://docs.rs/python3-dll-a/latest/python3_dll_a/
 
 <!-- Crates -->
 

CHANGELOG.md

@@ -8,6 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Added
+
+- Add an experimental `generate-abi3-import-lib` feature to auto-generate `python3.dll` import libraries for Windows. [#2282](https://github.com/PyO3/pyo3/pull/2282)
+
 ### Changed
 
 - Default to "m" ABI tag when choosing `libpython` link name for CPython 3.7 on Unix. [#2288](https://github.com/PyO3/pyo3/pull/2288)

Cargo.toml

@@ -78,6 +78,9 @@ abi3-py38 = ["abi3-py39", "pyo3-build-config/abi3-py38", "pyo3-ffi/abi3-py38"]
 abi3-py39 = ["abi3-py310", "pyo3-build-config/abi3-py39", "pyo3-ffi/abi3-py39"]
 abi3-py310 = ["abi3", "pyo3-build-config/abi3-py310", "pyo3-ffi/abi3-py310"]
 
+# Automatically generates `python3.dll` import libraries for Windows targets.
+generate-abi3-import-lib = ["pyo3-build-config/python3-dll-a"]
+
 # Changes `Python::with_gil` and `Python::acquire_gil` to automatically initialize the
 # Python interpreter if needed.
 auto-initialize = []

examples/maturin-starter/Cargo.toml

@@ -10,4 +10,7 @@ crate-type = ["cdylib"]
 [dependencies]
 pyo3 = { path = "../../", features = ["extension-module"] }
 
+[features]
+abi3 = ["pyo3/abi3-py37", "pyo3/generate-abi3-import-lib"]
+
 [workspace]

guide/src/building_and_distribution.md

@@ -152,10 +152,20 @@ As your extension module may be run with multiple different Python versions you
 
 PyO3 is only able to link your extension module to api3 version up to and including your host Python version. E.g., if you set `abi3-py38` and try to compile the crate with a host of Python 3.7, the build will fail.
 
-As an advanced feature, you can build PyO3 wheel without calling Python interpreter with the environment variable `PYO3_NO_PYTHON` set. On unix systems this works unconditionally; on Windows you must also set the `RUSTFLAGS` evironment variable to contain `-L native=/path/to/python/libs` so that the linker can find `python3.lib`.
-
 > Note: If you set more that one of these api version feature flags the lowest version always wins. For example, with both `abi3-py37` and `abi3-py38` set, PyO3 would build a wheel which supports Python 3.7 and up.
 
+#### Building `abi3` extensions without a Python interpreter
+
+As an advanced feature, you can build PyO3 wheel without calling Python interpreter with the environment variable `PYO3_NO_PYTHON` set.
+On Unix-like systems this works unconditionally; on Windows you must also set the `RUSTFLAGS` environment variable
+to contain `-L native=/path/to/python/libs` so that the linker can find `python3.lib`.
+
+If the `python3.dll` import library is not available, an experimental `generate-abi3-import-lib` crate
+feature may be enabled, and the required library will be created and used by PyO3 automatically.
+
+*Note*: MSVC targets require LLVM binutils (`llvm-dlltool`) to be available in `PATH` for
+the automatic import library generation feature to work.
+
 #### Missing features
 
 Due to limitations in the Python API, there are a few `pyo3` features that do
@@ -218,8 +228,8 @@ Thanks to Rust's great cross-compilation support, cross-compiling using PyO3 is
 
 * A toolchain for your target.
 * The appropriate options in your Cargo `.config` for the platform you're targeting and the toolchain you are using.
-* A Python interpreter that's already been compiled for your target.
-* A Python interpreter that is built for your host and available through the `PATH` or setting the [`PYO3_PYTHON`](#python-version) variable.
+* A Python interpreter that's already been compiled for your target (optional when building "abi3" extension modules).
+* A Python interpreter that is built for your host and available through the `PATH` or setting the [`PYO3_PYTHON`](#python-version) variable (optional when building "abi3" extension modules).
 
 After you've obtained the above, you can build a cross-compiled PyO3 module by using Cargo's `--target` flag. PyO3's build script will detect that you are attempting a cross-compile based on your host machine and the desired target.
 
@@ -230,6 +240,14 @@ When cross-compiling, PyO3's build script cannot execute the target Python inter
 * `PYO3_CROSS_PYTHON_VERSION`: Major and minor version (e.g. 3.9) of the target Python installation. This variable is only needed if PyO3 cannot determine the version to target from `abi3-py3*` features, or if `PYO3_CROSS_LIB_DIR` is not set, or if there are multiple versions of Python present in `PYO3_CROSS_LIB_DIR`.
 * `PYO3_CROSS_PYTHON_IMPLEMENTATION`: Python implementation name ("CPython" or "PyPy") of the target Python installation. CPython is assumed by default when this variable is not set, unless `PYO3_CROSS_LIB_DIR` is set for a Unix-like target and PyO3 can get the interpreter configuration from `_sysconfigdata*.py`.
 
+An experimental `pyo3` crate feature `generate-abi3-import-lib` enables the user to cross-compile
+"abi3" extension modules for Windows targets without setting the `PYO3_CROSS_LIB_DIR` environment
+variable or providing any Windows Python library files. It uses an external [`python3-dll-a`] crate
+to generate import libraries for the Stable ABI Python DLL for MinGW-w64 and MSVC compile targets.
+*Note*: MSVC targets require LLVM binutils to be available on the host system.
+More specifically, `python3-dll-a` requires `llvm-dlltool` executable to be present in `PATH` when
+targeting `*-pc-windows-msvc`.
+
 An example might look like the following (assuming your target's sysroot is at `/home/pyo3/cross/sysroot` and that your target is `armv7`):
 
 ```sh
@@ -255,7 +273,10 @@ cargo build --target x86_64-pc-windows-gnu
 ```
 
 Any of the `abi3-py3*` features can be enabled instead of setting `PYO3_CROSS_PYTHON_VERSION` in the above examples.
-`PYO3_CROSS_LIB_DIR` can often be omitted when cross compiling extension modules for Unix and macOS targets.
+
+`PYO3_CROSS_LIB_DIR` can often be omitted when cross compiling extension modules for Unix and macOS targets,
+or when cross compiling "abi3" extension modules for Windows and the experimental `generate-abi3-import-lib`
+crate feature is enabled.
 
 The following resources may also be useful for cross-compiling:
  - [github.com/japaric/rust-cross](https://github.com/japaric/rust-cross) is a primer on cross compiling Rust.
@@ -267,3 +288,4 @@ The following resources may also be useful for cross-compiling:
 [`maturin`]: https://github.com/PyO3/maturin
 [`setuptools-rust`]: https://github.com/PyO3/setuptools-rust
 [PyOxidizer]: https://github.com/indygreg/PyOxidizer
+[`python3-dll-a`]: https://docs.rs/python3-dll-a/latest/python3_dll_a/

guide/src/features.md

@@ -30,6 +30,17 @@ These features are extensions of the `abi3` feature to specify the exact minimum
 
 See the [building and distribution](building_and_distribution.md#minimum-python-version-for-abi3) section for further detail.
 
+### `generate-abi3-import-lib`
+
+This experimental feature is used to generate import libraries for the Stable ABI Python DLL
+for MinGW-w64 and MSVC (cross-)compile targets.
+
+Enabling it allows to (cross-)compile `abi3` extension modules to any Windows targets
+without having to install the Windows Python distribution files for the target.
+
+See the [building and distribution](building_and_distribution.md#building-abi3-extensions-without-a-python-interpreter)
+section for further detail.
+
 ## Features for embedding Python in Rust
 
 ### `auto-initialize`

pyo3-build-config/Cargo.toml

@@ -12,9 +12,11 @@ edition = "2018"
 
 [dependencies]
 once_cell = "1"
+python3-dll-a = { version = "0.2", optional = true }
 target-lexicon = "0.12"
 
 [build-dependencies]
+python3-dll-a = { version = "0.2", optional = true }
 target-lexicon = "0.12"
 
 [features]

pyo3-build-config/src/abi3_import_lib.rs

@@ -0,0 +1,48 @@
+//! Optional `python3.dll` import library generator for Windows
+
+use std::env;
+use std::path::PathBuf;
+
+use python3_dll_a::generate_implib_for_target;
+
+use crate::errors::{Context, Result};
+
+use super::{Architecture, OperatingSystem, Triple};
+
+/// Generates the `python3.dll` import library for Windows targets.
+///
+/// Places the generated import library into the build script output directory
+/// and returns the full library directory path.
+///
+/// Does nothing if the target OS is not Windows.
+pub(super) fn generate_abi3_import_lib(target: &Triple) -> Result<Option<String>> {
+    if target.operating_system != OperatingSystem::Windows {
+        return Ok(None);
+    }
+
+    let out_dir = env::var_os("OUT_DIR")
+        .expect("generate_abi3_import_lib() must be called from a build script");
+
+    // Put the newly created import library into the build script output directory.
+    let mut out_lib_dir = PathBuf::from(out_dir);
+    out_lib_dir.push("lib");
+
+    // Convert `Architecture` enum to rustc `target_arch` option format.
+    let arch = match target.architecture {
+        // i686, i586, etc.
+        Architecture::X86_32(_) => "x86".to_string(),
+        other => other.to_string(),
+    };
+
+    let env = target.environment.to_string();
+
+    generate_implib_for_target(&out_lib_dir, &arch, &env)
+        .context("failed to generate python3.dll import library")?;
+
+    let out_lib_dir_string = out_lib_dir
+        .to_str()
+        .ok_or("build directory is not a valid UTF-8 string")?
+        .to_owned();
+
+    Ok(Some(out_lib_dir_string))
+}

pyo3-build-config/src/impl_.rs

@@ -1,3 +1,11 @@
+//! Main implementation module included in both the `pyo3-build-config` library crate
+//! and its build script.
+
+// Optional python3.dll import library generator for Windows
+#[cfg(feature = "python3-dll-a")]
+#[path = "abi3_import_lib.rs"]
+mod abi3_import_lib;
+
 use std::{
     collections::{HashMap, HashSet},
     convert::AsRef,
@@ -1352,6 +1360,7 @@ fn cross_compile_from_sysconfigdata(
 /// Windows, macOS and Linux.
 ///
 /// Must be called from a PyO3 crate build script.
+#[allow(unused_mut)]
 fn default_cross_compile(cross_compile_config: &CrossCompileConfig) -> Result<InterpreterConfig> {
     let version = cross_compile_config
         .version
@@ -1381,7 +1390,13 @@ fn default_cross_compile(cross_compile_config: &CrossCompileConfig) -> Result<In
         None
     };
 
-    let lib_dir = cross_compile_config.lib_dir_string();
+    let mut lib_dir = cross_compile_config.lib_dir_string();
+
+    // Auto generate python3.dll import libraries for Windows targets.
+    #[cfg(feature = "python3-dll-a")]
+    if abi3 && lib_dir.is_none() {
+        lib_dir = self::abi3_import_lib::generate_abi3_import_lib(&cross_compile_config.target)?;
+    }
 
     Ok(InterpreterConfig {
         implementation,
@@ -1649,7 +1664,7 @@ pub fn make_cross_compile_config() -> Result<Option<InterpreterConfig>> {
 
 /// Generates an interpreter config which will be hard-coded into the pyo3-build-config crate.
 /// Only used by `pyo3-build-config` build script.
-#[allow(dead_code)]
+#[allow(dead_code, unused_mut)]
 pub fn make_interpreter_config() -> Result<InterpreterConfig> {
     let abi3_version = get_abi3_version();
 
@@ -1659,7 +1674,15 @@ pub fn make_interpreter_config() -> Result<InterpreterConfig> {
         Ok(interpreter_config)
     } else if let Some(version) = abi3_version {
         let host = Triple::host();
-        Ok(default_abi3_config(&host, version))
+        let mut interpreter_config = default_abi3_config(&host, version);
+
+        // Auto generate python3.dll import libraries for Windows targets.
+        #[cfg(feature = "python3-dll-a")]
+        {
+            interpreter_config.lib_dir = self::abi3_import_lib::generate_abi3_import_lib(&host)?;
+        }
+
+        Ok(interpreter_config)
     } else {
         bail!("An abi3-py3* feature must be specified when compiling without a Python interpreter.")
     }