luxolus
/
pyo3
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

pyo3-build-config: many tidy ups

This commit is contained in:
David Hewitt 2021-08-05 22:43:26 +01:00
parent 9507979d93
commit 49387e9a70
6 changed files with 448 additions and 359 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

151
build.rs
View File

@ -1,9 +1,11 @@
use std::{env, io::Cursor, path::Path, process::Command};
use std::{env, process::Command};
use pyo3_build_config::{
bail, cargo_env_var, ensure, env_var,
errors::{Context, Result},
make_cross_compile_config, InterpreterConfig, PythonVersion,
bail, ensure,
pyo3_build_script_impl::{
cargo_env_var, env_var, errors::Result, resolve_interpreter_config, InterpreterConfig,
PythonVersion,
},
};
/// Minimum Python version PyO3 supports.
@ -20,24 +22,54 @@ fn ensure_python_version(interpreter_config: &InterpreterConfig) -> Result<()> {
Ok(())
}
fn ensure_target_pointer_width(pointer_width: u32) -> Result<()> {
// Try to check whether the target architecture matches the python library
let rust_target = match cargo_env_var("CARGO_CFG_TARGET_POINTER_WIDTH")
.unwrap()
.as_str()
{
"64" => 64,
"32" => 32,
x => bail!("unexpected Rust target pointer width: {}", x),
};
fn ensure_target_pointer_width(interpreter_config: &InterpreterConfig) -> Result<()> {
if let Some(pointer_width) = interpreter_config.pointer_width {
// Try to check whether the target architecture matches the python library
let rust_target = match cargo_env_var("CARGO_CFG_TARGET_POINTER_WIDTH")
.unwrap()
.as_str()
{
"64" => 64,
"32" => 32,
x => bail!("unexpected Rust target pointer width: {}", x),
};
ensure!(
rust_target == pointer_width,
"your Rust target architecture ({}-bit) does not match your python interpreter ({}-bit)",
rust_target,
pointer_width
);
ensure!(
rust_target == pointer_width,
"your Rust target architecture ({}-bit) does not match your python interpreter ({}-bit)",
rust_target,
pointer_width
);
}
Ok(())
}
fn ensure_auto_initialize_ok(interpreter_config: &InterpreterConfig) -> Result<()> {
if cargo_env_var("CARGO_FEATURE_AUTO_INITIALIZE").is_some() {
if !interpreter_config.shared {
bail!(
"The `auto-initialize` feature is enabled, but your python installation only supports \
embedding the Python interpreter statically. If you are attempting to run tests, or a \
binary which is okay to link dynamically, install a Python distribution which ships \
with the Python shared library.\n\
\n\
Embedding the Python interpreter statically does not yet have first-class support in \
PyO3. If you are sure you intend to do this, disable the `auto-initialize` feature.\n\
\n\
For more information, see \
https://pyo3.rs/v{pyo3_version}/\
building_and_distribution.html#embedding-python-in-rust",
pyo3_version = env::var("CARGO_PKG_VERSION").unwrap()
);
}
// TODO: PYO3_CI env is a hack to workaround CI with PyPy, where the `dev-dependencies`
// currently cause `auto-initialize` to be enabled in CI.
// Once MSRV is 1.51 or higher, use cargo's `resolver = "2"` instead.
if interpreter_config.implementation.is_pypy() && env::var_os("PYO3_CI").is_none() {
bail!("the `auto-initialize` feature is not supported with PyPy");
}
}
Ok(())
}
@ -70,89 +102,36 @@ fn emit_cargo_configuration(interpreter_config: &InterpreterConfig) -> Result<()
} else {
""
},
lib_name = interpreter_config
.lib_name
.as_ref()
.ok_or("config does not contain lib_name")?,
lib_name = interpreter_config.lib_name.as_ref().ok_or(
"attempted to link to Python shared library but config does not contain lib_name"
)?,
);
if let Some(lib_dir) = &interpreter_config.lib_dir {
println!("cargo:rustc-link-search=native={}", lib_dir);
}
}
if cargo_env_var("CARGO_FEATURE_AUTO_INITIALIZE").is_some() {
if !interpreter_config.shared {
bail!(
"The `auto-initialize` feature is enabled, but your python installation only supports \
embedding the Python interpreter statically. If you are attempting to run tests, or a \
binary which is okay to link dynamically, install a Python distribution which ships \
with the Python shared library.\n\
\n\
Embedding the Python interpreter statically does not yet have first-class support in \
PyO3. If you are sure you intend to do this, disable the `auto-initialize` feature.\n\
\n\
For more information, see \
https://pyo3.rs/v{pyo3_version}/\
building_and_distribution.html#embedding-python-in-rust",
pyo3_version = env::var("CARGO_PKG_VERSION").unwrap()
);
}
// TODO: PYO3_CI env is a hack to workaround CI with PyPy, where the `dev-dependencies`
// currently cause `auto-initialize` to be enabled in CI.
// Once MSRV is 1.51 or higher, use cargo's `resolver = "2"` instead.
if interpreter_config.is_pypy() && env::var_os("PYO3_CI").is_none() {
bail!("the `auto-initialize` feature is not supported with PyPy");
}
}
Ok(())
}
/// Generates the interpreter config suitable for the host / target / cross-compilation at hand.
/// Prepares the PyO3 crate for compilation.
///
/// The result is written to pyo3_build_config::PATH, which downstream scripts can read from
/// (including `pyo3-macros-backend` during macro expansion).
/// This loads the config from pyo3-build-config and then makes some additional checks to improve UX
/// for users.
///
/// Emits the cargo configuration based on this config as well as a few checks of the Rust compiler
/// version to enable features which aren't supported on MSRV.
fn configure_pyo3() -> Result<()> {
let interpreter_config = if let Some(path) = env_var("PYO3_CONFIG_FILE") {
let path = Path::new(&path);
// This is necessary because the compilations that access PYO3_CONFIG_FILE (build scripts,
// proc macros) have many different working directories, so a relative path is no good.
ensure!(path.is_absolute(), "PYO3_CONFIG_FILE must be an absolute path");
println!("cargo:rerun-if-changed={}", path.display());
InterpreterConfig::from_path(path)?
} else if let Some(interpreter_config) = make_cross_compile_config()? {
// This is a cross compile, need to write the config file.
let path = Path::new(&pyo3_build_config::DEFAULT_CROSS_COMPILE_CONFIG_PATH);
let parent_dir = path.parent().ok_or_else(|| {
format!(
"failed to resolve parent directory of config file {}",
path.display()
)
})?;
std::fs::create_dir_all(&parent_dir).with_context(|| {
format!(
"failed to create config file directory {}",
parent_dir.display()
)
})?;
interpreter_config
.to_writer(&mut std::fs::File::create(&path).with_context(|| {
format!("failed to create config file at {}", path.display())
})?)?;
interpreter_config
} else {
InterpreterConfig::from_reader(Cursor::new(pyo3_build_config::HOST_CONFIG))?
};
let interpreter_config = resolve_interpreter_config()?;
if env_var("PYO3_PRINT_CONFIG").map_or(false, |os_str| os_str == "1") {
print_config_and_exit(&interpreter_config);
}
ensure_python_version(&interpreter_config)?;
if let Some(pointer_width) = interpreter_config.pointer_width {
ensure_target_pointer_width(pointer_width)?;
}
ensure_target_pointer_width(&interpreter_config)?;
ensure_auto_initialize_ok(&interpreter_config)?;
emit_cargo_configuration(&interpreter_config)?;
interpreter_config.emit_pyo3_cfgs();

23
guide/src/building_and_distribution.md
View File

@ -30,23 +30,22 @@ Caused by:
cargo:rerun-if-env-changed=PYO3_CROSS
cargo:rerun-if-env-changed=PYO3_CROSS_LIB_DIR
cargo:rerun-if-env-changed=PYO3_CROSS_PYTHON_VERSION
cargo:rerun-if-env-changed=PYO3_PYTHON
cargo:rerun-if-env-changed=VIRTUAL_ENV
cargo:rerun-if-env-changed=CONDA_PREFIX
cargo:rerun-if-env-changed=PATH
cargo:rerun-if-env-changed=PYO3_PRINT_CONFIG
-- PYO3_PRINT_CONFIG=1 is set, printing configuration and halting compile --
implementation: CPython
interpreter version: 3.8
interpreter path: Some("/usr/bin/python")
libdir: Some("/usr/lib")
shared: true
base prefix: Some("/usr")
ld_version: Some("3.8")
pointer width: Some(8)
implementation=CPython
version=3.8
shared=true
abi3=false
lib_name=python3.8
lib_dir=/usr/lib
executable=/usr/bin/python
pointer_width=64
build_flags=WITH_THREAD
```
> Note: if you safe the output config to a file, it is possible to manually override the and feed it back into PyO3 using the `PYO3_CONFIG_FILE` env var. For now, this is an advanced feature that should not be needed for most users. The format of the config file and its contents are deliberately unstable and undocumented. If you have a production use-case for this config file, please file an issue and help us stabilize it!
## Building Python extension modules
Python extension modules need to be compiled differently depending on the OS (and architecture) that they are being compiled for. As well as multiple OSes (and architectures), there are also many different Python versions which are actively supported. Packages uploaded to [PyPI](https://pypi.org/) usually want to upload prebuilt "wheels" covering many OS/arch/version combinations so that users on all these different platforms don't have to compile the package themselves. Package vendors can opt-in to the "abi3" limited Python API which allows their wheels to be used on multiple Python versions, reducing the number of wheels they need to compile, but restricts the functionality they can use.

84
pyo3-build-config/build.rs
View File

@ -11,21 +11,85 @@ mod errors;
use std::{env, path::Path};
use errors::{Result, Context};
use errors::{Context, Result};
use impl_::{
env_var, get_abi3_version, make_interpreter_config, BuildFlags, InterpreterConfig,
PythonImplementation,
};
fn generate_build_config() -> Result<()> {
// Create new interpreter config and write it to the default location
let interpreter_config = impl_::make_interpreter_config()?;
fn configure(interpreter_config: Option<InterpreterConfig>, name: &str) -> Result<bool> {
let target = Path::new(&env::var_os("OUT_DIR").unwrap()).join(name);
if let Some(config) = interpreter_config {
config
.to_writer(&mut std::fs::File::create(&target).with_context(|| {
format!("failed to write config file at {}", target.display())
})?)?;
Ok(true)
} else {
std::fs::File::create(&target)
.with_context(|| format!("failed to create new file at {}", target.display()))?;
Ok(false)
}
}
let path = Path::new(&env::var_os("OUT_DIR").unwrap()).join("pyo3-build-config.txt");
interpreter_config
.to_writer(&mut std::fs::File::create(&path).with_context(|| {
format!("failed to create config file at {}", path.display())
})?)
/// If PYO3_CONFIG_FILE is set, copy it into the crate.
fn config_file() -> Result<Option<InterpreterConfig>> {
if let Some(path) = env_var("PYO3_CONFIG_FILE") {
let path = Path::new(&path);
println!("cargo:rerun-if-changed={}", path.display());
// Absolute path is necessary because this build script is run with a cwd different to the
// original `cargo build` instruction.
ensure!(
path.is_absolute(),
"PYO3_CONFIG_FILE must be an absolute path"
);
let interpreter_config = InterpreterConfig::from_path(path)
.context("failed to parse contents of PYO3_CONFIG_FILE")?;
Ok(Some(interpreter_config))
} else {
Ok(None)
}
}
/// If PYO3_NO_PYTHON is set with abi3, use standard abi3 settings.
pub fn abi3_config() -> Option<InterpreterConfig> {
if let Some(version) = get_abi3_version() {
if env_var("PYO3_NO_PYTHON").is_some() {
return Some(InterpreterConfig {
version,
// NB PyPy doesn't support abi3 yet
implementation: PythonImplementation::CPython,
abi3: true,
lib_name: None,
lib_dir: None,
build_flags: BuildFlags::abi3(),
pointer_width: None,
executable: None,
shared: true,
});
}
}
None
}
fn generate_build_configs() -> Result<()> {
let mut configured = false;
configured |= configure(config_file()?, "pyo3-build-config-file.txt")?;
configured |= configure(abi3_config(), "pyo3-build-config-abi3.txt")?;
if configured {
// Don't bother trying to find an interpreter on the host system if at least one of the
// config file or abi3 settings are present
configure(None, "pyo3-build-config.txt")?;
} else {
configure(Some(make_interpreter_config()?), "pyo3-build-config.txt")?;
}
Ok(())
}
fn main() {
if let Err(e) = generate_build_config() {
if let Err(e) = generate_build_configs() {
eprintln!("error: {}", e.report());
std::process::exit(1)
}

5
pyo3-build-config/src/errors.rs
View File

@ -1,5 +1,6 @@
/// A simple macro for returning an error. Resembles anyhow::bail.
#[macro_export]
#[doc(hidden)]
macro_rules! bail {
($msg: expr) => { return Err($msg.into()); };
($fmt: literal $($args: tt)+) => { return Err(format!($fmt $($args)+).into()); };
@ -7,12 +8,14 @@ macro_rules! bail {
/// A simple macro for checking a condition. Resembles anyhow::ensure.
#[macro_export]
#[doc(hidden)]
macro_rules! ensure {
($condition:expr, $($args: tt)+) => { if !($condition) { bail!($($args)+) } };
}
/// Show warning. If needed, please extend this macro to support arguments.
#[macro_export]
#[doc(hidden)]
macro_rules! warn {
($msg: literal) => {
println!(concat!("cargo:warning=", $msg));
@ -27,7 +30,7 @@ pub struct Error {
}
/// Error report inspired by
/// https://blog.rust-lang.org/inside-rust/2021/07/01/What-the-error-handling-project-group-is-working-towards.html#2-error-reporter
/// <https://blog.rust-lang.org/inside-rust/2021/07/01/What-the-error-handling-project-group-is-working-towards.html#2-error-reporter>
pub struct ErrorReport<'a>(&'a Error);
impl Error {

395
pyo3-build-config/src/impl_.rs
View File

@ -38,9 +38,11 @@ pub fn env_var(var: &str) -> Option<OsString> {
/// Configuration needed by PyO3 to build for the correct Python implementation.
///
/// Usually this is queried directly from the Python interpreter. When the `PYO3_NO_PYTHON` variable
/// is set, or during cross compile situations, then alternative strategies are used to populate
/// this type.
/// Usually this is queried directly from the Python interpreter, or overridden using the
/// `PYO3_CONFIG_FILE` environment variable.
///
/// When the `PYO3_NO_PYTHON` variable is set, or during cross compile situations, then alternative
/// strategies are used to populate this type.
#[cfg_attr(test, derive(Debug, PartialEq))]
pub struct InterpreterConfig {
pub implementation: PythonImplementation,
@ -55,6 +57,7 @@ pub struct InterpreterConfig {
}
impl InterpreterConfig {
#[doc(hidden)]
pub fn emit_pyo3_cfgs(&self) {
// This should have been checked during pyo3-build-config build time.
assert!(self.version >= MINIMUM_SUPPORTED_VERSION);
@ -66,7 +69,7 @@ impl InterpreterConfig {
println!("cargo:rustc-cfg=Py_LIMITED_API");
}
if self.is_pypy() {
if self.implementation.is_pypy() {
println!("cargo:rustc-cfg=PyPy");
if self.abi3 {
warn!(
@ -81,20 +84,119 @@ impl InterpreterConfig {
}
}
pub fn is_pypy(&self) -> bool {
self.implementation == PythonImplementation::PyPy
#[doc(hidden)]
pub fn from_interpreter(interpreter: impl AsRef<Path>) -> Result<Self> {
let script = r#"
# Allow the script to run on Python 2, so that nicer error can be printed later.
from __future__ import print_function
import os.path
import platform
import struct
import sys
from sysconfig import get_config_var
PYPY = platform.python_implementation() == "PyPy"
# sys.base_prefix is missing on Python versions older than 3.3; this allows the script to continue
# so that the version mismatch can be reported in a nicer way later.
base_prefix = getattr(sys, "base_prefix", None)
if base_prefix:
# Anaconda based python distributions have a static python executable, but include
# the shared library. Use the shared library for embedding to avoid rust trying to
# LTO the static library (and failing with newer gcc's, because it is old).
ANACONDA = os.path.exists(os.path.join(base_prefix, "conda-meta"))
else:
ANACONDA = False
def print_if_set(varname, value):
if value is not None:
print(varname, value)
# Windows always uses shared linking
WINDOWS = hasattr(platform, "win32_ver")
# macOS framework packages use shared linking
FRAMEWORK = bool(get_config_var("PYTHONFRAMEWORK"))
# unix-style shared library enabled
SHARED = bool(get_config_var("Py_ENABLE_SHARED"))
print("implementation", platform.python_implementation())
print("version_major", sys.version_info[0])
print("version_minor", sys.version_info[1])
print("shared", PYPY or ANACONDA or WINDOWS or FRAMEWORK or SHARED)
print_if_set("ld_version", get_config_var("LDVERSION"))
print_if_set("libdir", get_config_var("LIBDIR"))
print_if_set("base_prefix", base_prefix)
print("executable", sys.executable)
print("calcsize_pointer", struct.calcsize("P"))
"#;
let output = run_python_script(interpreter.as_ref(), script)?;
let map: HashMap<String, String> = parse_script_output(&output);
let shared = map["shared"].as_str() == "True";
let version = PythonVersion {
major: map["version_major"]
.parse()
.context("failed to parse major version")?,
minor: map["version_minor"]
.parse()
.context("failed to parse minor version")?,
};
let abi3 = is_abi3();
let implementation = map["implementation"].parse()?;
let lib_name = if cfg!(windows) {
default_lib_name_windows(
&version,
abi3,
&cargo_env_var("CARGO_CFG_TARGET_ENV").unwrap(),
)
} else {
default_lib_name_unix(
&version,
implementation,
map.get("ld_version").map(String::as_str),
)?
};
let lib_dir = if cfg!(windows) {
map.get("base_prefix")
.map(|base_prefix| format!("{}\\libs", base_prefix))
} else {
map.get("libdir").cloned()
};
// The reason we don't use platform.architecture() here is that it's not
// reliable on macOS. See https://stackoverflow.com/a/1405971/823869.
// Similarly, sys.maxsize is not reliable on Windows. See
// https://stackoverflow.com/questions/1405913/how-do-i-determine-if-my-python-shell-is-executing-in-32bit-or-64bit-mode-on-os/1405971#comment6209952_1405971
// and https://stackoverflow.com/a/3411134/823869.
let calcsize_pointer: u32 = map["calcsize_pointer"]
.parse()
.context("failed to parse calcsize_pointer")?;
Ok(InterpreterConfig {
version,
implementation,
shared,
abi3,
lib_name: Some(lib_name),
lib_dir,
executable: map.get("executable").cloned(),
pointer_width: Some(calcsize_pointer * 8),
build_flags: BuildFlags::from_interpreter(interpreter)?.fixup(version, implementation),
})
}
#[doc(hidden)]
pub fn from_path(path: impl AsRef<Path>) -> Result<Self> {
let path = path.as_ref();
let config_file =
std::fs::File::open(path).with_context(|| {
format!(
"failed to open PyO3 config file at {}",
path.display()
)
})?;
let config_file = std::fs::File::open(path)
.with_context(|| format!("failed to open PyO3 config file at {}", path.display()))?;
let reader = std::io::BufReader::new(config_file);
InterpreterConfig::from_reader(reader)
}
@ -255,6 +357,13 @@ pub enum PythonImplementation {
PyPy,
}
impl PythonImplementation {
#[doc(hidden)]
pub fn is_pypy(self) -> bool {
self == PythonImplementation::PyPy
}
}
impl Display for PythonImplementation {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
@ -312,7 +421,7 @@ impl GetPrimitive for HashMap<String, String> {
struct CrossCompileConfig {
lib_dir: PathBuf,
version: Option<String>,
version: Option<PythonVersion>,
os: String,
arch: String,
}
@ -377,10 +486,12 @@ fn cross_compiling() -> Result<Option<CrossCompileConfig>> {
arch: target_arch.unwrap(),
version: cross_python_version
.map(|os_string| {
os_string
let utf8_str = os_string
.to_str()
.ok_or("PYO3_CROSS_PYTHON_VERSION is not valid utf-8.")
.map(str::to_owned)
.ok_or("PYO3_CROSS_PYTHON_VERSION is not valid utf-8.")?;
utf8_str
.parse()
.context("failed to parse PYO3_CROSS_PYTHON_VERSION")
})
.transpose()?,
}))
@ -462,7 +573,7 @@ impl BuildFlags {
/// Examine python's compile flags to pass to cfg by launching
/// the interpreter and printing variables of interest from
/// sysconfig.get_config_vars.
fn from_interpreter(interpreter: &Path) -> Result<Self> {
fn from_interpreter(interpreter: impl AsRef<Path>) -> Result<Self> {
// If we're on a Windows host, then Python won't have any useful config vars
if cfg!(windows) {
return Ok(Self::windows_hardcoded());
@ -475,7 +586,7 @@ impl BuildFlags {
script.push_str(&format!("print(config.get('{}', '0'))\n", k));
}
let stdout = run_python_script(interpreter, &script)?;
let stdout = run_python_script(interpreter.as_ref(), &script)?;
let split_stdout: Vec<&str> = stdout.trim_end().lines().collect();
ensure!(
split_stdout.len() == BuildFlags::ALL.len(),
@ -500,7 +611,7 @@ impl BuildFlags {
Self(flags)
}
fn abi3() -> Self {
pub fn abi3() -> Self {
let mut flags = HashSet::new();
flags.insert(BuildFlag::WITH_THREAD);
Self(flags)
@ -776,26 +887,8 @@ fn load_cross_compile_from_sysconfigdata(
fn windows_hardcoded_cross_compile(
cross_compile_config: CrossCompileConfig,
) -> Result<InterpreterConfig> {
let (major, minor) = if let Some(version) = cross_compile_config.version {
let mut parts = version.split('.');
match (
parts.next().and_then(|major| major.parse().ok()),
parts.next().and_then(|minor| minor.parse().ok()),
parts.next(),
) {
(Some(major), Some(minor), None) => (major, minor),
_ => bail!(
"Expected major.minor version (e.g. 3.9) for PYO3_CROSS_PYTHON_VERSION, got `{}`",
version
),
}
} else if let Some(minor_version) = get_abi3_minor_version() {
(3, minor_version)
} else {
bail!("PYO3_CROSS_PYTHON_VERSION or an abi3-py3* feature must be specified when cross-compiling for Windows.")
};
let version = PythonVersion { major, minor };
let version = cross_compile_config.version.or_else(get_abi3_version)
.ok_or("PYO3_CROSS_PYTHON_VERSION or an abi3-py3* feature must be specified when cross-compiling for Windows.")?;
Ok(InterpreterConfig {
implementation: PythonImplementation::CPython,
@ -810,7 +903,9 @@ fn windows_hardcoded_cross_compile(
})
}
fn load_cross_compile_info(cross_compile_config: CrossCompileConfig) -> Result<InterpreterConfig> {
fn load_cross_compile_config(
cross_compile_config: CrossCompileConfig,
) -> Result<InterpreterConfig> {
match cargo_env_var("CARGO_CFG_TARGET_FAMILY") {
// Configure for unix platforms using the sysconfigdata file
Some(os) if os == "unix" => load_cross_compile_from_sysconfigdata(cross_compile_config),
@ -820,7 +915,10 @@ fn load_cross_compile_info(cross_compile_config: CrossCompileConfig) -> Result<I
Some(os) if os == "wasm" => load_cross_compile_from_sysconfigdata(cross_compile_config),
// Waiting for users to tell us what they expect on their target platform
Some(os) => bail!(
"Unsupported target OS family for cross-compilation: {:?}",
"Unknown target OS family for cross-compilation: {:?}.\n\
\n\
Please set the PYO3_CONFIG_FILE environment variable to a config suitable for your \
target interpreter.",
os
),
// Unknown os family - try to do something useful
@ -903,11 +1001,13 @@ fn get_venv_path() -> Option<PathBuf> {
}
}
/// Attempts to locate a python interpreter. Locations are checked in the order listed:
/// 1. If `PYO3_PYTHON` is set, this intepreter is used.
/// 2. If in a virtualenv, that environment's interpreter is used.
/// 3. `python`, if this is functional a Python 3.x interpreter
/// 4. `python3`, as above
/// Attempts to locate a python interpreter.
///
/// Locations are checked in the order listed:
/// 1. If `PYO3_PYTHON` is set, this intepreter is used.
/// 2. If in a virtualenv, that environment's interpreter is used.
/// 3. `python`, if this is functional a Python 3.x interpreter
/// 4. `python3`, as above
pub fn find_interpreter() -> Result<PathBuf> {
if let Some(exe) = env_var("PYO3_PYTHON") {
Ok(exe.into())
@ -936,187 +1036,52 @@ pub fn find_interpreter() -> Result<PathBuf> {
}
}
/// Extract compilation vars from the specified interpreter.
pub fn get_config_from_interpreter(interpreter: &Path) -> Result<InterpreterConfig> {
let script = r#"
# Allow the script to run on Python 2, so that nicer error can be printed later.
from __future__ import print_function
import os.path
import platform
import struct
import sys
from sysconfig import get_config_var
PYPY = platform.python_implementation() == "PyPy"
# sys.base_prefix is missing on Python versions older than 3.3; this allows the script to continue
# so that the version mismatch can be reported in a nicer way later.
base_prefix = getattr(sys, "base_prefix", None)
if base_prefix:
# Anaconda based python distributions have a static python executable, but include
# the shared library. Use the shared library for embedding to avoid rust trying to
# LTO the static library (and failing with newer gcc's, because it is old).
ANACONDA = os.path.exists(os.path.join(base_prefix, "conda-meta"))
else:
ANACONDA = False
def print_if_set(varname, value):
if value is not None:
print(varname, value)
# Windows always uses shared linking
WINDOWS = hasattr(platform, "win32_ver")
# macOS framework packages use shared linking
FRAMEWORK = bool(get_config_var("PYTHONFRAMEWORK"))
# unix-style shared library enabled
SHARED = bool(get_config_var("Py_ENABLE_SHARED"))
print("implementation", platform.python_implementation())
print("version_major", sys.version_info[0])
print("version_minor", sys.version_info[1])
print("shared", PYPY or ANACONDA or WINDOWS or FRAMEWORK or SHARED)
print_if_set("ld_version", get_config_var("LDVERSION"))
print_if_set("libdir", get_config_var("LIBDIR"))
print_if_set("base_prefix", base_prefix)
print("executable", sys.executable)
print("calcsize_pointer", struct.calcsize("P"))
"#;
let output = run_python_script(interpreter, script)?;
let map: HashMap<String, String> = parse_script_output(&output);
let shared = map["shared"].as_str() == "True";
let version = PythonVersion {
major: map["version_major"]
.parse()
.context("failed to parse major version")?,
minor: map["version_minor"]
.parse()
.context("failed to parse minor version")?,
};
let abi3 = is_abi3();
let implementation = map["implementation"].parse()?;
let lib_name = if cfg!(windows) {
default_lib_name_windows(
&version,
abi3,
&cargo_env_var("CARGO_CFG_TARGET_ENV").unwrap(),
)
} else {
default_lib_name_unix(
&version,
implementation,
map.get("ld_version").map(String::as_str),
)?
};
let lib_dir = if cfg!(windows) {
map.get("base_prefix")
.map(|base_prefix| format!("cargo:rustc-link-search=native={}\\libs", base_prefix))
} else {
map.get("libdir").cloned()
};
// The reason we don't use platform.architecture() here is that it's not
// reliable on macOS. See https://stackoverflow.com/a/1405971/823869.
// Similarly, sys.maxsize is not reliable on Windows. See
// https://stackoverflow.com/questions/1405913/how-do-i-determine-if-my-python-shell-is-executing-in-32bit-or-64bit-mode-on-os/1405971#comment6209952_1405971
// and https://stackoverflow.com/a/3411134/823869.
let calcsize_pointer: u32 = map["calcsize_pointer"]
.parse()
.context("failed to parse calcsize_pointer")?;
Ok(InterpreterConfig {
version,
implementation,
shared,
abi3,
lib_name: Some(lib_name),
lib_dir,
executable: map.get("executable").cloned(),
pointer_width: Some(calcsize_pointer * 8),
build_flags: BuildFlags::from_interpreter(interpreter)?.fixup(version, implementation),
})
pub fn get_abi3_version() -> Option<PythonVersion> {
let minor_version = (MINIMUM_SUPPORTED_VERSION.minor..=ABI3_MAX_MINOR)
.find(|i| cargo_env_var(&format!("CARGO_FEATURE_ABI3_PY3{}", i)).is_some());
minor_version.map(|minor| PythonVersion { major: 3, minor })
}
fn get_abi3_minor_version() -> Option<u8> {
(MINIMUM_SUPPORTED_VERSION.minor..=ABI3_MAX_MINOR)
.find(|i| cargo_env_var(&format!("CARGO_FEATURE_ABI3_PY3{}", i)).is_some())
/// Lowers the configured version to the abi3 version, if set.
fn fixup_config_for_abi3(
config: &mut InterpreterConfig,
abi3_version: Option<PythonVersion>,
) -> Result<()> {
if let Some(version) = abi3_version {
ensure!(
version <= config.version,
"cannot set a mininimum Python version {} higher than the interpreter version {} \
(the minimum Python version is implied by the abi3-py3{} feature)",
version,
config.version,
version.minor,
);
config.version = version;
}
Ok(())
}
/// Generates an interpreter config suitable for cross-compilation.
///
/// This must be called from PyO3's build script, because it relies on environment variables such as
/// CARGO_CFG_TARGET_OS which aren't available at any other time.
pub fn make_cross_compile_config() -> Result<Option<InterpreterConfig>> {
let abi3_version = get_abi3_minor_version();
let mut interpreter_config = if let Some(paths) = cross_compiling()? {
load_cross_compile_info(paths)?
load_cross_compile_config(paths)?
} else {
return Ok(None);
};
// Fixup minor version if abi3-pyXX feature set
if let Some(abi3_minor_version) = abi3_version {
ensure!(
abi3_minor_version <= interpreter_config.version.minor,
"You cannot set a mininimum Python version 3.{} higher than the interpreter version 3.{}",
abi3_minor_version,
interpreter_config.version.minor
);
interpreter_config.version.minor = abi3_minor_version;
}
fixup_config_for_abi3(&mut interpreter_config, get_abi3_version())?;
Ok(Some(interpreter_config))
}
/// Generates an interpreter config which will be hard-coded into the pyo3-build-config crate.
/// Only used by `pyo3-build-config` build script.
#[allow(dead_code)]
pub fn make_interpreter_config() -> Result<InterpreterConfig> {
let abi3_version = get_abi3_minor_version();
// If PYO3_NO_PYTHON is set with abi3, we can build PyO3 without calling Python.
if let Some(abi3_minor_version) = abi3_version {
if env_var("PYO3_NO_PYTHON").is_some() {
let version = PythonVersion {
major: 3,
minor: abi3_minor_version,
};
let implementation = PythonImplementation::CPython;
let lib_name = if cfg!(windows) {
Some(WINDOWS_ABI3_LIB_NAME.to_owned())
} else {
None
};
return Ok(InterpreterConfig {
version,
implementation,
abi3: true,
lib_name,
lib_dir: None,
build_flags: BuildFlags::abi3(),
pointer_width: None,
executable: None,
shared: true,
});
}
}
let mut interpreter_config = get_config_from_interpreter(&find_interpreter()?)?;
// Fixup minor version if abi3-pyXX feature set
if let Some(abi3_minor_version) = abi3_version {
ensure!(
abi3_minor_version <= interpreter_config.version.minor,
"You cannot set a mininimum Python version 3.{} higher than the interpreter version 3.{}",
abi3_minor_version,
interpreter_config.version.minor
);
interpreter_config.version.minor = abi3_minor_version;
}
let mut interpreter_config = InterpreterConfig::from_interpreter(&find_interpreter()?)?;
fixup_config_for_abi3(&mut interpreter_config, get_abi3_version())?;
Ok(interpreter_config)
}

149
pyo3-build-config/src/lib.rs
View File

@ -1,48 +1,23 @@
//! Configuration used by PyO3 for conditional support of varying Python versions.
//!
//! The public APIs exposed, [`use_pyo3_cfgs`] and [`add_extension_module_link_args`] are intended
//! to be called from build scripts to simplify building crates which depend on PyO3.
//! This crate exposes two functions, [`use_pyo3_cfgs`] and [`add_extension_module_link_args`],
//! which are intended to be called from build scripts to simplify building crates which depend on
//! PyO3.
//!
//! It used internally by the PyO3 crate's build script to apply the same configuration.
#[doc(hidden)]
pub mod errors;
mod errors;
mod impl_;
use std::io::Cursor;
use once_cell::sync::OnceCell;
// Used in PyO3's build.rs
#[doc(hidden)]
pub use impl_::{
cargo_env_var, env_var, find_interpreter, get_config_from_interpreter, make_interpreter_config, make_cross_compile_config,
InterpreterConfig, PythonImplementation, PythonVersion,
};
use impl_::InterpreterConfig;
/// Reads the configuration written by PyO3's build.rs
///
/// Because this will never change in a given compilation run, this is cached in a `once_cell`.
// Used in `pyo3-macros-backend`; may expose this in a future release.
#[doc(hidden)]
pub fn get() -> &'static InterpreterConfig {
static CONFIG: OnceCell<InterpreterConfig> = OnceCell::new();
CONFIG.get_or_init(|| {
if let Some(path) = std::env::var_os("PYO3_CONFIG_FILE") {
// Config file set - use that
InterpreterConfig::from_path(path)
} else if impl_::any_cross_compiling_env_vars_set() {
InterpreterConfig::from_path(DEFAULT_CROSS_COMPILE_CONFIG_PATH)
} else {
InterpreterConfig::from_reader(Cursor::new(HOST_CONFIG))
}.expect("failed to parse PyO3 config file")
})
}
/// Path where PyO3's build.rs will write configuration by default.
#[doc(hidden)]
pub const DEFAULT_CROSS_COMPILE_CONFIG_PATH: &str = concat!(env!("OUT_DIR"), "/pyo3-cross-compile-config.txt");
/// Build configuration discovered by `pyo3-build-config` build script. Not aware of
/// cross-compilation settings.
pub const HOST_CONFIG: &str = include_str!(concat!(env!("OUT_DIR"), "/pyo3-build-config.txt"));
pub use impl_::PythonVersion;
/// Adds all the [`#[cfg]` flags](index.html) to the current compilation.
///
@ -68,8 +43,112 @@ pub fn use_pyo3_cfgs() {
/// This is currently a no-op on non-macOS platforms, however may emit additional linker arguments
/// in future if deemed necessarys.
pub fn add_extension_module_link_args() {
if cargo_env_var("CARGO_CFG_TARGET_OS").unwrap() == "macos" {
if impl_::cargo_env_var("CARGO_CFG_TARGET_OS").unwrap() == "macos" {
println!("cargo:rustc-cdylib-link-arg=-undefined");
println!("cargo:rustc-cdylib-link-arg=dynamic_lookup");
}
}
/// Loads the configuration determined from the build environment.
///
/// Because this will never change in a given compilation run, this is cached in a `once_cell`.
#[doc(hidden)]
pub fn get() -> &'static InterpreterConfig {
static CONFIG: OnceCell<InterpreterConfig> = OnceCell::new();
CONFIG.get_or_init(|| {
if !CONFIG_FILE.is_empty() {
InterpreterConfig::from_reader(Cursor::new(CONFIG_FILE))
} else if !ABI3_CONFIG.is_empty() {
Ok(abi3_config())
} else if impl_::any_cross_compiling_env_vars_set() {
InterpreterConfig::from_path(DEFAULT_CROSS_COMPILE_CONFIG_PATH)
} else {
InterpreterConfig::from_reader(Cursor::new(HOST_CONFIG))
}
.expect("failed to parse PyO3 config file")
})
}
/// Path where PyO3's build.rs will write configuration by default.
#[doc(hidden)]
const DEFAULT_CROSS_COMPILE_CONFIG_PATH: &str =
concat!(env!("OUT_DIR"), "/pyo3-cross-compile-config.txt");
/// Build configuration provided by `PYO3_CONFIG_FILE`. May be empty if env var not set.
#[doc(hidden)]
const CONFIG_FILE: &str = include_str!(concat!(env!("OUT_DIR"), "/pyo3-build-config-file.txt"));
/// Build configuration set if abi3 features enabled and `PYO3_NO_PYTHON` env var present. Empty if
/// not both present.
#[doc(hidden)]
const ABI3_CONFIG: &str = include_str!(concat!(env!("OUT_DIR"), "/pyo3-build-config-abi3.txt"));
/// Build configuration discovered by `pyo3-build-config` build script. Not aware of
/// cross-compilation settings.
#[doc(hidden)]
const HOST_CONFIG: &str = include_str!(concat!(env!("OUT_DIR"), "/pyo3-build-config.txt"));
fn abi3_config() -> InterpreterConfig {
let mut interpreter_config = InterpreterConfig::from_reader(Cursor::new(ABI3_CONFIG))
.expect("failed to parse hardcoded PyO3 abi3 config");
// If running from a build script on Windows, tweak the hardcoded abi3 config to contain
// the standard lib name (this is necessary so that abi3 extension modules using
// PYO3_NO_PYTHON on Windows can link)
if std::env::var("CARGO_CFG_TARGET_OS").map_or(false, |target_os| target_os == "windows") {
assert_eq!(interpreter_config.lib_name, None);
interpreter_config.lib_name = Some("python3".to_owned())
}
interpreter_config
}
/// Private exports used in PyO3's build.rs
///
/// Please don't use these - they could change at any time.
#[doc(hidden)]
pub mod pyo3_build_script_impl {
use crate::errors::{Context, Result};
use std::path::Path;
use super::*;
pub mod errors {
pub use crate::errors::*;
}
pub use crate::impl_::{
cargo_env_var, env_var, make_cross_compile_config, InterpreterConfig, PythonVersion,
};
/// Gets the configuration for use from PyO3's build script.
///
/// Differs from .get() above only in the cross-compile case, where PyO3's build script is
/// required to generate a new config (as it's the first build script which has access to the
/// correct value for CARGO_CFG_TARGET_OS).
pub fn resolve_interpreter_config() -> Result<InterpreterConfig> {
if !CONFIG_FILE.is_empty() {
InterpreterConfig::from_reader(Cursor::new(CONFIG_FILE))
} else if !ABI3_CONFIG.is_empty() {
Ok(abi3_config())
} else if let Some(interpreter_config) = impl_::make_cross_compile_config()? {
// This is a cross compile and need to write the config file.
let path = Path::new(DEFAULT_CROSS_COMPILE_CONFIG_PATH);
let parent_dir = path.parent().ok_or_else(|| {
format!(
"failed to resolve parent directory of config file {}",
path.display()
)
})?;
std::fs::create_dir_all(&parent_dir).with_context(|| {
format!(
"failed to create config file directory {}",
parent_dir.display()
)
})?;
interpreter_config.to_writer(&mut std::fs::File::create(&path).with_context(
|| format!("failed to create config file at {}", path.display()),
)?)?;
Ok(interpreter_config)
} else {
InterpreterConfig::from_reader(Cursor::new(HOST_CONFIG))
}
}
}