luxolus
/
pyo3
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1622 from davidhewitt/pyo3-build-config 

pyo3-build-config: new crate to re-use build.rs across crates
This commit is contained in:
David Hewitt 2021-05-25 07:11:24 +01:00 committed by GitHub
parent cfdd535eea 1e1605f0db
commit 9a4e7b5877
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 1108 additions and 871 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
Cargo.toml
View File

@ -11,7 +11,6 @@ documentation = "https://docs.rs/crate/pyo3/"
categories = ["api-bindings", "development-tools::ffi"]
license = "Apache-2.0"
exclude = ["/.gitignore", ".cargo/config", "/codecov.yml", "/Makefile", "/pyproject.toml", "/tox.ini"]
build = "build.rs"
edition = "2018"
[dependencies]
@ -40,6 +39,9 @@ proptest = { version = "0.10.1", default-features = false, features = ["std"] }
pyo3 = { path = ".", default-features = false, features = ["macros", "auto-initialize"] }
serde_json = "1.0.61"
[build-dependencies]
pyo3-build-config = { path = "pyo3-build-config", version = "=0.14.0-alpha.0" }
[features]
default = ["macros"]
@ -55,13 +57,13 @@ multiple-pymethods = ["inventory"]
extension-module = []
# Use the Python limited API. See https://www.python.org/dev/peps/pep-0384/ for more.
abi3 = []
abi3 = ["pyo3-build-config/abi3"]
# With abi3, we can manually set the minimum Python version.
abi3-py36 = ["abi3-py37"]
abi3-py37 = ["abi3-py38"]
abi3-py38 = ["abi3-py39"]
abi3-py39 = ["abi3"]
abi3-py36 = ["abi3-py37", "pyo3-build-config/abi3-py36"]
abi3-py37 = ["abi3-py38", "pyo3-build-config/abi3-py37"]
abi3-py38 = ["abi3-py39", "pyo3-build-config/abi3-py38"]
abi3-py39 = ["abi3", "pyo3-build-config/abi3-py39"]
# Changes `Python::with_gil` and `Python::acquire_gil` to automatically initialize the
# Python interpreter if needed.

905
build.rs
View File

@ -1,21 +1,11 @@
use std::{
collections::{HashMap, HashSet},
convert::AsRef,
env,
ffi::OsString,
fmt::Display,
fs::{self, DirEntry},
path::{Path, PathBuf},
process::{Command, Stdio},
str::FromStr,
};
use std::{env, process::Command};
use pyo3_build_config::{InterpreterConfig, PythonImplementation, PythonVersion};
type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
/// Minimum Python version PyO3 supports.
const MINIMUM_SUPPORTED_VERSION: PythonVersion = PythonVersion { major: 3, minor: 6 };
/// Maximum Python version that can be used as minimum required Python version with abi3.
const ABI3_MAX_MINOR: u8 = 9;
type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
// A simple macro for returning an error. Resembles anyhow::bail.
macro_rules! bail {
@ -28,564 +18,60 @@ macro_rules! ensure {
($condition:expr, $($args: tt)+) => { if !($condition) { bail!($($args)+) } };
}
// Show warning. If needed, please extend this macro to support arguments.
macro_rules! warn {
($msg: literal) => {
println!(concat!("cargo:warning=", $msg));
};
fn ensure_python_version(interpreter_config: &InterpreterConfig) -> Result<()> {
ensure!(
interpreter_config.version >= MINIMUM_SUPPORTED_VERSION,
"the configured Python interpreter version ({}) is lower than PyO3's minimum supported version ({})",
interpreter_config.version,
MINIMUM_SUPPORTED_VERSION,
);
Ok(())
}
/// Gets an environment variable owned by cargo.
///
/// Environment variables set by cargo are expected to be valid UTF8.
fn cargo_env_var(var: &str) -> Option<String> {
env::var_os(var).map(|os_string| os_string.to_str().unwrap().into())
}
/// Gets an external environment variable, and registers the build script to rerun if
/// the variable changes.
fn env_var(var: &str) -> Option<OsString> {
println!("cargo:rerun-if-env-changed={}", var);
env::var_os(var)
}
/// Configuration needed by PyO3 to build for the correct Python implementation.
///
/// Usually this is queried directly from the Python interpreter. When the `PYO3_NO_PYTHON` variable
/// is set, or during cross compile situations, then alternative strategies are used to populate
/// this type.
#[derive(Debug)]
struct InterpreterConfig {
version: PythonVersion,
libdir: Option<String>,
shared: bool,
ld_version: Option<String>,
base_prefix: Option<String>,
executable: Option<String>,
calcsize_pointer: Option<u32>,
implementation: PythonImplementation,
build_flags: BuildFlags,
}
impl InterpreterConfig {
fn is_pypy(&self) -> bool {
self.implementation == PythonImplementation::PyPy
}
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)]
struct PythonVersion {
major: u8,
minor: u8,
}
impl PythonVersion {
const PY37: Self = PythonVersion { major: 3, minor: 7 };
}
impl Display for PythonVersion {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}.{}", self.major, self.minor)
}
}
#[derive(Debug, Copy, Clone, PartialEq)]
pub enum PythonImplementation {
CPython,
PyPy,
}
impl FromStr for PythonImplementation {
type Err = Box<dyn std::error::Error>;
fn from_str(s: &str) -> Result<Self> {
match s {
"CPython" => Ok(PythonImplementation::CPython),
"PyPy" => Ok(PythonImplementation::PyPy),
_ => bail!("Invalid interpreter: {}", s),
}
}
}
fn is_abi3() -> bool {
cargo_env_var("CARGO_FEATURE_ABI3").is_some()
}
trait GetPrimitive {
fn get_bool(&self, key: &str) -> Result<bool>;
fn get_numeric<T: FromStr>(&self, key: &str) -> Result<T>;
}
impl GetPrimitive for HashMap<String, String> {
fn get_bool(&self, key: &str) -> Result<bool> {
match self
.get(key)
.map(|x| x.as_str())
.ok_or(format!("{} is not defined", key))?
{
"1" | "true" | "True" => Ok(true),
"0" | "false" | "False" => Ok(false),
_ => bail!("{} must be a bool (1/true/True or 0/false/False", key),
}
}
fn get_numeric<T: FromStr>(&self, key: &str) -> Result<T> {
self.get(key)
.ok_or(format!("{} is not defined", key))?
.parse::<T>()
.map_err(|_| format!("Could not parse value of {}", key).into())
}
}
struct CrossCompileConfig {
lib_dir: PathBuf,
version: Option<String>,
os: String,
arch: String,
}
fn cross_compiling() -> Result<Option<CrossCompileConfig>> {
let cross = env_var("PYO3_CROSS");
let cross_lib_dir = env_var("PYO3_CROSS_LIB_DIR");
let cross_python_version = env_var("PYO3_CROSS_PYTHON_VERSION");
let target_arch = cargo_env_var("CARGO_CFG_TARGET_ARCH");
let target_vendor = cargo_env_var("CARGO_CFG_TARGET_VENDOR");
let target_os = cargo_env_var("CARGO_CFG_TARGET_OS");
if cross.is_none() && cross_lib_dir.is_none() && cross_python_version.is_none() {
// No cross-compiling environment variables set; try to determine if this is a known case
// which is not cross-compilation.
let target = cargo_env_var("TARGET").unwrap();
let host = cargo_env_var("HOST").unwrap();
if target == host {
// Not cross-compiling
return Ok(None);
}
if target == "i686-pc-windows-msvc" && host == "x86_64-pc-windows-msvc" {
// Not cross-compiling to compile for 32-bit Python from windows 64-bit
return Ok(None);
}
if target == "x86_64-apple-darwin" && host == "aarch64-apple-darwin" {
// Not cross-compiling to compile for x86-64 Python from macOS arm64
return Ok(None);
}
if target == "aarch64-apple-darwin" && host == "x86_64-apple-darwin" {
// Not cross-compiling to compile for arm64 Python from macOS x86_64
return Ok(None);
}
if let (Some(arch), Some(vendor), Some(os)) = (&target_arch, &target_vendor, &target_os) {
if host.starts_with(&format!("{}-{}-{}", arch, vendor, os)) {
// Not cross-compiling if arch-vendor-os is all the same
// e.g. x86_64-unknown-linux-musl on x86_64-unknown-linux-gnu host
return Ok(None);
}
}
}
// At this point we assume that we are cross compiling.
Ok(Some(CrossCompileConfig {
lib_dir: cross_lib_dir
.ok_or("The PYO3_CROSS_LIB_DIR environment variable must be set when cross-compiling")?
.into(),
os: target_os.unwrap(),
arch: target_arch.unwrap(),
version: cross_python_version
.map(|os_string| {
os_string
.to_str()
.ok_or("PYO3_CROSS_PYTHON_VERSION is not valid utf-8.")
.map(str::to_owned)
})
.transpose()?,
}))
}
/// A list of python interpreter compile-time preprocessor defines that
/// we will pick up and pass to rustc via `--cfg=py_sys_config={varname}`;
/// this allows using them conditional cfg attributes in the .rs files, so
///
/// #[cfg(py_sys_config="{varname}"]
///
/// is the equivalent of `#ifdef {varname}` in C.
///
/// see Misc/SpecialBuilds.txt in the python source for what these mean.
#[derive(Debug)]
struct BuildFlags(HashSet<&'static str>);
impl BuildFlags {
const ALL: [&'static str; 5] = [
// TODO: Remove WITH_THREAD once Python 3.6 support dropped (as it's always on).
"WITH_THREAD",
"Py_DEBUG",
"Py_REF_DEBUG",
"Py_TRACE_REFS",
"COUNT_ALLOCS",
];
fn from_config_map(config_map: &HashMap<String, String>) -> Self {
Self(
BuildFlags::ALL
.iter()
.copied()
.filter(|flag| config_map.get(*flag).map_or(false, |value| value == "1"))
.collect(),
)
}
/// Examine python's compile flags to pass to cfg by launching
/// the interpreter and printing variables of interest from
/// sysconfig.get_config_vars.
fn from_interpreter(interpreter: &Path) -> Result<Self> {
if cargo_env_var("CARGO_CFG_TARGET_OS").unwrap() == "windows" {
return Ok(Self::windows_hardcoded());
}
let mut script = String::from("import sysconfig\n");
script.push_str("config = sysconfig.get_config_vars()\n");
for k in BuildFlags::ALL.iter() {
script.push_str(&format!("print(config.get('{}', '0'))\n", k));
}
let stdout = run_python_script(&interpreter, &script)?;
let split_stdout: Vec<&str> = stdout.trim_end().lines().collect();
ensure!(
split_stdout.len() == BuildFlags::ALL.len(),
"Python stdout len didn't return expected number of lines: {}",
split_stdout.len()
);
let flags = BuildFlags::ALL
.iter()
.zip(split_stdout)
.filter(|(_, flag_value)| *flag_value == "1")
.map(|(&flag, _)| flag)
.collect();
Ok(Self(flags))
}
fn windows_hardcoded() -> Self {
// sysconfig is missing all the flags on windows, so we can't actually
// query the interpreter directly for its build flags.
let mut flags = HashSet::new();
flags.insert("WITH_THREAD");
// Uncomment these manually if your python was built with these and you want
// the cfg flags to be set in rust.
//
// map.insert("Py_DEBUG", "1");
// map.insert("Py_REF_DEBUG", "1");
// map.insert("Py_TRACE_REFS", "1");
// map.insert("COUNT_ALLOCS", 1");
Self(flags)
}
fn abi3() -> Self {
let mut flags = HashSet::new();
flags.insert("WITH_THREAD");
Self(flags)
}
fn fixup(mut self, version: PythonVersion, implementation: PythonImplementation) -> Self {
if self.0.contains("Py_DEBUG") {
self.0.insert("Py_REF_DEBUG");
if version <= PythonVersion::PY37 {
// Py_DEBUG only implies Py_TRACE_REFS until Python 3.7
self.0.insert("Py_TRACE_REFS");
}
}
// WITH_THREAD is always on for Python 3.7, and for PyPy.
if implementation == PythonImplementation::PyPy || version >= PythonVersion::PY37 {
self.0.insert("WITH_THREAD");
}
self
}
}
fn parse_script_output(output: &str) -> HashMap<String, String> {
output
.lines()
.filter_map(|line| {
let mut i = line.splitn(2, ' ');
Some((i.next()?.into(), i.next()?.into()))
})
.collect()
}
/// Parse sysconfigdata file
///
/// The sysconfigdata is simply a dictionary containing all the build time variables used for the
/// python executable and library. Here it is read and added to a script to extract only what is
/// necessary. This necessitates a python interpreter for the host machine to work.
fn parse_sysconfigdata(config_path: impl AsRef<Path>) -> Result<HashMap<String, String>> {
let mut script = fs::read_to_string(config_path)?;
script += r#"
print("version_major", build_time_vars["VERSION"][0]) # 3
print("version_minor", build_time_vars["VERSION"][2]) # E.g., 8
KEYS = [
"WITH_THREAD",
"Py_DEBUG",
"Py_REF_DEBUG",
"Py_TRACE_REFS",
"COUNT_ALLOCS",
"Py_ENABLE_SHARED",
"LDVERSION",
"SIZEOF_VOID_P"
]
for key in KEYS:
print(key, build_time_vars.get(key, 0))
"#;
let output = run_python_script(&find_interpreter()?, &script)?;
Ok(parse_script_output(&output))
}
fn starts_with(entry: &DirEntry, pat: &str) -> bool {
let name = entry.file_name();
name.to_string_lossy().starts_with(pat)
}
fn ends_with(entry: &DirEntry, pat: &str) -> bool {
let name = entry.file_name();
name.to_string_lossy().ends_with(pat)
}
/// Finds the `_sysconfigdata*.py` file in the library path.
///
/// From the python source for `_sysconfigdata*.py` is always going to be located at
/// `build/lib.{PLATFORM}-{PY_MINOR_VERSION}` when built from source. The [exact line][1] is defined as:
///
/// ```py
/// pybuilddir = 'build/lib.%s-%s' % (get_platform(), sys.version_info[:2])
/// ```
///
/// Where get_platform returns a kebab-case formated string containing the os, the architecture and
/// possibly the os' kernel version (not the case on linux). However, when installed using a package
/// manager, the `_sysconfigdata*.py` file is installed in the `${PREFIX}/lib/python3.Y/` directory.
/// The `_sysconfigdata*.py` is generally in a sub-directory of the location of `libpython3.Y.so`.
/// So we must find the file in the following possible locations:
///
/// ```sh
/// # distribution from package manager, lib_dir should include lib/
/// ${INSTALL_PREFIX}/lib/python3.Y/_sysconfigdata*.py
/// ${INSTALL_PREFIX}/lib/libpython3.Y.so
/// ${INSTALL_PREFIX}/lib/python3.Y/config-3.Y-${HOST_TRIPLE}/libpython3.Y.so
///
/// # Built from source from host
/// ${CROSS_COMPILED_LOCATION}/build/lib.linux-x86_64-Y/_sysconfigdata*.py
/// ${CROSS_COMPILED_LOCATION}/libpython3.Y.so
///
/// # if cross compiled, kernel release is only present on certain OS targets.
/// ${CROSS_COMPILED_LOCATION}/build/lib.{OS}(-{OS-KERNEL-RELEASE})?-{ARCH}-Y/_sysconfigdata*.py
/// ${CROSS_COMPILED_LOCATION}/libpython3.Y.so
/// ```
///
/// [1]: https://github.com/python/cpython/blob/3.5/Lib/sysconfig.py#L389
fn find_sysconfigdata(cross: &CrossCompileConfig) -> Result<PathBuf> {
let sysconfig_paths = search_lib_dir(&cross.lib_dir, &cross);
let sysconfig_name = env_var("_PYTHON_SYSCONFIGDATA_NAME");
let mut sysconfig_paths = sysconfig_paths
.iter()
.filter_map(|p| {
let canonical = fs::canonicalize(p).ok();
match &sysconfig_name {
Some(_) => canonical.filter(|p| p.file_stem() == sysconfig_name.as_deref()),
None => canonical,
}
})
.collect::<Vec<PathBuf>>();
sysconfig_paths.dedup();
if sysconfig_paths.is_empty() {
bail!(
"Could not find either libpython.so or _sysconfigdata*.py in {}",
cross.lib_dir.display()
);
} else if sysconfig_paths.len() > 1 {
let mut error_msg = String::from(
"Detected multiple possible Python versions. Please set either the \
PYO3_CROSS_PYTHON_VERSION variable to the wanted version or the \
_PYTHON_SYSCONFIGDATA_NAME variable to the wanted sysconfigdata file name.\n\n\
sysconfigdata files found:",
);
for path in sysconfig_paths {
error_msg += &format!("\n\t{}", path.display());
}
bail!("{}", error_msg);
}
Ok(sysconfig_paths.remove(0))
}
/// recursive search for _sysconfigdata, returns all possibilities of sysconfigdata paths
fn search_lib_dir(path: impl AsRef<Path>, cross: &CrossCompileConfig) -> Vec<PathBuf> {
let mut sysconfig_paths = vec![];
let version_pat = if let Some(v) = &cross.version {
format!("python{}", v)
} else {
"python3.".into()
};
for f in fs::read_dir(path).expect("Path does not exist") {
let sysc = match &f {
Ok(f) if starts_with(f, "_sysconfigdata") && ends_with(f, "py") => vec![f.path()],
Ok(f) if starts_with(f, "build") => search_lib_dir(f.path(), cross),
Ok(f) if starts_with(f, "lib.") => {
let name = f.file_name();
// check if right target os
if !name.to_string_lossy().contains(if cross.os == "android" {
"linux"
} else {
&cross.os
}) {
continue;
}
// Check if right arch
if !name.to_string_lossy().contains(&cross.arch) {
continue;
}
search_lib_dir(f.path(), cross)
}
Ok(f) if starts_with(f, &version_pat) => search_lib_dir(f.path(), cross),
_ => continue,
};
sysconfig_paths.extend(sysc);
}
sysconfig_paths
}
/// Find cross compilation information from sysconfigdata file
///
/// first find sysconfigdata file which follows the pattern [`_sysconfigdata_{abi}_{platform}_{multiarch}`][1]
/// on python 3.6 or greater. On python 3.5 it is simply `_sysconfigdata.py`.
///
/// [1]: https://github.com/python/cpython/blob/3.8/Lib/sysconfig.py#L348
fn load_cross_compile_from_sysconfigdata(
cross_compile_config: CrossCompileConfig,
) -> Result<InterpreterConfig> {
let sysconfig_path = find_sysconfigdata(&cross_compile_config)?;
let sysconfig_data = parse_sysconfigdata(sysconfig_path)?;
let major = sysconfig_data.get_numeric("version_major")?;
let minor = sysconfig_data.get_numeric("version_minor")?;
let ld_version = match sysconfig_data.get("LDVERSION") {
Some(s) => s.clone(),
None => format!("{}.{}", major, minor),
};
let calcsize_pointer = sysconfig_data.get_numeric("SIZEOF_VOID_P").ok();
let version = PythonVersion { major, minor };
let implementation = PythonImplementation::CPython;
Ok(InterpreterConfig {
version,
libdir: cross_compile_config.lib_dir.to_str().map(String::from),
shared: sysconfig_data.get_bool("Py_ENABLE_SHARED")?,
ld_version: Some(ld_version),
base_prefix: None,
executable: None,
calcsize_pointer,
implementation,
build_flags: BuildFlags::from_config_map(&sysconfig_data).fixup(version, implementation),
})
}
fn windows_hardcoded_cross_compile(
cross_compile_config: CrossCompileConfig,
) -> Result<InterpreterConfig> {
let (major, minor) = if let Some(version) = cross_compile_config.version {
let mut parts = version.split('.');
match (
parts.next().and_then(|major| major.parse().ok()),
parts.next().and_then(|minor| minor.parse().ok()),
parts.next(),
) {
(Some(major), Some(minor), None) => (major, minor),
_ => bail!(
"Expected major.minor version (e.g. 3.9) for PYO3_CROSS_PYTHON_VERSION, got `{}`",
version
),
}
} else if let Some(minor_version) = get_abi3_minor_version() {
(3, minor_version)
} else {
bail!("PYO3_CROSS_PYTHON_VERSION or an abi3-py3* feature must be specified when cross-compiling for Windows.")
fn ensure_target_architecture(interpreter_config: &InterpreterConfig) -> Result<()> {
// Try to check whether the target architecture matches the python library
let rust_target = match env::var("CARGO_CFG_TARGET_POINTER_WIDTH").unwrap().as_str() {
"64" => "64-bit",
"32" => "32-bit",
x => bail!("unexpected Rust target pointer width: {}", x),
};
Ok(InterpreterConfig {
version: PythonVersion { major, minor },
libdir: cross_compile_config.lib_dir.to_str().map(String::from),
shared: true,
ld_version: None,
base_prefix: None,
executable: None,
calcsize_pointer: None,
implementation: PythonImplementation::CPython,
build_flags: BuildFlags::windows_hardcoded(),
})
}
// The reason we don't use platform.architecture() here is that it's not
// reliable on macOS. See https://stackoverflow.com/a/1405971/823869.
// Similarly, sys.maxsize is not reliable on Windows. See
// https://stackoverflow.com/questions/1405913/how-do-i-determine-if-my-python-shell-is-executing-in-32bit-or-64bit-mode-on-os/1405971#comment6209952_1405971
// and https://stackoverflow.com/a/3411134/823869.
let python_target = match interpreter_config.calcsize_pointer {
Some(8) => "64-bit",
Some(4) => "32-bit",
None => {
// Unset, e.g. because we're cross-compiling. Don't check anything
// in this case.
return Ok(());
}
Some(n) => bail!("unexpected Python calcsize_pointer value: {}", n),
};
fn load_cross_compile_info(cross_compile_config: CrossCompileConfig) -> Result<InterpreterConfig> {
match cargo_env_var("CARGO_CFG_TARGET_FAMILY") {
// Configure for unix platforms using the sysconfigdata file
Some(os) if os == "unix" => load_cross_compile_from_sysconfigdata(cross_compile_config),
// Use hardcoded interpreter config when targeting Windows
Some(os) if os == "windows" => windows_hardcoded_cross_compile(cross_compile_config),
// sysconfigdata works fine on wasm/wasi
Some(os) if os == "wasm" => load_cross_compile_from_sysconfigdata(cross_compile_config),
// Waiting for users to tell us what they expect on their target platform
Some(os) => bail!(
"Unsupported target OS family for cross-compilation: {:?}",
os
),
// Unknown os family - try to do something useful
None => load_cross_compile_from_sysconfigdata(cross_compile_config),
}
}
ensure!(
rust_target == python_target,
"Your Rust target architecture ({}) does not match your python interpreter ({})",
rust_target,
python_target
);
/// Run a python script using the specified interpreter binary.
fn run_python_script(interpreter: &Path, script: &str) -> Result<String> {
let out = Command::new(interpreter)
.env("PYTHONIOENCODING", "utf-8")
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::inherit())
.spawn()
.and_then(|mut child| {
use std::io::Write;
child
.stdin
.as_mut()
.expect("piped stdin")
.write_all(script.as_bytes())?;
child.wait_with_output()
});
match out {
Err(err) => bail!(
"failed to run the Python interpreter at {}: {}",
interpreter.display(),
err
),
Ok(ok) if !ok.status.success() => bail!("Python script failed"),
Ok(ok) => Ok(String::from_utf8(ok.stdout)?),
}
Ok(())
}
fn get_rustc_link_lib(config: &InterpreterConfig) -> Result<String> {
let link_name = if cargo_env_var("CARGO_CFG_TARGET_OS").unwrap() == "windows" {
if is_abi3() {
let link_name = if env::var_os("CARGO_CFG_TARGET_OS").unwrap() == "windows" {
if config.abi3 {
// Link against python3.lib for the stable ABI on Windows.
// See https://www.python.org/dev/peps/pep-0384/#linkage
//
// This contains only the limited ABI symbols.
"pythonXY:python3".to_owned()
} else if cargo_env_var("CARGO_CFG_TARGET_ENV").unwrap() == "gnu" {
} else if env::var_os("CARGO_CFG_TARGET_ENV").unwrap() == "gnu" {
// https://packages.msys2.org/base/mingw-w64-python
format!(
"pythonXY:python{}.{}",
@ -601,7 +87,9 @@ fn get_rustc_link_lib(config: &InterpreterConfig) -> Result<String> {
match config.implementation {
PythonImplementation::CPython => match &config.ld_version {
Some(ld_version) => format!("python{}", ld_version),
None => bail!("failed to configure `ld_version` when compiling for unix"),
None => {
return Err("failed to configure `ld_version` when compiling for unix".into())
}
},
PythonImplementation::PyPy => format!("pypy{}-c", config.version.major),
}
@ -614,140 +102,6 @@ fn get_rustc_link_lib(config: &InterpreterConfig) -> Result<String> {
))
}
fn get_venv_path() -> Option<PathBuf> {
match (env_var("VIRTUAL_ENV"), env_var("CONDA_PREFIX")) {
(Some(dir), None) => Some(PathBuf::from(dir)),
(None, Some(dir)) => Some(PathBuf::from(dir)),
(Some(_), Some(_)) => {
warn!(
"Both VIRTUAL_ENV and CONDA_PREFIX are set. PyO3 will ignore both of these for \
locating the Python interpreter until you unset one of them."
);
None
}
(None, None) => None,
}
}
/// Attempts to locate a python interpreter. Locations are checked in the order listed:
/// 1. If `PYO3_PYTHON` is set, this intepreter is used.
/// 2. If in a virtualenv, that environment's interpreter is used.
/// 3. `python`, if this is functional a Python 3.x interpreter
/// 4. `python3`, as above
fn find_interpreter() -> Result<PathBuf> {
if let Some(exe) = env_var("PYO3_PYTHON") {
Ok(exe.into())
} else if let Some(venv_path) = get_venv_path() {
match cargo_env_var("CARGO_CFG_TARGET_OS").unwrap().as_str() {
"windows" => Ok(venv_path.join("Scripts\\python")),
_ => Ok(venv_path.join("bin/python")),
}
} else {
println!("cargo:rerun-if-env-changed=PATH");
["python", "python3"]
.iter()
.find(|bin| {
if let Ok(out) = Command::new(bin).arg("--version").output() {
// begin with `Python 3.X.X :: additional info`
out.stdout.starts_with(b"Python 3") || out.stderr.starts_with(b"Python 3")
} else {
false
}
})
.map(PathBuf::from)
.ok_or_else(|| "no Python 3.x interpreter found".into())
}
}
/// Extract compilation vars from the specified interpreter.
fn get_config_from_interpreter(interpreter: &Path) -> Result<InterpreterConfig> {
let script = r#"
# Allow the script to run on Python 2, so that nicer error can be printed later.
from __future__ import print_function
import os.path
import platform
import struct
import sys
from sysconfig import get_config_var
PYPY = platform.python_implementation() == "PyPy"
# sys.base_prefix is missing on Python versions older than 3.3; this allows the script to continue
# so that the version mismatch can be reported in a nicer way later.
base_prefix = getattr(sys, "base_prefix", None)
if base_prefix:
# Anaconda based python distributions have a static python executable, but include
# the shared library. Use the shared library for embedding to avoid rust trying to
# LTO the static library (and failing with newer gcc's, because it is old).
ANACONDA = os.path.exists(os.path.join(base_prefix, "conda-meta"))
else:
ANACONDA = False
def print_if_set(varname, value):
if value is not None:
print(varname, value)
libdir = get_config_var("LIBDIR")
print("version_major", sys.version_info[0])
print("version_minor", sys.version_info[1])
print("implementation", platform.python_implementation())
print_if_set("libdir", libdir)
print_if_set("ld_version", get_config_var("LDVERSION"))
print_if_set("base_prefix", base_prefix)
print("framework", bool(get_config_var("PYTHONFRAMEWORK")))
print("shared", PYPY or ANACONDA or bool(get_config_var("Py_ENABLE_SHARED")))
print("executable", sys.executable)
print("calcsize_pointer", struct.calcsize("P"))
"#;
let output = run_python_script(interpreter, script)?;
let map: HashMap<String, String> = parse_script_output(&output);
let shared = match (
cargo_env_var("CARGO_CFG_TARGET_OS").unwrap().as_str(),
map["framework"].as_str(),
map["shared"].as_str(),
) {
(_, _, "True") // Py_ENABLE_SHARED is set
| ("windows", _, _) // Windows always uses shared linking
| ("macos", "True", _) // MacOS framework package uses shared linking
=> true,
(_, _, "False") => false, // Any other platform, Py_ENABLE_SHARED not set
_ => bail!("Unrecognised link model combination")
};
let version = PythonVersion {
major: map["version_major"].parse()?,
minor: map["version_minor"].parse()?,
};
let implementation = map["implementation"].parse()?;
Ok(InterpreterConfig {
version,
implementation,
libdir: map.get("libdir").cloned(),
shared,
ld_version: map.get("ld_version").cloned(),
base_prefix: map.get("base_prefix").cloned(),
executable: map.get("executable").cloned(),
calcsize_pointer: Some(map["calcsize_pointer"].parse()?),
build_flags: BuildFlags::from_interpreter(interpreter)?.fixup(version, implementation),
})
}
fn ensure_python_version(interpreter_config: &InterpreterConfig) -> Result<()> {
ensure!(
interpreter_config.version >= MINIMUM_SUPPORTED_VERSION,
"the configured Python interpreter version ({}) is lower than PyO3's minimum supported version ({})",
interpreter_config.version,
MINIMUM_SUPPORTED_VERSION,
);
Ok(())
}
fn rustc_minor_version() -> Option<u32> {
let rustc = env::var_os("RUSTC")?;
let output = Command::new(rustc).arg("--version").output().ok()?;
@ -760,8 +114,8 @@ fn rustc_minor_version() -> Option<u32> {
}
fn emit_cargo_configuration(interpreter_config: &InterpreterConfig) -> Result<()> {
let target_os = cargo_env_var("CARGO_CFG_TARGET_OS").unwrap();
let is_extension_module = cargo_env_var("CARGO_FEATURE_EXTENSION_MODULE").is_some();
let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap();
let is_extension_module = env::var_os("CARGO_FEATURE_EXTENSION_MODULE").is_some();
match (is_extension_module, target_os.as_str()) {
(_, "windows") => {
// always link on windows, even with extension module
@ -802,69 +156,43 @@ fn emit_cargo_configuration(interpreter_config: &InterpreterConfig) -> Result<()
}
if env::var_os("CARGO_FEATURE_AUTO_INITIALIZE").is_some() {
ensure!(
interpreter_config.shared,
"The `auto-initialize` feature is enabled, but your python installation only supports \
embedding the Python interpreter statically. If you are attempting to run tests, or a \
binary which is okay to link dynamically, install a Python distribution which ships \
with the Python shared library.\n\
\n\
Embedding the Python interpreter statically does not yet have first-class support in \
PyO3. If you are sure you intend to do this, disable the `auto-initialize` feature.\n\
\n\
For more information, see \
https://pyo3.rs/v{pyo3_version}/\
building_and_distribution.html#embedding-python-in-rust",
pyo3_version = env::var("CARGO_PKG_VERSION").unwrap()
);
if !interpreter_config.shared {
return Err(format!(
"The `auto-initialize` feature is enabled, but your python installation only supports \
embedding the Python interpreter statically. If you are attempting to run tests, or a \
binary which is okay to link dynamically, install a Python distribution which ships \
with the Python shared library.\n\
\n\
Embedding the Python interpreter statically does not yet have first-class support in \
PyO3. If you are sure you intend to do this, disable the `auto-initialize` feature.\n\
\n\
For more information, see \
https://pyo3.rs/v{pyo3_version}/\
building_and_distribution.html#embedding-python-in-rust",
pyo3_version = env::var("CARGO_PKG_VERSION").unwrap()
)
.into());
}
// TODO: PYO3_CI env is a hack to workaround CI with PyPy, where the `dev-dependencies`
// currently cause `auto-initialize` to be enabled in CI.
// Once cargo's `resolver = "2"` is stable (~ MSRV Rust 1.52), remove this.
ensure!(
!interpreter_config.is_pypy() || env::var_os("PYO3_CI").is_some(),
"The `auto-initialize` feature is not supported with PyPy."
);
}
let is_abi3 = is_abi3();
if interpreter_config.is_pypy() {
println!("cargo:rustc-cfg=PyPy");
if is_abi3 {
warn!(
"PyPy does not yet support abi3 so the build artifacts will be version-specific. \
See https://foss.heptapod.net/pypy/pypy/-/issues/3397 for more information."
)
if interpreter_config.is_pypy() && env::var_os("PYO3_CI").is_none() {
return Err("The `auto-initialize` feature is not supported with PyPy.".into());
}
};
let minor = if is_abi3 {
println!("cargo:rustc-cfg=Py_LIMITED_API");
// Check any `abi3-py3*` feature is set. If not, use the interpreter version.
match get_abi3_minor_version() {
Some(minor) if minor > interpreter_config.version.minor => bail!(
"You cannot set a mininimum Python version 3.{} higher than the interpreter version 3.{}",
minor,
interpreter_config.version.minor
),
Some(minor) => minor,
None => interpreter_config.version.minor
}
} else {
interpreter_config.version.minor
};
for i in MINIMUM_SUPPORTED_VERSION.minor..=minor {
println!("cargo:rustc-cfg=Py_3_{}", i);
}
for flag in &interpreter_config.build_flags.0 {
println!("cargo:rustc-cfg=py_sys_config=\"{}\"", flag)
}
Ok(())
}
fn configure_pyo3() -> Result<()> {
let interpreter_config = pyo3_build_config::get();
ensure_python_version(&interpreter_config)?;
ensure_target_architecture(&interpreter_config)?;
emit_cargo_configuration(&interpreter_config)?;
interpreter_config.emit_pyo3_cfgs();
// Enable use of const generics on Rust 1.51 and greater
if rustc_minor_version().unwrap_or(0) >= 51 {
println!("cargo:rustc-cfg=min_const_generics");
}
@ -872,83 +200,6 @@ fn emit_cargo_configuration(interpreter_config: &InterpreterConfig) -> Result<()
Ok(())
}
fn ensure_target_architecture(interpreter_config: &InterpreterConfig) -> Result<()> {
// Try to check whether the target architecture matches the python library
let rust_target = match cargo_env_var("CARGO_CFG_TARGET_POINTER_WIDTH")
.unwrap()
.as_str()
{
"64" => "64-bit",
"32" => "32-bit",
x => bail!("unexpected Rust target pointer width: {}", x),
};
// The reason we don't use platform.architecture() here is that it's not
// reliable on macOS. See https://stackoverflow.com/a/1405971/823869.
// Similarly, sys.maxsize is not reliable on Windows. See
// https://stackoverflow.com/questions/1405913/how-do-i-determine-if-my-python-shell-is-executing-in-32bit-or-64bit-mode-on-os/1405971#comment6209952_1405971
// and https://stackoverflow.com/a/3411134/823869.
let python_target = match interpreter_config.calcsize_pointer {
Some(8) => "64-bit",
Some(4) => "32-bit",
None => {
// Unset, e.g. because we're cross-compiling. Don't check anything
// in this case.
return Ok(());
}
Some(n) => bail!("unexpected Python calcsize_pointer value: {}", n),
};
ensure!(
rust_target == python_target,
"Your Rust target architecture ({}) does not match your python interpreter ({})",
rust_target,
python_target
);
Ok(())
}
fn get_abi3_minor_version() -> Option<u8> {
(MINIMUM_SUPPORTED_VERSION.minor..=ABI3_MAX_MINOR)
.find(|i| cargo_env_var(&format!("CARGO_FEATURE_ABI3_PY3{}", i)).is_some())
}
fn get_interpreter_config() -> Result<InterpreterConfig> {
// If PYO3_NO_PYTHON is set with abi3, we can build PyO3 without calling Python.
if let Some(abi3_version) = get_abi3_minor_version() {
if env_var("PYO3_NO_PYTHON").is_some() {
return Ok(InterpreterConfig {
version: PythonVersion {
major: 3,
minor: abi3_version,
},
implementation: PythonImplementation::CPython,
libdir: None,
build_flags: BuildFlags::abi3(),
base_prefix: None,
calcsize_pointer: None,
executable: None,
ld_version: None,
shared: true,
});
}
}
if let Some(paths) = cross_compiling()? {
load_cross_compile_info(paths)
} else {
get_config_from_interpreter(&find_interpreter()?)
}
}
fn configure_pyo3() -> Result<()> {
let interpreter_config = get_interpreter_config()?;
ensure_python_version(&interpreter_config)?;
ensure_target_architecture(&interpreter_config)?;
emit_cargo_configuration(&interpreter_config)
}
fn main() {
// Print out error messages using display, to get nicer formatting.
if let Err(e) = configure_pyo3() {

6
examples/pyo3-pytests/Cargo.toml
View File

@ -6,10 +6,10 @@ description = "Python-based tests for PyO3"
edition = "2018"
[dependencies]
pyo3 = { path = "../../", features = ["extension-module"] }
[dependencies.pyo3]
path = "../../"
features = ["extension-module"]
[build-dependencies]
pyo3-build-config = { path = "../../pyo3-build-config" }
[lib]
name = "pyo3_pytests"

27
examples/pyo3-pytests/build.rs
View File

@ -1,28 +1,3 @@
use std::process::Command;
fn main() {
let out = Command::new("python")
.args(&["-c", "import sys; import platform; print(sys.version_info[1]); print(platform.python_implementation())"])
.output()
.expect("python version did not print");
let output = String::from_utf8_lossy(&out.stdout);
let mut lines = output.trim().lines();
println!("{}", output);
let version: u8 = lines
.next()
.unwrap()
.parse()
.expect("python version was not parsed");
let implementation = lines.next().unwrap();
for each in 6..version {
println!("cargo:rustc-cfg=Py_3_{}", each);
}
if implementation == "PyPy" {
println!("cargo:rustc-cfg=PyPy");
}
pyo3_build_config::use_pyo3_cfgs();
}

1
guide/src/SUMMARY.md
View File

@ -19,6 +19,7 @@
- [Features Reference](features.md)
- [Advanced Topics](advanced.md)
- [Building and Distribution](building_and_distribution.md)
- [Supporting multiple Python versions](building_and_distribution/multiple_python_versions.md)
- [PyPy support](building_and_distribution/pypy.md)
- [Useful Crates](ecosystem.md)
- [Logging](ecosystem/logging.md)

13
guide/src/building_and_distribution.md
View File

@ -43,8 +43,7 @@ There are two ways to distribute your module as a Python package: [setuptools-ru
By default, Python extension modules can only be used with the same Python version they were compiled against -- if you build an extension module with Python 3.5, you can't import it using Python 3.8. [PEP 384](https://www.python.org/dev/peps/pep-0384/) introduced the idea of the limited Python API, which would have a stable ABI enabling extension modules built with it to be used against multiple Python versions. This is also known as `abi3`.
Note that [maturin] >= 0.9.0 or [setuptools-rust] >= 0.11.4 support `abi3` wheels.
See the [corresponding](https://github.com/PyO3/maturin/pull/353) [PRs](https://github.com/PyO3/setuptools-rust/pull/82) for more.
The advantage of building extension module using the limited Python API is that you only need to build and distribute a single copy (for each OS / architecture), and your users can install it on all Python versions from your [minimum version](#minimum-python-version-for-abi3) and up. The downside of this is that PyO3 can't use optimizations which rely on being compiled against a known exact Python version. It's up to you to decide whether this matters for your extension module. It's also possible to design your extension module such that you can distribute `abi3` wheels but allow users compiling from source to benefit from additional optimizations - see the [support for multiple python versions](./building_and_distribution/multiple_python_versions.html) section of this guide, in particular the `#[cfg(Py_LIMITED_API)]` flag.
There are three steps involved in making use of `abi3` when building Python packages as wheels:
@ -55,7 +54,8 @@ There are three steps involved in making use of `abi3` when building Python pack
pyo3 = { {{#PYO3_CRATE_VERSION}}, features = ["abi3"] }
```
2. Ensure that the built shared objects are correctly marked as `abi3`. This is accomplished by telling your build system that you're using the limited API.
2. Ensure that the built shared objects are correctly marked as `abi3`. This is accomplished by telling your build system that you're using the limited API. [maturin] >= 0.9.0 and [setuptools-rust] >= 0.11.4 support `abi3` wheels.
See the [corresponding](https://github.com/PyO3/maturin/pull/353) [PRs](https://github.com/PyO3/setuptools-rust/pull/82) for more.
3. Ensure that the `.whl` is correctly marked as `abi3`. For projects using `setuptools`, this is accomplished by passing `--py-limited-api=cp3x` (where `x` is the minimum Python version supported by the wheel, e.g. `--py-limited-api=cp35` for Python 3.5) to `setup.py bdist_wheel`.
@ -63,11 +63,15 @@ pyo3 = { {{#PYO3_CRATE_VERSION}}, features = ["abi3"] }
Because a single `abi3` wheel can be used with many different Python versions, PyO3 has feature flags `abi3-py36`, `abi3-py37`, `abi-py38` etc. to set the minimum required Python version for your `abi3` wheel.
For example, if you set the `abi3-py36` feature, your extension wheel can be used on all Python 3 versions from Python 3.6 and up. `maturin` and `setuptools-rust` will give the wheel a name like `my-extension-1.0-cp36-abi3-manylinux2020_x86_64.whl`.
If you set more that one of these api version feature flags the highest version always wins. For example, with both `abi3-py36` and `abi3-py38` set, PyO3 would build a wheel which supports Python 3.8 and up.
As your extension module may be run with multiple different Python versions you may occasionally find you need to check the Python version at runtime to customize behavior. See [the relevant section of this guide](./building_and_distribution/multiple_python_versions.html#checking-the-python-version-at-runtime) on supporting multiple Python versions at runtime.
PyO3 is only able to link your extension module to api3 version up to and including your host Python version. E.g., if you set `abi3-py38` and try to compile the crate with a host of Python 3.6, the build will fail.
As an advanced feature, you can build PyO3 wheel without calling Python interpreter with the environment variable `PYO3_NO_PYTHON` set. On unix systems this works unconditionally; on Windows you must also set the `RUSTFLAGS` evironment variable to contain `-L native=/path/to/python/libs` so that the linker can find `python3.lib`.
> Note: If you set more that one of these api version feature flags the highest version always wins. For example, with both `abi3-py36` and `abi3-py38` set, PyO3 would build a wheel which supports Python 3.8 and up.
### Missing features
Due to limitations in the Python API, there are a few `pyo3` features that do
@ -76,6 +80,7 @@ not work when compiling for `abi3`. These are:
- `#[text_signature]` does not work on classes until Python 3.10 or greater.
- The `dict` and `weakref` options on classes are not supported until Python 3.9 or greater.
- The buffer API is not supported.
- Optimizations which rely on knowledge of the exact Python version compiled against.
## Cross Compiling

103
guide/src/building_and_distribution/multiple_python_versions.md Normal file
View File

@ -0,0 +1,103 @@
# Supporting multiple Python versions
PyO3 supports all actively-supported Python 3 and PyPy versions. As much as possible, this is done internally to PyO3 so that your crate's code does not need to adapt to the differences between each version. However, as Python features grow and change between versions, PyO3 cannot a completely identical API for every Python version. This may require you to add conditional compilation to your crate or runtime checks for the Python version.
This section of the guide first introduces the `pyo3-build-config` crate, which you can use as a `build-dependency` to add additional `#[cfg]` flags which allow you to support multiple Python versions at compile-time.
Second, we'll show how to check the Python version at runtime. This can be useful when building for multiple versions with the `abi3` feature, where the Python API compiled against is not always the same as the one in use.
## Conditional compilation for different Python versions
The `pyo3-build-config` exposes multiple [`#[cfg]` flags](https://doc.rust-lang.org/rust-by-example/attribute/cfg.html) which can be used to conditionally compile code for a given Python version. PyO3 itself depends on this crate, so by using it you can be sure that you are configured correctly for the Python version PyO3 is building against.
This allows us to write code like the following
```rust,ignore
#[cfg(Py_3_7)]
fn function_only_supported_on_python_3_7_and_up() { }
#[cfg(not(Py_3_8))]
fn function_only_supported_before_python_3_8() { }
#[cfg(not(Py_LIMITED_API))]
fn function_incompatible_with_abi3_feature() { }
```
The following sections first show how to add these `#[cfg]` flags to your build process, and then cover some common patterns flags in a little more detail.
To see a full reference of all the `#[cfg]` flags provided, see the [`pyo3-build-cfg` docs](https://docs.rs/pyo3-build-config).
### Using `pyo3-build-config`
You can use the `#[cfg]` flags in just two steps:
1. Add `pyo3-build-config` it to your crate's build dependencies in `Cargo.toml`:
```toml
[build-dependencies]
pyo3-build-config = "{{#PYO3_CRATE_VERSION}}"
```
2. Add a [`build.rs`](https://doc.rust-lang.org/cargo/reference/build-scripts.html) file to your crate with the following contents:
```rust,ignore
fn main() {
// If you have an existing build.rs file, just add this line to it.
pyo3_build_config::use_pyo3_cfgs();
}
```
After these steps you are ready to annotate your code!
### Common usages of `pyo3-build-cfg` flags
The `#[cfg]` flags added by `pyo3-build-cfg` can be combined with all of Rust's logic in the `#[cfg]` attribute to create very precise conditional code generation. The following are some common patterns implemented using these flags:
```
#[cfg(Py_3_7)]
```
This `#[cfg]` marks code that will only be present on Python 3.7 and upwards. There are similar options `Py_3_8`, `Py_3_9`, `Py_3_10` and so on for each minor version.
```
#[cfg(not(Py_3_7))]
```
This `#[cfg]` marks code that will only be present on Python versions before (but not including) Python 3.7.
```
#[cfg(not(Py_LIMITED_API))]
```
This `#[cfg]` marks code that is only available when building for the unlimited Python API (i.e. PyO3's `abi3` feature is not enabled). This might be useful if you want to ship your extension module as an `abi3` wheel and also allow users to compile it from source to make use of optimizations only possible with the unlimited API.
```
#[cfg(any(Py_3_9, not(Py_LIMITED_API)))]
```
This `#[cfg]` marks code which is available when running Python 3.9 or newer, or when using the unlimited API with an older Python version. Patterns like this are commonly seen on Python APIs which were added to the limited Python API in a specific minor version.
```
#[cfg(PyPy)]
```
This `#[cfg]` marks code which is running on PyPy.
## Checking the Python version at runtime
When building with PyO3's `abi3` feature, your extension module will be compiled against a specific [minimum version](../building_and_distribution.html#minimum-python-version-for-abi3) of Python, but may be running on newer Python versions.
For example with PyO3's `abi3-py38` feature, your extension will be compiled as if it were for Python 3.8. If you were using `pyo3-build-config`, `#[cfg(Py_3_8)]` would be present. Your user could freely install and run your abi3 extension on Python 3.9.
There's no way to detect your user doing that at compile time, so instead you need to fall back to runtime checks.
PyO3 provides the APIs [`Python::version()`] and [`Python::version_info()`] to query the running Python version. This allows you to do the following, for example:
```rust,ignore
if py.version_info() >= (3, 9) {
// run this code only if Python 3.9 or up
}
```
[`Python::version()`]: {{#PYO3_DOCS_URL}}/pyo3/struct.Python.html#method.version
[`Python::version_info()`]: {{#PYO3_DOCS_URL}}/pyo3/struct.Python.html#method.version_info

2
guide/src/faq.md
View File

@ -21,7 +21,7 @@ Currently, [#341](https://github.com/PyO3/pyo3/issues/341) causes `cargo test` t
```toml
[dependencies.pyo3]
version = "{{#PYO3_VERSION}}"
version = "{{#PYO3_CRATE_VERSION}}"
[features]
extension-module = ["pyo3/extension-module"]

22
pyo3-build-config/Cargo.toml Normal file
View File

@ -0,0 +1,22 @@
[package]
name = "pyo3-build-config"
version = "0.14.0-alpha.0"
description = "Build configuration for the PyO3 ecosystem"
authors = ["PyO3 Project and Contributors <https://github.com/PyO3>"]
keywords = ["pyo3", "python", "cpython", "ffi"]
homepage = "https://github.com/pyo3/pyo3"
repository = "https://github.com/pyo3/pyo3"
categories = ["api-bindings", "development-tools::ffi"]
license = "Apache-2.0"
edition = "2018"
[dependencies]
[features]
default = []
abi3 = []
abi3-py36 = ["abi3-py37"]
abi3-py37 = ["abi3-py38"]
abi3-py38 = ["abi3-py39"]
abi3-py39 = ["abi3"]

1
pyo3-build-config/LICENSE Symbolic link
View File

@ -0,0 +1 @@
../LICENSE

11
pyo3-build-config/build.rs Normal file
View File

@ -0,0 +1,11 @@
#[allow(dead_code)]
#[path = "src/impl_.rs"]
mod impl_;
fn main() {
// Print out error messages using display, to get nicer formatting.
if let Err(e) = impl_::configure() {
eprintln!("error: {}", e);
std::process::exit(1)
}
}

832
pyo3-build-config/src/impl_.rs Normal file
View File

@ -0,0 +1,832 @@
use std::{
collections::{HashMap, HashSet},
convert::AsRef,
env,
ffi::OsString,
fmt::Display,
fs::{self, DirEntry, File},
io::Write,
path::{Path, PathBuf},
process::{Command, Stdio},
str::FromStr,
};
/// Minimum Python version PyO3 supports.
const MINIMUM_SUPPORTED_VERSION: PythonVersion = PythonVersion { major: 3, minor: 6 };
/// Maximum Python version that can be used as minimum required Python version with abi3.
const ABI3_MAX_MINOR: u8 = 9;
type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
// A simple macro for returning an error. Resembles anyhow::bail.
macro_rules! bail {
($msg: expr) => { return Err($msg.into()); };
($fmt: literal $($args: tt)+) => { return Err(format!($fmt $($args)+).into()); };
}
// A simple macro for checking a condition. Resembles anyhow::ensure.
macro_rules! ensure {
($condition:expr, $($args: tt)+) => { if !($condition) { bail!($($args)+) } };
}
// Show warning. If needed, please extend this macro to support arguments.
macro_rules! warn {
($msg: literal) => {
println!(concat!("cargo:warning=", $msg));
};
}
/// Gets an environment variable owned by cargo.
///
/// Environment variables set by cargo are expected to be valid UTF8.
fn cargo_env_var(var: &str) -> Option<String> {
env::var_os(var).map(|os_string| os_string.to_str().unwrap().into())
}
/// Gets an external environment variable, and registers the build script to rerun if
/// the variable changes.
fn env_var(var: &str) -> Option<OsString> {
println!("cargo:rerun-if-env-changed={}", var);
env::var_os(var)
}
/// Configuration needed by PyO3 to build for the correct Python implementation.
///
/// Usually this is queried directly from the Python interpreter. When the `PYO3_NO_PYTHON` variable
/// is set, or during cross compile situations, then alternative strategies are used to populate
/// this type.
#[derive(Debug)]
pub struct InterpreterConfig {
pub version: PythonVersion,
pub libdir: Option<String>,
pub shared: bool,
pub abi3: bool,
pub ld_version: Option<String>,
pub base_prefix: Option<String>,
pub executable: Option<String>,
pub calcsize_pointer: Option<u32>,
pub implementation: PythonImplementation,
pub build_flags: BuildFlags,
}
impl InterpreterConfig {
pub fn emit_pyo3_cfgs(&self) {
// This should have been checked during pyo3-build-config build time.
assert!(self.version >= MINIMUM_SUPPORTED_VERSION);
for i in MINIMUM_SUPPORTED_VERSION.minor..=self.version.minor {
println!("cargo:rustc-cfg=Py_3_{}", i);
}
if self.abi3 {
println!("cargo:rustc-cfg=Py_LIMITED_API");
}
if self.is_pypy() {
println!("cargo:rustc-cfg=PyPy");
if self.abi3 {
warn!(
"PyPy does not yet support abi3 so the build artifacts will be version-specific. \
See https://foss.heptapod.net/pypy/pypy/-/issues/3397 for more information."
)
}
};
for flag in &self.build_flags.0 {
println!("cargo:rustc-cfg=py_sys_config=\"{}\"", flag)
}
}
pub fn is_pypy(&self) -> bool {
self.implementation == PythonImplementation::PyPy
}
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub struct PythonVersion {
pub major: u8,
pub minor: u8,
}
impl PythonVersion {
const PY37: Self = PythonVersion { major: 3, minor: 7 };
}
impl Display for PythonVersion {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}.{}", self.major, self.minor)
}
}
#[derive(Debug, Copy, Clone, PartialEq)]
pub enum PythonImplementation {
CPython,
PyPy,
}
impl FromStr for PythonImplementation {
type Err = Box<dyn std::error::Error>;
fn from_str(s: &str) -> Result<Self> {
match s {
"CPython" => Ok(PythonImplementation::CPython),
"PyPy" => Ok(PythonImplementation::PyPy),
_ => bail!("Invalid interpreter: {}", s),
}
}
}
fn is_abi3() -> bool {
cargo_env_var("CARGO_FEATURE_ABI3").is_some()
}
trait GetPrimitive {
fn get_bool(&self, key: &str) -> Result<bool>;
fn get_numeric<T: FromStr>(&self, key: &str) -> Result<T>;
}
impl GetPrimitive for HashMap<String, String> {
fn get_bool(&self, key: &str) -> Result<bool> {
match self
.get(key)
.map(|x| x.as_str())
.ok_or(format!("{} is not defined", key))?
{
"1" | "true" | "True" => Ok(true),
"0" | "false" | "False" => Ok(false),
_ => bail!("{} must be a bool (1/true/True or 0/false/False", key),
}
}
fn get_numeric<T: FromStr>(&self, key: &str) -> Result<T> {
self.get(key)
.ok_or(format!("{} is not defined", key))?
.parse::<T>()
.map_err(|_| format!("Could not parse value of {}", key).into())
}
}
struct CrossCompileConfig {
lib_dir: PathBuf,
version: Option<String>,
os: String,
arch: String,
}
fn cross_compiling() -> Result<Option<CrossCompileConfig>> {
let cross = env_var("PYO3_CROSS");
let cross_lib_dir = env_var("PYO3_CROSS_LIB_DIR");
let cross_python_version = env_var("PYO3_CROSS_PYTHON_VERSION");
let target_arch = cargo_env_var("CARGO_CFG_TARGET_ARCH");
let target_vendor = cargo_env_var("CARGO_CFG_TARGET_VENDOR");
let target_os = cargo_env_var("CARGO_CFG_TARGET_OS");
if cross.is_none() && cross_lib_dir.is_none() && cross_python_version.is_none() {
// No cross-compiling environment variables set; try to determine if this is a known case
// which is not cross-compilation.
let target = cargo_env_var("TARGET").unwrap();
let host = cargo_env_var("HOST").unwrap();
if target == host {
// Not cross-compiling
return Ok(None);
}
if target == "i686-pc-windows-msvc" && host == "x86_64-pc-windows-msvc" {
// Not cross-compiling to compile for 32-bit Python from windows 64-bit
return Ok(None);
}
if target == "x86_64-apple-darwin" && host == "aarch64-apple-darwin" {
// Not cross-compiling to compile for x86-64 Python from macOS arm64
return Ok(None);
}
if target == "aarch64-apple-darwin" && host == "x86_64-apple-darwin" {
// Not cross-compiling to compile for arm64 Python from macOS x86_64
return Ok(None);
}
if let (Some(arch), Some(vendor), Some(os)) = (&target_arch, &target_vendor, &target_os) {
if host.starts_with(&format!("{}-{}-{}", arch, vendor, os)) {
// Not cross-compiling if arch-vendor-os is all the same
// e.g. x86_64-unknown-linux-musl on x86_64-unknown-linux-gnu host
return Ok(None);
}
}
}
// At this point we assume that we are cross compiling.
Ok(Some(CrossCompileConfig {
lib_dir: cross_lib_dir
.ok_or("The PYO3_CROSS_LIB_DIR environment variable must be set when cross-compiling")?
.into(),
os: target_os.unwrap(),
arch: target_arch.unwrap(),
version: cross_python_version
.map(|os_string| {
os_string
.to_str()
.ok_or("PYO3_CROSS_PYTHON_VERSION is not valid utf-8.")
.map(str::to_owned)
})
.transpose()?,
}))
}
/// A list of python interpreter compile-time preprocessor defines that
/// we will pick up and pass to rustc via `--cfg=py_sys_config={varname}`;
/// this allows using them conditional cfg attributes in the .rs files, so
///
/// #[cfg(py_sys_config="{varname}"]
///
/// is the equivalent of `#ifdef {varname}` in C.
///
/// see Misc/SpecialBuilds.txt in the python source for what these mean.
#[derive(Debug)]
pub struct BuildFlags(pub HashSet<&'static str>);
impl BuildFlags {
const ALL: [&'static str; 5] = [
// TODO: Remove WITH_THREAD once Python 3.6 support dropped (as it's always on).
"WITH_THREAD",
"Py_DEBUG",
"Py_REF_DEBUG",
"Py_TRACE_REFS",
"COUNT_ALLOCS",
];
fn from_config_map(config_map: &HashMap<String, String>) -> Self {
Self(
BuildFlags::ALL
.iter()
.copied()
.filter(|flag| config_map.get(*flag).map_or(false, |value| value == "1"))
.collect(),
)
}
/// Examine python's compile flags to pass to cfg by launching
/// the interpreter and printing variables of interest from
/// sysconfig.get_config_vars.
fn from_interpreter(interpreter: &Path) -> Result<Self> {
if cargo_env_var("CARGO_CFG_TARGET_OS").unwrap() == "windows" {
return Ok(Self::windows_hardcoded());
}
let mut script = String::from("import sysconfig\n");
script.push_str("config = sysconfig.get_config_vars()\n");
for k in BuildFlags::ALL.iter() {
script.push_str(&format!("print(config.get('{}', '0'))\n", k));
}
let stdout = run_python_script(&interpreter, &script)?;
let split_stdout: Vec<&str> = stdout.trim_end().lines().collect();
ensure!(
split_stdout.len() == BuildFlags::ALL.len(),
"Python stdout len didn't return expected number of lines: {}",
split_stdout.len()
);
let flags = BuildFlags::ALL
.iter()
.zip(split_stdout)
.filter(|(_, flag_value)| *flag_value == "1")
.map(|(&flag, _)| flag)
.collect();
Ok(Self(flags))
}
fn windows_hardcoded() -> Self {
// sysconfig is missing all the flags on windows, so we can't actually
// query the interpreter directly for its build flags.
let mut flags = HashSet::new();
flags.insert("WITH_THREAD");
// Uncomment these manually if your python was built with these and you want
// the cfg flags to be set in rust.
//
// map.insert("Py_DEBUG", "1");
// map.insert("Py_REF_DEBUG", "1");
// map.insert("Py_TRACE_REFS", "1");
// map.insert("COUNT_ALLOCS", 1");
Self(flags)
}
fn abi3() -> Self {
let mut flags = HashSet::new();
flags.insert("WITH_THREAD");
Self(flags)
}
fn fixup(mut self, version: PythonVersion, implementation: PythonImplementation) -> Self {
if self.0.contains("Py_DEBUG") {
self.0.insert("Py_REF_DEBUG");
if version <= PythonVersion::PY37 {
// Py_DEBUG only implies Py_TRACE_REFS until Python 3.7
self.0.insert("Py_TRACE_REFS");
}
}
// WITH_THREAD is always on for Python 3.7, and for PyPy.
if implementation == PythonImplementation::PyPy || version >= PythonVersion::PY37 {
self.0.insert("WITH_THREAD");
}
self
}
}
fn parse_script_output(output: &str) -> HashMap<String, String> {
output
.lines()
.filter_map(|line| {
let mut i = line.splitn(2, ' ');
Some((i.next()?.into(), i.next()?.into()))
})
.collect()
}
/// Parse sysconfigdata file
///
/// The sysconfigdata is simply a dictionary containing all the build time variables used for the
/// python executable and library. Here it is read and added to a script to extract only what is
/// necessary. This necessitates a python interpreter for the host machine to work.
fn parse_sysconfigdata(config_path: impl AsRef<Path>) -> Result<HashMap<String, String>> {
let mut script = fs::read_to_string(config_path)?;
script += r#"
print("version_major", build_time_vars["VERSION"][0]) # 3
print("version_minor", build_time_vars["VERSION"][2]) # E.g., 8
KEYS = [
"WITH_THREAD",
"Py_DEBUG",
"Py_REF_DEBUG",
"Py_TRACE_REFS",
"COUNT_ALLOCS",
"Py_ENABLE_SHARED",
"LDVERSION",
"SIZEOF_VOID_P"
]
for key in KEYS:
print(key, build_time_vars.get(key, 0))
"#;
let output = run_python_script(&find_interpreter()?, &script)?;
Ok(parse_script_output(&output))
}
fn starts_with(entry: &DirEntry, pat: &str) -> bool {
let name = entry.file_name();
name.to_string_lossy().starts_with(pat)
}
fn ends_with(entry: &DirEntry, pat: &str) -> bool {
let name = entry.file_name();
name.to_string_lossy().ends_with(pat)
}
/// Finds the `_sysconfigdata*.py` file in the library path.
///
/// From the python source for `_sysconfigdata*.py` is always going to be located at
/// `build/lib.{PLATFORM}-{PY_MINOR_VERSION}` when built from source. The [exact line][1] is defined as:
///
/// ```py
/// pybuilddir = 'build/lib.%s-%s' % (get_platform(), sys.version_info[:2])
/// ```
///
/// Where get_platform returns a kebab-case formated string containing the os, the architecture and
/// possibly the os' kernel version (not the case on linux). However, when installed using a package
/// manager, the `_sysconfigdata*.py` file is installed in the `${PREFIX}/lib/python3.Y/` directory.
/// The `_sysconfigdata*.py` is generally in a sub-directory of the location of `libpython3.Y.so`.
/// So we must find the file in the following possible locations:
///
/// ```sh
/// # distribution from package manager, lib_dir should include lib/
/// ${INSTALL_PREFIX}/lib/python3.Y/_sysconfigdata*.py
/// ${INSTALL_PREFIX}/lib/libpython3.Y.so
/// ${INSTALL_PREFIX}/lib/python3.Y/config-3.Y-${HOST_TRIPLE}/libpython3.Y.so
///
/// # Built from source from host
/// ${CROSS_COMPILED_LOCATION}/build/lib.linux-x86_64-Y/_sysconfigdata*.py
/// ${CROSS_COMPILED_LOCATION}/libpython3.Y.so
///
/// # if cross compiled, kernel release is only present on certain OS targets.
/// ${CROSS_COMPILED_LOCATION}/build/lib.{OS}(-{OS-KERNEL-RELEASE})?-{ARCH}-Y/_sysconfigdata*.py
/// ${CROSS_COMPILED_LOCATION}/libpython3.Y.so
/// ```
///
/// [1]: https://github.com/python/cpython/blob/3.5/Lib/sysconfig.py#L389
fn find_sysconfigdata(cross: &CrossCompileConfig) -> Result<PathBuf> {
let sysconfig_paths = search_lib_dir(&cross.lib_dir, &cross);
let sysconfig_name = env_var("_PYTHON_SYSCONFIGDATA_NAME");
let mut sysconfig_paths = sysconfig_paths
.iter()
.filter_map(|p| {
let canonical = fs::canonicalize(p).ok();
match &sysconfig_name {
Some(_) => canonical.filter(|p| p.file_stem() == sysconfig_name.as_deref()),
None => canonical,
}
})
.collect::<Vec<PathBuf>>();
sysconfig_paths.dedup();
if sysconfig_paths.is_empty() {
bail!(
"Could not find either libpython.so or _sysconfigdata*.py in {}",
cross.lib_dir.display()
);
} else if sysconfig_paths.len() > 1 {
let mut error_msg = String::from(
"Detected multiple possible Python versions. Please set either the \
PYO3_CROSS_PYTHON_VERSION variable to the wanted version or the \
_PYTHON_SYSCONFIGDATA_NAME variable to the wanted sysconfigdata file name.\n\n\
sysconfigdata files found:",
);
for path in sysconfig_paths {
error_msg += &format!("\n\t{}", path.display());
}
bail!("{}", error_msg);
}
Ok(sysconfig_paths.remove(0))
}
/// recursive search for _sysconfigdata, returns all possibilities of sysconfigdata paths
fn search_lib_dir(path: impl AsRef<Path>, cross: &CrossCompileConfig) -> Vec<PathBuf> {
let mut sysconfig_paths = vec![];
let version_pat = if let Some(v) = &cross.version {
format!("python{}", v)
} else {
"python3.".into()
};
for f in fs::read_dir(path).expect("Path does not exist") {
let sysc = match &f {
Ok(f) if starts_with(f, "_sysconfigdata") && ends_with(f, "py") => vec![f.path()],
Ok(f) if starts_with(f, "build") => search_lib_dir(f.path(), cross),
Ok(f) if starts_with(f, "lib.") => {
let name = f.file_name();
// check if right target os
if !name.to_string_lossy().contains(if cross.os == "android" {
"linux"
} else {
&cross.os
}) {
continue;
}
// Check if right arch
if !name.to_string_lossy().contains(&cross.arch) {
continue;
}
search_lib_dir(f.path(), cross)
}
Ok(f) if starts_with(f, &version_pat) => search_lib_dir(f.path(), cross),
_ => continue,
};
sysconfig_paths.extend(sysc);
}
sysconfig_paths
}
/// Find cross compilation information from sysconfigdata file
///
/// first find sysconfigdata file which follows the pattern [`_sysconfigdata_{abi}_{platform}_{multiarch}`][1]
/// on python 3.6 or greater. On python 3.5 it is simply `_sysconfigdata.py`.
///
/// [1]: https://github.com/python/cpython/blob/3.8/Lib/sysconfig.py#L348
fn load_cross_compile_from_sysconfigdata(
cross_compile_config: CrossCompileConfig,
) -> Result<InterpreterConfig> {
let sysconfig_path = find_sysconfigdata(&cross_compile_config)?;
let sysconfig_data = parse_sysconfigdata(sysconfig_path)?;
let major = sysconfig_data.get_numeric("version_major")?;
let minor = sysconfig_data.get_numeric("version_minor")?;
let ld_version = match sysconfig_data.get("LDVERSION") {
Some(s) => s.clone(),
None => format!("{}.{}", major, minor),
};
let calcsize_pointer = sysconfig_data.get_numeric("SIZEOF_VOID_P").ok();
let version = PythonVersion { major, minor };
let implementation = PythonImplementation::CPython;
Ok(InterpreterConfig {
version,
libdir: cross_compile_config.lib_dir.to_str().map(String::from),
shared: sysconfig_data.get_bool("Py_ENABLE_SHARED")?,
abi3: is_abi3(),
ld_version: Some(ld_version),
base_prefix: None,
executable: None,
calcsize_pointer,
implementation,
build_flags: BuildFlags::from_config_map(&sysconfig_data).fixup(version, implementation),
})
}
fn windows_hardcoded_cross_compile(
cross_compile_config: CrossCompileConfig,
) -> Result<InterpreterConfig> {
let (major, minor) = if let Some(version) = cross_compile_config.version {
let mut parts = version.split('.');
match (
parts.next().and_then(|major| major.parse().ok()),
parts.next().and_then(|minor| minor.parse().ok()),
parts.next(),
) {
(Some(major), Some(minor), None) => (major, minor),
_ => bail!(
"Expected major.minor version (e.g. 3.9) for PYO3_CROSS_PYTHON_VERSION, got `{}`",
version
),
}
} else if let Some(minor_version) = get_abi3_minor_version() {
(3, minor_version)
} else {
bail!("PYO3_CROSS_PYTHON_VERSION or an abi3-py3* feature must be specified when cross-compiling for Windows.")
};
Ok(InterpreterConfig {
version: PythonVersion { major, minor },
libdir: cross_compile_config.lib_dir.to_str().map(String::from),
shared: true,
abi3: is_abi3(),
ld_version: None,
base_prefix: None,
executable: None,
calcsize_pointer: None,
implementation: PythonImplementation::CPython,
build_flags: BuildFlags::windows_hardcoded(),
})
}
fn load_cross_compile_info(cross_compile_config: CrossCompileConfig) -> Result<InterpreterConfig> {
match cargo_env_var("CARGO_CFG_TARGET_FAMILY") {
// Configure for unix platforms using the sysconfigdata file
Some(os) if os == "unix" => load_cross_compile_from_sysconfigdata(cross_compile_config),
// Use hardcoded interpreter config when targeting Windows
Some(os) if os == "windows" => windows_hardcoded_cross_compile(cross_compile_config),
// sysconfigdata works fine on wasm/wasi
Some(os) if os == "wasm" => load_cross_compile_from_sysconfigdata(cross_compile_config),
// Waiting for users to tell us what they expect on their target platform
Some(os) => bail!(
"Unsupported target OS family for cross-compilation: {:?}",
os
),
// Unknown os family - try to do something useful
None => load_cross_compile_from_sysconfigdata(cross_compile_config),
}
}
/// Run a python script using the specified interpreter binary.
fn run_python_script(interpreter: &Path, script: &str) -> Result<String> {
let out = Command::new(interpreter)
.env("PYTHONIOENCODING", "utf-8")
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::inherit())
.spawn()
.and_then(|mut child| {
child
.stdin
.as_mut()
.expect("piped stdin")
.write_all(script.as_bytes())?;
child.wait_with_output()
});
match out {
Err(err) => bail!(
"failed to run the Python interpreter at {}: {}",
interpreter.display(),
err
),
Ok(ok) if !ok.status.success() => bail!("Python script failed"),
Ok(ok) => Ok(String::from_utf8(ok.stdout)?),
}
}
fn get_venv_path() -> Option<PathBuf> {
match (env_var("VIRTUAL_ENV"), env_var("CONDA_PREFIX")) {
(Some(dir), None) => Some(PathBuf::from(dir)),
(None, Some(dir)) => Some(PathBuf::from(dir)),
(Some(_), Some(_)) => {
warn!(
"Both VIRTUAL_ENV and CONDA_PREFIX are set. PyO3 will ignore both of these for \
locating the Python interpreter until you unset one of them."
);
None
}
(None, None) => None,
}
}
/// Attempts to locate a python interpreter. Locations are checked in the order listed:
/// 1. If `PYO3_PYTHON` is set, this intepreter is used.
/// 2. If in a virtualenv, that environment's interpreter is used.
/// 3. `python`, if this is functional a Python 3.x interpreter
/// 4. `python3`, as above
fn find_interpreter() -> Result<PathBuf> {
if let Some(exe) = env_var("PYO3_PYTHON") {
Ok(exe.into())
} else if let Some(venv_path) = get_venv_path() {
match cargo_env_var("CARGO_CFG_TARGET_OS").unwrap().as_str() {
"windows" => Ok(venv_path.join("Scripts\\python")),
_ => Ok(venv_path.join("bin/python")),
}
} else {
println!("cargo:rerun-if-env-changed=PATH");
["python", "python3"]
.iter()
.find(|bin| {
if let Ok(out) = Command::new(bin).arg("--version").output() {
// begin with `Python 3.X.X :: additional info`
out.stdout.starts_with(b"Python 3") || out.stderr.starts_with(b"Python 3")
} else {
false
}
})
.map(PathBuf::from)
.ok_or_else(|| "no Python 3.x interpreter found".into())
}
}
/// Extract compilation vars from the specified interpreter.
fn get_config_from_interpreter(interpreter: &Path) -> Result<InterpreterConfig> {
let script = r#"
# Allow the script to run on Python 2, so that nicer error can be printed later.
from __future__ import print_function
import os.path
import platform
import struct
import sys
from sysconfig import get_config_var
PYPY = platform.python_implementation() == "PyPy"
# sys.base_prefix is missing on Python versions older than 3.3; this allows the script to continue
# so that the version mismatch can be reported in a nicer way later.
base_prefix = getattr(sys, "base_prefix", None)
if base_prefix:
# Anaconda based python distributions have a static python executable, but include
# the shared library. Use the shared library for embedding to avoid rust trying to
# LTO the static library (and failing with newer gcc's, because it is old).
ANACONDA = os.path.exists(os.path.join(base_prefix, "conda-meta"))
else:
ANACONDA = False
def print_if_set(varname, value):
if value is not None:
print(varname, value)
libdir = get_config_var("LIBDIR")
print("version_major", sys.version_info[0])
print("version_minor", sys.version_info[1])
print("implementation", platform.python_implementation())
print_if_set("libdir", libdir)
print_if_set("ld_version", get_config_var("LDVERSION"))
print_if_set("base_prefix", base_prefix)
print("framework", bool(get_config_var("PYTHONFRAMEWORK")))
print("shared", PYPY or ANACONDA or bool(get_config_var("Py_ENABLE_SHARED")))
print("executable", sys.executable)
print("calcsize_pointer", struct.calcsize("P"))
"#;
let output = run_python_script(interpreter, script)?;
let map: HashMap<String, String> = parse_script_output(&output);
let shared = match (
cargo_env_var("CARGO_CFG_TARGET_OS").unwrap().as_str(),
map["framework"].as_str(),
map["shared"].as_str(),
) {
(_, _, "True") // Py_ENABLE_SHARED is set
| ("windows", _, _) // Windows always uses shared linking
| ("macos", "True", _) // MacOS framework package uses shared linking
=> true,
(_, _, "False") => false, // Any other platform, Py_ENABLE_SHARED not set
_ => bail!("Unrecognised link model combination")
};
let version = PythonVersion {
major: map["version_major"].parse()?,
minor: map["version_minor"].parse()?,
};
let implementation = map["implementation"].parse()?;
Ok(InterpreterConfig {
version,
implementation,
libdir: map.get("libdir").cloned(),
shared,
abi3: is_abi3(),
ld_version: map.get("ld_version").cloned(),
base_prefix: map.get("base_prefix").cloned(),
executable: map.get("executable").cloned(),
calcsize_pointer: Some(map["calcsize_pointer"].parse()?),
build_flags: BuildFlags::from_interpreter(interpreter)?.fixup(version, implementation),
})
}
fn get_abi3_minor_version() -> Option<u8> {
(MINIMUM_SUPPORTED_VERSION.minor..=ABI3_MAX_MINOR)
.find(|i| cargo_env_var(&format!("CARGO_FEATURE_ABI3_PY3{}", i)).is_some())
}
fn get_interpreter_config() -> Result<InterpreterConfig> {
let abi3_version = get_abi3_minor_version();
// If PYO3_NO_PYTHON is set with abi3, we can build PyO3 without calling Python.
if let Some(abi3_minor_version) = abi3_version {
if env_var("PYO3_NO_PYTHON").is_some() {
return Ok(InterpreterConfig {
version: PythonVersion {
major: 3,
minor: abi3_minor_version,
},
implementation: PythonImplementation::CPython,
abi3: true,
libdir: None,
build_flags: BuildFlags::abi3(),
base_prefix: None,
calcsize_pointer: None,
executable: None,
ld_version: None,
shared: true,
});
}
}
let mut interpreter_config = if let Some(paths) = cross_compiling()? {
load_cross_compile_info(paths)?
} else {
get_config_from_interpreter(&find_interpreter()?)?
};
// Fixup minor version if abi3-pyXX feature set
if let Some(abi3_minor_version) = abi3_version {
ensure!(
abi3_minor_version <= interpreter_config.version.minor,
"You cannot set a mininimum Python version 3.{} higher than the interpreter version 3.{}",
abi3_minor_version,
interpreter_config.version.minor
);
interpreter_config.version.minor = abi3_minor_version;
}
Ok(interpreter_config)
}
pub fn configure() -> Result<()> {
let interpreter_config = get_interpreter_config()?;
write_interpreter_config(&interpreter_config)
}
fn write_interpreter_config(interpreter_config: &InterpreterConfig) -> Result<()> {
let out_dir = env::var_os("OUT_DIR").unwrap();
let mut out = File::create(Path::new(&out_dir).join("pyo3-build-config.rs"))?;
writeln!(out, "{{")?;
writeln!(
out,
"let mut build_flags = std::collections::HashSet::new();"
)?;
for flag in &interpreter_config.build_flags.0 {
writeln!(out, "build_flags.insert({:?});", flag)?;
}
writeln!(
out,
r#"crate::impl_::InterpreterConfig {{
version: crate::impl_::PythonVersion {{
major: {major},
minor: {minor},
}},
implementation: crate::impl_::PythonImplementation::{implementation:?},
libdir: {libdir:?}.map(|str: &str| str.to_string()),
abi3: {abi3},
build_flags: crate::impl_::BuildFlags(build_flags),
base_prefix: {base_prefix:?}.map(|str: &str| str.to_string()),
calcsize_pointer: {calcsize_pointer:?},
executable: {executable:?}.map(|str: &str| str.to_string()),
ld_version: {ld_version:?}.map(|str: &str| str.to_string()),
shared: {shared:?}
}}"#,
major = interpreter_config.version.major,
minor = interpreter_config.version.minor,
implementation = interpreter_config.implementation,
base_prefix = interpreter_config.base_prefix,
calcsize_pointer = interpreter_config.calcsize_pointer,
executable = interpreter_config.executable,
ld_version = interpreter_config.ld_version,
libdir = interpreter_config.libdir,
shared = interpreter_config.shared,
abi3 = interpreter_config.abi3,
)?;
writeln!(out, "}}")?;
Ok(())
}

30
pyo3-build-config/src/lib.rs Normal file
View File

@ -0,0 +1,30 @@
//! Configuration used by PyO3 for conditional support of varying Python versions.
//!
//! The only public API currently exposed is [`use_pyo3_cfgs`], which is intended to be used in
//! build scripts to add a standard set of `#[cfg]` attributes for handling multiple Python
//! versions.
//!
//! The full list of attributes added are the following:
//!
//! | Flag | Description |
//! | ---- | ----------- |
//! | `#[cfg(Py_3_6)]`, `#[cfg(Py_3_7)]`, `#[cfg(Py_3_8)]`, `#[cfg(Py_3_9)]`, `#[cfg(Py_3_10)]` | These attributes mark code only for a given Python version and up. For example, `#[cfg(Py_3_6)]` marks code which can run on Python 3.6 **and newer**. |
//! | `#[cfg(Py_LIMITED_API)]` | This marks code which is run when compiling with PyO3's `abi3` feature enabled. |
//! | `#[cfg(PyPy)]` | This marks code which is run when compiling for PyPy. |
//!
//! For examples of how to use these attributes, [see PyO3's guide](https://pyo3.rs/main/building_and_distribution/multiple_python_versions.html).
#[allow(dead_code)] // TODO cover this using tests
mod impl_;
#[doc(hidden)]
pub use crate::impl_::{InterpreterConfig, PythonImplementation, PythonVersion};
#[doc(hidden)]
pub fn get() -> InterpreterConfig {
include!(concat!(env!("OUT_DIR"), "/pyo3-build-config.rs"))
}
pub fn use_pyo3_cfgs() {
get().emit_pyo3_cfgs();
}

3
pyo3-macros-backend/Cargo.toml
View File

@ -21,3 +21,6 @@ proc-macro2 = { version = "1", default-features = false }
version = "1"
default-features = false
features = ["derive", "parsing", "printing", "clone-impls", "full", "extra-traits"]
[build-dependencies]
pyo3-build-config = { path = "../pyo3-build-config", version = "=0.14.0-alpha.0" }

3
pyo3-macros-backend/build.rs Normal file
View File

@ -0,0 +1,3 @@
fn main() {
pyo3_build_config::use_pyo3_cfgs();
}

6
pyo3-macros-backend/src/pyproto.rs
View File

@ -132,10 +132,8 @@ fn impl_proto_methods(
let slots_trait_slots = proto.slots_trait_slots();
let mut maybe_buffer_methods = None;
if proto.name == "Buffer" {
// On Python 3.9 we have to use PyBufferProcs to set buffer slots.
// For now we emit this always for buffer methods, even on 3.9+.
// Maybe in the future we can access Py_3_9 here and define it.
if cfg!(not(Py_3_9)) && proto.name == "Buffer" {
maybe_buffer_methods = Some(quote! {
impl pyo3::class::impl_::PyBufferProtocolProcs<#ty>
for pyo3::class::impl_::PyClassImplCollector<#ty>