Expose cross compiling configuration from pyo3-build-config (#1996)
* Initial refactor - expose cross-compiling functions and add necessary fields to InterpreterConfig * Refactor cross_compiling to take arch/vendor/os separately. * Address review comments. * Update changelog with note about pyo3-build-config APIs. * Fix panic when parsing ABI tag on Windows. * Update parse_sysconfigdata test to best-guess values for linux. * Revert added fields in InterpreterConfig. * Refactor parse_sysconfigdata to return Sysconfigdata (HashMap). Add InterpreterConfig::from_sysconfigdata. * Update BuildFlags test to use from_sysconfigdata. * Add tests for from_sysconfigdata. Refactor Sysconfigdata API to be more open. * Add basic tests for not cross compiling. Add some error handling. * Address review comments. * Update search_lib_dir to recurse into lib and pypy dirs. * Look even harder for sysconfigdata. * Add skip-build-config feature. * Revert skip-build-config feature. * Suppress cargo:rerun-if-env-changed without resolve-config feature.
This commit is contained in:
Ashley Anderson
GitHub
parent
233a14fa1e
commit
c30ca03b60
|
|
@ -55,6 +55,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|||
|
||||
- Add implementations for `Py::as_ref()` and `Py::into_ref()` for `Py<PySequence>`, `Py<PyIterator>` and `Py<PyMapping>`. [#1682](https://github.com/PyO3/pyo3/pull/1682)
|
||||
- Add `PyTraceback` type to represent and format Python tracebacks. [#1977](https://github.com/PyO3/pyo3/pull/1977)
|
||||
- Expose `pyo3-build-config` APIs for cross-compiling and Python configuration discovery for use in other projects. [#1996](https://github.com/PyO3/pyo3/pull/1996)
|
||||
|
||||
### Changed
|
||||
|
||||
|
|
|
|||
|
|
@ -32,7 +32,9 @@ pub fn cargo_env_var(var: &str) -> Option<String> {
|
|||
/// Gets an external environment variable, and registers the build script to rerun if
|
||||
/// the variable changes.
|
||||
pub fn env_var(var: &str) -> Option<OsString> {
|
||||
println!("cargo:rerun-if-env-changed={}", var);
|
||||
if cfg!(feature = "resolve-config") {
|
||||
println!("cargo:rerun-if-env-changed={}", var);
|
||||
}
|
||||
env::var_os(var)
|
||||
}
|
||||
|
||||
|
|
@ -219,6 +221,7 @@ print("mingw", get_platform().startswith("mingw"))
|
|||
};
|
||||
|
||||
let abi3 = is_abi3();
|
||||
|
||||
let implementation = map["implementation"].parse()?;
|
||||
|
||||
let lib_name = if cfg!(windows) {
|
||||
|
|
@ -267,6 +270,61 @@ print("mingw", get_platform().startswith("mingw"))
|
|||
})
|
||||
}
|
||||
|
||||
/// Generate from parsed sysconfigdata file
|
||||
///
|
||||
/// Use [`parse_sysconfigdata`] to generate a hash map of configuration values which may be
|
||||
/// used to build an [`InterpreterConfig`].
|
||||
pub fn from_sysconfigdata(sysconfigdata: &Sysconfigdata) -> Result<Self> {
|
||||
macro_rules! get_key {
|
||||
($sysconfigdata:expr, $key:literal) => {
|
||||
$sysconfigdata
|
||||
.get_value($key)
|
||||
.ok_or(concat!($key, " not found in sysconfigdata file"))
|
||||
};
|
||||
}
|
||||
|
||||
macro_rules! parse_key {
|
||||
($sysconfigdata:expr, $key:literal) => {
|
||||
get_key!($sysconfigdata, $key)?
|
||||
.parse()
|
||||
.context(concat!("could not parse value of ", $key))
|
||||
};
|
||||
}
|
||||
|
||||
let soabi = get_key!(sysconfigdata, "SOABI")?;
|
||||
let implementation = PythonImplementation::from_soabi(soabi)?;
|
||||
let version = parse_key!(sysconfigdata, "VERSION")?;
|
||||
let shared = match sysconfigdata.get_value("Py_ENABLE_SHARED") {
|
||||
Some("1") | Some("true") | Some("True") => true,
|
||||
Some("0") | Some("false") | Some("False") => false,
|
||||
_ => bail!("expected a bool (1/true/True or 0/false/False) for Py_ENABLE_SHARED"),
|
||||
};
|
||||
let lib_dir = get_key!(sysconfigdata, "LIBDIR").ok().map(str::to_string);
|
||||
let lib_name = Some(default_lib_name_unix(
|
||||
version,
|
||||
implementation,
|
||||
sysconfigdata.get_value("LDVERSION"),
|
||||
));
|
||||
let pointer_width = parse_key!(sysconfigdata, "SIZEOF_VOID_P")
|
||||
.map(|bytes_width: u32| bytes_width * 8)
|
||||
.ok();
|
||||
let build_flags = BuildFlags::from_sysconfigdata(sysconfigdata).fixup(version);
|
||||
|
||||
Ok(InterpreterConfig {
|
||||
implementation,
|
||||
version,
|
||||
shared,
|
||||
abi3: is_abi3(),
|
||||
lib_dir,
|
||||
lib_name,
|
||||
executable: None,
|
||||
pointer_width,
|
||||
build_flags,
|
||||
suppress_build_script_link_lines: false,
|
||||
extra_build_script_lines: vec![],
|
||||
})
|
||||
}
|
||||
|
||||
#[doc(hidden)]
|
||||
pub fn from_path(path: impl AsRef<Path>) -> Result<Self> {
|
||||
let path = path.as_ref();
|
||||
|
|
@ -445,6 +503,17 @@ impl PythonImplementation {
|
|||
pub fn is_pypy(self) -> bool {
|
||||
self == PythonImplementation::PyPy
|
||||
}
|
||||
|
||||
#[doc(hidden)]
|
||||
pub fn from_soabi(soabi: &str) -> Result<Self> {
|
||||
if soabi.starts_with("pypy") {
|
||||
Ok(PythonImplementation::PyPy)
|
||||
} else if soabi.starts_with("cpython") {
|
||||
Ok(PythonImplementation::CPython)
|
||||
} else {
|
||||
bail!("unsupported Python interpreter");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Display for PythonImplementation {
|
||||
|
|
@ -471,11 +540,32 @@ fn is_abi3() -> bool {
|
|||
cargo_env_var("CARGO_FEATURE_ABI3").is_some()
|
||||
}
|
||||
|
||||
struct CrossCompileConfig {
|
||||
lib_dir: PathBuf,
|
||||
/// Configuration needed by PyO3 to cross-compile for a target platform.
|
||||
///
|
||||
/// Usually this is collected from the environment (i.e. `PYO3_CROSS_*` and `CARGO_CFG_TARGET_*`)
|
||||
/// when a cross-compilation configuration is detected.
|
||||
#[derive(Debug, PartialEq)]
|
||||
pub struct CrossCompileConfig {
|
||||
/// The directory containing the Python library to link against.
|
||||
pub lib_dir: PathBuf,
|
||||
|
||||
/// The version of the Python library to link against.
|
||||
version: Option<PythonVersion>,
|
||||
os: String,
|
||||
|
||||
/// The `arch` component of the compilaton target triple.
|
||||
///
|
||||
/// e.g. x86_64, i386, arm, thumb, mips, etc.
|
||||
arch: String,
|
||||
|
||||
/// The `vendor` component of the compilaton target triple.
|
||||
///
|
||||
/// e.g. apple, pc, unknown, etc.
|
||||
vendor: String,
|
||||
|
||||
/// The `os` component of the compilaton target triple.
|
||||
///
|
||||
/// e.g. darwin, freebsd, linux, windows, etc.
|
||||
os: String,
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
|
|
@ -485,47 +575,74 @@ pub fn any_cross_compiling_env_vars_set() -> bool {
|
|||
|| env::var_os("PYO3_CROSS_PYTHON_VERSION").is_some()
|
||||
}
|
||||
|
||||
fn cross_compiling() -> Result<Option<CrossCompileConfig>> {
|
||||
fn cross_compiling_from_cargo_env() -> Result<Option<CrossCompileConfig>> {
|
||||
let host = cargo_env_var("HOST").ok_or("expected HOST env var")?;
|
||||
let target = cargo_env_var("TARGET").ok_or("expected TARGET env var")?;
|
||||
|
||||
if host == target {
|
||||
// Definitely not cross compiling if the host matches the target
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
if target == "i686-pc-windows-msvc" && host == "x86_64-pc-windows-msvc" {
|
||||
// Not cross-compiling to compile for 32-bit Python from windows 64-bit
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let target_arch =
|
||||
cargo_env_var("CARGO_CFG_TARGET_ARCH").ok_or("expected CARGO_CFG_TARGET_ARCH env var")?;
|
||||
let target_vendor = cargo_env_var("CARGO_CFG_TARGET_VENDOR")
|
||||
.ok_or("expected CARGO_CFG_TARGET_VENDOR env var")?;
|
||||
let target_os =
|
||||
cargo_env_var("CARGO_CFG_TARGET_OS").ok_or("expected CARGO_CFG_TARGET_OS env var")?;
|
||||
|
||||
cross_compiling(&host, &target_arch, &target_vendor, &target_os)
|
||||
}
|
||||
|
||||
/// Detect whether we are cross compiling and return an assembled CrossCompileConfig if so.
|
||||
///
|
||||
/// This function relies on PyO3 cross-compiling environment variables:
|
||||
///
|
||||
/// * `PYO3_CROSS`: If present, forces PyO3 to configure as a cross-compilation.
|
||||
/// * `PYO3_CROSS_LIB_DIR`: Must be set to the directory containing the target's libpython DSO and
|
||||
/// the associated `_sysconfigdata*.py` file for Unix-like targets, or the Python DLL import
|
||||
/// libraries for the Windows target.
|
||||
/// * `PYO3_CROSS_PYTHON_VERSION`: Major and minor version (e.g. 3.9) of the target Python
|
||||
/// installation. This variable is only needed if PyO3 cannnot determine the version to target
|
||||
/// from `abi3-py3*` features, or if there are multiple versions of Python present in
|
||||
/// `PYO3_CROSS_LIB_DIR`.
|
||||
///
|
||||
/// See the [PyO3 User Guide](https://pyo3.rs/) for more info on cross-compiling.
|
||||
pub fn cross_compiling(
|
||||
host: &str,
|
||||
target_arch: &str,
|
||||
target_vendor: &str,
|
||||
target_os: &str,
|
||||
) -> Result<Option<CrossCompileConfig>> {
|
||||
let cross = env_var("PYO3_CROSS");
|
||||
let cross_lib_dir = env_var("PYO3_CROSS_LIB_DIR");
|
||||
let cross_python_version = env_var("PYO3_CROSS_PYTHON_VERSION");
|
||||
|
||||
let target_arch = cargo_env_var("CARGO_CFG_TARGET_ARCH");
|
||||
let target_vendor = cargo_env_var("CARGO_CFG_TARGET_VENDOR");
|
||||
let target_os = cargo_env_var("CARGO_CFG_TARGET_OS");
|
||||
let target_triple = format!("{}-{}-{}", target_arch, target_vendor, target_os);
|
||||
|
||||
if cross.is_none() && cross_lib_dir.is_none() && cross_python_version.is_none() {
|
||||
// No cross-compiling environment variables set; try to determine if this is a known case
|
||||
// which is not cross-compilation.
|
||||
|
||||
let target = cargo_env_var("TARGET").unwrap();
|
||||
let host = cargo_env_var("HOST").unwrap();
|
||||
if target == host {
|
||||
// Not cross-compiling
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
if target == "i686-pc-windows-msvc" && host == "x86_64-pc-windows-msvc" {
|
||||
// Not cross-compiling to compile for 32-bit Python from windows 64-bit
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
if target == "x86_64-apple-darwin" && host == "aarch64-apple-darwin" {
|
||||
if target_triple == "x86_64-apple-darwin" && host == "aarch64-apple-darwin" {
|
||||
// Not cross-compiling to compile for x86-64 Python from macOS arm64
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
if target == "aarch64-apple-darwin" && host == "x86_64-apple-darwin" {
|
||||
if target_triple == "aarch64-apple-darwin" && host == "x86_64-apple-darwin" {
|
||||
// Not cross-compiling to compile for arm64 Python from macOS x86_64
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
if let (Some(arch), Some(vendor), Some(os)) = (&target_arch, &target_vendor, &target_os) {
|
||||
if host.starts_with(&format!("{}-{}-{}", arch, vendor, os)) {
|
||||
// Not cross-compiling if arch-vendor-os is all the same
|
||||
// e.g. x86_64-unknown-linux-musl on x86_64-unknown-linux-gnu host
|
||||
return Ok(None);
|
||||
}
|
||||
if host.starts_with(&target_triple) {
|
||||
// Not cross-compiling if arch-vendor-os is all the same
|
||||
// e.g. x86_64-unknown-linux-musl on x86_64-unknown-linux-gnu host
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -535,8 +652,9 @@ fn cross_compiling() -> Result<Option<CrossCompileConfig>> {
|
|||
lib_dir: cross_lib_dir
|
||||
.ok_or("The PYO3_CROSS_LIB_DIR environment variable must be set when cross-compiling")?
|
||||
.into(),
|
||||
os: target_os.unwrap(),
|
||||
arch: target_arch.unwrap(),
|
||||
arch: target_arch.into(),
|
||||
vendor: target_vendor.into(),
|
||||
os: target_os.into(),
|
||||
version: cross_python_version
|
||||
.map(|os_string| {
|
||||
let utf8_str = os_string
|
||||
|
|
@ -610,14 +728,14 @@ impl BuildFlags {
|
|||
BuildFlags(HashSet::new())
|
||||
}
|
||||
|
||||
fn from_config_map(config_map: &HashMap<String, String>) -> Self {
|
||||
fn from_sysconfigdata(config_map: &Sysconfigdata) -> Self {
|
||||
Self(
|
||||
BuildFlags::ALL
|
||||
.iter()
|
||||
.cloned()
|
||||
.filter(|flag| {
|
||||
config_map
|
||||
.get(&flag.to_string())
|
||||
.get_value(&flag.to_string())
|
||||
.map_or(false, |value| value == "1")
|
||||
})
|
||||
.collect(),
|
||||
|
|
@ -708,12 +826,35 @@ fn parse_script_output(output: &str) -> HashMap<String, String> {
|
|||
.collect()
|
||||
}
|
||||
|
||||
/// Parsed data from Python sysconfigdata file
|
||||
///
|
||||
/// A hash map of all values from a sysconfigdata file.
|
||||
pub struct Sysconfigdata(HashMap<String, String>);
|
||||
|
||||
impl Sysconfigdata {
|
||||
pub fn get_value<S: AsRef<str>>(&self, k: S) -> Option<&str> {
|
||||
self.0.get(k.as_ref()).map(String::as_str)
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
fn new() -> Self {
|
||||
Sysconfigdata(HashMap::new())
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
fn insert<S: Into<String>>(&mut self, k: S, v: S) {
|
||||
self.0.insert(k.into(), v.into());
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse sysconfigdata file
|
||||
///
|
||||
/// The sysconfigdata is simply a dictionary containing all the build time variables used for the
|
||||
/// python executable and library. Here it is read and added to a script to extract only what is
|
||||
/// necessary. This necessitates a python interpreter for the host machine to work.
|
||||
fn parse_sysconfigdata(sysconfigdata_path: impl AsRef<Path>) -> Result<InterpreterConfig> {
|
||||
/// python executable and library. This function necessitates a python interpreter on the host
|
||||
/// machine to work. Here it is read into a `Sysconfigdata` (hash map), which can be turned into an
|
||||
/// [`InterpreterConfig`](InterpreterConfig) using
|
||||
/// [`from_sysconfigdata`](InterpreterConfig::from_sysconfigdata).
|
||||
pub fn parse_sysconfigdata(sysconfigdata_path: impl AsRef<Path>) -> Result<Sysconfigdata> {
|
||||
let sysconfigdata_path = sysconfigdata_path.as_ref();
|
||||
let mut script = fs::read_to_string(&sysconfigdata_path).with_context(|| {
|
||||
format!(
|
||||
|
|
@ -722,83 +863,13 @@ fn parse_sysconfigdata(sysconfigdata_path: impl AsRef<Path>) -> Result<Interpret
|
|||
)
|
||||
})?;
|
||||
script += r#"
|
||||
print("version", build_time_vars["VERSION"])
|
||||
print("SOABI", build_time_vars.get("SOABI", ""))
|
||||
if "LIBDIR" in build_time_vars:
|
||||
print("LIBDIR", build_time_vars["LIBDIR"])
|
||||
KEYS = [
|
||||
"WITH_THREAD",
|
||||
"Py_DEBUG",
|
||||
"Py_REF_DEBUG",
|
||||
"Py_TRACE_REFS",
|
||||
"COUNT_ALLOCS",
|
||||
"Py_ENABLE_SHARED",
|
||||
"LDVERSION",
|
||||
"SIZEOF_VOID_P"
|
||||
]
|
||||
for key in KEYS:
|
||||
print(key, build_time_vars.get(key, 0))
|
||||
for key, val in build_time_vars.items():
|
||||
print(key, val)
|
||||
"#;
|
||||
|
||||
let output = run_python_script(&find_interpreter()?, &script)?;
|
||||
|
||||
let sysconfigdata = parse_script_output(&output);
|
||||
|
||||
macro_rules! get_key {
|
||||
($sysconfigdata:expr, $key:literal) => {
|
||||
$sysconfigdata
|
||||
.get($key)
|
||||
.ok_or(concat!($key, " not found in sysconfigdata file"))
|
||||
};
|
||||
}
|
||||
|
||||
macro_rules! parse_key {
|
||||
($sysconfigdata:expr, $key:literal) => {
|
||||
get_key!($sysconfigdata, $key)?
|
||||
.parse()
|
||||
.context(concat!("could not parse value of ", $key))
|
||||
};
|
||||
}
|
||||
|
||||
let version = parse_key!(sysconfigdata, "version")?;
|
||||
let pointer_width = parse_key!(sysconfigdata, "SIZEOF_VOID_P")
|
||||
.map(|bytes_width: u32| bytes_width * 8)
|
||||
.ok();
|
||||
|
||||
let soabi = get_key!(sysconfigdata, "SOABI")?;
|
||||
let implementation = if soabi.starts_with("pypy") {
|
||||
PythonImplementation::PyPy
|
||||
} else if soabi.starts_with("cpython") {
|
||||
PythonImplementation::CPython
|
||||
} else {
|
||||
bail!("unsupported Python interpreter");
|
||||
};
|
||||
|
||||
let shared = match sysconfigdata
|
||||
.get("Py_ENABLE_SHARED")
|
||||
.map(|string| string.as_str())
|
||||
{
|
||||
Some("1") | Some("true") | Some("True") => true,
|
||||
Some("0") | Some("false") | Some("False") | None => false,
|
||||
_ => bail!("expected a bool (1/true/True or 0/false/False) for Py_ENABLE_SHARED"),
|
||||
};
|
||||
|
||||
Ok(InterpreterConfig {
|
||||
implementation,
|
||||
version,
|
||||
shared,
|
||||
abi3: is_abi3(),
|
||||
lib_dir: get_key!(sysconfigdata, "LIBDIR").ok().cloned(),
|
||||
lib_name: Some(default_lib_name_unix(
|
||||
version,
|
||||
implementation,
|
||||
sysconfigdata.get("LDVERSION").map(String::as_str),
|
||||
)),
|
||||
executable: None,
|
||||
pointer_width,
|
||||
build_flags: BuildFlags::from_config_map(&sysconfigdata).fixup(version),
|
||||
suppress_build_script_link_lines: false,
|
||||
extra_build_script_lines: vec![],
|
||||
})
|
||||
Ok(Sysconfigdata(parse_script_output(&output)))
|
||||
}
|
||||
|
||||
fn starts_with(entry: &DirEntry, pat: &str) -> bool {
|
||||
|
|
@ -810,52 +881,8 @@ fn ends_with(entry: &DirEntry, pat: &str) -> bool {
|
|||
name.to_string_lossy().ends_with(pat)
|
||||
}
|
||||
|
||||
/// Finds the `_sysconfigdata*.py` file in the library path.
|
||||
///
|
||||
/// From the python source for `_sysconfigdata*.py` is always going to be located at
|
||||
/// `build/lib.{PLATFORM}-{PY_MINOR_VERSION}` when built from source. The [exact line][1] is defined as:
|
||||
///
|
||||
/// ```py
|
||||
/// pybuilddir = 'build/lib.%s-%s' % (get_platform(), sys.version_info[:2])
|
||||
/// ```
|
||||
///
|
||||
/// Where get_platform returns a kebab-case formatted string containing the os, the architecture and
|
||||
/// possibly the os' kernel version (not the case on linux). However, when installed using a package
|
||||
/// manager, the `_sysconfigdata*.py` file is installed in the `${PREFIX}/lib/python3.Y/` directory.
|
||||
/// The `_sysconfigdata*.py` is generally in a sub-directory of the location of `libpython3.Y.so`.
|
||||
/// So we must find the file in the following possible locations:
|
||||
///
|
||||
/// ```sh
|
||||
/// # distribution from package manager, lib_dir should include lib/
|
||||
/// ${INSTALL_PREFIX}/lib/python3.Y/_sysconfigdata*.py
|
||||
/// ${INSTALL_PREFIX}/lib/libpython3.Y.so
|
||||
/// ${INSTALL_PREFIX}/lib/python3.Y/config-3.Y-${HOST_TRIPLE}/libpython3.Y.so
|
||||
///
|
||||
/// # Built from source from host
|
||||
/// ${CROSS_COMPILED_LOCATION}/build/lib.linux-x86_64-Y/_sysconfigdata*.py
|
||||
/// ${CROSS_COMPILED_LOCATION}/libpython3.Y.so
|
||||
///
|
||||
/// # if cross compiled, kernel release is only present on certain OS targets.
|
||||
/// ${CROSS_COMPILED_LOCATION}/build/lib.{OS}(-{OS-KERNEL-RELEASE})?-{ARCH}-Y/_sysconfigdata*.py
|
||||
/// ${CROSS_COMPILED_LOCATION}/libpython3.Y.so
|
||||
/// ```
|
||||
///
|
||||
/// [1]: https://github.com/python/cpython/blob/3.5/Lib/sysconfig.py#L389
|
||||
fn find_sysconfigdata(cross: &CrossCompileConfig) -> Result<PathBuf> {
|
||||
let sysconfig_paths = search_lib_dir(&cross.lib_dir, cross);
|
||||
let sysconfig_name = env_var("_PYTHON_SYSCONFIGDATA_NAME");
|
||||
let mut sysconfig_paths = sysconfig_paths
|
||||
.iter()
|
||||
.filter_map(|p| {
|
||||
let canonical = fs::canonicalize(p).ok();
|
||||
match &sysconfig_name {
|
||||
Some(_) => canonical.filter(|p| p.file_stem() == sysconfig_name.as_deref()),
|
||||
None => canonical,
|
||||
}
|
||||
})
|
||||
.collect::<Vec<PathBuf>>();
|
||||
sysconfig_paths.sort();
|
||||
sysconfig_paths.dedup();
|
||||
let mut sysconfig_paths = find_all_sysconfigdata(cross);
|
||||
if sysconfig_paths.is_empty() {
|
||||
bail!(
|
||||
"Could not find either libpython.so or _sysconfigdata*.py in {}",
|
||||
|
|
@ -877,14 +904,82 @@ fn find_sysconfigdata(cross: &CrossCompileConfig) -> Result<PathBuf> {
|
|||
Ok(sysconfig_paths.remove(0))
|
||||
}
|
||||
|
||||
/// recursive search for _sysconfigdata, returns all possibilities of sysconfigdata paths
|
||||
fn search_lib_dir(path: impl AsRef<Path>, cross: &CrossCompileConfig) -> Vec<PathBuf> {
|
||||
let mut sysconfig_paths = vec![];
|
||||
let version_pat = if let Some(v) = &cross.version {
|
||||
/// Finds `_sysconfigdata*.py` files for detected Python interpreters.
|
||||
///
|
||||
/// From the python source for `_sysconfigdata*.py` is always going to be located at
|
||||
/// `build/lib.{PLATFORM}-{PY_MINOR_VERSION}` when built from source. The [exact line][1] is defined as:
|
||||
///
|
||||
/// ```py
|
||||
/// pybuilddir = 'build/lib.%s-%s' % (get_platform(), sys.version_info[:2])
|
||||
/// ```
|
||||
///
|
||||
/// Where get_platform returns a kebab-case formatted string containing the os, the architecture and
|
||||
/// possibly the os' kernel version (not the case on linux). However, when installed using a package
|
||||
/// manager, the `_sysconfigdata*.py` file is installed in the `${PREFIX}/lib/python3.Y/` directory.
|
||||
/// The `_sysconfigdata*.py` is generally in a sub-directory of the location of `libpython3.Y.so`.
|
||||
/// So we must find the file in the following possible locations:
|
||||
///
|
||||
/// ```sh
|
||||
/// # distribution from package manager, (lib_dir may or may not include lib/)
|
||||
/// ${INSTALL_PREFIX}/lib/python3.Y/_sysconfigdata*.py
|
||||
/// ${INSTALL_PREFIX}/lib/libpython3.Y.so
|
||||
/// ${INSTALL_PREFIX}/lib/python3.Y/config-3.Y-${HOST_TRIPLE}/libpython3.Y.so
|
||||
///
|
||||
/// # Built from source from host
|
||||
/// ${CROSS_COMPILED_LOCATION}/build/lib.linux-x86_64-Y/_sysconfigdata*.py
|
||||
/// ${CROSS_COMPILED_LOCATION}/libpython3.Y.so
|
||||
///
|
||||
/// # if cross compiled, kernel release is only present on certain OS targets.
|
||||
/// ${CROSS_COMPILED_LOCATION}/build/lib.{OS}(-{OS-KERNEL-RELEASE})?-{ARCH}-Y/_sysconfigdata*.py
|
||||
/// ${CROSS_COMPILED_LOCATION}/libpython3.Y.so
|
||||
///
|
||||
/// # PyPy includes a similar file since v73
|
||||
/// ${INSTALL_PREFIX}/lib/pypy3.Y/_sysconfigdata.py
|
||||
/// ${INSTALL_PREFIX}/lib_pypy/_sysconfigdata.py
|
||||
/// ```
|
||||
///
|
||||
/// [1]: https://github.com/python/cpython/blob/3.5/Lib/sysconfig.py#L389
|
||||
pub fn find_all_sysconfigdata(cross: &CrossCompileConfig) -> Vec<PathBuf> {
|
||||
let sysconfig_paths = search_lib_dir(&cross.lib_dir, cross);
|
||||
let sysconfig_name = env_var("_PYTHON_SYSCONFIGDATA_NAME");
|
||||
let mut sysconfig_paths = sysconfig_paths
|
||||
.iter()
|
||||
.filter_map(|p| {
|
||||
let canonical = fs::canonicalize(p).ok();
|
||||
match &sysconfig_name {
|
||||
Some(_) => canonical.filter(|p| p.file_stem() == sysconfig_name.as_deref()),
|
||||
None => canonical,
|
||||
}
|
||||
})
|
||||
.collect::<Vec<PathBuf>>();
|
||||
|
||||
sysconfig_paths.sort();
|
||||
sysconfig_paths.dedup();
|
||||
|
||||
sysconfig_paths
|
||||
}
|
||||
|
||||
fn is_pypy_lib_dir(path: &str, v: &Option<PythonVersion>) -> bool {
|
||||
let pypy_version_pat = if let Some(v) = v {
|
||||
format!("pypy{}", v)
|
||||
} else {
|
||||
"pypy3.".into()
|
||||
};
|
||||
path == "lib_pypy" || path.starts_with(&pypy_version_pat)
|
||||
}
|
||||
|
||||
fn is_cpython_lib_dir(path: &str, v: &Option<PythonVersion>) -> bool {
|
||||
let cpython_version_pat = if let Some(v) = v {
|
||||
format!("python{}", v)
|
||||
} else {
|
||||
"python3.".into()
|
||||
};
|
||||
path.starts_with(&cpython_version_pat)
|
||||
}
|
||||
|
||||
/// recursive search for _sysconfigdata, returns all possibilities of sysconfigdata paths
|
||||
fn search_lib_dir(path: impl AsRef<Path>, cross: &CrossCompileConfig) -> Vec<PathBuf> {
|
||||
let mut sysconfig_paths = vec![];
|
||||
for f in fs::read_dir(path).expect("Path does not exist") {
|
||||
sysconfig_paths.extend(match &f {
|
||||
// Python 3.7+ sysconfigdata with platform specifics
|
||||
|
|
@ -892,7 +987,7 @@ fn search_lib_dir(path: impl AsRef<Path>, cross: &CrossCompileConfig) -> Vec<Pat
|
|||
Ok(f) if f.metadata().map_or(false, |metadata| metadata.is_dir()) => {
|
||||
let file_name = f.file_name();
|
||||
let file_name = file_name.to_string_lossy();
|
||||
if file_name.starts_with("build") {
|
||||
if file_name == "build" || file_name == "lib" {
|
||||
search_lib_dir(f.path(), cross)
|
||||
} else if file_name.starts_with("lib.") {
|
||||
// check if right target os
|
||||
|
|
@ -908,7 +1003,9 @@ fn search_lib_dir(path: impl AsRef<Path>, cross: &CrossCompileConfig) -> Vec<Pat
|
|||
continue;
|
||||
}
|
||||
search_lib_dir(f.path(), cross)
|
||||
} else if file_name.starts_with(&version_pat) {
|
||||
} else if is_cpython_lib_dir(&file_name, &cross.version)
|
||||
|| is_pypy_lib_dir(&file_name, &cross.version)
|
||||
{
|
||||
search_lib_dir(f.path(), cross)
|
||||
} else {
|
||||
continue;
|
||||
|
|
@ -947,7 +1044,7 @@ fn load_cross_compile_from_sysconfigdata(
|
|||
cross_compile_config: CrossCompileConfig,
|
||||
) -> Result<InterpreterConfig> {
|
||||
let sysconfigdata_path = find_sysconfigdata(&cross_compile_config)?;
|
||||
parse_sysconfigdata(sysconfigdata_path)
|
||||
InterpreterConfig::from_sysconfigdata(&parse_sysconfigdata(sysconfigdata_path)?)
|
||||
}
|
||||
|
||||
fn windows_hardcoded_cross_compile(
|
||||
|
|
@ -957,9 +1054,10 @@ fn windows_hardcoded_cross_compile(
|
|||
.ok_or("PYO3_CROSS_PYTHON_VERSION or an abi3-py3* feature must be specified when cross-compiling for Windows.")?;
|
||||
|
||||
let abi3 = is_abi3();
|
||||
let implementation = PythonImplementation::CPython;
|
||||
|
||||
Ok(InterpreterConfig {
|
||||
implementation: PythonImplementation::CPython,
|
||||
implementation,
|
||||
version,
|
||||
shared: true,
|
||||
abi3,
|
||||
|
|
@ -1159,7 +1257,7 @@ fn fixup_config_for_abi3(
|
|||
/// This must be called from PyO3's build script, because it relies on environment variables such as
|
||||
/// CARGO_CFG_TARGET_OS which aren't available at any other time.
|
||||
pub fn make_cross_compile_config() -> Result<Option<InterpreterConfig>> {
|
||||
let mut interpreter_config = if let Some(paths) = cross_compiling()? {
|
||||
let mut interpreter_config = if let Some(paths) = cross_compiling_from_cargo_env()? {
|
||||
load_cross_compile_config(paths)?
|
||||
} else {
|
||||
return Ok(None);
|
||||
|
|
@ -1265,24 +1363,33 @@ mod tests {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn build_flags_from_config_map() {
|
||||
let mut config_map = HashMap::new();
|
||||
fn build_flags_from_sysconfigdata() {
|
||||
let mut sysconfigdata = Sysconfigdata::new();
|
||||
|
||||
assert_eq!(BuildFlags::from_config_map(&config_map).0, HashSet::new());
|
||||
assert_eq!(
|
||||
BuildFlags::from_sysconfigdata(&sysconfigdata).0,
|
||||
HashSet::new()
|
||||
);
|
||||
|
||||
for flag in &BuildFlags::ALL {
|
||||
config_map.insert(flag.to_string(), "0".into());
|
||||
sysconfigdata.insert(flag.to_string(), "0".into());
|
||||
}
|
||||
|
||||
assert_eq!(BuildFlags::from_config_map(&config_map).0, HashSet::new());
|
||||
assert_eq!(
|
||||
BuildFlags::from_sysconfigdata(&sysconfigdata).0,
|
||||
HashSet::new()
|
||||
);
|
||||
|
||||
let mut expected_flags = HashSet::new();
|
||||
for flag in &BuildFlags::ALL {
|
||||
config_map.insert(flag.to_string(), "1".into());
|
||||
sysconfigdata.insert(flag.to_string(), "1".into());
|
||||
expected_flags.insert(flag.clone());
|
||||
}
|
||||
|
||||
assert_eq!(BuildFlags::from_config_map(&config_map).0, expected_flags);
|
||||
assert_eq!(
|
||||
BuildFlags::from_sysconfigdata(&sysconfigdata).0,
|
||||
expected_flags
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -1325,6 +1432,41 @@ mod tests {
|
|||
assert!(make_interpreter_config().is_ok())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_from_empty_sysconfigdata() {
|
||||
let sysconfigdata = Sysconfigdata::new();
|
||||
assert!(InterpreterConfig::from_sysconfigdata(&sysconfigdata).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_from_sysconfigdata() {
|
||||
let mut sysconfigdata = Sysconfigdata::new();
|
||||
// these are the minimal values required such that InterpreterConfig::from_sysconfigdata
|
||||
// does not error
|
||||
sysconfigdata.insert("SOABI", "cpython-37m-x86_64-linux-gnu");
|
||||
sysconfigdata.insert("VERSION", "3.7");
|
||||
sysconfigdata.insert("Py_ENABLE_SHARED", "1");
|
||||
sysconfigdata.insert("LIBDIR", "/usr/lib");
|
||||
sysconfigdata.insert("LDVERSION", "3.7m");
|
||||
sysconfigdata.insert("SIZEOF_VOID_P", "8");
|
||||
assert_eq!(
|
||||
InterpreterConfig::from_sysconfigdata(&sysconfigdata).unwrap(),
|
||||
InterpreterConfig {
|
||||
abi3: false,
|
||||
build_flags: BuildFlags::from_sysconfigdata(&sysconfigdata),
|
||||
pointer_width: Some(64),
|
||||
executable: None,
|
||||
implementation: PythonImplementation::CPython,
|
||||
lib_dir: Some("/usr/lib".into()),
|
||||
lib_name: Some("python3.7m".into()),
|
||||
shared: true,
|
||||
version: PythonVersion::PY37,
|
||||
suppress_build_script_link_lines: false,
|
||||
extra_build_script_lines: vec![],
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn windows_hardcoded_cross_compile() {
|
||||
let cross_config = CrossCompileConfig {
|
||||
|
|
@ -1332,6 +1474,7 @@ mod tests {
|
|||
version: Some(PythonVersion { major: 3, minor: 7 }),
|
||||
os: "os".into(),
|
||||
arch: "arch".into(),
|
||||
vendor: "vendor".into(),
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
|
|
@ -1497,8 +1640,9 @@ mod tests {
|
|||
let cross = CrossCompileConfig {
|
||||
lib_dir: lib_dir.into(),
|
||||
version: Some(interpreter_config.version),
|
||||
os: "linux".into(),
|
||||
arch: "x86_64".into(),
|
||||
vendor: "unknown".into(),
|
||||
os: "linux".into(),
|
||||
};
|
||||
|
||||
let sysconfigdata_path = match find_sysconfigdata(&cross) {
|
||||
|
|
@ -1506,7 +1650,8 @@ mod tests {
|
|||
// Couldn't find a matching sysconfigdata; never mind!
|
||||
Err(_) => return,
|
||||
};
|
||||
let parsed_config = super::parse_sysconfigdata(sysconfigdata_path).unwrap();
|
||||
let sysconfigdata = super::parse_sysconfigdata(sysconfigdata_path).unwrap();
|
||||
let parsed_config = InterpreterConfig::from_sysconfigdata(&sysconfigdata).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
parsed_config,
|
||||
|
|
@ -1551,4 +1696,23 @@ mod tests {
|
|||
PathBuf::from_iter(&["base", "bin", "python"])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_not_cross_compiling() {
|
||||
assert!(
|
||||
cross_compiling("aarch64-apple-darwin", "x86_64", "apple", "darwin")
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
cross_compiling("x86_64-apple-darwin", "aarch64", "apple", "darwin")
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
cross_compiling("x86_64-unknown-linux-gnu", "x86_64", "unknown", "linux")
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,7 @@
|
|||
//! Configuration used by PyO3 for conditional support of varying Python versions.
|
||||
//!
|
||||
//! This crate exposes two functions, [`use_pyo3_cfgs`] and [`add_extension_module_link_args`],
|
||||
//! which are intended to be called from build scripts to simplify building crates which depend on
|
||||
//! PyO3.
|
||||
//! This crate exposes functionality to be called from build scripts to simplify building crates
|
||||
//! which depend on PyO3.
|
||||
//!
|
||||
//! It used internally by the PyO3 crate's build script to apply the same configuration.
|
||||
|
||||
|
|
@ -15,7 +14,10 @@ use std::io::Cursor;
|
|||
#[cfg(feature = "resolve-config")]
|
||||
use once_cell::sync::OnceCell;
|
||||
|
||||
pub use impl_::{BuildFlag, BuildFlags, InterpreterConfig, PythonImplementation, PythonVersion};
|
||||
pub use impl_::{
|
||||
cross_compiling, find_all_sysconfigdata, parse_sysconfigdata, BuildFlag, BuildFlags,
|
||||
CrossCompileConfig, InterpreterConfig, PythonImplementation, PythonVersion,
|
||||
};
|
||||
|
||||
/// Adds all the [`#[cfg]` flags](index.html) to the current compilation.
|
||||
///
|
||||
|
|
|
|||
Loading…
Reference in New Issue