auto-initialize: new feature to control initializing Python
This commit is contained in:
parent
42ca48a8b4
commit
59707f0b81
|
@ -85,27 +85,28 @@ jobs:
|
|||
run: echo LD_LIBRARY_PATH=${pythonLocation}/lib >> $GITHUB_ENV
|
||||
|
||||
- name: Build docs
|
||||
run: cargo doc --features "num-bigint num-complex hashbrown" --verbose --target ${{ matrix.platform.rust-target }}
|
||||
run: cargo doc --no-default-features --features "macros num-bigint num-complex hashbrown" --verbose --target ${{ matrix.platform.rust-target }}
|
||||
|
||||
- name: Build without default features
|
||||
- name: Build (no features)
|
||||
run: cargo build --no-default-features --verbose --target ${{ matrix.platform.rust-target }}
|
||||
|
||||
- name: Build with default features
|
||||
run: cargo build --features "num-bigint num-complex hashbrown" --verbose --target ${{ matrix.platform.rust-target }}
|
||||
- name: Build (all additive features)
|
||||
run: cargo build --no-default-features --features "macros num-bigint num-complex hashbrown" --verbose --target ${{ matrix.platform.rust-target }}
|
||||
|
||||
# Run tests (except on PyPy, because no embedding API).
|
||||
- if: matrix.python-version != 'pypy-3.6'
|
||||
name: Test
|
||||
run: cargo test --features "num-bigint num-complex hashbrown" --target ${{ matrix.platform.rust-target }}
|
||||
run: cargo test --no-default-features --features "macros num-bigint num-complex hashbrown" --target ${{ matrix.platform.rust-target }}
|
||||
|
||||
# Run tests again, but in abi3 mode
|
||||
- if: matrix.python-version != 'pypy-3.6'
|
||||
name: Test (abi3)
|
||||
run: cargo test --no-default-features --features "abi3,macros" --target ${{ matrix.platform.rust-target }}
|
||||
run: cargo test --no-default-features --features "abi3 macros num-bigint num-complex hashbrown" --target ${{ matrix.platform.rust-target }}
|
||||
|
||||
# Run tests again, for abi3-py36 (the minimal Python version)
|
||||
- if: (matrix.python-version != 'pypy-3.6') && (matrix.python-version != '3.6')
|
||||
name: Test (abi3-py36)
|
||||
run: cargo test --no-default-features --features "abi3-py36,macros" --target ${{ matrix.platform.rust-target }}
|
||||
run: cargo test --no-default-features --features "abi3-py36 macros num-bigint num-complex hashbrown" --target ${{ matrix.platform.rust-target }}
|
||||
|
||||
- name: Test proc-macro code
|
||||
run: cargo test --manifest-path=pyo3-macros-backend/Cargo.toml --target ${{ matrix.platform.rust-target }}
|
||||
|
@ -125,6 +126,9 @@ jobs:
|
|||
env:
|
||||
RUST_BACKTRACE: 1
|
||||
RUSTFLAGS: "-D warnings"
|
||||
# TODO: this is a hack to workaround compile_error! warnings about auto-initialize on PyPy
|
||||
# Once cargo's `resolver = "2"` is stable (~ MSRV Rust 1.52), remove this.
|
||||
PYO3_CI: 1
|
||||
|
||||
coverage:
|
||||
needs: [fmt]
|
||||
|
|
24
Cargo.toml
24
Cargo.toml
|
@ -33,26 +33,36 @@ assert_approx_eq = "1.1.0"
|
|||
trybuild = "1.0.23"
|
||||
rustversion = "1.0"
|
||||
proptest = { version = "0.10.1", default-features = false, features = ["std"] }
|
||||
# features needed to run the PyO3 test suite
|
||||
pyo3 = { path = ".", default-features = false, features = ["macros", "auto-initialize"] }
|
||||
|
||||
[features]
|
||||
default = ["macros"]
|
||||
macros = ["ctor", "indoc", "inventory", "paste", "pyo3-macros", "unindent"]
|
||||
default = ["macros", "auto-initialize"]
|
||||
|
||||
# Enables macros: #[pyclass], #[pymodule], #[pyfunction] etc.
|
||||
macros = ["pyo3-macros", "ctor", "indoc", "inventory", "paste", "unindent"]
|
||||
|
||||
# Use this feature when building an extension module.
|
||||
# It tells the linker to keep the python symbols unresolved,
|
||||
# so that the module can also be used with statically linked python interpreters.
|
||||
extension-module = []
|
||||
|
||||
# Use the Python limited API. See https://www.python.org/dev/peps/pep-0384/ for more.
|
||||
abi3 = []
|
||||
|
||||
# With abi3, we can manually set the minimum Python version.
|
||||
abi3-py36 = ["abi3-py37"]
|
||||
abi3-py37 = ["abi3-py38"]
|
||||
abi3-py38 = ["abi3-py39"]
|
||||
abi3-py39 = ["abi3"]
|
||||
|
||||
# Changes `Python::with_gil` and `Python::acquire_gil` to automatically initialize the
|
||||
# Python interpreter if needed.
|
||||
auto-initialize = []
|
||||
|
||||
# Optimizes PyObject to Vec conversion and so on.
|
||||
nightly = []
|
||||
|
||||
# Use this feature when building an extension module.
|
||||
# It tells the linker to keep the python symbols unresolved,
|
||||
# so that the module can also be used with statically linked python interpreters.
|
||||
extension-module = []
|
||||
|
||||
[workspace]
|
||||
members = [
|
||||
"pyo3-macros",
|
||||
|
|
|
@ -104,8 +104,9 @@ If you want your Rust application to create a Python interpreter internally and
|
|||
use it to run Python code, add `pyo3` to your `Cargo.toml` like this:
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
pyo3 = "0.13.0"
|
||||
[dependencies.pyo3]
|
||||
version = "0.13.0"
|
||||
features = ["auto-initialize"]
|
||||
```
|
||||
|
||||
Example program displaying the value of `sys.version` and the current user name:
|
||||
|
|
10
build.rs
10
build.rs
|
@ -749,6 +749,10 @@ fn configure(interpreter_config: &InterpreterConfig) -> Result<()> {
|
|||
}
|
||||
}
|
||||
|
||||
if interpreter_config.shared {
|
||||
println!("cargo:rustc-cfg=Py_SHARED");
|
||||
}
|
||||
|
||||
if interpreter_config.version.implementation == PythonInterpreterKind::PyPy {
|
||||
println!("cargo:rustc-cfg=PyPy");
|
||||
};
|
||||
|
@ -883,5 +887,11 @@ fn main() -> Result<()> {
|
|||
}
|
||||
}
|
||||
|
||||
// TODO: this is a hack to workaround compile_error! warnings about auto-initialize on PyPy
|
||||
// Once cargo's `resolver = "2"` is stable (~ MSRV Rust 1.52), remove this.
|
||||
if env::var_os("PYO3_CI").is_some() {
|
||||
println!("cargo:rustc-cfg=__pyo3_ci");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
|
89
src/gil.rs
89
src/gil.rs
|
@ -3,11 +3,11 @@
|
|||
//! Interaction with python's global interpreter lock
|
||||
|
||||
use crate::{ffi, internal_tricks::Unsendable, Python};
|
||||
use parking_lot::{const_mutex, Mutex};
|
||||
use parking_lot::{const_mutex, Mutex, Once};
|
||||
use std::cell::{Cell, RefCell};
|
||||
use std::{mem::ManuallyDrop, ptr::NonNull, sync};
|
||||
use std::{mem::ManuallyDrop, ptr::NonNull};
|
||||
|
||||
static START: sync::Once = sync::Once::new();
|
||||
static START: Once = Once::new();
|
||||
|
||||
thread_local! {
|
||||
/// This is a internal counter in pyo3 monitoring whether this thread has the GIL.
|
||||
|
@ -45,16 +45,20 @@ pub(crate) fn gil_is_acquired() -> bool {
|
|||
/// If both the Python interpreter and Python threading are already initialized,
|
||||
/// this function has no effect.
|
||||
///
|
||||
/// # Availability
|
||||
///
|
||||
/// This function is only available when linking against Python distributions that contain a
|
||||
/// shared library.
|
||||
///
|
||||
/// This function is not available on PyPy.
|
||||
///
|
||||
/// # Panic
|
||||
/// If the Python interpreter is initialized but Python threading is not,
|
||||
/// a panic occurs.
|
||||
/// It is not possible to safely access the Python runtime unless the main
|
||||
/// thread (the thread which originally initialized Python) also initializes
|
||||
/// threading.
|
||||
///
|
||||
/// When writing an extension module, the `#[pymodule]` macro
|
||||
/// will ensure that Python threading is initialized.
|
||||
///
|
||||
#[cfg(all(Py_SHARED, not(PyPy)))]
|
||||
pub fn prepare_freethreaded_python() {
|
||||
// Protect against race conditions when Python is not yet initialized
|
||||
// and multiple threads concurrently call 'prepare_freethreaded_python()'.
|
||||
|
@ -72,22 +76,18 @@ pub fn prepare_freethreaded_python() {
|
|||
// Note that the 'main thread' notion in Python isn't documented properly;
|
||||
// and running Python without one is not officially supported.
|
||||
|
||||
// PyPy does not support the embedding API
|
||||
#[cfg(not(PyPy))]
|
||||
{
|
||||
ffi::Py_InitializeEx(0);
|
||||
ffi::Py_InitializeEx(0);
|
||||
|
||||
// Make sure Py_Finalize will be called before exiting.
|
||||
extern "C" fn finalize() {
|
||||
unsafe {
|
||||
if ffi::Py_IsInitialized() != 0 {
|
||||
ffi::PyGILState_Ensure();
|
||||
ffi::Py_Finalize();
|
||||
}
|
||||
// Make sure Py_Finalize will be called before exiting.
|
||||
extern "C" fn finalize() {
|
||||
unsafe {
|
||||
if ffi::Py_IsInitialized() != 0 {
|
||||
ffi::PyGILState_Ensure();
|
||||
ffi::Py_Finalize();
|
||||
}
|
||||
}
|
||||
libc::atexit(finalize);
|
||||
}
|
||||
libc::atexit(finalize);
|
||||
|
||||
// > Changed in version 3.7: This function is now called by Py_Initialize(), so you don’t have
|
||||
// > to call it yourself anymore.
|
||||
|
@ -95,11 +95,10 @@ pub fn prepare_freethreaded_python() {
|
|||
if ffi::PyEval_ThreadsInitialized() == 0 {
|
||||
ffi::PyEval_InitThreads();
|
||||
}
|
||||
// PyEval_InitThreads() will acquire the GIL,
|
||||
// but we don't want to hold it at this point
|
||||
|
||||
// Py_InitializeEx() will acquire the GIL, but we don't want to hold it at this point
|
||||
// (it's not acquired in the other code paths)
|
||||
// So immediately release the GIL:
|
||||
#[cfg(not(PyPy))]
|
||||
let _thread_state = ffi::PyEval_SaveThread();
|
||||
// Note that the PyThreadState returned by PyEval_SaveThread is also held in TLS by the Python runtime,
|
||||
// and will be restored by PyGILState_Ensure.
|
||||
|
@ -137,7 +136,51 @@ impl GILGuard {
|
|||
/// If PyO3 does not yet have a `GILPool` for tracking owned PyObject references, then this
|
||||
/// new `GILGuard` will also contain a `GILPool`.
|
||||
pub(crate) fn acquire() -> GILGuard {
|
||||
prepare_freethreaded_python();
|
||||
// Maybe auto-initialize the GIL:
|
||||
// - If auto-initialize feature set and supported, try to initalize the interpreter.
|
||||
// - If the auto-initialize feature is set but unsupported, emit hard errors only when
|
||||
// the extension-module feature is not activated - extension modules don't care about
|
||||
// auto-initialize so this avoids breaking existing builds.
|
||||
// - Otherwise, just check the GIL is initialized.
|
||||
cfg_if::cfg_if! {
|
||||
if #[cfg(all(feature = "auto-initialize", Py_SHARED, not(PyPy)))] {
|
||||
prepare_freethreaded_python();
|
||||
} else if #[cfg(all(feature = "auto-initialize", not(feature = "extension-module"), not(Py_SHARED), not(__pyo3_ci)))] {
|
||||
compile_error!(concat!(
|
||||
"The `auto-initialize` feature is not supported when linking Python ",
|
||||
"statically instead of with a shared library.\n\n",
|
||||
"Please disable the `auto-initialize` feature, for example by entering the following ",
|
||||
"in your cargo.toml:\n\n",
|
||||
" pyo3 = { version = \"0.13.0\", default-features = false }\n\n",
|
||||
"Alternatively, compile PyO3 using a Python distribution which contains a shared ",
|
||||
"libary."
|
||||
));
|
||||
} else if #[cfg(all(feature = "auto-initialize", not(feature = "extension-module"), PyPy, not(__pyo3_ci)))] {
|
||||
compile_error!(concat!(
|
||||
"The `auto-initialize` feature is not supported by PyPy.\n\n",
|
||||
"Please disable the `auto-initialize` feature, for example by entering the following ",
|
||||
"in your cargo.toml:\n\n",
|
||||
" pyo3 = { version = \"0.13.0\", default-features = false }",
|
||||
));
|
||||
} else {
|
||||
// extension module feature enabled and PyPy or static linking
|
||||
// OR auto-initialize feature not enabled
|
||||
START.call_once_force(|_| unsafe {
|
||||
// Use call_once_force because if there is a panic because the interpreter is not
|
||||
// initialized, it's fine for the user to initialize the interpreter and retry.
|
||||
assert_ne!(
|
||||
ffi::Py_IsInitialized(),
|
||||
0,
|
||||
"The Python interpreter is not initalized and the `auto-initialize` feature is not enabled."
|
||||
);
|
||||
assert_ne!(
|
||||
ffi::PyEval_ThreadsInitialized(),
|
||||
0,
|
||||
"Python threading is not initalized and the `auto-initialize` feature is not enabled."
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let gstate = unsafe { ffi::PyGILState_Ensure() }; // acquire GIL
|
||||
|
||||
|
|
|
@ -114,8 +114,9 @@
|
|||
//! Add `pyo3` to your `Cargo.toml`:
|
||||
//!
|
||||
//! ```toml
|
||||
//! [dependencies]
|
||||
//! pyo3 = "0.13.0"
|
||||
//! [dependencies.pyo3]
|
||||
//! version = "0.13.0"
|
||||
//! features = ["auto-initialize"]
|
||||
//! ```
|
||||
//!
|
||||
//! Example program displaying the value of `sys.version`:
|
||||
|
@ -145,12 +146,14 @@ pub use crate::conversion::{
|
|||
ToBorrowedObject, ToPyObject,
|
||||
};
|
||||
pub use crate::err::{PyDowncastError, PyErr, PyErrArguments, PyResult};
|
||||
#[cfg(all(Py_SHARED, not(PyPy)))]
|
||||
pub use crate::gil::prepare_freethreaded_python;
|
||||
pub use crate::gil::{GILGuard, GILPool};
|
||||
pub use crate::instance::{Py, PyNativeType, PyObject};
|
||||
pub use crate::pycell::{PyCell, PyRef, PyRefMut};
|
||||
pub use crate::pyclass::PyClass;
|
||||
pub use crate::pyclass_init::PyClassInitializer;
|
||||
pub use crate::python::{prepare_freethreaded_python, Python, PythonVersionInfo};
|
||||
pub use crate::python::{Python, PythonVersionInfo};
|
||||
pub use crate::type_object::{type_flags, PyTypeInfo};
|
||||
// Since PyAny is as important as PyObject, we expose it to the top level.
|
||||
pub use crate::types::PyAny;
|
||||
|
|
|
@ -11,8 +11,6 @@ use std::ffi::{CStr, CString};
|
|||
use std::marker::PhantomData;
|
||||
use std::os::raw::{c_char, c_int};
|
||||
|
||||
pub use gil::prepare_freethreaded_python;
|
||||
|
||||
/// Represents the major, minor, and patch (if any) versions of this interpreter.
|
||||
///
|
||||
/// See [Python::version].
|
||||
|
@ -134,8 +132,13 @@ impl Python<'_> {
|
|||
/// Acquires the global interpreter lock, which allows access to the Python runtime. The
|
||||
/// provided closure F will be executed with the acquired `Python` marker token.
|
||||
///
|
||||
/// If the Python runtime is not already initialized, this function will initialize it.
|
||||
/// See [prepare_freethreaded_python()](fn.prepare_freethreaded_python.html) for details.
|
||||
/// If the `auto-initialize` feature is enabled and the Python runtime is not already
|
||||
/// initialized, this function will initialize it. See
|
||||
/// [prepare_freethreaded_python()](fn.prepare_freethreaded_python.html) for details.
|
||||
///
|
||||
/// # Panics
|
||||
/// - If the `auto-initialize` feature is not enabled and the Python interpreter is not
|
||||
/// initialized.
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
|
@ -158,8 +161,9 @@ impl Python<'_> {
|
|||
impl<'p> Python<'p> {
|
||||
/// Acquires the global interpreter lock, which allows access to the Python runtime.
|
||||
///
|
||||
/// If the Python runtime is not already initialized, this function will initialize it.
|
||||
/// See [prepare_freethreaded_python()](fn.prepare_freethreaded_python.html) for details.
|
||||
/// If the `auto-initialize` feature is enabled and the Python runtime is not already
|
||||
/// initialized, this function will initialize it. See
|
||||
/// [prepare_freethreaded_python()](fn.prepare_freethreaded_python.html) for details.
|
||||
///
|
||||
/// Most users should not need to use this API directly, and should prefer one of two options:
|
||||
/// 1. When implementing `#[pymethods]` or `#[pyfunction]` add a function argument
|
||||
|
@ -172,6 +176,10 @@ impl<'p> Python<'p> {
|
|||
/// allowed, and will not deadlock. However, `GILGuard`s must be dropped in the reverse order
|
||||
/// to acquisition. If PyO3 detects this order is not maintained, it may be forced to begin
|
||||
/// an irrecoverable panic.
|
||||
///
|
||||
/// # Panics
|
||||
/// - If the `auto-initialize` feature is not enabled and the Python interpreter is not
|
||||
/// initialized.
|
||||
#[inline]
|
||||
pub fn acquire_gil() -> GILGuard {
|
||||
GILGuard::acquire()
|
||||
|
|
Loading…
Reference in New Issue