Merge pull request #2165 from mejrs/auto_trait

Implement Auto trait

.github/workflows/ci.yml

@@ -72,7 +72,7 @@ jobs:
 
   build:
     needs: [fmt] # don't wait for clippy as fails rarely and takes longer
-    name: python${{ matrix.python-version }}-${{ matrix.platform.python-architecture }} ${{ matrix.platform.os }} ${{ matrix.msrv }}
+    name: python${{ matrix.python-version }}-${{ matrix.platform.python-architecture }} ${{ matrix.platform.os }} rust-${{ matrix.rust }}
     runs-on: ${{ matrix.platform.os }}
     strategy:
       fail-fast: false # If one platform fails, allow the rest to keep testing.
@@ -120,6 +120,16 @@ jobs:
               }
             msrv: "MSRV"
 
+          # Test the `nightly` feature
+          - rust: nightly
+            python-version: "3.10"
+            platform:
+              {
+                os: "ubuntu-latest",
+                python-architecture: "x64",
+                rust-target: "x86_64-unknown-linux-gnu",
+              }
+            extra_features: "nightly"
     steps:
       - uses: actions/checkout@v2
 
@@ -155,7 +165,7 @@ jobs:
           cargo update -p hashbrown:0.11.2 --precise 0.9.1
 
       - name: Build docs
-        run: cargo doc --no-deps --no-default-features --features full
+        run: cargo doc --no-deps --no-default-features --features "full ${{ matrix.extra_features }}"
 
       - name: Build (no features)
         run: cargo build --lib --tests --no-default-features
@@ -178,26 +188,26 @@ jobs:
           cargo test --no-default-features
 
       - name: Build (all additive features)
-        run: cargo build --lib --tests --no-default-features --features full
+        run: cargo build --lib --tests --no-default-features --features "full ${{ matrix.extra_features }}"
 
       - if: ${{ startsWith(matrix.python-version, 'pypy') }}
         name: Build PyPy (abi3-py37)
-        run: cargo build --lib --tests --no-default-features --features "abi3-py37 full"
+        run: cargo build --lib --tests --no-default-features --features "abi3-py37 full ${{ matrix.extra_features }}"
 
       # Run tests (except on PyPy, because no embedding API).
       - if: ${{ !startsWith(matrix.python-version, 'pypy') }}
         name: Test
-        run: cargo test --no-default-features --features full
+        run: cargo test --no-default-features --features "full ${{ matrix.extra_features }}"
 
       # Run tests again, but in abi3 mode
       - if: ${{ !startsWith(matrix.python-version, 'pypy') }}
         name: Test (abi3)
-        run: cargo test --no-default-features --features "abi3 full"
+        run: cargo test --no-default-features --features "abi3 full ${{ matrix.extra_features }}"
 
       # Run tests again, for abi3-py37 (the minimal Python version)
       - if: ${{ (!startsWith(matrix.python-version, 'pypy')) && (matrix.python-version != '3.7') }}
         name: Test (abi3-py37)
-        run: cargo test --no-default-features --features "abi3-py37 full"
+        run: cargo test --no-default-features --features "abi3-py37 full ${{ matrix.extra_features }}"
 
       - name: Test proc-macro code
         run: cargo test --manifest-path=pyo3-macros-backend/Cargo.toml

Cargo.toml

@@ -46,6 +46,7 @@ trybuild = "1.0.49"
 rustversion = "1.0"
 # 1.0.0 requires Rust 1.50
 proptest = { version = "0.10.1", default-features = false, features = ["std"] }
+send_wrapper = "0.5"
 serde_json = "1.0.61"
 
 [build-dependencies]

guide/src/features.md

@@ -73,9 +73,7 @@ This feature enables the `#[pyproto]` macro, which is an alternative (older, soo
 
 ### `nightly`
 
-The `nightly` feature needs the nightly Rust compiler. This allows PyO3 to use Rust's unstable specialization feature to apply the following optimizations:
+The `nightly` feature needs the nightly Rust compiler. This allows PyO3 to use the auto_traits and negative_impls features to fix the `Python::allow_threads` function.
-- `FromPyObject` for `Vec` and `[T;N]` can perform a `memcpy` when the object supports the Python buffer protocol.
-- `ToBorrowedObject` can skip a reference count increase when the provided object is a Python native type.
 
 ### `resolve-config`
 

pytests/src/othermod.rs

@@ -1,4 +1,4 @@
-//! https://github.com/PyO3/pyo3/issues/233
+//! <https://github.com/PyO3/pyo3/issues/233>
 //!
 //! The code below just tries to use the most important code generation paths
 

src/conversion.rs

@@ -97,38 +97,7 @@ pub trait ToBorrowedObject: ToPyObject {
     }
 }
 
-impl<T> ToBorrowedObject for T
+impl<T> ToBorrowedObject for T where T: ToPyObject {}
-where
-    T: ToPyObject,
-{
-    #[cfg(feature = "nightly")]
-    #[cfg_attr(docsrs, doc(cfg(feature = "nightly")))]
-    default fn with_borrowed_ptr<F, R>(&self, py: Python, f: F) -> R
-    where
-        F: FnOnce(*mut ffi::PyObject) -> R,
-    {
-        let ptr = self.to_object(py).into_ptr();
-        let result = f(ptr);
-        unsafe {
-            ffi::Py_XDECREF(ptr);
-        }
-        result
-    }
-}
-
-#[cfg(feature = "nightly")]
-#[cfg_attr(docsrs, doc(cfg(feature = "nightly")))]
-impl<T> ToBorrowedObject for T
-where
-    T: ToPyObject + AsPyPointer,
-{
-    fn with_borrowed_ptr<F, R>(&self, _py: Python, f: F) -> R
-    where
-        F: FnOnce(*mut ffi::PyObject) -> R,
-    {
-        f(self.as_ptr())
-    }
-}
 
 /// Defines a conversion from a Rust type to a Python object.
 ///

src/conversions/array.rs

@@ -18,37 +18,9 @@ mod min_const_generics {
     where
         T: FromPyObject<'a>,
     {
-        #[cfg(not(feature = "nightly"))]
         fn extract(obj: &'a PyAny) -> PyResult<Self> {
             create_array_from_obj(obj)
         }
-
-        #[cfg(feature = "nightly")]
-        default fn extract(obj: &'a PyAny) -> PyResult<Self> {
-            create_array_from_obj(obj)
-        }
-    }
-
-    #[cfg(all(feature = "nightly", not(Py_LIMITED_API)))]
-    impl<'source, T, const N: usize> FromPyObject<'source> for [T; N]
-    where
-        for<'a> T: Default + FromPyObject<'a> + crate::buffer::Element,
-    {
-        fn extract(obj: &'source PyAny) -> PyResult<Self> {
-            use crate::AsPyPointer;
-            // first try buffer protocol
-            if unsafe { crate::ffi::PyObject_CheckBuffer(obj.as_ptr()) } == 1 {
-                if let Ok(buf) = crate::buffer::PyBuffer::get(obj) {
-                    let mut array = [T::default(); N];
-                    if buf.dimensions() == 1 && buf.copy_to_slice(obj.py(), &mut array).is_ok() {
-                        buf.release(obj.py());
-                        return Ok(array);
-                    }
-                    buf.release(obj.py());
-                }
-            }
-            create_array_from_obj(obj)
-        }
     }
 
     fn create_array_from_obj<'s, T, const N: usize>(obj: &'s PyAny) -> PyResult<[T; N]>
@@ -185,42 +157,11 @@ mod array_impls {
                 where
                     T: Copy + Default + FromPyObject<'a>,
                 {
-                    #[cfg(not(feature = "nightly"))]
                     fn extract(obj: &'a PyAny) -> PyResult<Self> {
                         let mut array = [T::default(); $N];
                         extract_sequence_into_slice(obj, &mut array)?;
                         Ok(array)
                     }
-
-                    #[cfg(feature = "nightly")]
-                    default fn extract(obj: &'a PyAny) -> PyResult<Self> {
-                        let mut array = [T::default(); $N];
-                        extract_sequence_into_slice(obj, &mut array)?;
-                        Ok(array)
-                    }
-                }
-
-                #[cfg(feature = "nightly")]
-                impl<'source, T> FromPyObject<'source> for [T; $N]
-                where
-                    for<'a> T: Default + FromPyObject<'a> + crate::buffer::Element,
-                {
-                    fn extract(obj: &'source PyAny) -> PyResult<Self> {
-                        let mut array = [T::default(); $N];
-                        // first try buffer protocol
-                        if unsafe { crate::ffi::PyObject_CheckBuffer(obj.as_ptr()) } == 1 {
-                            if let Ok(buf) = crate::buffer::PyBuffer::get(obj) {
-                                if buf.dimensions() == 1 && buf.copy_to_slice(obj.py(), &mut array).is_ok() {
-                                    buf.release(obj.py());
-                                    return Ok(array);
-                                }
-                                buf.release(obj.py());
-                            }
-                        }
-                        // fall back to sequence protocol
-                        extract_sequence_into_slice(obj, &mut array)?;
-                        Ok(array)
-                    }
                 }
             )+
         }

src/err/mod.rs

@@ -38,6 +38,9 @@ pub struct PyErr {
     state: UnsafeCell<Option<PyErrState>>,
 }
 
+// The inner value is only accessed through ways that require proving the gil is held
+#[cfg(feature = "nightly")]
+unsafe impl crate::marker::Ungil for PyErr {}
 unsafe impl Send for PyErr {}
 unsafe impl Sync for PyErr {}
 

src/instance.rs

@@ -223,6 +223,9 @@ pub unsafe trait PyNativeType: Sized {
 #[repr(transparent)]
 pub struct Py<T>(NonNull<ffi::PyObject>, PhantomData<T>);
 
+// The inner value is only accessed through ways that require proving the gil is held
+#[cfg(feature = "nightly")]
+unsafe impl<T> crate::marker::Ungil for Py<T> {}
 unsafe impl<T> Send for Py<T> {}
 unsafe impl<T> Sync for Py<T> {}
 

src/lib.rs

@@ -1,4 +1,4 @@
-#![cfg_attr(feature = "nightly", feature(specialization))]
+#![cfg_attr(feature = "nightly", feature(auto_traits, negative_impls))]
 #![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))]
 #![cfg_attr(
     docsrs, // rustdoc:: is not supported on msrv
@@ -94,9 +94,7 @@
 //!
 //! ## Unstable features
 //!
-//! - `nightly`: Gates some optimizations that rely on
+//! - `nightly`: Uses  `#![feature(auto_traits, negative_impls)]` to define [`Ungil`] as an auto trait.
-//! [`#![feature(specialization)]`](https://github.com/rust-lang/rfcs/blob/master/text/1210-impl-specialization.md),
-//! for which you'd also need nightly Rust. You should not use this feature.
 //
 //! ## `rustc` environment flags
 //!
@@ -283,6 +281,7 @@
 //! [Python from Rust]: https://github.com/PyO3/pyo3#using-python-from-rust
 //! [Rust from Python]: https://github.com/PyO3/pyo3#using-rust-from-python
 //! [Features chapter of the guide]: https://pyo3.rs/latest/features.html#features-reference "Features Reference - PyO3 user guide"
+//! [`Ungil`]: crate::marker::Ungil
 pub use crate::class::*;
 pub use crate::conversion::{
     AsPyPointer, FromPyObject, FromPyPointer, IntoPy, IntoPyPointer, PyTryFrom, PyTryInto,
@@ -293,12 +292,13 @@ pub use crate::err::{PyDowncastError, PyErr, PyErrArguments, PyResult};
 pub use crate::gil::{prepare_freethreaded_python, with_embedded_python_interpreter};
 pub use crate::gil::{GILGuard, GILPool};
 pub use crate::instance::{Py, PyNativeType, PyObject};
+pub use crate::marker::Python;
 pub use crate::pycell::{PyCell, PyRef, PyRefMut};
 pub use crate::pyclass::PyClass;
 pub use crate::pyclass_init::PyClassInitializer;
-pub use crate::python::{Python, PythonVersionInfo};
 pub use crate::type_object::PyTypeInfo;
 pub use crate::types::PyAny;
+pub use crate::version::PythonVersionInfo;
 
 // Old directory layout, to be rethought?
 #[cfg(not(feature = "pyproto"))]
@@ -355,6 +355,7 @@ mod gil;
 #[doc(hidden)]
 pub mod impl_;
 mod instance;
+pub mod marker;
 pub mod marshal;
 pub mod once_cell;
 pub mod panic;
@@ -362,9 +363,10 @@ pub mod prelude;
 pub mod pycell;
 pub mod pyclass;
 pub mod pyclass_init;
-mod python;
+
 pub mod type_object;
 pub mod types;
+mod version;
 
 pub use crate::conversions::*;
 

src/macros.rs

@@ -5,7 +5,7 @@
 /// This macro internally calls [`Python::run`](crate::Python::run) and panics
 /// if it returns `Err`, after printing the error to stdout.
 ///
-/// If you need to handle failures, please use [`Python::run`](crate::python::Python::run) instead.
+/// If you need to handle failures, please use [`Python::run`](crate::marker::Python::run) instead.
 ///
 /// # Examples
 /// ```

src/marker.rs

@@ -2,102 +2,188 @@
 //
 // based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython
 
+//! Fundamental properties of objects tied to the Python interpreter.
+//!
+//! The Python interpreter is not threadsafe. To protect the Python interpreter in multithreaded
+//! scenarios there is a global lock, the *global interpreter lock* (hereafter referred to as *GIL*)
+//! that must be held to safely interact with Python objects. This is why in PyO3 when you acquire
+//! the GIL you get a [`Python`] marker token that carries the *lifetime* of holding the GIL and all
+//! borrowed references to Python objects carry this lifetime as well. This will statically ensure
+//! that you can never use Python objects after dropping the lock - if you mess this up it will be
+//! caught at compile time and your program will fail to compile.
+//!
+//! It also supports this pattern that many extension modules employ:
+//! - Drop the GIL, so that other Python threads can acquire it and make progress themselves
+//! - Do something independently of the Python interpreter, like IO, a long running calculation or
+//! awaiting a future
+//! - Once that is done, reacquire the GIL
+//!
+//! That API is provided by [`Python::allow_threads`] and enforced via the [`Ungil`] bound on the
+//! closure and the return type. This is done by relying on the [`Send`] auto trait. `Ungil` is
+//! defined as the following:
+//!
+//! ```rust
+//! pub unsafe trait Ungil {}
+//!
+//! unsafe impl<T: Send> Ungil for T {}
+//! ```
+//!
+//! We piggy-back off the `Send` auto trait because it is not possible to implement custom auto
+//! traits on stable Rust. This is the solution which enables it for as many types as possible while
+//! making the API usable.
+//!
+//! In practice this API works quite well, but it comes with some drawbacks:
+//!
+//! ## Drawbacks
+//!
+//! There is no reason to prevent `!Send` types like [`Rc`] from crossing the closure. After all,
+//! [`Python::allow_threads`] just lets other Python threads run - it does not itself launch a new
+//! thread.
+//!
+//! ```rust, compile_fail
+//! # #[cfg(feature = "nightly")]
+//! # compile_error!("this actually works on nightly")
+//! use pyo3::prelude::*;
+//! use std::rc::Rc;
+//!
+//! fn main() {
+//!     Python::with_gil(|py| {
+//!         let rc = Rc::new(5);
+//!
+//!         py.allow_threads(|| {
+//!             // This would actually be fine...
+//!             println!("{:?}", *rc);
+//!         });
+//!     });
+//! }
+//! ```
+//!
+//! Because we are using `Send` for something it's not quite meant for, other code that
+//! (correctly) upholds the invariants of [`Send`] can cause problems.
+//!
+//! [`SendWrapper`] is one of those. Per its documentation:
+//!
+//! > A wrapper which allows you to move around non-Send-types between threads, as long as you
+//! > access the contained value only from within the original thread and make sure that it is
+//! > dropped from within the original thread.
+//!
+//! This will "work" to smuggle Python references across the closure, because we're not actually
+//! doing anything with threads:
+//!
+//! ```rust, no_run
+//! use pyo3::prelude::*;
+//! use pyo3::types::PyString;
+//! use send_wrapper::SendWrapper;
+//!
+//! Python::with_gil(|py| {
+//!     let string = PyString::new(py, "foo");
+//!
+//!     let wrapped = SendWrapper::new(string);
+//!
+//!     py.allow_threads(|| {
+//! # #[cfg(not(feature = "nightly"))]
+//! # {
+//!         // 💥 Unsound! 💥
+//!         let smuggled: &PyString = *wrapped;
+//!         println!("{:?}", smuggled);
+//! # }
+//!     });
+//! });
+//! ```
+//!
+//! For now the answer to that is "don't do that".
+//!
+//! # A proper implementation using an auto trait
+//!
+//! However on nightly Rust and when PyO3's `nightly` feature is
+//! enabled, `Ungil` is defined as the following:
+//!
+//! ```rust
+//! # #[cfg(FALSE)]
+//! # {
+//! #![feature(auto_traits, negative_impls)]
+//!
+//! pub unsafe auto trait Ungil {}
+//!
+//! // It is unimplemented for the `Python` struct and Python objects.
+//! impl !Ungil for Python<'_> {}
+//! impl !Ungil for ffi::PyObject {}
+//!
+//! // `Py` wraps it in  a safe api, so this is OK
+//! unsafe impl <T> Ungil for Py<T> {}
+//! # }
+//! ```
+//!
+//! With this feature enabled, the above two examples will start working and not working, respectively.
+//!
+//! [`SendWrapper`]: https://docs.rs/send_wrapper/latest/send_wrapper/struct.SendWrapper.html
+//! [`Rc`]: std::rc::Rc
+//! [`Py`]: crate::Py
 use crate::err::{self, PyDowncastError, PyErr, PyResult};
 use crate::gil::{self, GILGuard, GILPool};
 use crate::impl_::not_send::NotSend;
 use crate::type_object::{PyTypeInfo, PyTypeObject};
 use crate::types::{PyAny, PyDict, PyModule, PyType};
+use crate::version::PythonVersionInfo;
 use crate::{ffi, AsPyPointer, FromPyPointer, IntoPyPointer, PyNativeType, PyObject, PyTryFrom};
 use std::ffi::{CStr, CString};
 use std::marker::PhantomData;
 use std::os::raw::c_int;
 
-/// Represents the major, minor, and patch (if any) versions of this interpreter.
+/// Types that are safe to access while the GIL is not held.
 ///
-/// See [Python::version].
+/// # Safety
-#[derive(Debug)]
+///
-pub struct PythonVersionInfo<'py> {
+/// The type must not carry borrowed Python references or, if it does, not allow access to them if
-    /// Python major version (e.g. `3`).
+/// the GIL is not held.
-    pub major: u8,
+///
-    /// Python minor version (e.g. `11`).
+/// See the [module-level documentation](self) for more information.
-    pub minor: u8,
+#[cfg_attr(docsrs, doc(cfg(all())))] // Hide the cfg flag
-    /// Python patch version (e.g. `0`).
+#[cfg(not(feature = "nightly"))]
-    pub patch: u8,
+pub unsafe trait Ungil {}
-    /// Python version suffix, if applicable (e.g. `a0`).
-    pub suffix: Option<&'py str>,
-}
 
-impl<'py> PythonVersionInfo<'py> {
+#[cfg_attr(docsrs, doc(cfg(all())))] // Hide the cfg flag
-    /// Parses a hard-coded Python interpreter version string (e.g. 3.9.0a4+).
+#[cfg(not(feature = "nightly"))]
-    ///
+unsafe impl<T: Send> Ungil for T {}
-    /// Panics if the string is ill-formatted.
-    fn from_str(version_number_str: &'py str) -> Self {
-        fn split_and_parse_number(version_part: &str) -> (u8, Option<&str>) {
-            match version_part.find(|c: char| !c.is_ascii_digit()) {
-                None => (version_part.parse().unwrap(), None),
-                Some(version_part_suffix_start) => {
-                    let (version_part, version_part_suffix) =
-                        version_part.split_at(version_part_suffix_start);
-                    (version_part.parse().unwrap(), Some(version_part_suffix))
-                }
-            }
-        }
 
-        let mut parts = version_number_str.split('.');
+/// Types that are safe to access while the GIL is not held.
-        let major_str = parts.next().expect("Python major version missing");
+///
-        let minor_str = parts.next().expect("Python minor version missing");
+/// # Safety
-        let patch_str = parts.next();
+///
-        assert!(
+/// The type must not carry borrowed Python references or, if it does, not allow access to them if
-            parts.next().is_none(),
+/// the GIL is not held.
-            "Python version string has too many parts"
+///
-        );
+/// See the [module-level documentation](self) for more information.
+#[cfg(feature = "nightly")]
+pub unsafe auto trait Ungil {}
 
-        let major = major_str
+#[cfg(feature = "nightly")]
-            .parse()
+mod negative_impls {
-            .expect("Python major version not an integer");
+    use super::Ungil;
-        let (minor, suffix) = split_and_parse_number(minor_str);
-        if suffix.is_some() {
-            assert!(patch_str.is_none());
-            return PythonVersionInfo {
-                major,
-                minor,
-                patch: 0,
-                suffix,
-            };
-        }
 
-        let (patch, suffix) = patch_str.map(split_and_parse_number).unwrap_or_default();
+    impl !Ungil for crate::Python<'_> {}
-        PythonVersionInfo {
-            major,
-            minor,
-            patch,
-            suffix,
-        }
-    }
-}
 
-impl PartialEq<(u8, u8)> for PythonVersionInfo<'_> {
+    // This means that PyString, PyList, etc all inherit !Ungil from  this.
-    fn eq(&self, other: &(u8, u8)) -> bool {
+    impl !Ungil for crate::PyAny {}
-        self.major == other.0 && self.minor == other.1
-    }
-}
 
-impl PartialEq<(u8, u8, u8)> for PythonVersionInfo<'_> {
+    // All the borrowing wrappers
-    fn eq(&self, other: &(u8, u8, u8)) -> bool {
+    impl<T> !Ungil for crate::PyCell<T> {}
-        self.major == other.0 && self.minor == other.1 && self.patch == other.2
+    impl<T> !Ungil for crate::PyRef<'_, T> {}
-    }
+    impl<T> !Ungil for crate::PyRefMut<'_, T> {}
-}
 
-impl PartialOrd<(u8, u8)> for PythonVersionInfo<'_> {
+    // FFI pointees
-    fn partial_cmp(&self, other: &(u8, u8)) -> Option<std::cmp::Ordering> {
+    impl !Ungil for crate::ffi::PyObject {}
-        (self.major, self.minor).partial_cmp(other)
+    impl !Ungil for crate::ffi::PyLongObject {}
-    }
-}
 
-impl PartialOrd<(u8, u8, u8)> for PythonVersionInfo<'_> {
+    impl !Ungil for crate::ffi::PyThreadState {}
-    fn partial_cmp(&self, other: &(u8, u8, u8)) -> Option<std::cmp::Ordering> {
+    impl !Ungil for crate::ffi::PyInterpreterState {}
-        (self.major, self.minor, self.patch).partial_cmp(other)
+    impl !Ungil for crate::ffi::PyWeakReference {}
-    }
+    impl !Ungil for crate::ffi::PyFrameObject {}
+    impl !Ungil for crate::ffi::PyCodeObject {}
+    #[cfg(not(Py_LIMITED_API))]
+    impl !Ungil for crate::ffi::PyDictKeysObject {}
+    #[cfg(not(Py_LIMITED_API))]
+    impl !Ungil for crate::ffi::PyArena {}
 }
 
 /// A marker token that represents holding the GIL.
@@ -305,12 +391,8 @@ impl<'py> Python<'py> {
     /// interpreter for some time and have other Python threads around, this will let you run
     /// Rust-only code while letting those other Python threads make progress.
     ///
-    /// The closure is impermeable to types that are tied to holding the GIL, such as `&`[`PyAny`]
+    /// Only types that implement [`Ungil`] can cross the closure. See the
-    /// and its concrete-typed siblings like `&`[`PyString`]. This is achieved via the [`Send`]
+    /// [module level documentation](self) for more information.
-    /// bound on the closure and the return type. This is slightly
-    /// more restrictive than necessary, but it's the most fitting solution available in stable
-    /// Rust. In the future this bound may be relaxed by using an "auto-trait" instead, if
-    /// [auto-traits] ever become a stable feature of the Rust language.
     ///
     /// If you need to pass Python objects into the closure you can use [`Py`]`<T>`to create a
     /// reference independent of the GIL lifetime. However, you cannot do much with those without a
@@ -365,8 +447,8 @@ impl<'py> Python<'py> {
     /// [Parallelism]: https://pyo3.rs/main/parallelism.html
     pub fn allow_threads<T, F>(self, f: F) -> T
     where
-        F: Send + FnOnce() -> T,
+        F: Ungil + FnOnce() -> T,
-        T: Send,
+        T: Ungil,
     {
         // Use a guard pattern to handle reacquiring the GIL, so that the GIL will be reacquired
         // even if `f` panics.
@@ -532,7 +614,7 @@ impl<'py> Python<'py> {
     /// # use pyo3::Python;
     /// Python::with_gil(|py| {
     ///     // The full string could be, for example:
-    ///     // "3.0a5+ (py3k:63103M, May 12 2008, 00:53:55) \n[GCC 4.2.3]"
+    ///     // "3.10.0 (tags/v3.10.0:b494f59, Oct  4 2021, 19:00:18) [MSC v.1929 64 bit (AMD64)]"
     ///     assert!(py.version().starts_with("3."));
     /// });
     /// ```
@@ -563,7 +645,7 @@ impl<'py> Python<'py> {
         // version number.
         let version_number_str = version_str.split(' ').next().unwrap_or(version_str);
 
-        PythonVersionInfo::from_str(version_number_str)
+        PythonVersionInfo::from_str(version_number_str).unwrap()
     }
 
     /// Registers the object in the release pool, and tries to downcast to specific type.
@@ -804,6 +886,8 @@ impl<'py> Python<'py> {
 mod tests {
     use super::*;
     use crate::types::{IntoPyDict, PyList};
+    use crate::Py;
+    use std::sync::Arc;
 
     #[test]
     fn test_eval() {
@@ -889,36 +973,19 @@ mod tests {
     }
 
     #[test]
-    fn test_python_version_info() {
+    fn test_allow_threads_pass_stuff_in() {
-        Python::with_gil(|py| {
+        let list: Py<PyList> = Python::with_gil(|py| {
-            let version = py.version_info();
+            let list = PyList::new(py, vec!["foo", "bar"]);
-            #[cfg(Py_3_7)]
+            list.into()
-            assert!(version >= (3, 7));
-            #[cfg(Py_3_7)]
-            assert!(version >= (3, 7, 0));
-            #[cfg(Py_3_8)]
-            assert!(version >= (3, 8));
-            #[cfg(Py_3_8)]
-            assert!(version >= (3, 8, 0));
-            #[cfg(Py_3_9)]
-            assert!(version >= (3, 9));
-            #[cfg(Py_3_9)]
-            assert!(version >= (3, 9, 0));
         });
-    }
+        let mut v = vec![1, 2, 3];
+        let a = Arc::new(String::from("foo"));
 
-    #[test]
+        Python::with_gil(|py| {
-    fn test_python_version_info_parse() {
+            py.allow_threads(|| {
-        assert!(PythonVersionInfo::from_str("3.5.0a1") >= (3, 5, 0));
+                drop((list, &mut v, a));
-        assert!(PythonVersionInfo::from_str("3.5+") >= (3, 5, 0));
+            });
-        assert!(PythonVersionInfo::from_str("3.5+") == (3, 5, 0));
+        });
-        assert!(PythonVersionInfo::from_str("3.5+") != (3, 5, 1));
-        assert!(PythonVersionInfo::from_str("3.5.2a1+") < (3, 5, 3));
-        assert!(PythonVersionInfo::from_str("3.5.2a1+") == (3, 5, 2));
-        assert!(PythonVersionInfo::from_str("3.5.2a1+") == (3, 5));
-        assert!(PythonVersionInfo::from_str("3.5+") == (3, 5));
-        assert!(PythonVersionInfo::from_str("3.5.2a1+") < (3, 6));
-        assert!(PythonVersionInfo::from_str("3.5.2a1+") > (3, 4));
     }
 
     #[test]

src/prelude.rs

@@ -16,9 +16,9 @@ pub use crate::conversion::{
 pub use crate::err::{PyErr, PyResult};
 pub use crate::gil::GILGuard;
 pub use crate::instance::{Py, PyObject};
+pub use crate::marker::Python;
 pub use crate::pycell::{PyCell, PyRef, PyRefMut};
 pub use crate::pyclass_init::PyClassInitializer;
-pub use crate::python::Python;
 pub use crate::types::{PyAny, PyModule};
 
 #[cfg(feature = "macros")]

src/types/sequence.rs

@@ -269,35 +269,9 @@ impl<'a, T> FromPyObject<'a> for Vec<T>
 where
     T: FromPyObject<'a>,
 {
-    #[cfg(not(feature = "nightly"))]
     fn extract(obj: &'a PyAny) -> PyResult<Self> {
         extract_sequence(obj)
     }
-    #[cfg(feature = "nightly")]
-    default fn extract(obj: &'a PyAny) -> PyResult<Self> {
-        extract_sequence(obj)
-    }
-}
-
-#[cfg(all(feature = "nightly", not(Py_LIMITED_API)))]
-impl<'source, T> FromPyObject<'source> for Vec<T>
-where
-    for<'a> T: FromPyObject<'a> + crate::buffer::Element,
-{
-    fn extract(obj: &'source PyAny) -> PyResult<Self> {
-        // first try buffer protocol
-        if let Ok(buf) = crate::buffer::PyBuffer::get(obj) {
-            if buf.dimensions() == 1 {
-                if let Ok(v) = buf.to_vec(obj.py()) {
-                    buf.release(obj.py());
-                    return Ok(v);
-                }
-            }
-            buf.release(obj.py());
-        }
-        // fall back to sequence protocol
-        extract_sequence(obj)
-    }
 }
 
 fn extract_sequence<'s, T>(obj: &'s PyAny) -> PyResult<Vec<T>>

src/version.rs

@@ -0,0 +1,143 @@
+/// Represents the major, minor, and patch (if any) versions of this interpreter.
+///
+/// This struct is usually created with [`Python::version`].
+///
+/// # Examples
+///
+/// ```rust
+/// # use pyo3::Python;
+/// Python::with_gil(|py| {
+///     // PyO3 supports Python 3.7 and up.
+///     assert!(py.version_info() >= (3, 7));
+///     assert!(py.version_info() >= (3, 7, 0));
+/// });
+/// ```
+///
+/// [`Python::version`]: crate::marker::Python::version
+#[derive(Debug)]
+pub struct PythonVersionInfo<'py> {
+    /// Python major version (e.g. `3`).
+    pub major: u8,
+    /// Python minor version (e.g. `11`).
+    pub minor: u8,
+    /// Python patch version (e.g. `0`).
+    pub patch: u8,
+    /// Python version suffix, if applicable (e.g. `a0`).
+    pub suffix: Option<&'py str>,
+}
+
+impl<'py> PythonVersionInfo<'py> {
+    /// Parses a hard-coded Python interpreter version string (e.g. 3.9.0a4+).
+    pub(crate) fn from_str(version_number_str: &'py str) -> Result<Self, &str> {
+        fn split_and_parse_number(version_part: &str) -> (u8, Option<&str>) {
+            match version_part.find(|c: char| !c.is_ascii_digit()) {
+                None => (version_part.parse().unwrap(), None),
+                Some(version_part_suffix_start) => {
+                    let (version_part, version_part_suffix) =
+                        version_part.split_at(version_part_suffix_start);
+                    (version_part.parse().unwrap(), Some(version_part_suffix))
+                }
+            }
+        }
+
+        let mut parts = version_number_str.split('.');
+        let major_str = parts.next().ok_or("Python major version missing")?;
+        let minor_str = parts.next().ok_or("Python minor version missing")?;
+        let patch_str = parts.next();
+        if parts.next().is_some() {
+            return Err("Python version string has too many parts");
+        };
+
+        let major = major_str
+            .parse()
+            .map_err(|_| "Python major version not an integer")?;
+        let (minor, suffix) = split_and_parse_number(minor_str);
+        if suffix.is_some() {
+            assert!(patch_str.is_none());
+            return Ok(PythonVersionInfo {
+                major,
+                minor,
+                patch: 0,
+                suffix,
+            });
+        }
+
+        let (patch, suffix) = patch_str.map(split_and_parse_number).unwrap_or_default();
+        Ok(PythonVersionInfo {
+            major,
+            minor,
+            patch,
+            suffix,
+        })
+    }
+}
+
+impl PartialEq<(u8, u8)> for PythonVersionInfo<'_> {
+    fn eq(&self, other: &(u8, u8)) -> bool {
+        self.major == other.0 && self.minor == other.1
+    }
+}
+
+impl PartialEq<(u8, u8, u8)> for PythonVersionInfo<'_> {
+    fn eq(&self, other: &(u8, u8, u8)) -> bool {
+        self.major == other.0 && self.minor == other.1 && self.patch == other.2
+    }
+}
+
+impl PartialOrd<(u8, u8)> for PythonVersionInfo<'_> {
+    fn partial_cmp(&self, other: &(u8, u8)) -> Option<std::cmp::Ordering> {
+        (self.major, self.minor).partial_cmp(other)
+    }
+}
+
+impl PartialOrd<(u8, u8, u8)> for PythonVersionInfo<'_> {
+    fn partial_cmp(&self, other: &(u8, u8, u8)) -> Option<std::cmp::Ordering> {
+        (self.major, self.minor, self.patch).partial_cmp(other)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::Python;
+    #[test]
+    fn test_python_version_info() {
+        Python::with_gil(|py| {
+            let version = py.version_info();
+            #[cfg(Py_3_7)]
+            assert!(version >= (3, 7));
+            #[cfg(Py_3_7)]
+            assert!(version >= (3, 7, 0));
+            #[cfg(Py_3_8)]
+            assert!(version >= (3, 8));
+            #[cfg(Py_3_8)]
+            assert!(version >= (3, 8, 0));
+            #[cfg(Py_3_9)]
+            assert!(version >= (3, 9));
+            #[cfg(Py_3_9)]
+            assert!(version >= (3, 9, 0));
+            #[cfg(Py_3_10)]
+            assert!(version >= (3, 10));
+            #[cfg(Py_3_10)]
+            assert!(version >= (3, 10, 0));
+            #[cfg(Py_3_11)]
+            assert!(version >= (3, 11));
+            #[cfg(Py_3_11)]
+            assert!(version >= (3, 11, 0));
+        });
+    }
+
+    #[test]
+    fn test_python_version_info_parse() {
+        assert!(PythonVersionInfo::from_str("3.5.0a1").unwrap() >= (3, 5, 0));
+        assert!(PythonVersionInfo::from_str("3.5+").unwrap() >= (3, 5, 0));
+        assert!(PythonVersionInfo::from_str("3.5+").unwrap() == (3, 5, 0));
+        assert!(PythonVersionInfo::from_str("3.5+").unwrap() != (3, 5, 1));
+        assert!(PythonVersionInfo::from_str("3.5.2a1+").unwrap() < (3, 5, 3));
+        assert!(PythonVersionInfo::from_str("3.5.2a1+").unwrap() == (3, 5, 2));
+        assert!(PythonVersionInfo::from_str("3.5.2a1+").unwrap() == (3, 5));
+        assert!(PythonVersionInfo::from_str("3.5+").unwrap() == (3, 5));
+        assert!(PythonVersionInfo::from_str("3.5.2a1+").unwrap() < (3, 6));
+        assert!(PythonVersionInfo::from_str("3.5.2a1+").unwrap() > (3, 4));
+    }
+}

tests/test_compile_error.rs

@@ -7,6 +7,7 @@ fn test_compile_errors() {
     _test_compile_errors()
 }
 
+#[cfg(not(feature = "nightly"))]
 #[rustversion::nightly]
 #[test]
 fn test_compile_errors() {
@@ -15,8 +16,19 @@ fn test_compile_errors() {
     let _ = std::panic::catch_unwind(_test_compile_errors);
 }
 
+#[cfg(feature = "nightly")]
+#[rustversion::nightly]
+#[test]
+fn test_compile_errors() {
+    // nightly - don't care if test output is potentially wrong, to avoid churn in PyO3's CI thanks
+    // to diagnostics changing on nightly.
+    _test_compile_errors()
+}
+
+#[cfg(not(feature = "nightly"))]
 fn _test_compile_errors() {
     let t = trybuild::TestCases::new();
+
     t.compile_fail("tests/ui/invalid_macro_args.rs");
     t.compile_fail("tests/ui/invalid_need_module_arg_position.rs");
     t.compile_fail("tests/ui/invalid_property_args.rs");
@@ -71,6 +83,8 @@ fn _test_compile_errors() {
         t.compile_fail("tests/ui/invalid_pymethods.rs");
         t.compile_fail("tests/ui/missing_clone.rs");
         t.compile_fail("tests/ui/not_send.rs");
+        t.compile_fail("tests/ui/not_send2.rs");
+        t.compile_fail("tests/ui/not_send3.rs");
         #[cfg(Py_LIMITED_API)]
         t.compile_fail("tests/ui/abi3_nativetype_inheritance.rs");
     }
@@ -78,3 +92,12 @@ fn _test_compile_errors() {
     #[rustversion::before(1.58)]
     fn tests_rust_1_58(_t: &trybuild::TestCases) {}
 }
+
+#[cfg(feature = "nightly")]
+fn _test_compile_errors() {
+    let t = trybuild::TestCases::new();
+
+    t.compile_fail("tests/ui/not_send_auto_trait.rs");
+    t.compile_fail("tests/ui/not_send_auto_trait2.rs");
+    t.compile_fail("tests/ui/send_wrapper.rs");
+}

tests/ui/not_send.stderr

@@ -12,8 +12,9 @@ error[E0277]: `*mut pyo3::Python<'static>` cannot be shared between threads safe
     = note: required because it appears within the type `pyo3::Python<'_>`
     = note: required because of the requirements on the impl of `Send` for `&pyo3::Python<'_>`
     = note: required because it appears within the type `[closure@$DIR/tests/ui/not_send.rs:4:22: 4:38]`
+    = note: required because of the requirements on the impl of `Ungil` for `[closure@$DIR/tests/ui/not_send.rs:4:22: 4:38]`
 note: required by a bound in `pyo3::Python::<'py>::allow_threads`
-   --> src/python.rs
+   --> src/marker.rs
     |
-    |         F: Send + FnOnce() -> T,
+    |         F: Ungil + FnOnce() -> T,
-    |            ^^^^ required by this bound in `pyo3::Python::<'py>::allow_threads`
+    |            ^^^^^ required by this bound in `pyo3::Python::<'py>::allow_threads`

tests/ui/not_send2.rs

@@ -0,0 +1,12 @@
+use pyo3::prelude::*;
+use pyo3::types::PyString;
+
+fn main() {
+    Python::with_gil(|py| {
+        let string = PyString::new(py, "foo");
+
+        py.allow_threads(|| {
+            println!("{:?}", string);
+        });
+    });
+}

tests/ui/not_send2.stderr

@@ -0,0 +1,18 @@
+error[E0277]: `UnsafeCell<PyObject>` cannot be shared between threads safely
+   --> tests/ui/not_send2.rs:8:12
+    |
+8   |         py.allow_threads(|| {
+    |            ^^^^^^^^^^^^^ `UnsafeCell<PyObject>` cannot be shared between threads safely
+    |
+    = help: within `&PyString`, the trait `Sync` is not implemented for `UnsafeCell<PyObject>`
+    = note: required because it appears within the type `PyAny`
+    = note: required because it appears within the type `PyString`
+    = note: required because it appears within the type `&PyString`
+    = note: required because of the requirements on the impl of `Send` for `&&PyString`
+    = note: required because it appears within the type `[closure@$DIR/tests/ui/not_send2.rs:8:26: 10:10]`
+    = note: required because of the requirements on the impl of `Ungil` for `[closure@$DIR/tests/ui/not_send2.rs:8:26: 10:10]`
+note: required by a bound in `pyo3::Python::<'py>::allow_threads`
+   --> src/marker.rs
+    |
+    |         F: Ungil + FnOnce() -> T,
+    |            ^^^^^ required by this bound in `pyo3::Python::<'py>::allow_threads`

tests/ui/not_send3.rs

@@ -0,0 +1,12 @@
+use pyo3::prelude::*;
+use std::rc::Rc;
+
+fn main() {
+    Python::with_gil(|py| {
+        let rc = Rc::new(5);
+
+        py.allow_threads(|| {
+            println!("{:?}", rc);
+        });
+    });
+}

tests/ui/not_send3.stderr

@@ -0,0 +1,15 @@
+error[E0277]: `Rc<i32>` cannot be shared between threads safely
+   --> tests/ui/not_send3.rs:8:12
+    |
+8   |         py.allow_threads(|| {
+    |            ^^^^^^^^^^^^^ `Rc<i32>` cannot be shared between threads safely
+    |
+    = help: the trait `Sync` is not implemented for `Rc<i32>`
+    = note: required because of the requirements on the impl of `Send` for `&Rc<i32>`
+    = note: required because it appears within the type `[closure@$DIR/tests/ui/not_send3.rs:8:26: 10:10]`
+    = note: required because of the requirements on the impl of `Ungil` for `[closure@$DIR/tests/ui/not_send3.rs:8:26: 10:10]`
+note: required by a bound in `pyo3::Python::<'py>::allow_threads`
+   --> src/marker.rs
+    |
+    |         F: Ungil + FnOnce() -> T,
+    |            ^^^^^ required by this bound in `pyo3::Python::<'py>::allow_threads`

tests/ui/not_send_auto_trait.rs

@@ -0,0 +1,11 @@
+use pyo3::prelude::*;
+
+fn test_not_send_allow_threads(py: Python) {
+    py.allow_threads(|| { drop(py); });
+}
+
+fn main() {
+    Python::with_gil(|py| {
+        test_not_send_allow_threads(py);
+    })
+}

tests/ui/not_send_auto_trait.stderr

@@ -0,0 +1,15 @@
+error[E0277]: the trait bound `pyo3::Python<'_>: Ungil` is not satisfied in `[closure@$DIR/tests/ui/not_send_auto_trait.rs:4:22: 4:38]`
+   --> tests/ui/not_send_auto_trait.rs:4:8
+    |
+4   |     py.allow_threads(|| { drop(py); });
+    |        ^^^^^^^^^^^^^ ---------------- within this `[closure@$DIR/tests/ui/not_send_auto_trait.rs:4:22: 4:38]`
+    |        |
+    |        within `[closure@$DIR/tests/ui/not_send_auto_trait.rs:4:22: 4:38]`, the trait `Ungil` is not implemented for `pyo3::Python<'_>`
+    |
+    = note: required because it appears within the type `&pyo3::Python<'_>`
+    = note: required because it appears within the type `[closure@$DIR/tests/ui/not_send_auto_trait.rs:4:22: 4:38]`
+note: required by a bound in `pyo3::Python::<'py>::allow_threads`
+   --> src/marker.rs
+    |
+    |         F: Ungil + FnOnce() -> T,
+    |            ^^^^^ required by this bound in `pyo3::Python::<'py>::allow_threads`

tests/ui/not_send_auto_trait2.rs

@@ -0,0 +1,12 @@
+use pyo3::prelude::*;
+use pyo3::types::PyString;
+
+fn main() {
+    Python::with_gil(|py| {
+        let string = PyString::new(py, "foo");
+
+        py.allow_threads(|| {
+            println!("{:?}", string);
+        });
+    });
+}

tests/ui/not_send_auto_trait2.stderr

@@ -0,0 +1,20 @@
+error[E0277]: the trait bound `PyAny: Ungil` is not satisfied in `[closure@$DIR/tests/ui/not_send_auto_trait2.rs:8:26: 10:10]`
+   --> tests/ui/not_send_auto_trait2.rs:8:12
+    |
+8   |           py.allow_threads(|| {
+    |  ____________^^^^^^^^^^^^^_-
+    | |            |
+    | |            within `[closure@$DIR/tests/ui/not_send_auto_trait2.rs:8:26: 10:10]`, the trait `Ungil` is not implemented for `PyAny`
+9   | |             println!("{:?}", string);
+10  | |         });
+    | |_________- within this `[closure@$DIR/tests/ui/not_send_auto_trait2.rs:8:26: 10:10]`
+    |
+    = note: required because it appears within the type `PyString`
+    = note: required because it appears within the type `&PyString`
+    = note: required because it appears within the type `&&PyString`
+    = note: required because it appears within the type `[closure@$DIR/tests/ui/not_send_auto_trait2.rs:8:26: 10:10]`
+note: required by a bound in `pyo3::Python::<'py>::allow_threads`
+   --> src/marker.rs
+    |
+    |         F: Ungil + FnOnce() -> T,
+    |            ^^^^^ required by this bound in `pyo3::Python::<'py>::allow_threads`

tests/ui/send_wrapper.rs

@@ -0,0 +1,16 @@
+use pyo3::prelude::*;
+use pyo3::types::PyString;
+use send_wrapper::SendWrapper;
+
+fn main() {
+    Python::with_gil(|py| {
+        let string = PyString::new(py, "foo");
+
+        let wrapped = SendWrapper::new(string);
+
+        py.allow_threads(|| {
+            let smuggled: &PyString = *wrapped;
+            println!("{:?}", smuggled);
+        });
+    });
+}

tests/ui/send_wrapper.stderr

@@ -0,0 +1,23 @@
+error[E0277]: the trait bound `PyAny: Ungil` is not satisfied in `[closure@$DIR/tests/ui/send_wrapper.rs:11:26: 14:10]`
+   --> tests/ui/send_wrapper.rs:11:12
+    |
+11  |           py.allow_threads(|| {
+    |  ____________^^^^^^^^^^^^^_-
+    | |            |
+    | |            within `[closure@$DIR/tests/ui/send_wrapper.rs:11:26: 14:10]`, the trait `Ungil` is not implemented for `PyAny`
+12  | |             let smuggled: &PyString = *wrapped;
+13  | |             println!("{:?}", smuggled);
+14  | |         });
+    | |_________- within this `[closure@$DIR/tests/ui/send_wrapper.rs:11:26: 14:10]`
+    |
+    = note: required because it appears within the type `PyString`
+    = note: required because it appears within the type `&PyString`
+    = note: required because it appears within the type `*mut &PyString`
+    = note: required because it appears within the type `SendWrapper<&PyString>`
+    = note: required because it appears within the type `&SendWrapper<&PyString>`
+    = note: required because it appears within the type `[closure@$DIR/tests/ui/send_wrapper.rs:11:26: 14:10]`
+note: required by a bound in `pyo3::Python::<'py>::allow_threads`
+   --> src/marker.rs
+    |
+    |         F: Ungil + FnOnce() -> T,
+    |            ^^^^^ required by this bound in `pyo3::Python::<'py>::allow_threads`