Merge pull request #3707 from davidhewitt/gil-refs-feature

add `gil-refs` feature to aid migration

Cargo.toml

@@ -96,6 +96,9 @@ generate-import-lib = ["pyo3-ffi/generate-import-lib"]
 # Changes `Python::with_gil` to automatically initialize the Python interpreter if needed.
 auto-initialize = []
 
+# Allows use of the deprecated "GIL Refs" APIs.
+gil-refs = []
+
 # Optimizes PyObject to Vec conversion and so on.
 nightly = []
 

guide/src/features.md

@@ -57,6 +57,12 @@ This feature adds the `pyo3::inspect` module, as well as `IntoPy::type_output` a
 
 This is a first step towards adding first-class support for generating type annotations automatically in PyO3, however work is needed to finish this off. All feedback and offers of help welcome on [issue #2454](https://github.com/PyO3/pyo3/issues/2454).
 
+### `gil-refs`
+
+This feature is a backwards-compatibility feature to allow continued use of the "GIL Refs" APIs deprecated in PyO3 0.21. These APIs have performance drawbacks and soundness edge cases which the newer `Bound<T>` smart pointer and accompanying APIs resolve.
+
+This feature and the APIs it enables is expected to be removed in a future PyO3 version.
+
 ### `macros`
 
 This feature enables a dependency on the `pyo3-macros` crate, which provides the procedural macros portion of PyO3's API:

guide/src/migration.md

@@ -5,6 +5,42 @@ For a detailed list of all changes, see the [CHANGELOG](changelog.md).
 
 ## from 0.20.* to 0.21
 
+PyO3 0.21 introduces a new `Bound<'py, T>` smart pointer which replaces the existing "GIL Refs" API to interact with Python objects. For example, in PyO3 0.20 the reference `&'py PyAny` would be used to interact with Python objects. In PyO3 0.21 the updated type is `Bound<'py, PyAny>`. Making this change moves Rust ownership semantics out of PyO3's internals and into user code. This change fixes [a known soundness edge case of interaction with gevent](https://github.com/PyO3/pyo3/issues/3668) as well as improves CPU and [memory performance](https://github.com/PyO3/pyo3/issues/1056). For a full history of discussion see https://github.com/PyO3/pyo3/issues/3382.
+
+The "GIL Ref" `&'py PyAny` and similar types such as `&'py PyDict` continue to be available as a deprecated API. Due to the advantages of the new API it is advised that all users make the effort to upgrade as soon as possible.
+
+In addition to the major API type overhaul, PyO3 has needed to make a few small breaking adjustments to other APIs to close correctness and soundness gaps.
+
+The recommended steps to update to PyO3 0.21 is as follows:
+  1. Enable the `gil-refs` feature to silence deprecations related to the API change
+  2. Fix all other PyO3 0.21 migration steps
+  3. Disable the `gil-refs` feature and migrate off the deprecated APIs
+
+The following sections are laid out in this order.
+
+### Enable the `gil-refs` feature
+
+To make the transition for the PyO3 ecosystem away from the GIL Refs API as smooth as possible, in PyO3 0.21 no APIs consuming or producing GIL Refs have been altered. Instead, variants using `Bound<T>` smart pointers have been introduced, for example `PyTuple::new_bound` which returns `Bound<PyTuple>` is the replacement form of `PyTuple::new`. The GIL Ref APIs have been deprecated, but to make migration easier it is possible to disable these deprecation warnings by enabling the `gil-refs` feature.
+
+It is recommended that users do this as a first step of updating to PyO3 0.21 so that the deprecation warnings do not get in the way of resolving the rest of the migration steps.
+
+Before:
+
+```toml
+# Cargo.toml
+[dependencies]
+pyo3 = "0.20"
+```
+
+After:
+
+```toml
+# Cargo.toml
+[dependencies]
+pyo3 = { version = "0.21", features = ["gil-refs"] }
+```
+
+
 ### `PyTypeInfo` and `PyTryFrom` have been adjusted
 
 The `PyTryFrom` trait has aged poorly, its [`try_from`] method now conflicts with `try_from` in the 2021 edition prelude. A lot of its functionality was also duplicated with `PyTypeInfo`.
@@ -196,6 +232,10 @@ impl PyClassAsyncIter {
 
 `PyType::name` has been renamed to `PyType::qualname` to indicate that it does indeed return the [qualified name](https://docs.python.org/3/glossary.html#term-qualified-name), matching the `__qualname__` attribute. The newly added `PyType::name` yields the full name including the module name now which corresponds to `__module__.__name__` on the level of attributes.
 
+### Migrating from the GIL-Refs API to `Bound<T>`
+
+TODO
+
 ## from 0.19.* to 0.20
 
 ### Drop support for older technologies

newsfragments/3707.added.md

@@ -0,0 +1 @@
+Add `gil-refs` feature to allow continued use of the deprecated GIL Refs APIs.

src/err/mod.rs

@@ -536,9 +536,12 @@ impl PyErr {
     }
 
     /// Deprecated form of `PyErr::write_unraisable_bound`.
-    #[deprecated(
-        since = "0.21.0",
-        note = "`PyErr::write_unraisable` will be replaced by `PyErr::write_unraisable_bound` in a future PyO3 version"
+    #[cfg_attr(
+        not(feature = "gil-refs"),
+        deprecated(
+            since = "0.21.0",
+            note = "`PyErr::write_unraisable` will be replaced by `PyErr::write_unraisable_bound` in a future PyO3 version"
+        )
     )]
     #[inline]
     pub fn write_unraisable(self, py: Python<'_>, obj: Option<&PyAny>) {

src/types/datetime.rs

@@ -164,9 +164,12 @@ pub trait PyTimeAccess {
 /// Trait for accessing the components of a struct containing a tzinfo.
 pub trait PyTzInfoAccess<'py> {
     /// Deprecated form of `get_tzinfo_bound`.
-    #[deprecated(
-        since = "0.21.0",
-        note = "`get_tzinfo` will be replaced by `get_tzinfo_bound` in a future PyO3 version"
+    #[cfg_attr(
+        not(feature = "gil-refs"),
+        deprecated(
+            since = "0.21.0",
+            note = "`get_tzinfo` will be replaced by `get_tzinfo_bound` in a future PyO3 version"
+        )
     )]
     fn get_tzinfo(&self) -> Option<&'py PyTzInfo> {
         self.get_tzinfo_bound().map(Bound::into_gil_ref)
@@ -734,7 +737,7 @@ mod tests {
 
     #[test]
     #[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons
-    #[allow(deprecated)]
+    #[cfg_attr(not(feature = "gil-refs"), allow(deprecated))]
     fn test_get_tzinfo() {
         crate::Python::with_gil(|py| {
             let utc = timezone_utc(py);

src/types/pysuper.rs

@@ -17,9 +17,12 @@ pyobject_native_type_core!(
 
 impl PySuper {
     /// Deprecated form of `PySuper::new_bound`.
-    #[deprecated(
-        since = "0.21.0",
-        note = "`PySuper::new` will be replaced by `PySuper::new_bound` in a future PyO3 version"
+    #[cfg_attr(
+        not(feature = "gil-refs"),
+        deprecated(
+            since = "0.21.0",
+            note = "`PySuper::new` will be replaced by `PySuper::new_bound` in a future PyO3 version"
+        )
     )]
     pub fn new<'py>(ty: &'py PyType, obj: &'py PyAny) -> PyResult<&'py PySuper> {
         Self::new_bound(&ty.as_borrowed(), &obj.as_borrowed()).map(Bound::into_gil_ref)

tests/test_exceptions.rs

@@ -100,7 +100,7 @@ fn test_exception_nosegfault() {
 
 #[test]
 #[cfg(Py_3_8)]
-#[allow(deprecated)]
+#[cfg_attr(not(feature = "gil-refs"), allow(deprecated))]
 fn test_write_unraisable() {
     use common::UnraisableCapture;
     use pyo3::{exceptions::PyRuntimeError, ffi};

tests/test_super.rs

@@ -35,7 +35,7 @@ impl SubClass {
     }
 
     fn method_super_new(self_: &PyCell<Self>) -> PyResult<&PyAny> {
-        #[allow(deprecated)]
+        #[cfg_attr(not(feature = "gil-refs"), allow(deprecated))]
         let super_ = PySuper::new(self_.get_type(), self_)?;
         super_.call_method("method", (), None)
     }