Merge pull request #795 from PyO3/relase-0.9

Write migration guide

Cargo.toml

@@ -22,8 +22,8 @@ appveyor = { repository = "fafhrd91/pyo3" }
 indoc = "0.3.4"
 inventory = "0.1.4"
 libc = "0.2.62"
-num-bigint = { version = ">= 0.2", optional = true }
-num-complex = { version = ">= 0.2", optional = true }
+num-bigint = { version = "0.2", optional = true }
+num-complex = { version = "0.2", optional = true }
 num-traits = "0.2.8"
 parking_lot = { version = "0.10", features = ["nightly"] }
 paste = "0.1.6"
@@ -32,7 +32,7 @@ unindent = "0.1.4"
 
 [dev-dependencies]
 assert_approx_eq = "1.1.0"
-trybuild = "1.0.14"
+trybuild = "1.0.23"
 
 [build-dependencies]
 regex = "1"

guide/src/SUMMARY.md

@@ -12,4 +12,5 @@
 - [Advanced Topics](advanced.md)
 - [Building and Distribution](building_and_distribution.md)
 - [PyPy support](pypy.md)
-- [Appendix: PyO3 and rust-cpython](rust_cpython.md)
+- [Appendix A: PyO3 and rust-cpython](rust_cpython.md)
+- [Appendix B: Migration Guide](migration.md)

guide/src/class.md

@@ -38,6 +38,7 @@ unsafe impl pyo3::PyTypeInfo for MyClass {
     type BaseLayout = pyo3::pycell::PyCellBase<PyAny>;
     type Layout = PyCell<Self>;
     type Initializer = PyClassInitializer<Self>;
+    type AsRefTarget = PyCell<Self>;
 
     const NAME: &'static str = "MyClass";
     const MODULE: Option<&'static str> = None;

guide/src/migration.md

@@ -0,0 +1,190 @@
+# Appendix B: Migrating from older PyO3 versions
+This guide can help you upgrade code through breaking changes from one PyO3 version to the next. For a detailed list of all changes, see [CHANGELOG.md](https://github.com/PyO3/pyo3/blob/master/CHANGELOG.md)
+## from 0.8.* to 0.9
+
+### `#[new]` interface
+[`PyRawObject`](https://docs.rs/pyo3/0.8.5/pyo3/type_object/struct.PyRawObject.html)
+is now removed and our syntax for constructors has changed.
+
+Before:
+```compile_fail
+#[pyclass]
+struct MyClass {}
+
+#[pymethods]
+impl MyClass {
+   #[new]
+   fn new(obj: &PyRawObject) {
+       obj.init(MyClass { })
+   }
+}
+```
+
+After:
+```
+# use pyo3::prelude::*;
+#[pyclass]
+struct MyClass {}
+
+#[pymethods]
+impl MyClass {
+   #[new]
+   fn new() -> Self {
+       MyClass {}
+   }
+}
+```
+
+Basically you can return `Self` or `Result<Self>` directly.
+For more, see [the constructor section](https://pyo3.rs/master/class.html#constructor) of this guide.
+
+### PyCell
+PyO3 0.9 introduces [`PyCell`](https://pyo3.rs/master/doc/pyo3/pycell/struct.PyCell.html), which is
+a [`RefCell`](https://doc.rust-lang.org/std/cell/struct.RefCell.html)-like object wrapper
+for ensuring Rust's rules regarding aliasing of references are upheld.
+For more detail, see the
+[Rust Book's section on Rust's rules of references](https://doc.rust-lang.org/book/ch04-02-references-and-borrowing.html#the-rules-of-references)
+
+For `#[pymethods]` or `#[pyfunction]`s, your existing code should continue to work without any change.
+Python exceptions will automatically be raised when your functions are used in a way which breaks Rust's
+rules of references.
+
+Here is an example.
+```
+# use pyo3::prelude::*;
+#[pyclass]
+struct Names {
+    names: Vec<String>
+}
+
+#[pymethods]
+impl Names {
+    #[new]
+    fn new() -> Self {
+        Names { names: vec![] }
+    }
+    fn merge(&mut self, other: &mut Names) {
+        self.names.append(&mut other.names)
+    }
+}
+# let gil = Python::acquire_gil();
+# let py = gil.python();
+# let names = PyCell::new(py, Names::new()).unwrap();
+# let borrow_mut_err = py.get_type::<pyo3::pycell::PyBorrowMutError>();
+# pyo3::py_run!(py, names borrow_mut_err, r"
+# try:
+#    names.merge(names)
+#    assert False, 'Unreachable'
+# except Exception as e:
+#    isinstance(e, borrow_mut_err)
+# ");
+```
+`Names` has a `merge` method, which takes `&mut self` and another argument of type `&mut Self`.
+Given this `#[pyclass]`, calling `names.merge(names)` in Python raises a `PyBorrowMutError` exception,
+since it requires two mutable borrows of `names`.
+
+However, for `#[pyproto]` and some functions, you need to manually fix the code.
+
+#### Object creation
+In 0.8 object creation was done with `PyRef::new` and `PyRefMut::new`.
+In 0.9 these have both been removed.
+To upgrade code, please use `PyCell::new` instead.
+If a `PyRef` or `PyRefMut` is needed, just call `.borrow()` or `.borrow_mut()`
+on the newly-created `PyCell`.
+
+Before:
+```compile_fail
+# use pyo3::prelude::*;
+# #[pyclass]
+# struct MyClass {}
+let gil = Python::acquire_gil();
+let py = gil.python();
+let obj_ref = PyRef::new(py, MyClass {}).unwrap();
+```
+
+After:
+```
+# use pyo3::prelude::*;
+# #[pyclass]
+# struct MyClass {}
+let gil = Python::acquire_gil();
+let py = gil.python();
+let obj = PyCell::new(py, MyClass {}).unwrap();
+let obj_ref = obj.borrow();
+```
+
+#### Object extraction
+For `PyClass` types `T`, `&T` and `&mut T` no longer have `FromPyObject` implementations.
+Instead you should extract `PyRef<T>` or `PyRefMut<T>`, respectively.
+If `T` implements `Clone`, you can extract `T` itself.
+In addition, you can also extract `&PyCell<T>`, though you rarely need it.
+
+Before:
+```ignore
+let obj: &PyAny = create_obj();
+let obj_ref: &MyClass = obj.extract().unwrap();
+let obj_ref_mut: &mut MyClass = obj.extract().unwrap();
+```
+
+After:
+```
+# use pyo3::prelude::*;
+# use pyo3::types::{PyAny, IntoPyDict};
+# #[pyclass] #[derive(Clone)] struct MyClass {}
+# #[pymethods] impl MyClass { #[new]fn new() -> Self { MyClass {} }}
+# let gil = Python::acquire_gil();
+# let py = gil.python();
+# let typeobj = py.get_type::<MyClass>();
+# let d = [("c", typeobj)].into_py_dict(py);
+# let create_obj = || py.eval("c()", None, Some(d)).unwrap();
+let obj: &PyAny = create_obj();
+let obj_cell: &PyCell<MyClass> = obj.extract().unwrap();
+let obj_cloned: MyClass = obj.extract().unwrap(); // extracted by cloning the object
+{
+    let obj_ref: PyRef<MyClass> = obj.extract().unwrap();
+    // we need to drop obj_ref before we can extract a PyRefMut due to Rust's rules of references
+}
+let obj_ref_mut: PyRefMut<MyClass> = obj.extract().unwrap();
+```
+
+
+#### `#[pyproto]`
+Most of the arguments to methods in `#[pyproto]` impls require a [`FromPyObject`] implementation.
+So if your protocol methods take `&T` or `&mut T` (where `T: PyClass`),
+please use `PyRef` or `PyRefMut` instead.
+
+Before:
+```compile_fail
+# use pyo3::prelude::*;
+# use pyo3::class::PySequenceProtocol;
+#[pyclass]
+struct ByteSequence {
+    elements: Vec<u8>,
+}
+#[pyproto]
+impl PySequenceProtocol for ByteSequence {
+    fn __concat__(&self, other: &Self) -> PyResult<Self> {
+        let mut elements = self.elements.clone();
+        elements.extend_from_slice(&other.elements);
+        Ok(Self { elements })
+    }
+}
+```
+
+After:
+```
+# use pyo3::prelude::*;
+# use pyo3::class::PySequenceProtocol;
+#[pyclass]
+struct ByteSequence {
+    elements: Vec<u8>,
+}
+#[pyproto]
+impl PySequenceProtocol for ByteSequence {
+    fn __concat__(&self, other: PyRef<'p, Self>) -> PyResult<Self> {
+        let mut elements = self.elements.clone();
+        elements.extend_from_slice(&other.elements);
+        Ok(Self { elements })
+    }
+}
+```

guide/src/rust_cpython.md

@@ -1,4 +1,4 @@
-# Appendix: PyO3 and rust-cpython
+# Appendix A: PyO3 and rust-cpython
 
 PyO3 began as fork of [rust-cpython](https://github.com/dgrunwald/rust-cpython) when rust-cpython wasn't maintained. Over the time PyO3 has become fundamentally different from rust-cpython.
 

pyo3-derive-backend/src/pyclass.rs

@@ -389,9 +389,10 @@ fn impl_class(
         unsafe impl pyo3::type_object::PyTypeInfo for #cls {
             type Type = #cls;
             type BaseType = #base;
-            type Layout = pyo3::pycell::PyCell<Self>;
+            type Layout = pyo3::PyCell<Self>;
             type BaseLayout = #base_layout;
             type Initializer = pyo3::pyclass_init::PyClassInitializer<Self>;
+            type AsRefTarget = pyo3::PyCell<Self>;
 
             const NAME: &'static str = #cls_name;
             const MODULE: Option<&'static str> = #module;
@@ -416,6 +417,7 @@ fn impl_class(
         {
             type Target = pyo3::PyRef<'a, #cls>;
         }
+
         impl<'a> pyo3::derive_utils::ExtractExt<'a> for &'a mut #cls
         {
             type Target = pyo3::PyRefMut<'a, #cls>;

src/instance.rs

@@ -7,7 +7,7 @@ use crate::type_object::{PyBorrowFlagLayout, PyDowncastImpl};
 use crate::types::PyAny;
 use crate::{
     ffi, AsPyPointer, FromPyObject, IntoPy, IntoPyPointer, PyCell, PyClass, PyClassInitializer,
-    PyRef, PyRefMut, Python, ToPyObject,
+    PyRef, PyRefMut, PyTypeInfo, Python, ToPyObject,
 };
 use std::marker::PhantomData;
 use std::mem;
@@ -158,9 +158,12 @@ pub trait AsPyRef: Sized {
     fn as_ref(&self, py: Python<'_>) -> &Self::Target;
 }
 
-impl<'p, T: PyClass> AsPyRef for Py<T> {
-    type Target = PyCell<T>;
-    fn as_ref(&self, _py: Python) -> &PyCell<T> {
+impl<T> AsPyRef for Py<T>
+where
+    T: PyTypeInfo,
+{
+    type Target = T::AsRefTarget;
+    fn as_ref(&self, _py: Python) -> &Self::Target {
         let any = self as *const Py<T> as *const PyAny;
         unsafe { PyDowncastImpl::unchecked_downcast(&*any) }
     }

src/lib.rs

@@ -328,4 +328,5 @@ pub mod doc_test {
     doctest!("../guide/src/parallelism.md", guide_parallelism_md);
     doctest!("../guide/src/pypy.md", guide_pypy_md);
     doctest!("../guide/src/rust_cpython.md", guide_rust_cpython_md);
+    doctest!("../guide/src/migration.md", guide_migration_md);
 }

src/pycell.rs

@@ -440,7 +440,13 @@ pub struct PyRef<'p, T: PyClass> {
     inner: &'p PyCellInner<T>,
 }
 
-unsafe impl<'p, T: PyClass> crate::PyNativeType for PyRef<'p, T> {}
+impl<'p, T: PyClass> PyRef<'p, T> {
+    /// Returns `Python` token.
+    /// This function is safe since PyRef has the same lifetime as a `GILGuard`.
+    pub fn py(&self) -> Python {
+        unsafe { Python::assume_gil_acquired() }
+    }
+}
 
 impl<'p, T, U> AsRef<U> for PyRef<'p, T>
 where
@@ -551,7 +557,13 @@ pub struct PyRefMut<'p, T: PyClass> {
     inner: &'p PyCellInner<T>,
 }
 
-unsafe impl<'p, T: PyClass> crate::PyNativeType for PyRefMut<'p, T> {}
+impl<'p, T: PyClass> PyRefMut<'p, T> {
+    /// Returns `Python` token.
+    /// This function is safe since PyRefMut has the same lifetime as a `GILGuard`.
+    pub fn py(&self) -> Python {
+        unsafe { Python::assume_gil_acquired() }
+    }
+}
 
 impl<'p, T, U> AsRef<U> for PyRefMut<'p, T>
 where

src/type_object.rs

@@ -123,6 +123,9 @@ pub unsafe trait PyTypeInfo: Sized {
     /// Initializer for layout
     type Initializer: PyObjectInit<Self>;
 
+    /// Utility type to make AsPyRef work
+    type AsRefTarget: PyDowncastImpl;
+
     /// PyTypeObject instance for this type.
     fn type_object() -> &'static ffi::PyTypeObject;
 

src/types/mod.rs

@@ -128,6 +128,7 @@ macro_rules! pyobject_native_type_convert(
             type Layout = $layout;
             type BaseLayout = ffi::PyObject;
             type Initializer = $crate::pyclass_init::PyNativeTypeInitializer<Self>;
+            type AsRefTarget = Self;
 
             const NAME: &'static str = stringify!($name);
             const MODULE: Option<&'static str> = $module;
@@ -153,14 +154,6 @@ macro_rules! pyobject_native_type_convert(
             }
         }
 
-        impl $crate::AsPyRef for $crate::Py<$name> {
-            type Target = $name;
-            fn as_ref(&self, _py: $crate::Python) -> &$name {
-                let any = self as *const $crate::Py<$name> as *const $crate::types::PyAny;
-                unsafe { $crate::type_object::PyDowncastImpl::unchecked_downcast(&*any) }
-            }
-        }
-
         impl<$($type_param,)*> ::std::fmt::Debug for $name {
             fn fmt(&self, f: &mut ::std::fmt::Formatter)
                    -> Result<(), ::std::fmt::Error>