Make PyClassShell have dict&weakref

pyo3-derive-backend/src/pyclass.rs

@@ -126,10 +126,10 @@ impl PyClassArgs {
         let flag = exp.path.segments.first().unwrap().ident.to_string();
         let path = match flag.as_str() {
             "gc" => {
-                parse_quote! {pyo3::type_object::PY_TYPE_FLAG_GC}
+                parse_quote! {pyo3::type_flags::GC}
             }
             "weakref" => {
-                parse_quote! {pyo3::type_object::PY_TYPE_FLAG_WEAKREF}
+                parse_quote! {pyo3::type_flags::WEAKREF}
             }
             "subclass" => {
                 if cfg!(not(feature = "unsound-subclass")) {
@@ -138,10 +138,10 @@ impl PyClassArgs {
                         "You need to activate the `unsound-subclass` feature if you want to use subclassing",
                     ));
                 }
-                parse_quote! {pyo3::type_object::PY_TYPE_FLAG_BASETYPE}
+                parse_quote! {pyo3::type_flags::BASETYPE}
             }
             "dict" => {
-                parse_quote! {pyo3::type_object::PY_TYPE_FLAG_DICT}
+                parse_quote! {pyo3::type_flags::DICT}
             }
             _ => {
                 return Err(syn::Error::new_spanned(
@@ -267,7 +267,7 @@ fn impl_class(
     let extra = {
         if let Some(freelist) = &attr.freelist {
             quote! {
-                impl pyo3::freelist::PyObjectWithFreeList for #cls {
+                impl pyo3::freelist::PyClassWithFreeList for #cls {
                     #[inline]
                     fn get_free_list() -> &'static mut pyo3::freelist::FreeList<*mut pyo3::ffi::PyObject> {
                         static mut FREELIST: *mut pyo3::freelist::FreeList<*mut pyo3::ffi::PyObject> = 0 as *mut _;
@@ -285,7 +285,7 @@ fn impl_class(
             }
         } else {
             quote! {
-                impl pyo3::type_object::PyObjectAlloc for #cls {}
+                impl pyo3::pyclass::PyClassAlloc for #cls {}
             }
         }
     };
@@ -308,24 +308,25 @@ fn impl_class(
     let mut has_gc = false;
     for f in attr.flags.iter() {
         if let syn::Expr::Path(ref epath) = f {
-            if epath.path == parse_quote! {pyo3::type_object::PY_TYPE_FLAG_WEAKREF} {
+            if epath.path == parse_quote! { pyo3::type_flags::WEAKREF } {
                 has_weakref = true;
-            } else if epath.path == parse_quote! {pyo3::type_object::PY_TYPE_FLAG_DICT} {
+            } else if epath.path == parse_quote! { pyo3::type_flags::DICT } {
                 has_dict = true;
-            } else if epath.path == parse_quote! {pyo3::type_object::PY_TYPE_FLAG_GC} {
+            } else if epath.path == parse_quote! { pyo3::type_flags::GC } {
                 has_gc = true;
             }
         }
     }
+    // TODO: implement dict and weakref
     let weakref = if has_weakref {
-        quote! {std::mem::size_of::<*const pyo3::ffi::PyObject>()}
+        quote! { type WeakRef = pyo3::pyclass_slots::PyClassWeakRefSlot; }
     } else {
-        quote! {0}
+        quote! { type WeakRef = pyo3::pyclass_slots::PyClassDummySlot; }
     };
     let dict = if has_dict {
-        quote! {std::mem::size_of::<*const pyo3::ffi::PyObject>()}
+        quote! { type Dict = pyo3::pyclass_slots::PyClassDictSlot; }
     } else {
-        quote! {0}
+        quote! { type Dict = pyo3::pyclass_slots::PyClassDummySlot; }
     };
     let module = if let Some(m) = &attr.module {
         quote! { Some(#m) }
@@ -358,25 +359,13 @@ fn impl_class(
         impl pyo3::type_object::PyTypeInfo for #cls {
             type Type = #cls;
             type BaseType = #base;
+            type ConcreteLayout = pyo3::pyclass::PyClassShell<Self>;
 
             const NAME: &'static str = #cls_name;
             const MODULE: Option<&'static str> = #module;
             const DESCRIPTION: &'static str = #doc;
             const FLAGS: usize = #(#flags)|*;
 
-            const SIZE: usize = {
-                Self::OFFSET as usize +
-                ::std::mem::size_of::<#cls>() + #weakref + #dict
-            };
-            const OFFSET: isize = {
-                // round base_size up to next multiple of align
-                (
-                    (<#base as pyo3::type_object::PyTypeInfo>::SIZE +
-                     ::std::mem::align_of::<#cls>() - 1)  /
-                        ::std::mem::align_of::<#cls>() * ::std::mem::align_of::<#cls>()
-                ) as isize
-            };
-
             #[inline]
             unsafe fn type_object() -> &'static mut pyo3::ffi::PyTypeObject {
                 static mut TYPE_OBJECT: pyo3::ffi::PyTypeObject = pyo3::ffi::PyTypeObject_INIT;
@@ -384,6 +373,11 @@ fn impl_class(
             }
         }
 
+        impl pyo3::PyClass for #cls {
+            #dict
+            #weakref
+        }
+
         impl pyo3::IntoPy<PyObject> for #cls {
             fn into_py(self, py: pyo3::Python) -> pyo3::PyObject {
                 pyo3::IntoPy::into_py(pyo3::Py::new(py, self).unwrap(), py)

pyo3-derive-backend/src/pymethod.rs

@@ -237,25 +237,14 @@ pub fn impl_wrap_new(cls: &syn::Type, name: &syn::Ident, spec: &FnSpec<'_>) -> T
             const _LOCATION: &'static str = concat!(stringify!(#cls),".",stringify!(#name),"()");
             let _py = pyo3::Python::assume_gil_acquired();
             let _pool = pyo3::GILPool::new(_py);
-            match pyo3::type_object::PyRawObject::new(_py, #cls::type_object(), _cls) {
+            let _args = _py.from_borrowed_ptr::<pyo3::types::PyTuple>(_args);
-                Ok(_obj) => {
+            let _kwargs: Option<&pyo3::types::PyDict> = _py.from_borrowed_ptr_or_opt(_kwargs);
-                    let _args = _py.from_borrowed_ptr::<pyo3::types::PyTuple>(_args);
-                    let _kwargs: Option<&pyo3::types::PyDict> = _py.from_borrowed_ptr_or_opt(_kwargs);
 
-                    #body
+            #body
 
-                    match _result {
+            match <<#cls as pyo3::PyTypeInfo>::ConcreteLayout as pyo3::pyclass::PyClassNew>::new(_py, _result) {
-                        Ok(_) => pyo3::IntoPyPointer::into_ptr(_obj),
+                Ok(_slf) => _slf as _,
-                        Err(e) => {
+                Err(e) => e.restore_and_null(),
-                            e.restore(_py);
-                            ::std::ptr::null_mut()
-                        }
-                    }
-                }
-                Err(e) => {
-                    e.restore(_py);
-                    ::std::ptr::null_mut()
-                }
             }
         }
     }

src/class/iter.rs

@@ -4,7 +4,7 @@
 
 use crate::callback::{CallbackConverter, PyObjectCallbackConverter};
 use crate::err::PyResult;
-use crate::{ffi, IntoPy, PyClass, PyClassShell, PyObject};
+use crate::{ffi, pyclass::PyClassShell, IntoPy, PyClass, PyObject};
 use crate::{IntoPyPointer, Python};
 use std::ptr;
 

src/class/macros.rs

@@ -35,7 +35,7 @@ macro_rules! py_unary_pyref_func {
         where
             T: for<'p> $trait<'p>,
         {
-            use $crate::{FromPyPointer, PyClassShell};
+            use $crate::{pyclass::PyClassShell, FromPyPointer};
             let py = $crate::Python::assume_gil_acquired();
             let _pool = $crate::GILPool::new(py);
             let slf: &mut PyClassShell<T> = FromPyPointer::from_borrowed_ptr_or_panic(py, slf);

src/err.rs

@@ -326,6 +326,13 @@ impl PyErr {
         unsafe { ffi::PyErr_Restore(ptype.into_ptr(), pvalue, ptraceback.into_ptr()) }
     }
 
+    #[doc(hidden)]
+    /// Utility method for proc-macro code
+    pub fn restore_and_null<T>(self, py: Python) -> *mut T {
+        self.restore(py);
+        std::ptr::null_mut()
+    }
+
     /// Issue a warning message.
     /// May return a PyErr if warnings-as-errors is enabled.
     pub fn warn(py: Python, category: &PyAny, message: &str, stacklevel: i32) -> PyResult<()> {

src/freelist.rs

@@ -12,7 +12,7 @@ use std::os::raw::c_void;
 /// Implementing this trait for custom class adds free allocation list to class.
 /// The performance improvement applies to types that are often created and deleted in a row,
 /// so that they can benefit from a freelist.
-pub trait PyObjectWithFreeList {
+pub trait PyClassWithFreeList {
     fn get_free_list() -> &'static mut FreeList<*mut ffi::PyObject>;
 }
 
@@ -70,10 +70,10 @@ impl<T> FreeList<T> {
 
 impl<T> PyClassAlloc for T
 where
-    T: PyTypeInfo + PyObjectWithFreeList,
+    T: PyTypeInfo + PyClassWithFreeList,
 {
     unsafe fn alloc(_py: Python) -> *mut Self::ConcreteLayout {
-        if let Some(obj) = <Self as PyObjectWithFreeList>::get_free_list().pop() {
+        if let Some(obj) = <Self as PyClassWithFreeList>::get_free_list().pop() {
             ffi::PyObject_Init(obj, <Self as PyTypeInfo>::type_object());
             obj as _
         } else {
@@ -89,7 +89,7 @@ where
             return;
         }
 
-        if let Some(obj) = <Self as PyObjectWithFreeList>::get_free_list().insert(obj) {
+        if let Some(obj) = <Self as PyClassWithFreeList>::get_free_list().insert(obj) {
             match Self::type_object().tp_free {
                 Some(free) => free(obj as *mut c_void),
                 None => tp_free_fallback(obj),

src/instance.rs

@@ -36,11 +36,16 @@ unsafe impl<T> Sync for Py<T> {}
 
 impl<T> Py<T> {
     /// Create new instance of T and move it under python management
+    ///
+    /// **NOTE**
+    /// This method's `where` bound is actually the same as `PyClass`.
+    /// However, since Rust still doesn't have higher order generics, we cannot represent
+    /// this bound by `PyClass`.
     pub fn new(py: Python, value: T) -> PyResult<Py<T>>
     where
         T: PyClass,
     {
-        let obj = unsafe { PyClassShell::<T>::new(py, value) };
+        let obj = unsafe { PyClassShell::new(py, value)? };
         let ob = unsafe { Py::from_owned_ptr(obj as _) };
         Ok(ob)
     }

src/internal_tricks.rs

@@ -4,3 +4,22 @@ use std::rc::Rc;
 /// A marker type that makes the type !Send.
 /// Temporal hack until https://github.com/rust-lang/rust/issues/13231 is resolved.
 pub(crate) type Unsendable = PhantomData<Rc<()>>;
+
+pub struct PrivateMarker;
+
+macro_rules! private_decl {
+    () => {
+        /// This trait is private to implement; this method exists to make it
+        /// impossible to implement outside the crate.
+        fn __private__(&self) -> crate::internal_tricks::PrivateMarker;
+    }
+}
+
+macro_rules! private_impl {
+    () => {
+        #[doc(hidden)]
+        fn __private__(&self) -> crate::internal_tricks::PrivateMarker {
+            crate::internal_tricks::PrivateMarker
+        }
+    }
+}

src/lib.rs

@@ -124,12 +124,12 @@ pub use crate::conversion::{
 };
 pub use crate::err::{PyDowncastError, PyErr, PyErrArguments, PyErrValue, PyResult};
 pub use crate::gil::{init_once, GILGuard, GILPool};
-pub use crate::instance::{ManagedPyRef, Py, PyNativeType};
+pub use crate::instance::{AsPyRef, ManagedPyRef, Py, PyNativeType};
 pub use crate::object::PyObject;
 pub use crate::objectprotocol::ObjectProtocol;
-pub use crate::pyclass::{PyClass, PyClassAlloc, PyClassShell};
+pub use crate::pyclass::{PyClass, PyClassShell};
 pub use crate::python::{prepare_freethreaded_python, Python};
-pub use crate::type_object::{PyConcreteObject, PyTypeInfo};
+pub use crate::type_object::{type_flags, PyConcreteObject, PyTypeInfo};
 
 // Re-exported for wrap_function
 #[doc(hidden)]
@@ -161,12 +161,14 @@ pub mod ffi;
 pub mod freelist;
 mod gil;
 mod instance;
+#[macro_use]
 mod internal_tricks;
 pub mod marshal;
 mod object;
 mod objectprotocol;
 pub mod prelude;
 pub mod pyclass;
+pub mod pyclass_slots;
 mod python;
 pub mod type_object;
 pub mod types;

src/object.rs

@@ -7,7 +7,6 @@ use crate::instance::{AsPyRef, PyNativeType};
 use crate::types::{PyAny, PyDict, PyTuple};
 use crate::{AsPyPointer, Py, Python};
 use crate::{FromPyObject, IntoPy, IntoPyPointer, PyTryFrom, ToBorrowedObject, ToPyObject};
-use std::convert::AsRef;
 use std::ptr::NonNull;
 
 /// A python object

src/pyclass.rs

@@ -1,8 +1,9 @@
 //! An experiment module which has all codes related only to #[pyclass]
 use crate::class::methods::{PyMethodDefType, PyMethodsProtocol};
-use crate::conversion::FromPyPointer;
+use crate::conversion::{AsPyPointer, FromPyPointer, ToPyObject};
-use crate::type_object::{PyConcreteObject, PyTypeObject};
+use crate::pyclass_slots::{PyClassDict, PyClassWeakRef};
-use crate::{class, ffi, gil, PyErr, PyResult, PyTypeInfo, Python};
+use crate::type_object::{type_flags, PyConcreteObject, PyTypeObject};
+use crate::{class, ffi, gil, PyErr, PyObject, PyResult, PyTypeInfo, Python};
 use std::ffi::CString;
 use std::mem::ManuallyDrop;
 use std::os::raw::c_void;
@@ -46,7 +47,10 @@ pub unsafe fn tp_free_fallback(obj: *mut ffi::PyObject) {
     }
 }
 
-pub trait PyClass: PyTypeInfo + Sized + PyClassAlloc + PyMethodsProtocol {}
+pub trait PyClass: PyTypeInfo + Sized + PyClassAlloc + PyMethodsProtocol {
+    type Dict: PyClassDict;
+    type WeakRef: PyClassWeakRef;
+}
 
 unsafe impl<T> PyTypeObject for T
 where
@@ -75,21 +79,44 @@ where
 pub struct PyClassShell<T: PyClass> {
     ob_base: <T::BaseType as PyTypeInfo>::ConcreteLayout,
     pyclass: ManuallyDrop<T>,
+    dict: T::Dict,
+    weakref: T::WeakRef,
 }
 
 impl<T: PyClass> PyClassShell<T> {
-    pub unsafe fn new(py: Python, value: T) -> *mut Self {
+    pub fn new_ref(py: Python, value: T) -> PyResult<&Self> {
+        unsafe {
+            let ptr = Self::new(py, value)?;
+            FromPyPointer::from_owned_ptr_or_err(py, ptr as _)
+        }
+    }
+
+    pub fn new_mut(py: Python, value: T) -> PyResult<&mut Self> {
+        unsafe {
+            let ptr = Self::new(py, value)?;
+            FromPyPointer::from_owned_ptr_or_err(py, ptr as _)
+        }
+    }
+
+    pub unsafe fn new(py: Python, value: T) -> PyResult<*mut Self> {
         T::init_type();
         let base = T::alloc(py);
+        if base.is_null() {
+            return Err(PyErr::fetch(py));
+        }
         let self_ = base as *mut Self;
         (*self_).pyclass = ManuallyDrop::new(value);
-        self_
+        (*self_).dict = T::Dict::new();
+        (*self_).weakref = T::WeakRef::new();
+        Ok(self_)
     }
 }
 
 impl<T: PyClass> PyConcreteObject for PyClassShell<T> {
     unsafe fn py_drop(&mut self, py: Python) {
         ManuallyDrop::drop(&mut self.pyclass);
+        self.dict.clear_dict(py);
+        self.weakref.clear_weakrefs(self.as_ptr(), py);
         self.ob_base.py_drop(py);
     }
 }
@@ -107,6 +134,12 @@ impl<T: PyClass> std::ops::DerefMut for PyClassShell<T> {
     }
 }
 
+impl<T: PyClass> ToPyObject for PyClassShell<T> {
+    fn to_object(&self, py: Python<'_>) -> PyObject {
+        unsafe { PyObject::from_borrowed_ptr(py, self.as_ptr()) }
+    }
+}
+
 unsafe impl<'p, T> FromPyPointer<'p> for &'p PyClassShell<T>
 where
     T: PyClass,
@@ -131,18 +164,6 @@ where
     }
 }
 
-/// type object supports python GC
-const PY_TYPE_FLAG_GC: usize = 1;
-
-/// Type object supports python weak references
-const PY_TYPE_FLAG_WEAKREF: usize = 1 << 1;
-
-/// Type object can be used as the base type of another type
-const PY_TYPE_FLAG_BASETYPE: usize = 1 << 2;
-
-/// The instances of this type have a dictionary containing instance variables
-const PY_TYPE_FLAG_DICT: usize = 1 << 3;
-
 /// Register new type in python object system.
 #[cfg(not(Py_LIMITED_API))]
 pub fn initialize_type<T>(py: Python, module_name: Option<&str>) -> PyResult<*mut ffi::PyTypeObject>
@@ -182,19 +203,20 @@ where
     type_object.tp_dealloc = Some(tp_dealloc_callback::<T>);
 
     // type size
-    type_object.tp_basicsize = std::mem::size_of::<T::ConcreteLayout>() as isize;
+    type_object.tp_basicsize = std::mem::size_of::<T::ConcreteLayout>() as ffi::Py_ssize_t;
 
-    // weakref support (check py3cls::py_class::impl_class)
+    let mut offset = type_object.tp_basicsize;
-    if T::FLAGS & PY_TYPE_FLAG_WEAKREF != 0 {
-        // STUB
-        type_object.tp_weaklistoffset = 0isize;
-    }
 
     // __dict__ support
-    let has_dict = T::FLAGS & PY_TYPE_FLAG_DICT != 0;
+    if let Some(dict_offset) = T::Dict::OFFSET {
-    if has_dict {
+        offset += dict_offset as ffi::Py_ssize_t;
-        // STUB
+        type_object.tp_dictoffset = offset;
-        type_object.tp_dictoffset = 0isize;
+    }
+
+    // weakref support
+    if let Some(weakref_offset) = T::WeakRef::OFFSET {
+        offset += weakref_offset as ffi::Py_ssize_t;
+        type_object.tp_weaklistoffset = offset;
     }
 
     // GC support
@@ -243,7 +265,7 @@ where
     // properties
     let mut props = py_class_properties::<T>();
 
-    if has_dict {
+    if T::Dict::OFFSET.is_some() {
         props.push(ffi::PyGetSetDef_DICT);
     }
     if !props.is_empty() {
@@ -267,13 +289,13 @@ where
 fn py_class_flags<T: PyTypeInfo>(type_object: &mut ffi::PyTypeObject) {
     if type_object.tp_traverse != None
         || type_object.tp_clear != None
-        || T::FLAGS & PY_TYPE_FLAG_GC != 0
+        || T::FLAGS & type_flags::GC != 0
     {
         type_object.tp_flags = ffi::Py_TPFLAGS_DEFAULT | ffi::Py_TPFLAGS_HAVE_GC;
     } else {
         type_object.tp_flags = ffi::Py_TPFLAGS_DEFAULT;
     }
-    if T::FLAGS & PY_TYPE_FLAG_BASETYPE != 0 {
+    if T::FLAGS & type_flags::BASETYPE != 0 {
         type_object.tp_flags |= ffi::Py_TPFLAGS_BASETYPE;
     }
 }

src/pyclass_slots.rs

@@ -0,0 +1,72 @@
+//! This module contains additional fields pf pyclass
+// TODO(kngwyu): Add vectorcall support
+use crate::{ffi, Python};
+
+const POINTER_SIZE: isize = std::mem::size_of::<*mut ffi::PyObject>() as _;
+
+/// Represents `__dict__`.
+pub trait PyClassDict {
+    const OFFSET: Option<isize> = None;
+    fn new() -> Self;
+    fn clear_dict(&mut self, _py: Python) {}
+    private_decl! {}
+}
+
+/// Represents `__weakref__`.
+pub trait PyClassWeakRef {
+    const OFFSET: Option<isize> = None;
+    fn new() -> Self;
+    fn clear_weakrefs(&mut self, _obj: *mut ffi::PyObject, _py: Python) {}
+    private_decl! {}
+}
+
+/// Dummy slot means the function doesn't has such a feature.
+pub struct PyClassDummySlot;
+
+impl PyClassDict for PyClassDummySlot {
+    private_impl! {}
+    fn new() -> Self {
+        PyClassDummySlot
+    }
+}
+
+impl PyClassWeakRef for PyClassDummySlot {
+    private_impl! {}
+    fn new() -> Self {
+        PyClassDummySlot
+    }
+}
+
+/// actural dict field
+#[repr(transparent)]
+pub struct PyClassDictSlot(*mut ffi::PyObject);
+
+impl PyClassDict for PyClassDictSlot {
+    private_impl! {}
+    const OFFSET: Option<isize> = Some(-POINTER_SIZE);
+    fn new() -> Self {
+        Self(std::ptr::null_mut())
+    }
+    fn clear_dict(&mut self, _py: Python) {
+        if self.0 != std::ptr::null_mut() {
+            unsafe { ffi::PyDict_Clear(self.0) }
+        }
+    }
+}
+
+/// actural weakref field
+#[repr(transparent)]
+pub struct PyClassWeakRefSlot(*mut ffi::PyObject);
+
+impl PyClassWeakRef for PyClassWeakRefSlot {
+    private_impl! {}
+    const OFFSET: Option<isize> = Some(-POINTER_SIZE);
+    fn new() -> Self {
+        Self(std::ptr::null_mut())
+    }
+    fn clear_weakrefs(&mut self, obj: *mut ffi::PyObject, _py: Python) {
+        if self.0 != std::ptr::null_mut() {
+            unsafe { ffi::PyObject_ClearWeakRefs(obj) }
+        }
+    }
+}

src/type_object.rs

@@ -10,6 +10,7 @@ use crate::AsPyPointer;
 use crate::Python;
 use std::ptr::NonNull;
 
+/// TODO: write document
 pub trait PyConcreteObject: Sized {
     unsafe fn py_drop(&mut self, _py: Python) {}
 }
@@ -22,6 +23,21 @@ impl<T: PyConcreteObject> AsPyPointer for T {
 
 impl PyConcreteObject for ffi::PyObject {}
 
+/// Our custom type flags
+pub mod type_flags {
+    /// type object supports python GC
+    pub const GC: usize = 1;
+
+    /// Type object supports python weak references
+    pub const WEAKREF: usize = 1 << 1;
+
+    /// Type object can be used as the base type of another type
+    pub const BASETYPE: usize = 1 << 2;
+
+    /// The instances of this type have a dictionary containing instance variables
+    pub const DICT: usize = 1 << 3;
+}
+
 /// Python type information.
 pub trait PyTypeInfo {
     /// Type of objects to store in PyObject struct
@@ -36,9 +52,6 @@ pub trait PyTypeInfo {
     /// Class doc string
     const DESCRIPTION: &'static str = "\0";
 
-    /// `Type` instance offset inside PyObject structure
-    const OFFSET: isize;
-
     /// Type flags (ie PY_TYPE_FLAG_GC, PY_TYPE_FLAG_WEAKREF)
     const FLAGS: usize = 0;
 

src/types/any.rs

@@ -1,4 +1,3 @@
-use crate::conversion::AsPyPointer;
 use crate::conversion::PyTryFrom;
 use crate::err::PyDowncastError;
 use crate::internal_tricks::Unsendable;

src/types/mod.rs

@@ -108,7 +108,6 @@ macro_rules! pyobject_native_type_convert(
 
             const NAME: &'static str = stringify!($name);
             const MODULE: Option<&'static str> = $module;
-            const OFFSET: isize = 0;
 
             #[inline]
             unsafe fn type_object() -> &'static mut $crate::ffi::PyTypeObject {