From bdb66afe0a8ca5d212449a9f0a87bbd00d3fccfe Mon Sep 17 00:00:00 2001 From: kngwyu Date: Sun, 8 Dec 2019 17:18:25 +0900 Subject: [PATCH] Make PyClassShell have dict&weakref --- pyo3-derive-backend/src/pyclass.rs | 46 +++++++--------- pyo3-derive-backend/src/pymethod.rs | 23 +++----- src/class/iter.rs | 2 +- src/class/macros.rs | 2 +- src/err.rs | 7 +++ src/freelist.rs | 8 +-- src/instance.rs | 7 ++- src/internal_tricks.rs | 19 +++++++ src/lib.rs | 8 +-- src/object.rs | 1 - src/pyclass.rs | 84 ++++++++++++++++++----------- src/pyclass_slots.rs | 72 +++++++++++++++++++++++++ src/type_object.rs | 19 +++++-- src/types/any.rs | 1 - src/types/mod.rs | 1 - 15 files changed, 210 insertions(+), 90 deletions(-) create mode 100644 src/pyclass_slots.rs diff --git a/pyo3-derive-backend/src/pyclass.rs b/pyo3-derive-backend/src/pyclass.rs index b4a8d6fe..76336aef 100644 --- a/pyo3-derive-backend/src/pyclass.rs +++ b/pyo3-derive-backend/src/pyclass.rs @@ -126,10 +126,10 @@ impl PyClassArgs { let flag = exp.path.segments.first().unwrap().ident.to_string(); let path = match flag.as_str() { "gc" => { - parse_quote! {pyo3::type_object::PY_TYPE_FLAG_GC} + parse_quote! {pyo3::type_flags::GC} } "weakref" => { - parse_quote! {pyo3::type_object::PY_TYPE_FLAG_WEAKREF} + parse_quote! {pyo3::type_flags::WEAKREF} } "subclass" => { if cfg!(not(feature = "unsound-subclass")) { @@ -138,10 +138,10 @@ impl PyClassArgs { "You need to activate the `unsound-subclass` feature if you want to use subclassing", )); } - parse_quote! {pyo3::type_object::PY_TYPE_FLAG_BASETYPE} + parse_quote! {pyo3::type_flags::BASETYPE} } "dict" => { - parse_quote! {pyo3::type_object::PY_TYPE_FLAG_DICT} + parse_quote! {pyo3::type_flags::DICT} } _ => { return Err(syn::Error::new_spanned( @@ -267,7 +267,7 @@ fn impl_class( let extra = { if let Some(freelist) = &attr.freelist { quote! { - impl pyo3::freelist::PyObjectWithFreeList for #cls { + impl pyo3::freelist::PyClassWithFreeList for #cls { #[inline] fn get_free_list() -> &'static mut pyo3::freelist::FreeList<*mut pyo3::ffi::PyObject> { static mut FREELIST: *mut pyo3::freelist::FreeList<*mut pyo3::ffi::PyObject> = 0 as *mut _; @@ -285,7 +285,7 @@ fn impl_class( } } else { quote! { - impl pyo3::type_object::PyObjectAlloc for #cls {} + impl pyo3::pyclass::PyClassAlloc for #cls {} } } }; @@ -308,24 +308,25 @@ fn impl_class( let mut has_gc = false; for f in attr.flags.iter() { if let syn::Expr::Path(ref epath) = f { - if epath.path == parse_quote! {pyo3::type_object::PY_TYPE_FLAG_WEAKREF} { + if epath.path == parse_quote! { pyo3::type_flags::WEAKREF } { has_weakref = true; - } else if epath.path == parse_quote! {pyo3::type_object::PY_TYPE_FLAG_DICT} { + } else if epath.path == parse_quote! { pyo3::type_flags::DICT } { has_dict = true; - } else if epath.path == parse_quote! {pyo3::type_object::PY_TYPE_FLAG_GC} { + } else if epath.path == parse_quote! { pyo3::type_flags::GC } { has_gc = true; } } } + // TODO: implement dict and weakref let weakref = if has_weakref { - quote! {std::mem::size_of::<*const pyo3::ffi::PyObject>()} + quote! { type WeakRef = pyo3::pyclass_slots::PyClassWeakRefSlot; } } else { - quote! {0} + quote! { type WeakRef = pyo3::pyclass_slots::PyClassDummySlot; } }; let dict = if has_dict { - quote! {std::mem::size_of::<*const pyo3::ffi::PyObject>()} + quote! { type Dict = pyo3::pyclass_slots::PyClassDictSlot; } } else { - quote! {0} + quote! { type Dict = pyo3::pyclass_slots::PyClassDummySlot; } }; let module = if let Some(m) = &attr.module { quote! { Some(#m) } @@ -358,25 +359,13 @@ fn impl_class( impl pyo3::type_object::PyTypeInfo for #cls { type Type = #cls; type BaseType = #base; + type ConcreteLayout = pyo3::pyclass::PyClassShell; const NAME: &'static str = #cls_name; const MODULE: Option<&'static str> = #module; const DESCRIPTION: &'static str = #doc; const FLAGS: usize = #(#flags)|*; - const SIZE: usize = { - Self::OFFSET as usize + - ::std::mem::size_of::<#cls>() + #weakref + #dict - }; - const OFFSET: isize = { - // round base_size up to next multiple of align - ( - (<#base as pyo3::type_object::PyTypeInfo>::SIZE + - ::std::mem::align_of::<#cls>() - 1) / - ::std::mem::align_of::<#cls>() * ::std::mem::align_of::<#cls>() - ) as isize - }; - #[inline] unsafe fn type_object() -> &'static mut pyo3::ffi::PyTypeObject { static mut TYPE_OBJECT: pyo3::ffi::PyTypeObject = pyo3::ffi::PyTypeObject_INIT; @@ -384,6 +373,11 @@ fn impl_class( } } + impl pyo3::PyClass for #cls { + #dict + #weakref + } + impl pyo3::IntoPy for #cls { fn into_py(self, py: pyo3::Python) -> pyo3::PyObject { pyo3::IntoPy::into_py(pyo3::Py::new(py, self).unwrap(), py) diff --git a/pyo3-derive-backend/src/pymethod.rs b/pyo3-derive-backend/src/pymethod.rs index ba32667c..4d01f99c 100644 --- a/pyo3-derive-backend/src/pymethod.rs +++ b/pyo3-derive-backend/src/pymethod.rs @@ -237,25 +237,14 @@ pub fn impl_wrap_new(cls: &syn::Type, name: &syn::Ident, spec: &FnSpec<'_>) -> T const _LOCATION: &'static str = concat!(stringify!(#cls),".",stringify!(#name),"()"); let _py = pyo3::Python::assume_gil_acquired(); let _pool = pyo3::GILPool::new(_py); - match pyo3::type_object::PyRawObject::new(_py, #cls::type_object(), _cls) { - Ok(_obj) => { - let _args = _py.from_borrowed_ptr::(_args); - let _kwargs: Option<&pyo3::types::PyDict> = _py.from_borrowed_ptr_or_opt(_kwargs); + let _args = _py.from_borrowed_ptr::(_args); + let _kwargs: Option<&pyo3::types::PyDict> = _py.from_borrowed_ptr_or_opt(_kwargs); - #body + #body - match _result { - Ok(_) => pyo3::IntoPyPointer::into_ptr(_obj), - Err(e) => { - e.restore(_py); - ::std::ptr::null_mut() - } - } - } - Err(e) => { - e.restore(_py); - ::std::ptr::null_mut() - } + match <<#cls as pyo3::PyTypeInfo>::ConcreteLayout as pyo3::pyclass::PyClassNew>::new(_py, _result) { + Ok(_slf) => _slf as _, + Err(e) => e.restore_and_null(), } } } diff --git a/src/class/iter.rs b/src/class/iter.rs index 92fe028b..802bd159 100644 --- a/src/class/iter.rs +++ b/src/class/iter.rs @@ -4,7 +4,7 @@ use crate::callback::{CallbackConverter, PyObjectCallbackConverter}; use crate::err::PyResult; -use crate::{ffi, IntoPy, PyClass, PyClassShell, PyObject}; +use crate::{ffi, pyclass::PyClassShell, IntoPy, PyClass, PyObject}; use crate::{IntoPyPointer, Python}; use std::ptr; diff --git a/src/class/macros.rs b/src/class/macros.rs index f188d177..fc86b62f 100644 --- a/src/class/macros.rs +++ b/src/class/macros.rs @@ -35,7 +35,7 @@ macro_rules! py_unary_pyref_func { where T: for<'p> $trait<'p>, { - use $crate::{FromPyPointer, PyClassShell}; + use $crate::{pyclass::PyClassShell, FromPyPointer}; let py = $crate::Python::assume_gil_acquired(); let _pool = $crate::GILPool::new(py); let slf: &mut PyClassShell = FromPyPointer::from_borrowed_ptr_or_panic(py, slf); diff --git a/src/err.rs b/src/err.rs index 3f653e36..3868ebd8 100644 --- a/src/err.rs +++ b/src/err.rs @@ -326,6 +326,13 @@ impl PyErr { unsafe { ffi::PyErr_Restore(ptype.into_ptr(), pvalue, ptraceback.into_ptr()) } } + #[doc(hidden)] + /// Utility method for proc-macro code + pub fn restore_and_null(self, py: Python) -> *mut T { + self.restore(py); + std::ptr::null_mut() + } + /// Issue a warning message. /// May return a PyErr if warnings-as-errors is enabled. pub fn warn(py: Python, category: &PyAny, message: &str, stacklevel: i32) -> PyResult<()> { diff --git a/src/freelist.rs b/src/freelist.rs index 6fb0c287..11308802 100644 --- a/src/freelist.rs +++ b/src/freelist.rs @@ -12,7 +12,7 @@ use std::os::raw::c_void; /// Implementing this trait for custom class adds free allocation list to class. /// The performance improvement applies to types that are often created and deleted in a row, /// so that they can benefit from a freelist. -pub trait PyObjectWithFreeList { +pub trait PyClassWithFreeList { fn get_free_list() -> &'static mut FreeList<*mut ffi::PyObject>; } @@ -70,10 +70,10 @@ impl FreeList { impl PyClassAlloc for T where - T: PyTypeInfo + PyObjectWithFreeList, + T: PyTypeInfo + PyClassWithFreeList, { unsafe fn alloc(_py: Python) -> *mut Self::ConcreteLayout { - if let Some(obj) = ::get_free_list().pop() { + if let Some(obj) = ::get_free_list().pop() { ffi::PyObject_Init(obj, ::type_object()); obj as _ } else { @@ -89,7 +89,7 @@ where return; } - if let Some(obj) = ::get_free_list().insert(obj) { + if let Some(obj) = ::get_free_list().insert(obj) { match Self::type_object().tp_free { Some(free) => free(obj as *mut c_void), None => tp_free_fallback(obj), diff --git a/src/instance.rs b/src/instance.rs index 5b7b436f..4fd52a4c 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -36,11 +36,16 @@ unsafe impl Sync for Py {} impl Py { /// Create new instance of T and move it under python management + /// + /// **NOTE** + /// This method's `where` bound is actually the same as `PyClass`. + /// However, since Rust still doesn't have higher order generics, we cannot represent + /// this bound by `PyClass`. pub fn new(py: Python, value: T) -> PyResult> where T: PyClass, { - let obj = unsafe { PyClassShell::::new(py, value) }; + let obj = unsafe { PyClassShell::new(py, value)? }; let ob = unsafe { Py::from_owned_ptr(obj as _) }; Ok(ob) } diff --git a/src/internal_tricks.rs b/src/internal_tricks.rs index 1cf8a56a..1d6a5eb9 100644 --- a/src/internal_tricks.rs +++ b/src/internal_tricks.rs @@ -4,3 +4,22 @@ use std::rc::Rc; /// A marker type that makes the type !Send. /// Temporal hack until https://github.com/rust-lang/rust/issues/13231 is resolved. pub(crate) type Unsendable = PhantomData>; + +pub struct PrivateMarker; + +macro_rules! private_decl { + () => { + /// This trait is private to implement; this method exists to make it + /// impossible to implement outside the crate. + fn __private__(&self) -> crate::internal_tricks::PrivateMarker; + } +} + +macro_rules! private_impl { + () => { + #[doc(hidden)] + fn __private__(&self) -> crate::internal_tricks::PrivateMarker { + crate::internal_tricks::PrivateMarker + } + } +} diff --git a/src/lib.rs b/src/lib.rs index 293b6456..88d62b69 100755 --- a/src/lib.rs +++ b/src/lib.rs @@ -124,12 +124,12 @@ pub use crate::conversion::{ }; pub use crate::err::{PyDowncastError, PyErr, PyErrArguments, PyErrValue, PyResult}; pub use crate::gil::{init_once, GILGuard, GILPool}; -pub use crate::instance::{ManagedPyRef, Py, PyNativeType}; +pub use crate::instance::{AsPyRef, ManagedPyRef, Py, PyNativeType}; pub use crate::object::PyObject; pub use crate::objectprotocol::ObjectProtocol; -pub use crate::pyclass::{PyClass, PyClassAlloc, PyClassShell}; +pub use crate::pyclass::{PyClass, PyClassShell}; pub use crate::python::{prepare_freethreaded_python, Python}; -pub use crate::type_object::{PyConcreteObject, PyTypeInfo}; +pub use crate::type_object::{type_flags, PyConcreteObject, PyTypeInfo}; // Re-exported for wrap_function #[doc(hidden)] @@ -161,12 +161,14 @@ pub mod ffi; pub mod freelist; mod gil; mod instance; +#[macro_use] mod internal_tricks; pub mod marshal; mod object; mod objectprotocol; pub mod prelude; pub mod pyclass; +pub mod pyclass_slots; mod python; pub mod type_object; pub mod types; diff --git a/src/object.rs b/src/object.rs index 731ec088..d52440dd 100644 --- a/src/object.rs +++ b/src/object.rs @@ -7,7 +7,6 @@ use crate::instance::{AsPyRef, PyNativeType}; use crate::types::{PyAny, PyDict, PyTuple}; use crate::{AsPyPointer, Py, Python}; use crate::{FromPyObject, IntoPy, IntoPyPointer, PyTryFrom, ToBorrowedObject, ToPyObject}; -use std::convert::AsRef; use std::ptr::NonNull; /// A python object diff --git a/src/pyclass.rs b/src/pyclass.rs index 56d8c708..2b1fd1b6 100644 --- a/src/pyclass.rs +++ b/src/pyclass.rs @@ -1,8 +1,9 @@ //! An experiment module which has all codes related only to #[pyclass] use crate::class::methods::{PyMethodDefType, PyMethodsProtocol}; -use crate::conversion::FromPyPointer; -use crate::type_object::{PyConcreteObject, PyTypeObject}; -use crate::{class, ffi, gil, PyErr, PyResult, PyTypeInfo, Python}; +use crate::conversion::{AsPyPointer, FromPyPointer, ToPyObject}; +use crate::pyclass_slots::{PyClassDict, PyClassWeakRef}; +use crate::type_object::{type_flags, PyConcreteObject, PyTypeObject}; +use crate::{class, ffi, gil, PyErr, PyObject, PyResult, PyTypeInfo, Python}; use std::ffi::CString; use std::mem::ManuallyDrop; use std::os::raw::c_void; @@ -46,7 +47,10 @@ pub unsafe fn tp_free_fallback(obj: *mut ffi::PyObject) { } } -pub trait PyClass: PyTypeInfo + Sized + PyClassAlloc + PyMethodsProtocol {} +pub trait PyClass: PyTypeInfo + Sized + PyClassAlloc + PyMethodsProtocol { + type Dict: PyClassDict; + type WeakRef: PyClassWeakRef; +} unsafe impl PyTypeObject for T where @@ -75,21 +79,44 @@ where pub struct PyClassShell { ob_base: ::ConcreteLayout, pyclass: ManuallyDrop, + dict: T::Dict, + weakref: T::WeakRef, } impl PyClassShell { - pub unsafe fn new(py: Python, value: T) -> *mut Self { + pub fn new_ref(py: Python, value: T) -> PyResult<&Self> { + unsafe { + let ptr = Self::new(py, value)?; + FromPyPointer::from_owned_ptr_or_err(py, ptr as _) + } + } + + pub fn new_mut(py: Python, value: T) -> PyResult<&mut Self> { + unsafe { + let ptr = Self::new(py, value)?; + FromPyPointer::from_owned_ptr_or_err(py, ptr as _) + } + } + + pub unsafe fn new(py: Python, value: T) -> PyResult<*mut Self> { T::init_type(); let base = T::alloc(py); + if base.is_null() { + return Err(PyErr::fetch(py)); + } let self_ = base as *mut Self; (*self_).pyclass = ManuallyDrop::new(value); - self_ + (*self_).dict = T::Dict::new(); + (*self_).weakref = T::WeakRef::new(); + Ok(self_) } } impl PyConcreteObject for PyClassShell { unsafe fn py_drop(&mut self, py: Python) { ManuallyDrop::drop(&mut self.pyclass); + self.dict.clear_dict(py); + self.weakref.clear_weakrefs(self.as_ptr(), py); self.ob_base.py_drop(py); } } @@ -107,6 +134,12 @@ impl std::ops::DerefMut for PyClassShell { } } +impl ToPyObject for PyClassShell { + fn to_object(&self, py: Python<'_>) -> PyObject { + unsafe { PyObject::from_borrowed_ptr(py, self.as_ptr()) } + } +} + unsafe impl<'p, T> FromPyPointer<'p> for &'p PyClassShell where T: PyClass, @@ -131,18 +164,6 @@ where } } -/// type object supports python GC -const PY_TYPE_FLAG_GC: usize = 1; - -/// Type object supports python weak references -const PY_TYPE_FLAG_WEAKREF: usize = 1 << 1; - -/// Type object can be used as the base type of another type -const PY_TYPE_FLAG_BASETYPE: usize = 1 << 2; - -/// The instances of this type have a dictionary containing instance variables -const PY_TYPE_FLAG_DICT: usize = 1 << 3; - /// Register new type in python object system. #[cfg(not(Py_LIMITED_API))] pub fn initialize_type(py: Python, module_name: Option<&str>) -> PyResult<*mut ffi::PyTypeObject> @@ -182,19 +203,20 @@ where type_object.tp_dealloc = Some(tp_dealloc_callback::); // type size - type_object.tp_basicsize = std::mem::size_of::() as isize; + type_object.tp_basicsize = std::mem::size_of::() as ffi::Py_ssize_t; - // weakref support (check py3cls::py_class::impl_class) - if T::FLAGS & PY_TYPE_FLAG_WEAKREF != 0 { - // STUB - type_object.tp_weaklistoffset = 0isize; - } + let mut offset = type_object.tp_basicsize; // __dict__ support - let has_dict = T::FLAGS & PY_TYPE_FLAG_DICT != 0; - if has_dict { - // STUB - type_object.tp_dictoffset = 0isize; + if let Some(dict_offset) = T::Dict::OFFSET { + offset += dict_offset as ffi::Py_ssize_t; + type_object.tp_dictoffset = offset; + } + + // weakref support + if let Some(weakref_offset) = T::WeakRef::OFFSET { + offset += weakref_offset as ffi::Py_ssize_t; + type_object.tp_weaklistoffset = offset; } // GC support @@ -243,7 +265,7 @@ where // properties let mut props = py_class_properties::(); - if has_dict { + if T::Dict::OFFSET.is_some() { props.push(ffi::PyGetSetDef_DICT); } if !props.is_empty() { @@ -267,13 +289,13 @@ where fn py_class_flags(type_object: &mut ffi::PyTypeObject) { if type_object.tp_traverse != None || type_object.tp_clear != None - || T::FLAGS & PY_TYPE_FLAG_GC != 0 + || T::FLAGS & type_flags::GC != 0 { type_object.tp_flags = ffi::Py_TPFLAGS_DEFAULT | ffi::Py_TPFLAGS_HAVE_GC; } else { type_object.tp_flags = ffi::Py_TPFLAGS_DEFAULT; } - if T::FLAGS & PY_TYPE_FLAG_BASETYPE != 0 { + if T::FLAGS & type_flags::BASETYPE != 0 { type_object.tp_flags |= ffi::Py_TPFLAGS_BASETYPE; } } diff --git a/src/pyclass_slots.rs b/src/pyclass_slots.rs new file mode 100644 index 00000000..698816bb --- /dev/null +++ b/src/pyclass_slots.rs @@ -0,0 +1,72 @@ +//! This module contains additional fields pf pyclass +// TODO(kngwyu): Add vectorcall support +use crate::{ffi, Python}; + +const POINTER_SIZE: isize = std::mem::size_of::<*mut ffi::PyObject>() as _; + +/// Represents `__dict__`. +pub trait PyClassDict { + const OFFSET: Option = None; + fn new() -> Self; + fn clear_dict(&mut self, _py: Python) {} + private_decl! {} +} + +/// Represents `__weakref__`. +pub trait PyClassWeakRef { + const OFFSET: Option = None; + fn new() -> Self; + fn clear_weakrefs(&mut self, _obj: *mut ffi::PyObject, _py: Python) {} + private_decl! {} +} + +/// Dummy slot means the function doesn't has such a feature. +pub struct PyClassDummySlot; + +impl PyClassDict for PyClassDummySlot { + private_impl! {} + fn new() -> Self { + PyClassDummySlot + } +} + +impl PyClassWeakRef for PyClassDummySlot { + private_impl! {} + fn new() -> Self { + PyClassDummySlot + } +} + +/// actural dict field +#[repr(transparent)] +pub struct PyClassDictSlot(*mut ffi::PyObject); + +impl PyClassDict for PyClassDictSlot { + private_impl! {} + const OFFSET: Option = Some(-POINTER_SIZE); + fn new() -> Self { + Self(std::ptr::null_mut()) + } + fn clear_dict(&mut self, _py: Python) { + if self.0 != std::ptr::null_mut() { + unsafe { ffi::PyDict_Clear(self.0) } + } + } +} + +/// actural weakref field +#[repr(transparent)] +pub struct PyClassWeakRefSlot(*mut ffi::PyObject); + +impl PyClassWeakRef for PyClassWeakRefSlot { + private_impl! {} + const OFFSET: Option = Some(-POINTER_SIZE); + fn new() -> Self { + Self(std::ptr::null_mut()) + } + fn clear_weakrefs(&mut self, obj: *mut ffi::PyObject, _py: Python) { + if self.0 != std::ptr::null_mut() { + unsafe { ffi::PyObject_ClearWeakRefs(obj) } + } + } +} diff --git a/src/type_object.rs b/src/type_object.rs index b5af7b18..e7ff7dc5 100644 --- a/src/type_object.rs +++ b/src/type_object.rs @@ -10,6 +10,7 @@ use crate::AsPyPointer; use crate::Python; use std::ptr::NonNull; +/// TODO: write document pub trait PyConcreteObject: Sized { unsafe fn py_drop(&mut self, _py: Python) {} } @@ -22,6 +23,21 @@ impl AsPyPointer for T { impl PyConcreteObject for ffi::PyObject {} +/// Our custom type flags +pub mod type_flags { + /// type object supports python GC + pub const GC: usize = 1; + + /// Type object supports python weak references + pub const WEAKREF: usize = 1 << 1; + + /// Type object can be used as the base type of another type + pub const BASETYPE: usize = 1 << 2; + + /// The instances of this type have a dictionary containing instance variables + pub const DICT: usize = 1 << 3; +} + /// Python type information. pub trait PyTypeInfo { /// Type of objects to store in PyObject struct @@ -36,9 +52,6 @@ pub trait PyTypeInfo { /// Class doc string const DESCRIPTION: &'static str = "\0"; - /// `Type` instance offset inside PyObject structure - const OFFSET: isize; - /// Type flags (ie PY_TYPE_FLAG_GC, PY_TYPE_FLAG_WEAKREF) const FLAGS: usize = 0; diff --git a/src/types/any.rs b/src/types/any.rs index aa962f66..b299c6c9 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -1,4 +1,3 @@ -use crate::conversion::AsPyPointer; use crate::conversion::PyTryFrom; use crate::err::PyDowncastError; use crate::internal_tricks::Unsendable; diff --git a/src/types/mod.rs b/src/types/mod.rs index 6f13c2b5..48eb96cb 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -108,7 +108,6 @@ macro_rules! pyobject_native_type_convert( const NAME: &'static str = stringify!($name); const MODULE: Option<&'static str> = $module; - const OFFSET: isize = 0; #[inline] unsafe fn type_object() -> &'static mut $crate::ffi::PyTypeObject {