abi3: add support for dict and weakref from Python 3.9

CHANGELOG.md

@@ -6,6 +6,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- Add support for `#[pyclass(dict)]` and `#[pyclass(weakref)]` with the `abi3` feature on Python 3.9 and up. [#1342](https://github.com/PyO3/pyo3/pull/1342)
+
 ### Fixed
 - Stop including `Py_TRACE_REFS` config setting automatically if `Py_DEBUG` is set on Python 3.8 and up. [#1334](https://github.com/PyO3/pyo3/pull/1334)
 - Remove `#[deny(warnings)]` attribute (and instead refuse warnings only in CI). [#1340](https://github.com/PyO3/pyo3/pull/1340)

guide/src/building_and_distribution.md

@@ -74,7 +74,7 @@ Due to limitations in the Python API, there are a few `pyo3` features that do
 not work when compiling for `abi3`. These are:
 
 - `#[text_signature]` does not work on classes until Python 3.10 or greater.
-- The `dict` and `weakref` options on classes are not supported.
+- The `dict` and `weakref` options on classes are not supported until Python 3.9 or greater.
 - The buffer API is not supported.
 
 ## Cross Compiling

src/ffi/object.rs

@@ -706,7 +706,7 @@ extern "C" {
         arg2: *mut PyObject,
         arg3: *mut PyObject,
     ) -> c_int;
-    #[cfg(not(Py_LIMITED_API))]
+    #[cfg(not(all(Py_LIMITED_API, not(Py_3_10))))]
     pub fn PyObject_GenericGetDict(arg1: *mut PyObject, arg2: *mut c_void) -> *mut PyObject;
     pub fn PyObject_GenericSetDict(
         arg1: *mut PyObject,

src/pycell.rs

@@ -170,7 +170,7 @@ pub struct PyCell<T: PyClass> {
 
 impl<T: PyClass> PyCell<T> {
     /// Get the offset of the dictionary from the start of the struct in bytes.
-    #[cfg(not(Py_LIMITED_API))]
+    #[cfg(not(all(Py_LIMITED_API, not(Py_3_9))))]
     pub(crate) fn dict_offset() -> Option<usize> {
         if T::Dict::IS_DUMMY {
             None
@@ -184,7 +184,7 @@ impl<T: PyClass> PyCell<T> {
     }
 
     /// Get the offset of the weakref list from the start of the struct in bytes.
-    #[cfg(not(Py_LIMITED_API))]
+    #[cfg(not(all(Py_LIMITED_API, not(Py_3_9))))]
     pub(crate) fn weakref_offset() -> Option<usize> {
         if T::WeakRef::IS_DUMMY {
             None

src/pyclass.rs

@@ -178,6 +178,14 @@ where
     slots.maybe_push(ffi::Py_tp_new, new.map(|v| v as _));
     slots.maybe_push(ffi::Py_tp_call, call.map(|v| v as _));
 
+    #[cfg(Py_3_9)]
+    {
+        let members = py_class_members::<T>();
+        if !members.is_empty() {
+            slots.push(ffi::Py_tp_members, into_raw(members))
+        }
+    }
+
     // normal methods
     if !methods.is_empty() {
         slots.push(ffi::Py_tp_methods, into_raw(methods));
@@ -203,7 +211,7 @@ where
         basicsize: std::mem::size_of::<T::Layout>() as c_int,
         itemsize: 0,
         flags: py_class_flags::<T>(has_gc_methods),
-        slots: slots.0.as_mut_slice().as_mut_ptr(),
+        slots: slots.0.as_mut_ptr(),
     };
 
     let type_object = unsafe { ffi::PyType_FromSpec(&mut spec) };
@@ -215,7 +223,8 @@ where
     }
 }
 
-#[cfg(not(Py_LIMITED_API))]
+/// Additional type initializations necessary before Python 3.10
+#[cfg(all(not(Py_LIMITED_API), not(Py_3_10)))]
 fn tp_init_additional<T: PyClass>(type_object: *mut ffi::PyTypeObject) {
     // Just patch the type objects for the things there's no
     // PyType_FromSpec API for... there's no reason this should work,
@@ -247,21 +256,27 @@ fn tp_init_additional<T: PyClass>(type_object: *mut ffi::PyTypeObject) {
             (*(*type_object).tp_as_buffer).bf_releasebuffer = buffer.bf_releasebuffer;
         }
     }
-    // __dict__ support
-    if let Some(dict_offset) = PyCell::<T>::dict_offset() {
-        unsafe {
-            (*type_object).tp_dictoffset = dict_offset as ffi::Py_ssize_t;
+
+    // Setting tp_dictoffset and tp_weaklistoffset via slots doesn't work until Python 3.9, so on
+    // older versions again we must fixup the type object.
+    #[cfg(not(Py_3_9))]
+    {
+        // __dict__ support
+        if let Some(dict_offset) = PyCell::<T>::dict_offset() {
+            unsafe {
+                (*type_object).tp_dictoffset = dict_offset as ffi::Py_ssize_t;
+            }
         }
-    }
-    // weakref support
-    if let Some(weakref_offset) = PyCell::<T>::weakref_offset() {
-        unsafe {
-            (*type_object).tp_weaklistoffset = weakref_offset as ffi::Py_ssize_t;
+        // weakref support
+        if let Some(weakref_offset) = PyCell::<T>::weakref_offset() {
+            unsafe {
+                (*type_object).tp_weaklistoffset = weakref_offset as ffi::Py_ssize_t;
+            }
         }
     }
 }
 
-#[cfg(Py_LIMITED_API)]
+#[cfg(any(Py_LIMITED_API, Py_3_10))]
 fn tp_init_additional<T: PyClass>(_type_object: *mut ffi::PyTypeObject) {}
 
 fn py_class_flags<T: PyClass + PyTypeInfo>(has_gc_methods: bool) -> c_uint {
@@ -333,6 +348,43 @@ fn py_class_method_defs<T: PyMethods>() -> (
     (new, call, defs)
 }
 
+/// Generates the __dictoffset__ and __weaklistoffset__ members, to set tp_dictoffset and
+/// tp_weaklistoffset.
+///
+/// Only works on Python 3.9 and up.
+#[cfg(Py_3_9)]
+fn py_class_members<T: PyClass>() -> Vec<ffi::structmember::PyMemberDef> {
+    macro_rules! offset_def {
+        ($name:literal, $offset:expr) => {
+            ffi::structmember::PyMemberDef {
+                name: $name.as_ptr() as _,
+                type_code: ffi::structmember::T_PYSSIZET,
+                offset: $offset,
+                flags: ffi::structmember::READONLY,
+                doc: std::ptr::null_mut(),
+            }
+        };
+    }
+
+    let mut members = Vec::new();
+
+    // __dict__ support
+    if let Some(dict_offset) = PyCell::<T>::dict_offset() {
+        members.push(offset_def!("__dictoffset__\0", dict_offset as _));
+    }
+
+    // weakref support
+    if let Some(weakref_offset) = PyCell::<T>::weakref_offset() {
+        members.push(offset_def!("__weaklistoffset__\0", weakref_offset as _));
+    }
+
+    if !members.is_empty() {
+        members.push(unsafe { std::mem::zeroed() });
+    }
+
+    members
+}
+
 fn py_class_properties<T: PyClass>() -> Vec<ffi::PyGetSetDef> {
     let mut defs = std::collections::HashMap::new();
 
@@ -357,11 +409,21 @@ fn py_class_properties<T: PyClass>() -> Vec<ffi::PyGetSetDef> {
     }
 
     let mut props: Vec<_> = defs.values().cloned().collect();
+
+    // PyPy doesn't automatically adds __dict__ getter / setter.
+    // PyObject_GenericGetDict not in the limited API until Python 3.10.
+    #[cfg(not(any(PyPy, all(Py_LIMITED_API, not(Py_3_10)))))]
     if !T::Dict::IS_DUMMY {
-        props.push(ffi::PyGetSetDef_DICT);
+        props.push(ffi::PyGetSetDef {
+            name: "__dict__\0".as_ptr() as *mut c_char,
+            get: Some(ffi::PyObject_GenericGetDict),
+            set: Some(ffi::PyObject_GenericSetDict),
+            doc: ptr::null_mut(),
+            closure: ptr::null_mut(),
+        });
     }
     if !props.is_empty() {
-        props.push(ffi::PyGetSetDef_INIT);
+        props.push(unsafe { std::mem::zeroed() });
     }
     props
 }

tests/test_dunder.rs

@@ -457,7 +457,7 @@ fn test_cls_impl() {
 struct DunderDictSupport {}
 
 #[test]
-#[cfg_attr(Py_LIMITED_API, ignore)]
+#[cfg_attr(all(Py_LIMITED_API, not(Py_3_9)), ignore)]
 fn dunder_dict_support() {
     let gil = Python::acquire_gil();
     let py = gil.python();
@@ -472,8 +472,9 @@ fn dunder_dict_support() {
     );
 }
 
+// Accessing inst.__dict__ only supported in limited API from Python 3.10
 #[test]
-#[cfg_attr(Py_LIMITED_API, ignore)]
+#[cfg_attr(all(Py_LIMITED_API, not(Py_3_10)), ignore)]
 fn access_dunder_dict() {
     let gil = Python::acquire_gil();
     let py = gil.python();
@@ -495,7 +496,7 @@ struct InheritDict {
 }
 
 #[test]
-#[cfg_attr(Py_LIMITED_API, ignore)]
+#[cfg_attr(all(Py_LIMITED_API, not(Py_3_9)), ignore)]
 fn inherited_dict() {
     let gil = Python::acquire_gil();
     let py = gil.python();
@@ -505,7 +506,7 @@ fn inherited_dict() {
         inst,
         r#"
         inst.a = 1
-        assert inst.__dict__ == {'a': 1}
+        assert inst.a == 1
     "#
     );
 }
@@ -514,7 +515,7 @@ fn inherited_dict() {
 struct WeakRefDunderDictSupport {}
 
 #[test]
-#[cfg_attr(Py_LIMITED_API, ignore)]
+#[cfg_attr(all(Py_LIMITED_API, not(Py_3_9)), ignore)]
 fn weakref_dunder_dict_support() {
     let gil = Python::acquire_gil();
     let py = gil.python();

tests/test_gc.rs

@@ -150,7 +150,7 @@ fn gc_integration2() {
 struct WeakRefSupport {}
 
 #[test]
-#[cfg_attr(Py_LIMITED_API, ignore)]
+#[cfg_attr(all(Py_LIMITED_API, not(Py_3_9)), ignore)]
 fn weakref_support() {
     let gil = Python::acquire_gil();
     let py = gil.python();
@@ -169,7 +169,7 @@ struct InheritWeakRef {
 }
 
 #[test]
-#[cfg_attr(Py_LIMITED_API, ignore)]
+#[cfg_attr(all(Py_LIMITED_API, not(Py_3_9)), ignore)]
 fn inherited_weakref() {
     let gil = Python::acquire_gil();
     let py = gil.python();

tests/test_unsendable_dict.rs

@@ -13,7 +13,7 @@ impl UnsendableDictClass {
 }
 
 #[test]
-#[cfg_attr(Py_LIMITED_API, ignore)]
+#[cfg_attr(all(Py_LIMITED_API, not(Py_3_10)), ignore)]
 fn test_unsendable_dict() {
     let gil = Python::acquire_gil();
     let py = gil.python();
@@ -33,7 +33,7 @@ impl UnsendableDictClassWithWeakRef {
 }
 
 #[test]
-#[cfg_attr(Py_LIMITED_API, ignore)]
+#[cfg_attr(all(Py_LIMITED_API, not(Py_3_10)), ignore)]
 fn test_unsendable_dict_with_weakref() {
     let gil = Python::acquire_gil();
     let py = gil.python();

tests/test_various.rs

@@ -149,7 +149,7 @@ fn add_module(py: Python, module: &PyModule) -> PyResult<()> {
 }
 
 #[test]
-#[cfg_attr(Py_LIMITED_API, ignore)]
+#[cfg_attr(all(Py_LIMITED_API, not(Py_3_10)), ignore)]
 fn test_pickle() {
     let gil = Python::acquire_gil();
     let py = gil.python();