Allow 0..n pymethod blocks without specialization

Cargo.toml

@@ -26,6 +26,7 @@ num-traits = "0.2.6"
 pyo3cls = { path = "pyo3cls", version = "=0.6.0-alpha.2" }
 mashup = "0.1.9"
 num-complex = { version = "0.2.1", optional = true }
+inventory = "0.1.2"
 
 [dev-dependencies]
 assert_approx_eq = "1.0.0"

examples/rustapi_module/src/lib.rs

@@ -1,4 +1,3 @@
-#![feature(specialization)]
 
 pub mod datetime;
 pub mod dict_iter;

examples/word-count/src/lib.rs

@@ -1,6 +1,5 @@
 // Source adopted from
 // https://github.com/tildeio/helix-website/blob/master/crates/word_count/src/lib.rs
-#![feature(specialization)]
 
 use pyo3::prelude::*;
 use pyo3::wrap_pyfunction;

guide/src/class.md

@@ -556,3 +556,13 @@ impl PyIterProtocol for MyIterator {
     }
 }
 ```
+
+## Manually implementing pyclass
+
+TODO: Which traits to implement (basically `PyTypeCreate: PyObjectAlloc + PyTypeInfo + PyMethodsProtocol + Sized`) and what they mean.
+
+## How methods are implemented
+
+Users should be able to define a `#[pyclass]` with or without `#[pymethods]`, while pyo3 needs a trait with a function that returns all methods. Since it's impossible make the code generation in pyclass dependent on whether there is an impl block, we'd need to make to implement the trait on `#[pyclass]` and override the implementation in `#[pymethods]`, which is to my best knowledge only possible with the specialization feature, which is can't be used on stable.
+
+To escape this we use [inventory](https://github.com/dtolnay/inventory), which allows us to collect `impl`s from arbitrary source code by exploiting some binary trick. See [inventory: how it works](https://github.com/dtolnay/inventory#how-it-works) and `pyo3_derive_backend::py_class::impl_inventory` for more details.

pyo3-derive-backend/src/py_class.rs

@@ -63,6 +63,40 @@ fn parse_descriptors(item: &mut syn::Field) -> Vec<FnType> {
     descs
 }
 
+/// The orphan rule disallows using a generic inventory struct, so we create the whole boilerplate
+/// once per class
+fn impl_inventory(cls: &syn::Ident) -> TokenStream {
+    // Try to build a unique type that gives a hint about it's function when
+    // it comes up in error messages
+    let name = cls.to_string() + "GeneratedPyo3Inventory";
+    let inventory_cls = syn::Ident::new(&name, Span::call_site());
+
+    quote! {
+        #[doc(hidden)]
+        pub struct #inventory_cls {
+            methods: &'static [::pyo3::class::PyMethodDefType],
+        }
+
+        impl ::pyo3::class::methods::PyMethodsInventory for #inventory_cls {
+            fn new(methods: &'static [::pyo3::class::PyMethodDefType]) -> Self {
+                Self {
+                    methods
+                }
+            }
+
+            fn get_methods(&self) -> &'static [::pyo3::class::PyMethodDefType] {
+                self.methods
+            }
+        }
+
+        impl ::pyo3::class::methods::PyMethodsInventoryDispatch for #cls {
+            type InventoryType = #inventory_cls;
+        }
+
+        ::pyo3::inventory::collect!(#inventory_cls);
+    }
+}
+
 fn impl_class(
     cls: &syn::Ident,
     base: &syn::TypePath,
@@ -136,6 +170,8 @@ fn impl_class(
         quote! {0}
     };
 
+    let inventory_impl = impl_inventory(&cls);
+
     quote! {
         impl ::pyo3::typeob::PyTypeInfo for #cls {
             type Type = #cls;
@@ -197,6 +233,8 @@ fn impl_class(
             }
         }
 
+        #inventory_impl
+
         #extra
     }
 }
@@ -287,12 +325,11 @@ fn impl_descriptors(cls: &syn::Type, descriptors: Vec<(syn::Field, Vec<FnType>)>
     quote! {
         #(#methods)*
 
-        impl ::pyo3::class::methods::PyPropMethodsProtocolImpl for #cls {
-            fn py_methods() -> &'static [::pyo3::class::PyMethodDefType] {
-                static METHODS: &'static [::pyo3::class::PyMethodDefType] = &[
-                    #(#py_methods),*
-                ];
-                METHODS
+        ::pyo3::inventory::submit! {
+            #![crate = pyo3]
+            {
+                type ClsInventory = <#cls as ::pyo3::class::methods::PyMethodsInventoryDispatch>::InventoryType;
+                <ClsInventory as ::pyo3::class::methods::PyMethodsInventory>::new(&[#(#py_methods),*])
             }
         }
     }

pyo3-derive-backend/src/py_impl.rs

@@ -31,12 +31,11 @@ pub fn impl_methods(ty: &syn::Type, impls: &mut Vec<syn::ImplItem>) -> TokenStre
     }
 
     quote! {
-        impl ::pyo3::class::methods::PyMethodsProtocolImpl for #ty {
-            fn py_methods() -> &'static [::pyo3::class::PyMethodDefType] {
-                static METHODS: &'static [::pyo3::class::PyMethodDefType] = &[
-                    #(#methods),*
-                ];
-                METHODS
+        ::pyo3::inventory::submit! {
+            #![crate = pyo3]
+            {
+                type TyInventory = <#ty as ::pyo3::class::methods::PyMethodsInventoryDispatch>::InventoryType;
+                <TyInventory as ::pyo3::class::methods::PyMethodsInventory>::new(&[#(#methods),*])
             }
         }
     }

src/class/methods.rs

@@ -58,10 +58,13 @@ pub struct PySetterDef {
 }
 
 unsafe impl Sync for PyMethodDef {}
+
 unsafe impl Sync for ffi::PyMethodDef {}
 
 unsafe impl Sync for PyGetterDef {}
+
 unsafe impl Sync for PySetterDef {}
+
 unsafe impl Sync for ffi::PyGetSetDef {}
 
 impl PyMethodDef {
@@ -110,21 +113,40 @@ impl PySetterDef {
     }
 }
 
-#[doc(hidden)]
-/// The pymethods macro implements this trait so the methods are added to the object
-pub trait PyMethodsProtocolImpl {
-    fn py_methods() -> &'static [PyMethodDefType] {
-        &[]
-    }
+#[doc(hidden)] // Only to be used through the proc macros, use PyMethodsProtocol in custom code
+/// This trait is implemented for all pyclass so to implement the [PyMethodsProtocol]
+/// through inventory
+pub trait PyMethodsInventoryDispatch {
+    /// This allows us to get the inventory type when only the pyclass is in scope
+    type InventoryType: PyMethodsInventory;
 }
 
-impl<T> PyMethodsProtocolImpl for T {}
+#[doc(hidden)] // Only to be used through the proc macros, use PyMethodsProtocol in custom code
+/// Allows arbitrary pymethod blocks to submit their methods, which are eventually collected by pyclass
+pub trait PyMethodsInventory: inventory::Collect {
+    /// Create a new instance
+    fn new(methods: &'static [PyMethodDefType]) -> Self;
 
-#[doc(hidden)]
-pub trait PyPropMethodsProtocolImpl {
-    fn py_methods() -> &'static [PyMethodDefType] {
-        &[]
-    }
+    /// Returns the methods for a single impl block
+    fn get_methods(&self) -> &'static [PyMethodDefType];
 }
 
-impl<T> PyPropMethodsProtocolImpl for T {}
+/// The implementation of tis trait defines which methods a python type has.
+///
+/// For pyclass derived structs this is implemented by collecting all impl blocks through inventory
+pub trait PyMethodsProtocol {
+    /// Returns all methods that are defined for a class
+    fn py_methods() -> Vec<&'static PyMethodDefType>;
+}
+
+impl<T> PyMethodsProtocol for T
+where
+    T: PyMethodsInventoryDispatch,
+{
+    fn py_methods() -> Vec<&'static PyMethodDefType> {
+        inventory::iter::<T::InventoryType>
+            .into_iter()
+            .flat_map(PyMethodsInventory::get_methods)
+            .collect()
+    }
+}

src/freelist.rs

@@ -6,6 +6,7 @@ use crate::err::PyResult;
 use crate::ffi;
 use crate::python::Python;
 use crate::typeob::{pytype_drop, PyObjectAlloc, PyTypeInfo};
+use class::methods::PyMethodsProtocol;
 use std::mem;
 use std::os::raw::c_void;
 
@@ -70,7 +71,7 @@ impl<T> FreeList<T> {
 
 impl<T> PyObjectAlloc for T
 where
-    T: PyObjectWithFreeList,
+    T: PyObjectWithFreeList + PyMethodsProtocol,
 {
     unsafe fn alloc(_py: Python) -> PyResult<*mut ffi::PyObject> {
         let obj = if let Some(obj) = <Self as PyObjectWithFreeList>::get_free_list().pop() {

src/typeob.rs

@@ -10,6 +10,7 @@ use crate::python::{IntoPyPointer, Python};
 use crate::types::PyObjectRef;
 use crate::types::PyType;
 use crate::{class, ffi, pythonrun};
+use class::methods::PyMethodsProtocol;
 use std::collections::HashMap;
 use std::ffi::CString;
 use std::os::raw::c_void;
@@ -196,7 +197,7 @@ pub(crate) unsafe fn pytype_drop<T: PyTypeInfo>(py: Python, obj: *mut ffi::PyObj
 ///
 /// All native types and all `#[pyclass]` types use the default functions, while
 /// [PyObjectWithFreeList](crate::freelist::PyObjectWithFreeList) gets a special version.
-pub trait PyObjectAlloc: PyTypeInfo + Sized {
+pub trait PyObjectAlloc: PyTypeInfo + PyMethodsProtocol + Sized {
     unsafe fn alloc(_py: Python) -> PyResult<*mut ffi::PyObject> {
         // TODO: remove this
         <Self as PyTypeCreate>::init_type();
@@ -258,7 +259,7 @@ pub trait PyTypeObject {
 
 /// Python object types that have a corresponding type object and be
 /// instanciated with [Self::create()]
-pub trait PyTypeCreate: PyObjectAlloc + PyTypeInfo + Sized {
+pub trait PyTypeCreate: PyObjectAlloc + PyTypeInfo + PyMethodsProtocol + Sized {
     #[inline]
     fn init_type() {
         let type_object = unsafe { *<Self as PyTypeInfo>::type_object() };
@@ -297,7 +298,7 @@ pub trait PyTypeCreate: PyObjectAlloc + PyTypeInfo + Sized {
     }
 }
 
-impl<T> PyTypeCreate for T where T: PyObjectAlloc + PyTypeInfo + Sized {}
+impl<T> PyTypeCreate for T where T: PyObjectAlloc + PyTypeInfo + PyMethodsProtocol {}
 
 impl<T> PyTypeObject for T
 where
@@ -316,7 +317,7 @@ where
 #[cfg(not(Py_LIMITED_API))]
 pub fn initialize_type<T>(py: Python, module_name: Option<&str>) -> PyResult<()>
 where
-    T: PyObjectAlloc + PyTypeInfo,
+    T: PyObjectAlloc + PyTypeInfo + PyMethodsProtocol,
 {
     // type name
     let name = match module_name {
@@ -493,7 +494,7 @@ fn py_class_flags<T: PyTypeInfo>(type_object: &mut ffi::PyTypeObject) {
     }
 }
 
-fn py_class_method_defs<T>() -> PyResult<(
+fn py_class_method_defs<T: PyMethodsProtocol>() -> PyResult<(
     Option<ffi::newfunc>,
     Option<ffi::initproc>,
     Option<ffi::PyCFunctionWithKeywords>,
@@ -504,7 +505,7 @@ fn py_class_method_defs<T>() -> PyResult<(
     let mut new = None;
     let mut init = None;
 
-    for def in <T as class::methods::PyMethodsProtocolImpl>::py_methods() {
+    for def in T::py_methods() {
         match *def {
             PyMethodDefType::New(ref def) => {
                 if let class::methods::PyMethodType::PyNewFunc(meth) = def.ml_meth {
@@ -565,13 +566,10 @@ fn py_class_async_methods<T>(defs: &mut Vec<ffi::PyMethodDef>) {
 #[cfg(not(Py_3))]
 fn py_class_async_methods<T>(_defs: &mut Vec<ffi::PyMethodDef>) {}
 
-fn py_class_properties<T>() -> Vec<ffi::PyGetSetDef> {
+fn py_class_properties<T: PyMethodsProtocol>() -> Vec<ffi::PyGetSetDef> {
     let mut defs = HashMap::new();
 
-    for def in <T as class::methods::PyMethodsProtocolImpl>::py_methods()
-        .iter()
-        .chain(<T as class::methods::PyPropMethodsProtocolImpl>::py_methods().iter())
-    {
+    for def in T::py_methods() {
         match *def {
             PyMethodDefType::Getter(ref getter) => {
                 let name = getter.name.to_string();

src/types/mod.rs

@@ -127,6 +127,14 @@ macro_rules! pyobject_native_type_convert(
             }
         }
 
+        // We currently need to fulfill this trait bound for PyTypeCreate, even though we know
+        // that the function will never actuall be called
+        impl<$($type_param,)*> $crate::class::methods::PyMethodsProtocol for $name {
+            fn py_methods() -> Vec<&'static $crate::class::methods::PyMethodDefType> {
+                unreachable!();
+            }
+        }
+
         impl<$($type_param,)*> $crate::typeob::PyObjectAlloc for $name {}
 
         impl<$($type_param,)*> $crate::typeob::PyTypeCreate for $name {

src/types/module.rs

@@ -12,6 +12,7 @@ use crate::python::{Python, ToPyPointer};
 use crate::typeob::{initialize_type, PyTypeInfo};
 use crate::types::{exceptions, PyDict, PyObjectRef, PyType};
 use crate::PyObjectAlloc;
+use class::methods::PyMethodsProtocol;
 use std::ffi::{CStr, CString};
 use std::os::raw::c_char;
 use std::str;
@@ -150,7 +151,7 @@ impl PyModule {
     /// and adds the type to this module.
     pub fn add_class<T>(&self) -> PyResult<()>
     where
-        T: PyTypeInfo + PyObjectAlloc,
+        T: PyTypeInfo + PyObjectAlloc + PyMethodsProtocol,
     {
         let ty = unsafe {
             let ty = <T as PyTypeInfo>::type_object();