luxolus/pyo3
2
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Merge pull request #332 from PyO3/inventory

Browse source 
Allow 0..n pymethod blocks without specialization
This commit is contained in:
konstin 2019-02-01 17:40:23 +01:00 committed by GitHub
parent 9c3ac7e526 328d61ca3b
commit 853fc7c7d1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 164 additions and 130 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
Cargo.toml
View file

@ -20,20 +20,21 @@ appveyor = { repository = "fafhrd91/pyo3" }
codecov = { repository = "PyO3/pyo3", branch = "master", service = "github" }
[dependencies]
libc = "0.2.43"
libc = "0.2.48"
spin = "0.5.0"
num-traits = "0.2.6"
pyo3cls = { path = "pyo3cls", version = "=0.6.0-alpha.2" }
mashup = "0.1.9"
num-complex = { version = "0.2.1", optional = true }
inventory = "0.1.3"
[dev-dependencies]
assert_approx_eq = "1.0.0"
assert_approx_eq = "1.1.0"
docmatic = "0.1.2"
indoc = "0.3.1"
[build-dependencies]
regex = "1.0.5"
regex = "1.1.0"
version_check = "0.1.5"
[features]

6
Makefile
View file

@ -4,7 +4,11 @@ test:
cargo test
cargo clippy
tox
for example in examples/*; do tox -e py --workdir $$example; done
for example in examples/*; do tox -e py -c $$example/tox.ini; done
test_py3:
tox -e py3
for example in examples/*; do tox -e py3 -c $$example/tox.ini; done
publish:
cargo test

2
examples/rustapi_module/src/lib.rs
View file

@ -1,5 +1,3 @@
#![feature(specialization)]
pub mod datetime;
pub mod dict_iter;
pub mod othermod;

2
examples/rustapi_module/tests/test_datetime.py
View file

@ -296,4 +296,4 @@ def test_tz_class_introspection():
tzi = rdt.TzClass()
assert tzi.__class__ == rdt.TzClass
assert repr(tzi).startswith("<rustapi_module.datetime.TzClass object at")
assert repr(tzi).startswith("<TzClass object at")

1
examples/word-count/src/lib.rs
View file

@ -1,6 +1,5 @@
// Source adopted from
// https://github.com/tildeio/helix-website/blob/master/crates/word_count/src/lib.rs
#![feature(specialization)]
use pyo3::prelude::*;
use pyo3::wrap_pyfunction;

25
guide/src/class.md
View file

@ -5,7 +5,6 @@
To define python custom class, rust struct needs to be annotated with `#[pyclass]` attribute.
```rust
# #![feature(specialization)]
# use pyo3::prelude::*;
#[pyclass]
@ -42,11 +41,8 @@ To declare a constructor, you need to define a class method and annotate it with
attribute. Only the python `__new__` method can be specified, `__init__` is not available.
```rust
# #![feature(specialization)]
#
# use pyo3::prelude::*;
# use pyo3::PyRawObject;
#[pyclass]
struct MyClass {
num: i32,
@ -86,7 +82,6 @@ By default `PyObject` is used as default base class. To override default base cl
with value of custom class struct. Subclass must call parent's `__new__` method.
```rust
# #![feature(specialization)]
# use pyo3::prelude::*;
# use pyo3::PyRawObject;
#[pyclass]
@ -136,7 +131,6 @@ Descriptor methods can be defined in
attributes. i.e.
```rust
# #![feature(specialization)]
# use pyo3::prelude::*;
# #[pyclass]
# struct MyClass {
@ -161,7 +155,6 @@ Descriptor name becomes function name with prefix removed. This is useful in cas
rust's special keywords like `type`.
```rust
# #![feature(specialization)]
# use pyo3::prelude::*;
# #[pyclass]
# struct MyClass {
@ -190,7 +183,6 @@ Also both `#[getter]` and `#[setter]` attributes accepts one parameter.
If parameter is specified, it is used and property name. i.e.
```rust
# #![feature(specialization)]
# use pyo3::prelude::*;
# #[pyclass]
# struct MyClass {
@ -218,7 +210,6 @@ In this case property `number` is defined. And it is available from python code
For simple cases you can also define getters and setters in your Rust struct field definition, for example:
```rust
# #![feature(specialization)]
# use pyo3::prelude::*;
#[pyclass]
struct MyClass {
@ -237,7 +228,6 @@ wrappers for all functions in this block with some variations, like descriptors,
class method static methods, etc.
```rust
# #![feature(specialization)]
# use pyo3::prelude::*;
# #[pyclass]
# struct MyClass {
@ -265,7 +255,6 @@ The return type must be `PyResult<T>` for some `T` that implements `IntoPyObject
get injected by method wrapper. i.e
```rust
# #![feature(specialization)]
# use pyo3::prelude::*;
# #[pyclass]
# struct MyClass {
@ -289,7 +278,6 @@ To specify class method for custom class, method needs to be annotated
with`#[classmethod]` attribute.
```rust
# #![feature(specialization)]
# use pyo3::prelude::*;
# #[pyclass]
# struct MyClass {
@ -321,7 +309,6 @@ with `#[staticmethod]` attribute. The return type must be `PyResult<T>`
for some `T` that implements `IntoPyObject`.
```rust
# #![feature(specialization)]
# use pyo3::prelude::*;
# #[pyclass]
# struct MyClass {
@ -344,7 +331,6 @@ To specify custom `__call__` method for custom class, call method needs to be an
with `#[call]` attribute. Arguments of the method are specified same as for instance method.
```rust
# #![feature(specialization)]
# use pyo3::prelude::*;
# #[pyclass]
# struct MyClass {
@ -387,7 +373,6 @@ Each parameter could one of following type:
Example:
```rust
# #![feature(specialization)]
# use pyo3::prelude::*;
#
# #[pyclass]
@ -556,3 +541,13 @@ impl PyIterProtocol for MyIterator {
}
}
```
## Manually implementing pyclass
TODO: Which traits to implement (basically `PyTypeCreate: PyObjectAlloc + PyTypeInfo + PyMethodsProtocol + Sized`) and what they mean.
## How methods are implemented
Users should be able to define a `#[pyclass]` with or without `#[pymethods]`, while pyo3 needs a trait with a function that returns all methods. Since it's impossible make the code generation in pyclass dependent on whether there is an impl block, we'd need to make to implement the trait on `#[pyclass]` and override the implementation in `#[pymethods]`, which is to my best knowledge only possible with the specialization feature, which is can't be used on stable.
To escape this we use [inventory](https://github.com/dtolnay/inventory), which allows us to collect `impl`s from arbitrary source code by exploiting some binary trick. See [inventory: how it works](https://github.com/dtolnay/inventory#how-it-works) and `pyo3_derive_backend::py_class::impl_inventory` for more details.

48
pyo3-derive-backend/src/py_class.rs
View file

@ -63,6 +63,40 @@ fn parse_descriptors(item: &mut syn::Field) -> Vec<FnType> {
descs
}
/// The orphan rule disallows using a generic inventory struct, so we create the whole boilerplate
/// once per class
fn impl_inventory(cls: &syn::Ident) -> TokenStream {
// Try to build a unique type that gives a hint about it's function when
// it comes up in error messages
let name = cls.to_string() + "GeneratedPyo3Inventory";
let inventory_cls = syn::Ident::new(&name, Span::call_site());
quote! {
#[doc(hidden)]
pub struct #inventory_cls {
methods: &'static [::pyo3::class::PyMethodDefType],
}
impl ::pyo3::class::methods::PyMethodsInventory for #inventory_cls {
fn new(methods: &'static [::pyo3::class::PyMethodDefType]) -> Self {
Self {
methods
}
}
fn get_methods(&self) -> &'static [::pyo3::class::PyMethodDefType] {
self.methods
}
}
impl ::pyo3::class::methods::PyMethodsInventoryDispatch for #cls {
type InventoryType = #inventory_cls;
}
::pyo3::inventory::collect!(#inventory_cls);
}
}
fn impl_class(
cls: &syn::Ident,
base: &syn::TypePath,
@ -136,6 +170,8 @@ fn impl_class(
quote! {0}
};
let inventory_impl = impl_inventory(&cls);
quote! {
impl ::pyo3::typeob::PyTypeInfo for #cls {
type Type = #cls;
@ -197,6 +233,8 @@ fn impl_class(
}
}
#inventory_impl
#extra
}
}
@ -287,12 +325,10 @@ fn impl_descriptors(cls: &syn::Type, descriptors: Vec<(syn::Field, Vec<FnType>)>
quote! {
#(#methods)*
impl ::pyo3::class::methods::PyPropMethodsProtocolImpl for #cls {
fn py_methods() -> &'static [::pyo3::class::PyMethodDefType] {
static METHODS: &'static [::pyo3::class::PyMethodDefType] = &[
#(#py_methods),*
];
METHODS
::pyo3::inventory::submit! {
#![crate = pyo3] {
type ClsInventory = <#cls as ::pyo3::class::methods::PyMethodsInventoryDispatch>::InventoryType;
<ClsInventory as ::pyo3::class::methods::PyMethodsInventory>::new(&[#(#py_methods),*])
}
}
}

10
pyo3-derive-backend/src/py_impl.rs
View file

@ -31,12 +31,10 @@ pub fn impl_methods(ty: &syn::Type, impls: &mut Vec<syn::ImplItem>) -> TokenStre
}
quote! {
impl ::pyo3::class::methods::PyMethodsProtocolImpl for #ty {
fn py_methods() -> &'static [::pyo3::class::PyMethodDefType] {
static METHODS: &'static [::pyo3::class::PyMethodDefType] = &[
#(#methods),*
];
METHODS
::pyo3::inventory::submit! {
#![crate = pyo3] {
type TyInventory = <#ty as ::pyo3::class::methods::PyMethodsInventoryDispatch>::InventoryType;
<TyInventory as ::pyo3::class::methods::PyMethodsInventory>::new(&[#(#methods),*])
}
}
}

48
src/class/methods.rs
View file

@ -58,10 +58,13 @@ pub struct PySetterDef {
}
unsafe impl Sync for PyMethodDef {}
unsafe impl Sync for ffi::PyMethodDef {}
unsafe impl Sync for PyGetterDef {}
unsafe impl Sync for PySetterDef {}
unsafe impl Sync for ffi::PyGetSetDef {}
impl PyMethodDef {
@ -110,21 +113,40 @@ impl PySetterDef {
}
}
#[doc(hidden)]
/// The pymethods macro implements this trait so the methods are added to the object
pub trait PyMethodsProtocolImpl {
fn py_methods() -> &'static [PyMethodDefType] {
&[]
}
#[doc(hidden)] // Only to be used through the proc macros, use PyMethodsProtocol in custom code
/// This trait is implemented for all pyclass so to implement the [PyMethodsProtocol]
/// through inventory
pub trait PyMethodsInventoryDispatch {
/// This allows us to get the inventory type when only the pyclass is in scope
type InventoryType: PyMethodsInventory;
}
impl<T> PyMethodsProtocolImpl for T {}
#[doc(hidden)] // Only to be used through the proc macros, use PyMethodsProtocol in custom code
/// Allows arbitrary pymethod blocks to submit their methods, which are eventually collected by pyclass
pub trait PyMethodsInventory: inventory::Collect {
/// Create a new instance
fn new(methods: &'static [PyMethodDefType]) -> Self;
#[doc(hidden)]
pub trait PyPropMethodsProtocolImpl {
fn py_methods() -> &'static [PyMethodDefType] {
&[]
}
/// Returns the methods for a single impl block
fn get_methods(&self) -> &'static [PyMethodDefType];
}
impl<T> PyPropMethodsProtocolImpl for T {}
/// The implementation of tis trait defines which methods a python type has.
///
/// For pyclass derived structs this is implemented by collecting all impl blocks through inventory
pub trait PyMethodsProtocol {
/// Returns all methods that are defined for a class
fn py_methods() -> Vec<&'static PyMethodDefType>;
}
impl<T> PyMethodsProtocol for T
where
T: PyMethodsInventoryDispatch,
{
fn py_methods() -> Vec<&'static PyMethodDefType> {
inventory::iter::<T::InventoryType>
.into_iter()
.flat_map(PyMethodsInventory::get_methods)
.collect()
}
}

14
src/lib.rs
View file

@ -55,8 +55,6 @@
//! **`src/lib.rs`**
//!
//! ```rust
//! #![feature(specialization)]
//!
//! use pyo3::prelude::*;
//! use pyo3::wrap_pyfunction;
//!
@ -101,8 +99,6 @@
//! Example program displaying the value of `sys.version`:
//!
//! ```rust
//! #![feature(specialization)]
//!
//! use pyo3::prelude::*;
//! use pyo3::types::PyDict;
//!
@ -146,12 +142,12 @@ pub use crate::pythonrun::{init_once, prepare_freethreaded_python, GILGuard, GIL
pub use crate::typeob::{PyObjectAlloc, PyRawObject, PyTypeInfo};
pub use crate::types::exceptions;
// We need those types in the macro exports
#[doc(hidden)]
pub use libc;
// We need that reexport for wrap_function
#[doc(hidden)]
pub use mashup;
// We need that reexport for pymethods
#[doc(hidden)]
pub use inventory;
/// Rust FFI declarations for Python
pub mod ffi;
@ -207,7 +203,7 @@ pub mod proc_macro {
macro_rules! wrap_pyfunction {
($function_name:ident) => {{
// Get the mashup macro and its helpers into scope
use $crate::mashup::*;
use pyo3::mashup::*;
mashup! {
// Make sure this ident matches the one in function_wrapper_ident
@ -227,7 +223,7 @@ macro_rules! wrap_pyfunction {
#[macro_export]
macro_rules! wrap_pymodule {
($module_name:ident) => {{
use $crate::mashup::*;
use pyo3::mashup::*;
mashup! {
m["method"] = PyInit_ $module_name;

83
src/typeob.rs
View file

@ -2,6 +2,12 @@
//! Python type object information
use std::collections::HashMap;
use std::ffi::CString;
use std::os::raw::c_void;
use class::methods::PyMethodsProtocol;
use crate::class::methods::PyMethodDefType;
use crate::err::{PyErr, PyResult};
use crate::instance::{Py, PyObjectWithGIL};
@ -10,9 +16,6 @@ use crate::python::{IntoPyPointer, Python};
use crate::types::PyObjectRef;
use crate::types::PyType;
use crate::{class, ffi, pythonrun};
use std::collections::HashMap;
use std::ffi::CString;
use std::os::raw::c_void;
/// Python type information.
pub trait PyTypeInfo {
@ -71,8 +74,6 @@ pub const PY_TYPE_FLAG_DICT: usize = 1 << 3;
///
/// Example of custom class implementation with `__new__` method:
/// ```
/// #![feature(specialization)]
///
/// use pyo3::prelude::*;
///
/// #[pyclass]
@ -198,9 +199,6 @@ pub(crate) unsafe fn pytype_drop<T: PyTypeInfo>(py: Python, obj: *mut ffi::PyObj
/// [PyObjectWithFreeList](crate::freelist::PyObjectWithFreeList) gets a special version.
pub trait PyObjectAlloc: PyTypeInfo + Sized {
unsafe fn alloc(_py: Python) -> PyResult<*mut ffi::PyObject> {
// TODO: remove this
<Self as PyTypeCreate>::init_type();
let tp_ptr = Self::type_object();
let alloc = (*tp_ptr).tp_alloc.unwrap_or(ffi::PyType_GenericAlloc);
let obj = alloc(tp_ptr, 0);
@ -259,20 +257,7 @@ pub trait PyTypeObject {
/// Python object types that have a corresponding type object and be
/// instanciated with [Self::create()]
pub trait PyTypeCreate: PyObjectAlloc + PyTypeInfo + Sized {
#[inline]
fn init_type() {
let type_object = unsafe { *<Self as PyTypeInfo>::type_object() };
if (type_object.tp_flags & ffi::Py_TPFLAGS_READY) == 0 {
// automatically initialize the class on-demand
let gil = Python::acquire_gil();
let py = gil.python();
initialize_type::<Self>(py, None).unwrap_or_else(|_| {
panic!("An error occurred while initializing class {}", Self::NAME)
});
}
}
fn init_type();
#[inline]
fn type_object() -> Py<PyType> {
@ -297,7 +282,25 @@ pub trait PyTypeCreate: PyObjectAlloc + PyTypeInfo + Sized {
}
}
impl<T> PyTypeCreate for T where T: PyObjectAlloc + PyTypeInfo + Sized {}
impl<T> PyTypeCreate for T
where
T: PyObjectAlloc + PyTypeInfo + PyMethodsProtocol,
{
#[inline]
fn init_type() {
let type_object = unsafe { *<Self as PyTypeInfo>::type_object() };
if (type_object.tp_flags & ffi::Py_TPFLAGS_READY) == 0 {
// automatically initialize the class on-demand
let gil = Python::acquire_gil();
let py = gil.python();
initialize_type::<Self>(py).unwrap_or_else(|_| {
panic!("An error occurred while initializing class {}", Self::NAME)
});
}
}
}
impl<T> PyTypeObject for T
where
@ -313,25 +316,20 @@ where
}
/// Register new type in python object system.
///
/// Currently, module_name is always None, so it defaults to pyo3_extension
#[cfg(not(Py_LIMITED_API))]
pub fn initialize_type<T>(py: Python, module_name: Option<&str>) -> PyResult<()>
pub fn initialize_type<T>(py: Python) -> PyResult<*mut ffi::PyTypeObject>
where
T: PyObjectAlloc + PyTypeInfo,
T: PyObjectAlloc + PyTypeInfo + PyMethodsProtocol,
{
// type name
let name = match module_name {
Some(module_name) => CString::new(format!("{}.{}", module_name, T::NAME)),
None => CString::new(T::NAME),
};
let name = name
.expect("Module name/type name must not contain NUL byte")
.into_raw();
let type_name = CString::new(T::NAME).expect("class name must not contain NUL byte");
let type_object: &mut ffi::PyTypeObject = unsafe { &mut *T::type_object() };
let type_object: &mut ffi::PyTypeObject = unsafe { T::type_object() };
let base_type_object: &mut ffi::PyTypeObject =
unsafe { &mut *<T::BaseType as PyTypeInfo>::type_object() };
unsafe { <T::BaseType as PyTypeInfo>::type_object() };
type_object.tp_name = name;
type_object.tp_name = type_name.into_raw();
type_object.tp_doc = T::DESCRIPTION.as_ptr() as *const _;
type_object.tp_base = base_type_object;
@ -431,7 +429,7 @@ where
// register type object
unsafe {
if ffi::PyType_Ready(type_object) == 0 {
Ok(())
Ok(type_object as *mut ffi::PyTypeObject)
} else {
PyErr::fetch(py).into()
}
@ -493,7 +491,7 @@ fn py_class_flags<T: PyTypeInfo>(type_object: &mut ffi::PyTypeObject) {
}
}
fn py_class_method_defs<T>() -> PyResult<(
fn py_class_method_defs<T: PyMethodsProtocol>() -> PyResult<(
Option<ffi::newfunc>,
Option<ffi::initproc>,
Option<ffi::PyCFunctionWithKeywords>,
@ -504,7 +502,7 @@ fn py_class_method_defs<T>() -> PyResult<(
let mut new = None;
let mut init = None;
for def in <T as class::methods::PyMethodsProtocolImpl>::py_methods() {
for def in T::py_methods() {
match *def {
PyMethodDefType::New(ref def) => {
if let class::methods::PyMethodType::PyNewFunc(meth) = def.ml_meth {
@ -565,13 +563,10 @@ fn py_class_async_methods<T>(defs: &mut Vec<ffi::PyMethodDef>) {
#[cfg(not(Py_3))]
fn py_class_async_methods<T>(_defs: &mut Vec<ffi::PyMethodDef>) {}
fn py_class_properties<T>() -> Vec<ffi::PyGetSetDef> {
fn py_class_properties<T: PyMethodsProtocol>() -> Vec<ffi::PyGetSetDef> {
let mut defs = HashMap::new();
for def in <T as class::methods::PyMethodsProtocolImpl>::py_methods()
.iter()
.chain(<T as class::methods::PyPropMethodsProtocolImpl>::py_methods().iter())
{
for def in T::py_methods() {
match *def {
PyMethodDefType::Getter(ref getter) => {
let name = getter.name.to_string();

23
src/types/module.rs
View file

@ -9,9 +9,8 @@ use crate::instance::PyObjectWithGIL;
use crate::object::PyObject;
use crate::objectprotocol::ObjectProtocol;
use crate::python::{Python, ToPyPointer};
use crate::typeob::{initialize_type, PyTypeInfo};
use crate::types::{exceptions, PyDict, PyObjectRef, PyType};
use crate::PyObjectAlloc;
use crate::typeob::PyTypeCreate;
use crate::types::{exceptions, PyDict, PyObjectRef};
use std::ffi::{CStr, CString};
use std::os::raw::c_char;
use std::str;
@ -150,23 +149,9 @@ impl PyModule {
/// and adds the type to this module.
pub fn add_class<T>(&self) -> PyResult<()>
where
T: PyTypeInfo + PyObjectAlloc,
T: PyTypeCreate,
{
let ty = unsafe {
let ty = <T as PyTypeInfo>::type_object();
if ((*ty).tp_flags & ffi::Py_TPFLAGS_READY) != 0 {
PyType::new::<T>()
} else {
// automatically initialize the class
initialize_type::<T>(self.py(), Some(self.name()?)).unwrap_or_else(|_| {
panic!("An error occurred while initializing class {}", T::NAME)
});
PyType::new::<T>()
}
};
self.setattr(T::NAME, ty)
self.setattr(T::NAME, <T as PyTypeCreate>::type_object())
}
/// Adds a function or a (sub)module to a module, using the functions __name__ as name.

3
src/types/string2.rs
View file

@ -2,6 +2,7 @@
//
// based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython
use super::PyObjectRef;
use crate::err::{PyErr, PyResult};
use crate::ffi;
use crate::instance::{Py, PyObjectWithGIL};
@ -14,8 +15,6 @@ use std::borrow::Cow;
use std::os::raw::c_char;
use std::str;
use super::PyObjectRef;
/// Represents a Python `string`.
#[repr(transparent)]
pub struct PyString(PyObject);

20
tests/test_class_basics.rs
View file

@ -66,11 +66,17 @@ fn empty_class_in_module() {
ty.getattr("__name__").unwrap().extract::<String>().unwrap(),
"EmptyClassInModule"
);
assert_eq!(
ty.getattr("__module__")
.unwrap()
.extract::<String>()
.unwrap(),
"test_module.nested"
);
let builtin = if cfg!(feature = "python2") {
"__builtin__"
} else {
"builtins"
};
let module: String = ty.getattr("__module__").unwrap().extract().unwrap();
// Rationale: The class can be added to many modules, but will only be initialized once.
// We currently have no way of determining a canonical module, so builtins is better
// than using whatever calls init first.
assert_eq!(module, builtin);
}

2
tests/test_module.rs
View file

@ -29,7 +29,7 @@ fn double(x: usize) -> usize {
#[pymodule]
#[cfg(Py_3)]
fn module_with_functions(py: Python, m: &PyModule) -> PyResult<()> {
use pyo3::{wrap_pyfunction, wrap_pymodule};
use pyo3::wrap_pyfunction;
#[pyfn(m, "sum_as_string")]
fn sum_as_string_py(_py: Python, a: i64, b: i64) -> PyResult<String> {