luxolus
/
pyo3
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1457 from davidhewitt/multiple-pymethods 

pymethods: make inventory optional
This commit is contained in:
David Hewitt 2021-03-06 23:09:02 +00:00 committed by GitHub
parent fa50c11ac4 bb6d4df497
commit a45f520ec1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 386 additions and 175 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
.github/workflows/ci.yml vendored
View File

@ -88,7 +88,7 @@ jobs:
id: settings
shell: bash
run: |
echo "::set-output name=all_additive_features::macros num-bigint num-complex hashbrown serde"
echo "::set-output name=all_additive_features::macros num-bigint num-complex hashbrown serde multiple-pymethods"
- name: Build docs
run: cargo doc --no-default-features --features "${{ steps.settings.outputs.all_additive_features }}"
@ -156,16 +156,15 @@ jobs:
toolchain: nightly
override: true
profile: minimal
- uses: actions-rs/cargo@v1
with:
command: test
args: --features "num-bigint num-complex hashbrown serde" --no-fail-fast
env:
CARGO_INCREMENTAL: 0
RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=abort -Zpanic_abort_tests"
RUSTDOCFLAGS: "-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=abort -Zpanic_abort_tests"
- run: cargo test --no-default-features --no-fail-fast
- run: cargo test --no-default-features --no-fail-fast --features "macros num-bigint num-complex hashbrown serde multiple-pymethods"
- uses: actions-rs/grcov@v0.1
id: coverage
- uses: codecov/codecov-action@v1
with:
file: ${{ steps.coverage.outputs.report }}
env:
CARGO_TERM_VERBOSE: true
CARGO_INCREMENTAL: 0
RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=abort -Zpanic_abort_tests"
RUSTDOCFLAGS: "-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=abort -Zpanic_abort_tests"

5
Cargo.toml
View File

@ -43,7 +43,10 @@ serde_json = "1.0.61"
default = ["macros"]
# Enables macros: #[pyclass], #[pymodule], #[pyfunction] etc.
macros = ["pyo3-macros", "indoc", "inventory", "paste", "unindent"]
macros = ["pyo3-macros", "indoc", "paste", "unindent"]
# Enables multiple #[pymethods] per #[pyclass]
multiple-pymethods = ["inventory"]
# Use this feature when building an extension module.
# It tells the linker to keep the python symbols unresolved,

60
guide/src/class.md
View File

@ -1,8 +1,10 @@
# Python Classes
PyO3 exposes a group of attributes powered by Rust's proc macro system for defining Python classes as Rust structs. This chapter will discuss the functionality and configuration they offer.
PyO3 exposes a group of attributes powered by Rust's proc macro system for defining Python classes as Rust structs.
For ease of discovery, below is a list of all custom attributes with links to the relevant section of this chapter:
The main attribute is `#[pyclass]`, which is placed upon a Rust `struct` to generate a Python type for it. A struct will usually also have *one* `#[pymethods]`-annotated `impl` block for the struct, which is used to define Python methods and constants for the generated Python type. (If the [`multiple-pymethods`] feature is enabled each `#[pyclass]` is allowed to have multiple `#[pymethods]` blocks.) Finally, there may be multiple `#[pyproto]` trait implementations for the struct, which are used to define certain python magic methods such as `__str__`.
This chapter will discuss the functionality and configuration these attributes offer. Below is a list of links to the relevant section of this chapter for each:
- [`#[pyclass]`](#defining-a-new-class)
- [`#[pyo3(get, set)]`](#object-properties-using-pyo3get-set)
@ -31,9 +33,9 @@ struct MyClass {
}
```
Because Python objects are freely shared between threads by the Python interpreter, all structs annotated with `#[pyclass]` must implement `Send`.
Because Python objects are freely shared between threads by the Python interpreter, all structs annotated with `#[pyclass]` must implement `Send` (unless annotated with [`#[pyclass(unsendable)]`](#customizing-the-class)).
The above example generates implementations for [`PyTypeInfo`], [`PyTypeObject`], and [`PyClass`] for `MyClass`. To see these generated implementations, refer to the section [How methods are implemented](#how-methods-are-implemented) at the end of this chapter.
The above example generates implementations for [`PyTypeInfo`], [`PyTypeObject`], and [`PyClass`] for `MyClass`. To see these generated implementations, refer to the [implementation details](#implementation-details) at the end of this chapter.
## Adding the class to a module
@ -332,7 +334,7 @@ impl SubClass {
PyO3 supports two ways to add properties to your `#[pyclass]`:
- For simple fields with no side effects, a `#[pyo3(get, set)]` attribute can be added directly to the field definition in the `#[pyclass]`.
- For properties which require computation you can define `#[getter]` and `#[setter]` functions in the `#[pymethods]` block.
- For properties which require computation you can define `#[getter]` and `#[setter]` functions in the [`#[pymethods]`](#instance-methods) block.
We'll cover each of these in the following sections.
@ -444,7 +446,8 @@ To define a Python compatible method, an `impl` block for your struct has to be
block with some variations, like descriptors, class method static methods, etc.
Since Rust allows any number of `impl` blocks, you can easily split methods
between those accessible to Python (and Rust) and those accessible only to Rust.
between those accessible to Python (and Rust) and those accessible only to Rust. However to have multiple
`#[pymethods]`-annotated `impl` blocks for the same struct you must enable the [`multiple-pymethods`] feature of PyO3.
```rust
# use pyo3::prelude::*;
@ -698,20 +701,21 @@ num=44, debug=false
num=-1, debug=false
```
## How methods are implemented
## Implementation details
Users should be able to define a `#[pyclass]` with or without `#[pymethods]`, while PyO3 needs a
trait with a function that returns all methods. Since it's impossible to make the code generation in
pyclass dependent on whether there is an impl block, we'd need to implement the trait on
`#[pyclass]` and override the implementation in `#[pymethods]`.
To enable this, we use a static registry type provided by [inventory](https://github.com/dtolnay/inventory),
which allows us to collect `impl`s from arbitrary source code by exploiting some binary trick.
See [inventory: how it works](https://github.com/dtolnay/inventory#how-it-works) and `pyo3_macros_backend::py_class` for more details.
The `#[pyclass]` macros rely on a lot of conditional code generation: each `#[pyclass]` can optionally have a `#[pymethods]` block as well as several different possible `#[pyproto]` trait implementations.
Specifically, the following implementation is generated:
To support this flexibility the `#[pyclass]` macro expands to a blob of boilerplate code which sets up the structure for ["dtolnay specialization"](https://github.com/dtolnay/case-studies/blob/master/autoref-specialization/README.md). This implementation pattern enables the Rust compiler to use `#[pymethods]` and `#[pyproto]` implementations when they are present, and fall back to default (empty) definitions when they are not.
This simple technique works for the case when there is zero or one implementations. To support multiple `#[pymethods]` for a `#[pyclass]` (in the [`multiple-pymethods`] feature), a registry mechanism provided by the [`inventory`](https://github.com/dtolnay/inventory) crate is used instead. This collects `impl`s at library load time, but isn't supported on all platforms. See [inventory: how it works](https://github.com/dtolnay/inventory#how-it-works) for more details.
The `#[pyclass]` macro expands to roughly the code seen below. The `PyClassImplCollector` is the type used internally by PyO3 for dtolnay specialization:
```rust
use pyo3::prelude::*;
# #[cfg(not(feature = "multiple-pymethods"))]
# {
# use pyo3::prelude::*;
// Note: the implementation differs slightly with the `multiple-pymethods` feature enabled.
/// Class for demonstration
struct MyClass {
@ -754,31 +758,14 @@ impl pyo3::IntoPy<PyObject> for MyClass {
}
}
pub struct Pyo3MethodsInventoryForMyClass {
methods: Vec<pyo3::class::PyMethodDefType>,
}
impl pyo3::class::methods::PyMethodsInventory for Pyo3MethodsInventoryForMyClass {
fn new(methods: Vec<pyo3::class::PyMethodDefType>) -> Self {
Self { methods }
}
fn get(&'static self) -> &'static [pyo3::class::PyMethodDefType] {
&self.methods
}
}
impl pyo3::class::methods::HasMethodsInventory for MyClass {
type Methods = Pyo3MethodsInventoryForMyClass;
}
pyo3::inventory::collect!(Pyo3MethodsInventoryForMyClass);
impl pyo3::class::impl_::PyClassImpl for MyClass {
type ThreadChecker = pyo3::class::impl_::ThreadCheckerStub<MyClass>;
fn for_each_method_def(visitor: impl FnMut(&pyo3::class::PyMethodDefType)) {
use pyo3::class::impl_::*;
let collector = PyClassImplCollector::<MyClass>::new();
pyo3::inventory::iter::<<MyClass as pyo3::class::methods::HasMethodsInventory>::Methods>
.into_iter()
.flat_map(pyo3::class::methods::PyMethodsInventory::get)
collector.py_methods().iter()
.chain(collector.py_class_descriptors())
.chain(collector.object_protocol_methods())
.chain(collector.async_protocol_methods())
.chain(collector.context_protocol_methods())
@ -824,6 +811,7 @@ impl pyo3::class::impl_::PyClassImpl for MyClass {
# let py = gil.python();
# let cls = py.get_type::<MyClass>();
# pyo3::py_run!(py, cls, "assert cls.__name__ == 'MyClass'")
# }
```
@ -840,3 +828,5 @@ impl pyo3::class::impl_::PyClassImpl for MyClass {
[`RefCell`]: https://doc.rust-lang.org/std/cell/struct.RefCell.html
[classattr]: https://docs.python.org/3/tutorial/classes.html#class-and-instance-variables
[`multiple-pymethods`]: features.md#multiple-pymethods

8
guide/src/features.md
View File

@ -55,6 +55,14 @@ These macros require a number of dependencies which may not be needed by users w
> This feature is enabled by default. To disable it, set `default-features = false` for the `pyo3` entry in your Cargo.toml.
### `multiple-pymethods`
This feature enables a dependency on `inventory`, which enables each `#[pyclass]` to have more than one `#[pymethods]` block.
Most users should only need a single `#[pymethods]` per `#[pyclass]`. In addition, not all platforms (e.g. Wasm) are supported by `inventory`. For this reason this feature is not enabled by default, meaning fewer dependencies and faster compilation for the majority of users.
See [the `#[pyclass]` implementation details](class.md#implementation-details) for more information.
### `nightly`
The `nightly` feature needs the nightly Rust compiler. This allows PyO3 to use Rust's unstable specialization feature to apply the following optimizations:

6
guide/src/migration.md
View File

@ -9,6 +9,12 @@ For a detailed list of all changes, see the [CHANGELOG](changelog.md).
For projects embedding Python in Rust, PyO3 no longer automatically initalizes a Python interpreter on the first call to `Python::with_gil` (or `Python::acquire_gil`) unless the [`auto-initalize` feature](features.md#auto-initalize) is enabled.
### New `multiple-pymethods` feature
`#[pymethods]` have been reworked with a simpler default implementation which removes the dependency on the `inventory` crate. This reduces dependencies and compile times for the majority of users.
The limitation of the new default implementation is that it cannot support multiple `#[pymethods]` blocks for the same `#[pyclass]`. If you need this functionality, you must enable the `multiple-pymethods` feature which will switch `#[pymethods]` to the inventory-based implementation.
## from 0.12.* to 0.13
### Minimum Rust version increased to Rust 1.45

2
pyo3-macros-backend/src/lib.rs
View File

@ -24,6 +24,6 @@ pub use from_pyobject::build_derive_from_pyobject;
pub use module::{add_fn_to_module, process_functions_in_module, py_init};
pub use pyclass::{build_py_class, PyClassArgs};
pub use pyfunction::{build_py_function, PyFunctionAttr};
pub use pyimpl::build_py_methods;
pub use pyimpl::{build_py_methods, PyClassMethodsType};
pub use pyproto::build_py_proto;
pub use utils::get_doc;

43
pyo3-macros-backend/src/pyclass.rs
View File

@ -1,6 +1,7 @@
// Copyright (c) 2017-present PyO3 Project and Contributors
use crate::method::{FnType, SelfType};
use crate::pyimpl::PyClassMethodsType;
use crate::pymethod::{
impl_py_getter_def, impl_py_setter_def, impl_wrap_getter, impl_wrap_setter, PropertyType,
};
@ -154,7 +155,11 @@ impl PyClassArgs {
}
}
pub fn build_py_class(class: &mut syn::ItemStruct, attr: &PyClassArgs) -> syn::Result<TokenStream> {
pub fn build_py_class(
class: &mut syn::ItemStruct,
attr: &PyClassArgs,
methods_type: PyClassMethodsType,
) -> syn::Result<TokenStream> {
let text_signature = utils::parse_text_signature_attrs(
&mut class.attrs,
&get_class_python_name(&class.ident, attr),
@ -178,7 +183,7 @@ pub fn build_py_class(class: &mut syn::ItemStruct, attr: &PyClassArgs) -> syn::R
bail_spanned!(class.fields.span() => "#[pyclass] can only be used with C-style structs");
}
impl_class(&class.ident, &attr, doc, descriptors)
impl_class(&class.ident, &attr, doc, descriptors, methods_type)
}
/// Parses `#[pyo3(get, set)]`
@ -210,7 +215,7 @@ fn parse_descriptors(item: &mut syn::Field) -> syn::Result<Vec<FnType>> {
Ok(descs)
}
/// To allow multiple #[pymethods]/#[pyproto] block, we define inventory types.
/// To allow multiple #[pymethods] block, we define inventory types.
fn impl_methods_inventory(cls: &syn::Ident) -> TokenStream {
// Try to build a unique type for better error messages
let name = format!("Pyo3MethodsInventoryFor{}", cls.unraw());
@ -221,7 +226,7 @@ fn impl_methods_inventory(cls: &syn::Ident) -> TokenStream {
pub struct #inventory_cls {
methods: Vec<pyo3::class::PyMethodDefType>,
}
impl pyo3::class::methods::PyMethodsInventory for #inventory_cls {
impl pyo3::class::impl_::PyMethodsInventory for #inventory_cls {
fn new(methods: Vec<pyo3::class::PyMethodDefType>) -> Self {
Self { methods }
}
@ -230,7 +235,7 @@ fn impl_methods_inventory(cls: &syn::Ident) -> TokenStream {
}
}
impl pyo3::class::methods::HasMethodsInventory for #cls {
impl pyo3::class::impl_::HasMethodsInventory for #cls {
type Methods = #inventory_cls;
}
@ -247,6 +252,7 @@ fn impl_class(
attr: &PyClassArgs,
doc: syn::LitStr,
descriptors: Vec<(syn::Field, Vec<FnType>)>,
methods_type: PyClassMethodsType,
) -> syn::Result<TokenStream> {
let cls_name = get_class_python_name(cls, attr).to_string();
@ -338,7 +344,17 @@ fn impl_class(
quote! {}
};
let impl_inventory = impl_methods_inventory(&cls);
let (impl_inventory, iter_py_methods) = match methods_type {
PyClassMethodsType::Specialization => (None, quote! { collector.py_methods().iter() }),
PyClassMethodsType::Inventory => (
Some(impl_methods_inventory(&cls)),
quote! {
pyo3::inventory::iter::<<Self as pyo3::class::impl_::HasMethodsInventory>::Methods>
.into_iter()
.flat_map(pyo3::class::impl_::PyMethodsInventory::get)
},
),
};
let base = &attr.base;
let flags = &attr.flags;
@ -429,9 +445,8 @@ fn impl_class(
fn for_each_method_def(visitor: impl FnMut(&pyo3::class::PyMethodDefType)) {
use pyo3::class::impl_::*;
let collector = PyClassImplCollector::<Self>::new();
pyo3::inventory::iter::<<Self as pyo3::class::methods::HasMethodsInventory>::Methods>
.into_iter()
.flat_map(pyo3::class::methods::PyMethodsInventory::get)
#iter_py_methods
.chain(collector.py_class_descriptors())
.chain(collector.object_protocol_methods())
.chain(collector.async_protocol_methods())
.chain(collector.context_protocol_methods())
@ -513,10 +528,12 @@ fn impl_descriptors(
.collect::<syn::Result<_>>()?;
Ok(quote! {
pyo3::inventory::submit! {
#![crate = pyo3] {
type Inventory = <#cls as pyo3::class::methods::HasMethodsInventory>::Methods;
<Inventory as pyo3::class::methods::PyMethodsInventory>::new(vec![#(#py_methods),*])
impl pyo3::class::impl_::PyClassDescriptors<#cls>
for pyo3::class::impl_::PyClassImplCollector<#cls>
{
fn py_class_descriptors(self) -> &'static [pyo3::class::methods::PyMethodDefType] {
static METHODS: &[pyo3::class::methods::PyMethodDefType] = &[#(#py_methods),*];
METHODS
}
}
})

57
pyo3-macros-backend/src/pyimpl.rs
View File

@ -6,7 +6,16 @@ use pymethod::GeneratedPyMethod;
use quote::quote;
use syn::spanned::Spanned;
pub fn build_py_methods(ast: &mut syn::ItemImpl) -> syn::Result<TokenStream> {
/// The mechanism used to collect `#[pymethods]` into the type object
pub enum PyClassMethodsType {
Specialization,
Inventory,
}
pub fn build_py_methods(
ast: &mut syn::ItemImpl,
methods_type: PyClassMethodsType,
) -> syn::Result<TokenStream> {
if let Some((_, path, _)) = &ast.trait_ {
bail_spanned!(path.span() => "#[pymethods] cannot be used on trait impl blocks");
} else if ast.generics != Default::default() {
@ -15,11 +24,15 @@ pub fn build_py_methods(ast: &mut syn::ItemImpl) -> syn::Result<TokenStream> {
"#[pymethods] cannot be used with lifetime parameters or generics"
);
} else {
impl_methods(&ast.self_ty, &mut ast.items)
impl_methods(&ast.self_ty, &mut ast.items, methods_type)
}
}
pub fn impl_methods(ty: &syn::Type, impls: &mut Vec<syn::ImplItem>) -> syn::Result<TokenStream> {
pub fn impl_methods(
ty: &syn::Type,
impls: &mut Vec<syn::ImplItem>,
methods_type: PyClassMethodsType,
) -> syn::Result<TokenStream> {
let mut new_impls = Vec::new();
let mut call_impls = Vec::new();
let mut methods = Vec::new();
@ -51,18 +64,46 @@ pub fn impl_methods(ty: &syn::Type, impls: &mut Vec<syn::ImplItem>) -> syn::Resu
}
}
let methods_registration = match methods_type {
PyClassMethodsType::Specialization => impl_py_methods(ty, methods),
PyClassMethodsType::Inventory => submit_methods_inventory(ty, methods),
};
Ok(quote! {
#(#new_impls)*
#(#call_impls)*
pyo3::inventory::submit! {
#![crate = pyo3] {
type Inventory = <#ty as pyo3::class::methods::HasMethodsInventory>::Methods;
<Inventory as pyo3::class::methods::PyMethodsInventory>::new(vec![#(#methods),*])
#methods_registration
})
}
fn impl_py_methods(ty: &syn::Type, methods: Vec<TokenStream>) -> TokenStream {
quote! {
impl pyo3::class::impl_::PyMethods<#ty>
for pyo3::class::impl_::PyClassImplCollector<#ty>
{
fn py_methods(self) -> &'static [pyo3::class::methods::PyMethodDefType] {
static METHODS: &[pyo3::class::methods::PyMethodDefType] = &[#(#methods),*];
METHODS
}
}
})
}
}
fn submit_methods_inventory(ty: &syn::Type, methods: Vec<TokenStream>) -> TokenStream {
if methods.is_empty() {
return TokenStream::default();
}
quote! {
pyo3::inventory::submit! {
#![crate = pyo3] {
type Inventory = <#ty as pyo3::class::impl_::HasMethodsInventory>::Methods;
<Inventory as pyo3::class::impl_::PyMethodsInventory>::new(vec![#(#methods),*])
}
}
}
}
fn get_cfg_attributes(attrs: &[syn::Attribute]) -> Vec<&syn::Attribute> {

22
pyo3-macros-backend/src/pymethod.rs
View File

@ -689,7 +689,10 @@ pub fn impl_py_method_class_attribute(spec: &FnSpec<'_>, wrapper: &TokenStream)
pyo3::class::PyMethodDefType::ClassAttribute({
#wrapper
pyo3::class::PyClassAttributeDef::new(concat!(stringify!(#python_name), "\0"), __wrap)
pyo3::class::PyClassAttributeDef::new(
concat!(stringify!(#python_name), "\0"),
pyo3::class::methods::PyClassAttributeFactory(__wrap)
)
})
}
}
@ -700,7 +703,10 @@ pub fn impl_py_const_class_attribute(spec: &ConstSpec, wrapper: &TokenStream) ->
pyo3::class::PyMethodDefType::ClassAttribute({
#wrapper
pyo3::class::PyClassAttributeDef::new(concat!(stringify!(#python_name), "\0"), __wrap)
pyo3::class::PyClassAttributeDef::new(
concat!(stringify!(#python_name), "\0"),
pyo3::class::methods::PyClassAttributeFactory(__wrap)
)
})
}
}
@ -726,7 +732,11 @@ pub(crate) fn impl_py_setter_def(
pyo3::class::PyMethodDefType::Setter({
#wrapper
pyo3::class::PySetterDef::new(concat!(stringify!(#python_name), "\0"), __wrap, #doc)
pyo3::class::PySetterDef::new(
concat!(stringify!(#python_name), "\0"),
pyo3::class::methods::PySetter(__wrap),
#doc
)
})
}
}
@ -740,7 +750,11 @@ pub(crate) fn impl_py_getter_def(
pyo3::class::PyMethodDefType::Getter({
#wrapper
pyo3::class::PyGetterDef::new(concat!(stringify!(#python_name), "\0"), __wrap, #doc)
pyo3::class::PyGetterDef::new(
concat!(stringify!(#python_name), "\0"),
pyo3::class::methods::PyGetter(__wrap),
#doc
)
})
}
}

57
pyo3-macros/src/lib.rs
View File

@ -7,7 +7,8 @@ extern crate proc_macro;
use proc_macro::TokenStream;
use pyo3_macros_backend::{
build_derive_from_pyobject, build_py_class, build_py_function, build_py_methods,
build_py_proto, get_doc, process_functions_in_module, py_init, PyClassArgs, PyFunctionAttr,
build_py_proto, get_doc, process_functions_in_module, py_init, PyClassArgs, PyClassMethodsType,
PyFunctionAttr,
};
use quote::quote;
use syn::parse_macro_input;
@ -56,27 +57,22 @@ pub fn pyproto(_: TokenStream, input: TokenStream) -> TokenStream {
#[proc_macro_attribute]
pub fn pyclass(attr: TokenStream, input: TokenStream) -> TokenStream {
let mut ast = parse_macro_input!(input as syn::ItemStruct);
let args = parse_macro_input!(attr as PyClassArgs);
let expanded = build_py_class(&mut ast, &args).unwrap_or_else(|e| e.to_compile_error());
pyclass_impl(attr, input, PyClassMethodsType::Specialization)
}
quote!(
#ast
#expanded
)
.into()
#[proc_macro_attribute]
pub fn pyclass_with_inventory(attr: TokenStream, input: TokenStream) -> TokenStream {
pyclass_impl(attr, input, PyClassMethodsType::Inventory)
}
#[proc_macro_attribute]
pub fn pymethods(_: TokenStream, input: TokenStream) -> TokenStream {
let mut ast = parse_macro_input!(input as syn::ItemImpl);
let expanded = build_py_methods(&mut ast).unwrap_or_else(|e| e.to_compile_error());
pymethods_impl(input, PyClassMethodsType::Specialization)
}
quote!(
#ast
#expanded
)
.into()
#[proc_macro_attribute]
pub fn pymethods_with_inventory(_: TokenStream, input: TokenStream) -> TokenStream {
pymethods_impl(input, PyClassMethodsType::Inventory)
}
#[proc_macro_attribute]
@ -102,3 +98,32 @@ pub fn derive_from_py_object(item: TokenStream) -> TokenStream {
)
.into()
}
fn pyclass_impl(
attr: TokenStream,
input: TokenStream,
methods_type: PyClassMethodsType,
) -> TokenStream {
let mut ast = parse_macro_input!(input as syn::ItemStruct);
let args = parse_macro_input!(attr as PyClassArgs);
let expanded =
build_py_class(&mut ast, &args, methods_type).unwrap_or_else(|e| e.to_compile_error());
quote!(
#ast
#expanded
)
.into()
}
fn pymethods_impl(input: TokenStream, methods_type: PyClassMethodsType) -> TokenStream {
let mut ast = parse_macro_input!(input as syn::ItemImpl);
let expanded =
build_py_methods(&mut ast, methods_type).unwrap_or_else(|e| e.to_compile_error());
quote!(
#ast
#expanded
)
.into()
}

58
src/class/impl_.rs
View File

@ -76,6 +76,50 @@ impl<T> PyClassCallImpl<T> for &'_ PyClassImplCollector<T> {
}
}
// General methods implementation: either dtolnay specialization trait or inventory if
// multiple-pymethods feature is enabled.
macro_rules! methods_trait {
($name:ident, $function_name: ident) => {
pub trait $name<T> {
fn $function_name(self) -> &'static [PyMethodDefType];
}
impl<T> $name<T> for &'_ PyClassImplCollector<T> {
fn $function_name(self) -> &'static [PyMethodDefType] {
&[]
}
}
};
}
/// Implementation detail. Only to be used through our proc macro code.
/// Method storage for `#[pyclass]`.
/// Allows arbitrary `#[pymethod]` blocks to submit their methods,
/// which are eventually collected by `#[pyclass]`.
#[cfg(all(feature = "macros", feature = "multiple-pymethods"))]
pub trait PyMethodsInventory: inventory::Collect {
/// Create a new instance
fn new(methods: Vec<PyMethodDefType>) -> Self;
/// Returns the methods for a single `#[pymethods] impl` block
fn get(&'static self) -> &'static [PyMethodDefType];
}
/// Implemented for `#[pyclass]` in our proc macro code.
/// Indicates that the pyclass has its own method storage.
#[cfg(all(feature = "macros", feature = "multiple-pymethods"))]
pub trait HasMethodsInventory {
type Methods: PyMethodsInventory;
}
// Methods from #[pyo3(get, set)] on struct fields.
methods_trait!(PyClassDescriptors, py_class_descriptors);
// Methods from #[pymethods] if not using inventory.
#[cfg(not(feature = "multiple-pymethods"))]
methods_trait!(PyMethods, py_methods);
// All traits describing slots, as well as the fallback implementations for unimplemented protos
//
// Protos which are implemented use dtolnay specialization to implement for PyClassImplCollector<T>.
@ -106,20 +150,6 @@ slots_trait!(PyAsyncProtocolSlots, async_protocol_slots);
slots_trait!(PySequenceProtocolSlots, sequence_protocol_slots);
slots_trait!(PyBufferProtocolSlots, buffer_protocol_slots);
macro_rules! methods_trait {
($name:ident, $function_name: ident) => {
pub trait $name<T> {
fn $function_name(self) -> &'static [PyMethodDefType];
}
impl<T> $name<T> for &'_ PyClassImplCollector<T> {
fn $function_name(self) -> &'static [PyMethodDefType] {
&[]
}
}
};
}
methods_trait!(PyObjectProtocolMethods, object_protocol_methods);
methods_trait!(PyAsyncProtocolMethods, async_protocol_methods);
methods_trait!(PyContextProtocolMethods, context_protocol_methods);

98
src/class/methods.rs
View File

@ -1,7 +1,8 @@
// Copyright (c) 2017-present PyO3 Project and Contributors
use crate::internal_tricks::{extract_cstr_or_leak_cstring, NulByteInString};
use crate::{ffi, PyObject, Python};
use std::ffi::{CStr, CString};
use std::ffi::CStr;
use std::fmt;
use std::os::raw::c_int;
@ -29,16 +30,22 @@ pub enum PyMethodType {
PyCFunctionWithKeywords(PyCFunctionWithKeywords),
}
// These two newtype structs serve no purpose other than wrapping the raw ffi types (which are
// function pointers) - because function pointers aren't allowed in const fn, but types wrapping
// them are!
// These newtype structs serve no purpose other than wrapping which are function pointers - because
// function pointers aren't allowed in const fn, but types wrapping them are!
#[derive(Clone, Copy, Debug)]
pub struct PyCFunction(pub ffi::PyCFunction);
#[derive(Clone, Copy, Debug)]
pub struct PyCFunctionWithKeywords(pub ffi::PyCFunctionWithKeywords);
#[derive(Clone, Copy, Debug)]
pub struct PyGetter(pub ffi::getter);
#[derive(Clone, Copy, Debug)]
pub struct PySetter(pub ffi::setter);
#[derive(Clone, Copy)]
pub struct PyClassAttributeFactory(pub for<'p> fn(Python<'p>) -> PyObject);
// TODO: it would be nice to use CStr in these types, but then the constructors can't be const fn
// until `CStr::from_bytes_with_nul_unchecked` is const fn.
#[derive(Clone, Debug)]
pub struct PyMethodDef {
pub(crate) ml_name: &'static str,
@ -49,22 +56,22 @@ pub struct PyMethodDef {
#[derive(Copy, Clone)]
pub struct PyClassAttributeDef {
pub(crate) name: &'static CStr,
pub(crate) meth: for<'p> fn(Python<'p>) -> PyObject,
pub(crate) name: &'static str,
pub(crate) meth: PyClassAttributeFactory,
}
#[derive(Clone, Debug)]
pub struct PyGetterDef {
pub(crate) name: &'static CStr,
pub(crate) meth: ffi::getter,
doc: &'static CStr,
pub(crate) name: &'static str,
pub(crate) meth: PyGetter,
doc: &'static str,
}
#[derive(Clone, Debug)]
pub struct PySetterDef {
pub(crate) name: &'static CStr,
pub(crate) meth: ffi::setter,
doc: &'static CStr,
pub(crate) name: &'static str,
pub(crate) meth: PySetter,
doc: &'static str,
}
unsafe impl Sync for PyMethodDef {}
@ -117,11 +124,8 @@ impl PyMethodDef {
impl PyClassAttributeDef {
/// Define a class attribute.
pub fn new(name: &'static str, meth: for<'p> fn(Python<'p>) -> PyObject) -> Self {
Self {
name: get_name(name).unwrap(),
meth,
}
pub const fn new(name: &'static str, meth: PyClassAttributeFactory) -> Self {
Self { name, meth }
}
}
@ -137,73 +141,48 @@ impl fmt::Debug for PyClassAttributeDef {
impl PyGetterDef {
/// Define a getter.
pub fn new(name: &'static str, getter: ffi::getter, doc: &'static str) -> Self {
pub const fn new(name: &'static str, getter: PyGetter, doc: &'static str) -> Self {
Self {
name: get_name(name).unwrap(),
name,
meth: getter,
doc: get_doc(doc).unwrap(),
doc,
}
}
/// Copy descriptor information to `ffi::PyGetSetDef`
pub fn copy_to(&self, dst: &mut ffi::PyGetSetDef) {
if dst.name.is_null() {
dst.name = self.name.as_ptr() as _;
dst.name = get_name(self.name).unwrap().as_ptr() as _;
}
if dst.doc.is_null() {
dst.doc = self.doc.as_ptr() as _;
dst.doc = get_doc(self.doc).unwrap().as_ptr() as _;
}
dst.get = Some(self.meth);
dst.get = Some(self.meth.0);
}
}
impl PySetterDef {
/// Define a setter.
pub fn new(name: &'static str, setter: ffi::setter, doc: &'static str) -> Self {
pub const fn new(name: &'static str, setter: PySetter, doc: &'static str) -> Self {
Self {
name: get_name(name).unwrap(),
name,
meth: setter,
doc: get_doc(doc).unwrap(),
doc,
}
}
/// Copy descriptor information to `ffi::PyGetSetDef`
pub fn copy_to(&self, dst: &mut ffi::PyGetSetDef) {
if dst.name.is_null() {
dst.name = self.name.as_ptr() as _;
dst.name = get_name(self.name).unwrap().as_ptr() as _;
}
if dst.doc.is_null() {
dst.doc = self.doc.as_ptr() as _;
dst.doc = get_doc(self.doc).unwrap().as_ptr() as _;
}
dst.set = Some(self.meth);
dst.set = Some(self.meth.0);
}
}
/// Implementation detail. Only to be used through our proc macro code.
/// Method storage for `#[pyclass]`.
/// Allows arbitrary `#[pymethod]/#[pyproto]` blocks to submit their methods,
/// which are eventually collected by `#[pyclass]`.
#[doc(hidden)]
#[cfg(feature = "macros")]
pub trait PyMethodsInventory: inventory::Collect {
/// Create a new instance
fn new(methods: Vec<PyMethodDefType>) -> Self;
/// Returns the methods for a single `#[pymethods] impl` block
fn get(&'static self) -> &'static [PyMethodDefType];
}
/// Implemented for `#[pyclass]` in our proc macro code.
/// Indicates that the pyclass has its own method storage.
#[doc(hidden)]
#[cfg(feature = "macros")]
pub trait HasMethodsInventory {
type Methods: PyMethodsInventory;
}
#[derive(Debug)]
pub(crate) struct NulByteInString(pub(crate) &'static str);
fn get_name(name: &'static str) -> Result<&'static CStr, NulByteInString> {
extract_cstr_or_leak_cstring(name, "Function name cannot contain NUL byte.")
}
@ -211,14 +190,3 @@ fn get_name(name: &'static str) -> Result<&'static CStr, NulByteInString> {
fn get_doc(doc: &'static str) -> Result<&'static CStr, NulByteInString> {
extract_cstr_or_leak_cstring(doc, "Document cannot contain NUL byte.")
}
fn extract_cstr_or_leak_cstring(
src: &'static str,
err_msg: &'static str,
) -> Result<&'static CStr, NulByteInString> {
CStr::from_bytes_with_nul(src.as_bytes())
.or_else(|_| {
CString::new(src.as_bytes()).map(|c_string| &*Box::leak(c_string.into_boxed_c_str()))
})
.map_err(|_| NulByteInString(err_msg))
}

15
src/internal_tricks.rs
View File

@ -1,3 +1,4 @@
use std::ffi::{CStr, CString};
use std::marker::PhantomData;
use std::rc::Rc;
@ -36,3 +37,17 @@ macro_rules! pyo3_exception {
$crate::create_exception_type_object!(pyo3_runtime, $name, $base);
};
}
#[derive(Debug)]
pub(crate) struct NulByteInString(pub(crate) &'static str);
pub(crate) fn extract_cstr_or_leak_cstring(
src: &'static str,
err_msg: &'static str,
) -> Result<&'static CStr, NulByteInString> {
CStr::from_bytes_with_nul(src.as_bytes())
.or_else(|_| {
CString::new(src.as_bytes()).map(|c_string| &*Box::leak(c_string.into_boxed_c_str()))
})
.map_err(|_| NulByteInString(err_msg))
}

20
src/lib.rs
View File

@ -166,12 +166,14 @@ pub use crate::types::PyAny;
#[cfg(feature = "macros")]
#[doc(hidden)]
pub use {
indoc, // Re-exported for py_run
inventory, // Re-exported for pymethods
paste, // Re-exported for wrap_function
unindent, // Re-exported for py_run
indoc, // Re-exported for py_run
paste, // Re-exported for wrap_function
unindent, // Re-exported for py_run
};
#[cfg(all(feature = "macros", feature = "multiple-pymethods"))]
pub use inventory; // Re-exported for `#[pyclass]` and `#[pymethods]` with `multiple-pymethods`.
#[macro_use]
mod internal_tricks;
@ -216,7 +218,15 @@ pub mod serde;
pub mod proc_macro {
pub use pyo3_macros::pymodule;
/// The proc macro attributes
pub use pyo3_macros::{pyclass, pyfunction, pymethods, pyproto};
pub use pyo3_macros::{pyfunction, pyproto};
#[cfg(not(feature = "multiple-pymethods"))]
pub use pyo3_macros::{pyclass, pymethods};
#[cfg(feature = "multiple-pymethods")]
pub use pyo3_macros::{
pyclass_with_inventory as pyclass, pymethods_with_inventory as pymethods,
};
}
/// Returns a function that takes a [Python] instance and returns a Python function.

2
src/prelude.rs
View File

@ -20,4 +20,4 @@ pub use crate::{FromPyObject, IntoPy, IntoPyPointer, PyTryFrom, PyTryInto, ToPyO
// PyModule is only part of the prelude because we need it for the pymodule function
pub use crate::types::{PyAny, PyModule};
#[cfg(feature = "macros")]
pub use pyo3_macros::{pyclass, pyfunction, pymethods, pymodule, pyproto, FromPyObject};
pub use {crate::proc_macro::*, pyo3_macros::FromPyObject};

10
src/type_object.rs
View File

@ -1,6 +1,7 @@
// Copyright (c) 2017-present PyO3 Project and Contributors
//! Python type object information
use crate::internal_tricks::extract_cstr_or_leak_cstring;
use crate::once_cell::GILOnceCell;
use crate::pyclass::{create_type_object, PyClass};
use crate::pyclass_init::PyObjectInit;
@ -197,7 +198,14 @@ impl LazyStaticType {
let mut items = vec![];
T::for_each_method_def(|def| {
if let PyMethodDefType::ClassAttribute(attr) = def {
items.push((attr.name, (attr.meth)(py)));
items.push((
extract_cstr_or_leak_cstring(
attr.name,
"class attribute name cannot contain nul bytes",
)
.unwrap(),
(attr.meth.0)(py),
));
}
});

4
src/types/function.rs
View File

@ -1,10 +1,10 @@
use crate::derive_utils::PyFunctionArguments;
use crate::exceptions::PyValueError;
use crate::prelude::*;
use crate::{
class::methods::{self, PyMethodDef},
ffi, AsPyPointer,
};
use crate::{derive_utils::PyFunctionArguments, methods::NulByteInString};
/// Represents a builtin Python function object.
#[repr(transparent)]
@ -54,7 +54,7 @@ impl PyCFunction {
let (py, module) = py_or_module.into_py_and_maybe_module();
let def = method_def
.as_method_def()
.map_err(|NulByteInString(err)| PyValueError::new_err(err))?;
.map_err(|err| PyValueError::new_err(err.0))?;
let (mod_ptr, module_name) = if let Some(m) = module {
let mod_ptr = m.as_ptr();
let name = m.name()?.into_py(py);

77
tests/test_multiple_pymethods.rs Normal file
View File

@ -0,0 +1,77 @@
#![cfg(feature = "multiple-pymethods")]
use pyo3::prelude::*;
use pyo3::type_object::PyTypeObject;
use pyo3::types::PyType;
#[macro_use]
mod common;
#[pyclass]
struct PyClassWithMultiplePyMethods {}
#[pymethods]
impl PyClassWithMultiplePyMethods {
#[new]
fn new() -> Self {
Self {}
}
}
#[pymethods]
impl PyClassWithMultiplePyMethods {
#[call]
fn call(&self) -> &'static str {
"call"
}
}
#[pymethods]
impl PyClassWithMultiplePyMethods {
fn method(&self) -> &'static str {
"method"
}
}
#[pymethods]
impl PyClassWithMultiplePyMethods {
#[classmethod]
fn classmethod(_ty: &PyType) -> &'static str {
"classmethod"
}
}
#[pymethods]
impl PyClassWithMultiplePyMethods {
#[staticmethod]
fn staticmethod() -> &'static str {
"staticmethod"
}
}
#[pymethods]
impl PyClassWithMultiplePyMethods {
#[classattr]
fn class_attribute() -> &'static str {
"class_attribute"
}
}
#[pymethods]
impl PyClassWithMultiplePyMethods {
#[classattr]
const CLASS_ATTRIBUTE: &'static str = "CLASS_ATTRIBUTE";
}
#[test]
fn test_class_with_multiple_pymethods() {
Python::with_gil(|py| {
let cls = PyClassWithMultiplePyMethods::type_object(py);
py_assert!(py, cls, "cls()() == 'call'");
py_assert!(py, cls, "cls().method() == 'method'");
py_assert!(py, cls, "cls.classmethod() == 'classmethod'");
py_assert!(py, cls, "cls.staticmethod() == 'staticmethod'");
py_assert!(py, cls, "cls.class_attribute == 'class_attribute'");
py_assert!(py, cls, "cls.CLASS_ATTRIBUTE == 'CLASS_ATTRIBUTE'");
})
}