luxolus
/
pyo3
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

pyo3-macros-backend: support macros inside doc attributes

This commit is contained in:
David Hewitt 2021-07-31 00:21:13 +01:00
parent 3fa97f9086
commit f76535fd07
10 changed files with 135 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
### Added
- Add `PyList::get_item_unchecked` and `PyTuple::get_item_unchecked` to get items without bounds checks. [#1733](https://github.com/PyO3/pyo3/pull/1733)
- Support `#[doc = include_str!(...)]` attributes on Rust 1.54 and up. [#1746](https://github.com/PyO3/pyo3/issues/1746)
- Add `PyAny::py` as a convenience for `PyNativeType::py`. [#1751](https://github.com/PyO3/pyo3/pull/1751)
- Add implementation of `std::ops::Index<usize>` for `PyList`, `PyTuple` and `PySequence`. [#1825](https://github.com/PyO3/pyo3/pull/1825)
- Add range indexing implementations of `std::ops::Index` for `PyList`, `PyTuple` and `PySequence`. [#1829](https://github.com/PyO3/pyo3/pull/1829)

16
pyo3-macros-backend/src/method.rs
View File

@ -4,7 +4,7 @@ use crate::attributes::TextSignatureAttribute;
use crate::params::{accept_args_kwargs, impl_arg_params};
use crate::pyfunction::PyFunctionOptions;
use crate::pyfunction::{PyFunctionArgPyO3Attributes, PyFunctionSignature};
use crate::utils;
use crate::utils::{self, PythonDoc};
use crate::{deprecations::Deprecations, pyfunction::Argument};
use proc_macro2::{Span, TokenStream};
use quote::ToTokens;
@ -202,7 +202,7 @@ pub struct FnSpec<'a> {
pub attrs: Vec<Argument>,
pub args: Vec<FnArg<'a>>,
pub output: syn::Type,
pub doc: syn::LitStr,
pub doc: PythonDoc,
pub deprecations: Deprecations,
pub convention: CallingConvention,
}
@ -271,7 +271,7 @@ impl<'a> FnSpec<'a> {
.text_signature
.as_ref()
.map(|attr| (&python_name, attr)),
)?;
);
let arguments: Vec<_> = if skip_first_arg {
sig.inputs
@ -602,8 +602,8 @@ fn parse_method_attributes(
}
for attr in attrs.drain(..) {
match attr.parse_meta()? {
syn::Meta::Path(name) => {
match attr.parse_meta() {
Ok(syn::Meta::Path(name)) => {
if name.is_ident("new") || name.is_ident("__new__") {
set_ty!(MethodTypeAttribute::New, name);
} else if name.is_ident("init") || name.is_ident("__init__") {
@ -631,9 +631,9 @@ fn parse_method_attributes(
new_attrs.push(attr)
}
}
syn::Meta::List(syn::MetaList {
Ok(syn::Meta::List(syn::MetaList {
path, mut nested, ..
}) => {
})) => {
if path.is_ident("new") {
set_ty!(MethodTypeAttribute::New, path);
} else if path.is_ident("init") {
@ -689,7 +689,7 @@ fn parse_method_attributes(
new_attrs.push(attr)
}
}
syn::Meta::NameValue(_) => new_attrs.push(attr),
Ok(syn::Meta::NameValue(_)) | Err(_) => new_attrs.push(attr),
}
}

4
pyo3-macros-backend/src/module.rs
View File

@ -5,6 +5,7 @@ use crate::{
attributes::{self, take_pyo3_options},
deprecations::Deprecations,
pyfunction::{impl_wrap_pyfunction, PyFunctionOptions},
utils::PythonDoc,
};
use crate::{
attributes::{is_attribute_ident, take_attributes, NameAttribute},
@ -62,11 +63,10 @@ impl PyModuleOptions {
/// Generates the function that is called by the python interpreter to initialize the native
/// module
pub fn py_init(fnname: &Ident, options: PyModuleOptions, doc: syn::LitStr) -> TokenStream {
pub fn py_init(fnname: &Ident, options: PyModuleOptions, doc: PythonDoc) -> TokenStream {
let name = options.name.unwrap_or_else(|| fnname.unraw());
let deprecations = options.deprecations;
let cb_name = Ident::new(&format!("PyInit_{}", name), Span::call_site());
assert!(doc.value().ends_with('\0'));
quote! {
#[no_mangle]

6
pyo3-macros-backend/src/pyclass.rs
View File

@ -7,7 +7,7 @@ use crate::attributes::{
use crate::deprecations::Deprecations;
use crate::pyimpl::PyClassMethodsType;
use crate::pymethod::{impl_py_getter_def, impl_py_setter_def, PropertyType};
use crate::utils::{self, unwrap_group};
use crate::utils::{self, unwrap_group, PythonDoc};
use proc_macro2::{Span, TokenStream};
use quote::quote;
use syn::ext::IdentExt;
@ -230,7 +230,7 @@ pub fn build_py_class(
.text_signature
.as_ref()
.map(|attr| (get_class_python_name(&class.ident, args), attr)),
)?;
);
ensure_spanned!(
class.generics.params.is_empty(),
@ -371,7 +371,7 @@ fn get_class_python_name<'a>(cls: &'a syn::Ident, attr: &'a PyClassArgs) -> &'a
fn impl_class(
cls: &syn::Ident,
attr: &PyClassArgs,
doc: syn::LitStr,
doc: PythonDoc,
field_options: Vec<(&syn::Field, FieldPyO3Options)>,
methods_type: PyClassMethodsType,
deprecations: Deprecations,

2
pyo3-macros-backend/src/pyfunction.rs
View File

@ -406,7 +406,7 @@ pub fn impl_wrap_pyfunction(
.text_signature
.as_ref()
.map(|attr| (&python_name, attr)),
)?;
);
let function_wrapper_ident = function_wrapper_ident(&func.sig.ident);

8
pyo3-macros-backend/src/pymethod.rs
View File

@ -3,7 +3,7 @@
use std::borrow::Cow;
use crate::attributes::NameAttribute;
use crate::utils::ensure_not_async_fn;
use crate::utils::{ensure_not_async_fn, PythonDoc};
use crate::{deprecations::Deprecations, utils};
use crate::{
method::{FnArg, FnSpec, FnType, SelfType},
@ -355,12 +355,10 @@ impl PropertyType<'_> {
}
}
fn doc(&self) -> Cow<syn::LitStr> {
fn doc(&self) -> Cow<PythonDoc> {
match self {
PropertyType::Descriptor { field, .. } => {
let doc = utils::get_doc(&field.attrs, None)
.unwrap_or_else(|_| syn::LitStr::new("", Span::call_site()));
Cow::Owned(doc)
Cow::Owned(utils::get_doc(&field.attrs, None))
}
PropertyType::Function { spec, .. } => Cow::Borrowed(&spec.doc),
}

104
pyo3-macros-backend/src/utils.rs
View File

@ -1,5 +1,6 @@
// Copyright (c) 2017-present PyO3 Project and Contributors
use proc_macro2::Span;
use proc_macro2::{Span, TokenStream};
use quote::ToTokens;
use syn::spanned::Spanned;
use crate::attributes::TextSignatureAttribute;
@ -54,59 +55,94 @@ pub fn option_type_argument(ty: &syn::Type) -> Option<&syn::Type> {
None
}
// Returns a null-terminated syn::LitStr for use as a Python docstring.
/// A syntax tree which evaluates to a null-terminated docstring for Python.
///
/// It's built as a `concat!` evaluation, so it's hard to do anything with this
/// contents such as parse the string contents.
#[derive(Clone)]
pub struct PythonDoc(TokenStream);
// TODO(#1782) use strip_prefix on Rust 1.45 or greater
#[allow(clippy::manual_strip)]
/// Collects all #[doc = "..."] attributes into a TokenStream evaluating to a null-terminated string
/// e.g. concat!("...", "\n", "\0")
pub fn get_doc(
attrs: &[syn::Attribute],
text_signature: Option<(&syn::Ident, &TextSignatureAttribute)>,
) -> syn::Result<syn::LitStr> {
let mut doc = String::new();
let mut span = Span::call_site();
) -> PythonDoc {
let mut tokens = TokenStream::new();
let comma = syn::token::Comma(Span::call_site());
let newline = syn::LitStr::new("\n", Span::call_site());
if let Some((python_name, text_signature)) = text_signature {
// create special doc string lines to set `__text_signature__`
doc.push_str(&python_name.to_string());
span = text_signature.lit.span();
doc.push_str(&text_signature.lit.value());
doc.push_str("\n--\n\n");
}
syn::Ident::new("concat", Span::call_site()).to_tokens(&mut tokens);
syn::token::Bang(Span::call_site()).to_tokens(&mut tokens);
syn::token::Bracket(Span::call_site()).surround(&mut tokens, |tokens| {
if let Some((python_name, text_signature)) = text_signature {
// create special doc string lines to set `__text_signature__`
let signature_lines = format!(
"{}{}\n--\n\n",
python_name.to_string(),
text_signature.lit.value()
);
signature_lines.to_tokens(tokens);
comma.to_tokens(tokens);
}
let mut separator = "";
let mut first = true;
let mut first = true;
for attr in attrs.iter() {
if attr.path.is_ident("doc") {
if let Ok(DocArgs { _eq_token, lit_str }) = syn::parse2(attr.tokens.clone()) {
if first {
first = false;
span = lit_str.span();
for attr in attrs.iter() {
if attr.path.is_ident("doc") {
if let Ok(DocArgs {
_eq_token,
token_stream,
}) = syn::parse2(attr.tokens.clone())
{
if !first {
newline.to_tokens(tokens);
comma.to_tokens(tokens);
} else {
first = false;
}
if let Ok(syn::Lit::Str(lit_str)) = syn::parse2(token_stream.clone()) {
// Strip single left space from literal strings, if needed.
// e.g. `/// Hello world` expands to #[doc = " Hello world"]
let doc_line = lit_str.value();
if doc_line.starts_with(' ') {
syn::LitStr::new(&doc_line[1..], lit_str.span()).to_tokens(tokens)
} else {
lit_str.to_tokens(tokens)
}
} else {
// This is probably a macro doc from Rust 1.54, e.g. #[doc = include_str!(...)]
token_stream.to_tokens(tokens)
}
comma.to_tokens(tokens);
}
let d = lit_str.value();
doc.push_str(separator);
if d.starts_with(' ') {
doc.push_str(&d[1..d.len()]);
} else {
doc.push_str(&d);
};
separator = "\n";
}
}
syn::LitStr::new("\0", Span::call_site()).to_tokens(tokens);
});
PythonDoc(tokens)
}
impl quote::ToTokens for PythonDoc {
fn to_tokens(&self, tokens: &mut TokenStream) {
self.0.to_tokens(tokens)
}
doc.push('\0');
Ok(syn::LitStr::new(&doc, span))
}
struct DocArgs {
_eq_token: syn::Token![=],
lit_str: syn::LitStr,
token_stream: TokenStream,
}
impl syn::parse::Parse for DocArgs {
fn parse(input: syn::parse::ParseStream) -> syn::Result<Self> {
let this = Self {
_eq_token: input.parse()?,
lit_str: input.parse()?,
token_stream: input.parse()?,
};
ensure_spanned!(input.is_empty(), input.span() => "expected end of doc attribute");
Ok(this)

5
pyo3-macros/src/lib.rs
View File

@ -52,10 +52,7 @@ pub fn pymodule(attr: TokenStream, input: TokenStream) -> TokenStream {
return err.to_compile_error().into();
}
let doc = match get_doc(&ast.attrs, None) {
Ok(doc) => doc,
Err(err) => return err.to_compile_error().into(),
};
let doc = get_doc(&ast.attrs, None);
let expanded = py_init(&ast.sig.ident, options, doc);

36
tests/not_msrv/requires_1_54.rs Normal file
View File

@ -0,0 +1,36 @@
use pyo3::prelude::*;
use pyo3::types::IntoPyDict;
#[macro_use]
#[path = "../common.rs"]
mod common;
#[pyclass]
/// The MacroDocs class.
#[doc = concat!("Some macro ", "class ", "docs.")]
/// A very interesting type!
struct MacroDocs {}
#[pymethods]
impl MacroDocs {
#[doc = concat!("A macro ", "example.")]
/// With mixed doc types.
fn macro_doc(&self) {}
}
#[test]
fn meth_doc() {
Python::with_gil(|py| {
let d = [("C", py.get_type::<MacroDocs>())].into_py_dict(py);
py_assert!(
py,
*d,
"C.__doc__ == 'The MacroDocs class.\\nSome macro class docs.\\nA very interesting type!'"
);
py_assert!(
py,
*d,
"C.macro_doc.__doc__ == 'A macro example.\\nWith mixed doc types.'"
);
});
}

10
tests/test_not_msrv.rs Normal file
View File

@ -0,0 +1,10 @@
//! Functionality which is not only not supported on MSRV,
//! but can't even be cfg-ed out on MSRV because the compiler doesn't support
//! the syntax.
// TODO(#1782) rustversion attribute can't go on modules until Rust 1.42, so this
// funky dance has to happen...
mod requires_1_54 {
#[rustversion::since(1.54)]
include!("not_msrv/requires_1_54.rs");
}