luxolus/pyo3
2
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Merge pull request #2134 from davidhewitt/optimize-extract-argument

Browse source 
opt: improve generated code for argument extraction
This commit is contained in:
David Hewitt 2022-02-05 00:33:45 +00:00 committed by GitHub
parent 6814740d20 1beb2bbb2d
commit abd7eafafb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 506 additions and 251 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

102
pyo3-macros-backend/src/params.rs
View file

@ -70,7 +70,7 @@ pub fn impl_arg_params(
arg_convert.push(impl_arg_param(arg, spec, i, None, &mut 0, py, &args_array)?);
}
return Ok(quote! {
let _args = ::std::option::Option::Some(#py.from_borrowed_ptr::<_pyo3::types::PyTuple>(_args));
let _args = #py.from_borrowed_ptr::<_pyo3::types::PyTuple>(_args);
let _kwargs: ::std::option::Option<&_pyo3::types::PyDict> = #py.from_borrowed_ptr_or_opt(_kwargs);
#(#arg_convert)*
});
@ -126,6 +126,16 @@ pub fn impl_arg_params(
}
let (accept_args, accept_kwargs) = accept_args_kwargs(&spec.attrs);
let args_handler = if accept_args {
quote! { _pyo3::impl_::extract_argument::TupleVarargs }
} else {
quote! { _pyo3::impl_::extract_argument::NoVarargs }
};
let kwargs_handler = if accept_kwargs {
quote! { _pyo3::impl_::extract_argument::DictVarkeywords }
} else {
quote! { _pyo3::impl_::extract_argument::NoVarkeywords }
};
let cls_name = if let Some(cls) = self_ {
quote! { ::std::option::Option::Some(<#cls as _pyo3::type_object::PyTypeInfo>::NAME) }
@ -136,7 +146,7 @@ pub fn impl_arg_params(
let extract_expression = if fastcall {
quote! {
DESCRIPTION.extract_arguments_fastcall(
DESCRIPTION.extract_arguments_fastcall::<#args_handler, #kwargs_handler>(
#py,
_args,
_nargs,
@ -146,7 +156,7 @@ pub fn impl_arg_params(
}
} else {
quote! {
DESCRIPTION.extract_arguments_tuple_dict(
DESCRIPTION.extract_arguments_tuple_dict::<#args_handler, #kwargs_handler>(
#py,
_args,
_kwargs,
@ -164,8 +174,6 @@ pub fn impl_arg_params(
positional_only_parameters: #positional_only_parameters,
required_positional_parameters: #required_positional_parameters,
keyword_only_parameters: &[#(#keyword_only_parameters),*],
accept_varargs: #accept_args,
accept_varkeywords: #accept_kwargs,
};
let mut #args_array = [::std::option::Option::None; #num_params];
@ -201,9 +209,6 @@ fn impl_arg_param(
let ty = arg.ty;
let name = arg.name;
let name_str = name.to_string();
let transform_error = quote! {
|e| _pyo3::impl_::extract_argument::argument_extraction_error(#py, #name_str, e)
};
if is_args(&spec.attrs, name) {
ensure_spanned!(
@ -211,7 +216,7 @@ fn impl_arg_param(
arg.name.span() => "args cannot be optional"
);
return Ok(quote_arg_span! {
let #arg_name = _pyo3::impl_::extract_argument::extract_argument(_args.unwrap(), #name_str)?;
let #arg_name = _pyo3::impl_::extract_argument::extract_argument(_args, #name_str)?;
});
} else if is_kwargs(&spec.attrs, name) {
ensure_spanned!(
@ -219,43 +224,64 @@ fn impl_arg_param(
arg.name.span() => "kwargs must be Option<_>"
);
return Ok(quote_arg_span! {
let #arg_name = _kwargs.map(|kwargs| _pyo3::impl_::extract_argument::extract_argument(kwargs, #name_str))
.transpose()?;
let #arg_name = _pyo3::impl_::extract_argument::extract_optional_argument(_kwargs.map(|kwargs| kwargs.as_ref()), #name_str)?;
});
}
let arg_value = quote_arg_span!(#args_array[#option_pos]);
*option_pos += 1;
let extract = if let Some(FromPyWithAttribute(expr_path)) = &arg.attrs.from_py_with {
quote_arg_span! { #expr_path(_obj).map_err(#transform_error) }
let arg_value_or_default = if let Some(FromPyWithAttribute(expr_path)) = &arg.attrs.from_py_with
{
match (spec.default_value(name), arg.optional.is_some()) {
(Some(default), true) if default.to_string() != "None" => {
quote_arg_span! {
_pyo3::impl_::extract_argument::from_py_with_with_default(#arg_value, #name_str, #expr_path, || Some(#default))
}
}
(Some(default), _) => {
quote_arg_span! {
_pyo3::impl_::extract_argument::from_py_with_with_default(#arg_value, #name_str, #expr_path, || #default)
}
}
(None, true) => {
quote_arg_span! {
_pyo3::impl_::extract_argument::from_py_with_with_default(#arg_value, #name_str, #expr_path, || Some(None))
}
}
(None, false) => {
quote_arg_span! {
_pyo3::impl_::extract_argument::from_py_with(
_pyo3::impl_::extract_argument::unwrap_required_argument(#arg_value),
#name_str,
#expr_path,
)
}
}
}
} else {
quote_arg_span! { _pyo3::impl_::extract_argument::extract_argument(_obj, #name_str) }
};
let arg_value_or_default = match (spec.default_value(name), arg.optional.is_some()) {
(Some(default), true) if default.to_string() != "None" => {
quote_arg_span! {
#arg_value.map_or_else(|| ::std::result::Result::Ok(::std::option::Option::Some(#default)),
|_obj| #extract)?
match (spec.default_value(name), arg.optional.is_some()) {
(Some(default), true) if default.to_string() != "None" => {
quote_arg_span! {
_pyo3::impl_::extract_argument::extract_argument_with_default(#arg_value, #name_str, || Some(#default))
}
}
}
(Some(default), _) => {
quote_arg_span! {
#arg_value.map_or_else(|| ::std::result::Result::Ok(#default), |_obj| #extract)?
(Some(default), _) => {
quote_arg_span! {
_pyo3::impl_::extract_argument::extract_argument_with_default(#arg_value, #name_str, || #default)
}
}
}
(None, true) => {
quote_arg_span! {
#arg_value.map_or(::std::result::Result::Ok(::std::option::Option::None),
|_obj| #extract)?
(None, true) => {
quote_arg_span! {
_pyo3::impl_::extract_argument::extract_optional_argument(#arg_value, #name_str)
}
}
}
(None, false) => {
quote_arg_span! {
{
let _obj = #arg_value.expect("Failed to extract required method argument");
#extract?
(None, false) => {
quote_arg_span! {
_pyo3::impl_::extract_argument::extract_argument(
_pyo3::impl_::extract_argument::unwrap_required_argument(#arg_value),
#name_str
)
}
}
}
@ -286,13 +312,13 @@ fn impl_arg_param(
};
Ok(quote_arg_span! {
let #mut_ _tmp: #target_ty = #arg_value_or_default;
let #mut_ _tmp: #target_ty = #arg_value_or_default?;
#[allow(clippy::needless_option_as_deref)]
let #arg_name = #borrow_tmp;
})
} else {
Ok(quote_arg_span! {
let #arg_name = #arg_value_or_default;
let #arg_name = #arg_value_or_default?;
})
};
}

2
pytests/pyo3-benchmarks/noxfile.py
View file

@ -15,4 +15,4 @@ def test(session):
def bench(session):
session.install("-rrequirements-dev.txt")
session.install(".")
session.run("pytest", "--benchmark-enable")
session.run("pytest", "--benchmark-enable", *session.posargs)

1
pytests/pyo3-pytests/build.rs
View file

@ -1,3 +1,4 @@
fn main() {
pyo3_build_config::use_pyo3_cfgs();
pyo3_build_config::add_extension_module_link_args();
}

650
src/impl_/extract_argument.rs
View file

@ -6,6 +6,7 @@ use crate::{
FromPyObject, PyAny, PyErr, PyResult, Python,
};
/// The standard implementation of how PyO3 extracts a `#[pyfunction]` or `#[pymethod]` function argument.
#[doc(hidden)]
#[inline]
pub fn extract_argument<'py, T>(obj: &'py PyAny, arg_name: &str) -> PyResult<T>
@ -13,11 +14,87 @@ where
T: FromPyObject<'py>,
{
match obj.extract() {
Ok(e) => Ok(e),
Ok(value) => Ok(value),
Err(e) => Err(argument_extraction_error(obj.py(), arg_name, e)),
}
}
/// Alternative to [`extract_argument`] used for `Option<T>` arguments (because they are implicitly treated
/// as optional if at the end of the positional parameters).
#[doc(hidden)]
#[inline]
pub fn extract_optional_argument<'py, T>(
obj: Option<&'py PyAny>,
arg_name: &str,
) -> PyResult<Option<T>>
where
T: FromPyObject<'py>,
{
match obj {
Some(obj) => match obj.extract() {
Ok(value) => Ok(value),
Err(e) => Err(argument_extraction_error(obj.py(), arg_name, e)),
},
None => Ok(None),
}
}
/// Alternative to [`extract_argument`] used when the argument has a default value provided by an annotation.
#[doc(hidden)]
#[inline]
pub fn extract_argument_with_default<'py, T>(
obj: Option<&'py PyAny>,
arg_name: &str,
default: impl FnOnce() -> T,
) -> PyResult<T>
where
T: FromPyObject<'py>,
{
match obj {
Some(obj) => match obj.extract() {
Ok(value) => Ok(value),
Err(e) => Err(argument_extraction_error(obj.py(), arg_name, e)),
},
None => Ok(default()),
}
}
/// Alternative to [`extract_argument`] used when the argument has a `#[pyo3(from_py_with)]` annotation.
///
/// # Safety
/// - `obj` must not be None (this helper is only used for required function arguments).
#[doc(hidden)]
#[inline]
pub fn from_py_with<'py, T>(
obj: &'py PyAny,
arg_name: &str,
extractor: impl FnOnce(&'py PyAny) -> PyResult<T>,
) -> PyResult<T> {
// Safety: obj is not None (see safety
match extractor(obj) {
Ok(value) => Ok(value),
Err(e) => Err(argument_extraction_error(obj.py(), arg_name, e)),
}
}
/// Alternative to [`extract_argument`] used when the argument has a `#[pyo3(from_py_with)]` annotation and also a default value.
#[doc(hidden)]
#[inline]
pub fn from_py_with_with_default<'py, T>(
obj: Option<&'py PyAny>,
arg_name: &str,
extractor: impl FnOnce(&'py PyAny) -> PyResult<T>,
default: impl FnOnce() -> T,
) -> PyResult<T> {
match obj {
Some(obj) => match extractor(obj) {
Ok(value) => Ok(value),
Err(e) => Err(argument_extraction_error(obj.py(), arg_name, e)),
},
None => Ok(default()),
}
}
/// Adds the argument name to the error message of an error which occurred during argument extraction.
///
/// Only modifies TypeError. (Cannot guarantee all exceptions have constructors from
@ -32,6 +109,23 @@ pub fn argument_extraction_error(py: Python, arg_name: &str, error: PyErr) -> Py
}
}
/// Unwraps the Option<&PyAny> produced by the FunctionDescription `extract_arguments_` methods.
/// They check if required methods are all provided.
///
/// # Safety
/// `argument` must not be `None`
#[doc(hidden)]
#[inline]
pub unsafe fn unwrap_required_argument(argument: Option<&PyAny>) -> &PyAny {
match argument {
Some(value) => value,
#[cfg(debug_assertions)]
None => unreachable!("required method argument was not extracted"),
#[cfg(not(debug_assertions))]
None => std::hint::unreachable_unchecked(),
}
}
pub struct KeywordOnlyParameterDescription {
pub name: &'static str,
pub required: bool,
@ -45,8 +139,6 @@ pub struct FunctionDescription {
pub positional_only_parameters: usize,
pub required_positional_parameters: usize,
pub keyword_only_parameters: &'static [KeywordOnlyParameterDescription],
pub accept_varargs: bool,
pub accept_varkeywords: bool,
}
impl FunctionDescription {
@ -58,81 +150,30 @@ impl FunctionDescription {
}
}
/// Wrapper around `extract_arguments` which uses the Python C-API "fastcall" convention.
/// Equivalent of `extract_arguments_tuple_dict` which uses the Python C-API "fastcall" convention.
///
/// # Safety
/// - `args` must be a pointer to a C-style array of valid `ffi::PyObject` pointers.
/// - `kwnames` must be a pointer to a PyTuple, or NULL.
/// - `nargs + kwnames.len()` is the total length of the `args` array.
#[cfg(not(Py_LIMITED_API))]
pub unsafe fn extract_arguments_fastcall<'py>(
pub unsafe fn extract_arguments_fastcall<'py, V, K>(
&self,
py: Python<'py>,
args: *const *mut ffi::PyObject,
nargs: ffi::Py_ssize_t,
kwnames: *mut ffi::PyObject,
output: &mut [Option<&'py PyAny>],
) -> PyResult<(Option<&'py PyTuple>, Option<&'py PyDict>)> {
let kwnames: Option<&PyTuple> = py.from_borrowed_ptr_or_opt(kwnames);
// Safety: &PyAny has the same memory layout as `*mut ffi::PyObject`
let args = args as *const &PyAny;
let kwargs = if let Option::Some(kwnames) = kwnames {
::std::slice::from_raw_parts(args.offset(nargs), kwnames.len())
} else {
&[]
};
let args = std::slice::from_raw_parts(args, nargs as usize);
self.extract_arguments(
py,
args.iter().copied(),
kwnames.map(|kwnames| {
kwnames
.as_slice()
.iter()
.copied()
.zip(kwargs.iter().copied())
}),
output,
)
}
) -> PyResult<(V::Varargs, K::Varkeywords)>
where
V: VarargsHandler<'py>,
K: VarkeywordsHandler<'py>,
{
// Safety: Option<&PyAny> has the same memory layout as `*mut ffi::PyObject`
let args = args as *const Option<&PyAny>;
let positional_args_provided = nargs as usize;
let args_slice = std::slice::from_raw_parts(args, positional_args_provided);
/// Wrapper around `extract_arguments` which uses the
/// tuple-and-dict Python call convention.
///
/// # Safety
/// - `args` must be a pointer to a PyTuple.
/// - `kwargs` must be a pointer to a PyDict, or NULL.
pub unsafe fn extract_arguments_tuple_dict<'py>(
&self,
py: Python<'py>,
args: *mut ffi::PyObject,
kwargs: *mut ffi::PyObject,
output: &mut [Option<&'py PyAny>],
) -> PyResult<(Option<&'py PyTuple>, Option<&'py PyDict>)> {
let args = py.from_borrowed_ptr::<PyTuple>(args);
let kwargs: ::std::option::Option<&PyDict> = py.from_borrowed_ptr_or_opt(kwargs);
self.extract_arguments(py, args.iter(), kwargs.map(|dict| dict.iter()), output)
}
/// Extracts the `args` and `kwargs` provided into `output`, according to this function
/// definition.
///
/// `output` must have the same length as this function has positional and keyword-only
/// parameters (as per the `positional_parameter_names` and `keyword_only_parameters`
/// respectively).
///
/// If `accept_varargs` or `accept_varkeywords`, then the returned `&PyTuple` and `&PyDict` may
/// be `Some` if there are extra arguments.
///
/// Unexpected, duplicate or invalid arguments will cause this function to return `TypeError`.
#[inline]
fn extract_arguments<'py>(
&self,
py: Python<'py>,
mut args: impl ExactSizeIterator<Item = &'py PyAny>,
kwargs: Option<impl Iterator<Item = (&'py PyAny, &'py PyAny)>>,
output: &mut [Option<&'py PyAny>],
) -> PyResult<(Option<&'py PyTuple>, Option<&'py PyDict>)> {
let num_positional_parameters = self.positional_parameter_names.len();
debug_assert!(self.positional_only_parameters <= num_positional_parameters);
@ -142,129 +183,202 @@ impl FunctionDescription {
num_positional_parameters + self.keyword_only_parameters.len()
);
// Handle positional arguments
let args_provided = {
let args_provided = args.len();
if self.accept_varargs {
std::cmp::min(num_positional_parameters, args_provided)
} else if args_provided > num_positional_parameters {
return Err(self.too_many_positional_arguments(args_provided));
} else {
args_provided
}
};
// Copy positional arguments into output
for (out, arg) in output[..args_provided].iter_mut().zip(args.by_ref()) {
*out = Some(arg);
}
// Collect varargs into tuple
let varargs = if self.accept_varargs {
Some(PyTuple::new(py, args))
let varargs = if positional_args_provided > num_positional_parameters {
let (positional_parameters, varargs) = args_slice.split_at(num_positional_parameters);
output[..num_positional_parameters].copy_from_slice(positional_parameters);
V::handle_varargs_fastcall(py, varargs, self)?
} else {
None
output[..positional_args_provided].copy_from_slice(args_slice);
V::handle_varargs_fastcall(py, &[], self)?
};
// Handle keyword arguments
let varkeywords = match (kwargs, self.accept_varkeywords) {
(Some(kwargs), true) => {
let mut varkeywords = None;
self.extract_keyword_arguments(kwargs, output, |name, value| {
varkeywords
.get_or_insert_with(|| PyDict::new(py))
.set_item(name, value)
})?;
varkeywords
}
(Some(kwargs), false) => {
self.extract_keyword_arguments(
kwargs,
output,
#[cold]
|name, _| Err(self.unexpected_keyword_argument(name)),
)?;
None
}
(None, _) => None,
};
let mut varkeywords = Default::default();
if let Some(kwnames) = py.from_borrowed_ptr_or_opt::<PyTuple>(kwnames) {
let mut positional_only_keyword_arguments = Vec::new();
// Check that there's sufficient positional arguments once keyword arguments are specified
if args_provided < self.required_positional_parameters {
for out in &output[..self.required_positional_parameters] {
// Safety: &PyAny has the same memory layout as `*mut ffi::PyObject`
let kwargs =
::std::slice::from_raw_parts((args as *const &PyAny).offset(nargs), kwnames.len());
for (kwarg_name_py, &value) in kwnames.iter().zip(kwargs) {
// All keyword arguments should be UTF8 strings, but we'll check, just in case.
if let Ok(kwarg_name) = kwarg_name_py.downcast::<PyString>()?.to_str() {
// Try to place parameter in keyword only parameters
if let Some(i) = self.find_keyword_parameter_in_keyword_only(kwarg_name) {
if output[i + num_positional_parameters]
.replace(value)
.is_some()
{
return Err(self.multiple_values_for_argument(kwarg_name));
}
continue;
}
// Repeat for positional parameters
if let Some(i) = self.find_keyword_parameter_in_positional(kwarg_name) {
if i < self.positional_only_parameters {
positional_only_keyword_arguments.push(kwarg_name);
} else if output[i].replace(value).is_some() {
return Err(self.multiple_values_for_argument(kwarg_name));
}
continue;
}
};
K::handle_unexpected_keyword(&mut varkeywords, kwarg_name_py, value, self)?
}
if !positional_only_keyword_arguments.is_empty() {
return Err(
self.positional_only_keyword_arguments(&positional_only_keyword_arguments)
);
}
}
// Once all inputs have been processed, check that all required arguments have been provided.
self.ensure_no_missing_required_positional_arguments(output, positional_args_provided)?;
self.ensure_no_missing_required_keyword_arguments(output)?;
Ok((varargs, varkeywords))
}
/// Extracts the `args` and `kwargs` provided into `output`, according to this function
/// definition.
///
/// `output` must have the same length as this function has positional and keyword-only
/// parameters (as per the `positional_parameter_names` and `keyword_only_parameters`
/// respectively).
///
/// Unexpected, duplicate or invalid arguments will cause this function to return `TypeError`.
///
/// # Safety
/// - `args` must be a pointer to a PyTuple.
/// - `kwargs` must be a pointer to a PyDict, or NULL.
pub unsafe fn extract_arguments_tuple_dict<'py, V, K>(
&self,
py: Python<'py>,
args: *mut ffi::PyObject,
kwargs: *mut ffi::PyObject,
output: &mut [Option<&'py PyAny>],
) -> PyResult<(V::Varargs, K::Varkeywords)>
where
V: VarargsHandler<'py>,
K: VarkeywordsHandler<'py>,
{
let args = py.from_borrowed_ptr::<PyTuple>(args);
let kwargs: ::std::option::Option<&PyDict> = py.from_borrowed_ptr_or_opt(kwargs);
let num_positional_parameters = self.positional_parameter_names.len();
debug_assert!(self.positional_only_parameters <= num_positional_parameters);
debug_assert!(self.required_positional_parameters <= num_positional_parameters);
debug_assert_eq!(
output.len(),
num_positional_parameters + self.keyword_only_parameters.len()
);
// Copy positional arguments into output
for (i, arg) in args.iter().take(num_positional_parameters).enumerate() {
output[i] = Some(arg);
}
// If any arguments remain, push them to varargs (if possible) or error
let varargs = V::handle_varargs_tuple(args, self)?;
// Handle keyword arguments
let mut varkeywords = Default::default();
if let Some(kwargs) = kwargs {
let mut positional_only_keyword_arguments = Vec::new();
for (kwarg_name_py, value) in kwargs {
// All keyword arguments should be UTF8 strings, but we'll check, just in case.
if let Ok(kwarg_name) = kwarg_name_py.downcast::<PyString>()?.to_str() {
// Try to place parameter in keyword only parameters
if let Some(i) = self.find_keyword_parameter_in_keyword_only(kwarg_name) {
if output[i + num_positional_parameters]
.replace(value)
.is_some()
{
return Err(self.multiple_values_for_argument(kwarg_name));
}
continue;
}
// Repeat for positional parameters
if let Some(i) = self.find_keyword_parameter_in_positional(kwarg_name) {
if i < self.positional_only_parameters {
positional_only_keyword_arguments.push(kwarg_name);
} else if output[i].replace(value).is_some() {
return Err(self.multiple_values_for_argument(kwarg_name));
}
continue;
}
};
K::handle_unexpected_keyword(&mut varkeywords, kwarg_name_py, value, self)?
}
if !positional_only_keyword_arguments.is_empty() {
return Err(
self.positional_only_keyword_arguments(&positional_only_keyword_arguments)
);
}
}
// Once all inputs have been processed, check that all required arguments have been provided.
self.ensure_no_missing_required_positional_arguments(output, args.len())?;
self.ensure_no_missing_required_keyword_arguments(output)?;
Ok((varargs, varkeywords))
}
#[inline]
fn find_keyword_parameter_in_positional(&self, kwarg_name: &str) -> Option<usize> {
self.positional_parameter_names
.iter()
.position(|&param_name| param_name == kwarg_name)
}
#[inline]
fn find_keyword_parameter_in_keyword_only(&self, kwarg_name: &str) -> Option<usize> {
// Compare the keyword name against each parameter in turn. This is exactly the same method
// which CPython uses to map keyword names. Although it's O(num_parameters), the number of
// parameters is expected to be small so it's not worth constructing a mapping.
self.keyword_only_parameters
.iter()
.position(|param_desc| param_desc.name == kwarg_name)
}
#[inline]
fn ensure_no_missing_required_positional_arguments(
&self,
output: &[Option<&PyAny>],
positional_args_provided: usize,
) -> PyResult<()> {
if positional_args_provided < self.required_positional_parameters {
for out in &output[positional_args_provided..self.required_positional_parameters] {
if out.is_none() {
return Err(self.missing_required_positional_arguments(output));
}
}
}
Ok(())
}
// Check no missing required keyword arguments
let keyword_output = &output[num_positional_parameters..];
#[inline]
fn ensure_no_missing_required_keyword_arguments(
&self,
output: &[Option<&PyAny>],
) -> PyResult<()> {
let keyword_output = &output[self.positional_parameter_names.len()..];
for (param, out) in self.keyword_only_parameters.iter().zip(keyword_output) {
if param.required && out.is_none() {
return Err(self.missing_required_keyword_arguments(keyword_output));
}
}
Ok((varargs, varkeywords))
}
fn extract_keyword_arguments<'py>(
&self,
kwargs: impl Iterator<Item = (&'py PyAny, &'py PyAny)>,
output: &mut [Option<&'py PyAny>],
mut unexpected_keyword_handler: impl FnMut(&'py PyAny, &'py PyAny) -> PyResult<()>,
) -> PyResult<()> {
let positional_args_count = self.positional_parameter_names.len();
let mut positional_only_keyword_arguments = Vec::new();
'for_each_kwarg: for (kwarg_name_py, value) in kwargs {
let kwarg_name = match kwarg_name_py.downcast::<PyString>()?.to_str() {
Ok(kwarg_name) => kwarg_name,
// This keyword is not a UTF8 string: all PyO3 argument names are guaranteed to be
// UTF8 by construction.
Err(_) => {
unexpected_keyword_handler(kwarg_name_py, value)?;
continue;
}
};
// Compare the keyword name against each parameter in turn. This is exactly the same method
// which CPython uses to map keyword names. Although it's O(num_parameters), the number of
// parameters is expected to be small so it's not worth constructing a mapping.
for (i, param) in self.keyword_only_parameters.iter().enumerate() {
if param.name == kwarg_name {
output[positional_args_count + i] = Some(value);
continue 'for_each_kwarg;
}
}
// Repeat for positional parameters
if let Some(i) = self.find_keyword_parameter_in_positionals(kwarg_name) {
if i < self.positional_only_parameters {
positional_only_keyword_arguments.push(kwarg_name);
} else if output[i].replace(value).is_some() {
return Err(self.multiple_values_for_argument(kwarg_name));
}
continue;
}
unexpected_keyword_handler(kwarg_name_py, value)?;
}
if positional_only_keyword_arguments.is_empty() {
Ok(())
} else {
Err(self.positional_only_keyword_arguments(&positional_only_keyword_arguments))
}
}
fn find_keyword_parameter_in_positionals(&self, kwarg_name: &str) -> Option<usize> {
for (i, param_name) in self.positional_parameter_names.iter().enumerate() {
if *param_name == kwarg_name {
return Some(i);
}
}
None
Ok(())
}
#[cold]
@ -373,6 +487,129 @@ impl FunctionDescription {
}
}
/// A trait used to control whether to accept varargs in FunctionDescription::extract_argument_(method) functions.
pub trait VarargsHandler<'py> {
type Varargs;
/// Called by `FunctionDescription::extract_arguments_fastcall` with any additional arguments.
fn handle_varargs_fastcall(
py: Python<'py>,
varargs: &[Option<&PyAny>],
function_description: &FunctionDescription,
) -> PyResult<Self::Varargs>;
/// Called by `FunctionDescription::extract_arguments_tuple_dict` with the original tuple.
///
/// Additional arguments are those in the tuple slice starting from `function_description.positional_parameter_names.len()`.
fn handle_varargs_tuple(
args: &'py PyTuple,
function_description: &FunctionDescription,
) -> PyResult<Self::Varargs>;
}
/// Marker struct which indicates varargs are not allowed.
pub struct NoVarargs;
impl<'py> VarargsHandler<'py> for NoVarargs {
type Varargs = ();
#[inline]
fn handle_varargs_fastcall(
_py: Python<'py>,
varargs: &[Option<&PyAny>],
function_description: &FunctionDescription,
) -> PyResult<Self::Varargs> {
let extra_arguments = varargs.len();
if extra_arguments > 0 {
return Err(function_description.too_many_positional_arguments(
function_description.positional_parameter_names.len() + extra_arguments,
));
}
Ok(())
}
#[inline]
fn handle_varargs_tuple(
args: &'py PyTuple,
function_description: &FunctionDescription,
) -> PyResult<Self::Varargs> {
let positional_parameter_count = function_description.positional_parameter_names.len();
let provided_args_count = args.len();
if provided_args_count <= positional_parameter_count {
Ok(())
} else {
Err(function_description.too_many_positional_arguments(provided_args_count))
}
}
}
/// Marker struct which indicates varargs should be collected into a `PyTuple`.
pub struct TupleVarargs;
impl<'py> VarargsHandler<'py> for TupleVarargs {
type Varargs = &'py PyTuple;
#[inline]
fn handle_varargs_fastcall(
py: Python<'py>,
varargs: &[Option<&PyAny>],
_function_description: &FunctionDescription,
) -> PyResult<Self::Varargs> {
Ok(PyTuple::new(py, varargs))
}
#[inline]
fn handle_varargs_tuple(
args: &'py PyTuple,
function_description: &FunctionDescription,
) -> PyResult<Self::Varargs> {
let positional_parameters = function_description.positional_parameter_names.len();
Ok(args.get_slice(positional_parameters, args.len()))
}
}
/// A trait used to control whether to accept unrecognised keywords in FunctionDescription::extract_argument_(method) functions.
pub trait VarkeywordsHandler<'py> {
type Varkeywords: Default;
fn handle_unexpected_keyword(
varkeywords: &mut Self::Varkeywords,
name: &'py PyAny,
value: &'py PyAny,
function_description: &FunctionDescription,
) -> PyResult<()>;
}
/// Marker struct which indicates unknown keywords are not permitted.
pub struct NoVarkeywords;
impl<'py> VarkeywordsHandler<'py> for NoVarkeywords {
type Varkeywords = ();
#[inline]
fn handle_unexpected_keyword(
_varkeywords: &mut Self::Varkeywords,
name: &'py PyAny,
_value: &'py PyAny,
function_description: &FunctionDescription,
) -> PyResult<()> {
Err(function_description.unexpected_keyword_argument(name))
}
}
/// Marker struct which indicates unknown keywords should be collected into a `PyDict`.
pub struct DictVarkeywords;
impl<'py> VarkeywordsHandler<'py> for DictVarkeywords {
type Varkeywords = Option<&'py PyDict>;
#[inline]
fn handle_unexpected_keyword(
varkeywords: &mut Self::Varkeywords,
name: &'py PyAny,
value: &'py PyAny,
_function_description: &FunctionDescription,
) -> PyResult<()> {
varkeywords
.get_or_insert_with(|| PyDict::new(name.py()))
.set_item(name, value)
}
}
fn push_parameter_list(msg: &mut String, parameter_names: &[&str]) {
for (i, parameter) in parameter_names.iter().enumerate() {
if i != 0 {
@ -395,9 +632,12 @@ fn push_parameter_list(msg: &mut String, parameter_names: &[&str]) {
#[cfg(test)]
mod tests {
use crate::{types::PyTuple, AsPyPointer, PyAny, Python, ToPyObject};
use crate::{
types::{IntoPyDict, PyTuple},
AsPyPointer, PyAny, Python, ToPyObject,
};
use super::{push_parameter_list, FunctionDescription};
use super::{push_parameter_list, FunctionDescription, NoVarargs, NoVarkeywords};
#[test]
fn unexpected_keyword_argument() {
@ -408,26 +648,21 @@ mod tests {
positional_only_parameters: 0,
required_positional_parameters: 0,
keyword_only_parameters: &[],
accept_varargs: false,
accept_varkeywords: false,
};
Python::with_gil(|py| {
let err = function_description
.extract_arguments(
py,
[].iter().copied(),
Some(
[(
"foo".to_object(py).into_ref(py),
1u8.to_object(py).into_ref(py),
)]
.iter()
.copied(),
),
&mut [],
)
.unwrap_err();
let err = unsafe {
function_description
.extract_arguments_tuple_dict::<NoVarargs, NoVarkeywords>(
py,
PyTuple::new(py, Vec::<&PyAny>::new()).as_ptr(),
[("foo".to_object(py).into_ref(py), 0u8)]
.into_py_dict(py)
.as_ptr(),
&mut [],
)
.unwrap_err()
};
assert_eq!(
err.to_string(),
"TypeError: example() got an unexpected keyword argument 'foo'"
@ -444,26 +679,21 @@ mod tests {
positional_only_parameters: 0,
required_positional_parameters: 0,
keyword_only_parameters: &[],
accept_varargs: false,
accept_varkeywords: false,
};
Python::with_gil(|py| {
let err = function_description
.extract_arguments(
py,
[].iter().copied(),
Some(
[(
1u8.to_object(py).into_ref(py),
1u8.to_object(py).into_ref(py),
)]
.iter()
.copied(),
),
&mut [],
)
.unwrap_err();
let err = unsafe {
function_description
.extract_arguments_tuple_dict::<NoVarargs, NoVarkeywords>(
py,
PyTuple::new(py, Vec::<&PyAny>::new()).as_ptr(),
[(1u8.to_object(py).into_ref(py), 1u8)]
.into_py_dict(py)
.as_ptr(),
&mut [],
)
.unwrap_err()
};
assert_eq!(
err.to_string(),
"TypeError: 'int' object cannot be converted to 'PyString'"
@ -480,14 +710,12 @@ mod tests {
positional_only_parameters: 0,
required_positional_parameters: 2,
keyword_only_parameters: &[],
accept_varargs: false,
accept_varkeywords: false,
};
Python::with_gil(|py| {
let mut output = [None, None];
let err = unsafe {
function_description.extract_arguments_tuple_dict(
function_description.extract_arguments_tuple_dict::<NoVarargs, NoVarkeywords>(
py,
PyTuple::new(py, Vec::<&PyAny>::new()).as_ptr(),
std::ptr::null_mut(),

2
tests/ui/static_ref.stderr
View file

@ -1,4 +1,4 @@
error[E0495]: cannot infer an appropriate lifetime for lifetime parameter 'py in function call due to conflicting requirements
error[E0495]: cannot infer an appropriate lifetime for lifetime parameter `'py` due to conflicting requirements
--> tests/ui/static_ref.rs:4:1
|
4 | #[pyfunction]