luxolus
/
pyo3
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #3445 from davidhewitt/py2-internals 

add `Py2` as an internal API for optimization and dogfooding
This commit is contained in:
David Hewitt 2023-10-13 06:49:08 +00:00 committed by GitHub
parent 642b335ce3 cac95f31c7
commit db13a97790
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 1668 additions and 276 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

368
src/instance.rs
View File

@ -3,13 +3,15 @@ use crate::err::{self, PyDowncastError, PyErr, PyResult};
use crate::gil;
use crate::pycell::{PyBorrowError, PyBorrowMutError, PyCell};
use crate::pyclass::boolean_struct::{False, True};
use crate::types::any::PyAnyMethods;
use crate::types::{PyDict, PyString, PyTuple};
use crate::{
ffi, AsPyPointer, FromPyObject, IntoPy, PyAny, PyClass, PyClassInitializer, PyRef, PyRefMut,
PyTypeInfo, Python, ToPyObject,
};
use std::marker::PhantomData;
use std::mem;
use std::mem::{self, ManuallyDrop};
use std::ops::Deref;
use std::ptr::NonNull;
/// Types that are built into the Python interpreter.
@ -38,6 +40,180 @@ pub unsafe trait PyNativeType: Sized {
}
}
/// A GIL-attached equivalent to `Py`.
#[repr(transparent)]
pub(crate) struct Py2<'py, T>(Python<'py>, ManuallyDrop<Py<T>>);
impl<'py> Py2<'py, PyAny> {
/// Constructs a new Py2 from a pointer. Panics if ptr is null.
pub(crate) unsafe fn from_owned_ptr(py: Python<'py>, ptr: *mut ffi::PyObject) -> Self {
Self(py, ManuallyDrop::new(Py::from_owned_ptr(py, ptr)))
}
// /// Constructs a new Py2 from a pointer. Returns None if ptr is null.
// ///
// /// Safety: ptr must be a valid pointer to a Python object, or NULL.
// pub unsafe fn from_owned_ptr_or_opt(py: Python<'py>, ptr: *mut ffi::PyObject) -> Option<Self> {
// Py::from_owned_ptr_or_opt(py, ptr).map(|obj| Self(py, ManuallyDrop::new(obj)))
// }
/// Constructs a new Py2 from a pointer. Returns error if ptr is null.
pub(crate) unsafe fn from_owned_ptr_or_err(
py: Python<'py>,
ptr: *mut ffi::PyObject,
) -> PyResult<Self> {
Py::from_owned_ptr_or_err(py, ptr).map(|obj| Self(py, ManuallyDrop::new(obj)))
}
}
impl<'py, T> Py2<'py, T> {
/// Helper to cast to Py2<'py, PyAny>
pub(crate) fn as_any(&self) -> &Py2<'py, PyAny> {
// Safety: all Py2<T> have the same memory layout, and all Py2<T> are valid Py2<PyAny>
unsafe { std::mem::transmute(self) }
}
}
impl<'py, T> std::fmt::Debug for Py2<'py, T> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
let any = self.as_any();
python_format(any, any.repr(), f)
}
}
impl<'py, T> std::fmt::Display for Py2<'py, T> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
let any = self.as_any();
python_format(any, any.str(), f)
}
}
fn python_format(
any: &Py2<'_, PyAny>,
format_result: PyResult<Py2<'_, PyString>>,
f: &mut std::fmt::Formatter<'_>,
) -> Result<(), std::fmt::Error> {
match format_result {
Result::Ok(s) => return f.write_str(&s.as_gil_ref().to_string_lossy()),
Result::Err(err) => {
err.write_unraisable(any.py(), std::option::Option::Some(any.as_gil_ref()))
}
}
match any.get_type().name() {
Result::Ok(name) => std::write!(f, "<unprintable {} object>", name),
Result::Err(_err) => f.write_str("<unprintable object>"),
}
}
impl<'py, T> Deref for Py2<'py, T>
where
T: AsRef<PyAny>,
{
type Target = Py2<'py, PyAny>;
#[inline]
fn deref(&self) -> &Py2<'py, PyAny> {
self.as_any()
}
}
impl<'py, T> AsRef<Py2<'py, PyAny>> for Py2<'py, T>
where
T: AsRef<PyAny>,
{
fn as_ref(&self) -> &Py2<'py, PyAny> {
self.as_any()
}
}
impl<T> Clone for Py2<'_, T> {
fn clone(&self) -> Self {
Self(self.0, ManuallyDrop::new(self.1.clone_ref(self.0)))
}
}
impl<T> Drop for Py2<'_, T> {
fn drop(&mut self) {
unsafe { ffi::Py_DECREF(self.1.as_ptr()) }
}
}
impl<'py, T> Py2<'py, T> {
/// Returns the GIL token associated with this object.
pub fn py(&self) -> Python<'py> {
self.0
}
/// Returns the raw FFI pointer represented by self.
///
/// # Safety
///
/// Callers are responsible for ensuring that the pointer does not outlive self.
///
/// The reference is borrowed; callers should not decrease the reference count
/// when they are finished with the pointer.
#[inline]
pub fn as_ptr(&self) -> *mut ffi::PyObject {
self.1.as_ptr()
}
/// Returns an owned raw FFI pointer represented by self.
///
/// # Safety
///
/// The reference is owned; when finished the caller should either transfer ownership
/// of the pointer or decrease the reference count (e.g. with [`pyo3::ffi::Py_DecRef`](crate::ffi::Py_DecRef)).
#[inline]
pub fn into_ptr(self) -> *mut ffi::PyObject {
self.into_non_null().as_ptr()
}
/// Internal helper to convert e.g. &'a &'py PyDict to &'a Py2<'py, PyDict> for
/// backwards-compatibility during migration to removal of pool.
#[doc(hidden)] // public and doc(hidden) to use in examples and tests for now
pub fn borrowed_from_gil_ref<'a>(gil_ref: &'a &'py T::AsRefTarget) -> &'a Self
where
T: PyTypeInfo,
{
// Safety: &'py T::AsRefTarget is expected to be a Python pointer,
// so &'a &'py T::AsRefTarget has the same layout as &'a Py2<'py, T>
unsafe { std::mem::transmute(gil_ref) }
}
/// Internal helper to get to pool references for backwards compatibility
#[doc(hidden)] // public and doc(hidden) to use in examples and tests for now
pub fn as_gil_ref(&'py self) -> &'py T::AsRefTarget
where
T: PyTypeInfo,
{
unsafe { self.py().from_borrowed_ptr(self.as_ptr()) }
}
/// Internal helper to get to pool references for backwards compatibility
#[doc(hidden)] // public but hidden, to use for tests for now
pub fn into_gil_ref(self) -> &'py T::AsRefTarget
where
T: PyTypeInfo,
{
unsafe { self.py().from_owned_ptr(self.into_ptr()) }
}
// Internal helper to convert `self` into a `NonNull` which owns the
// Python reference.
pub(crate) fn into_non_null(self) -> NonNull<ffi::PyObject> {
// wrap in ManuallyDrop to avoid running Drop for self and decreasing
// the reference count
ManuallyDrop::new(self).1 .0
}
}
unsafe impl<T> AsPyPointer for Py2<'_, T> {
fn as_ptr(&self) -> *mut ffi::PyObject {
self.1.as_ptr()
}
}
/// A GIL-independent reference to an object allocated on the Python heap.
///
/// This type does not auto-dereference to the inner object because you must prove you hold the GIL to access it.
@ -285,8 +461,9 @@ where
/// #
/// Python::with_gil(|py| {
/// let list: Py<PyList> = PyList::empty(py).into();
/// let list: &PyList = list.as_ref(py);
/// assert_eq!(list.len(), 0);
/// // FIXME as_ref() no longer makes sense with new Py API, remove this doc
/// // let list: &PyList = list.as_ref(py);
/// // assert_eq!(list.len(), 0);
/// });
/// ```
///
@ -536,6 +713,17 @@ where
}
impl<T> Py<T> {
/// Attaches this `Py` to the given Python context, allowing access to further Python APIs.
pub(crate) fn attach<'py>(&self, _py: Python<'py>) -> &Py2<'py, T> {
// Safety: `Py2` has the same layout as `Py`
unsafe { &*(self as *const Py<T>).cast() }
}
/// Same as `attach` but takes ownership of `self`.
pub(crate) fn attach_into(self, py: Python<'_>) -> Py2<'_, T> {
Py2(py, ManuallyDrop::new(self))
}
/// Returns whether `self` and `other` point to the same object. To compare
/// the equality of two objects (the `==` operator), use [`eq`](PyAny::eq).
///
@ -638,14 +826,7 @@ impl<T> Py<T> {
where
N: IntoPy<Py<PyString>>,
{
let attr_name = attr_name.into_py(py);
unsafe {
PyObject::from_owned_ptr_or_err(
py,
ffi::PyObject_GetAttr(self.as_ptr(), attr_name.as_ptr()),
)
}
self.attach(py).as_any().getattr(attr_name).map(Into::into)
}
/// Sets an attribute value.
@ -675,12 +856,9 @@ impl<T> Py<T> {
N: IntoPy<Py<PyString>>,
V: IntoPy<Py<PyAny>>,
{
let attr_name = attr_name.into_py(py);
let value = value.into_py(py);
err::error_on_minusone(py, unsafe {
ffi::PyObject_SetAttr(self.as_ptr(), attr_name.as_ptr(), value.as_ptr())
})
self.attach(py)
.as_any()
.setattr(attr_name, value.into_py(py).attach_into(py))
}
/// Calls the object.
@ -692,43 +870,21 @@ impl<T> Py<T> {
args: impl IntoPy<Py<PyTuple>>,
kwargs: Option<&PyDict>,
) -> PyResult<PyObject> {
let args = args.into_py(py);
let kwargs = kwargs.map_or(std::ptr::null_mut(), |p| p.into_ptr());
unsafe {
let ret = PyObject::from_owned_ptr_or_err(
py,
ffi::PyObject_Call(self.as_ptr(), args.as_ptr(), kwargs),
);
ffi::Py_XDECREF(kwargs);
ret
}
self.attach(py).as_any().call(args, kwargs).map(Into::into)
}
/// Calls the object with only positional arguments.
///
/// This is equivalent to the Python expression `self(*args)`.
pub fn call1(&self, py: Python<'_>, args: impl IntoPy<Py<PyTuple>>) -> PyResult<PyObject> {
self.call(py, args, None)
self.attach(py).as_any().call1(args).map(Into::into)
}
/// Calls the object without arguments.
///
/// This is equivalent to the Python expression `self()`.
pub fn call0(&self, py: Python<'_>) -> PyResult<PyObject> {
cfg_if::cfg_if! {
if #[cfg(all(
not(PyPy),
any(Py_3_10, all(not(Py_LIMITED_API), Py_3_9)) // PyObject_CallNoArgs was added to python in 3.9 but to limited API in 3.10
))] {
// Optimized path on python 3.9+
unsafe {
PyObject::from_owned_ptr_or_err(py, ffi::PyObject_CallNoArgs(self.as_ptr()))
}
} else {
self.call(py, (), None)
}
}
self.attach(py).as_any().call0().map(Into::into)
}
/// Calls a method on the object.
@ -748,18 +904,10 @@ impl<T> Py<T> {
N: IntoPy<Py<PyString>>,
A: IntoPy<Py<PyTuple>>,
{
let callee = self.getattr(py, name)?;
let args: Py<PyTuple> = args.into_py(py);
let kwargs = kwargs.map_or(std::ptr::null_mut(), |p| p.into_ptr());
unsafe {
let result = PyObject::from_owned_ptr_or_err(
py,
ffi::PyObject_Call(callee.as_ptr(), args.as_ptr(), kwargs),
);
ffi::Py_XDECREF(kwargs);
result
}
self.attach(py)
.as_any()
.call_method(name, args, kwargs)
.map(Into::into)
}
/// Calls a method on the object with only positional arguments.
@ -773,7 +921,10 @@ impl<T> Py<T> {
N: IntoPy<Py<PyString>>,
A: IntoPy<Py<PyTuple>>,
{
self.call_method(py, name, args, None)
self.attach(py)
.as_any()
.call_method1(name, args)
.map(Into::into)
}
/// Calls a method on the object with no arguments.
@ -786,17 +937,7 @@ impl<T> Py<T> {
where
N: IntoPy<Py<PyString>>,
{
cfg_if::cfg_if! {
if #[cfg(all(Py_3_9, not(any(Py_LIMITED_API, PyPy))))] {
// Optimized path on python 3.9+
unsafe {
let name: Py<PyString> = name.into_py(py);
PyObject::from_owned_ptr_or_err(py, ffi::PyObject_CallMethodNoArgs(self.as_ptr(), name.as_ptr()))
}
} else {
self.call_method(py, name, (), None)
}
}
self.attach(py).as_any().call_method0(name).map(Into::into)
}
/// Create a `Py<T>` instance by taking ownership of the given FFI pointer.
@ -932,6 +1073,31 @@ impl<T> IntoPy<PyObject> for &'_ Py<T> {
}
}
impl<T> ToPyObject for Py2<'_, T> {
/// Converts `Py` instance -> PyObject.
fn to_object(&self, py: Python<'_>) -> PyObject {
unsafe { PyObject::from_borrowed_ptr(py, self.as_ptr()) }
}
}
impl<T> IntoPy<PyObject> for Py2<'_, T> {
/// Converts a `Py` instance to `PyObject`.
/// Consumes `self` without calling `Py_DECREF()`.
#[inline]
fn into_py(self, _py: Python<'_>) -> PyObject {
unsafe { PyObject::from_non_null(self.into_non_null()) }
}
}
impl<T> IntoPy<PyObject> for &Py2<'_, T> {
/// Converts a `Py` instance to `PyObject`.
/// Consumes `self` without calling `Py_DECREF()`.
#[inline]
fn into_py(self, _py: Python<'_>) -> PyObject {
unsafe { PyObject::from_non_null(self.clone().into_non_null()) }
}
}
unsafe impl<T> crate::AsPyPointer for Py<T> {
/// Gets the underlying FFI pointer, returns a borrowed pointer.
#[inline]
@ -965,6 +1131,24 @@ where
}
}
impl<T> std::convert::From<Py2<'_, T>> for PyObject
where
T: AsRef<PyAny>,
{
#[inline]
fn from(other: Py2<'_, T>) -> Self {
let py = other.py();
other.into_py(py)
}
}
impl<T> std::convert::From<Py2<'_, T>> for Py<T> {
#[inline]
fn from(other: Py2<'_, T>) -> Self {
unsafe { Self::from_non_null(other.into_non_null()) }
}
}
// `&PyCell<T>` can be converted to `Py<T>`
impl<T> std::convert::From<&PyCell<T>> for Py<T>
where
@ -1029,6 +1213,19 @@ where
}
}
impl<'a, T> FromPyObject<'a> for Py2<'a, T>
where
T: PyTypeInfo,
{
/// Extracts `Self` from the source `PyObject`.
fn extract(ob: &'a PyAny) -> PyResult<Self> {
Py2::<PyAny>::borrowed_from_gil_ref(&ob)
.downcast()
.map(Clone::clone)
.map_err(Into::into)
}
}
/// `Py<T>` can be used as an error when T is an Error.
///
/// However for GIL lifetime reasons, cause() cannot be implemented for `Py<T>`.
@ -1139,7 +1336,7 @@ impl PyObject {
#[cfg(test)]
mod tests {
use super::{Py, PyObject};
use super::{Py, Py2, PyObject};
use crate::types::{PyDict, PyString};
use crate::{PyAny, PyResult, Python, ToPyObject};
@ -1255,6 +1452,27 @@ a = A()
})
}
#[test]
fn test_py2_from_py_object() {
Python::with_gil(|py| {
let instance: &PyAny = py.eval("object()", None, None).unwrap();
let ptr = instance.as_ptr();
let instance: Py2<'_, PyAny> = instance.extract().unwrap();
assert_eq!(instance.as_ptr(), ptr);
})
}
#[test]
fn test_py2_into_py_object() {
Python::with_gil(|py| {
let instance: Py2<'_, PyAny> =
Py2::borrowed_from_gil_ref(&py.eval("object()", None, None).unwrap()).clone();
let ptr = instance.as_ptr();
let instance: PyObject = instance.clone().into();
assert_eq!(instance.as_ptr(), ptr);
})
}
#[test]
fn test_is_ellipsis() {
Python::with_gil(|py| {
@ -1271,6 +1489,22 @@ a = A()
});
}
#[test]
fn test_debug_fmt() {
Python::with_gil(|py| {
let obj = "hello world".to_object(py).attach_into(py);
assert_eq!(format!("{:?}", obj), "'hello world'");
});
}
#[test]
fn test_display_fmt() {
Python::with_gil(|py| {
let obj = "hello world".to_object(py).attach_into(py);
assert_eq!(format!("{}", obj), "hello world");
});
}
#[cfg(feature = "macros")]
mod using_macros {
use super::*;

3
src/lib.rs
View File

@ -312,6 +312,9 @@ pub use crate::type_object::PyTypeInfo;
pub use crate::types::PyAny;
pub use crate::version::PythonVersionInfo;
// Expected to become public API in 0.21 under a different name
pub(crate) use crate::instance::Py2;
/// Old module which contained some implementation details of the `#[pyproto]` module.
///
/// Prefer using the same content from `pyo3::pyclass`, e.g. `use pyo3::pyclass::CompareOp` instead

4
src/prelude.rs
View File

@ -21,3 +21,7 @@ pub use pyo3_macros::{pyclass, pyfunction, pymethods, pymodule, FromPyObject};
#[cfg(feature = "macros")]
pub use crate::wrap_pyfunction;
// Expected to become public API in 0.21
// pub(crate) use crate::instance::Py2; // Will be stabilized with a different name
// pub(crate) use crate::types::any::PyAnyMethods;

1518
src/types/any.rs
View File

File diff suppressed because it is too large Load Diff

16
src/types/iterator.rs
View File

@ -1,6 +1,8 @@
use crate::{ffi, AsPyPointer, Py, PyAny, PyErr, PyNativeType, PyResult, Python};
use crate::{ffi, AsPyPointer, Py, Py2, PyAny, PyErr, PyNativeType, PyResult, Python};
use crate::{PyDowncastError, PyTryFrom};
use super::any::PyAnyMethods;
/// A Python iterator object.
///
/// # Examples
@ -31,9 +33,17 @@ impl PyIterator {
///
/// Equivalent to Python's built-in `iter` function.
pub fn from_object(obj: &PyAny) -> PyResult<&PyIterator> {
Self::from_object2(Py2::borrowed_from_gil_ref(&obj)).map(|py2| {
// Can't use into_gil_ref here because T: PyTypeInfo bound is not satisfied
// Safety: into_ptr produces a valid pointer to PyIterator object
unsafe { obj.py().from_owned_ptr(py2.into_ptr()) }
})
}
pub(crate) fn from_object2<'py>(obj: &Py2<'py, PyAny>) -> PyResult<Py2<'py, PyIterator>> {
unsafe {
obj.py()
.from_owned_ptr_or_err(ffi::PyObject_GetIter(obj.as_ptr()))
Py2::from_owned_ptr_or_err(obj.py(), ffi::PyObject_GetIter(obj.as_ptr()))
.map(|any| any.downcast_into_unchecked())
}
}
}

2
src/types/mod.rs
View File

@ -269,7 +269,7 @@ macro_rules! pyobject_native_type {
};
}
mod any;
pub(crate) mod any;
mod boolobject;
mod bytearray;
mod bytes;

25
src/types/pysuper.rs
View File

@ -1,5 +1,7 @@
use crate::ffi;
use crate::instance::Py2;
use crate::types::any::PyAnyMethods;
use crate::types::PyType;
use crate::{ffi, PyTypeInfo};
use crate::{PyAny, PyResult};
/// Represents a Python `super` object.
@ -55,9 +57,22 @@ impl PySuper {
/// }
/// ```
pub fn new<'py>(ty: &'py PyType, obj: &'py PyAny) -> PyResult<&'py PySuper> {
let py = ty.py();
let super_ = py.get_type::<PySuper>().call1((ty, obj))?;
let super_ = super_.downcast::<PySuper>()?;
Ok(super_)
Self::new2(
Py2::borrowed_from_gil_ref(&ty),
Py2::borrowed_from_gil_ref(&obj),
)
.map(Py2::into_gil_ref)
}
pub(crate) fn new2<'py>(
ty: &Py2<'py, PyType>,
obj: &Py2<'py, PyAny>,
) -> PyResult<Py2<'py, PySuper>> {
Py2::<PyType>::borrowed_from_gil_ref(&PySuper::type_object(ty.py()))
.call1((ty, obj))
.map(|any| {
// Safety: super() always returns instance of super
unsafe { any.downcast_into_unchecked() }
})
}
}

8
tests/test_super.rs
View File

@ -1,6 +1,6 @@
#![cfg(all(feature = "macros", not(PyPy)))]
use pyo3::prelude::*;
use pyo3::{prelude::*, types::PySuper};
#[pyclass(subclass)]
struct BaseClass {
@ -33,6 +33,11 @@ impl SubClass {
let super_ = self_.py_super()?;
super_.call_method("method", (), None)
}
fn method_super_new(self_: &PyCell<Self>) -> PyResult<&PyAny> {
let super_ = PySuper::new(self_.get_type(), self_)?;
super_.call_method("method", (), None)
}
}
#[test]
@ -45,6 +50,7 @@ fn test_call_super_method() {
r#"
obj = cls()
assert obj.method() == 10
assert obj.method_super_new() == 10
"#
)
});