luxolus
/
pyo3
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #887 from kngwyu/new-nativetypes 

New Native Types and Lighter GILPool
This commit is contained in:
Yuji Kanagawa 2020-05-03 12:10:43 +09:00 committed by GitHub
parent 17cf97df7d 75c807f31d
commit e9bec070e1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 230 additions and 409 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
guide/src/rust_cpython.md
View File

@ -47,7 +47,8 @@ impl MyClass {
## Ownership and lifetimes
All objects are owned by the PyO3 library and all APIs available with references, while in rust-cpython, you own python objects.
While in rust-cpython you always own python objects, PyO3 allows efficient *borrowed objects*
and most APIs are available with references.
Here is an example of the PyList API:
@ -73,7 +74,8 @@ impl PyList {
}
```
Because PyO3 allows only references to Python objects, all references have the GIL lifetime. So the owned Python object is not required, and it is safe to have functions like `fn py<'p>(&'p self) -> Python<'p> {}`.
In PyO3, all object references are bounded by the GIL lifetime.
So the owned Python object is not required, and it is safe to have functions like `fn py<'p>(&'p self) -> Python<'p> {}`.
## Error handling

28
src/conversion.rs
View File

@ -3,7 +3,7 @@
//! Conversions between various states of Rust and Python types and their wrappers.
use crate::err::{self, PyDowncastError, PyResult};
use crate::object::PyObject;
use crate::type_object::{PyDowncastImpl, PyTypeInfo};
use crate::type_object::PyTypeInfo;
use crate::types::PyTuple;
use crate::{ffi, gil, Py, PyAny, PyCell, PyClass, PyNativeType, PyRef, PyRefMut, Python};
use std::ptr::NonNull;
@ -311,7 +311,7 @@ where
/// If `T` implements `PyTryFrom`, we can convert `&PyAny` to `&T`.
///
/// This trait is similar to `std::convert::TryFrom`
pub trait PyTryFrom<'v>: Sized + PyDowncastImpl {
pub trait PyTryFrom<'v>: Sized + PyNativeType {
/// Cast from a concrete Python object type to PyObject.
fn try_from<V: Into<&'v PyAny>>(value: V) -> Result<&'v Self, PyDowncastError>;
@ -348,7 +348,7 @@ where
impl<'v, T> PyTryFrom<'v> for T
where
T: PyDowncastImpl + PyTypeInfo + PyNativeType,
T: PyTypeInfo + PyNativeType,
{
fn try_from<V: Into<&'v PyAny>>(value: V) -> Result<&'v Self, PyDowncastError> {
let value = value.into();
@ -460,28 +460,14 @@ where
T: 'p + crate::PyNativeType,
{
unsafe fn from_owned_ptr_or_opt(py: Python<'p>, ptr: *mut ffi::PyObject) -> Option<&'p Self> {
NonNull::new(ptr).map(|p| Self::unchecked_downcast(gil::register_owned(py, p)))
gil::register_owned(py, NonNull::new(ptr)?);
Some(&*(ptr as *mut Self))
}
unsafe fn from_borrowed_ptr_or_opt(
py: Python<'p>,
_py: Python<'p>,
ptr: *mut ffi::PyObject,
) -> Option<&'p Self> {
NonNull::new(ptr).map(|p| Self::unchecked_downcast(gil::register_borrowed(py, p)))
}
}
unsafe impl<'p, T> FromPyPointer<'p> for PyCell<T>
where
T: PyClass,
{
unsafe fn from_owned_ptr_or_opt(py: Python<'p>, ptr: *mut ffi::PyObject) -> Option<&'p Self> {
NonNull::new(ptr).map(|p| &*(gil::register_owned(py, p).as_ptr() as *const PyCell<T>))
}
unsafe fn from_borrowed_ptr_or_opt(
py: Python<'p>,
ptr: *mut ffi::PyObject,
) -> Option<&'p Self> {
NonNull::new(ptr).map(|p| &*(gil::register_borrowed(py, p).as_ptr() as *const PyCell<T>))
NonNull::new(ptr as *mut Self).map(|p| &*p.as_ptr())
}
}

360
src/gil.rs
View File

@ -2,8 +2,9 @@
//! Interaction with python's global interpreter lock
use crate::{ffi, internal_tricks::Unsendable, PyAny, Python};
use std::cell::{Cell, UnsafeCell};
use crate::{ffi, internal_tricks::Unsendable, Python};
use parking_lot::Mutex;
use std::cell::{Cell, RefCell, UnsafeCell};
use std::{any, mem::ManuallyDrop, ptr::NonNull, sync};
static START: sync::Once = sync::Once::new();
@ -16,6 +17,13 @@ thread_local! {
///
/// As a result, if this thread has the GIL, GIL_COUNT is greater than zero.
static GIL_COUNT: Cell<u32> = Cell::new(0);
/// These are objects owned by the current thread, to be released when the GILPool drops.
static OWNED_OBJECTS: RefCell<Vec<NonNull<ffi::PyObject>>> = RefCell::new(Vec::with_capacity(256));
/// These are non-python objects such as (String) owned by the current thread, to be released
/// when the GILPool drops.
static OWNED_ANYS: RefCell<Vec<Box<dyn any::Any>>> = RefCell::new(Vec::with_capacity(256));
}
/// Check whether the GIL is acquired.
@ -136,90 +144,75 @@ impl GILGuard {
impl Drop for GILGuard {
fn drop(&mut self) {
unsafe {
// Must drop the objects in the pool before releasing the GILGuard
ManuallyDrop::drop(&mut self.pool);
ffi::PyGILState_Release(self.gstate);
}
}
}
/// Implementation of release pool
struct ReleasePoolImpl {
owned: ArrayList<NonNull<ffi::PyObject>>,
borrowed: ArrayList<NonNull<ffi::PyObject>>,
pointers: *mut Vec<NonNull<ffi::PyObject>>,
obj: Vec<Box<dyn any::Any>>,
p: parking_lot::Mutex<*mut Vec<NonNull<ffi::PyObject>>>,
}
impl ReleasePoolImpl {
fn new() -> Self {
Self {
owned: ArrayList::new(),
borrowed: ArrayList::new(),
pointers: Box::into_raw(Box::new(Vec::with_capacity(256))),
obj: Vec::with_capacity(8),
p: parking_lot::Mutex::new(Box::into_raw(Box::new(Vec::with_capacity(256)))),
}
}
unsafe fn release_pointers(&mut self) {
let mut v = self.p.lock();
let vec = &mut **v;
if vec.is_empty() {
return;
}
// switch vectors
std::mem::swap(&mut self.pointers, &mut *v);
drop(v);
// release PyObjects
for ptr in vec.iter_mut() {
ffi::Py_DECREF(ptr.as_ptr());
}
vec.set_len(0);
}
pub unsafe fn drain(&mut self, _py: Python, owned: usize, borrowed: usize) {
// Release owned objects(call decref)
while owned < self.owned.len() {
let last = self.owned.pop_back().unwrap();
ffi::Py_DECREF(last.as_ptr());
}
// Release borrowed objects(don't call decref)
self.borrowed.truncate(borrowed);
self.release_pointers();
self.obj.clear();
}
}
/// Sync wrapper of ReleasePoolImpl
/// Thread-safe storage for objects which were dropped while the GIL was not held.
struct ReleasePool {
value: UnsafeCell<Option<ReleasePoolImpl>>,
pointers_to_drop: Mutex<*mut Vec<NonNull<ffi::PyObject>>>,
pointers_being_dropped: UnsafeCell<*mut Vec<NonNull<ffi::PyObject>>>,
}
impl ReleasePool {
const fn new() -> Self {
Self {
value: UnsafeCell::new(None),
pointers_to_drop: parking_lot::const_mutex(std::ptr::null_mut()),
pointers_being_dropped: UnsafeCell::new(std::ptr::null_mut()),
}
}
/// # Safety
/// This function is not thread safe. Thus, the caller has to have GIL.
#[allow(clippy::mut_from_ref)]
unsafe fn get_or_init(&self) -> &mut ReleasePoolImpl {
(*self.value.get()).get_or_insert_with(ReleasePoolImpl::new)
fn register_pointer(&self, obj: NonNull<ffi::PyObject>) {
let mut storage = self.pointers_to_drop.lock();
if storage.is_null() {
*storage = Box::into_raw(Box::new(Vec::with_capacity(256)))
}
unsafe {
(**storage).push(obj);
}
}
fn release_pointers(&self, _py: Python) {
let mut v = self.pointers_to_drop.lock();
if v.is_null() {
// No pointers have been registered
return;
}
unsafe {
// Function is safe to call because GIL is held, so only one thread can be inside this
// block at a time
let vec = &mut **v;
if vec.is_empty() {
return;
}
// switch vectors
std::mem::swap(&mut *self.pointers_being_dropped.get(), &mut *v);
drop(v);
// release PyObjects
for ptr in vec.iter_mut() {
ffi::Py_DECREF(ptr.as_ptr());
}
vec.set_len(0);
}
}
}
static POOL: ReleasePool = ReleasePool::new();
unsafe impl Sync for ReleasePool {}
static POOL: ReleasePool = ReleasePool::new();
#[doc(hidden)]
pub struct GILPool {
owned: usize,
borrowed: usize,
owned_objects_start: usize,
owned_anys_start: usize,
// Stable solution for impl !Send
no_send: Unsendable,
}
@ -231,11 +224,10 @@ impl GILPool {
pub unsafe fn new() -> GILPool {
increment_gil_count();
// Release objects that were dropped since last GIL acquisition
let pool = POOL.get_or_init();
pool.release_pointers();
POOL.release_pointers(Python::assume_gil_acquired());
GILPool {
owned: pool.owned.len(),
borrowed: pool.borrowed.len(),
owned_objects_start: OWNED_OBJECTS.with(|o| o.borrow().len()),
owned_anys_start: OWNED_ANYS.with(|o| o.borrow().len()),
no_send: Unsendable::default(),
}
}
@ -247,42 +239,66 @@ impl GILPool {
impl Drop for GILPool {
fn drop(&mut self) {
unsafe {
let pool = POOL.get_or_init();
pool.drain(self.python(), self.owned, self.borrowed);
OWNED_OBJECTS.with(|owned_objects| {
// Note: inside this closure we must be careful to not hold a borrow too long, because
// while calling Py_DECREF we may cause other callbacks to run which will need to
// register objects into the GILPool.
let len = owned_objects.borrow().len();
if self.owned_objects_start < len {
let rest = owned_objects
.borrow_mut()
.split_off(self.owned_objects_start);
for obj in rest {
ffi::Py_DECREF(obj.as_ptr());
}
}
});
OWNED_ANYS.with(|owned_anys| owned_anys.borrow_mut().truncate(self.owned_anys_start));
}
decrement_gil_count();
}
}
pub unsafe fn register_any<'p, T: 'static>(obj: T) -> &'p T {
let pool = POOL.get_or_init();
pool.obj.push(Box::new(obj));
pool.obj
.last()
.unwrap()
.as_ref()
.downcast_ref::<T>()
.unwrap()
}
/// Register a Python object pointer inside the release pool, to have reference count decreased
/// next time the GIL is acquired in pyo3.
///
/// # Safety
/// The object must be an owned Python reference.
pub unsafe fn register_pointer(obj: NonNull<ffi::PyObject>) {
let pool = POOL.get_or_init();
if gil_is_acquired() {
ffi::Py_DECREF(obj.as_ptr())
} else {
(**pool.p.lock()).push(obj);
POOL.register_pointer(obj);
}
}
pub unsafe fn register_owned(_py: Python, obj: NonNull<ffi::PyObject>) -> &PyAny {
let pool = POOL.get_or_init();
&*(pool.owned.push_back(obj) as *const _ as *const PyAny)
/// Register an owned object inside the GILPool.
///
/// # Safety
/// The object must be an owned Python reference.
pub unsafe fn register_owned(_py: Python, obj: NonNull<ffi::PyObject>) {
debug_assert!(gil_is_acquired());
OWNED_OBJECTS.with(|objs| objs.borrow_mut().push(obj));
}
pub unsafe fn register_borrowed(_py: Python, obj: NonNull<ffi::PyObject>) -> &PyAny {
let pool = POOL.get_or_init();
&*(pool.borrowed.push_back(obj) as *const _ as *const PyAny)
/// Register any value inside the GILPool.
///
/// # Safety
/// It is the caller's responsibility to ensure that the inferred lifetime 'p is not inferred by
/// the Rust compiler to outlast the current GILPool.
pub unsafe fn register_any<'p, T: 'static>(obj: T) -> &'p T {
debug_assert!(gil_is_acquired());
OWNED_ANYS.with(|owned_anys| {
let boxed = Box::new(obj);
let value_ref: &T = &*boxed;
// Sneaky - extend the lifetime of the reference so that the box can be moved
let value_ref_extended_lifetime = std::mem::transmute(value_ref);
owned_anys.borrow_mut().push(boxed);
value_ref_extended_lifetime
})
}
/// Increment pyo3's internal GIL count - to be called whenever GILPool or GILGuard is created.
@ -304,70 +320,11 @@ fn decrement_gil_count() {
})
}
use self::array_list::ArrayList;
mod array_list {
use std::collections::LinkedList;
const BLOCK_SIZE: usize = 256;
/// A container type for Release Pool
/// See #271 for why this is crated
pub(super) struct ArrayList<T> {
inner: LinkedList<[Option<T>; BLOCK_SIZE]>,
length: usize,
}
impl<T: Copy> ArrayList<T> {
pub fn new() -> Self {
ArrayList {
inner: LinkedList::new(),
length: 0,
}
}
pub fn push_back(&mut self, item: T) -> &T {
let next_idx = self.next_idx();
if next_idx == 0 {
self.inner.push_back([None; BLOCK_SIZE]);
}
self.inner.back_mut().unwrap()[next_idx] = Some(item);
self.length += 1;
self.inner.back().unwrap()[next_idx].as_ref().unwrap()
}
pub fn pop_back(&mut self) -> Option<T> {
self.length -= 1;
let current_idx = self.next_idx();
if current_idx == 0 {
let last_list = self.inner.pop_back()?;
return last_list[0];
}
self.inner.back().and_then(|arr| arr[current_idx])
}
pub fn len(&self) -> usize {
self.length
}
pub fn truncate(&mut self, new_len: usize) {
if self.length <= new_len {
return;
}
while self.inner.len() > (new_len + BLOCK_SIZE - 1) / BLOCK_SIZE {
self.inner.pop_back();
}
self.length = new_len;
}
fn next_idx(&self) -> usize {
self.length % BLOCK_SIZE
}
}
}
#[cfg(test)]
mod test {
use super::{GILPool, NonNull, GIL_COUNT, POOL};
use crate::object::PyObject;
use crate::AsPyPointer;
use crate::Python;
use crate::ToPyObject;
use crate::{ffi, gil};
use super::{GILPool, GIL_COUNT, OWNED_OBJECTS};
use crate::{ffi, gil, AsPyPointer, IntoPyPointer, PyObject, Python, ToPyObject};
use std::ptr::NonNull;
fn get_object() -> PyObject {
// Convenience function for getting a single unique object
@ -379,6 +336,10 @@ mod test {
obj.to_object(py)
}
fn owned_object_count() -> usize {
OWNED_OBJECTS.with(|objs| objs.borrow().len())
}
#[test]
fn test_owned() {
let gil = Python::acquire_gil();
@ -389,19 +350,16 @@ mod test {
let _ref = obj.clone_ref(py);
unsafe {
let p = POOL.get_or_init();
{
let gil = Python::acquire_gil();
let py = gil.python();
let _ = gil::register_owned(py, obj.into_nonnull());
gil::register_owned(gil.python(), NonNull::new_unchecked(obj.into_ptr()));
assert_eq!(ffi::Py_REFCNT(obj_ptr), 2);
assert_eq!(p.owned.len(), 1);
assert_eq!(owned_object_count(), 1);
}
{
let _gil = Python::acquire_gil();
assert_eq!(p.owned.len(), 0);
assert_eq!(owned_object_count(), 0);
assert_eq!(ffi::Py_REFCNT(obj_ptr), 1);
}
}
@ -417,86 +375,24 @@ mod test {
let obj_ptr = obj.as_ptr();
unsafe {
let p = POOL.get_or_init();
{
let _pool = GILPool::new();
assert_eq!(p.owned.len(), 0);
assert_eq!(owned_object_count(), 0);
let _ = gil::register_owned(py, obj.into_nonnull());
gil::register_owned(py, NonNull::new_unchecked(obj.into_ptr()));
assert_eq!(p.owned.len(), 1);
assert_eq!(owned_object_count(), 1);
assert_eq!(ffi::Py_REFCNT(obj_ptr), 2);
{
let _pool = GILPool::new();
let obj = get_object();
let _ = gil::register_owned(py, obj.into_nonnull());
assert_eq!(p.owned.len(), 2);
gil::register_owned(py, NonNull::new_unchecked(obj.into_ptr()));
assert_eq!(owned_object_count(), 2);
}
assert_eq!(p.owned.len(), 1);
assert_eq!(owned_object_count(), 1);
}
{
assert_eq!(p.owned.len(), 0);
assert_eq!(ffi::Py_REFCNT(obj_ptr), 1);
}
}
}
#[test]
fn test_borrowed() {
unsafe {
let p = POOL.get_or_init();
let obj = get_object();
let obj_ptr = obj.as_ptr();
{
let gil = Python::acquire_gil();
let py = gil.python();
assert_eq!(p.borrowed.len(), 0);
gil::register_borrowed(py, NonNull::new(obj_ptr).unwrap());
assert_eq!(p.borrowed.len(), 1);
assert_eq!(ffi::Py_REFCNT(obj_ptr), 1);
}
{
let _gil = Python::acquire_gil();
assert_eq!(p.borrowed.len(), 0);
assert_eq!(ffi::Py_REFCNT(obj_ptr), 1);
}
}
}
#[test]
fn test_borrowed_nested() {
unsafe {
let p = POOL.get_or_init();
let obj = get_object();
let obj_ptr = obj.as_ptr();
{
let gil = Python::acquire_gil();
let py = gil.python();
assert_eq!(p.borrowed.len(), 0);
gil::register_borrowed(py, NonNull::new(obj_ptr).unwrap());
assert_eq!(p.borrowed.len(), 1);
assert_eq!(ffi::Py_REFCNT(obj_ptr), 1);
{
let _pool = GILPool::new();
assert_eq!(p.borrowed.len(), 1);
gil::register_borrowed(py, NonNull::new(obj_ptr).unwrap());
assert_eq!(p.borrowed.len(), 2);
}
assert_eq!(p.borrowed.len(), 1);
assert_eq!(ffi::Py_REFCNT(obj_ptr), 1);
}
{
let _gil = Python::acquire_gil();
assert_eq!(p.borrowed.len(), 0);
assert_eq!(owned_object_count(), 0);
assert_eq!(ffi::Py_REFCNT(obj_ptr), 1);
}
}
@ -512,10 +408,8 @@ mod test {
let obj_ptr = obj.as_ptr();
unsafe {
let p = POOL.get_or_init();
{
assert_eq!(p.owned.len(), 0);
assert_eq!(owned_object_count(), 0);
assert_eq!(ffi::Py_REFCNT(obj_ptr), 2);
}
@ -535,10 +429,8 @@ mod test {
let obj_ptr = obj.as_ptr();
unsafe {
let p = POOL.get_or_init();
{
assert_eq!(p.owned.len(), 0);
assert_eq!(owned_object_count(), 0);
assert_eq!(ffi::Py_REFCNT(obj_ptr), 2);
}

15
src/instance.rs
View File

@ -3,7 +3,7 @@ use crate::err::{PyErr, PyResult};
use crate::gil;
use crate::object::PyObject;
use crate::objectprotocol::ObjectProtocol;
use crate::type_object::{PyBorrowFlagLayout, PyDowncastImpl};
use crate::type_object::PyBorrowFlagLayout;
use crate::{
ffi, AsPyPointer, FromPyObject, IntoPy, IntoPyPointer, PyAny, PyCell, PyClass,
PyClassInitializer, PyRef, PyRefMut, PyTypeInfo, Python, ToPyObject,
@ -21,6 +21,15 @@ pub unsafe trait PyNativeType: Sized {
fn py(&self) -> Python {
unsafe { Python::assume_gil_acquired() }
}
/// Cast `&PyAny` to `&Self` without no type checking.
///
/// # Safety
///
/// `obj` must have the same layout as `*const ffi::PyObject` and must be
/// an instance of a type corresponding to `Self`.
unsafe fn unchecked_downcast(obj: &PyAny) -> &Self {
&*(obj.as_ptr() as *const Self)
}
}
/// A Python object of known type.
@ -176,8 +185,8 @@ where
{
type Target = T::AsRefTarget;
fn as_ref<'p>(&'p self, _py: Python<'p>) -> &'p Self::Target {
let any = self as *const Py<T> as *const PyAny;
unsafe { PyDowncastImpl::unchecked_downcast(&*any) }
let any = self.as_ptr() as *const PyAny;
unsafe { PyNativeType::unchecked_downcast(&*any) }
}
}

8
src/object.rs
View File

@ -32,12 +32,6 @@ impl PyObject {
PyObject(ptr)
}
pub(crate) unsafe fn into_nonnull(self) -> NonNull<ffi::PyObject> {
let res = self.0;
std::mem::forget(self); // Avoid Drop
res
}
/// Creates a `PyObject` instance for the given FFI pointer.
/// This moves ownership over the pointer into the `PyObject`.
/// Undefined behavior if the pointer is NULL or invalid.
@ -268,7 +262,7 @@ impl PyObject {
impl AsPyRef for PyObject {
type Target = PyAny;
fn as_ref<'p>(&'p self, _py: Python<'p>) -> &'p PyAny {
unsafe { &*(self as *const _ as *const PyAny) }
unsafe { &*(self.as_ptr() as *const PyAny) }
}
}

13
src/pycell.rs
View File

@ -2,8 +2,8 @@
use crate::conversion::{AsPyPointer, FromPyPointer, ToPyObject};
use crate::pyclass_init::PyClassInitializer;
use crate::pyclass_slots::{PyClassDict, PyClassWeakRef};
use crate::type_object::{PyBorrowFlagLayout, PyDowncastImpl, PyLayout, PySizedLayout, PyTypeInfo};
use crate::{ffi, FromPy, PyAny, PyClass, PyErr, PyNativeType, PyObject, PyResult, Python};
use crate::type_object::{PyBorrowFlagLayout, PyLayout, PySizedLayout, PyTypeInfo};
use crate::{ffi, FromPy, PyClass, PyErr, PyNativeType, PyObject, PyResult, Python};
use std::cell::{Cell, UnsafeCell};
use std::fmt;
use std::mem::ManuallyDrop;
@ -159,6 +159,8 @@ pub struct PyCell<T: PyClass> {
weakref: T::WeakRef,
}
unsafe impl<T: PyClass> PyNativeType for PyCell<T> {}
impl<T: PyClass> PyCell<T> {
/// Make new `PyCell` on the Python heap and returns the reference of it.
///
@ -360,13 +362,6 @@ unsafe impl<T: PyClass> PyLayout<T> for PyCell<T> {
}
}
unsafe impl<T: PyClass> PyDowncastImpl for PyCell<T> {
unsafe fn unchecked_downcast(obj: &PyAny) -> &Self {
&*(obj.as_ptr() as *const Self)
}
private_impl! {}
}
impl<T: PyClass> AsPyPointer for PyCell<T> {
fn as_ptr(&self) -> *mut ffi::PyObject {
self.inner.as_ptr()

29
src/python.rs
View File

@ -3,17 +3,15 @@
// based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython
use crate::err::{PyDowncastError, PyErr, PyResult};
use crate::ffi;
use crate::gil::{self, GILGuard};
use crate::instance::AsPyRef;
use crate::object::PyObject;
use crate::type_object::{PyDowncastImpl, PyTypeInfo, PyTypeObject};
use crate::type_object::{PyTypeInfo, PyTypeObject};
use crate::types::{PyAny, PyDict, PyModule, PyType};
use crate::{AsPyPointer, FromPyPointer, IntoPyPointer, PyTryFrom};
use crate::{
ffi, AsPyPointer, AsPyRef, FromPyPointer, IntoPyPointer, PyNativeType, PyObject, PyTryFrom,
};
use std::ffi::CString;
use std::marker::PhantomData;
use std::os::raw::c_int;
use std::ptr::NonNull;
pub use gil::prepare_freethreaded_python;
@ -293,27 +291,18 @@ impl<'p> Python<'p> {
where
T: PyTryFrom<'p>,
{
let obj = unsafe { gil::register_owned(self, obj.into_nonnull()) };
<T as PyTryFrom>::try_from(obj)
let any: &PyAny = unsafe { self.from_owned_ptr(obj.into_ptr()) };
<T as PyTryFrom>::try_from(any)
}
/// Registers the object in the release pool, and does an unchecked downcast
/// to the specific type.
pub unsafe fn cast_as<T>(self, obj: PyObject) -> &'p T
where
T: PyDowncastImpl + PyTypeInfo,
T: PyNativeType + PyTypeInfo,
{
let obj = gil::register_owned(self, obj.into_nonnull());
T::unchecked_downcast(obj)
}
/// Registers the object pointer in the release pool.
#[allow(clippy::wrong_self_convention)]
pub unsafe fn from_borrowed_ptr_to_obj(self, ptr: *mut ffi::PyObject) -> &'p PyAny {
match NonNull::new(ptr) {
Some(p) => gil::register_borrowed(self, p),
None => crate::err::panic_after_error(),
}
let any: &PyAny = self.from_owned_ptr(obj.into_ptr());
T::unchecked_downcast(any)
}
/// Registers the object pointer in the release pool,

28
src/type_object.rs
View File

@ -63,32 +63,6 @@ pub mod type_flags {
pub const EXTENDED: usize = 1 << 4;
}
/// Reference abstraction for `PyClass` and `PyNativeType`. Used internaly.
// NOTE(kngwyu):
// `&PyCell` is a pointer of `ffi::PyObject` but `&PyAny` is a pointer of a pointer,
// so we need abstraction.
// This mismatch eventually should be fixed(e.g., https://github.com/PyO3/pyo3/issues/679).
pub unsafe trait PyDowncastImpl {
/// Cast `&PyAny` to `&Self` without no type checking.
///
/// # Safety
///
/// Unless obj is not an instance of a type corresponding to Self,
/// this method causes undefined behavior.
unsafe fn unchecked_downcast(obj: &PyAny) -> &Self;
private_decl! {}
}
unsafe impl<'py, T> PyDowncastImpl for T
where
T: 'py + crate::PyNativeType,
{
unsafe fn unchecked_downcast(obj: &PyAny) -> &Self {
&*(obj as *const _ as *const Self)
}
private_impl! {}
}
/// Python type information.
/// All Python native types(e.g., `PyDict`) and `#[pyclass]` structs implement this trait.
///
@ -124,7 +98,7 @@ pub unsafe trait PyTypeInfo: Sized {
type Initializer: PyObjectInit<Self>;
/// Utility type to make AsPyRef work
type AsRefTarget: PyDowncastImpl;
type AsRefTarget: crate::PyNativeType;
/// PyTypeObject instance for this type.
fn type_object() -> &'static ffi::PyTypeObject;

27
src/types/any.rs
View File

@ -1,7 +1,7 @@
use crate::conversion::PyTryFrom;
use crate::conversion::{AsPyPointer, PyTryFrom};
use crate::err::PyDowncastError;
use crate::internal_tricks::Unsendable;
use crate::{ffi, PyObject};
use crate::ffi;
use std::cell::UnsafeCell;
/// A Python object with GIL lifetime
///
@ -28,10 +28,26 @@ use crate::{ffi, PyObject};
/// assert!(any.downcast::<PyList>().is_err());
/// ```
#[repr(transparent)]
pub struct PyAny(PyObject, Unsendable);
pub struct PyAny(UnsafeCell<ffi::PyObject>);
impl crate::AsPyPointer for PyAny {
#[inline]
fn as_ptr(&self) -> *mut ffi::PyObject {
self.0.get()
}
}
impl PartialEq for PyAny {
#[inline]
fn eq(&self, o: &PyAny) -> bool {
self.as_ptr() == o.as_ptr()
}
}
unsafe impl crate::PyNativeType for PyAny {}
unsafe impl crate::type_object::PyLayout<PyAny> for ffi::PyObject {}
impl crate::type_object::PySizedLayout<PyAny> for ffi::PyObject {}
pyobject_native_type_named!(PyAny);
pyobject_native_type_convert!(
PyAny,
ffi::PyObject,
@ -39,6 +55,7 @@ pyobject_native_type_convert!(
Some("builtins"),
ffi::PyObject_Check
);
pyobject_native_type_extract!(PyAny);
impl PyAny {

3
src/types/boolobject.rs
View File

@ -1,5 +1,4 @@
// Copyright (c) 2017-present PyO3 Project and Contributors
use crate::internal_tricks::Unsendable;
use crate::{
ffi, AsPyPointer, FromPy, FromPyObject, PyAny, PyObject, PyResult, PyTryFrom, Python,
ToPyObject,
@ -7,7 +6,7 @@ use crate::{
/// Represents a Python `bool`.
#[repr(transparent)]
pub struct PyBool(PyObject, Unsendable);
pub struct PyBool(PyAny);
pyobject_native_type!(PyBool, ffi::PyObject, ffi::PyBool_Type, ffi::PyBool_Check);

16
src/types/bytearray.rs
View File

@ -1,17 +1,13 @@
// Copyright (c) 2017-present PyO3 Project and Contributors
use crate::err::{PyErr, PyResult};
use crate::ffi;
use crate::instance::PyNativeType;
use crate::internal_tricks::Unsendable;
use crate::object::PyObject;
use crate::AsPyPointer;
use crate::Python;
use crate::{ffi, AsPyPointer, PyAny, Python};
use std::os::raw::c_char;
use std::slice;
/// Represents a Python `bytearray`.
#[repr(transparent)]
pub struct PyByteArray(PyObject, Unsendable);
pub struct PyByteArray(PyAny);
pyobject_native_var_type!(PyByteArray, ffi::PyByteArray_Type, ffi::PyByteArray_Check);
@ -38,7 +34,7 @@ impl PyByteArray {
#[inline]
pub fn len(&self) -> usize {
// non-negative Py_ssize_t should always fit into Rust usize
unsafe { ffi::PyByteArray_Size(self.0.as_ptr()) as usize }
unsafe { ffi::PyByteArray_Size(self.as_ptr()) as usize }
}
/// Checks if the bytearray is empty.
@ -69,8 +65,8 @@ impl PyByteArray {
/// ```
pub fn to_vec(&self) -> Vec<u8> {
let slice = unsafe {
let buffer = ffi::PyByteArray_AsString(self.0.as_ptr()) as *mut u8;
let length = ffi::PyByteArray_Size(self.0.as_ptr()) as usize;
let buffer = ffi::PyByteArray_AsString(self.as_ptr()) as *mut u8;
let length = ffi::PyByteArray_Size(self.as_ptr()) as usize;
slice::from_raw_parts_mut(buffer, length)
};
slice.to_vec()
@ -79,7 +75,7 @@ impl PyByteArray {
/// Resizes the bytearray object to the new length `len`.
pub fn resize(&self, len: usize) -> PyResult<()> {
unsafe {
let result = ffi::PyByteArray_Resize(self.0.as_ptr(), len as ffi::Py_ssize_t);
let result = ffi::PyByteArray_Resize(self.as_ptr(), len as ffi::Py_ssize_t);
if result == 0 {
Ok(())
} else {

3
src/types/bytes.rs
View File

@ -1,4 +1,3 @@
use crate::internal_tricks::Unsendable;
use crate::{
ffi, AsPyPointer, FromPy, FromPyObject, PyAny, PyObject, PyResult, PyTryFrom, Python,
ToPyObject,
@ -12,7 +11,7 @@ use std::str;
///
/// This type is immutable.
#[repr(transparent)]
pub struct PyBytes(PyObject, Unsendable);
pub struct PyBytes(PyAny);
pyobject_native_var_type!(PyBytes, ffi::PyBytes_Type, ffi::PyBytes_Check);

10
src/types/complex.rs
View File

@ -1,17 +1,13 @@
use crate::ffi;
#[cfg(not(PyPy))]
use crate::instance::PyNativeType;
use crate::internal_tricks::Unsendable;
use crate::AsPyPointer;
use crate::PyObject;
use crate::Python;
use crate::{ffi, AsPyPointer, PyAny, Python};
#[cfg(not(PyPy))]
use std::ops::*;
use std::os::raw::c_double;
/// Represents a Python `complex`.
#[repr(transparent)]
pub struct PyComplex(PyObject, Unsendable);
pub struct PyComplex(PyAny);
pyobject_native_type!(
PyComplex,
@ -133,7 +129,7 @@ impl<'py> Neg for &'py PyComplex {
#[cfg(feature = "num-complex")]
mod complex_conversion {
use super::*;
use crate::{FromPyObject, PyAny, PyErr, PyResult, ToPyObject};
use crate::{FromPyObject, PyAny, PyErr, PyObject, PyResult, ToPyObject};
use num_complex::Complex;
impl PyComplex {

20
src/types/datetime.rs
View File

@ -25,12 +25,9 @@ use crate::ffi::{
PyDateTime_TIME_GET_HOUR, PyDateTime_TIME_GET_MICROSECOND, PyDateTime_TIME_GET_MINUTE,
PyDateTime_TIME_GET_SECOND,
};
use crate::internal_tricks::Unsendable;
use crate::object::PyObject;
use crate::types::PyTuple;
use crate::AsPyPointer;
use crate::Python;
use crate::ToPyObject;
use crate::{AsPyPointer, PyAny, Python, ToPyObject};
use std::os::raw::c_int;
#[cfg(not(PyPy))]
use std::ptr;
@ -66,7 +63,8 @@ pub trait PyTimeAccess {
}
/// Bindings around `datetime.date`
pub struct PyDate(PyObject, Unsendable);
#[repr(transparent)]
pub struct PyDate(PyAny);
pyobject_native_type!(
PyDate,
crate::ffi::PyDateTime_Date,
@ -122,7 +120,8 @@ impl PyDateAccess for PyDate {
}
/// Bindings for `datetime.datetime`
pub struct PyDateTime(PyObject, Unsendable);
#[repr(transparent)]
pub struct PyDateTime(PyAny);
pyobject_native_type!(
PyDateTime,
crate::ffi::PyDateTime_DateTime,
@ -232,7 +231,8 @@ impl PyTimeAccess for PyDateTime {
}
/// Bindings for `datetime.time`
pub struct PyTime(PyObject, Unsendable);
#[repr(transparent)]
pub struct PyTime(PyAny);
pyobject_native_type!(
PyTime,
crate::ffi::PyDateTime_Time,
@ -317,7 +317,8 @@ impl PyTimeAccess for PyTime {
/// Bindings for `datetime.tzinfo`
///
/// This is an abstract base class and should not be constructed directly.
pub struct PyTzInfo(PyObject, Unsendable);
#[repr(transparent)]
pub struct PyTzInfo(PyAny);
pyobject_native_type!(
PyTzInfo,
crate::ffi::PyObject,
@ -327,7 +328,8 @@ pyobject_native_type!(
);
/// Bindings for `datetime.timedelta`
pub struct PyDelta(PyObject, Unsendable);
#[repr(transparent)]
pub struct PyDelta(PyAny);
pyobject_native_type!(
PyDelta,
crate::ffi::PyDateTime_Delta,

11
src/types/dict.rs
View File

@ -2,22 +2,19 @@
use crate::err::{self, PyErr, PyResult};
use crate::instance::PyNativeType;
use crate::internal_tricks::Unsendable;
use crate::object::PyObject;
use crate::types::{PyAny, PyList};
use crate::AsPyPointer;
#[cfg(not(PyPy))]
use crate::IntoPyPointer;
use crate::Python;
use crate::{ffi, IntoPy};
use crate::{FromPyObject, PyTryFrom};
use crate::{ToBorrowedObject, ToPyObject};
use crate::{
ffi, AsPyPointer, FromPyObject, IntoPy, PyTryFrom, Python, ToBorrowedObject, ToPyObject,
};
use std::collections::{BTreeMap, HashMap};
use std::{cmp, collections, hash};
/// Represents a Python `dict`.
#[repr(transparent)]
pub struct PyDict(PyObject, Unsendable);
pub struct PyDict(PyAny);
pyobject_native_type!(
PyDict,

5
src/types/floatob.rs
View File

@ -1,7 +1,6 @@
// Copyright (c) 2017-present PyO3 Project and Contributors
//
// based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython
use crate::internal_tricks::Unsendable;
use crate::{
ffi, AsPyPointer, FromPy, FromPyObject, ObjectProtocol, PyAny, PyErr, PyNativeType, PyObject,
PyResult, Python, ToPyObject,
@ -15,7 +14,7 @@ use std::os::raw::c_double;
/// and [extract](struct.PyObject.html#method.extract)
/// with `f32`/`f64`.
#[repr(transparent)]
pub struct PyFloat(PyObject, Unsendable);
pub struct PyFloat(PyAny);
pyobject_native_type!(
PyFloat,
@ -32,7 +31,7 @@ impl PyFloat {
/// Gets the value of this float.
pub fn value(&self) -> c_double {
unsafe { ffi::PyFloat_AsDouble(self.0.as_ptr()) }
unsafe { ffi::PyFloat_AsDouble(self.as_ptr()) }
}
}

3
src/types/list.rs
View File

@ -4,7 +4,6 @@
use crate::err::{self, PyResult};
use crate::ffi::{self, Py_ssize_t};
use crate::internal_tricks::Unsendable;
use crate::{
AsPyPointer, IntoPy, IntoPyPointer, PyAny, PyNativeType, PyObject, Python, ToBorrowedObject,
ToPyObject,
@ -12,7 +11,7 @@ use crate::{
/// Represents a Python `list`.
#[repr(transparent)]
pub struct PyList(PyObject, Unsendable);
pub struct PyList(PyAny);
pyobject_native_var_type!(PyList, ffi::PyList_Type, ffi::PyList_Check);

4
src/types/mod.rs
View File

@ -31,7 +31,7 @@ macro_rules! pyobject_native_type_named (
impl<$($type_param,)*> ::std::convert::AsRef<$crate::PyAny> for $name {
#[inline]
fn as_ref(&self) -> &$crate::PyAny {
unsafe{&*(self as *const $name as *const $crate::PyAny)}
unsafe { &*(self.as_ptr() as *const $crate::PyAny) }
}
}
@ -150,7 +150,7 @@ macro_rules! pyobject_native_type_convert(
#[inline]
fn to_object(&self, py: $crate::Python) -> $crate::PyObject {
use $crate::AsPyPointer;
unsafe {$crate::PyObject::from_borrowed_ptr(py, self.0.as_ptr())}
unsafe { $crate::PyObject::from_borrowed_ptr(py, self.as_ptr()) }
}
}

3
src/types/module.rs
View File

@ -6,7 +6,6 @@ use crate::err::{PyErr, PyResult};
use crate::exceptions;
use crate::ffi;
use crate::instance::PyNativeType;
use crate::internal_tricks::Unsendable;
use crate::object::PyObject;
use crate::objectprotocol::ObjectProtocol;
use crate::pyclass::PyClass;
@ -20,7 +19,7 @@ use std::str;
/// Represents a Python `module` object.
#[repr(transparent)]
pub struct PyModule(PyObject, Unsendable);
pub struct PyModule(PyAny);
pyobject_native_var_type!(PyModule, ffi::PyModule_Type, ffi::PyModule_Check);

3
src/types/num.rs
View File

@ -2,7 +2,6 @@
//
// based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython
use crate::internal_tricks::Unsendable;
use crate::{
exceptions, ffi, AsPyPointer, FromPyObject, IntoPy, PyAny, PyErr, PyNativeType, PyObject,
PyResult, Python, ToPyObject,
@ -111,7 +110,7 @@ macro_rules! int_convert_128 {
/// and [extract](struct.PyObject.html#method.extract)
/// with the primitive Rust integer types.
#[repr(transparent)]
pub struct PyLong(PyObject, Unsendable);
pub struct PyLong(PyAny);
pyobject_native_var_type!(PyLong, ffi::PyLong_Type, ffi::PyLong_Check);

4
src/types/sequence.rs
View File

@ -5,8 +5,6 @@ use crate::err::{self, PyDowncastError, PyErr, PyResult};
use crate::exceptions;
use crate::ffi::{self, Py_ssize_t};
use crate::instance::PyNativeType;
use crate::internal_tricks::Unsendable;
use crate::object::PyObject;
use crate::objectprotocol::ObjectProtocol;
use crate::types::{PyAny, PyList, PyTuple};
use crate::AsPyPointer;
@ -14,7 +12,7 @@ use crate::{FromPyObject, PyTryFrom, ToBorrowedObject};
/// Represents a reference to a Python object supporting the sequence protocol.
#[repr(transparent)]
pub struct PySequence(PyObject, Unsendable);
pub struct PySequence(PyAny);
pyobject_native_type_named!(PySequence);
pyobject_native_type_extract!(PySequence);

5
src/types/set.rs
View File

@ -2,7 +2,6 @@
//
use crate::err::{self, PyErr, PyResult};
use crate::internal_tricks::Unsendable;
use crate::{
ffi, AsPyPointer, FromPy, FromPyObject, IntoPy, PyAny, PyNativeType, PyObject, Python,
ToBorrowedObject, ToPyObject,
@ -13,11 +12,11 @@ use std::{collections, hash, ptr};
/// Represents a Python `set`
#[repr(transparent)]
pub struct PySet(PyObject, Unsendable);
pub struct PySet(PyAny);
/// Represents a Python `frozenset`
#[repr(transparent)]
pub struct PyFrozenSet(PyObject, Unsendable);
pub struct PyFrozenSet(PyAny);
pyobject_native_type!(PySet, ffi::PySetObject, ffi::PySet_Type, ffi::PySet_Check);
pyobject_native_type!(

7
src/types/slice.rs
View File

@ -3,17 +3,14 @@
use crate::err::{PyErr, PyResult};
use crate::ffi::{self, Py_ssize_t};
use crate::instance::PyNativeType;
use crate::internal_tricks::Unsendable;
use crate::object::PyObject;
use crate::Python;
use crate::{AsPyPointer, ToPyObject};
use crate::{AsPyPointer, PyAny, PyObject, Python, ToPyObject};
use std::os::raw::c_long;
/// Represents a Python `slice`.
///
/// Only `c_long` indices supported at the moment by the `PySlice` object.
#[repr(transparent)]
pub struct PySlice(PyObject, Unsendable);
pub struct PySlice(PyAny);
pyobject_native_type!(
PySlice,

7
src/types/string.rs
View File

@ -1,6 +1,5 @@
// Copyright (c) 2017-present PyO3 Project and Contributors
use crate::internal_tricks::Unsendable;
use crate::{
ffi, gil, AsPyPointer, FromPy, FromPyObject, IntoPy, PyAny, PyErr, PyNativeType, PyObject,
PyResult, PyTryFrom, Python, ToPyObject,
@ -15,7 +14,7 @@ use std::str;
///
/// This type is immutable.
#[repr(transparent)]
pub struct PyString(PyObject, Unsendable);
pub struct PyString(PyAny);
pyobject_native_var_type!(PyString, ffi::PyUnicode_Type, ffi::PyUnicode_Check);
@ -48,7 +47,7 @@ impl PyString {
pub fn as_bytes(&self) -> PyResult<&[u8]> {
unsafe {
let mut size: ffi::Py_ssize_t = 0;
let data = ffi::PyUnicode_AsUTF8AndSize(self.0.as_ptr(), &mut size) as *const u8;
let data = ffi::PyUnicode_AsUTF8AndSize(self.as_ptr(), &mut size) as *const u8;
if data.is_null() {
Err(PyErr::fetch(self.py()))
} else {
@ -74,7 +73,7 @@ impl PyString {
Err(_) => {
unsafe {
let py_bytes = ffi::PyUnicode_AsEncodedString(
self.0.as_ptr(),
self.as_ptr(),
CStr::from_bytes_with_nul(b"utf-8\0").unwrap().as_ptr(),
CStr::from_bytes_with_nul(b"surrogatepass\0")
.unwrap()

3
src/types/tuple.rs
View File

@ -1,7 +1,6 @@
// Copyright (c) 2017-present PyO3 Project and Contributors
use crate::ffi::{self, Py_ssize_t};
use crate::internal_tricks::Unsendable;
use crate::{
exceptions, AsPyPointer, AsPyRef, FromPy, FromPyObject, IntoPy, IntoPyPointer, Py, PyAny,
PyErr, PyNativeType, PyObject, PyResult, PyTryFrom, Python, ToPyObject,
@ -12,7 +11,7 @@ use std::slice;
///
/// This type is immutable.
#[repr(transparent)]
pub struct PyTuple(PyObject, Unsendable);
pub struct PyTuple(PyAny);
pyobject_native_var_type!(PyTuple, ffi::PyTuple_Type, ffi::PyTuple_Check);

8
src/types/typeobject.rs
View File

@ -3,19 +3,15 @@
// based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython
use crate::err::{PyErr, PyResult};
use crate::ffi;
use crate::instance::{Py, PyNativeType};
use crate::internal_tricks::Unsendable;
use crate::object::PyObject;
use crate::type_object::PyTypeObject;
use crate::AsPyPointer;
use crate::Python;
use crate::{ffi, AsPyPointer, PyAny, Python};
use std::borrow::Cow;
use std::ffi::CStr;
/// Represents a reference to a Python `type object`.
#[repr(transparent)]
pub struct PyType(PyObject, Unsendable);
pub struct PyType(PyAny);
pyobject_native_var_type!(PyType, ffi::PyType_Type, ffi::PyType_Check);

10
tests/test_datetime.rs
View File

@ -54,14 +54,8 @@ macro_rules! assert_check_only {
};
}
// Because of the relase pool unsoundness reported in https://github.com/PyO3/pyo3/issues/756,
// we need to stop other threads before calling `py.import()`.
// TODO(kngwyu): Remove this variable
static MUTEX: parking_lot::Mutex<()> = parking_lot::const_mutex(());
#[test]
fn test_date_check() {
let _lock = MUTEX.lock();
let gil = Python::acquire_gil();
let py = gil.python();
let (obj, sub_obj, sub_sub_obj) = _get_subclasses(&py, "date", "2018, 1, 1").unwrap();
@ -73,7 +67,6 @@ fn test_date_check() {
#[test]
fn test_time_check() {
let _lock = MUTEX.lock();
let gil = Python::acquire_gil();
let py = gil.python();
let (obj, sub_obj, sub_sub_obj) = _get_subclasses(&py, "time", "12, 30, 15").unwrap();
@ -85,7 +78,6 @@ fn test_time_check() {
#[test]
fn test_datetime_check() {
let _lock = MUTEX.lock();
let gil = Python::acquire_gil();
let py = gil.python();
let (obj, sub_obj, sub_sub_obj) = _get_subclasses(&py, "datetime", "2018, 1, 1, 13, 30, 15")
@ -100,7 +92,6 @@ fn test_datetime_check() {
#[test]
fn test_delta_check() {
let _lock = MUTEX.lock();
let gil = Python::acquire_gil();
let py = gil.python();
let (obj, sub_obj, sub_sub_obj) = _get_subclasses(&py, "timedelta", "1, -3").unwrap();
@ -115,7 +106,6 @@ fn test_datetime_utc() {
use assert_approx_eq::assert_approx_eq;
use pyo3::types::PyDateTime;
let _lock = MUTEX.lock();
let gil = Python::acquire_gil();
let py = gil.python();
let datetime = py.import("datetime").map_err(|e| e.print(py)).unwrap();