Merge #2882

2882: inspect: gate behind `experimental-inspect` feature r=davidhewitt a=davidhewitt

This is the last thing I want to do before preparing 0.18 release.

The `pyo3::inspect` functionality looks useful as a first step towards #2454. However, we don't actually make use of this anywhere within PyO3 yet (we could probably use it for better error messages). I think we also have open questions about the traits which I'd like to resolve before committing to these additional APIs. (For example, this PR adds `IntoPy::type_output`, which seems potentially misplaced to me, the `type_output` function probably wants to be on a non-generic trait e.g. `ToPyObject` or maybe #2316.) 

As such, I propose putting these APIs behind an `experimental-inspect` feature gate for now, and invite users who find them useful to contribute a finished-off design.

Co-authored-by: David Hewitt <1939362+davidhewitt@users.noreply.github.com>

Cargo.toml

@@ -61,6 +61,10 @@ pyo3-build-config = { path = "pyo3-build-config", version = "0.17.3", features =
 [features]
 default = ["macros"]
 
+# Enables pyo3::inspect module and additional type information on FromPyObject
+# and IntoPy traits
+experimental-inspect = []
+
 # Enables macros: #[pyclass], #[pymodule], #[pyfunction] etc.
 macros = ["pyo3-macros", "indoc", "unindent"]
 
@@ -105,6 +109,7 @@ full = [
     "indexmap",
     "eyre",
     "anyhow",
+    "experimental-inspect",
 ]
 
 [[bench]]

guide/src/features.md

@@ -51,6 +51,12 @@ If you do not enable this feature, you should call `pyo3::prepare_freethreaded_p
 
 ## Advanced Features
 
+### `experimental-inspect`
+
+This feature adds the `pyo3::inspect` module, as well as `IntoPy::type_output` and `FromPyObject::type_input` APIs to produce Python type "annotations" for Rust types.
+
+This is a first step towards adding first-class support for generating type annotations automatically in PyO3, however work is needed to finish this off. All feedback and offers of help welcome on [issue #2454](https://github.com/PyO3/pyo3/issues/2454).
+
 ### `macros`
 
 This feature enables a dependency on the `pyo3-macros` crate, which provides the procedural macros portion of PyO3's API:

guide/src/python_typing_hints.md

@@ -4,6 +4,8 @@ PyO3 provides an easy to use interface to code native Python libraries in Rust.
 
 Currently the best solution for the problem is to manually maintain `*.pyi` files and ship them along with the package.
 
+There is a sketch of a roadmap towards completing [the `experimental-inspect` feature](./features.md#experimental-inspect) which may eventually lead to automatic type annotations generated by PyO3. This needs more testing and implementation, please see [issue #2454](https://github.com/PyO3/pyo3/issues/2454).
+
 ## Introduction to `pyi` files
 
 `pyi` files (an abbreviation for `Python Interface`) are called "stub files" in most of the documentation related to them. A very good definition of what it is can be found in [old MyPy documentation](https://github.com/python/mypy/wiki/Creating-Stubs-For-Python-Modules):

src/conversion.rs

@@ -2,6 +2,7 @@
 
 //! Defines conversions between Rust and Python types.
 use crate::err::{self, PyDowncastError, PyResult};
+#[cfg(feature = "experimental-inspect")]
 use crate::inspect::types::TypeInfo;
 use crate::pyclass::boolean_struct::False;
 use crate::type_object::PyTypeInfo;
@@ -253,6 +254,7 @@ pub trait IntoPy<T>: Sized {
     ///
     /// For most types, the return value for this method will be identical to that of [`FromPyObject::type_input`].
     /// It may be different for some types, such as `Dict`, to allow duck-typing: functions return `Dict` but take `Mapping` as argument.
+    #[cfg(feature = "experimental-inspect")]
     fn type_output() -> TypeInfo {
         TypeInfo::Any
     }
@@ -309,6 +311,7 @@ pub trait FromPyObject<'source>: Sized {
     ///
     /// For most types, the return value for this method will be identical to that of [`IntoPy::type_output`].
     /// It may be different for some types, such as `Dict`, to allow duck-typing: functions return `Dict` but take `Mapping` as argument.
+    #[cfg(feature = "experimental-inspect")]
     fn type_input() -> TypeInfo {
         TypeInfo::Any
     }

src/conversions/std/map.rs

@@ -1,7 +1,8 @@
 use std::{cmp, collections, hash};
 
+#[cfg(feature = "experimental-inspect")]
+use crate::inspect::types::TypeInfo;
 use crate::{
-    inspect::types::TypeInfo,
     types::{IntoPyDict, PyDict},
     FromPyObject, IntoPy, PyAny, PyErr, PyObject, Python, ToPyObject,
 };
@@ -40,6 +41,7 @@ where
         IntoPyDict::into_py_dict(iter, py).into()
     }
 
+    #[cfg(feature = "experimental-inspect")]
     fn type_output() -> TypeInfo {
         TypeInfo::dict_of(K::type_output(), V::type_output())
     }
@@ -57,6 +59,7 @@ where
         IntoPyDict::into_py_dict(iter, py).into()
     }
 
+    #[cfg(feature = "experimental-inspect")]
     fn type_output() -> TypeInfo {
         TypeInfo::dict_of(K::type_output(), V::type_output())
     }
@@ -77,6 +80,7 @@ where
         Ok(ret)
     }
 
+    #[cfg(feature = "experimental-inspect")]
     fn type_input() -> TypeInfo {
         TypeInfo::mapping_of(K::type_input(), V::type_input())
     }
@@ -96,6 +100,7 @@ where
         Ok(ret)
     }
 
+    #[cfg(feature = "experimental-inspect")]
     fn type_input() -> TypeInfo {
         TypeInfo::mapping_of(K::type_input(), V::type_input())
     }

src/conversions/std/num.rs

@@ -1,6 +1,8 @@
+#[cfg(feature = "experimental-inspect")]
+use crate::inspect::types::TypeInfo;
 use crate::{
-    exceptions, ffi, inspect::types::TypeInfo, AsPyPointer, FromPyObject, IntoPy, PyAny, PyErr,
-    PyObject, PyResult, Python, ToPyObject,
+    exceptions, ffi, AsPyPointer, FromPyObject, IntoPy, PyAny, PyErr, PyObject, PyResult, Python,
+    ToPyObject,
 };
 use std::convert::TryFrom;
 use std::num::{
@@ -22,6 +24,7 @@ macro_rules! int_fits_larger_int {
                 (self as $larger_type).into_py(py)
             }
 
+            #[cfg(feature = "experimental-inspect")]
             fn type_output() -> TypeInfo {
                 <$larger_type>::type_output()
             }
@@ -34,6 +37,7 @@ macro_rules! int_fits_larger_int {
                     .map_err(|e| exceptions::PyOverflowError::new_err(e.to_string()))
             }
 
+            #[cfg(feature = "experimental-inspect")]
             fn type_input() -> TypeInfo {
                 <$larger_type>::type_input()
             }
@@ -55,6 +59,7 @@ macro_rules! int_convert_u64_or_i64 {
                 unsafe { PyObject::from_owned_ptr(py, $pylong_from_ll_or_ull(self)) }
             }
 
+            #[cfg(feature = "experimental-inspect")]
             fn type_output() -> TypeInfo {
                 TypeInfo::builtin("int")
             }
@@ -74,6 +79,7 @@ macro_rules! int_convert_u64_or_i64 {
                 }
             }
 
+            #[cfg(feature = "experimental-inspect")]
             fn type_input() -> TypeInfo {
                 Self::type_output()
             }
@@ -93,6 +99,7 @@ macro_rules! int_fits_c_long {
                 unsafe { PyObject::from_owned_ptr(py, ffi::PyLong_FromLong(self as c_long)) }
             }
 
+            #[cfg(feature = "experimental-inspect")]
             fn type_output() -> TypeInfo {
                 TypeInfo::builtin("int")
             }
@@ -115,6 +122,7 @@ macro_rules! int_fits_c_long {
                     .map_err(|e| exceptions::PyOverflowError::new_err(e.to_string()))
             }
 
+            #[cfg(feature = "experimental-inspect")]
             fn type_input() -> TypeInfo {
                 Self::type_output()
             }
@@ -183,6 +191,7 @@ mod fast_128bit_int_conversion {
                     }
                 }
 
+                #[cfg(feature = "experimental-inspect")]
                 fn type_output() -> TypeInfo {
                     TypeInfo::builtin("int")
                 }
@@ -209,6 +218,7 @@ mod fast_128bit_int_conversion {
                     }
                 }
 
+                #[cfg(feature = "experimental-inspect")]
                 fn type_input() -> TypeInfo {
                     Self::type_output()
                 }
@@ -255,6 +265,7 @@ mod slow_128bit_int_conversion {
                     }
                 }
 
+                #[cfg(feature = "experimental-inspect")]
                 fn type_output() -> TypeInfo {
                     TypeInfo::builtin("int")
                 }
@@ -278,6 +289,7 @@ mod slow_128bit_int_conversion {
                     }
                 }
 
+                #[cfg(feature = "experimental-inspect")]
                 fn type_input() -> TypeInfo {
                     Self::type_output()
                 }
@@ -324,6 +336,7 @@ macro_rules! nonzero_int_impl {
                     .map_err(|_| exceptions::PyValueError::new_err("invalid zero value"))
             }
 
+            #[cfg(feature = "experimental-inspect")]
             fn type_input() -> TypeInfo {
                 <$primitive_type>::type_input()
             }

src/conversions/std/set.rs

@@ -1,8 +1,10 @@
 use std::{cmp, collections, hash};
 
+#[cfg(feature = "experimental-inspect")]
+use crate::inspect::types::TypeInfo;
 use crate::{
-    inspect::types::TypeInfo, types::set::new_from_iter, types::PySet, FromPyObject, IntoPy, PyAny,
-    PyObject, PyResult, Python, ToPyObject,
+    types::set::new_from_iter, types::PySet, FromPyObject, IntoPy, PyAny, PyObject, PyResult,
+    Python, ToPyObject,
 };
 
 impl<T, S> ToPyObject for collections::HashSet<T, S>
@@ -39,6 +41,7 @@ where
             .into()
     }
 
+    #[cfg(feature = "experimental-inspect")]
     fn type_output() -> TypeInfo {
         TypeInfo::set_of(K::type_output())
     }
@@ -54,6 +57,7 @@ where
         set.iter().map(K::extract).collect()
     }
 
+    #[cfg(feature = "experimental-inspect")]
     fn type_input() -> TypeInfo {
         TypeInfo::set_of(K::type_input())
     }
@@ -69,6 +73,7 @@ where
             .into()
     }
 
+    #[cfg(feature = "experimental-inspect")]
     fn type_output() -> TypeInfo {
         TypeInfo::set_of(K::type_output())
     }
@@ -83,6 +88,7 @@ where
         set.iter().map(K::extract).collect()
     }
 
+    #[cfg(feature = "experimental-inspect")]
     fn type_input() -> TypeInfo {
         TypeInfo::set_of(K::type_input())
     }

src/conversions/std/slice.rs

@@ -1,13 +1,13 @@
-use crate::{
-    inspect::types::TypeInfo, types::PyBytes, FromPyObject, IntoPy, PyAny, PyObject, PyResult,
-    Python, ToPyObject,
-};
+#[cfg(feature = "experimental-inspect")]
+use crate::inspect::types::TypeInfo;
+use crate::{types::PyBytes, FromPyObject, IntoPy, PyAny, PyObject, PyResult, Python, ToPyObject};
 
 impl<'a> IntoPy<PyObject> for &'a [u8] {
     fn into_py(self, py: Python<'_>) -> PyObject {
         PyBytes::new(py, self).to_object(py)
     }
 
+    #[cfg(feature = "experimental-inspect")]
     fn type_output() -> TypeInfo {
         TypeInfo::builtin("bytes")
     }
@@ -18,6 +18,7 @@ impl<'a> FromPyObject<'a> for &'a [u8] {
         Ok(obj.downcast::<PyBytes>()?.as_bytes())
     }
 
+    #[cfg(feature = "experimental-inspect")]
     fn type_input() -> TypeInfo {
         Self::type_output()
     }

src/conversions/std/string.rs

@@ -1,8 +1,9 @@
 use std::borrow::Cow;
 
+#[cfg(feature = "experimental-inspect")]
+use crate::inspect::types::TypeInfo;
 use crate::{
-    inspect::types::TypeInfo, types::PyString, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult,
-    Python, ToPyObject,
+    types::PyString, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject,
 };
 
 /// Converts a Rust `str` to a Python object.
@@ -20,6 +21,7 @@ impl<'a> IntoPy<PyObject> for &'a str {
         PyString::new(py, self).into()
     }
 
+    #[cfg(feature = "experimental-inspect")]
     fn type_output() -> TypeInfo {
         <String>::type_output()
     }
@@ -31,6 +33,7 @@ impl<'a> IntoPy<Py<PyString>> for &'a str {
         PyString::new(py, self).into()
     }
 
+    #[cfg(feature = "experimental-inspect")]
     fn type_output() -> TypeInfo {
         <String>::type_output()
     }
@@ -51,6 +54,7 @@ impl IntoPy<PyObject> for Cow<'_, str> {
         self.to_object(py)
     }
 
+    #[cfg(feature = "experimental-inspect")]
     fn type_output() -> TypeInfo {
         <String>::type_output()
     }
@@ -77,6 +81,7 @@ impl IntoPy<PyObject> for char {
         PyString::new(py, self.encode_utf8(&mut bytes)).into()
     }
 
+    #[cfg(feature = "experimental-inspect")]
     fn type_output() -> TypeInfo {
         <String>::type_output()
     }
@@ -87,6 +92,7 @@ impl IntoPy<PyObject> for String {
         PyString::new(py, &self).into()
     }
 
+    #[cfg(feature = "experimental-inspect")]
     fn type_output() -> TypeInfo {
         TypeInfo::builtin("str")
     }
@@ -98,6 +104,7 @@ impl<'a> IntoPy<PyObject> for &'a String {
         PyString::new(py, self).into()
     }
 
+    #[cfg(feature = "experimental-inspect")]
     fn type_output() -> TypeInfo {
         <String>::type_output()
     }
@@ -110,6 +117,7 @@ impl<'source> FromPyObject<'source> for &'source str {
         ob.downcast::<PyString>()?.to_str()
     }
 
+    #[cfg(feature = "experimental-inspect")]
     fn type_input() -> TypeInfo {
         <String>::type_input()
     }
@@ -122,6 +130,7 @@ impl FromPyObject<'_> for String {
         obj.downcast::<PyString>()?.to_str().map(ToOwned::to_owned)
     }
 
+    #[cfg(feature = "experimental-inspect")]
     fn type_input() -> TypeInfo {
         Self::type_output()
     }
@@ -140,6 +149,7 @@ impl FromPyObject<'_> for char {
         }
     }
 
+    #[cfg(feature = "experimental-inspect")]
     fn type_input() -> TypeInfo {
         <String>::type_input()
     }

src/conversions/std/vec.rs

@@ -1,3 +1,4 @@
+#[cfg(feature = "experimental-inspect")]
 use crate::inspect::types::TypeInfo;
 use crate::types::list::new_from_iter;
 use crate::{IntoPy, PyObject, Python, ToPyObject};
@@ -32,6 +33,7 @@ where
         list.into()
     }
 
+    #[cfg(feature = "experimental-inspect")]
     fn type_output() -> TypeInfo {
         TypeInfo::list_of(T::type_output())
     }

src/lib.rs

@@ -445,6 +445,7 @@ mod macros;
 #[cfg(all(test, feature = "macros"))]
 mod test_hygiene;
 
+#[cfg(feature = "experimental-inspect")]
 pub mod inspect;
 
 /// Test readme and user guide

src/types/boolobject.rs

@@ -1,4 +1,5 @@
 // Copyright (c) 2017-present PyO3 Project and Contributors
+#[cfg(feature = "experimental-inspect")]
 use crate::inspect::types::TypeInfo;
 use crate::{
     ffi, AsPyPointer, FromPyObject, IntoPy, PyAny, PyObject, PyResult, Python, ToPyObject,
@@ -47,6 +48,7 @@ impl IntoPy<PyObject> for bool {
         PyBool::new(py, self).into()
     }
 
+    #[cfg(feature = "experimental-inspect")]
     fn type_output() -> TypeInfo {
         TypeInfo::builtin("bool")
     }
@@ -60,6 +62,7 @@ impl<'source> FromPyObject<'source> for bool {
         Ok(obj.downcast::<PyBool>()?.is_true())
     }
 
+    #[cfg(feature = "experimental-inspect")]
     fn type_input() -> TypeInfo {
         Self::type_output()
     }

src/types/floatob.rs

@@ -1,6 +1,7 @@
 // Copyright (c) 2017-present PyO3 Project and Contributors
 //
 // based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython
+#[cfg(feature = "experimental-inspect")]
 use crate::inspect::types::TypeInfo;
 use crate::{
     ffi, AsPyPointer, FromPyObject, IntoPy, PyAny, PyErr, PyObject, PyResult, Python, ToPyObject,
@@ -46,6 +47,7 @@ impl IntoPy<PyObject> for f64 {
         PyFloat::new(py, self).into()
     }
 
+    #[cfg(feature = "experimental-inspect")]
     fn type_output() -> TypeInfo {
         TypeInfo::builtin("float")
     }
@@ -66,6 +68,7 @@ impl<'source> FromPyObject<'source> for f64 {
         Ok(v)
     }
 
+    #[cfg(feature = "experimental-inspect")]
     fn type_input() -> TypeInfo {
         Self::type_output()
     }
@@ -82,6 +85,7 @@ impl IntoPy<PyObject> for f32 {
         PyFloat::new(py, f64::from(self)).into()
     }
 
+    #[cfg(feature = "experimental-inspect")]
     fn type_output() -> TypeInfo {
         TypeInfo::builtin("float")
     }
@@ -92,6 +96,7 @@ impl<'source> FromPyObject<'source> for f32 {
         Ok(obj.extract::<f64>()? as f32)
     }
 
+    #[cfg(feature = "experimental-inspect")]
     fn type_input() -> TypeInfo {
         Self::type_output()
     }

src/types/sequence.rs

@@ -1,6 +1,7 @@
 // Copyright (c) 2017-present PyO3 Project and Contributors
 use crate::err::{self, PyDowncastError, PyErr, PyResult};
 use crate::exceptions::PyTypeError;
+#[cfg(feature = "experimental-inspect")]
 use crate::inspect::types::TypeInfo;
 use crate::internal_tricks::get_ssize_index;
 use crate::once_cell::GILOnceCell;
@@ -290,6 +291,7 @@ where
         extract_sequence(obj)
     }
 
+    #[cfg(feature = "experimental-inspect")]
     fn type_input() -> TypeInfo {
         TypeInfo::sequence_of(T::type_input())
     }

src/types/tuple.rs

@@ -3,6 +3,7 @@
 use std::convert::TryInto;
 
 use crate::ffi::{self, Py_ssize_t};
+#[cfg(feature = "experimental-inspect")]
 use crate::inspect::types::TypeInfo;
 use crate::internal_tricks::get_ssize_index;
 use crate::types::PySequence;
@@ -293,6 +294,7 @@ macro_rules! tuple_conversion ({$length:expr,$(($refN:ident, $n:tt, $T:ident)),+
             }
         }
 
+        #[cfg(feature = "experimental-inspect")]
 fn type_output() -> TypeInfo {
             TypeInfo::Tuple(Some(vec![$( $T::type_output() ),+]))
         }
@@ -308,6 +310,7 @@ macro_rules! tuple_conversion ({$length:expr,$(($refN:ident, $n:tt, $T:ident)),+
             }
         }
 
+        #[cfg(feature = "experimental-inspect")]
         fn type_output() -> TypeInfo {
             TypeInfo::Tuple(Some(vec![$( $T::type_output() ),+]))
         }
@@ -328,6 +331,7 @@ macro_rules! tuple_conversion ({$length:expr,$(($refN:ident, $n:tt, $T:ident)),+
             }
         }
 
+        #[cfg(feature = "experimental-inspect")]
 fn type_input() -> TypeInfo {
             TypeInfo::Tuple(Some(vec![$( $T::type_input() ),+]))
         }