From 463543ad583e4aef23fa8058a40d18beedd2f5fd Mon Sep 17 00:00:00 2001 From: mejrs Date: Sat, 29 Jan 2022 01:03:57 +0100 Subject: [PATCH 1/2] Annotate Rust blocks with "rust" --- .../multiple_python_versions.md | 10 +++++----- guide/src/conversions/traits.md | 18 +++++++++--------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/guide/src/building_and_distribution/multiple_python_versions.md b/guide/src/building_and_distribution/multiple_python_versions.md index ff442072..e7bdd986 100644 --- a/guide/src/building_and_distribution/multiple_python_versions.md +++ b/guide/src/building_and_distribution/multiple_python_versions.md @@ -53,31 +53,31 @@ After these steps you are ready to annotate your code! The `#[cfg]` flags added by `pyo3-build-cfg` can be combined with all of Rust's logic in the `#[cfg]` attribute to create very precise conditional code generation. The following are some common patterns implemented using these flags: -``` +```rust,ignore #[cfg(Py_3_7)] ``` This `#[cfg]` marks code that will only be present on Python 3.7 and upwards. There are similar options `Py_3_8`, `Py_3_9`, `Py_3_10` and so on for each minor version. -``` +```rust,ignore #[cfg(not(Py_3_7))] ``` This `#[cfg]` marks code that will only be present on Python versions before (but not including) Python 3.7. -``` +```rust,ignore #[cfg(not(Py_LIMITED_API))] ``` This `#[cfg]` marks code that is only available when building for the unlimited Python API (i.e. PyO3's `abi3` feature is not enabled). This might be useful if you want to ship your extension module as an `abi3` wheel and also allow users to compile it from source to make use of optimizations only possible with the unlimited API. -``` +```rust,ignore #[cfg(any(Py_3_9, not(Py_LIMITED_API)))] ``` This `#[cfg]` marks code which is available when running Python 3.9 or newer, or when using the unlimited API with an older Python version. Patterns like this are commonly seen on Python APIs which were added to the limited Python API in a specific minor version. -``` +```rust,ignore #[cfg(PyPy)] ``` diff --git a/guide/src/conversions/traits.md b/guide/src/conversions/traits.md index 619aac80..4de449e3 100644 --- a/guide/src/conversions/traits.md +++ b/guide/src/conversions/traits.md @@ -35,7 +35,7 @@ structs is not supported. The derivation generates code that will attempt to access the attribute `my_string` on the Python object, i.e. `obj.getattr("my_string")`, and call `extract()` on the attribute. -``` +```rust use pyo3::prelude::*; #[derive(FromPyObject)] @@ -65,7 +65,7 @@ struct RustyStruct { By setting the `#[pyo3(item)]` attribute on the field, PyO3 will attempt to extract the value by calling the `get_item` method on the Python object. -``` +```rust use pyo3::prelude::*; @@ -90,7 +90,7 @@ struct RustyStruct { The argument passed to `getattr` and `get_item` can also be configured: -``` +```rust use pyo3::prelude::*; #[derive(FromPyObject)] @@ -135,7 +135,7 @@ Tuple structs are also supported but do not allow customizing the extraction. Th always assumed to be a Python tuple with the same length as the Rust type, the `n`th field is extracted from the `n`th item in the Python tuple. -``` +```rust use pyo3::prelude::*; #[derive(FromPyObject)] @@ -158,7 +158,7 @@ struct RustyTuple(String, String); Tuple structs with a single field are treated as wrapper types which are described in the following section. To override this behaviour and ensure that the input is in fact a tuple, specify the struct as -``` +```rust use pyo3::prelude::*; #[derive(FromPyObject)] @@ -184,7 +184,7 @@ in extracting directly from the input object, i.e. `obj.extract()`, rather than an item or attribute. This behaviour is enabled per default for newtype structs and tuple-variants with a single field. -``` +```rust use pyo3::prelude::*; #[derive(FromPyObject)] @@ -223,7 +223,7 @@ i.e. a tuple variant assumes that the input is a Python tuple, and a struct vari extracting fields as attributes but can be configured in the same manner. The `transparent` attribute can be applied to single-field-variants. -``` +```rust use pyo3::prelude::*; #[derive(FromPyObject)] @@ -370,7 +370,7 @@ tested variants is returned. The names reported in the error message can be cust through the `#[pyo3(annotation = "name")]` attribute, e.g. to use conventional Python type names: -``` +```rust use pyo3::prelude::*; #[derive(FromPyObject)] @@ -454,7 +454,7 @@ All types in PyO3 implement this trait, as does a `#[pyclass]` which doesn't use Occasionally you may choose to implement this for custom types which are mapped to Python types _without_ having a unique python type. -``` +```rust use pyo3::prelude::*; struct MyPyObjectWrapper(PyObject); From be84e0fbb901b8d5cbef43a42e2dc525adb3f350 Mon Sep 17 00:00:00 2001 From: mejrs Date: Wed, 23 Feb 2022 10:08:42 +0100 Subject: [PATCH 2/2] Add numeric and object protocol examples --- .github/pull_request_template.md | 14 +- .github/workflows/bench.yml | 6 +- .github/workflows/ci.yml | 40 +- Architecture.md | 7 +- CHANGELOG.md | 35 +- Cargo.toml | 28 +- Contributing.md | 6 +- Makefile | 59 -- README.md | 5 +- build.rs | 128 +-- guide/src/SUMMARY.md | 2 + guide/src/building_and_distribution.md | 2 +- guide/src/class.md | 297 ++++--- guide/src/class/numeric.md | 449 ++++++++++ guide/src/class/object.md | 232 ++++++ guide/src/class/protocols.md | 23 +- guide/src/features.md | 10 +- guide/src/migration.md | 32 +- noxfile.py | 88 ++ pyo3-build-config/src/lib.rs | 31 + pyo3-ffi/Cargo.toml | 39 + {src/ffi => pyo3-ffi}/LICENSE | 0 pyo3-ffi/README.md | 199 +++++ pyo3-ffi/build.rs | 122 +++ {src/ffi => pyo3-ffi/src}/abstract_.rs | 9 +- {src/ffi => pyo3-ffi/src}/bltinmodule.rs | 2 +- {src/ffi => pyo3-ffi/src}/boolobject.rs | 10 +- pyo3-ffi/src/buffer.rs | 133 +++ {src/ffi => pyo3-ffi/src}/bytearrayobject.rs | 8 +- {src/ffi => pyo3-ffi/src}/bytesobject.rs | 6 +- {src/ffi => pyo3-ffi/src}/ceval.rs | 10 +- {src/ffi => pyo3-ffi/src}/code.rs | 0 {src/ffi => pyo3-ffi/src}/codecs.rs | 2 +- {src/ffi => pyo3-ffi/src}/compile.rs | 0 {src/ffi => pyo3-ffi/src}/complexobject.rs | 6 +- {src/ffi => pyo3-ffi/src}/context.rs | 8 +- .../ffi => pyo3-ffi/src}/cpython/abstract_.rs | 26 +- .../src}/cpython/bytesobject.rs | 4 +- {src/ffi => pyo3-ffi/src}/cpython/ceval.rs | 9 +- {src/ffi => pyo3-ffi/src}/cpython/code.rs | 8 +- {src/ffi => pyo3-ffi/src}/cpython/compile.rs | 8 +- .../src}/cpython/dictobject.rs | 8 +- .../src}/cpython/frameobject.rs | 8 +- {src/ffi => pyo3-ffi/src}/cpython/import.rs | 2 +- .../src}/cpython/initconfig.rs | 2 +- .../src}/cpython/listobject.rs | 4 +- {src/ffi => pyo3-ffi/src}/cpython/mod.rs | 0 {src/ffi => pyo3-ffi/src}/cpython/object.rs | 30 +- {src/ffi => pyo3-ffi/src}/cpython/pydebug.rs | 0 .../src}/cpython/pylifecycle.rs | 2 +- {src/ffi => pyo3-ffi/src}/cpython/pymem.rs | 0 {src/ffi => pyo3-ffi/src}/cpython/pystate.rs | 10 +- .../ffi => pyo3-ffi/src}/cpython/pythonrun.rs | 8 +- .../src}/cpython/tupleobject.rs | 4 +- .../src}/cpython/unicodeobject.rs | 139 +--- {src/ffi => pyo3-ffi/src}/datetime.rs | 203 ++--- {src/ffi => pyo3-ffi/src}/descrobject.rs | 6 +- {src/ffi => pyo3-ffi/src}/dictobject.rs | 12 +- {src/ffi => pyo3-ffi/src}/enumobject.rs | 2 +- {src/ffi => pyo3-ffi/src}/eval.rs | 2 +- {src/ffi => pyo3-ffi/src}/fileobject.rs | 2 +- {src/ffi => pyo3-ffi/src}/fileutils.rs | 2 +- {src/ffi => pyo3-ffi/src}/floatobject.rs | 6 +- {src/ffi => pyo3-ffi/src}/funcobject.rs | 4 +- {src/ffi => pyo3-ffi/src}/genobject.rs | 14 +- {src/ffi => pyo3-ffi/src}/import.rs | 2 +- {src/ffi => pyo3-ffi/src}/intrcheck.rs | 0 {src/ffi => pyo3-ffi/src}/iterobject.rs | 6 +- pyo3-ffi/src/lib.rs | 447 ++++++++++ {src/ffi => pyo3-ffi/src}/listobject.rs | 6 +- {src/ffi => pyo3-ffi/src}/longobject.rs | 12 +- {src/ffi => pyo3-ffi/src}/marshal.rs | 0 {src/ffi => pyo3-ffi/src}/memoryobject.rs | 6 +- {src/ffi => pyo3-ffi/src}/methodobject.rs | 68 +- {src/ffi => pyo3-ffi/src}/modsupport.rs | 8 +- {src/ffi => pyo3-ffi/src}/moduleobject.rs | 10 +- {src/ffi => pyo3-ffi/src}/object.rs | 17 +- {src/ffi => pyo3-ffi/src}/objimpl.rs | 4 +- {src/ffi => pyo3-ffi/src}/osmodule.rs | 2 +- {src/ffi => pyo3-ffi/src}/pyarena.rs | 0 {src/ffi => pyo3-ffi/src}/pycapsule.rs | 4 +- {src/ffi => pyo3-ffi/src}/pyerrors.rs | 6 +- {src/ffi => pyo3-ffi/src}/pyframe.rs | 2 +- {src/ffi => pyo3-ffi/src}/pyhash.rs | 2 +- {src/ffi => pyo3-ffi/src}/pylifecycle.rs | 2 +- {src/ffi => pyo3-ffi/src}/pymem.rs | 0 {src/ffi => pyo3-ffi/src}/pyport.rs | 0 {src/ffi => pyo3-ffi/src}/pystate.rs | 4 +- {src/ffi => pyo3-ffi/src}/pystrtod.rs | 2 +- {src/ffi => pyo3-ffi/src}/pythonrun.rs | 6 +- {src/ffi => pyo3-ffi/src}/rangeobject.rs | 4 +- {src/ffi => pyo3-ffi/src}/setobject.rs | 24 +- {src/ffi => pyo3-ffi/src}/sliceobject.rs | 8 +- {src/ffi => pyo3-ffi/src}/structmember.rs | 4 +- {src/ffi => pyo3-ffi/src}/structseq.rs | 10 +- {src/ffi => pyo3-ffi/src}/sysmodule.rs | 2 +- {src/ffi => pyo3-ffi/src}/traceback.rs | 6 +- {src/ffi => pyo3-ffi/src}/tupleobject.rs | 6 +- {src/ffi => pyo3-ffi/src}/typeslots.rs | 2 - {src/ffi => pyo3-ffi/src}/unicodeobject.rs | 6 +- {src/ffi => pyo3-ffi/src}/warnings.rs | 4 +- {src/ffi => pyo3-ffi/src}/weakrefobject.rs | 10 +- pyo3-macros-backend/Cargo.toml | 3 + pyo3-macros-backend/src/defs.rs | 22 +- pyo3-macros-backend/src/deprecations.rs | 2 + pyo3-macros-backend/src/lib.rs | 4 + pyo3-macros-backend/src/method.rs | 36 +- pyo3-macros-backend/src/module.rs | 3 +- pyo3-macros-backend/src/params.rs | 102 ++- pyo3-macros-backend/src/pyclass.rs | 444 +++++----- pyo3-macros-backend/src/pyimpl.rs | 113 +-- pyo3-macros-backend/src/pymethod.rs | 276 ++++--- pyo3-macros-backend/src/pyproto.rs | 51 +- pyo3-macros/Cargo.toml | 2 + pyo3-macros/src/lib.rs | 24 +- pytests/{pyo3-pytests => }/Cargo.toml | 4 +- pytests/{pyo3-pytests => }/MANIFEST.in | 0 pytests/README.md | 38 +- pytests/build.rs | 4 + pytests/{pyo3-benchmarks => }/noxfile.py | 2 +- pytests/pyo3-benchmarks/Cargo.toml | 16 - pytests/pyo3-benchmarks/MANIFEST.in | 3 - pytests/pyo3-benchmarks/README.md | 18 - pytests/pyo3-benchmarks/requirements-dev.txt | 4 - pytests/pyo3-pytests/README.md | 19 - pytests/pyo3-pytests/build.rs | 3 - pytests/pyo3-pytests/noxfile.py | 9 - pytests/pyo3-pytests/pyproject.toml | 3 - pytests/pyo3-pytests/requirements-dev.txt | 4 - pytests/pyo3-pytests/src/pyclass_iter.rs | 34 - .../pyo3_pytests/__init__.py | 0 pytests/{pyo3-benchmarks => }/pyproject.toml | 0 pytests/requirements-dev.txt | 4 + pytests/{pyo3-pytests => }/src/buf_and_str.rs | 0 pytests/{pyo3-pytests => }/src/datetime.rs | 0 pytests/{pyo3-pytests => }/src/dict_iter.rs | 0 pytests/{pyo3-pytests => }/src/lib.rs | 9 +- pytests/{pyo3-pytests => }/src/misc.rs | 0 pytests/{pyo3-pytests => }/src/objstore.rs | 0 pytests/{pyo3-pytests => }/src/othermod.rs | 2 +- pytests/{pyo3-pytests => }/src/path.rs | 0 pytests/src/pyclasses.rs | 43 + .../src/lib.rs => src/pyfunctions.rs} | 14 +- pytests/{pyo3-pytests => }/src/subclassing.rs | 3 - .../tests/test_buf_and_str.py | 0 .../{pyo3-pytests => }/tests/test_datetime.py | 0 .../tests/test_dict_iter.py | 0 pytests/{pyo3-pytests => }/tests/test_misc.py | 0 .../{pyo3-pytests => }/tests/test_objstore.py | 0 .../{pyo3-pytests => }/tests/test_othermod.py | 4 +- pytests/{pyo3-pytests => }/tests/test_path.py | 0 .../test_pyclasses.py} | 16 +- .../test_pyfunctions.py} | 38 +- .../tests/test_subclassing.py | 0 src/buffer.rs | 52 +- src/callback.rs | 21 +- src/class/basic.rs | 33 +- src/class/gc.rs | 16 + src/class/impl_.rs | 782 ------------------ src/class/iter.rs | 52 +- src/class/methods.rs | 220 ----- src/class/mod.rs | 8 +- src/class/pyasync.rs | 52 +- src/conversion.rs | 54 +- src/conversions/array.rs | 59 -- src/conversions/num_bigint.rs | 4 +- src/err/mod.rs | 3 + src/ffi/README.md | 15 - src/ffi/mod.rs | 186 +---- src/ffi/tests.rs | 183 ++++ src/impl_/deprecations.rs | 6 + src/impl_/extract_argument.rs | 655 ++++++++++----- src/impl_/pyclass.rs | 756 ++++++++++++++++- src/impl_/pyfunction.rs | 2 +- src/impl_/pymethods.rs | 223 ++++- src/impl_/pymodule.rs | 16 +- src/instance.rs | 27 +- src/lib.rs | 49 +- src/macros.rs | 2 +- src/{python.rs => marker.rs} | 297 ++++--- src/prelude.rs | 9 +- src/pycell.rs | 3 +- src/pyclass.rs | 397 ++++++--- src/pyclass_init.rs | 5 +- src/test_hygiene/pyclass.rs | 6 + src/type_object.rs | 9 +- src/types/any.rs | 6 +- src/types/datetime.rs | 129 ++- src/types/function.rs | 17 +- src/types/mapping.rs | 25 + src/types/mod.rs | 8 +- src/types/num.rs | 12 +- src/types/sequence.rs | 26 - src/version.rs | 143 ++++ tests/test_arithmetics.rs | 33 + tests/test_arithmetics_protos.rs | 1 + tests/test_buffer.rs | 77 +- tests/test_buffer_protocol.rs | 2 +- tests/test_buffer_protocol_pyproto.rs | 3 +- tests/test_compile_error.rs | 23 + tests/test_datetime.rs | 6 +- tests/test_enum.rs | 124 ++- tests/test_frompyobject.rs | 5 +- tests/test_gc.rs | 17 +- tests/test_gc_pyproto.rs | 282 +++++++ tests/test_mapping.rs | 4 - tests/test_mapping_pyproto.rs | 110 +++ tests/test_proto_methods.rs | 282 ++++++- tests/test_pyfunction.rs | 73 +- tests/test_pyproto.rs | 1 + tests/test_pyself.rs | 9 +- tests/test_sequence.rs | 18 +- tests/test_sequence_pyproto.rs | 304 +++++++ tests/ui/abi3_nativetype_inheritance.stderr | 2 +- tests/ui/deprecations.rs | 3 + tests/ui/deprecations.stderr | 6 + tests/ui/invalid_pymethods.rs | 11 + tests/ui/invalid_pymethods.stderr | 11 + tests/ui/not_send.stderr | 13 +- tests/ui/not_send2.rs | 12 + tests/ui/not_send2.stderr | 18 + tests/ui/not_send3.rs | 12 + tests/ui/not_send3.stderr | 15 + tests/ui/not_send_auto_trait.rs | 11 + tests/ui/not_send_auto_trait.stderr | 15 + tests/ui/not_send_auto_trait2.rs | 12 + tests/ui/not_send_auto_trait2.stderr | 20 + tests/ui/pyclass_send.stderr | 2 +- tests/ui/send_wrapper.rs | 16 + tests/ui/send_wrapper.stderr | 23 + tests/ui/static_ref.stderr | 26 +- xtask/src/main.rs | 19 +- 232 files changed, 7128 insertions(+), 3712 deletions(-) delete mode 100644 Makefile create mode 100644 guide/src/class/numeric.md create mode 100644 guide/src/class/object.md create mode 100644 noxfile.py create mode 100644 pyo3-ffi/Cargo.toml rename {src/ffi => pyo3-ffi}/LICENSE (100%) create mode 100644 pyo3-ffi/README.md create mode 100644 pyo3-ffi/build.rs rename {src/ffi => pyo3-ffi/src}/abstract_.rs (98%) rename {src/ffi => pyo3-ffi/src}/bltinmodule.rs (84%) rename {src/ffi => pyo3-ffi/src}/boolobject.rs (79%) create mode 100644 pyo3-ffi/src/buffer.rs rename {src/ffi => pyo3-ffi/src}/bytearrayobject.rs (87%) rename {src/ffi => pyo3-ffi/src}/bytesobject.rs (95%) rename {src/ffi => pyo3-ffi/src}/ceval.rs (91%) rename {src/ffi => pyo3-ffi/src}/code.rs (100%) rename {src/ffi => pyo3-ffi/src}/codecs.rs (98%) rename {src/ffi => pyo3-ffi/src}/compile.rs (100%) rename {src/ffi => pyo3-ffi/src}/complexobject.rs (92%) rename {src/ffi => pyo3-ffi/src}/context.rs (84%) rename {src/ffi => pyo3-ffi/src}/cpython/abstract_.rs (93%) rename {src/ffi => pyo3-ffi/src}/cpython/bytesobject.rs (64%) rename {src/ffi => pyo3-ffi/src}/cpython/ceval.rs (55%) rename {src/ffi => pyo3-ffi/src}/cpython/code.rs (96%) rename {src/ffi => pyo3-ffi/src}/cpython/compile.rs (95%) rename {src/ffi => pyo3-ffi/src}/cpython/dictobject.rs (92%) rename {src/ffi => pyo3-ffi/src}/cpython/frameobject.rs (94%) rename {src/ffi => pyo3-ffi/src}/cpython/import.rs (97%) rename {src/ffi => pyo3-ffi/src}/cpython/initconfig.rs (99%) rename {src/ffi => pyo3-ffi/src}/cpython/listobject.rs (92%) rename {src/ffi => pyo3-ffi/src}/cpython/mod.rs (100%) rename {src/ffi => pyo3-ffi/src}/cpython/object.rs (94%) rename {src/ffi => pyo3-ffi/src}/cpython/pydebug.rs (100%) rename {src/ffi => pyo3-ffi/src}/cpython/pylifecycle.rs (95%) rename {src/ffi => pyo3-ffi/src}/cpython/pymem.rs (100%) rename {src/ffi => pyo3-ffi/src}/cpython/pystate.rs (93%) rename {src/ffi => pyo3-ffi/src}/cpython/pythonrun.rs (98%) rename {src/ffi => pyo3-ffi/src}/cpython/tupleobject.rs (93%) rename {src/ffi => pyo3-ffi/src}/cpython/unicodeobject.rs (73%) rename {src/ffi => pyo3-ffi/src}/datetime.rs (72%) rename {src/ffi => pyo3-ffi/src}/descrobject.rs (95%) rename {src/ffi => pyo3-ffi/src}/dictobject.rs (92%) rename {src/ffi => pyo3-ffi/src}/enumobject.rs (80%) rename {src/ffi => pyo3-ffi/src}/eval.rs (95%) rename {src/ffi => pyo3-ffi/src}/fileobject.rs (97%) rename {src/ffi => pyo3-ffi/src}/fileutils.rs (87%) rename {src/ffi => pyo3-ffi/src}/floatobject.rs (91%) rename {src/ffi => pyo3-ffi/src}/funcobject.rs (93%) rename {src/ffi => pyo3-ffi/src}/genobject.rs (85%) rename {src/ffi => pyo3-ffi/src}/import.rs (98%) rename {src/ffi => pyo3-ffi/src}/intrcheck.rs (100%) rename {src/ffi => pyo3-ffi/src}/iterobject.rs (80%) create mode 100644 pyo3-ffi/src/lib.rs rename {src/ffi => pyo3-ffi/src}/listobject.rs (95%) rename {src/ffi => pyo3-ffi/src}/longobject.rs (94%) rename {src/ffi => pyo3-ffi/src}/marshal.rs (100%) rename {src/ffi => pyo3-ffi/src}/memoryobject.rs (90%) rename {src/ffi => pyo3-ffi/src}/methodobject.rs (60%) rename {src/ffi => pyo3-ffi/src}/modsupport.rs (97%) rename {src/ffi => pyo3-ffi/src}/moduleobject.rs (92%) rename {src/ffi => pyo3-ffi/src}/object.rs (96%) rename {src/ffi => pyo3-ffi/src}/objimpl.rs (98%) rename {src/ffi => pyo3-ffi/src}/osmodule.rs (69%) rename {src/ffi => pyo3-ffi/src}/pyarena.rs (100%) rename {src/ffi => pyo3-ffi/src}/pycapsule.rs (96%) rename {src/ffi => pyo3-ffi/src}/pyerrors.rs (99%) rename {src/ffi => pyo3-ffi/src}/pyframe.rs (87%) rename {src/ffi => pyo3-ffi/src}/pyhash.rs (96%) rename {src/ffi => pyo3-ffi/src}/pylifecycle.rs (97%) rename {src/ffi => pyo3-ffi/src}/pymem.rs (100%) rename {src/ffi => pyo3-ffi/src}/pyport.rs (100%) rename {src/ffi => pyo3-ffi/src}/pystate.rs (97%) rename {src/ffi => pyo3-ffi/src}/pystrtod.rs (97%) rename {src/ffi => pyo3-ffi/src}/pythonrun.rs (93%) rename {src/ffi => pyo3-ffi/src}/rangeobject.rs (81%) rename {src/ffi => pyo3-ffi/src}/setobject.rs (80%) rename {src/ffi => pyo3-ffi/src}/sliceobject.rs (92%) rename {src/ffi => pyo3-ffi/src}/structmember.rs (96%) rename {src/ffi => pyo3-ffi/src}/structseq.rs (88%) rename {src/ffi => pyo3-ffi/src}/sysmodule.rs (97%) rename {src/ffi => pyo3-ffi/src}/traceback.rs (79%) rename {src/ffi => pyo3-ffi/src}/tupleobject.rs (92%) rename {src/ffi => pyo3-ffi/src}/typeslots.rs (98%) rename {src/ffi => pyo3-ffi/src}/unicodeobject.rs (99%) rename {src/ffi => pyo3-ffi/src}/warnings.rs (92%) rename {src/ffi => pyo3-ffi/src}/weakrefobject.rs (82%) rename pytests/{pyo3-pytests => }/Cargo.toml (82%) rename pytests/{pyo3-pytests => }/MANIFEST.in (100%) create mode 100644 pytests/build.rs rename pytests/{pyo3-benchmarks => }/noxfile.py (79%) delete mode 100644 pytests/pyo3-benchmarks/Cargo.toml delete mode 100644 pytests/pyo3-benchmarks/MANIFEST.in delete mode 100644 pytests/pyo3-benchmarks/README.md delete mode 100644 pytests/pyo3-benchmarks/requirements-dev.txt delete mode 100644 pytests/pyo3-pytests/README.md delete mode 100644 pytests/pyo3-pytests/build.rs delete mode 100644 pytests/pyo3-pytests/noxfile.py delete mode 100644 pytests/pyo3-pytests/pyproject.toml delete mode 100644 pytests/pyo3-pytests/requirements-dev.txt delete mode 100644 pytests/pyo3-pytests/src/pyclass_iter.rs rename pytests/{pyo3-pytests => }/pyo3_pytests/__init__.py (100%) rename pytests/{pyo3-benchmarks => }/pyproject.toml (100%) create mode 100644 pytests/requirements-dev.txt rename pytests/{pyo3-pytests => }/src/buf_and_str.rs (100%) rename pytests/{pyo3-pytests => }/src/datetime.rs (100%) rename pytests/{pyo3-pytests => }/src/dict_iter.rs (100%) rename pytests/{pyo3-pytests => }/src/lib.rs (83%) rename pytests/{pyo3-pytests => }/src/misc.rs (100%) rename pytests/{pyo3-pytests => }/src/objstore.rs (100%) rename pytests/{pyo3-pytests => }/src/othermod.rs (93%) rename pytests/{pyo3-pytests => }/src/path.rs (100%) create mode 100644 pytests/src/pyclasses.rs rename pytests/{pyo3-benchmarks/src/lib.rs => src/pyfunctions.rs} (87%) rename pytests/{pyo3-pytests => }/src/subclassing.rs (88%) rename pytests/{pyo3-pytests => }/tests/test_buf_and_str.py (100%) rename pytests/{pyo3-pytests => }/tests/test_datetime.py (100%) rename pytests/{pyo3-pytests => }/tests/test_dict_iter.py (100%) rename pytests/{pyo3-pytests => }/tests/test_misc.py (100%) rename pytests/{pyo3-pytests => }/tests/test_objstore.py (100%) rename pytests/{pyo3-pytests => }/tests/test_othermod.py (85%) rename pytests/{pyo3-pytests => }/tests/test_path.py (100%) rename pytests/{pyo3-pytests/tests/test_pyclass_iter.py => tests/test_pyclasses.py} (51%) rename pytests/{pyo3-benchmarks/tests/test_benchmarks.py => tests/test_pyfunctions.py} (63%) rename pytests/{pyo3-pytests => }/tests/test_subclassing.py (100%) delete mode 100644 src/class/impl_.rs delete mode 100644 src/ffi/README.md create mode 100644 src/ffi/tests.rs rename src/{python.rs => marker.rs} (81%) create mode 100644 src/version.rs create mode 100644 tests/test_gc_pyproto.rs create mode 100644 tests/test_mapping_pyproto.rs create mode 100644 tests/test_sequence_pyproto.rs create mode 100644 tests/ui/not_send2.rs create mode 100644 tests/ui/not_send2.stderr create mode 100644 tests/ui/not_send3.rs create mode 100644 tests/ui/not_send3.stderr create mode 100644 tests/ui/not_send_auto_trait.rs create mode 100644 tests/ui/not_send_auto_trait.stderr create mode 100644 tests/ui/not_send_auto_trait2.rs create mode 100644 tests/ui/not_send_auto_trait2.stderr create mode 100644 tests/ui/send_wrapper.rs create mode 100644 tests/ui/send_wrapper.stderr diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 3429c6c3..8a3782ca 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -5,11 +5,9 @@ Please consider adding the following to your pull request: - docs to all new functions and / or detail in the guide - tests for all new or changed functions -Be aware the CI pipeline will check your pull request for the following: - - Rust tests (Just `cargo test` or `make test` if you need to test examples) - - Rust lints (`make clippy`) - - Rust formatting (`cargo fmt`) - - Python formatting (`black . --check`. You can install black with `pip install black`) - - Compatibility with all supported Python versions for all examples. This uses `tox`; you can do run it using `cargo xtask test-py`. - -You can run a similar set of checks as the CI pipeline using `make test`. +Be aware the CI pipeline will check your pull request for the following. This is done using `nox` (you can install with `pip install nox`): + - Rust tests (`cargo test` or `nox -s test-rust`) + - Examples (`nox -s test-py`) + - Rust lints (`nox -s clippy`) + - Rust formatting (`nox -s fmt-rust`) + - Python formatting (`nox -s fmt-py`) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index c1c1919d..d6d673f6 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -83,10 +83,8 @@ jobs: - name: Run benchmarks run: | - cd pytests/pyo3-benchmarks - pip install -r requirements-dev.txt - pip install . - pytest --benchmark-json ../../output.json --benchmark-enable + pip install nox + nox -f pytests/noxfile.py -s bench -- --benchmark-json $(pwd)/output.json - name: Store benchmark result uses: rhysd/github-action-benchmark@v1 with: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6dfe756d..aff00ee2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,27 +15,29 @@ jobs: steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - - run: pip install black==21.12b0 + - run: pip install nox - uses: actions-rs/toolchain@v1 with: toolchain: stable profile: minimal components: rustfmt - name: Check python formatting (black) - run: make fmt_py + run: nox -s fmt-py - name: Check rust formatting (rustfmt) - run: make fmt_rust + run: nox -s fmt-rust clippy: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - run: pip install nox - uses: actions-rs/toolchain@v1 with: toolchain: stable profile: minimal components: clippy - - run: make clippy + - run: nox -s clippy check-target: needs: [fmt] @@ -70,13 +72,13 @@ jobs: build: needs: [fmt] # don't wait for clippy as fails rarely and takes longer - name: python${{ matrix.python-version }}-${{ matrix.platform.python-architecture }} ${{ matrix.platform.os }} ${{ matrix.msrv }} + name: python${{ matrix.python-version }}-${{ matrix.platform.python-architecture }} ${{ matrix.platform.os }} rust-${{ matrix.rust }} runs-on: ${{ matrix.platform.os }} strategy: fail-fast: false # If one platform fails, allow the rest to keep testing. matrix: rust: [stable] - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11-dev", "pypy-3.7", "pypy-3.8"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11-dev", "pypy-3.7-v7.3.7", "pypy-3.8"] platform: [ { @@ -102,7 +104,7 @@ jobs: ] exclude: # PyPy doesn't release 32-bit Windows builds any more - - python-version: pypy-3.7 + - python-version: pypy-3.7-v7.3.7 platform: { os: "windows-latest", python-architecture: "x86" } - python-version: pypy-3.8 platform: { os: "windows-latest", python-architecture: "x86" } @@ -118,6 +120,16 @@ jobs: } msrv: "MSRV" + # Test the `nightly` feature + - rust: nightly + python-version: "3.10" + platform: + { + os: "ubuntu-latest", + python-architecture: "x64", + rust-target: "x86_64-unknown-linux-gnu", + } + extra_features: "nightly" steps: - uses: actions/checkout@v2 @@ -153,7 +165,7 @@ jobs: cargo update -p hashbrown:0.11.2 --precise 0.9.1 - name: Build docs - run: cargo doc --no-deps --no-default-features --features full + run: cargo doc --no-deps --no-default-features --features "full ${{ matrix.extra_features }}" - name: Build (no features) run: cargo build --lib --tests --no-default-features @@ -176,26 +188,26 @@ jobs: cargo test --no-default-features - name: Build (all additive features) - run: cargo build --lib --tests --no-default-features --features full + run: cargo build --lib --tests --no-default-features --features "full ${{ matrix.extra_features }}" - if: ${{ startsWith(matrix.python-version, 'pypy') }} name: Build PyPy (abi3-py37) - run: cargo build --lib --tests --no-default-features --features "abi3-py37 full" + run: cargo build --lib --tests --no-default-features --features "abi3-py37 full ${{ matrix.extra_features }}" # Run tests (except on PyPy, because no embedding API). - if: ${{ !startsWith(matrix.python-version, 'pypy') }} name: Test - run: cargo test --no-default-features --features full + run: cargo test --no-default-features --features "full ${{ matrix.extra_features }}" # Run tests again, but in abi3 mode - if: ${{ !startsWith(matrix.python-version, 'pypy') }} name: Test (abi3) - run: cargo test --no-default-features --features "abi3 full" + run: cargo test --no-default-features --features "abi3 full ${{ matrix.extra_features }}" # Run tests again, for abi3-py37 (the minimal Python version) - if: ${{ (!startsWith(matrix.python-version, 'pypy')) && (matrix.python-version != '3.7') }} name: Test (abi3-py37) - run: cargo test --no-default-features --features "abi3-py37 full" + run: cargo test --no-default-features --features "abi3-py37 full ${{ matrix.extra_features }}" - name: Test proc-macro code run: cargo test --manifest-path=pyo3-macros-backend/Cargo.toml @@ -259,7 +271,7 @@ jobs: components: llvm-tools-preview - name: Install cargo-llvm-cov uses: taiki-e/install-action@cargo-llvm-cov - - run: pip install -U pip nox + - run: python -m pip install -U pip nox - run: cargo xtask coverage --output-lcov coverage.lcov - uses: codecov/codecov-action@v2 with: diff --git a/Architecture.md b/Architecture.md index 8e393473..f46f6db5 100644 --- a/Architecture.md +++ b/Architecture.md @@ -19,7 +19,7 @@ Since implementing `PyClass` requires lots of boilerplate, we have a proc-macro To summarize, there are six main parts to the PyO3 codebase. 1. [Low-level bindings of Python/C API.](#1-low-level-bindings-of-python-capi) - - [`src/ffi`] + - [`pyo3-ffi`] and [`src/ffi`] 2. [Bindings to Python objects.](#2-bindings-to-python-objects) - [`src/instance.rs`] and [`src/types`] 3. [`PyClass` and related functionalities.](#3-pyclass-and-related-functionalities) @@ -34,7 +34,7 @@ To summarize, there are six main parts to the PyO3 codebase. ## 1. Low-level bindings of Python/C API -[`src/ffi`] contains wrappers of [Python/C API]. +[`pyo3-ffi`] contains wrappers of [Python/C API]. We aim to provide straight-forward Rust wrappers resembling the file structure of [`cpython/Include`](https://github.com/python/cpython/tree/v3.9.2/Include). @@ -43,7 +43,7 @@ However, we still lack some APIs and are continuously updating the module to mat the file contents upstream in CPython. The tracking issue is [#1289](https://github.com/PyO3/pyo3/issues/1289), and contribution is welcome. -In the [`src/ffi`] module, there is lots of conditional compilation such as `#[cfg(Py_LIMITED_API)]`, +In the [`pyo3-ffi`] crate, there is lots of conditional compilation such as `#[cfg(Py_LIMITED_API)]`, `#[cfg(Py_37)]`, and `#[cfg(PyPy)]`. `Py_LIMITED_API` corresponds to `#define Py_LIMITED_API` macro in Python/C API. With `Py_LIMITED_API`, we can build a Python-version-agnostic binary called an @@ -208,6 +208,7 @@ Some of the functionality of `pyo3-build-config`: [`pyo3-macros`]: https://github.com/PyO3/pyo3/tree/main/pyo3-macros [`pyo3-macros-backend`]: https://github.com/PyO3/pyo3/tree/main/pyo3-macros-backend [`pyo3-build-config`]: https://github.com/PyO3/pyo3/tree/main/pyo3-build-config +[`pyo3-ffi`]: https://github.com/PyO3/pyo3/tree/main/pyo3-ffi diff --git a/CHANGELOG.md b/CHANGELOG.md index dfb351f6..0e3a9e4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,19 +16,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `pyo3-build-config` no longer enables the `resolve-config` feature by default. [#2008](https://github.com/PyO3/pyo3/pull/2008) - Update `inventory` optional dependency to 0.2. [#2019](https://github.com/PyO3/pyo3/pull/2019) - Drop `paste` dependency. [#2081](https://github.com/PyO3/pyo3/pull/2081) +- The bindings found `pyo3::ffi` are now a re-export of the new `pyo3-ffi` crate. [#2126](https://github.com/PyO3/pyo3/pull/2126) ### Added - Add `Py::setattr` method. [#2009](https://github.com/PyO3/pyo3/pull/2009) - Add `PyCapsule`, exposing the [Capsule API](https://docs.python.org/3/c-api/capsule.html#capsules). [#1980](https://github.com/PyO3/pyo3/pull/1980) -- All PyO3 proc-macros except the deprecated `#[pyproto]` now accept a supplemental attribute `#[pyo3(crate = "some::path")]` that specifies - where to find the `pyo3` crate, in case it has been renamed or is re-exported and not found at the crate root. [#2022](https://github.com/PyO3/pyo3/pull/2022) - Expose `pyo3-build-config` APIs for cross-compiling and Python configuration discovery for use in other projects. [#1996](https://github.com/PyO3/pyo3/pull/1996) +- All PyO3 proc-macros except the deprecated `#[pyproto]` now accept a supplemental attribute `#[pyo3(crate = "some::path")]` that specifies where to find the `pyo3` crate, in case it has been renamed or is re-exported and not found at the crate root. [#2022](https://github.com/PyO3/pyo3/pull/2022) +- Enable `#[pyclass]` for fieldless (aka C-like) enums. [#2034](https://github.com/PyO3/pyo3/pull/2034) - Add buffer magic methods `__getbuffer__` and `__releasebuffer__` to `#[pymethods]`. [#2067](https://github.com/PyO3/pyo3/pull/2067) - Accept paths in `wrap_pyfunction` and `wrap_pymodule`. [#2081](https://github.com/PyO3/pyo3/pull/2081) - Add check for correct number of arguments on magic methods. [#2083](https://github.com/PyO3/pyo3/pull/2083) - `wrap_pyfunction!` can now wrap a `#[pyfunction]` which is implemented in a different Rust module or crate. [#2091](https://github.com/PyO3/pyo3/pull/2091) - Add `PyAny::contains` method (`in` operator for `PyAny`). [#2115](https://github.com/PyO3/pyo3/pull/2115) +- Add `PyMapping::contains` method (`in` operator for `PyMapping`). [#2133](https://github.com/PyO3/pyo3/pull/2133) +- Add garbage collection magic methods `__traverse__` and `__clear__` to `#[pymethods]`. [#2159](https://github.com/PyO3/pyo3/pull/2159) ### Changed @@ -43,17 +46,38 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `PyErr::new_type` now takes an optional docstring and now returns `PyResult>` rather than a `ffi::PyTypeObject` pointer. - The `create_exception!` macro can now take an optional docstring. This docstring, if supplied, is visible to users (with `.__doc__` and `help()`) and accompanies your error type in your crate's documentation. +- `__getitem__`, `__setitem__` and `__delitem__` in `#[pymethods]` now implement both a Python mapping and sequence by default. [#2065](https://github.com/PyO3/pyo3/pull/2065) - Improve performance and error messages for `#[derive(FromPyObject)]` for enums. [#2068](https://github.com/PyO3/pyo3/pull/2068) - Reduce generated LLVM code size (to improve compile times) for: - - internal `handle_panic` helper [#2074](https://github.com/PyO3/pyo3/pull/2074) + - internal `handle_panic` helper [#2074](https://github.com/PyO3/pyo3/pull/2074) [#2158](https://github.com/PyO3/pyo3/pull/2158) - `#[pyfunction]` and `#[pymethods]` argument extraction [#2075](https://github.com/PyO3/pyo3/pull/2075) [#2085](https://github.com/PyO3/pyo3/pull/2085) - - `#[pyclass]` type object creation [#2076](https://github.com/PyO3/pyo3/pull/2076) [#2081](https://github.com/PyO3/pyo3/pull/2081) + - `#[pyclass]` type object creation [#2076](https://github.com/PyO3/pyo3/pull/2076) [#2081](https://github.com/PyO3/pyo3/pull/2081) [#2157](https://github.com/PyO3/pyo3/pull/2157) - `__ipow__` now supports modulo argument on Python 3.8+. [#2083](https://github.com/PyO3/pyo3/pull/2083) - `pyo3-macros-backend` is now compiled with PyO3 cfgs to enable different magic method definitions based on version. [#2083](https://github.com/PyO3/pyo3/pull/2083) +- `_PyCFunctionFast` now correctly reflects the signature defined in the [Python docs](https://docs.python.org/3/c-api/structures.html#c._PyCFunctionFast). [#2126](https://github.com/PyO3/pyo3/pull/2126) +- `PyDateTimeAPI` and `PyDateTime_TimeZone_UTC` are are now unsafe functions instead of statics. [#2126](https://github.com/PyO3/pyo3/pull/2126) +- `PyDateTimeAPI` does not implicitly call `PyDateTime_IMPORT` anymore to reflect the original Python API more closely. Before the first call to `PyDateTime_IMPORT` a null pointer is returned. Therefore before calling any of the following FFI functions `PyDateTime_IMPORT` must be called to avoid undefined behaviour: [#2126](https://github.com/PyO3/pyo3/pull/2126) + - `PyDateTime_TimeZone_UTC` + - `PyDate_Check` + - `PyDate_CheckExact` + - `PyDateTime_Check` + - `PyDateTime_CheckExact` + - `PyTime_Check` + - `PyTime_CheckExact` + - `PyDelta_Check` + - `PyDelta_CheckExact` + - `PyTZInfo_Check` + - `PyTZInfo_CheckExact` + - `PyDateTime_FromTimestamp` + - `PyDate_FromTimestamp` +- Deprecate the `gc` option for `pyclass` (e.g. `#[pyclass(gc)]`). Just implement a `__traverse__` `#[pymethod]`. [#2159](https://github.com/PyO3/pyo3/pull/2159) +- The `ml_meth` field of `PyMethodDef` is now represented by the `PyMethodDefPointer` union [2166](https://github.com/PyO3/pyo3/pull/2166) +- The `PyTypeError` thrown when argument parsing failed now contains the nested causes [2177](https://github.com/PyO3/pyo3/pull/2178) ### Removed - Remove all functionality deprecated in PyO3 0.14. [#2007](https://github.com/PyO3/pyo3/pull/2007) +- Remove `Default` impl for `PyMethodDef` [2166](https://github.com/PyO3/pyo3/pull/2166) ### Fixed @@ -64,6 +88,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Use the Rust function path for `wrap_pymodule!` of a `#[pymodule]` with a `#[pyo3(name = "..")]` attribute, not the Python name. [#2081](https://github.com/PyO3/pyo3/pull/2081) - Fix panic in `#[pyfunction]` generated code when a required argument following an `Option` was not provided. [#2093](https://github.com/PyO3/pyo3/pull/2093) - Fixed undefined behaviour caused by incorrect `ExactSizeIterator` implementations. [#2124](https://github.com/PyO3/pyo3/pull/2124) +- Add missing FFI definitions `_PyLong_NumBits` and `_PyLong_AsByteArray` on PyPy. [#2146](https://github.com/PyO3/pyo3/pull/2146) +- Fix memory leak in implementation of `AsPyPointer` for `Option`. [#2160](https://github.com/PyO3/pyo3/pull/2160) +- Fix the signature of `_PyLong_NumBits` [#2161](https://github.com/PyO3/pyo3/pull/2161) ## [0.15.1] - 2021-11-19 diff --git a/Cargo.toml b/Cargo.toml index fc69989c..5eaf39eb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,9 @@ cfg-if = "1.0" libc = "0.2.62" parking_lot = "0.11.0" +# ffi bindings to the python interpreter, split into a seperate crate so they can be used independently +pyo3-ffi = { path = "pyo3-ffi", version = "=0.15.1" } + # support crates for macros feature pyo3-macros = { path = "pyo3-macros", version = "=0.15.1", optional = true } indoc = { version = "1.0.3", optional = true } @@ -43,13 +46,14 @@ trybuild = "1.0.49" rustversion = "1.0" # 1.0.0 requires Rust 1.50 proptest = { version = "0.10.1", default-features = false, features = ["std"] } +send_wrapper = "0.5" serde_json = "1.0.61" [build-dependencies] pyo3-build-config = { path = "pyo3-build-config", version = "0.15.1", features = ["resolve-config"] } [features] -default = ["macros"] +default = ["macros", "pyproto"] # Enables macros: #[pyclass], #[pymodule], #[pyfunction] etc. macros = ["pyo3-macros", "indoc", "unindent"] @@ -57,19 +61,22 @@ macros = ["pyo3-macros", "indoc", "unindent"] # Enables multiple #[pymethods] per #[pyclass] multiple-pymethods = ["inventory", "pyo3-macros/multiple-pymethods"] +# Enables deprecated #[pyproto] macro +pyproto = ["pyo3-macros/pyproto"] + # Use this feature when building an extension module. # It tells the linker to keep the python symbols unresolved, # so that the module can also be used with statically linked python interpreters. -extension-module = [] +extension-module = ["pyo3-ffi/extension-module"] # Use the Python limited API. See https://www.python.org/dev/peps/pep-0384/ for more. -abi3 = ["pyo3-build-config/abi3"] +abi3 = ["pyo3-build-config/abi3", "pyo3-ffi/abi3"] # With abi3, we can manually set the minimum Python version. -abi3-py37 = ["abi3-py38", "pyo3-build-config/abi3-py37"] -abi3-py38 = ["abi3-py39", "pyo3-build-config/abi3-py38"] -abi3-py39 = ["abi3-py310", "pyo3-build-config/abi3-py39"] -abi3-py310 = ["abi3", "pyo3-build-config/abi3-py310"] +abi3-py37 = ["abi3-py38", "pyo3-build-config/abi3-py37", "pyo3-ffi/abi3-py37"] +abi3-py38 = ["abi3-py39", "pyo3-build-config/abi3-py38", "pyo3-ffi/abi3-py38"] +abi3-py39 = ["abi3-py310", "pyo3-build-config/abi3-py39", "pyo3-ffi/abi3-py39"] +abi3-py310 = ["abi3", "pyo3-build-config/abi3-py310", "pyo3-ffi/abi3-py310"] # Changes `Python::with_gil` and `Python::acquire_gil` to automatically initialize the # Python interpreter if needed. @@ -80,7 +87,7 @@ nightly = [] # Activates all additional features # This is mostly intended for testing purposes - activating *all* of these isn't particularly useful. -full = ["macros", "multiple-pymethods", "num-bigint", "num-complex", "hashbrown", "serde", "indexmap", "eyre", "anyhow"] +full = ["macros", "pyproto", "multiple-pymethods", "num-bigint", "num-complex", "hashbrown", "serde", "indexmap", "eyre", "anyhow"] [[bench]] name = "bench_call" @@ -126,10 +133,11 @@ harness = false [workspace] members = [ + "pyo3-ffi", + "pyo3-build-config", "pyo3-macros", "pyo3-macros-backend", - "pytests/pyo3-benchmarks", - "pytests/pyo3-pytests", + "pytests", "examples", "xtask" ] diff --git a/Contributing.md b/Contributing.md index e7d9b69d..4fc2c246 100644 --- a/Contributing.md +++ b/Contributing.md @@ -9,7 +9,7 @@ If you want to become familiar with the codebase, see ## Getting started contributing -Please join in with any part of PyO3 which interests you. We use GitHub issues to record all bugs and ideas. Feel free to request an issue to be assigned to you if you want to work on it. +Please join in with any part of PyO3 which interests you. We use GitHub issues to record all bugs and ideas. Feel free to request an issue to be assigned to you if you want to work on it. You can browse the API of the non-public parts of PyO3 [here](https://pyo3.rs/internal/doc/pyo3/index.html). @@ -47,7 +47,7 @@ There are some specific areas of focus where help is currently needed for the do - Issues requesting documentation improvements are tracked with the [documentation](https://github.com/PyO3/pyo3/issues?q=is%3Aissue+is%3Aopen+label%3Adocumentation) label. - Not all APIs had docs or examples when they were made. The goal is to have documentation on all PyO3 APIs ([#306](https://github.com/PyO3/pyo3/issues/306)). If you see an API lacking a doc, please write one and open a PR! -You can build the docs (including all features) with +You can build the docs (including all features) with ```cargo +nightly pyo3_doc_scrape``` #### Doctests @@ -115,7 +115,7 @@ First, there are Rust-based benchmarks located in the `benches` subdirectory. As cargo +nightly bench -Second, there is a Python-based benchmark contained in the `pyo3-benchmarks` example. You can read more about it [here](examples/pyo3-benchmarks). +Second, there is a Python-based benchmark contained in the `pytests` subdirectory. You can read more about it [here](pytests). ## Sponsor this project diff --git a/Makefile b/Makefile deleted file mode 100644 index 5a6b6a18..00000000 --- a/Makefile +++ /dev/null @@ -1,59 +0,0 @@ -.PHONY: test test_py publish clippy lint fmt fmt_py fmt_rust - -ALL_ADDITIVE_FEATURES = macros multiple-pymethods num-bigint num-complex hashbrown serde indexmap eyre anyhow -COVERAGE_PACKAGES = --package pyo3 --package pyo3-build-config --package pyo3-macros-backend --package pyo3-macros - -list_all_additive_features: - @echo $(ALL_ADDITIVE_FEATURES) - -test: lint test_py - cargo test - cargo test --features="abi3" - cargo test --features="$(ALL_ADDITIVE_FEATURES)" - cargo test --features="abi3 $(ALL_ADDITIVE_FEATURES)" - -test_py: - @for example in examples/*/noxfile.py; do echo "-- Running nox for $$example --"; nox -f $$example/noxfile.py || exit 1; echo ""; done - @for package in pytests/*/noxfile.py; do echo "-- Running nox for $$package --"; nox -f $$package/noxfile.py || exit 1; echo ""; done - -fmt_py: - black . --check - -fmt_rust: - cargo fmt --all -- --check - for package in pytests/*/; do cargo fmt --manifest-path $$package/Cargo.toml -- --check || exit 1; done - -fmt: fmt_rust fmt_py - @true - -coverage: - # cargo llvm-cov clean --workspace - # cargo llvm-cov $(COVERAGE_PACKAGES) --no-report - # cargo llvm-cov $(COVERAGE_PACKAGES) --no-report --features abi3 - # cargo llvm-cov $(COVERAGE_PACKAGES) --no-report --features $(ALL_ADDITIVE_FEATURES) - # cargo llvm-cov $(COVERAGE_PACKAGES) --no-report --features abi3 $(ALL_ADDITIVE_FEATURES) - bash -c "\ - set -a\ - source <(cargo llvm-cov show-env)\ - make test_py\ - " - cargo llvm-cov $(COVERAGE_PACKAGES) --no-run --summary-only - - -clippy: - cargo clippy --features="$(ALL_ADDITIVE_FEATURES)" --all-targets --workspace -- -Dwarnings - cargo clippy --features="abi3 $(ALL_ADDITIVE_FEATURES)" --all-targets --workspace -- -Dwarnings - for example in examples/*/; do cargo clippy --manifest-path $$example/Cargo.toml -- -Dwarnings || exit 1; done - for package in pytests/*/; do cargo clippy --manifest-path $$package/Cargo.toml -- -Dwarnings || exit 1; done - -lint: fmt clippy - @true - -publish: test - cargo publish --manifest-path pyo3-build-config/Cargo.toml - sleep 10 - cargo publish --manifest-path pyo3-macros-backend/Cargo.toml - sleep 10 # wait for crates.io to update - cargo publish --manifest-path pyo3-macros/Cargo.toml - sleep 10 # wait for crates.io to update - cargo publish diff --git a/README.md b/README.md index 1acf9266..edf06319 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,9 @@ version = "0.1.0" edition = "2018" [lib] +# The name of the native library. This is the name which will be used in Python to import the +# library (i.e. `import string_sum`). If you change this, you must also change the name of the +# `#[pymodule]` in `src/lib.rs`. name = "string_sum" # "cdylib" is necessary to produce a shared library for Python to import from. # @@ -159,7 +162,7 @@ about this topic. ## Tools and libraries -- [maturin](https://github.com/PyO3/maturin) _Zero configuration build tool for Rust-made Python extensions_. +- [maturin](https://github.com/PyO3/maturin) _Build and publish crates with pyo3, rust-cpython or cffi bindings as well as rust binaries as python packages_ - [setuptools-rust](https://github.com/PyO3/setuptools-rust) _Setuptools plugin for Rust support_. - [pyo3-built](https://github.com/PyO3/pyo3-built) _Simple macro to expose metadata obtained with the [`built`](https://crates.io/crates/built) crate as a [`PyDict`](https://docs.rs/pyo3/*/pyo3/types/struct.PyDict.html)_ - [rust-numpy](https://github.com/PyO3/rust-numpy) _Rust binding of NumPy C-API_ diff --git a/build.rs b/build.rs index e0f2845b..78cebf9a 100644 --- a/build.rs +++ b/build.rs @@ -1,48 +1,7 @@ -use std::{env, process::Command}; +use std::env; -use pyo3_build_config::{ - bail, ensure, - pyo3_build_script_impl::{ - cargo_env_var, env_var, errors::Result, resolve_interpreter_config, InterpreterConfig, - PythonVersion, - }, -}; - -/// Minimum Python version PyO3 supports. -const MINIMUM_SUPPORTED_VERSION: PythonVersion = PythonVersion { major: 3, minor: 6 }; - -fn ensure_python_version(interpreter_config: &InterpreterConfig) -> Result<()> { - ensure!( - interpreter_config.version >= MINIMUM_SUPPORTED_VERSION, - "the configured Python interpreter version ({}) is lower than PyO3's minimum supported version ({})", - interpreter_config.version, - MINIMUM_SUPPORTED_VERSION, - ); - - Ok(()) -} - -fn ensure_target_pointer_width(interpreter_config: &InterpreterConfig) -> Result<()> { - if let Some(pointer_width) = interpreter_config.pointer_width { - // Try to check whether the target architecture matches the python library - let rust_target = match cargo_env_var("CARGO_CFG_TARGET_POINTER_WIDTH") - .unwrap() - .as_str() - { - "64" => 64, - "32" => 32, - x => bail!("unexpected Rust target pointer width: {}", x), - }; - - ensure!( - rust_target == pointer_width, - "your Rust target architecture ({}-bit) does not match your python interpreter ({}-bit)", - rust_target, - pointer_width - ); - } - Ok(()) -} +use pyo3_build_config::pyo3_build_script_impl::{cargo_env_var, errors::Result}; +use pyo3_build_config::{bail, print_feature_cfgs, InterpreterConfig}; fn ensure_auto_initialize_ok(interpreter_config: &InterpreterConfig) -> Result<()> { if cargo_env_var("CARGO_FEATURE_AUTO_INITIALIZE").is_some() { @@ -73,47 +32,6 @@ fn ensure_auto_initialize_ok(interpreter_config: &InterpreterConfig) -> Result<( Ok(()) } -fn rustc_minor_version() -> Option { - let rustc = env::var_os("RUSTC")?; - let output = Command::new(rustc).arg("--version").output().ok()?; - let version = core::str::from_utf8(&output.stdout).ok()?; - let mut pieces = version.split('.'); - if pieces.next() != Some("rustc 1") { - return None; - } - pieces.next()?.parse().ok() -} - -fn emit_link_config(interpreter_config: &InterpreterConfig) -> Result<()> { - let target_os = cargo_env_var("CARGO_CFG_TARGET_OS").unwrap(); - let is_extension_module = cargo_env_var("CARGO_FEATURE_EXTENSION_MODULE").is_some(); - if target_os == "windows" || target_os == "android" || !is_extension_module { - // windows and android - always link - // other systems - only link if not extension module - println!( - "cargo:rustc-link-lib={link_model}{alias}{lib_name}", - link_model = if interpreter_config.shared { - "" - } else { - "static=" - }, - alias = if target_os == "windows" { - "pythonXY:" - } else { - "" - }, - lib_name = interpreter_config.lib_name.as_ref().ok_or( - "attempted to link to Python shared library but config does not contain lib_name" - )?, - ); - if let Some(lib_dir) = &interpreter_config.lib_dir { - println!("cargo:rustc-link-search=native={}", lib_dir); - } - } - - Ok(()) -} - /// Prepares the PyO3 crate for compilation. /// /// This loads the config from pyo3-build-config and then makes some additional checks to improve UX @@ -122,50 +40,18 @@ fn emit_link_config(interpreter_config: &InterpreterConfig) -> Result<()> { /// Emits the cargo configuration based on this config as well as a few checks of the Rust compiler /// version to enable features which aren't supported on MSRV. fn configure_pyo3() -> Result<()> { - let interpreter_config = resolve_interpreter_config()?; - - if env_var("PYO3_PRINT_CONFIG").map_or(false, |os_str| os_str == "1") { - print_config_and_exit(&interpreter_config); - } - - ensure_python_version(&interpreter_config)?; - ensure_target_pointer_width(&interpreter_config)?; - ensure_auto_initialize_ok(&interpreter_config)?; - - if !interpreter_config.suppress_build_script_link_lines { - emit_link_config(&interpreter_config)?; - } + let interpreter_config = pyo3_build_config::get(); interpreter_config.emit_pyo3_cfgs(); - let rustc_minor_version = rustc_minor_version().unwrap_or(0); + ensure_auto_initialize_ok(interpreter_config)?; - // Enable use of const generics on Rust 1.51 and greater - if rustc_minor_version >= 51 { - println!("cargo:rustc-cfg=min_const_generics"); - } - - // Enable use of std::ptr::addr_of! on Rust 1.51 and greater - if rustc_minor_version >= 51 { - println!("cargo:rustc-cfg=addr_of"); - } - - // Extra lines come last, to support last write wins. - for line in &interpreter_config.extra_build_script_lines { - println!("{}", line); - } + // Emit cfgs like `addr_of` and `min_const_generics` + print_feature_cfgs(); Ok(()) } -fn print_config_and_exit(config: &InterpreterConfig) { - println!("\n-- PYO3_PRINT_CONFIG=1 is set, printing configuration and halting compile --"); - config - .to_writer(&mut std::io::stdout()) - .expect("failed to print config to stdout"); - std::process::exit(101); -} - fn main() { if let Err(e) = configure_pyo3() { eprintln!("error: {}", e.report()); diff --git a/guide/src/SUMMARY.md b/guide/src/SUMMARY.md index 0d5e4b1f..c1ed4711 100644 --- a/guide/src/SUMMARY.md +++ b/guide/src/SUMMARY.md @@ -8,6 +8,8 @@ - [Python Functions](function.md) - [Python Classes](class.md) - [Class customizations](class/protocols.md) + - [Basic object customization](class/object.md) + - [Emulating numeric types](class/numeric.md) - [Emulating callable objects](class/call.md) - [Type Conversions](conversions.md) - [Mapping of Rust types to Python types](conversions/tables.md)] diff --git a/guide/src/building_and_distribution.md b/guide/src/building_and_distribution.md index 0644859d..5a3a7ab5 100644 --- a/guide/src/building_and_distribution.md +++ b/guide/src/building_and_distribution.md @@ -163,7 +163,7 @@ not work when compiling for `abi3`. These are: - `#[pyo3(text_signature = "...")]` does not work on classes until Python 3.10 or greater. - The `dict` and `weakref` options on classes are not supported until Python 3.9 or greater. -- The buffer API is not supported. +- The buffer API is not supported until Python 3.11 or greater. - Optimizations which rely on knowledge of the exact Python version compiled against. ## Embedding Python in Rust diff --git a/guide/src/class.md b/guide/src/class.md index 51c6e01d..1814d766 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -2,7 +2,7 @@ PyO3 exposes a group of attributes powered by Rust's proc macro system for defining Python classes as Rust structs. -The main attribute is `#[pyclass]`, which is placed upon a Rust `struct` to generate a Python type for it. A struct will usually also have *one* `#[pymethods]`-annotated `impl` block for the struct, which is used to define Python methods and constants for the generated Python type. (If the [`multiple-pymethods`] feature is enabled each `#[pyclass]` is allowed to have multiple `#[pymethods]` blocks.) Finally, there may be multiple `#[pyproto]` trait implementations for the struct, which are used to define certain python magic methods such as `__str__`. +The main attribute is `#[pyclass]`, which is placed upon a Rust `struct` or a fieldless `enum` (a.k.a. C-like enum) to generate a Python type for it. They will usually also have *one* `#[pymethods]`-annotated `impl` block for the struct, which is used to define Python methods and constants for the generated Python type. (If the [`multiple-pymethods`] feature is enabled each `#[pyclass]` is allowed to have multiple `#[pymethods]` blocks.) Finally, there may be multiple `#[pyproto]` trait implementations for the struct, which are used to define certain python magic methods such as `__str__`. This chapter will discuss the functionality and configuration these attributes offer. Below is a list of links to the relevant section of this chapter for each: @@ -20,37 +20,87 @@ This chapter will discuss the functionality and configuration these attributes o ## Defining a new class -To define a custom Python class, a Rust struct needs to be annotated with the -`#[pyclass]` attribute. - +To define a custom Python class, add the `#[pyclass]` attribute to a Rust struct or a fieldless enum. ```rust # #![allow(dead_code)] -# use pyo3::prelude::*; +use pyo3::prelude::*; + #[pyclass] -struct MyClass { - # #[pyo3(get)] - num: i32, +struct Integer{ + inner: i32 +} + +// A "tuple" struct +#[pyclass] +struct Number(i32); + +// PyO3 supports custom discriminants in enums +#[pyclass] +enum HttpResponse { + Ok = 200, + NotFound = 404, + Teapot = 418, + // ... } ``` -Because Python objects are freely shared between threads by the Python interpreter, all structs annotated with `#[pyclass]` must implement `Send` (unless annotated with [`#[pyclass(unsendable)]`](#customizing-the-class)). +Because Python objects are freely shared between threads by the Python interpreter, all types annotated with `#[pyclass]` must implement `Send` (unless annotated with [`#[pyclass(unsendable)]`](#customizing-the-class)). -The above example generates implementations for [`PyTypeInfo`], [`PyTypeObject`], and [`PyClass`] for `MyClass`. To see these generated implementations, refer to the [implementation details](#implementation-details) at the end of this chapter. +The above example generates implementations for [`PyTypeInfo`], [`PyTypeObject`], and [`PyClass`] for `MyClass` and `MyEnum`. To see these generated implementations, refer to the [implementation details](#implementation-details) at the end of this chapter. -## Adding the class to a module +## Constructor -Custom Python classes can then be added to a module using `add_class()`. +By default it is not possible to create an instance of a custom class from Python code. +To declare a constructor, you need to define a method and annotate it with the `#[new]` +attribute. Only Python's `__new__` method can be specified, `__init__` is not available. ```rust # use pyo3::prelude::*; # #[pyclass] -# struct MyClass { -# #[allow(dead_code)] -# num: i32, -# } +# struct Number(i32); +# +#[pymethods] +impl Number { + #[new] + fn new(value: i32) -> Self { + Number(value) + } +} +``` + +Alternatively, if your `new` method may fail you can return `PyResult`. + +```rust +# use pyo3::prelude::*; +# #[pyclass] +# struct Number(i32); +# +#[pymethods] +impl Number { + #[new] + fn new(value: i32) -> Self { + Number(value) + } +} +``` + +If no method marked with `#[new]` is declared, object instances can only be +created from Rust, but not from Python. + +For arguments, see the `Method arguments` section below. + +## Adding the class to a module + +The next step is to create the module initializer and add our class to it + +```rust +# use pyo3::prelude::*; +# #[pyclass] +# struct Number(i32); +# #[pymodule] -fn mymodule(_py: Python, m: &PyModule) -> PyResult<()> { - m.add_class::()?; +fn my_module(_py: Python, m: &PyModule) -> PyResult<()> { + m.add_class::()?; Ok(()) } ``` @@ -140,60 +190,14 @@ so that they can benefit from a freelist. `XXX` is a number of items for the fre * `gc` - Classes with the `gc` parameter participate in Python garbage collection. If a custom class contains references to other Python objects that can be collected, the [`PyGCProtocol`]({{#PYO3_DOCS_URL}}/pyo3/class/gc/trait.PyGCProtocol.html) trait has to be implemented. * `weakref` - Adds support for Python weak references. -* `extends=BaseType` - Use a custom base class. The base `BaseType` must implement `PyTypeInfo`. -* `subclass` - Allows Python classes to inherit from this class. +* `extends=BaseType` - Use a custom base class. The base `BaseType` must implement `PyTypeInfo`. `enum` pyclasses can't use a custom base class. +* `subclass` - Allows Python classes to inherit from this class. `enum` pyclasses can't be inherited from. * `dict` - Adds `__dict__` support, so that the instances of this type have a dictionary containing arbitrary instance variables. * `unsendable` - Making it safe to expose `!Send` structs to Python, where all object can be accessed by multiple threads. A class marked with `unsendable` panics when accessed by another thread. * `module="XXX"` - Set the name of the module the class will be shown as defined in. If not given, the class will be a virtual member of the `builtins` module. -## Constructor - -By default it is not possible to create an instance of a custom class from Python code. -To declare a constructor, you need to define a method and annotate it with the `#[new]` -attribute. Only Python's `__new__` method can be specified, `__init__` is not available. - -```rust -# use pyo3::prelude::*; -#[pyclass] -struct MyClass { - # #[allow(dead_code)] - num: i32, -} - -#[pymethods] -impl MyClass { - #[new] - fn new(num: i32) -> Self { - MyClass { num } - } -} -``` - -Alternatively, if your `new` method may fail you can return `PyResult`. -```rust -# use pyo3::prelude::*; -#[pyclass] -struct MyClass { - # #[allow(dead_code)] - num: i32, -} - -#[pymethods] -impl MyClass { - #[new] - fn new(num: i32) -> PyResult { - Ok(MyClass { num }) - } -} -``` - -If no method marked with `#[new]` is declared, object instances can only be -created from Rust, but not from Python. - -For arguments, see the `Method arguments` section below. - ### Return type Generally, `#[new]` method have to return `T: Into>` or @@ -351,7 +355,7 @@ impl SubClass { ## Object properties PyO3 supports two ways to add properties to your `#[pyclass]`: -- For simple fields with no side effects, a `#[pyo3(get, set)]` attribute can be added directly to the field definition in the `#[pyclass]`. +- For simple struct fields with no side effects, a `#[pyo3(get, set)]` attribute can be added directly to the field definition in the `#[pyclass]`. - For properties which require computation you can define `#[getter]` and `#[setter]` functions in the [`#[pymethods]`](#instance-methods) block. We'll cover each of these in the following sections. @@ -802,6 +806,118 @@ impl MyClass { Note that `text_signature` on classes is not compatible with compilation in `abi3` mode until Python 3.10 or greater. +## #[pyclass] enums + +Currently PyO3 only supports fieldless enums. PyO3 adds a class attribute for each variant, so you can access them in Python without defining `#[new]`. PyO3 also provides default implementations of `__richcmp__` and `__int__`, so they can be compared using `==`: + +```rust +# use pyo3::prelude::*; +#[pyclass] +enum MyEnum { + Variant, + OtherVariant, +} + +Python::with_gil(|py| { + let x = Py::new(py, MyEnum::Variant).unwrap(); + let y = Py::new(py, MyEnum::OtherVariant).unwrap(); + let cls = py.get_type::(); + pyo3::py_run!(py, x y cls, r#" + assert x == cls.Variant + assert y == cls.OtherVariant + assert x != y + "#) +}) +``` + +You can also convert your enums into `int`: + +```rust +# use pyo3::prelude::*; +#[pyclass] +enum MyEnum { + Variant, + OtherVariant = 10, +} + +Python::with_gil(|py| { + let cls = py.get_type::(); + let x = MyEnum::Variant as i32; // The exact value is assigned by the compiler. + pyo3::py_run!(py, cls x, r#" + assert int(cls.Variant) == x + assert int(cls.OtherVariant) == 10 + assert cls.OtherVariant == 10 # You can also compare against int. + assert 10 == cls.OtherVariant + "#) +}) +``` + +PyO3 also provides `__repr__` for enums: + +```rust +# use pyo3::prelude::*; +#[pyclass] +enum MyEnum{ + Variant, + OtherVariant, +} + +Python::with_gil(|py| { + let cls = py.get_type::(); + let x = Py::new(py, MyEnum::Variant).unwrap(); + pyo3::py_run!(py, cls x, r#" + assert repr(x) == 'MyEnum.Variant' + assert repr(cls.OtherVariant) == 'MyEnum.OtherVariant' + "#) +}) +``` + +All methods defined by PyO3 can be overriden. For example here's how you override `__repr__`: + +```rust +# use pyo3::prelude::*; +#[pyclass] +enum MyEnum { + Answer = 42, +} + +#[pymethods] +impl MyEnum { + fn __repr__(&self) -> &'static str { + "42" + } +} + +Python::with_gil(|py| { + let cls = py.get_type::(); + pyo3::py_run!(py, cls, "assert repr(cls.Answer) == '42'") +}) +``` + +You may not use enums as a base class or let enums inherit from other classes. + +```rust,compile_fail +# use pyo3::prelude::*; +#[pyclass(subclass)] +enum BadBase{ + Var1, +} +``` + +```rust,compile_fail +# use pyo3::prelude::*; + +#[pyclass(subclass)] +struct Base; + +#[pyclass(extends=Base)] +enum BadSubclass{ + Var1, +} +``` + +`#[pyclass]` enums are currently not interoperable with `IntEnum` in Python. + ## Implementation details The `#[pyclass]` macros rely on a lot of conditional code generation: each `#[pyclass]` can optionally have a `#[pymethods]` block as well as several different possible `#[pyproto]` trait implementations. @@ -849,55 +965,20 @@ impl pyo3::IntoPy for MyClass { } } -impl pyo3::class::impl_::PyClassImpl for MyClass { +impl pyo3::impl_::pyclass::PyClassImpl for MyClass { const DOC: &'static str = "Class for demonstration\u{0}"; - const IS_GC: bool = false; const IS_BASETYPE: bool = false; const IS_SUBCLASS: bool = false; type Layout = PyCell; type BaseType = PyAny; - type ThreadChecker = pyo3::class::impl_::ThreadCheckerStub; + type ThreadChecker = pyo3::impl_::pyclass::ThreadCheckerStub; - fn for_each_method_def(visitor: &mut dyn FnMut(&[pyo3::class::PyMethodDefType])) { - use pyo3::class::impl_::*; + fn for_all_items(visitor: &mut dyn FnMut(&pyo3::impl_::pyclass::PyClassItems)) { + use pyo3::impl_::pyclass::*; let collector = PyClassImplCollector::::new(); + static INTRINSIC_ITEMS: PyClassItems = PyClassItems { slots: &[], methods: &[] }; + visitor(&INTRINSIC_ITEMS); visitor(collector.py_methods()); - visitor(collector.py_class_descriptors()); - visitor(collector.object_protocol_methods()); - visitor(collector.async_protocol_methods()); - visitor(collector.descr_protocol_methods()); - visitor(collector.mapping_protocol_methods()); - visitor(collector.number_protocol_methods()); - } - fn get_new() -> Option { - use pyo3::class::impl_::*; - let collector = PyClassImplCollector::::new(); - collector.new_impl() - } - fn get_alloc() -> Option { - use pyo3::class::impl_::*; - let collector = PyClassImplCollector::::new(); - collector.alloc_impl() - } - fn get_free() -> Option { - use pyo3::class::impl_::*; - let collector = PyClassImplCollector::::new(); - collector.free_impl() - } - fn for_each_proto_slot(visitor: &mut dyn FnMut(&[pyo3::ffi::PyType_Slot])) { - // Implementation which uses dtolnay specialization to load all slots. - use pyo3::class::impl_::*; - let collector = PyClassImplCollector::::new(); - visitor(collector.object_protocol_slots()); - visitor(collector.number_protocol_slots()); - visitor(collector.iter_protocol_slots()); - visitor(collector.gc_protocol_slots()); - visitor(collector.descr_protocol_slots()); - visitor(collector.mapping_protocol_slots()); - visitor(collector.sequence_protocol_slots()); - visitor(collector.async_protocol_slots()); - visitor(collector.buffer_protocol_slots()); - visitor(collector.methods_protocol_slots()); } } # Python::with_gil(|py| { diff --git a/guide/src/class/numeric.md b/guide/src/class/numeric.md new file mode 100644 index 00000000..0fc06c4e --- /dev/null +++ b/guide/src/class/numeric.md @@ -0,0 +1,449 @@ +# Emulating numeric types + +At this point we have a `Number` class that we can't actually do any math on! + +Before proceeding, we should think about how we want to handle overflows. There are three obvious solutions: +- We can have infinite precision just like Python's `int`. However that would be quite boring - we'd + be reinventing the wheel. +- We can raise exceptions whenever `Number` overflows, but that makes the API painful to use. +- We can wrap around the boundary of `i32`. This is the approach we'll take here. To do that we'll just forward to `i32`'s + `wrapping_*` methods. + +### Fixing our constructor + +Let's address the first overflow, in `Number`'s constructor: + +```python +from my_module import Number + +n = Number(1 << 1337) +``` + +```text +Traceback (most recent call last): + File "example.py", line 3, in + n = Number(1 << 1337) +OverflowError: Python int too large to convert to C long +``` + +Instead of relying on the default [`FromPyObject`] extraction to parse arguments, we can specify our +own extraction function, using the `#[pyo3(from_py_with = "...")]` attribute. Unfortunately PyO3 +doesn't provide a way to wrap Python integers out of the box, but we can do a Python call to mask it +and cast it to an `i32`. + +```rust +# #![allow(dead_code)] +use pyo3::prelude::*; + +fn wrap(obj: &PyAny) -> Result { + let val = obj.call_method1("__and__", (0xFFFFFFFF_u32,))?; + let val: u32 = val.extract()?; + // 👇 This intentionally overflows! + Ok(val as i32) +} +``` +We also add documentation, via `///` comments and the `#[pyo3(text_signature = "...")]` attribute, both of which are visible to Python users. + +```rust +# #![allow(dead_code)] +use pyo3::prelude::*; + +fn wrap(obj: &PyAny) -> Result { + let val = obj.call_method1("__and__", (0xFFFFFFFF_u32,))?; + let val: u32 = val.extract()?; + Ok(val as i32) +} + +/// Did you ever hear the tragedy of Darth Signed The Overfloweth? I thought not. +/// It's not a story C would tell you. It's a Rust legend. +#[pyclass(module = "my_module")] +#[pyo3(text_signature = "(int)")] +struct Number(i32); + +#[pymethods] +impl Number { + #[new] + fn new(#[pyo3(from_py_with = "wrap")] value: i32) -> Self { + Self(value) + } +} +``` + + +With that out of the way, let's implement some operators: +```rust +use std::convert::TryInto; +use pyo3::exceptions::{PyZeroDivisionError, PyValueError}; + +# use pyo3::prelude::*; +# +# #[pyclass] +# struct Number(i32); +# +#[pymethods] +impl Number { + fn __add__(&self, other: &Self) -> Self { + Self(self.0.wrapping_add(other.0)) + } + + fn __sub__(&self, other: &Self) -> Self { + Self(self.0.wrapping_sub(other.0)) + } + + fn __mul__(&self, other: &Self) -> Self { + Self(self.0.wrapping_mul(other.0)) + } + + fn __truediv__(&self, other: &Self) -> PyResult { + match self.0.checked_div(other.0) { + Some(i) => Ok(Self(i)), + None => Err(PyZeroDivisionError::new_err("division by zero")), + } + } + + fn __floordiv__(&self, other: &Self) -> PyResult { + match self.0.checked_div(other.0) { + Some(i) => Ok(Self(i)), + None => Err(PyZeroDivisionError::new_err("division by zero")), + } + } + + fn __rshift__(&self, other: &Self) -> PyResult { + match other.0.try_into() { + Ok(rhs) => Ok(Self(self.0.wrapping_shr(rhs))), + Err(_) => Err(PyValueError::new_err("negative shift count")), + } + } + + fn __lshift__(&self, other: &Self) -> PyResult { + match other.0.try_into() { + Ok(rhs) => Ok(Self(self.0.wrapping_shl(rhs))), + Err(_) => Err(PyValueError::new_err("negative shift count")), + } + } +} +``` + +### Unary arithmethic operations + +```rust +# use pyo3::prelude::*; +# +# #[pyclass] +# struct Number(i32); +# +#[pymethods] +impl Number { + fn __pos__(slf: PyRef) -> PyRef { + slf + } + + fn __neg__(&self) -> Self { + Self(-self.0) + } + + fn __abs__(&self) -> Self { + Self(self.0.abs()) + } + + fn __invert__(&self) -> Self { + Self(!self.0) + } +} +``` + +### Support for the `complex()`, `int()` and `float()` built-in functions. + +```rust +# use pyo3::prelude::*; +# +# #[pyclass] +# struct Number(i32); +# +use pyo3::types::PyComplex; + +#[pymethods] +impl Number { + fn __int__(&self) -> i32 { + self.0 + } + + fn __float__(&self) -> f64 { + self.0 as f64 + } + + fn __complex__<'py>(&self, py: Python<'py>) -> &'py PyComplex { + PyComplex::from_doubles(py, self.0 as f64, 0.0) + } +} +``` + +We do not implement the in-place operations like `__iadd__` because we do not wish to mutate `Number`. +Similarly we're not interested in supporting operations with different types, so we do not implement + the reflected operations like `__radd__` either. + +Now Python can use our `Number` class: + +```python +from my_module import Number + +def hash_djb2(s: str): + ''' + A version of Daniel J. Bernstein's djb2 string hashing algorithm + Like many hashing algorithms, it relies on integer wrapping. + ''' + + n = Number(0) + five = Number(5) + + for x in s: + n = Number(ord(x)) + ((n << five) - n) + return n + +assert hash_djb2('l50_50') == Number(-1152549421) +``` + +### Final code + +```rust +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; +use std::convert::TryInto; + +use pyo3::exceptions::{PyValueError, PyZeroDivisionError}; +use pyo3::prelude::*; +use pyo3::class::basic::CompareOp; +use pyo3::types::PyComplex; + +fn wrap(obj: &PyAny) -> Result { + let val = obj.call_method1("__and__", (0xFFFFFFFF_u32,))?; + let val: u32 = val.extract()?; + Ok(val as i32) +} +/// Did you ever hear the tragedy of Darth Signed The Overfloweth? I thought not. +/// It's not a story C would tell you. It's a Rust legend. +#[pyclass(module = "my_module")] +#[pyo3(text_signature = "(int)")] +struct Number(i32); + +#[pymethods] +impl Number { + #[new] + fn new(#[pyo3(from_py_with = "wrap")] value: i32) -> Self { + Self(value) + } + + fn __repr__(&self) -> String { + format!("Number({})", self.0) + } + + fn __str__(&self) -> String { + self.0.to_string() + } + + fn __hash__(&self) -> u64 { + let mut hasher = DefaultHasher::new(); + self.0.hash(&mut hasher); + hasher.finish() + } + + fn __richcmp__(&self, other: &Self, op: CompareOp) -> PyResult { + match op { + CompareOp::Lt => Ok(self.0 < other.0), + CompareOp::Le => Ok(self.0 <= other.0), + CompareOp::Eq => Ok(self.0 == other.0), + CompareOp::Ne => Ok(self.0 != other.0), + CompareOp::Gt => Ok(self.0 > other.0), + CompareOp::Ge => Ok(self.0 >= other.0), + } + } + + fn __bool__(&self) -> bool { + self.0 != 0 + } + + fn __add__(&self, other: &Self) -> Self { + Self(self.0.wrapping_add(other.0)) + } + + fn __sub__(&self, other: &Self) -> Self { + Self(self.0.wrapping_sub(other.0)) + } + + fn __mul__(&self, other: &Self) -> Self { + Self(self.0.wrapping_mul(other.0)) + } + + fn __truediv__(&self, other: &Self) -> PyResult { + match self.0.checked_div(other.0) { + Some(i) => Ok(Self(i)), + None => Err(PyZeroDivisionError::new_err("division by zero")), + } + } + + fn __floordiv__(&self, other: &Self) -> PyResult { + match self.0.checked_div(other.0) { + Some(i) => Ok(Self(i)), + None => Err(PyZeroDivisionError::new_err("division by zero")), + } + } + + fn __rshift__(&self, other: &Self) -> PyResult { + match other.0.try_into() { + Ok(rhs) => Ok(Self(self.0.wrapping_shr(rhs))), + Err(_) => Err(PyValueError::new_err("negative shift count")), + } + } + + fn __lshift__(&self, other: &Self) -> PyResult { + match other.0.try_into() { + Ok(rhs) => Ok(Self(self.0.wrapping_shl(rhs))), + Err(_) => Err(PyValueError::new_err("negative shift count")), + } + } + + fn __xor__(&self, other: &Self) -> Self { + Self(self.0 ^ other.0) + } + + fn __or__(&self, other: &Self) -> Self { + Self(self.0 | other.0) + } + + fn __and__(&self, other: &Self) -> Self { + Self(self.0 & other.0) + } + + fn __int__(&self) -> i32 { + self.0 + } + + fn __float__(&self) -> f64 { + self.0 as f64 + } + + fn __complex__<'py>(&self, py: Python<'py>) -> &'py PyComplex { + PyComplex::from_doubles(py, self.0 as f64, 0.0) + } +} + +#[pymodule] +fn my_module(_py: Python, m: &PyModule) -> PyResult<()> { + m.add_class::()?; + Ok(()) +} +# const SCRIPT: &'static str = r#" +# def hash_djb2(s: str): +# n = Number(0) +# five = Number(5) +# +# for x in s: +# n = Number(ord(x)) + ((n << five) - n) +# return n +# +# assert hash_djb2('l50_50') == Number(-1152549421) +# assert hash_djb2('logo') == Number(3327403) +# assert hash_djb2('horizon') == Number(1097468315) +# +# +# assert Number(2) + Number(2) == Number(4) +# assert Number(2) + Number(2) != Number(5) +# +# assert Number(13) - Number(7) == Number(6) +# assert Number(13) - Number(-7) == Number(20) +# +# assert Number(13) / Number(7) == Number(1) +# assert Number(13) // Number(7) == Number(1) +# +# assert Number(13) * Number(7) == Number(13*7) +# +# assert Number(13) > Number(7) +# assert Number(13) < Number(20) +# assert Number(13) == Number(13) +# assert Number(13) >= Number(7) +# assert Number(13) <= Number(20) +# assert Number(13) == Number(13) +# +# +# assert (True if Number(1) else False) +# assert (False if Number(0) else True) +# +# +# assert int(Number(13)) == 13 +# assert float(Number(13)) == 13 +# assert Number.__doc__ == "Did you ever hear the tragedy of Darth Signed The Overfloweth? I thought not.\nIt's not a story C would tell you. It's a Rust legend." +# assert Number(12345234523452) == Number(1498514748) +# try: +# import inspect +# assert inspect.signature(Number).__str__() == '(int)' +# except ValueError: +# # Not supported with `abi3` before Python 3.10 +# pass +# assert Number(1337).__str__() == '1337' +# assert Number(1337).__repr__() == 'Number(1337)' +"#; + +# +# use pyo3::type_object::PyTypeObject; +# +# fn main() -> PyResult<()> { +# Python::with_gil(|py| -> PyResult<()> { +# let globals = PyModule::import(py, "__main__")?.dict(); +# globals.set_item("Number", Number::type_object(py))?; +# +# py.run(SCRIPT, Some(globals), None)?; +# Ok(()) +# }) +# } + +``` + +## Appendix: Writing some unsafe code + +At the beginning of this chapter we said that PyO3 doesn't provide a way to wrap Python integers out +of the box but that's a half truth. There's not a PyO3 API for it, but there's a Python C API +function that does: + +```c +unsigned long PyLong_AsUnsignedLongMask(PyObject *obj) +``` + +We can call this function from Rust by using [`pyo3::ffi::PyLong_AsUnsignedLongMask`]. This is an *unsafe* +function, which means we have to use an unsafe block to call it and take responsibility for upholding +the contracts of this function. Let's review those contracts: +- The GIL must be held. If it's not, calling this function causes a data race. +- The pointer must be valid, i.e. it must be properly aligned and point to a valid Python object. + +Let's create that helper function. The signature has to be `fn(&PyAny) -> PyResult`. +- `&PyAny` represents a checked borrowed reference, so the pointer derived from it is valid (and not null). +- Whenever we have borrowed references to Python objects in scope, it is guaranteed that the GIL is held. This reference is also where we can get a [`Python`] token to use in our call to [`PyErr::take`]. + +```rust +# #![allow(dead_code)] +use std::os::raw::c_ulong; +use pyo3::prelude::*; +use pyo3::ffi; +use pyo3::conversion::AsPyPointer; + +fn wrap(obj: &PyAny) -> Result { + let py: Python = obj.py(); + + unsafe { + let ptr = obj.as_ptr(); + + let ret: c_ulong = ffi::PyLong_AsUnsignedLongMask(ptr); + if ret == c_ulong::MAX { + if let Some(err) = PyErr::take(py) { + return Err(err); + } + } + + Ok(ret as i32) + } +} +``` + +[`PyErr::take`]: https://docs.rs/pyo3/latest/pyo3/prelude/struct.PyErr.html#method.take +[`Python`]: https://docs.rs/pyo3/latest/pyo3/struct.Python.html +[`FromPyObject`]: https://docs.rs/pyo3/latest/pyo3/conversion/trait.FromPyObject.html +[`pyo3::ffi::PyLong_AsUnsignedLongMask`]: https://docs.rs/pyo3/latest/pyo3/ffi/fn.PyLong_AsUnsignedLongMask.html \ No newline at end of file diff --git a/guide/src/class/object.md b/guide/src/class/object.md new file mode 100644 index 00000000..7a20d03c --- /dev/null +++ b/guide/src/class/object.md @@ -0,0 +1,232 @@ +# Basic object customization + +Recall the `Number` class from the previous chapter: + +```rust +use pyo3::prelude::*; + +#[pyclass] +struct Number(i32); + +#[pymethods] +impl Number { + #[new] + fn new(value: i32) -> Self { + Self(value) + } +} + +#[pymodule] +fn my_module(_py: Python, m: &PyModule) -> PyResult<()> { + m.add_class::()?; + Ok(()) +} +``` + +At this point Python code can import the module, access the class and create class instances - but +nothing else. + +```python +from my_module import Number + +n = Number(5) +print(n) +``` + +```text + +``` + +### String representations + +It can't even print an user-readable representation of itself! We can fix that by defining the + `__repr__` and `__str__` methods inside a `#[pymethods]` block. We do this by accessing the value + contained inside `Number`. + + ```rust +# use pyo3::prelude::*; +# +# #[pyclass] +# struct Number(i32); +# +#[pymethods] +impl Number { + // For `__repr__` we want to return a string that Python code could use to recreate + // the `Number`, like `Number(5)` for example. + fn __repr__(&self) -> String { + // We use the `format!` macro to create a string. Its first argument is a + // format string, followed by any number of parameters which replace the + // `{}`'s in the format string. + // + // 👇 Tuple field access in Rust uses a dot + format!("Number({})", self.0) + } + + // `__str__` is generally used to create an "informal" representation, so we + // just forward to `i32`'s `ToString` trait implementation to print a bare number. + fn __str__(&self) -> String { + self.0.to_string() + } +} +``` + +### Hashing + + +Let's also implement hashing. We'll just hash the `i32`. For that we need a [`Hasher`]. The one +provided by `std` is [`DefaultHasher`], which uses the [SipHash] algorithm. + +```rust +use std::collections::hash_map::DefaultHasher; + +// Required to call the `.hash` and `.finish` methods, which are defined on traits. +use std::hash::{Hash, Hasher}; + +# use pyo3::prelude::*; +# +# #[pyclass] +# struct Number(i32); +# +#[pymethods] +impl Number { + fn __hash__(&self) -> u64 { + let mut hasher = DefaultHasher::new(); + self.0.hash(&mut hasher); + hasher.finish() + } +} +``` + +> **Note**: When implementing `__hash__` and comparisons, it is important that the following property holds: +> +> ```text +> k1 == k2 -> hash(k1) == hash(k2) +> ``` +> +> In other words, if two keys are equal, their hashes must also be equal. In addition you must take +> care that your classes' hash doesn't change during its lifetime. In this tutorial we do that by not +> letting Python code change our `Number` class. In other words, it is immutable. +> +> By default, all `#[pyclass]` types have a default hash implementation from Python. +> Types which should not be hashable can override this by setting `__hash__` to None. +> This is the same mechanism as for a pure-Python class. This is done like so: +> +> ```rust +> # use pyo3::prelude::*; +> #[pyclass] +> struct NotHashable { } +> +> #[pymethods] +> impl NotHashable { +> #[classattr] +> const __hash__: Option> = None; +>} +> ``` + +### Comparisons + +Unlike in Python, PyO3 does not provide the magic comparison methods you might expect like `__eq__`, + `__lt__` and so on. Instead you have to implement all six operations at once with `__richcmp__`. +This method will be called with a value of `CompareOp` depending on the operation. + +```rust +use pyo3::class::basic::CompareOp; + +# use pyo3::prelude::*; +# +# #[pyclass] +# struct Number(i32); +# +#[pymethods] +impl Number { + fn __richcmp__(&self, other: &Self, op: CompareOp) -> PyResult { + match op { + CompareOp::Lt => Ok(self.0 < other.0), + CompareOp::Le => Ok(self.0 <= other.0), + CompareOp::Eq => Ok(self.0 == other.0), + CompareOp::Ne => Ok(self.0 != other.0), + CompareOp::Gt => Ok(self.0 > other.0), + CompareOp::Ge => Ok(self.0 >= other.0), + } + } +} +``` + +### Truthyness + +We'll consider `Number` to be `True` if it is nonzero: + +```rust +# use pyo3::prelude::*; +# +# #[pyclass] +# struct Number(i32); +# +#[pymethods] +impl Number { + fn __bool__(&self) -> bool { + self.0 != 0 + } +} +``` + +### Final code + +```rust +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; + +use pyo3::prelude::*; +use pyo3::class::basic::CompareOp; + +#[pyclass] +struct Number(i32); + +#[pymethods] +impl Number { + #[new] + fn new(value: i32) -> Self { + Self(value) + } + + fn __repr__(&self) -> String { + format!("Number({})", self.0) + } + + fn __str__(&self) -> String { + self.0.to_string() + } + + fn __hash__(&self) -> u64 { + let mut hasher = DefaultHasher::new(); + self.0.hash(&mut hasher); + hasher.finish() + } + + fn __richcmp__(&self, other: &Self, op: CompareOp) -> PyResult { + match op { + CompareOp::Lt => Ok(self.0 < other.0), + CompareOp::Le => Ok(self.0 <= other.0), + CompareOp::Eq => Ok(self.0 == other.0), + CompareOp::Ne => Ok(self.0 != other.0), + CompareOp::Gt => Ok(self.0 > other.0), + CompareOp::Ge => Ok(self.0 >= other.0), + } + } + + fn __bool__(&self) -> bool { + self.0 != 0 + } +} + +#[pymodule] +fn my_module(_py: Python, m: &PyModule) -> PyResult<()> { + m.add_class::()?; + Ok(()) +} +``` + +[`Hash`]: https://doc.rust-lang.org/std/hash/trait.Hash.html +[`Hasher`]: https://doc.rust-lang.org/std/hash/trait.Hasher.html +[`DefaultHasher`]: https://doc.rust-lang.org/std/collections/hash_map/struct.DefaultHasher.html +[SipHash]: https://en.wikipedia.org/wiki/SipHash \ No newline at end of file diff --git a/guide/src/class/protocols.md b/guide/src/class/protocols.md index fc6867c8..e4566287 100644 --- a/guide/src/class/protocols.md +++ b/guide/src/class/protocols.md @@ -18,7 +18,6 @@ In PyO3 0.15, if a function name in `#[pymethods]` is a recognised magic method, The magic methods handled by PyO3 are very similar to the standard Python ones on [this page](https://docs.python.org/3/reference/datamodel.html#special-method-names) - in particular they are the the subset which have slots as [defined here](https://docs.python.org/3/c-api/typeobj.html). Some of the slots do not have a magic method in Python, which leads to a few additional magic methods defined only in PyO3: - Magic methods for garbage collection - Magic methods for the buffer protocol - - Magic methods for the sequence protocol When PyO3 handles a magic method, a couple of changes apply compared to other `#[pymethods]`: - The `#[pyo3(text_signature = "...")]` attribute is not allowed @@ -86,11 +85,7 @@ given signatures should be interpreted as follows: - `__aiter__() -> object` - `__anext__() -> Option or IterANextOutput` -#### Sequence types - -TODO; see [#1884](https://github.com/PyO3/pyo3/issues/1884) - -#### Mapping types +#### Mapping & Sequence types - `__len__() -> usize` - `__contains__(, object) -> bool` @@ -180,7 +175,8 @@ TODO; see [#1884](https://github.com/PyO3/pyo3/issues/1884) #### Garbage Collector Integration -TODO; see [#1884](https://github.com/PyO3/pyo3/issues/1884) + - `__traverse__(, visit: pyo3::class::gc::PyVisit) -> Result<(), pyo3::class::gc::PyTraverseError>` + - `__clear__() -> ()` ### `#[pyproto]` traits @@ -449,19 +445,6 @@ impl PyGCProtocol for ClassWithGCSupport { } ``` -Special protocol trait implementations have to be annotated with the `#[pyproto]` attribute. - -It is also possible to enable GC for custom classes using the `gc` parameter of the `pyclass` attribute. -i.e. `#[pyclass(gc)]`. In that case instances of custom class participate in Python garbage -collection, and it is possible to track them with `gc` module methods. When using the `gc` parameter, -it is *required* to implement the `PyGCProtocol` trait, failure to do so will result in an error -at compile time: - -```compile_fail -#[pyclass(gc)] -struct GCTracked {} // Fails because it does not implement PyGCProtocol -``` - #### Iterator Types Iterators can be defined using the diff --git a/guide/src/features.md b/guide/src/features.md index 1a29885c..19b8bc24 100644 --- a/guide/src/features.md +++ b/guide/src/features.md @@ -65,11 +65,15 @@ Most users should only need a single `#[pymethods]` per `#[pyclass]`. In additio See [the `#[pyclass]` implementation details](class.md#implementation-details) for more information. +### `pyproto` + +This feature enables the `#[pyproto]` macro, which is an alternative (older, soon-to-be-deprecated) to `#[pymethods]` for defining magic methods such as `__eq__`. + +> This feature is enabled by default. To disable it, set `default-features = false` for the `pyo3` entry in your Cargo.toml. + ### `nightly` -The `nightly` feature needs the nightly Rust compiler. This allows PyO3 to use Rust's unstable specialization feature to apply the following optimizations: -- `FromPyObject` for `Vec` and `[T;N]` can perform a `memcpy` when the object supports the Python buffer protocol. -- `ToBorrowedObject` can skip a reference count increase when the provided object is a Python native type. +The `nightly` feature needs the nightly Rust compiler. This allows PyO3 to use the auto_traits and negative_impls features to fix the `Python::allow_threads` function. ### `resolve-config` diff --git a/guide/src/migration.md b/guide/src/migration.md index 4c298bf2..46113b60 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -5,10 +5,40 @@ For a detailed list of all changes, see the [CHANGELOG](changelog.md). ## from 0.15.* to 0.16 -### Drop support for older technogies +### Drop support for older technologies PyO3 0.16 has increased minimum Rust version to 1.48 and minimum Python version to 3.7. This enables ore use of newer language features (enabling some of the other additions in 0.16) and simplifies maintenance of the project. +### Container magic methods now match Python behavior + +In PyO3 0.15, `__getitem__`, `__setitem__` and `__delitem__` in `#[pymethods]` would generate only the _mapping_ implementation for a `#[pyclass]`. To match the Python behavior, these methods now generate both the _mapping_ **and** _sequence_ implementations. + +This means that classes implementing these `#[pymethods]` will now also be treated as sequences, same as a Python `class` would be. Small differences in behavior may result: + - PyO3 will allow instances of these classes to be cast to `PySequence` as well as `PyMapping`. + - Python will provide a default implementation of `__iter__` (if the class did not have one) which repeatedly calls `__getitem__` with integers (starting at 0) until an `IndexError` is raised. + +To explain this in detail, consider the following Python class: + +```python +class ExampleContainer: + + def __len__(self): + return 5 + + def __getitem__(self, idx: int) -> int: + if idx < 0 or idx > 5: + raise IndexError() + return idx +``` + +This class implements a Python [sequence](https://docs.python.org/3/glossary.html#term-sequence). + +The `__len__` and `__getitem__` methods are also used to implement a Python [mapping](https://docs.python.org/3/glossary.html#term-mapping). In the Python C-API, these methods are not shared: the sequence `__len__` and `__getitem__` are defined by the `sq_len` and `sq_item` slots, and the mapping equivalents are `mp_len` and `mp_subscript`. There are similar distinctions for `__setitem__` and `__delitem__`. + +Because there is no such distinction from Python, implementing these methods will fill the mapping and sequence slots simultaneously. A Python class with `__len__` implemented, for example, will have both the `sq_len` and `mp_len` slots filled. + +The PyO3 behavior in 0.16 has been changed to be closer to this Python behavior by default. + ## from 0.14.* to 0.15 ### Changes in sequence indexing diff --git a/noxfile.py b/noxfile.py new file mode 100644 index 00000000..f6b85e1b --- /dev/null +++ b/noxfile.py @@ -0,0 +1,88 @@ +import time +from glob import glob + +import nox + +nox.options.sessions = ["test", "clippy", "fmt"] + + +@nox.session(venv_backend="none") +def test(session: nox.Session): + test_rust(session) + test_py(session) + + +@nox.session(name="test-rust", venv_backend="none") +def test_rust(session: nox.Session): + session.run("cargo", "test", external=True) + session.run("cargo", "test", "--features=abi3", external=True) + session.run("cargo", "test", "--features=full", external=True) + session.run("cargo", "test", "--features=abi3 full", external=True) + + +@nox.session(name="test-py", venv_backend="none") +def test_py(session): + session.run("nox", "-f", "pytests/noxfile.py", external=True) + for example in glob("examples/*/noxfile.py"): + session.run("nox", "-f", example, external=True) + + +@nox.session +def fmt(session: nox.Session): + fmt_rust(session) + fmt_py(session) + + +@nox.session(name="fmt-rust", venv_backend="none") +def fmt_rust(session: nox.Session): + session.run("cargo", "fmt", "--all", "--check", external=True) + + +@nox.session(name="fmt-py") +def fmt_py(session: nox.Session): + session.install("black==22.1.0") + session.run("black", ".", "--check") + + +@nox.session(venv_backend="none") +def clippy(session: nox.Session) -> None: + for feature_set in ["full", "abi3 full"]: + session.run( + "cargo", + "clippy", + f"--features={feature_set}", + "--all-targets", + "--workspace", + "--", + "--deny=warnings", + external=True, + ) + + +@nox.session(venv_backend="none") +def publish(session: nox.Session) -> None: + session.run( + "cargo", + "publish", + "--manifest-path", + "pyo3-build-config/Cargo.toml", + external=True, + ) + time.sleep(10) + session.run( + "cargo", + "publish", + "--manifest-path", + "pyo3-macros-backend/Cargo.toml", + external=True, + ) + time.sleep(10) + session.run( + "cargo", "publish", "--manifest-path", "pyo3-macros/Cargo.toml", external=True + ) + time.sleep(10) + session.run( + "cargo", "publish", "--manifest-path", "pyo3-ffi/Cargo.toml", external=True + ) + time.sleep(10) + session.run("cargo", "publish", external=True) diff --git a/pyo3-build-config/src/lib.rs b/pyo3-build-config/src/lib.rs index 0a4a147c..5a89ad9d 100644 --- a/pyo3-build-config/src/lib.rs +++ b/pyo3-build-config/src/lib.rs @@ -10,6 +10,7 @@ mod impl_; #[cfg(feature = "resolve-config")] use std::io::Cursor; +use std::{env, process::Command}; #[cfg(feature = "resolve-config")] use once_cell::sync::OnceCell; @@ -115,6 +116,36 @@ fn abi3_config() -> InterpreterConfig { interpreter_config } +/// Use certain features if we detect the compiler being used supports them. +/// +/// Features may be removed or added as MSRV gets bumped or new features become available, +/// so this function is unstable. +#[doc(hidden)] +pub fn print_feature_cfgs() { + fn rustc_minor_version() -> Option { + let rustc = env::var_os("RUSTC")?; + let output = Command::new(rustc).arg("--version").output().ok()?; + let version = core::str::from_utf8(&output.stdout).ok()?; + let mut pieces = version.split('.'); + if pieces.next() != Some("rustc 1") { + return None; + } + pieces.next()?.parse().ok() + } + + let rustc_minor_version = rustc_minor_version().unwrap_or(0); + + // Enable use of const generics on Rust 1.51 and greater + if rustc_minor_version >= 51 { + println!("cargo:rustc-cfg=min_const_generics"); + } + + // Enable use of std::ptr::addr_of! on Rust 1.51 and greater + if rustc_minor_version >= 51 { + println!("cargo:rustc-cfg=addr_of"); + } +} + /// Private exports used in PyO3's build.rs /// /// Please don't use these - they could change at any time. diff --git a/pyo3-ffi/Cargo.toml b/pyo3-ffi/Cargo.toml new file mode 100644 index 00000000..d9d15f9d --- /dev/null +++ b/pyo3-ffi/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "pyo3-ffi" +version = "0.15.1" +description = "Python-API bindings for the PyO3 ecosystem" +authors = ["PyO3 Project and Contributors "] +keywords = ["pyo3", "python", "cpython", "ffi"] +homepage = "https://github.com/pyo3/pyo3" +repository = "https://github.com/pyo3/pyo3" +categories = ["api-bindings", "development-tools::ffi"] +license = "Apache-2.0" +edition = "2018" + +[dependencies] +libc = "0.2.62" + +[features] + +default = [] + +# Use this feature when building an extension module. +# It tells the linker to keep the python symbols unresolved, +# so that the module can also be used with statically linked python interpreters. +extension-module = [] + +# Use the Python limited API. See https://www.python.org/dev/peps/pep-0384/ for more. +abi3 = ["pyo3-build-config/abi3"] + +# With abi3, we can manually set the minimum Python version. +abi3-py37 = ["abi3-py38", "pyo3-build-config/abi3-py37"] +abi3-py38 = ["abi3-py39", "pyo3-build-config/abi3-py38"] +abi3-py39 = ["abi3-py310", "pyo3-build-config/abi3-py39"] +abi3-py310 = ["abi3", "pyo3-build-config/abi3-py310"] + + + +[build-dependencies] +pyo3-build-config = { path = "../pyo3-build-config", version = "0.15.1", features = ["resolve-config"] } + + diff --git a/src/ffi/LICENSE b/pyo3-ffi/LICENSE similarity index 100% rename from src/ffi/LICENSE rename to pyo3-ffi/LICENSE diff --git a/pyo3-ffi/README.md b/pyo3-ffi/README.md new file mode 100644 index 00000000..d1473ee1 --- /dev/null +++ b/pyo3-ffi/README.md @@ -0,0 +1,199 @@ +# pyo3-ffi + +This crate provides [Rust](https://www.rust-lang.org/) FFI declarations for Python 3. +It supports both the stable and the unstable component of the ABI through the use of cfg flags. +Python Versions 3.7+ are supported. +It is meant for advanced users only - regular PyO3 users shouldn't +need to interact with this crate at all. + +The contents of this crate are not documented here, as it would entail +basically copying the documentation from CPython. Consult the [Python/C API Reference +Manual][capi] for up-to-date documentation. + +# Minimum supported Rust and Python versions + +PyO3 supports the following software versions: + - Python 3.7 and up (CPython and PyPy) + - Rust 1.48 and up + +# Example: Building Python Native modules + +PyO3 can be used to generate a native Python module. The easiest way to try this out for the +first time is to use [`maturin`]. `maturin` is a tool for building and publishing Rust-based +Python packages with minimal configuration. The following steps set up some files for an example +Python module, install `maturin`, and then show how to build and import the Python module. + +First, create a new folder (let's call it `string_sum`) containing the following two files: + +**`Cargo.toml`** + +```toml +[lib] +name = "string_sum" +# "cdylib" is necessary to produce a shared library for Python to import from. +# +# Downstream Rust code (including code in `bin/`, `examples/`, and `tests/`) will not be able +# to `use string_sum;` unless the "rlib" or "lib" crate type is also included, e.g.: +# crate-type = ["cdylib", "rlib"] +crate-type = ["cdylib"] + +[dependencies.pyo3-ffi] +version = "*" +features = ["extension-module"] +``` + +**`src/lib.rs`** +```rust +use std::os::raw::c_char; +use std::ptr; + +use pyo3_ffi::*; + +#[allow(non_snake_case)] +#[no_mangle] +pub unsafe extern "C" fn PyInit_string_sum() -> *mut PyObject { + let init = PyModuleDef { + m_base: PyModuleDef_HEAD_INIT, + m_name: "string_sum\0".as_ptr() as *const c_char, + m_doc: std::ptr::null(), + m_size: 0, + m_methods: std::ptr::null_mut(), + m_slots: std::ptr::null_mut(), + m_traverse: None, + m_clear: None, + m_free: None, + }; + + let mptr = PyModule_Create(Box::into_raw(Box::new(init))); + let version = env!("CARGO_PKG_VERSION"); + PyModule_AddObject( + mptr, + "__version__\0".as_ptr() as *const c_char, + PyUnicode_FromStringAndSize(version.as_ptr() as *const c_char, version.len() as isize), + ); + + let wrapped_sum_as_string = PyMethodDef { + ml_name: "sum_as_string\0".as_ptr() as *const c_char, + ml_meth: PyMethodDefPointer { + _PyCFunctionFast: sum_as_string + }, + ml_flags: METH_FASTCALL, + ml_doc: "returns the sum of two integers as a string\0".as_ptr() as *const c_char, + }; + + // PyModule_AddObject can technically fail. + // For more involved applications error checking may be necessary + PyModule_AddObject( + mptr, + "sum_as_string\0".as_ptr() as *const c_char, + PyCFunction_NewEx( + Box::into_raw(Box::new(wrapped_sum_as_string)), + std::ptr::null_mut(), + PyUnicode_InternFromString("string_sum\0".as_ptr() as *const c_char), + ), + ); + + let all = ["__all__\0", "__version__\0", "sum_as_string\0"]; + + let pyall = PyTuple_New(all.len() as isize); + for (i, obj) in all.iter().enumerate() { + PyTuple_SET_ITEM( + pyall, + i as isize, + PyUnicode_InternFromString(obj.as_ptr() as *const c_char), + ) + } + + PyModule_AddObject(mptr, "__all__\0".as_ptr() as *const c_char, pyall); + + mptr +} + +pub unsafe extern "C" fn sum_as_string( + _self: *mut PyObject, + args: *mut *mut PyObject, + nargs: Py_ssize_t, +) -> *mut PyObject { + if nargs != 2 { + return raise_type_error("sum_as_string() expected 2 positional arguments"); + } + + let arg1 = *args; + if PyLong_Check(arg1) == 0 { + return raise_type_error("sum_as_string() expected an int for positional argument 1"); + } + + let arg1 = PyLong_AsLong(arg1); + if !PyErr_Occurred().is_null() { + return ptr::null_mut() + } + + let arg2 = *args.add(1); + if PyLong_Check(arg2) == 0 { + return raise_type_error("sum_as_string() expected an int for positional argument 2"); + } + + let arg2 = PyLong_AsLong(arg2); + if !PyErr_Occurred().is_null() { + return ptr::null_mut() + } + + + + let res = (arg1 + arg2).to_string(); + PyUnicode_FromStringAndSize(res.as_ptr() as *const c_char, res.len() as isize) +} + +#[cold] +#[inline(never)] +fn raise_type_error(msg: &str) -> *mut PyObject { + unsafe { + let err_msg = + PyUnicode_FromStringAndSize(msg.as_ptr() as *const c_char, msg.len() as isize); + PyErr_SetObject(PyExc_TypeError, err_msg); + Py_DECREF(err_msg); + }; + std::ptr::null_mut() +} +``` + +With those two files in place, now `maturin` needs to be installed. This can be done using +Python's package manager `pip`. First, load up a new Python `virtualenv`, and install `maturin` +into it: +```bash +$ cd string_sum +$ python -m venv .env +$ source .env/bin/activate +$ pip install maturin +``` + +Now build and execute the module: +```bash +$ maturin develop +# lots of progress output as maturin runs the compilation... +$ python +>>> import string_sum +>>> string_sum.sum_as_string(5, 20) +'25' +``` + +As well as with `maturin`, it is possible to build using [setuptools-rust] or +[manually][manual_builds]. Both offer more flexibility than `maturin` but require further +configuration. + + +While most projects use the safe wrapper provided by PyO3, +you can take a look at the [`orjson`] library as an example on how to use `pyo3-ffi` directly. +For those well versed in C and Rust the [tutorials] from the CPython documentation +can be easily converted to rust as well. + +[tutorials]: https://docs.python.org/3/extending/ +[`orjson`]: https://github.com/ijl/orjson +[capi]: https://docs.python.org/3/c-api/index.html +[`maturin`]: https://github.com/PyO3/maturin "Build and publish crates with pyo3, rust-cpython and cffi bindings as well as rust binaries as python packages" +[`pyo3-build-config`]: https://docs.rs/pyo3-build-config +[feature flags]: https://doc.rust-lang.org/cargo/reference/features.html "Features - The Cargo Book" +[manual_builds]: https://pyo3.rs/latest/building_and_distribution.html#manual-builds "Manual builds - Building and Distribution - PyO3 user guide" +[setuptools-rust]: https://github.com/PyO3/setuptools-rust "Setuptools plugin for Rust extensions" +[PEP 384]: https://www.python.org/dev/peps/pep-0384 "PEP 384 -- Defining a Stable ABI" +[Features chapter of the guide]: https://pyo3.rs/latest/features.html#features-reference "Features Reference - PyO3 user guide" diff --git a/pyo3-ffi/build.rs b/pyo3-ffi/build.rs new file mode 100644 index 00000000..c8d65d5d --- /dev/null +++ b/pyo3-ffi/build.rs @@ -0,0 +1,122 @@ +use pyo3_build_config::{ + bail, ensure, print_feature_cfgs, + pyo3_build_script_impl::{ + cargo_env_var, env_var, errors::Result, resolve_interpreter_config, InterpreterConfig, + PythonVersion, + }, +}; + +/// Minimum Python version PyO3 supports. +const MINIMUM_SUPPORTED_VERSION: PythonVersion = PythonVersion { major: 3, minor: 7 }; + +fn ensure_python_version(interpreter_config: &InterpreterConfig) -> Result<()> { + ensure!( + interpreter_config.version >= MINIMUM_SUPPORTED_VERSION, + "the configured Python interpreter version ({}) is lower than PyO3's minimum supported version ({})", + interpreter_config.version, + MINIMUM_SUPPORTED_VERSION, + ); + + Ok(()) +} + +fn ensure_target_pointer_width(interpreter_config: &InterpreterConfig) -> Result<()> { + if let Some(pointer_width) = interpreter_config.pointer_width { + // Try to check whether the target architecture matches the python library + let rust_target = match cargo_env_var("CARGO_CFG_TARGET_POINTER_WIDTH") + .unwrap() + .as_str() + { + "64" => 64, + "32" => 32, + x => bail!("unexpected Rust target pointer width: {}", x), + }; + + ensure!( + rust_target == pointer_width, + "your Rust target architecture ({}-bit) does not match your python interpreter ({}-bit)", + rust_target, + pointer_width + ); + } + Ok(()) +} + +fn emit_link_config(interpreter_config: &InterpreterConfig) -> Result<()> { + let target_os = cargo_env_var("CARGO_CFG_TARGET_OS").unwrap(); + let is_extension_module = cargo_env_var("CARGO_FEATURE_EXTENSION_MODULE").is_some(); + if target_os == "windows" || target_os == "android" || !is_extension_module { + // windows and android - always link + // other systems - only link if not extension module + println!( + "cargo:rustc-link-lib={link_model}{alias}{lib_name}", + link_model = if interpreter_config.shared { + "" + } else { + "static=" + }, + alias = if target_os == "windows" { + "pythonXY:" + } else { + "" + }, + lib_name = interpreter_config.lib_name.as_ref().ok_or( + "attempted to link to Python shared library but config does not contain lib_name" + )?, + ); + if let Some(lib_dir) = &interpreter_config.lib_dir { + println!("cargo:rustc-link-search=native={}", lib_dir); + } + } + + Ok(()) +} + +/// Prepares the PyO3 crate for compilation. +/// +/// This loads the config from pyo3-build-config and then makes some additional checks to improve UX +/// for users. +/// +/// Emits the cargo configuration based on this config as well as a few checks of the Rust compiler +/// version to enable features which aren't supported on MSRV. +fn configure_pyo3() -> Result<()> { + let interpreter_config = resolve_interpreter_config()?; + + if env_var("PYO3_PRINT_CONFIG").map_or(false, |os_str| os_str == "1") { + print_config_and_exit(&interpreter_config); + } + + ensure_python_version(&interpreter_config)?; + ensure_target_pointer_width(&interpreter_config)?; + + if !interpreter_config.suppress_build_script_link_lines { + emit_link_config(&interpreter_config)?; + } + + interpreter_config.emit_pyo3_cfgs(); + + // Extra lines come last, to support last write wins. + for line in &interpreter_config.extra_build_script_lines { + println!("{}", line); + } + + // Emit cfgs like `addr_of` and `min_const_generics` + print_feature_cfgs(); + + Ok(()) +} + +fn print_config_and_exit(config: &InterpreterConfig) { + println!("\n-- PYO3_PRINT_CONFIG=1 is set, printing configuration and halting compile --"); + config + .to_writer(&mut std::io::stdout()) + .expect("failed to print config to stdout"); + std::process::exit(101); +} + +fn main() { + if let Err(e) = configure_pyo3() { + eprintln!("error: {}", e.report()); + std::process::exit(1) + } +} diff --git a/src/ffi/abstract_.rs b/pyo3-ffi/src/abstract_.rs similarity index 98% rename from src/ffi/abstract_.rs rename to pyo3-ffi/src/abstract_.rs index 07e02da4..172396aa 100644 --- a/src/ffi/abstract_.rs +++ b/pyo3-ffi/src/abstract_.rs @@ -1,5 +1,5 @@ -use crate::ffi::object::*; -use crate::ffi::pyport::Py_ssize_t; +use crate::object::*; +use crate::pyport::Py_ssize_t; use std::os::raw::{c_char, c_int}; use std::ptr; @@ -96,10 +96,9 @@ extern "C" { #[cfg(not(any(Py_LIMITED_API, PyPy)))] #[inline] pub unsafe fn PyIter_Check(o: *mut PyObject) -> c_int { - (match (*crate::ffi::Py_TYPE(o)).tp_iternext { + (match (*crate::Py_TYPE(o)).tp_iternext { Some(tp_iternext) => { - tp_iternext as *const std::os::raw::c_void - != crate::ffi::_PyObject_NextNotImplemented as _ + tp_iternext as *const std::os::raw::c_void != crate::_PyObject_NextNotImplemented as _ } None => false, }) as c_int diff --git a/src/ffi/bltinmodule.rs b/pyo3-ffi/src/bltinmodule.rs similarity index 84% rename from src/ffi/bltinmodule.rs rename to pyo3-ffi/src/bltinmodule.rs index f7069178..cd5be043 100644 --- a/src/ffi/bltinmodule.rs +++ b/pyo3-ffi/src/bltinmodule.rs @@ -1,4 +1,4 @@ -use crate::ffi::object::PyTypeObject; +use crate::object::PyTypeObject; #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { diff --git a/src/ffi/boolobject.rs b/pyo3-ffi/src/boolobject.rs similarity index 79% rename from src/ffi/boolobject.rs rename to pyo3-ffi/src/boolobject.rs index 92d77d01..17af974b 100644 --- a/src/ffi/boolobject.rs +++ b/pyo3-ffi/src/boolobject.rs @@ -1,5 +1,5 @@ -use crate::ffi::longobject::PyLongObject; -use crate::ffi::object::*; +use crate::longobject::PyLongObject; +use crate::object::*; use std::os::raw::{c_int, c_long}; #[cfg_attr(windows, link(name = "pythonXY"))] @@ -10,7 +10,7 @@ extern "C" { #[inline] pub unsafe fn PyBool_Check(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == &mut PyBool_Type) as c_int + (Py_TYPE(op) == addr_of_mut_shim!(PyBool_Type)) as c_int } #[cfg_attr(windows, link(name = "pythonXY"))] @@ -23,12 +23,12 @@ extern "C" { #[inline] pub unsafe fn Py_False() -> *mut PyObject { - &mut _Py_FalseStruct as *mut PyLongObject as *mut PyObject + addr_of_mut_shim!(_Py_FalseStruct) as *mut PyLongObject as *mut PyObject } #[inline] pub unsafe fn Py_True() -> *mut PyObject { - &mut _Py_TrueStruct as *mut PyLongObject as *mut PyObject + addr_of_mut_shim!(_Py_TrueStruct) as *mut PyLongObject as *mut PyObject } #[inline] diff --git a/pyo3-ffi/src/buffer.rs b/pyo3-ffi/src/buffer.rs new file mode 100644 index 00000000..bfa48c08 --- /dev/null +++ b/pyo3-ffi/src/buffer.rs @@ -0,0 +1,133 @@ +use crate::object::PyObject; +use crate::pyport::Py_ssize_t; +use std::os::raw::{c_char, c_int, c_void}; +use std::ptr; + +#[cfg(PyPy)] +const Py_MAX_NDIMS: usize = 36; + +#[repr(C)] +#[derive(Copy, Clone)] +pub struct Py_buffer { + pub buf: *mut c_void, + /// Owned reference + pub obj: *mut crate::PyObject, + pub len: Py_ssize_t, + pub itemsize: Py_ssize_t, + pub readonly: c_int, + pub ndim: c_int, + pub format: *mut c_char, + pub shape: *mut Py_ssize_t, + pub strides: *mut Py_ssize_t, + pub suboffsets: *mut Py_ssize_t, + pub internal: *mut c_void, + #[cfg(PyPy)] + pub flags: c_int, + #[cfg(PyPy)] + pub _strides: [Py_ssize_t; Py_MAX_NDIMS], + #[cfg(PyPy)] + pub _shape: [Py_ssize_t; Py_MAX_NDIMS], +} + +impl Py_buffer { + pub const fn new() -> Self { + Py_buffer { + buf: ptr::null_mut(), + obj: ptr::null_mut(), + len: 0, + itemsize: 0, + readonly: 0, + ndim: 0, + format: ptr::null_mut(), + shape: ptr::null_mut(), + strides: ptr::null_mut(), + suboffsets: ptr::null_mut(), + internal: ptr::null_mut(), + #[cfg(PyPy)] + flags: 0, + #[cfg(PyPy)] + _strides: [0; Py_MAX_NDIMS], + #[cfg(PyPy)] + _shape: [0; Py_MAX_NDIMS], + } + } +} + +/* Return 1 if the getbuffer function is available, otherwise return 0. */ +extern "C" { + #[cfg(not(PyPy))] + pub fn PyObject_CheckBuffer(obj: *mut PyObject) -> c_int; + + #[cfg_attr(PyPy, link_name = "PyPyObject_GetBuffer")] + pub fn PyObject_GetBuffer(obj: *mut PyObject, view: *mut Py_buffer, flags: c_int) -> c_int; + #[cfg_attr(PyPy, link_name = "PyPyBuffer_GetPointer")] + pub fn PyBuffer_GetPointer(view: *const Py_buffer, indices: *const Py_ssize_t) -> *mut c_void; + #[cfg_attr(PyPy, link_name = "PyPyBuffer_SizeFromFormat")] + pub fn PyBuffer_SizeFromFormat(format: *const c_char) -> Py_ssize_t; + #[cfg_attr(PyPy, link_name = "PyPyBuffer_ToContiguous")] + pub fn PyBuffer_ToContiguous( + buf: *mut c_void, + view: *const Py_buffer, + len: Py_ssize_t, + order: c_char, + ) -> c_int; + #[cfg_attr(PyPy, link_name = "PyPyBuffer_FromContiguous")] + pub fn PyBuffer_FromContiguous( + view: *const Py_buffer, + buf: *const c_void, + len: Py_ssize_t, + order: c_char, + ) -> c_int; + pub fn PyObject_CopyData(dest: *mut PyObject, src: *mut PyObject) -> c_int; + #[cfg_attr(PyPy, link_name = "PyPyBuffer_IsContiguous")] + pub fn PyBuffer_IsContiguous(view: *const Py_buffer, fort: c_char) -> c_int; + pub fn PyBuffer_FillContiguousStrides( + ndims: c_int, + shape: *mut Py_ssize_t, + strides: *mut Py_ssize_t, + itemsize: c_int, + fort: c_char, + ); + #[cfg_attr(PyPy, link_name = "PyPyBuffer_FillInfo")] + pub fn PyBuffer_FillInfo( + view: *mut Py_buffer, + o: *mut PyObject, + buf: *mut c_void, + len: Py_ssize_t, + readonly: c_int, + flags: c_int, + ) -> c_int; + #[cfg_attr(PyPy, link_name = "PyPyBuffer_Release")] + pub fn PyBuffer_Release(view: *mut Py_buffer); +} + +/// Maximum number of dimensions +pub const PyBUF_MAX_NDIM: c_int = 64; + +/* Flags for getting buffers */ +pub const PyBUF_SIMPLE: c_int = 0; +pub const PyBUF_WRITABLE: c_int = 0x0001; +/* we used to include an E, backwards compatible alias */ +pub const PyBUF_WRITEABLE: c_int = PyBUF_WRITABLE; +pub const PyBUF_FORMAT: c_int = 0x0004; +pub const PyBUF_ND: c_int = 0x0008; +pub const PyBUF_STRIDES: c_int = 0x0010 | PyBUF_ND; +pub const PyBUF_C_CONTIGUOUS: c_int = 0x0020 | PyBUF_STRIDES; +pub const PyBUF_F_CONTIGUOUS: c_int = 0x0040 | PyBUF_STRIDES; +pub const PyBUF_ANY_CONTIGUOUS: c_int = 0x0080 | PyBUF_STRIDES; +pub const PyBUF_INDIRECT: c_int = 0x0100 | PyBUF_STRIDES; + +pub const PyBUF_CONTIG: c_int = PyBUF_ND | PyBUF_WRITABLE; +pub const PyBUF_CONTIG_RO: c_int = PyBUF_ND; + +pub const PyBUF_STRIDED: c_int = PyBUF_STRIDES | PyBUF_WRITABLE; +pub const PyBUF_STRIDED_RO: c_int = PyBUF_STRIDES; + +pub const PyBUF_RECORDS: c_int = PyBUF_STRIDES | PyBUF_WRITABLE | PyBUF_FORMAT; +pub const PyBUF_RECORDS_RO: c_int = PyBUF_STRIDES | PyBUF_FORMAT; + +pub const PyBUF_FULL: c_int = PyBUF_INDIRECT | PyBUF_WRITABLE | PyBUF_FORMAT; +pub const PyBUF_FULL_RO: c_int = PyBUF_INDIRECT | PyBUF_FORMAT; + +pub const PyBUF_READ: c_int = 0x100; +pub const PyBUF_WRITE: c_int = 0x200; diff --git a/src/ffi/bytearrayobject.rs b/pyo3-ffi/src/bytearrayobject.rs similarity index 87% rename from src/ffi/bytearrayobject.rs rename to pyo3-ffi/src/bytearrayobject.rs index e7131a17..7afa7801 100644 --- a/src/ffi/bytearrayobject.rs +++ b/pyo3-ffi/src/bytearrayobject.rs @@ -1,5 +1,5 @@ -use crate::ffi::object::*; -use crate::ffi::pyport::Py_ssize_t; +use crate::object::*; +use crate::pyport::Py_ssize_t; use std::os::raw::{c_char, c_int}; #[cfg_attr(windows, link(name = "pythonXY"))] @@ -12,12 +12,12 @@ extern "C" { #[inline] pub unsafe fn PyByteArray_Check(op: *mut PyObject) -> c_int { - PyObject_TypeCheck(op, &mut PyByteArray_Type) + PyObject_TypeCheck(op, addr_of_mut_shim!(PyByteArray_Type)) } #[inline] pub unsafe fn PyByteArray_CheckExact(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == &mut PyByteArray_Type) as c_int + (Py_TYPE(op) == addr_of_mut_shim!(PyByteArray_Type)) as c_int } extern "C" { diff --git a/src/ffi/bytesobject.rs b/pyo3-ffi/src/bytesobject.rs similarity index 95% rename from src/ffi/bytesobject.rs rename to pyo3-ffi/src/bytesobject.rs index 1e52a47b..8cdcd7df 100644 --- a/src/ffi/bytesobject.rs +++ b/pyo3-ffi/src/bytesobject.rs @@ -1,5 +1,5 @@ -use crate::ffi::object::*; -use crate::ffi::pyport::Py_ssize_t; +use crate::object::*; +use crate::pyport::Py_ssize_t; use std::os::raw::{c_char, c_int}; #[cfg_attr(windows, link(name = "pythonXY"))] @@ -16,7 +16,7 @@ pub unsafe fn PyBytes_Check(op: *mut PyObject) -> c_int { #[inline] pub unsafe fn PyBytes_CheckExact(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == &mut PyBytes_Type) as c_int + (Py_TYPE(op) == addr_of_mut_shim!(PyBytes_Type)) as c_int } extern "C" { diff --git a/src/ffi/ceval.rs b/pyo3-ffi/src/ceval.rs similarity index 91% rename from src/ffi/ceval.rs rename to pyo3-ffi/src/ceval.rs index 4031daa7..a45b3059 100644 --- a/src/ffi/ceval.rs +++ b/pyo3-ffi/src/ceval.rs @@ -1,5 +1,5 @@ -use crate::ffi::object::PyObject; -use crate::ffi::pystate::PyThreadState; +use crate::object::PyObject; +use crate::pystate::PyThreadState; use std::os::raw::{c_char, c_int, c_void}; extern "C" { @@ -37,7 +37,7 @@ extern "C" { pub fn PyEval_GetGlobals() -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyEval_GetLocals")] pub fn PyEval_GetLocals() -> *mut PyObject; - pub fn PyEval_GetFrame() -> *mut crate::ffi::PyFrameObject; + pub fn PyEval_GetFrame() -> *mut crate::PyFrameObject; #[cfg_attr(PyPy, link_name = "PyPy_AddPendingCall")] pub fn Py_AddPendingCall( func: Option c_int>, @@ -59,8 +59,8 @@ extern "C" { pub fn PyEval_GetFuncName(arg1: *mut PyObject) -> *const c_char; pub fn PyEval_GetFuncDesc(arg1: *mut PyObject) -> *const c_char; pub fn PyEval_GetCallStats(arg1: *mut PyObject) -> *mut PyObject; - pub fn PyEval_EvalFrame(arg1: *mut crate::ffi::PyFrameObject) -> *mut PyObject; - pub fn PyEval_EvalFrameEx(f: *mut crate::ffi::PyFrameObject, exc: c_int) -> *mut PyObject; + pub fn PyEval_EvalFrame(arg1: *mut crate::PyFrameObject) -> *mut PyObject; + pub fn PyEval_EvalFrameEx(f: *mut crate::PyFrameObject, exc: c_int) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyEval_SaveThread")] pub fn PyEval_SaveThread() -> *mut PyThreadState; #[cfg_attr(PyPy, link_name = "PyPyEval_RestoreThread")] diff --git a/src/ffi/code.rs b/pyo3-ffi/src/code.rs similarity index 100% rename from src/ffi/code.rs rename to pyo3-ffi/src/code.rs diff --git a/src/ffi/codecs.rs b/pyo3-ffi/src/codecs.rs similarity index 98% rename from src/ffi/codecs.rs rename to pyo3-ffi/src/codecs.rs index 6c23e296..2fd214cb 100644 --- a/src/ffi/codecs.rs +++ b/pyo3-ffi/src/codecs.rs @@ -1,4 +1,4 @@ -use crate::ffi::object::PyObject; +use crate::object::PyObject; use std::os::raw::{c_char, c_int}; extern "C" { diff --git a/src/ffi/compile.rs b/pyo3-ffi/src/compile.rs similarity index 100% rename from src/ffi/compile.rs rename to pyo3-ffi/src/compile.rs diff --git a/src/ffi/complexobject.rs b/pyo3-ffi/src/complexobject.rs similarity index 92% rename from src/ffi/complexobject.rs rename to pyo3-ffi/src/complexobject.rs index beecb421..83b7620a 100644 --- a/src/ffi/complexobject.rs +++ b/pyo3-ffi/src/complexobject.rs @@ -1,4 +1,4 @@ -use crate::ffi::object::*; +use crate::object::*; use std::os::raw::{c_double, c_int}; #[repr(C)] @@ -40,12 +40,12 @@ extern "C" { #[inline] pub unsafe fn PyComplex_Check(op: *mut PyObject) -> c_int { - PyObject_TypeCheck(op, &mut PyComplex_Type) + PyObject_TypeCheck(op, addr_of_mut_shim!(PyComplex_Type)) } #[inline] pub unsafe fn PyComplex_CheckExact(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == &mut PyComplex_Type) as c_int + (Py_TYPE(op) == addr_of_mut_shim!(PyComplex_Type)) as c_int } extern "C" { diff --git a/src/ffi/context.rs b/pyo3-ffi/src/context.rs similarity index 84% rename from src/ffi/context.rs rename to pyo3-ffi/src/context.rs index f3872a69..a4ee09e3 100644 --- a/src/ffi/context.rs +++ b/pyo3-ffi/src/context.rs @@ -1,4 +1,4 @@ -use crate::ffi::object::{PyObject, PyTypeObject, Py_TYPE}; +use crate::object::{PyObject, PyTypeObject, Py_TYPE}; use std::os::raw::{c_char, c_int}; extern "C" { @@ -12,17 +12,17 @@ extern "C" { #[inline] pub unsafe fn PyContext_CheckExact(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == &mut PyContext_Type) as c_int + (Py_TYPE(op) == addr_of_mut_shim!(PyContext_Type)) as c_int } #[inline] pub unsafe fn PyContextVar_CheckExact(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == &mut PyContextVar_Type) as c_int + (Py_TYPE(op) == addr_of_mut_shim!(PyContextVar_Type)) as c_int } #[inline] pub unsafe fn PyContextToken_CheckExact(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == &mut PyContextToken_Type) as c_int + (Py_TYPE(op) == addr_of_mut_shim!(PyContextToken_Type)) as c_int } extern "C" { diff --git a/src/ffi/cpython/abstract_.rs b/pyo3-ffi/src/cpython/abstract_.rs similarity index 93% rename from src/ffi/cpython/abstract_.rs rename to pyo3-ffi/src/cpython/abstract_.rs index b86ed1a7..f495188c 100644 --- a/src/ffi/cpython/abstract_.rs +++ b/pyo3-ffi/src/cpython/abstract_.rs @@ -1,8 +1,11 @@ -use crate::ffi::{PyObject, Py_buffer, Py_ssize_t}; -use std::os::raw::{c_char, c_int, c_void}; +use crate::{PyObject, Py_ssize_t}; +use std::os::raw::{c_char, c_int}; + +#[cfg(not(Py_3_11))] +use crate::Py_buffer; #[cfg(all(Py_3_8, not(PyPy)))] -use crate::ffi::{ +use crate::{ pyport::PY_SSIZE_T_MAX, vectorcallfunc, PyCallable_Check, PyThreadState, PyThreadState_GET, PyTuple_Check, PyType_HasFeature, Py_TPFLAGS_HAVE_VECTORCALL, }; @@ -51,7 +54,7 @@ pub unsafe fn PyVectorcall_NARGS(n: size_t) -> Py_ssize_t { #[inline(always)] pub unsafe fn PyVectorcall_Function(callable: *mut PyObject) -> Option { assert!(!callable.is_null()); - let tp = crate::ffi::Py_TYPE(callable); + let tp = crate::Py_TYPE(callable); if PyType_HasFeature(tp, Py_TPFLAGS_HAVE_VECTORCALL) == 0 { return None; } @@ -211,6 +214,7 @@ extern "C" { #[cfg_attr(PyPy, link_name = "PyPyObject_LengthHint")] pub fn PyObject_LengthHint(o: *mut PyObject, arg1: Py_ssize_t) -> Py_ssize_t; + #[cfg(not(Py_3_11))] // moved to src/buffer.rs from 3.11 #[cfg(all(Py_3_9, not(PyPy)))] pub fn PyObject_CheckBuffer(obj: *mut PyObject) -> c_int; } @@ -218,20 +222,24 @@ extern "C" { #[cfg(not(any(Py_3_9, PyPy)))] #[inline] pub unsafe fn PyObject_CheckBuffer(o: *mut PyObject) -> c_int { - let tp_as_buffer = (*crate::ffi::Py_TYPE(o)).tp_as_buffer; + let tp_as_buffer = (*crate::Py_TYPE(o)).tp_as_buffer; (!tp_as_buffer.is_null() && (*tp_as_buffer).bf_getbuffer.is_some()) as c_int } +#[cfg(not(Py_3_11))] // moved to src/buffer.rs from 3.11 extern "C" { #[cfg_attr(PyPy, link_name = "PyPyObject_GetBuffer")] pub fn PyObject_GetBuffer(obj: *mut PyObject, view: *mut Py_buffer, flags: c_int) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyBuffer_GetPointer")] - pub fn PyBuffer_GetPointer(view: *mut Py_buffer, indices: *mut Py_ssize_t) -> *mut c_void; + pub fn PyBuffer_GetPointer( + view: *mut Py_buffer, + indices: *mut Py_ssize_t, + ) -> *mut std::os::raw::c_void; #[cfg_attr(PyPy, link_name = "PyPyBuffer_SizeFromFormat")] pub fn PyBuffer_SizeFromFormat(format: *const c_char) -> Py_ssize_t; #[cfg_attr(PyPy, link_name = "PyPyBuffer_ToContiguous")] pub fn PyBuffer_ToContiguous( - buf: *mut c_void, + buf: *mut std::os::raw::c_void, view: *mut Py_buffer, len: Py_ssize_t, order: c_char, @@ -239,7 +247,7 @@ extern "C" { #[cfg_attr(PyPy, link_name = "PyPyBuffer_FromContiguous")] pub fn PyBuffer_FromContiguous( view: *mut Py_buffer, - buf: *mut c_void, + buf: *mut std::os::raw::c_void, len: Py_ssize_t, order: c_char, ) -> c_int; @@ -257,7 +265,7 @@ extern "C" { pub fn PyBuffer_FillInfo( view: *mut Py_buffer, o: *mut PyObject, - buf: *mut c_void, + buf: *mut std::os::raw::c_void, len: Py_ssize_t, readonly: c_int, flags: c_int, diff --git a/src/ffi/cpython/bytesobject.rs b/pyo3-ffi/src/cpython/bytesobject.rs similarity index 64% rename from src/ffi/cpython/bytesobject.rs rename to pyo3-ffi/src/cpython/bytesobject.rs index 108dbb0c..9d02c32b 100644 --- a/src/ffi/cpython/bytesobject.rs +++ b/pyo3-ffi/src/cpython/bytesobject.rs @@ -1,5 +1,5 @@ -use crate::ffi::object::PyObject; -use crate::ffi::pyport::Py_ssize_t; +use crate::object::PyObject; +use crate::pyport::Py_ssize_t; use std::os::raw::c_int; extern "C" { diff --git a/src/ffi/cpython/ceval.rs b/pyo3-ffi/src/cpython/ceval.rs similarity index 55% rename from src/ffi/cpython/ceval.rs rename to pyo3-ffi/src/cpython/ceval.rs index b85c4d07..78678cd1 100644 --- a/src/ffi/cpython/ceval.rs +++ b/pyo3-ffi/src/cpython/ceval.rs @@ -1,12 +1,9 @@ -use crate::ffi::cpython::pystate::Py_tracefunc; -use crate::ffi::object::{freefunc, PyObject}; +use crate::cpython::pystate::Py_tracefunc; +use crate::object::{freefunc, PyObject}; use std::os::raw::c_int; extern "C" { - pub fn _PyEval_EvalFrameDefault( - arg1: *mut crate::ffi::PyFrameObject, - exc: c_int, - ) -> *mut PyObject; + pub fn _PyEval_EvalFrameDefault(arg1: *mut crate::PyFrameObject, exc: c_int) -> *mut PyObject; pub fn _PyEval_RequestCodeExtraIndex(func: freefunc) -> c_int; pub fn PyEval_SetProfile(trace_func: Option, arg1: *mut PyObject); pub fn PyEval_SetTrace(trace_func: Option, arg1: *mut PyObject); diff --git a/src/ffi/cpython/code.rs b/pyo3-ffi/src/cpython/code.rs similarity index 96% rename from src/ffi/cpython/code.rs rename to pyo3-ffi/src/cpython/code.rs index 7ebceb9f..760afb71 100644 --- a/src/ffi/cpython/code.rs +++ b/pyo3-ffi/src/cpython/code.rs @@ -1,5 +1,5 @@ -use crate::ffi::object::*; -use crate::ffi::pyport::Py_ssize_t; +use crate::object::*; +use crate::pyport::Py_ssize_t; use std::os::raw::{c_char, c_int, c_uchar, c_void}; // skipped _Py_CODEUNIT @@ -84,13 +84,13 @@ extern "C" { #[inline] #[cfg(not(PyPy))] pub unsafe fn PyCode_Check(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == &mut PyCode_Type) as c_int + (Py_TYPE(op) == addr_of_mut_shim!(PyCode_Type)) as c_int } #[inline] #[cfg(not(PyPy))] pub unsafe fn PyCode_GetNumFree(op: *mut PyCodeObject) -> Py_ssize_t { - crate::ffi::PyTuple_GET_SIZE((*op).co_freevars) + crate::PyTuple_GET_SIZE((*op).co_freevars) } extern "C" { diff --git a/src/ffi/cpython/compile.rs b/pyo3-ffi/src/cpython/compile.rs similarity index 95% rename from src/ffi/cpython/compile.rs rename to pyo3-ffi/src/cpython/compile.rs index 4308730e..aa459d33 100644 --- a/src/ffi/cpython/compile.rs +++ b/pyo3-ffi/src/cpython/compile.rs @@ -1,11 +1,11 @@ #[cfg(not(Py_3_10))] -use crate::ffi::object::PyObject; +use crate::object::PyObject; #[cfg(not(Py_3_10))] -use crate::ffi::pyarena::*; +use crate::pyarena::*; #[cfg(not(Py_3_10))] -use crate::ffi::pythonrun::*; +use crate::pythonrun::*; #[cfg(not(Py_3_10))] -use crate::ffi::PyCodeObject; +use crate::PyCodeObject; #[cfg(not(Py_3_10))] use std::os::raw::c_char; use std::os::raw::c_int; diff --git a/src/ffi/cpython/dictobject.rs b/pyo3-ffi/src/cpython/dictobject.rs similarity index 92% rename from src/ffi/cpython/dictobject.rs rename to pyo3-ffi/src/cpython/dictobject.rs index 3e58e7b9..480cecc7 100644 --- a/src/ffi/cpython/dictobject.rs +++ b/pyo3-ffi/src/cpython/dictobject.rs @@ -1,5 +1,5 @@ -use crate::ffi::object::*; -use crate::ffi::pyport::Py_ssize_t; +use crate::object::*; +use crate::pyport::Py_ssize_t; use std::os::raw::c_int; opaque_struct!(PyDictKeysObject); @@ -24,7 +24,7 @@ extern "C" { mp: *mut PyObject, key: *mut PyObject, item: *mut PyObject, - hash: crate::ffi::Py_hash_t, + hash: crate::Py_hash_t, ) -> c_int; // skipped _PyDict_DelItem_KnownHash // skipped _PyDict_DelItemIf @@ -34,7 +34,7 @@ extern "C" { pos: *mut Py_ssize_t, key: *mut *mut PyObject, value: *mut *mut PyObject, - hash: *mut crate::ffi::Py_hash_t, + hash: *mut crate::Py_hash_t, ) -> c_int; // skipped PyDict_GET_SIZE // skipped _PyDict_Contains_KnownHash diff --git a/src/ffi/cpython/frameobject.rs b/pyo3-ffi/src/cpython/frameobject.rs similarity index 94% rename from src/ffi/cpython/frameobject.rs rename to pyo3-ffi/src/cpython/frameobject.rs index 2f7d2b1d..b51a10cb 100644 --- a/src/ffi/cpython/frameobject.rs +++ b/pyo3-ffi/src/cpython/frameobject.rs @@ -1,6 +1,6 @@ -use crate::ffi::cpython::code::{PyCodeObject, CO_MAXBLOCKS}; -use crate::ffi::object::*; -use crate::ffi::pystate::PyThreadState; +use crate::cpython::code::{PyCodeObject, CO_MAXBLOCKS}; +use crate::object::*; +use crate::pystate::PyThreadState; use std::os::raw::{c_char, c_int}; // skipped _framestate @@ -60,7 +60,7 @@ extern "C" { #[inline] pub unsafe fn PyFrame_Check(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == &mut PyFrame_Type) as c_int + (Py_TYPE(op) == addr_of_mut_shim!(PyFrame_Type)) as c_int } extern "C" { diff --git a/src/ffi/cpython/import.rs b/pyo3-ffi/src/cpython/import.rs similarity index 97% rename from src/ffi/cpython/import.rs rename to pyo3-ffi/src/cpython/import.rs index f342c0e3..d85eb5fa 100644 --- a/src/ffi/cpython/import.rs +++ b/pyo3-ffi/src/cpython/import.rs @@ -1,4 +1,4 @@ -use crate::ffi::{PyInterpreterState, PyObject}; +use crate::{PyInterpreterState, PyObject}; use std::os::raw::{c_char, c_int, c_uchar}; // skipped PyInit__imp diff --git a/src/ffi/cpython/initconfig.rs b/pyo3-ffi/src/cpython/initconfig.rs similarity index 99% rename from src/ffi/cpython/initconfig.rs rename to pyo3-ffi/src/cpython/initconfig.rs index 95d28b02..552a3664 100644 --- a/src/ffi/cpython/initconfig.rs +++ b/pyo3-ffi/src/cpython/initconfig.rs @@ -1,6 +1,6 @@ /* --- PyStatus ----------------------------------------------- */ -use crate::ffi::Py_ssize_t; +use crate::Py_ssize_t; use libc::wchar_t; use std::os::raw::{c_char, c_int, c_ulong}; diff --git a/src/ffi/cpython/listobject.rs b/pyo3-ffi/src/cpython/listobject.rs similarity index 92% rename from src/ffi/cpython/listobject.rs rename to pyo3-ffi/src/cpython/listobject.rs index 04894078..13b82ff1 100644 --- a/src/ffi/cpython/listobject.rs +++ b/pyo3-ffi/src/cpython/listobject.rs @@ -1,5 +1,5 @@ -use crate::ffi::object::*; -use crate::ffi::pyport::Py_ssize_t; +use crate::object::*; +use crate::pyport::Py_ssize_t; #[repr(C)] #[derive(Copy, Clone)] diff --git a/src/ffi/cpython/mod.rs b/pyo3-ffi/src/cpython/mod.rs similarity index 100% rename from src/ffi/cpython/mod.rs rename to pyo3-ffi/src/cpython/mod.rs diff --git a/src/ffi/cpython/object.rs b/pyo3-ffi/src/cpython/object.rs similarity index 94% rename from src/ffi/cpython/object.rs rename to pyo3-ffi/src/cpython/object.rs index 31a7c3d5..6201b925 100644 --- a/src/ffi/cpython/object.rs +++ b/pyo3-ffi/src/cpython/object.rs @@ -1,4 +1,4 @@ -use crate::ffi::PyObject; +use crate::PyObject; use std::os::raw::c_int; // skipped _Py_NewReference @@ -11,8 +11,9 @@ use std::os::raw::c_int; // skipped _Py_static_string // skipped _Py_IDENTIFIER +#[cfg(not(Py_3_11))] // moved to src/buffer.rs from Python mod bufferinfo { - use crate::ffi::Py_ssize_t; + use crate::Py_ssize_t; use std::os::raw::{c_char, c_int, c_void}; use std::ptr; @@ -24,7 +25,7 @@ mod bufferinfo { pub struct Py_buffer { pub buf: *mut c_void, /// Owned reference - pub obj: *mut crate::ffi::PyObject, + pub obj: *mut crate::PyObject, pub len: Py_ssize_t, pub itemsize: Py_ssize_t, pub readonly: c_int, @@ -67,12 +68,12 @@ mod bufferinfo { } pub type getbufferproc = unsafe extern "C" fn( - arg1: *mut crate::ffi::PyObject, + arg1: *mut crate::PyObject, arg2: *mut Py_buffer, arg3: c_int, ) -> c_int; pub type releasebufferproc = - unsafe extern "C" fn(arg1: *mut crate::ffi::PyObject, arg2: *mut Py_buffer); + unsafe extern "C" fn(arg1: *mut crate::PyObject, arg2: *mut Py_buffer); /// Maximum number of dimensions pub const PyBUF_MAX_NDIM: c_int = 64; @@ -106,6 +107,7 @@ mod bufferinfo { pub const PyBUF_WRITE: c_int = 0x200; } +#[cfg(not(Py_3_11))] pub use self::bufferinfo::*; #[cfg(Py_3_8)] @@ -117,8 +119,8 @@ pub type vectorcallfunc = unsafe extern "C" fn( ) -> *mut PyObject; mod typeobject { - use crate::ffi::{self, object}; - use crate::ffi::{PyObject, Py_ssize_t}; + use crate::object; + use crate::{PyObject, Py_ssize_t}; use std::mem; use std::os::raw::{c_char, c_int, c_uint, c_ulong, c_void}; @@ -199,8 +201,8 @@ mod typeobject { #[repr(C)] #[derive(Clone, Default)] pub struct PyBufferProcs { - pub bf_getbuffer: Option, - pub bf_releasebuffer: Option, + pub bf_getbuffer: Option, + pub bf_releasebuffer: Option, } pub type printfunc = @@ -248,9 +250,9 @@ mod typeobject { pub tp_weaklistoffset: Py_ssize_t, pub tp_iter: Option, pub tp_iternext: Option, - pub tp_methods: *mut ffi::methodobject::PyMethodDef, - pub tp_members: *mut ffi::structmember::PyMemberDef, - pub tp_getset: *mut ffi::descrobject::PyGetSetDef, + pub tp_methods: *mut crate::methodobject::PyMethodDef, + pub tp_members: *mut crate::structmember::PyMemberDef, + pub tp_getset: *mut crate::descrobject::PyGetSetDef, pub tp_base: *mut PyTypeObject, pub tp_dict: *mut object::PyObject, pub tp_descr_get: Option, @@ -310,10 +312,10 @@ mod typeobject { #[inline] pub unsafe fn PyHeapType_GET_MEMBERS( etype: *mut PyHeapTypeObject, - ) -> *mut ffi::structmember::PyMemberDef { + ) -> *mut crate::structmember::PyMemberDef { let py_type = object::Py_TYPE(etype as *mut object::PyObject); let ptr = etype.offset((*py_type).tp_basicsize); - ptr as *mut ffi::structmember::PyMemberDef + ptr as *mut crate::structmember::PyMemberDef } } diff --git a/src/ffi/cpython/pydebug.rs b/pyo3-ffi/src/cpython/pydebug.rs similarity index 100% rename from src/ffi/cpython/pydebug.rs rename to pyo3-ffi/src/cpython/pydebug.rs diff --git a/src/ffi/cpython/pylifecycle.rs b/pyo3-ffi/src/cpython/pylifecycle.rs similarity index 95% rename from src/ffi/cpython/pylifecycle.rs rename to pyo3-ffi/src/cpython/pylifecycle.rs index b05cbe8c..3f1f16e2 100644 --- a/src/ffi/cpython/pylifecycle.rs +++ b/pyo3-ffi/src/cpython/pylifecycle.rs @@ -1,4 +1,4 @@ -use crate::ffi::{PyConfig, PyPreConfig, PyStatus, Py_ssize_t}; +use crate::{PyConfig, PyPreConfig, PyStatus, Py_ssize_t}; use libc::wchar_t; use std::os::raw::{c_char, c_int}; diff --git a/src/ffi/cpython/pymem.rs b/pyo3-ffi/src/cpython/pymem.rs similarity index 100% rename from src/ffi/cpython/pymem.rs rename to pyo3-ffi/src/cpython/pymem.rs diff --git a/src/ffi/cpython/pystate.rs b/pyo3-ffi/src/cpython/pystate.rs similarity index 93% rename from src/ffi/cpython/pystate.rs rename to pyo3-ffi/src/cpython/pystate.rs index 3f545450..67fa42d3 100644 --- a/src/ffi/cpython/pystate.rs +++ b/pyo3-ffi/src/cpython/pystate.rs @@ -1,6 +1,6 @@ #[cfg(not(PyPy))] -use crate::ffi::PyThreadState; -use crate::ffi::{PyFrameObject, PyInterpreterState, PyObject}; +use crate::PyThreadState; +use crate::{PyFrameObject, PyInterpreterState, PyObject}; use std::os::raw::c_int; // skipped _PyInterpreterState_RequiresIDRef @@ -60,10 +60,10 @@ extern "C" { #[cfg(Py_3_9)] pub type _PyFrameEvalFunction = extern "C" fn( - *mut crate::ffi::PyThreadState, - *mut crate::ffi::PyFrameObject, + *mut crate::PyThreadState, + *mut crate::PyFrameObject, c_int, -) -> *mut crate::ffi::object::PyObject; +) -> *mut crate::object::PyObject; #[cfg(Py_3_9)] extern "C" { diff --git a/src/ffi/cpython/pythonrun.rs b/pyo3-ffi/src/cpython/pythonrun.rs similarity index 98% rename from src/ffi/cpython/pythonrun.rs rename to pyo3-ffi/src/cpython/pythonrun.rs index 8c3a41f2..ba86fe33 100644 --- a/src/ffi/cpython/pythonrun.rs +++ b/pyo3-ffi/src/cpython/pythonrun.rs @@ -1,9 +1,9 @@ -use crate::ffi::object::*; +use crate::object::*; #[cfg(all(not(Py_LIMITED_API), not(Py_3_10)))] -use crate::ffi::pyarena::PyArena; -use crate::ffi::PyCompilerFlags; +use crate::pyarena::PyArena; +use crate::PyCompilerFlags; #[cfg(not(Py_3_10))] -use crate::ffi::{_mod, _node}; +use crate::{_mod, _node}; use libc::FILE; use std::os::raw::{c_char, c_int}; diff --git a/src/ffi/cpython/tupleobject.rs b/pyo3-ffi/src/cpython/tupleobject.rs similarity index 93% rename from src/ffi/cpython/tupleobject.rs rename to pyo3-ffi/src/cpython/tupleobject.rs index 16993a41..77baf95c 100644 --- a/src/ffi/cpython/tupleobject.rs +++ b/pyo3-ffi/src/cpython/tupleobject.rs @@ -1,6 +1,6 @@ -use crate::ffi::object::*; +use crate::object::*; #[cfg(not(PyPy))] -use crate::ffi::pyport::Py_ssize_t; +use crate::pyport::Py_ssize_t; #[repr(C)] pub struct PyTupleObject { diff --git a/src/ffi/cpython/unicodeobject.rs b/pyo3-ffi/src/cpython/unicodeobject.rs similarity index 73% rename from src/ffi/cpython/unicodeobject.rs rename to pyo3-ffi/src/cpython/unicodeobject.rs index 00b832fc..0fa355f9 100644 --- a/src/ffi/cpython/unicodeobject.rs +++ b/pyo3-ffi/src/cpython/unicodeobject.rs @@ -1,4 +1,4 @@ -use crate::ffi::{PyObject, Py_UCS1, Py_UCS2, Py_UCS4, Py_UNICODE, Py_hash_t, Py_ssize_t}; +use crate::{PyObject, Py_UCS1, Py_UCS2, Py_UCS4, Py_UNICODE, Py_hash_t, Py_ssize_t}; use libc::wchar_t; use std::os::raw::{c_char, c_int, c_uint, c_void}; @@ -119,7 +119,7 @@ pub const SSTATE_INTERNED_IMMORTAL: c_uint = 2; #[inline] #[cfg(target_endian = "little")] pub unsafe fn PyUnicode_IS_ASCII(op: *mut PyObject) -> c_uint { - debug_assert!(crate::ffi::PyUnicode_Check(op) != 0); + debug_assert!(crate::PyUnicode_Check(op) != 0); debug_assert!(PyUnicode_IS_READY(op) != 0); (*(op as *mut PyASCIIObject)).ascii() @@ -170,7 +170,7 @@ pub unsafe fn PyUnicode_4BYTE_DATA(op: *mut PyObject) -> *mut Py_UCS4 { #[inline] #[cfg(target_endian = "little")] pub unsafe fn PyUnicode_KIND(op: *mut PyObject) -> c_uint { - debug_assert!(crate::ffi::PyUnicode_Check(op) != 0); + debug_assert!(crate::PyUnicode_Check(op) != 0); debug_assert!(PyUnicode_IS_READY(op) != 0); (*(op as *mut PyASCIIObject)).kind() @@ -197,7 +197,7 @@ pub unsafe fn _PyUnicode_NONCOMPACT_DATA(op: *mut PyObject) -> *mut c_void { #[inline] #[cfg(target_endian = "little")] pub unsafe fn PyUnicode_DATA(op: *mut PyObject) -> *mut c_void { - debug_assert!(crate::ffi::PyUnicode_Check(op) != 0); + debug_assert!(crate::PyUnicode_Check(op) != 0); if PyUnicode_IS_COMPACT(op) != 0 { _PyUnicode_COMPACT_DATA(op) @@ -213,7 +213,7 @@ pub unsafe fn PyUnicode_DATA(op: *mut PyObject) -> *mut c_void { #[inline] #[cfg(target_endian = "little")] pub unsafe fn PyUnicode_GET_LENGTH(op: *mut PyObject) -> Py_ssize_t { - debug_assert!(crate::ffi::PyUnicode_Check(op) != 0); + debug_assert!(crate::PyUnicode_Check(op) != 0); debug_assert!(PyUnicode_IS_READY(op) != 0); (*(op as *mut PyASCIIObject)).length @@ -230,7 +230,7 @@ pub unsafe fn PyUnicode_IS_READY(op: *mut PyObject) -> c_uint { #[inline] #[cfg(target_endian = "little")] pub unsafe fn PyUnicode_READY(op: *mut PyObject) -> c_int { - debug_assert!(crate::ffi::PyUnicode_Check(op) != 0); + debug_assert!(crate::PyUnicode_Check(op) != 0); if PyUnicode_IS_READY(op) != 0 { 0 @@ -480,130 +480,3 @@ extern "C" { // skipped _PyUnicode_FromId // skipped _PyUnicode_EQ // skipped _PyUnicode_ScanIdentifier - -#[cfg(test)] -#[cfg(target_endian = "little")] -mod tests { - use super::*; - use crate::types::PyString; - use crate::{AsPyPointer, Python}; - - #[test] - fn ascii_object_bitfield() { - let ob_base: PyObject = unsafe { std::mem::zeroed() }; - - let mut o = PyASCIIObject { - ob_base, - length: 0, - hash: 0, - state: 0, - wstr: std::ptr::null_mut() as *mut wchar_t, - }; - - unsafe { - assert_eq!(o.interned(), 0); - assert_eq!(o.kind(), 0); - assert_eq!(o.compact(), 0); - assert_eq!(o.ascii(), 0); - assert_eq!(o.ready(), 0); - - for i in 0..4 { - o.state = i; - assert_eq!(o.interned(), i); - } - - for i in 0..8 { - o.state = i << 2; - assert_eq!(o.kind(), i); - } - - o.state = 1 << 5; - assert_eq!(o.compact(), 1); - - o.state = 1 << 6; - assert_eq!(o.ascii(), 1); - - o.state = 1 << 7; - assert_eq!(o.ready(), 1); - } - } - - #[test] - #[cfg_attr(Py_3_10, allow(deprecated))] - fn ascii() { - Python::with_gil(|py| { - // This test relies on implementation details of PyString. - let s = PyString::new(py, "hello, world"); - let ptr = s.as_ptr(); - - unsafe { - let ascii_ptr = ptr as *mut PyASCIIObject; - let ascii = ascii_ptr.as_ref().unwrap(); - - assert_eq!(ascii.interned(), 0); - assert_eq!(ascii.kind(), PyUnicode_1BYTE_KIND); - assert_eq!(ascii.compact(), 1); - assert_eq!(ascii.ascii(), 1); - assert_eq!(ascii.ready(), 1); - - assert_eq!(PyUnicode_IS_ASCII(ptr), 1); - assert_eq!(PyUnicode_IS_COMPACT(ptr), 1); - assert_eq!(PyUnicode_IS_COMPACT_ASCII(ptr), 1); - - assert!(!PyUnicode_1BYTE_DATA(ptr).is_null()); - // 2 and 4 byte macros return nonsense for this string instance. - assert_eq!(PyUnicode_KIND(ptr), PyUnicode_1BYTE_KIND); - - assert!(!_PyUnicode_COMPACT_DATA(ptr).is_null()); - // _PyUnicode_NONCOMPACT_DATA isn't valid for compact strings. - assert!(!PyUnicode_DATA(ptr).is_null()); - - assert_eq!(PyUnicode_GET_LENGTH(ptr), s.len().unwrap() as _); - assert_eq!(PyUnicode_IS_READY(ptr), 1); - - // This has potential to mutate object. But it should be a no-op since - // we're already ready. - assert_eq!(PyUnicode_READY(ptr), 0); - } - }) - } - - #[test] - #[cfg_attr(Py_3_10, allow(deprecated))] - fn ucs4() { - Python::with_gil(|py| { - let s = "哈哈🐈"; - let py_string = PyString::new(py, s); - let ptr = py_string.as_ptr(); - - unsafe { - let ascii_ptr = ptr as *mut PyASCIIObject; - let ascii = ascii_ptr.as_ref().unwrap(); - - assert_eq!(ascii.interned(), 0); - assert_eq!(ascii.kind(), PyUnicode_4BYTE_KIND); - assert_eq!(ascii.compact(), 1); - assert_eq!(ascii.ascii(), 0); - assert_eq!(ascii.ready(), 1); - - assert_eq!(PyUnicode_IS_ASCII(ptr), 0); - assert_eq!(PyUnicode_IS_COMPACT(ptr), 1); - assert_eq!(PyUnicode_IS_COMPACT_ASCII(ptr), 0); - - assert!(!PyUnicode_4BYTE_DATA(ptr).is_null()); - assert_eq!(PyUnicode_KIND(ptr), PyUnicode_4BYTE_KIND); - - assert!(!_PyUnicode_COMPACT_DATA(ptr).is_null()); - // _PyUnicode_NONCOMPACT_DATA isn't valid for compact strings. - assert!(!PyUnicode_DATA(ptr).is_null()); - - assert_eq!(PyUnicode_GET_LENGTH(ptr), py_string.len().unwrap() as _); - assert_eq!(PyUnicode_IS_READY(ptr), 1); - - // This has potential to mutate object. But it should be a no-op since - // we're already ready. - assert_eq!(PyUnicode_READY(ptr), 0); - } - }) - } -} diff --git a/src/ffi/datetime.rs b/pyo3-ffi/src/datetime.rs similarity index 72% rename from src/ffi/datetime.rs rename to pyo3-ffi/src/datetime.rs index c7d7b787..d6499632 100644 --- a/src/ffi/datetime.rs +++ b/pyo3-ffi/src/datetime.rs @@ -9,15 +9,13 @@ //! Support for `PyDateTime_CAPI` is limited as of PyPy 7.0.0. //! `DateTime_FromTimestamp` and `Date_FromTimestamp` are currently not supported. -use crate::ffi::{PyObject, PyTypeObject}; -use crate::ffi::{PyObject_TypeCheck, Py_TYPE}; -use crate::once_cell::GILOnceCell; -use crate::Python; -use std::ops::Deref; +use crate::{PyObject, PyObject_TypeCheck, PyTypeObject, Py_TYPE}; +use std::cell::UnsafeCell; use std::os::raw::{c_char, c_int, c_uchar}; +use std::ptr; #[cfg(not(PyPy))] use { - crate::ffi::{PyCapsule_Import, Py_hash_t}, + crate::{PyCapsule_Import, Py_hash_t}, std::ffi::CString, }; @@ -434,62 +432,38 @@ pub struct PyDateTime_CAPI { // Python already shares this object between threads, so it's no more evil for us to do it too! unsafe impl Sync for PyDateTime_CAPI {} -/// Safe wrapper around the Python datetime C-API global. Note that this object differs slightly -/// from the equivalent C object: in C, this is implemented as a `static PyDateTime_CAPI *`. Here -/// this is implemented as a wrapper which implements [`Deref`] to access a reference to a -/// [`PyDateTime_CAPI`] object. +/// Returns a pointer to a `PyDateTime_CAPI` instance /// -/// In the [`Deref`] implementation, if the underlying object has not yet been initialized, the GIL -/// will automatically be acquired and [`PyDateTime_IMPORT()`] called. -pub static PyDateTimeAPI: _PyDateTimeAPI_impl = _PyDateTimeAPI_impl { - inner: GILOnceCell::new(), -}; +/// # Note +/// This function will return a null pointer until +/// `PyDateTime_IMPORT` is called +#[inline] +pub unsafe fn PyDateTimeAPI() -> *mut PyDateTime_CAPI { + *PyDateTimeAPI_impl.0.get() +} -/// Safe wrapper around the Python C-API global `PyDateTime_TimeZone_UTC`. This follows a similar -/// strategy as [`PyDateTimeAPI`]: the Python datetime C-API will automatically be imported if this -/// type is deferenced. -/// -/// The type obtained by dereferencing this object is `&'static PyObject`. This may change in the -/// future to be a more specific type representing that this is a `datetime.timezone` object. #[cfg(not(all(PyPy, not(Py_3_8))))] -pub static PyDateTime_TimeZone_UTC: _PyDateTime_TimeZone_UTC_impl = _PyDateTime_TimeZone_UTC_impl { - inner: &PyDateTimeAPI, -}; +#[inline] +pub unsafe fn PyDateTime_TimeZone_UTC() -> *mut PyObject { + (*PyDateTimeAPI()).TimeZone_UTC +} /// Populates the `PyDateTimeAPI` object -/// -/// Unlike in C, this does *not* need to be actively invoked in Rust, which -/// will populate the `PyDateTimeAPI` struct automatically on first use. -/// Use this function only if you want to eagerly load the datetime module, -/// such as if you do not want the first call to a datetime function to be -/// slightly slower than subsequent calls. -/// -/// # Safety -/// The Python GIL must be held. -pub unsafe fn PyDateTime_IMPORT() -> &'static PyDateTime_CAPI { - PyDateTimeAPI - .inner - .get_or_init(Python::assume_gil_acquired(), || { - // Because `get_or_init` is called with `assume_gil_acquired()`, it's necessary to acquire - // the GIL here to prevent segfault in usage of the safe `PyDateTimeAPI::deref` impl. - Python::with_gil(|_py| { - // PyPy expects the C-API to be initialized via PyDateTime_Import, so trying to use - // `PyCapsule_Import` will behave unexpectedly in pypy. - #[cfg(PyPy)] - let py_datetime_c_api = PyDateTime_Import(); +pub unsafe fn PyDateTime_IMPORT() { + // PyPy expects the C-API to be initialized via PyDateTime_Import, so trying to use + // `PyCapsule_Import` will behave unexpectedly in pypy. + #[cfg(PyPy)] + let py_datetime_c_api = PyDateTime_Import(); - #[cfg(not(PyPy))] - let py_datetime_c_api = { - // PyDateTime_CAPSULE_NAME is a macro in C - let PyDateTime_CAPSULE_NAME = CString::new("datetime.datetime_CAPI").unwrap(); + #[cfg(not(PyPy))] + let py_datetime_c_api = { + // PyDateTime_CAPSULE_NAME is a macro in C + let PyDateTime_CAPSULE_NAME = CString::new("datetime.datetime_CAPI").unwrap(); - &*(PyCapsule_Import(PyDateTime_CAPSULE_NAME.as_ptr(), 1) - as *const PyDateTime_CAPI) - }; + PyCapsule_Import(PyDateTime_CAPSULE_NAME.as_ptr(), 1) as *mut PyDateTime_CAPI + }; - py_datetime_c_api - }) - }) + *PyDateTimeAPI_impl.0.get() = py_datetime_c_api; } // skipped non-limited PyDateTime_TimeZone_UTC @@ -502,61 +476,61 @@ pub unsafe fn PyDateTime_IMPORT() -> &'static PyDateTime_CAPI { #[inline] /// Check if `op` is a `PyDateTimeAPI.DateType` or subtype. pub unsafe fn PyDate_Check(op: *mut PyObject) -> c_int { - PyObject_TypeCheck(op, PyDateTimeAPI.DateType) as c_int + PyObject_TypeCheck(op, (*PyDateTimeAPI()).DateType) as c_int } #[inline] /// Check if `op`'s type is exactly `PyDateTimeAPI.DateType`. pub unsafe fn PyDate_CheckExact(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == PyDateTimeAPI.DateType) as c_int + (Py_TYPE(op) == (*PyDateTimeAPI()).DateType) as c_int } #[inline] /// Check if `op` is a `PyDateTimeAPI.DateTimeType` or subtype. pub unsafe fn PyDateTime_Check(op: *mut PyObject) -> c_int { - PyObject_TypeCheck(op, PyDateTimeAPI.DateTimeType) as c_int + PyObject_TypeCheck(op, (*PyDateTimeAPI()).DateTimeType) as c_int } #[inline] /// Check if `op`'s type is exactly `PyDateTimeAPI.DateTimeType`. pub unsafe fn PyDateTime_CheckExact(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == PyDateTimeAPI.DateTimeType) as c_int + (Py_TYPE(op) == (*PyDateTimeAPI()).DateTimeType) as c_int } #[inline] /// Check if `op` is a `PyDateTimeAPI.TimeType` or subtype. pub unsafe fn PyTime_Check(op: *mut PyObject) -> c_int { - PyObject_TypeCheck(op, PyDateTimeAPI.TimeType) as c_int + PyObject_TypeCheck(op, (*PyDateTimeAPI()).TimeType) as c_int } #[inline] /// Check if `op`'s type is exactly `PyDateTimeAPI.TimeType`. pub unsafe fn PyTime_CheckExact(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == PyDateTimeAPI.TimeType) as c_int + (Py_TYPE(op) == (*PyDateTimeAPI()).TimeType) as c_int } #[inline] /// Check if `op` is a `PyDateTimeAPI.DetaType` or subtype. pub unsafe fn PyDelta_Check(op: *mut PyObject) -> c_int { - PyObject_TypeCheck(op, PyDateTimeAPI.DeltaType) as c_int + PyObject_TypeCheck(op, (*PyDateTimeAPI()).DeltaType) as c_int } #[inline] /// Check if `op`'s type is exactly `PyDateTimeAPI.DeltaType`. pub unsafe fn PyDelta_CheckExact(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == PyDateTimeAPI.DeltaType) as c_int + (Py_TYPE(op) == (*PyDateTimeAPI()).DeltaType) as c_int } #[inline] /// Check if `op` is a `PyDateTimeAPI.TZInfoType` or subtype. pub unsafe fn PyTZInfo_Check(op: *mut PyObject) -> c_int { - PyObject_TypeCheck(op, PyDateTimeAPI.TZInfoType) as c_int + PyObject_TypeCheck(op, (*PyDateTimeAPI()).TZInfoType) as c_int } #[inline] /// Check if `op`'s type is exactly `PyDateTimeAPI.TZInfoType`. pub unsafe fn PyTZInfo_CheckExact(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == PyDateTimeAPI.TZInfoType) as c_int + (Py_TYPE(op) == (*PyDateTimeAPI()).TZInfoType) as c_int } // skipped non-limited PyDate_FromDate @@ -570,12 +544,14 @@ pub unsafe fn PyTZInfo_CheckExact(op: *mut PyObject) -> c_int { #[cfg(not(PyPy))] pub unsafe fn PyDateTime_FromTimestamp(args: *mut PyObject) -> *mut PyObject { - (PyDateTimeAPI.DateTime_FromTimestamp)(PyDateTimeAPI.DateTimeType, args, std::ptr::null_mut()) + let f = (*PyDateTimeAPI()).DateTime_FromTimestamp; + f((*PyDateTimeAPI()).DateTimeType, args, std::ptr::null_mut()) } #[cfg(not(PyPy))] pub unsafe fn PyDate_FromTimestamp(args: *mut PyObject) -> *mut PyObject { - (PyDateTimeAPI.Date_FromTimestamp)(PyDateTimeAPI.DateType, args) + let f = (*PyDateTimeAPI()).Date_FromTimestamp; + f((*PyDateTimeAPI()).DateType, args) } #[cfg(PyPy)] @@ -589,96 +565,13 @@ extern "C" { #[cfg(PyPy)] extern "C" { #[link_name = "_PyPyDateTime_Import"] - pub fn PyDateTime_Import() -> &'static PyDateTime_CAPI; + pub fn PyDateTime_Import() -> *mut PyDateTime_CAPI; } -// -- implementation details which are specific to Rust. -- +// Rust specific implementation details -#[doc(hidden)] -pub struct _PyDateTimeAPI_impl { - inner: GILOnceCell<&'static PyDateTime_CAPI>, -} +struct PyDateTimeAPISingleton(UnsafeCell<*mut PyDateTime_CAPI>); +unsafe impl Sync for PyDateTimeAPISingleton {} -impl Deref for _PyDateTimeAPI_impl { - type Target = PyDateTime_CAPI; - - #[inline] - fn deref(&self) -> &'static PyDateTime_CAPI { - unsafe { PyDateTime_IMPORT() } - } -} - -#[doc(hidden)] -#[cfg(not(all(PyPy, not(Py_3_8))))] -pub struct _PyDateTime_TimeZone_UTC_impl { - inner: &'static _PyDateTimeAPI_impl, -} - -#[cfg(not(all(PyPy, not(Py_3_8))))] -impl Deref for _PyDateTime_TimeZone_UTC_impl { - type Target = crate::PyObject; - - #[inline] - fn deref(&self) -> &crate::PyObject { - unsafe { - &*((&self.inner.TimeZone_UTC) as *const *mut crate::ffi::PyObject - as *const crate::PyObject) - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{types::PyDict, AsPyPointer, IntoPy, Py, PyAny, Python}; - - #[test] - fn test_datetime_fromtimestamp() { - Python::with_gil(|py| { - let args: Py = (100,).into_py(py); - unsafe { PyDateTime_IMPORT() }; - let dt: &PyAny = unsafe { py.from_owned_ptr(PyDateTime_FromTimestamp(args.as_ptr())) }; - let locals = PyDict::new(py); - locals.set_item("dt", dt).unwrap(); - py.run( - "import datetime; assert dt == datetime.datetime.fromtimestamp(100)", - None, - Some(locals), - ) - .unwrap(); - }) - } - - #[test] - fn test_date_fromtimestamp() { - Python::with_gil(|py| { - let args: Py = (100,).into_py(py); - unsafe { PyDateTime_IMPORT() }; - let dt: &PyAny = unsafe { py.from_owned_ptr(PyDate_FromTimestamp(args.as_ptr())) }; - let locals = PyDict::new(py); - locals.set_item("dt", dt).unwrap(); - py.run( - "import datetime; assert dt == datetime.date.fromtimestamp(100)", - None, - Some(locals), - ) - .unwrap(); - }) - } - - #[test] - #[cfg(not(all(PyPy, not(Py_3_8))))] - fn test_utc_timezone() { - Python::with_gil(|py| { - let utc_timezone = PyDateTime_TimeZone_UTC.as_ref(py); - let locals = PyDict::new(py); - locals.set_item("utc_timezone", utc_timezone).unwrap(); - py.run( - "import datetime; assert utc_timezone is datetime.timezone.utc", - None, - Some(locals), - ) - .unwrap(); - }) - } -} +static PyDateTimeAPI_impl: PyDateTimeAPISingleton = + PyDateTimeAPISingleton(UnsafeCell::new(ptr::null_mut())); diff --git a/src/ffi/descrobject.rs b/pyo3-ffi/src/descrobject.rs similarity index 95% rename from src/ffi/descrobject.rs rename to pyo3-ffi/src/descrobject.rs index 6b6b0b2d..253b40dc 100644 --- a/src/ffi/descrobject.rs +++ b/pyo3-ffi/src/descrobject.rs @@ -1,6 +1,6 @@ -use crate::ffi::methodobject::PyMethodDef; -use crate::ffi::object::{PyObject, PyTypeObject}; -use crate::ffi::structmember::PyMemberDef; +use crate::methodobject::PyMethodDef; +use crate::object::{PyObject, PyTypeObject}; +use crate::structmember::PyMemberDef; use std::os::raw::{c_char, c_int, c_void}; pub type getter = unsafe extern "C" fn(slf: *mut PyObject, closure: *mut c_void) -> *mut PyObject; diff --git a/src/ffi/dictobject.rs b/pyo3-ffi/src/dictobject.rs similarity index 92% rename from src/ffi/dictobject.rs rename to pyo3-ffi/src/dictobject.rs index a0d2b2b3..b03fbb30 100644 --- a/src/ffi/dictobject.rs +++ b/pyo3-ffi/src/dictobject.rs @@ -1,5 +1,5 @@ -use crate::ffi::object::*; -use crate::ffi::pyport::Py_ssize_t; +use crate::object::*; +use crate::pyport::Py_ssize_t; use std::os::raw::{c_char, c_int}; #[cfg_attr(windows, link(name = "pythonXY"))] @@ -15,7 +15,7 @@ pub unsafe fn PyDict_Check(op: *mut PyObject) -> c_int { #[inline] pub unsafe fn PyDict_CheckExact(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == &mut PyDict_Type) as c_int + (Py_TYPE(op) == addr_of_mut_shim!(PyDict_Type)) as c_int } extern "C" { @@ -76,17 +76,17 @@ extern "C" { #[inline] pub unsafe fn PyDictKeys_Check(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == &mut PyDictKeys_Type) as c_int + (Py_TYPE(op) == addr_of_mut_shim!(PyDictKeys_Type)) as c_int } #[inline] pub unsafe fn PyDictValues_Check(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == &mut PyDictValues_Type) as c_int + (Py_TYPE(op) == addr_of_mut_shim!(PyDictValues_Type)) as c_int } #[inline] pub unsafe fn PyDictItems_Check(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == &mut PyDictItems_Type) as c_int + (Py_TYPE(op) == addr_of_mut_shim!(PyDictItems_Type)) as c_int } #[inline] diff --git a/src/ffi/enumobject.rs b/pyo3-ffi/src/enumobject.rs similarity index 80% rename from src/ffi/enumobject.rs rename to pyo3-ffi/src/enumobject.rs index 6be7f1f3..e3f187d1 100644 --- a/src/ffi/enumobject.rs +++ b/pyo3-ffi/src/enumobject.rs @@ -1,4 +1,4 @@ -use crate::ffi::object::PyTypeObject; +use crate::object::PyTypeObject; #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { diff --git a/src/ffi/eval.rs b/pyo3-ffi/src/eval.rs similarity index 95% rename from src/ffi/eval.rs rename to pyo3-ffi/src/eval.rs index 1ce62c07..612617c0 100644 --- a/src/ffi/eval.rs +++ b/pyo3-ffi/src/eval.rs @@ -1,4 +1,4 @@ -use crate::ffi::object::PyObject; +use crate::object::PyObject; use std::os::raw::c_int; extern "C" { diff --git a/src/ffi/fileobject.rs b/pyo3-ffi/src/fileobject.rs similarity index 97% rename from src/ffi/fileobject.rs rename to pyo3-ffi/src/fileobject.rs index a72db985..525a7f10 100644 --- a/src/ffi/fileobject.rs +++ b/pyo3-ffi/src/fileobject.rs @@ -1,4 +1,4 @@ -use crate::ffi::object::PyObject; +use crate::object::PyObject; use std::os::raw::{c_char, c_int}; pub const PY_STDIOTEXTMODE: &str = "b"; diff --git a/src/ffi/fileutils.rs b/pyo3-ffi/src/fileutils.rs similarity index 87% rename from src/ffi/fileutils.rs rename to pyo3-ffi/src/fileutils.rs index eb1de025..3f053b72 100644 --- a/src/ffi/fileutils.rs +++ b/pyo3-ffi/src/fileutils.rs @@ -1,4 +1,4 @@ -use crate::ffi::pyport::Py_ssize_t; +use crate::pyport::Py_ssize_t; use libc::wchar_t; use std::os::raw::c_char; diff --git a/src/ffi/floatobject.rs b/pyo3-ffi/src/floatobject.rs similarity index 91% rename from src/ffi/floatobject.rs rename to pyo3-ffi/src/floatobject.rs index d33feb9a..15f71ba2 100644 --- a/src/ffi/floatobject.rs +++ b/pyo3-ffi/src/floatobject.rs @@ -1,4 +1,4 @@ -use crate::ffi::object::*; +use crate::object::*; use std::os::raw::{c_double, c_int}; #[cfg(Py_LIMITED_API)] @@ -20,12 +20,12 @@ extern "C" { #[inline] pub unsafe fn PyFloat_Check(op: *mut PyObject) -> c_int { - PyObject_TypeCheck(op, &mut PyFloat_Type) + PyObject_TypeCheck(op, addr_of_mut_shim!(PyFloat_Type)) } #[inline] pub unsafe fn PyFloat_CheckExact(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == &mut PyFloat_Type) as c_int + (Py_TYPE(op) == addr_of_mut_shim!(PyFloat_Type)) as c_int } // skipped Py_RETURN_NAN diff --git a/src/ffi/funcobject.rs b/pyo3-ffi/src/funcobject.rs similarity index 93% rename from src/ffi/funcobject.rs rename to pyo3-ffi/src/funcobject.rs index f5ec3d87..f3f067e5 100644 --- a/src/ffi/funcobject.rs +++ b/pyo3-ffi/src/funcobject.rs @@ -1,6 +1,6 @@ use std::os::raw::c_int; -use crate::ffi::object::{PyObject, PyTypeObject, Py_TYPE}; +use crate::object::{PyObject, PyTypeObject, Py_TYPE}; // skipped PyFunctionObject @@ -12,7 +12,7 @@ extern "C" { #[inline] pub unsafe fn PyFunction_Check(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == &mut PyFunction_Type) as c_int + (Py_TYPE(op) == addr_of_mut_shim!(PyFunction_Type)) as c_int } extern "C" { diff --git a/src/ffi/genobject.rs b/pyo3-ffi/src/genobject.rs similarity index 85% rename from src/ffi/genobject.rs rename to pyo3-ffi/src/genobject.rs index c486e202..c13edb07 100644 --- a/src/ffi/genobject.rs +++ b/pyo3-ffi/src/genobject.rs @@ -1,6 +1,6 @@ -use crate::ffi::object::*; -use crate::ffi::pyport::Py_ssize_t; -use crate::ffi::PyFrameObject; +use crate::object::*; +use crate::pyport::Py_ssize_t; +use crate::PyFrameObject; use std::os::raw::c_int; #[repr(C)] @@ -25,12 +25,12 @@ extern "C" { #[inline] pub unsafe fn PyGen_Check(op: *mut PyObject) -> c_int { - PyObject_TypeCheck(op, &mut PyGen_Type) + PyObject_TypeCheck(op, addr_of_mut_shim!(PyGen_Type)) } #[inline] pub unsafe fn PyGen_CheckExact(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == &mut PyGen_Type) as c_int + (Py_TYPE(op) == addr_of_mut_shim!(PyGen_Type)) as c_int } extern "C" { @@ -55,7 +55,7 @@ extern "C" { #[inline] pub unsafe fn PyCoro_CheckExact(op: *mut PyObject) -> c_int { - PyObject_TypeCheck(op, &mut PyCoro_Type) + PyObject_TypeCheck(op, addr_of_mut_shim!(PyCoro_Type)) } // skipped _PyCoro_GetAwaitableIter @@ -75,7 +75,7 @@ extern "C" { #[inline] pub unsafe fn PyAsyncGen_CheckExact(op: *mut PyObject) -> c_int { - PyObject_TypeCheck(op, &mut PyAsyncGen_Type) + PyObject_TypeCheck(op, addr_of_mut_shim!(PyAsyncGen_Type)) } // skipped _PyAsyncGenValueWrapperNew diff --git a/src/ffi/import.rs b/pyo3-ffi/src/import.rs similarity index 98% rename from src/ffi/import.rs rename to pyo3-ffi/src/import.rs index 0edb58cc..794e0ee5 100644 --- a/src/ffi/import.rs +++ b/pyo3-ffi/src/import.rs @@ -1,4 +1,4 @@ -use crate::ffi::object::PyObject; +use crate::object::PyObject; use std::os::raw::{c_char, c_int, c_long}; extern "C" { diff --git a/src/ffi/intrcheck.rs b/pyo3-ffi/src/intrcheck.rs similarity index 100% rename from src/ffi/intrcheck.rs rename to pyo3-ffi/src/intrcheck.rs diff --git a/src/ffi/iterobject.rs b/pyo3-ffi/src/iterobject.rs similarity index 80% rename from src/ffi/iterobject.rs rename to pyo3-ffi/src/iterobject.rs index b41fa008..657409c7 100644 --- a/src/ffi/iterobject.rs +++ b/pyo3-ffi/src/iterobject.rs @@ -1,4 +1,4 @@ -use crate::ffi::object::*; +use crate::object::*; use std::os::raw::c_int; #[cfg_attr(windows, link(name = "pythonXY"))] @@ -9,7 +9,7 @@ extern "C" { #[inline] pub unsafe fn PySeqIter_Check(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == &mut PySeqIter_Type) as c_int + (Py_TYPE(op) == addr_of_mut_shim!(PySeqIter_Type)) as c_int } extern "C" { @@ -19,7 +19,7 @@ extern "C" { #[inline] pub unsafe fn PyCallIter_Check(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == &mut PyCallIter_Type) as c_int + (Py_TYPE(op) == addr_of_mut_shim!(PyCallIter_Type)) as c_int } extern "C" { diff --git a/pyo3-ffi/src/lib.rs b/pyo3-ffi/src/lib.rs new file mode 100644 index 00000000..562cfcba --- /dev/null +++ b/pyo3-ffi/src/lib.rs @@ -0,0 +1,447 @@ +//! Raw FFI declarations for Python's C API. +//! +//! PyO3 can be used to write native Python modules or run Python code and modules from Rust. +//! +//! This crate just provides low level bindings to the Python interpreter. +//! It is meant for advanced users only - regular PyO3 users shouldn't +//! need to interact with this crate at all. +//! +//! The contents of this crate are not documented here, as it would entail +//! basically copying the documentation from CPython. Consult the [Python/C API Reference +//! Manual][capi] for up-to-date documentation. +//! +//! # Safety +//! +//! The functions in this crate lack individual safety documentation, but +//! generally the following apply: +//! - Pointer arguments have to point to a valid Python object of the correct type, +//! although null pointers are sometimes valid input. +//! - The vast majority can only be used safely while the GIL is held. +//! - Some functions have additional safety requirements, consult the +//! [Python/C API Reference Manual][capi] +//! for more information. +//! +//! +//! # Feature flags +//! +//! PyO3 uses [feature flags] to enable you to opt-in to additional functionality. For a detailed +//! description, see the [Features chapter of the guide]. +//! +//! ## Optional feature flags +//! +//! The following features customize PyO3's behavior: +//! +//! - `abi3`: Restricts PyO3's API to a subset of the full Python API which is guaranteed by +//! [PEP 384] to be forward-compatible with future Python versions. +//! - `extension-module`: This will tell the linker to keep the Python symbols unresolved, so that +//! your module can also be used with statically linked Python interpreters. Use this feature when +//! building an extension module. +//! +//! ## `rustc` environment flags +//! +//! PyO3 uses `rustc`'s `--cfg` flags to enable or disable code used for different Python versions. +//! If you want to do this for your own crate, you can do so with the [`pyo3-build-config`] crate. +//! +//! - `Py_3_7`, `Py_3_8`, `Py_3_9`, `Py_3_10`: Marks code that is only enabled when +//! compiling for a given minimum Python version. +//! - `Py_LIMITED_API`: Marks code enabled when the `abi3` feature flag is enabled. +//! - `PyPy` - Marks code enabled when compiling for PyPy. +//! +//! # Minimum supported Rust and Python versions +//! +//! PyO3 supports the following software versions: +//! - Python 3.7 and up (CPython and PyPy) +//! - Rust 1.48 and up +//! +//! # Example: Building Python Native modules +//! +//! PyO3 can be used to generate a native Python module. The easiest way to try this out for the +//! first time is to use [`maturin`]. `maturin` is a tool for building and publishing Rust-based +//! Python packages with minimal configuration. The following steps set up some files for an example +//! Python module, install `maturin`, and then show how to build and import the Python module. +//! +//! First, create a new folder (let's call it `string_sum`) containing the following two files: +//! +//! **`Cargo.toml`** +//! +//! ```toml +//! [lib] +//! name = "string_sum" +//! # "cdylib" is necessary to produce a shared library for Python to import from. +//! # +//! # Downstream Rust code (including code in `bin/`, `examples/`, and `tests/`) will not be able +//! # to `use string_sum;` unless the "rlib" or "lib" crate type is also included, e.g.: +//! # crate-type = ["cdylib", "rlib"] +//! crate-type = ["cdylib"] +//! +//! [dependencies.pyo3-ffi] +// workaround for `extended_key_value_attributes`: https://github.com/rust-lang/rust/issues/82768#issuecomment-803935643 +#![cfg_attr(docsrs, cfg_attr(docsrs, doc = concat!("version = \"", env!("CARGO_PKG_VERSION"), "\"")))] +#![cfg_attr(not(docsrs), doc = "version = \"*\"")] +//! features = ["extension-module"] +//! ``` +//! +//! **`src/lib.rs`** +//! ```rust +//! use std::os::raw::c_char; +//! use std::ptr; +//! +//! use pyo3_ffi::*; +//! +//! #[allow(non_snake_case)] +//! #[no_mangle] +//! pub unsafe extern "C" fn PyInit_string_sum() -> *mut PyObject { +//! let init = PyModuleDef { +//! m_base: PyModuleDef_HEAD_INIT, +//! m_name: "string_sum\0".as_ptr() as *const c_char, +//! m_doc: std::ptr::null(), +//! m_size: 0, +//! m_methods: std::ptr::null_mut(), +//! m_slots: std::ptr::null_mut(), +//! m_traverse: None, +//! m_clear: None, +//! m_free: None, +//! }; +//! +//! let mptr = PyModule_Create(Box::into_raw(Box::new(init))); +//! let version = env!("CARGO_PKG_VERSION"); +//! PyModule_AddObject( +//! mptr, +//! "__version__\0".as_ptr() as *const c_char, +//! PyUnicode_FromStringAndSize(version.as_ptr() as *const c_char, version.len() as isize), +//! ); +//! +//! let wrapped_sum_as_string = PyMethodDef { +//! ml_name: "sum_as_string\0".as_ptr() as *const c_char, +//! ml_meth: PyMethodDefPointer { +//! _PyCFunctionFast: sum_as_string +//! }, +//! ml_flags: METH_FASTCALL, +//! ml_doc: "returns the sum of two integers as a string\0".as_ptr() as *const c_char, +//! }; +//! +//! // PyModule_AddObject can technically fail. +//! // For more involved applications error checking may be necessary +//! PyModule_AddObject( +//! mptr, +//! "sum_as_string\0".as_ptr() as *const c_char, +//! PyCFunction_NewEx( +//! Box::into_raw(Box::new(wrapped_sum_as_string)), +//! std::ptr::null_mut(), +//! PyUnicode_InternFromString("string_sum\0".as_ptr() as *const c_char), +//! ), +//! ); +//! +//! let all = ["__all__\0", "__version__\0", "sum_as_string\0"]; +//! +//! let pyall = PyTuple_New(all.len() as isize); +//! for (i, obj) in all.iter().enumerate() { +//! PyTuple_SET_ITEM( +//! pyall, +//! i as isize, +//! PyUnicode_InternFromString(obj.as_ptr() as *const c_char), +//! ) +//! } +//! +//! PyModule_AddObject(mptr, "__all__\0".as_ptr() as *const c_char, pyall); +//! +//! mptr +//! } +//! +//! pub unsafe extern "C" fn sum_as_string( +//! _self: *mut PyObject, +//! args: *mut *mut PyObject, +//! nargs: Py_ssize_t, +//! ) -> *mut PyObject { +//! if nargs != 2 { +//! return raise_type_error("sum_as_string() expected 2 positional arguments"); +//! } +//! +//! let arg1 = *args; +//! if PyLong_Check(arg1) == 0 { +//! return raise_type_error("sum_as_string() expected an int for positional argument 1"); +//! } +//! +//! let arg1 = PyLong_AsLong(arg1); +//! if !PyErr_Occurred().is_null() { +//! return ptr::null_mut() +//! } +//! +//! let arg2 = *args.add(1); +//! if PyLong_Check(arg2) == 0 { +//! return raise_type_error("sum_as_string() expected an int for positional argument 2"); +//! } +//! +//! let arg2 = PyLong_AsLong(arg2); +//! if !PyErr_Occurred().is_null() { +//! return ptr::null_mut() +//! } +//! let res = (arg1 + arg2).to_string(); +//! PyUnicode_FromStringAndSize(res.as_ptr() as *const c_char, res.len() as isize) +//! } +//! +//! #[cold] +//! #[inline(never)] +//! fn raise_type_error(msg: &str) -> *mut PyObject { +//! unsafe { +//! let err_msg = +//! PyUnicode_FromStringAndSize(msg.as_ptr() as *const c_char, msg.len() as isize); +//! PyErr_SetObject(PyExc_TypeError, err_msg); +//! Py_DECREF(err_msg); +//! }; +//! std::ptr::null_mut() +//! } +//! ``` +//! +//! With those two files in place, now `maturin` needs to be installed. This can be done using +//! Python's package manager `pip`. First, load up a new Python `virtualenv`, and install `maturin` +//! into it: +//! ```bash +//! $ cd string_sum +//! $ python -m venv .env +//! $ source .env/bin/activate +//! $ pip install maturin +//! ``` +//! +//! Now build and execute the module: +//! ```bash +//! $ maturin develop +//! # lots of progress output as maturin runs the compilation... +//! $ python +//! >>> import string_sum +//! >>> string_sum.sum_as_string(5, 20) +//! '25' +//! ``` +//! +//! As well as with `maturin`, it is possible to build using [setuptools-rust] or +//! [manually][manual_builds]. Both offer more flexibility than `maturin` but require further +//! configuration. +//! +//! +//! # Using Python from Rust +//! +//! To embed Python into a Rust binary, you need to ensure that your Python installation contains a +//! shared library. The following steps demonstrate how to ensure this (for Ubuntu). +//! +//! To install the Python shared library on Ubuntu: +//! ```bash +//! sudo apt install python3-dev +//! ``` +//! +//! While most projects use the safe wrapper provided by pyo3, +//! you can take a look at the [`orjson`] library as an example on how to use `pyo3-ffi` directly. +//! For those well versed in C and Rust the [tutorials] from the CPython documentation +//! can be easily converted to rust as well. +//! +//! [tutorials]: https://docs.python.org/3/extending/ +//! [`orjson`]: https://github.com/ijl/orjson +//! [capi]: https://docs.python.org/3/c-api/index.html +//! [`maturin`]: https://github.com/PyO3/maturin "Build and publish crates with pyo3, rust-cpython and cffi bindings as well as rust binaries as python packages" +//! [`pyo3-build-config`]: https://docs.rs/pyo3-build-config +//! [feature flags]: https://doc.rust-lang.org/cargo/reference/features.html "Features - The Cargo Book" +//! [manual_builds]: https://pyo3.rs/latest/building_and_distribution.html#manual-builds "Manual builds - Building and Distribution - PyO3 user guide" +//! [setuptools-rust]: https://github.com/PyO3/setuptools-rust "Setuptools plugin for Rust extensions" +//! [PEP 384]: https://www.python.org/dev/peps/pep-0384 "PEP 384 -- Defining a Stable ABI" +//! [Features chapter of the guide]: https://pyo3.rs/latest/features.html#features-reference "Features Reference - PyO3 user guide" + +#![allow( + missing_docs, + non_camel_case_types, + non_snake_case, + non_upper_case_globals, + clippy::upper_case_acronyms, + clippy::missing_safety_doc +)] + +// Until `extern type` is stabilized, use the recommended approach to +// model opaque types: +// https://doc.rust-lang.org/nomicon/ffi.html#representing-opaque-structs +macro_rules! opaque_struct { + ($name:ident) => { + #[repr(C)] + pub struct $name([u8; 0]); + }; +} + +macro_rules! addr_of_mut_shim { + ($place:expr) => {{ + #[cfg(addr_of)] + { + ::std::ptr::addr_of_mut!($place) + } + #[cfg(not(addr_of))] + { + &mut $place as *mut _ + } + }}; +} + +pub use self::abstract_::*; +pub use self::bltinmodule::*; +pub use self::boolobject::*; +#[cfg(Py_3_11)] +pub use self::buffer::*; +pub use self::bytearrayobject::*; +pub use self::bytesobject::*; +pub use self::ceval::*; +pub use self::code::*; +pub use self::codecs::*; +pub use self::compile::*; +pub use self::complexobject::*; +#[cfg(all(Py_3_8, not(Py_LIMITED_API)))] +pub use self::context::*; +#[cfg(not(Py_LIMITED_API))] +pub use self::datetime::*; +pub use self::descrobject::*; +pub use self::dictobject::*; +pub use self::enumobject::*; +pub use self::eval::*; +pub use self::fileobject::*; +pub use self::fileutils::*; +pub use self::floatobject::*; +#[cfg(not(Py_LIMITED_API))] +pub use self::funcobject::*; +#[cfg(not(Py_LIMITED_API))] +pub use self::genobject::*; +pub use self::import::*; +pub use self::intrcheck::*; +pub use self::iterobject::*; +pub use self::listobject::*; +pub use self::longobject::*; +pub use self::marshal::*; +pub use self::memoryobject::*; +pub use self::methodobject::*; +pub use self::modsupport::*; +pub use self::moduleobject::*; +pub use self::object::*; +pub use self::objimpl::*; +pub use self::osmodule::*; +#[cfg(not(Py_LIMITED_API))] +pub use self::pyarena::*; +pub use self::pycapsule::*; +pub use self::pyerrors::*; +pub use self::pyframe::*; +pub use self::pyhash::*; +pub use self::pylifecycle::*; +pub use self::pymem::*; +pub use self::pyport::*; +pub use self::pystate::*; +pub use self::pystrtod::*; +pub use self::pythonrun::*; +pub use self::rangeobject::*; +pub use self::setobject::*; +pub use self::sliceobject::*; +pub use self::structseq::*; +pub use self::sysmodule::*; +pub use self::traceback::*; +pub use self::tupleobject::*; +pub use self::typeslots::*; +pub use self::unicodeobject::*; +pub use self::warnings::*; +pub use self::weakrefobject::*; + +mod abstract_; +// skipped asdl.h +// skipped ast.h +mod bltinmodule; +mod boolobject; +#[cfg(Py_3_11)] +mod buffer; +mod bytearrayobject; +mod bytesobject; +// skipped cellobject.h +mod ceval; +// skipped classobject.h +mod code; +mod codecs; +mod compile; +mod complexobject; +#[cfg(all(Py_3_8, not(Py_LIMITED_API)))] +mod context; // It's actually 3.7.1, but no cfg for patches. +#[cfg(not(Py_LIMITED_API))] +pub(crate) mod datetime; +mod descrobject; +mod dictobject; +// skipped dynamic_annotations.h +mod enumobject; +// skipped errcode.h +mod eval; +// skipped exports.h +mod fileobject; +mod fileutils; +mod floatobject; +// skipped empty frameobject.h +#[cfg(not(Py_LIMITED_API))] +pub(crate) mod funcobject; +// skipped genericaliasobject.h +#[cfg(not(Py_LIMITED_API))] +mod genobject; +mod import; +// skipped interpreteridobject.h +mod intrcheck; +mod iterobject; +mod listobject; +// skipped longintrepr.h +mod longobject; +pub(crate) mod marshal; +mod memoryobject; +mod methodobject; +mod modsupport; +mod moduleobject; +// skipped namespaceobject.h +mod object; +mod objimpl; +// skipped odictobject.h +// skipped opcode.h +// skipped osdefs.h +mod osmodule; +// skipped parser_interface.h +// skipped patchlevel.h +// skipped picklebufobject.h +// skipped pyctype.h +// skipped py_curses.h +#[cfg(not(Py_LIMITED_API))] +mod pyarena; +mod pycapsule; +// skipped pydecimal.h +// skipped pydtrace.h +mod pyerrors; +// skipped pyexpat.h +// skipped pyfpe.h +mod pyframe; +mod pyhash; +mod pylifecycle; +// skipped pymacconfig.h +// skipped pymacro.h +// skipped pymath.h +mod pymem; +mod pyport; +mod pystate; +mod pythonrun; +// skipped pystrhex.h +// skipped pystrcmp.h +mod pystrtod; +// skipped pythread.h +// skipped pytime.h +mod rangeobject; +mod setobject; +mod sliceobject; +mod structseq; +mod sysmodule; +mod traceback; +// skipped tracemalloc.h +mod tupleobject; +mod typeslots; +mod unicodeobject; +mod warnings; +mod weakrefobject; + +// Additional headers that are not exported by Python.h +pub mod structmember; + +// "Limited API" definitions matching Python's `include/cpython` directory. +#[cfg(not(Py_LIMITED_API))] +mod cpython; + +#[cfg(not(Py_LIMITED_API))] +pub use self::cpython::*; diff --git a/src/ffi/listobject.rs b/pyo3-ffi/src/listobject.rs similarity index 95% rename from src/ffi/listobject.rs rename to pyo3-ffi/src/listobject.rs index 79885bcf..9b10a917 100644 --- a/src/ffi/listobject.rs +++ b/pyo3-ffi/src/listobject.rs @@ -1,5 +1,5 @@ -use crate::ffi::object::*; -use crate::ffi::pyport::Py_ssize_t; +use crate::object::*; +use crate::pyport::Py_ssize_t; use std::os::raw::c_int; #[cfg_attr(windows, link(name = "pythonXY"))] @@ -17,7 +17,7 @@ pub unsafe fn PyList_Check(op: *mut PyObject) -> c_int { #[inline] pub unsafe fn PyList_CheckExact(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == &mut PyList_Type) as c_int + (Py_TYPE(op) == addr_of_mut_shim!(PyList_Type)) as c_int } extern "C" { diff --git a/src/ffi/longobject.rs b/pyo3-ffi/src/longobject.rs similarity index 94% rename from src/ffi/longobject.rs rename to pyo3-ffi/src/longobject.rs index 58d93d76..72e42b3d 100644 --- a/src/ffi/longobject.rs +++ b/pyo3-ffi/src/longobject.rs @@ -1,5 +1,5 @@ -use crate::ffi::object::*; -use crate::ffi::pyport::Py_ssize_t; +use crate::object::*; +use crate::pyport::Py_ssize_t; use libc::size_t; #[cfg(not(Py_LIMITED_API))] use std::os::raw::c_uchar; @@ -20,7 +20,7 @@ pub unsafe fn PyLong_Check(op: *mut PyObject) -> c_int { #[inline] pub unsafe fn PyLong_CheckExact(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == &mut PyLong_Type) as c_int + (Py_TYPE(op) == addr_of_mut_shim!(PyLong_Type)) as c_int } extern "C" { @@ -96,8 +96,8 @@ extern "C" { extern "C" { // skipped non-limited _PyLong_Sign - #[cfg(not(PyPy))] - pub fn _PyLong_NumBits(obj: *mut PyObject) -> c_int; + #[cfg_attr(PyPy, link_name = "_PyPyLong_NumBits")] + pub fn _PyLong_NumBits(obj: *mut PyObject) -> size_t; // skipped _PyLong_DivmodNear @@ -109,7 +109,7 @@ extern "C" { is_signed: c_int, ) -> *mut PyObject; - #[cfg(not(PyPy))] + #[cfg_attr(PyPy, link_name = "_PyPyLong_AsByteArrayO")] pub fn _PyLong_AsByteArray( v: *mut PyLongObject, bytes: *mut c_uchar, diff --git a/src/ffi/marshal.rs b/pyo3-ffi/src/marshal.rs similarity index 100% rename from src/ffi/marshal.rs rename to pyo3-ffi/src/marshal.rs diff --git a/src/ffi/memoryobject.rs b/pyo3-ffi/src/memoryobject.rs similarity index 90% rename from src/ffi/memoryobject.rs rename to pyo3-ffi/src/memoryobject.rs index 8ca27d79..a6569003 100644 --- a/src/ffi/memoryobject.rs +++ b/pyo3-ffi/src/memoryobject.rs @@ -1,5 +1,5 @@ -use crate::ffi::object::*; -use crate::ffi::pyport::Py_ssize_t; +use crate::object::*; +use crate::pyport::Py_ssize_t; use std::os::raw::{c_char, c_int}; // skipped non-limited _PyManagedBuffer_Type @@ -12,7 +12,7 @@ extern "C" { #[inline] pub unsafe fn PyMemoryView_Check(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == &mut PyMemoryView_Type) as c_int + (Py_TYPE(op) == addr_of_mut_shim!(PyMemoryView_Type)) as c_int } // skipped non-limited PyMemoryView_GET_BUFFER diff --git a/src/ffi/methodobject.rs b/pyo3-ffi/src/methodobject.rs similarity index 60% rename from src/ffi/methodobject.rs rename to pyo3-ffi/src/methodobject.rs index 24dac732..a3599933 100644 --- a/src/ffi/methodobject.rs +++ b/pyo3-ffi/src/methodobject.rs @@ -1,6 +1,6 @@ -use crate::ffi::object::{PyObject, PyTypeObject, Py_TYPE}; +use crate::object::{PyObject, PyTypeObject, Py_TYPE}; #[cfg(Py_3_9)] -use crate::ffi::PyObject_TypeCheck; +use crate::PyObject_TypeCheck; use std::mem; use std::os::raw::{c_char, c_int}; @@ -13,30 +13,29 @@ extern "C" { #[cfg(Py_3_9)] #[inline] pub unsafe fn PyCFunction_CheckExact(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == &mut PyCFunction_Type) as c_int + (Py_TYPE(op) == addr_of_mut_shim!(PyCFunction_Type)) as c_int } #[cfg(Py_3_9)] #[inline] pub unsafe fn PyCFunction_Check(op: *mut PyObject) -> c_int { - PyObject_TypeCheck(op, &mut PyCFunction_Type) + PyObject_TypeCheck(op, addr_of_mut_shim!(PyCFunction_Type)) } #[cfg(not(Py_3_9))] #[inline] pub unsafe fn PyCFunction_Check(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == &mut PyCFunction_Type) as c_int + (Py_TYPE(op) == addr_of_mut_shim!(PyCFunction_Type)) as c_int } pub type PyCFunction = unsafe extern "C" fn(slf: *mut PyObject, args: *mut PyObject) -> *mut PyObject; -#[cfg(not(Py_LIMITED_API))] +#[cfg(any(Py_3_10, not(Py_LIMITED_API)))] pub type _PyCFunctionFast = unsafe extern "C" fn( slf: *mut PyObject, args: *mut *mut PyObject, - nargs: crate::ffi::pyport::Py_ssize_t, - kwnames: *mut PyObject, + nargs: crate::pyport::Py_ssize_t, ) -> *mut PyObject; pub type PyCFunctionWithKeywords = unsafe extern "C" fn( @@ -49,11 +48,18 @@ pub type PyCFunctionWithKeywords = unsafe extern "C" fn( pub type _PyCFunctionFastWithKeywords = unsafe extern "C" fn( slf: *mut PyObject, args: *const *mut PyObject, - nargs: crate::ffi::pyport::Py_ssize_t, + nargs: crate::pyport::Py_ssize_t, kwnames: *mut PyObject, ) -> *mut PyObject; -// skipped PyCMethod (since 3.9) +#[cfg(all(Py_3_9, not(Py_LIMITED_API)))] +pub type PyCMethod = unsafe extern "C" fn( + slf: *mut PyObject, + defining_class: *mut PyTypeObject, + args: *const *mut PyObject, + nargs: crate::pyport::Py_ssize_t, + kwnames: *mut PyObject, +) -> *mut PyObject; extern "C" { #[cfg_attr(PyPy, link_name = "PyPyCFunction_GetFunction")] @@ -72,17 +78,45 @@ extern "C" { #[derive(Copy, Clone)] pub struct PyMethodDef { pub ml_name: *const c_char, - pub ml_meth: Option, + pub ml_meth: PyMethodDefPointer, pub ml_flags: c_int, pub ml_doc: *const c_char, } -impl Default for PyMethodDef { - fn default() -> PyMethodDef { - unsafe { mem::zeroed() } - } +/// Function types used to implement Python callables. +/// +/// This function pointer must be accompanied by the correct [ml_flags](PyMethodDef::ml_flags), +/// otherwise the behavior is undefined. +/// +/// See the [Python C API documentation][1] for more information. +/// +/// [1]: https://docs.python.org/3/c-api/structures.html#implementing-functions-and-methods +#[repr(C)] +#[derive(Copy, Clone)] +pub union PyMethodDefPointer { + /// This variant corresponds with [`METH_VARARGS`] *or* [`METH_NOARGS`] *or* [`METH_O`]. + pub PyCFunction: PyCFunction, + + /// This variant corresponds with [`METH_VARARGS`] | [`METH_KEYWORDS`]. + pub PyCFunctionWithKeywords: PyCFunctionWithKeywords, + + /// This variant corresponds with [`METH_FASTCALL`]. + #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] + pub _PyCFunctionFast: _PyCFunctionFast, + + /// This variant corresponds with [`METH_FASTCALL`] | [`METH_KEYWORDS`]. + #[cfg(not(Py_LIMITED_API))] + pub _PyCFunctionFastWithKeywords: _PyCFunctionFastWithKeywords, + + /// This variant corresponds with [`METH_METHOD`] | [`METH_FASTCALL`] | [`METH_KEYWORDS`]. + #[cfg(all(Py_3_9, not(Py_LIMITED_API)))] + pub PyCMethod: PyCMethod, } +// TODO: This can be a const assert on Rust 1.57 +const _: () = + [()][mem::size_of::() - mem::size_of::>()]; + extern "C" { #[cfg_attr(PyPy, link_name = "PyPyCFunction_New")] pub fn PyCFunction_New(ml: *mut PyMethodDef, slf: *mut PyObject) -> *mut PyObject; @@ -123,7 +157,9 @@ be specified alone or with METH_KEYWORDS. */ pub const METH_FASTCALL: c_int = 0x0080; // skipped METH_STACKLESS -// skipped METH_METHOD + +#[cfg(all(Py_3_9, not(Py_LIMITED_API)))] +pub const METH_METHOD: c_int = 0x0200; extern "C" { #[cfg(not(Py_3_9))] diff --git a/src/ffi/modsupport.rs b/pyo3-ffi/src/modsupport.rs similarity index 97% rename from src/ffi/modsupport.rs rename to pyo3-ffi/src/modsupport.rs index 30cba10e..fc0815a4 100644 --- a/src/ffi/modsupport.rs +++ b/pyo3-ffi/src/modsupport.rs @@ -1,7 +1,7 @@ -use crate::ffi::methodobject::PyMethodDef; -use crate::ffi::moduleobject::PyModuleDef; -use crate::ffi::object::PyObject; -use crate::ffi::pyport::Py_ssize_t; +use crate::methodobject::PyMethodDef; +use crate::moduleobject::PyModuleDef; +use crate::object::PyObject; +use crate::pyport::Py_ssize_t; use std::os::raw::{c_char, c_int, c_long}; extern "C" { diff --git a/src/ffi/moduleobject.rs b/pyo3-ffi/src/moduleobject.rs similarity index 92% rename from src/ffi/moduleobject.rs rename to pyo3-ffi/src/moduleobject.rs index 96d3e64b..63d9f44c 100644 --- a/src/ffi/moduleobject.rs +++ b/pyo3-ffi/src/moduleobject.rs @@ -1,6 +1,6 @@ -use crate::ffi::methodobject::PyMethodDef; -use crate::ffi::object::*; -use crate::ffi::pyport::Py_ssize_t; +use crate::methodobject::PyMethodDef; +use crate::object::*; +use crate::pyport::Py_ssize_t; use std::os::raw::{c_char, c_int, c_void}; #[cfg_attr(windows, link(name = "pythonXY"))] @@ -11,12 +11,12 @@ extern "C" { #[inline] pub unsafe fn PyModule_Check(op: *mut PyObject) -> c_int { - PyObject_TypeCheck(op, &mut PyModule_Type) + PyObject_TypeCheck(op, addr_of_mut_shim!(PyModule_Type)) } #[inline] pub unsafe fn PyModule_CheckExact(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == &mut PyModule_Type) as c_int + (Py_TYPE(op) == addr_of_mut_shim!(PyModule_Type)) as c_int } extern "C" { diff --git a/src/ffi/object.rs b/pyo3-ffi/src/object.rs similarity index 96% rename from src/ffi/object.rs rename to pyo3-ffi/src/object.rs index e113d1ee..a0333046 100644 --- a/src/ffi/object.rs +++ b/pyo3-ffi/src/object.rs @@ -1,6 +1,6 @@ // FFI note: this file changed a lot between 3.6 and 3.10. // Some missing definitions may not be marked "skipped". -use crate::ffi::pyport::{Py_hash_t, Py_ssize_t}; +use crate::pyport::{Py_hash_t, Py_ssize_t}; use std::mem; use std::os::raw::{c_char, c_int, c_uint, c_ulong, c_void}; use std::ptr; @@ -9,7 +9,7 @@ use std::ptr; opaque_struct!(PyTypeObject); #[cfg(not(Py_LIMITED_API))] -pub use crate::ffi::cpython::object::PyTypeObject; +pub use crate::cpython::object::PyTypeObject; // _PyObject_HEAD_EXTRA: conditionally defined in PyObject_HEAD_INIT // _PyObject_EXTRA_INIT: conditionally defined in PyObject_HEAD_INIT @@ -162,6 +162,11 @@ pub type newfunc = unsafe extern "C" fn( ) -> *mut PyObject; pub type allocfunc = unsafe extern "C" fn(arg1: *mut PyTypeObject, arg2: Py_ssize_t) -> *mut PyObject; +#[cfg(Py_3_11)] +pub type getbufferproc = + unsafe extern "C" fn(arg1: *mut PyObject, arg2: *mut crate::Py_buffer, arg3: c_int) -> c_int; +#[cfg(Py_3_11)] +pub type releasebufferproc = unsafe extern "C" fn(arg1: *mut PyObject, arg2: *mut crate::Py_buffer); #[repr(C)] #[derive(Copy, Clone)] @@ -407,7 +412,7 @@ pub unsafe fn Py_DECREF(op: *mut PyObject) { } #[inline] -pub unsafe fn Py_CLEAR(op: &mut *mut PyObject) { +pub unsafe fn Py_CLEAR(op: *mut *mut PyObject) { let tmp = *op; if !tmp.is_null() { *op = ptr::null_mut(); @@ -465,7 +470,7 @@ extern "C" { #[inline] pub unsafe fn Py_None() -> *mut PyObject { - &mut _Py_NoneStruct + addr_of_mut_shim!(_Py_NoneStruct) } #[inline] @@ -483,7 +488,7 @@ extern "C" { #[inline] pub unsafe fn Py_NotImplemented() -> *mut PyObject { - &mut _Py_NotImplementedStruct + addr_of_mut_shim!(_Py_NotImplementedStruct) } // skipped Py_RETURN_NOTIMPLEMENTED @@ -531,5 +536,5 @@ pub unsafe fn PyType_Check(op: *mut PyObject) -> c_int { #[inline] pub unsafe fn PyType_CheckExact(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == &mut PyType_Type) as c_int + (Py_TYPE(op) == addr_of_mut_shim!(PyType_Type)) as c_int } diff --git a/src/ffi/objimpl.rs b/pyo3-ffi/src/objimpl.rs similarity index 98% rename from src/ffi/objimpl.rs rename to pyo3-ffi/src/objimpl.rs index 8f805e3b..deeaceb1 100644 --- a/src/ffi/objimpl.rs +++ b/pyo3-ffi/src/objimpl.rs @@ -1,5 +1,5 @@ -use crate::ffi::object::*; -use crate::ffi::pyport::Py_ssize_t; +use crate::object::*; +use crate::pyport::Py_ssize_t; use libc::size_t; use std::os::raw::{c_int, c_void}; diff --git a/src/ffi/osmodule.rs b/pyo3-ffi/src/osmodule.rs similarity index 69% rename from src/ffi/osmodule.rs rename to pyo3-ffi/src/osmodule.rs index f15682bd..cacec443 100644 --- a/src/ffi/osmodule.rs +++ b/pyo3-ffi/src/osmodule.rs @@ -1,4 +1,4 @@ -use crate::ffi::object::PyObject; +use crate::object::PyObject; extern "C" { pub fn PyOS_FSPath(path: *mut PyObject) -> *mut PyObject; diff --git a/src/ffi/pyarena.rs b/pyo3-ffi/src/pyarena.rs similarity index 100% rename from src/ffi/pyarena.rs rename to pyo3-ffi/src/pyarena.rs diff --git a/src/ffi/pycapsule.rs b/pyo3-ffi/src/pycapsule.rs similarity index 96% rename from src/ffi/pycapsule.rs rename to pyo3-ffi/src/pycapsule.rs index f93cf098..382d7eb8 100644 --- a/src/ffi/pycapsule.rs +++ b/pyo3-ffi/src/pycapsule.rs @@ -1,4 +1,4 @@ -use crate::ffi::object::*; +use crate::object::*; use std::os::raw::{c_char, c_int, c_void}; #[cfg_attr(windows, link(name = "pythonXY"))] @@ -11,7 +11,7 @@ pub type PyCapsule_Destructor = unsafe extern "C" fn(o: *mut PyObject); #[inline] pub unsafe fn PyCapsule_CheckExact(ob: *mut PyObject) -> c_int { - (Py_TYPE(ob) == &mut PyCapsule_Type) as c_int + (Py_TYPE(ob) == addr_of_mut_shim!(PyCapsule_Type)) as c_int } extern "C" { diff --git a/src/ffi/pyerrors.rs b/pyo3-ffi/src/pyerrors.rs similarity index 99% rename from src/ffi/pyerrors.rs rename to pyo3-ffi/src/pyerrors.rs index 462d983d..3d3bfd4f 100644 --- a/src/ffi/pyerrors.rs +++ b/pyo3-ffi/src/pyerrors.rs @@ -1,5 +1,5 @@ -use crate::ffi::object::*; -use crate::ffi::pyport::Py_ssize_t; +use crate::object::*; +use crate::pyport::Py_ssize_t; use std::os::raw::{c_char, c_int}; #[repr(C)] @@ -158,7 +158,7 @@ pub unsafe fn PyUnicodeDecodeError_Create( end: Py_ssize_t, _reason: *const c_char, ) -> *mut PyObject { - crate::ffi::PyObject_CallFunction( + crate::PyObject_CallFunction( PyExc_UnicodeDecodeError, std::ffi::CStr::from_bytes_with_nul(b"sy#nns\0") .unwrap() diff --git a/src/ffi/pyframe.rs b/pyo3-ffi/src/pyframe.rs similarity index 87% rename from src/ffi/pyframe.rs rename to pyo3-ffi/src/pyframe.rs index e3c72268..c58bc1fb 100644 --- a/src/ffi/pyframe.rs +++ b/pyo3-ffi/src/pyframe.rs @@ -1,5 +1,5 @@ #[cfg(not(Py_LIMITED_API))] -use crate::ffi::PyFrameObject; +use crate::PyFrameObject; use std::os::raw::c_int; #[cfg(Py_LIMITED_API)] diff --git a/src/ffi/pyhash.rs b/pyo3-ffi/src/pyhash.rs similarity index 96% rename from src/ffi/pyhash.rs rename to pyo3-ffi/src/pyhash.rs index 703f6245..bfff7341 100644 --- a/src/ffi/pyhash.rs +++ b/pyo3-ffi/src/pyhash.rs @@ -1,5 +1,5 @@ #[cfg(not(Py_LIMITED_API))] -use crate::ffi::pyport::{Py_hash_t, Py_ssize_t}; +use crate::pyport::{Py_hash_t, Py_ssize_t}; #[cfg(not(Py_LIMITED_API))] use std::os::raw::{c_char, c_void}; diff --git a/src/ffi/pylifecycle.rs b/pyo3-ffi/src/pylifecycle.rs similarity index 97% rename from src/ffi/pylifecycle.rs rename to pyo3-ffi/src/pylifecycle.rs index 9970ceb9..7f73e3f0 100644 --- a/src/ffi/pylifecycle.rs +++ b/pyo3-ffi/src/pylifecycle.rs @@ -1,4 +1,4 @@ -use crate::ffi::pystate::PyThreadState; +use crate::pystate::PyThreadState; use libc::wchar_t; use std::os::raw::{c_char, c_int}; diff --git a/src/ffi/pymem.rs b/pyo3-ffi/src/pymem.rs similarity index 100% rename from src/ffi/pymem.rs rename to pyo3-ffi/src/pymem.rs diff --git a/src/ffi/pyport.rs b/pyo3-ffi/src/pyport.rs similarity index 100% rename from src/ffi/pyport.rs rename to pyo3-ffi/src/pyport.rs diff --git a/src/ffi/pystate.rs b/pyo3-ffi/src/pystate.rs similarity index 97% rename from src/ffi/pystate.rs rename to pyo3-ffi/src/pystate.rs index 4eb2049f..8bae6652 100644 --- a/src/ffi/pystate.rs +++ b/pyo3-ffi/src/pystate.rs @@ -1,6 +1,6 @@ #[cfg(not(PyPy))] -use crate::ffi::moduleobject::PyModuleDef; -use crate::ffi::object::PyObject; +use crate::moduleobject::PyModuleDef; +use crate::object::PyObject; use std::os::raw::c_int; #[cfg(not(PyPy))] diff --git a/src/ffi/pystrtod.rs b/pyo3-ffi/src/pystrtod.rs similarity index 97% rename from src/ffi/pystrtod.rs rename to pyo3-ffi/src/pystrtod.rs index 4937a88e..1f027686 100644 --- a/src/ffi/pystrtod.rs +++ b/pyo3-ffi/src/pystrtod.rs @@ -1,4 +1,4 @@ -use crate::ffi::object::PyObject; +use crate::object::PyObject; use std::os::raw::{c_char, c_double, c_int}; extern "C" { diff --git a/src/ffi/pythonrun.rs b/pyo3-ffi/src/pythonrun.rs similarity index 93% rename from src/ffi/pythonrun.rs rename to pyo3-ffi/src/pythonrun.rs index 66f6703d..5045abc1 100644 --- a/src/ffi/pythonrun.rs +++ b/pyo3-ffi/src/pythonrun.rs @@ -1,4 +1,4 @@ -use crate::ffi::object::*; +use crate::object::*; #[cfg(all(not(Py_LIMITED_API), not(Py_3_10)))] use libc::FILE; #[cfg(any(Py_LIMITED_API, not(Py_3_10)))] @@ -47,7 +47,7 @@ opaque_struct!(_node); #[inline] pub unsafe fn PyParser_SimpleParseString(s: *const c_char, b: c_int) -> *mut _node { #[allow(deprecated)] - crate::ffi::PyParser_SimpleParseStringFlags(s, b, 0) + crate::PyParser_SimpleParseStringFlags(s, b, 0) } #[cfg(all(not(Py_LIMITED_API), not(Py_3_10)))] @@ -55,7 +55,7 @@ pub unsafe fn PyParser_SimpleParseString(s: *const c_char, b: c_int) -> *mut _no #[inline] pub unsafe fn PyParser_SimpleParseFile(fp: *mut FILE, s: *const c_char, b: c_int) -> *mut _node { #[allow(deprecated)] - crate::ffi::PyParser_SimpleParseFileFlags(fp, s, b, 0) + crate::PyParser_SimpleParseFileFlags(fp, s, b, 0) } extern "C" { diff --git a/src/ffi/rangeobject.rs b/pyo3-ffi/src/rangeobject.rs similarity index 81% rename from src/ffi/rangeobject.rs rename to pyo3-ffi/src/rangeobject.rs index 1f2195bb..be617377 100644 --- a/src/ffi/rangeobject.rs +++ b/pyo3-ffi/src/rangeobject.rs @@ -1,4 +1,4 @@ -use crate::ffi::object::*; +use crate::object::*; use std::os::raw::c_int; #[cfg_attr(windows, link(name = "pythonXY"))] @@ -11,5 +11,5 @@ extern "C" { #[inline] pub unsafe fn PyRange_Check(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == &mut PyRange_Type) as c_int + (Py_TYPE(op) == addr_of_mut_shim!(PyRange_Type)) as c_int } diff --git a/src/ffi/setobject.rs b/pyo3-ffi/src/setobject.rs similarity index 80% rename from src/ffi/setobject.rs rename to pyo3-ffi/src/setobject.rs index 7471c5fe..c90d26f5 100644 --- a/src/ffi/setobject.rs +++ b/pyo3-ffi/src/setobject.rs @@ -1,7 +1,7 @@ -use crate::ffi::object::*; +use crate::object::*; #[cfg(not(Py_LIMITED_API))] -use crate::ffi::pyport::Py_hash_t; -use crate::ffi::pyport::Py_ssize_t; +use crate::pyport::Py_hash_t; +use crate::pyport::Py_ssize_t; use std::os::raw::c_int; pub const PySet_MINSIZE: usize = 8; @@ -82,7 +82,7 @@ extern "C" { #[inline] #[cfg(not(PyPy))] pub unsafe fn PyFrozenSet_CheckExact(ob: *mut PyObject) -> c_int { - (Py_TYPE(ob) == &mut PyFrozenSet_Type) as c_int + (Py_TYPE(ob) == addr_of_mut_shim!(PyFrozenSet_Type)) as c_int } extern "C" { @@ -94,8 +94,8 @@ extern "C" { #[inline] #[cfg(not(PyPy))] pub unsafe fn PyFrozenSet_Check(ob: *mut PyObject) -> c_int { - (Py_TYPE(ob) == &mut PyFrozenSet_Type - || PyType_IsSubtype(Py_TYPE(ob), &mut PyFrozenSet_Type) != 0) as c_int + (Py_TYPE(ob) == addr_of_mut_shim!(PyFrozenSet_Type) + || PyType_IsSubtype(Py_TYPE(ob), addr_of_mut_shim!(PyFrozenSet_Type)) != 0) as c_int } extern "C" { @@ -107,20 +107,21 @@ extern "C" { #[inline] #[cfg(not(PyPy))] pub unsafe fn PyAnySet_CheckExact(ob: *mut PyObject) -> c_int { - (Py_TYPE(ob) == &mut PySet_Type || Py_TYPE(ob) == &mut PyFrozenSet_Type) as c_int + (Py_TYPE(ob) == addr_of_mut_shim!(PySet_Type) + || Py_TYPE(ob) == addr_of_mut_shim!(PyFrozenSet_Type)) as c_int } #[inline] pub unsafe fn PyAnySet_Check(ob: *mut PyObject) -> c_int { (PyAnySet_CheckExact(ob) != 0 - || PyType_IsSubtype(Py_TYPE(ob), &mut PySet_Type) != 0 - || PyType_IsSubtype(Py_TYPE(ob), &mut PyFrozenSet_Type) != 0) as c_int + || PyType_IsSubtype(Py_TYPE(ob), addr_of_mut_shim!(PySet_Type)) != 0 + || PyType_IsSubtype(Py_TYPE(ob), addr_of_mut_shim!(PyFrozenSet_Type)) != 0) as c_int } #[inline] #[cfg(Py_3_10)] pub unsafe fn PySet_CheckExact(op: *mut PyObject) -> c_int { - crate::ffi::Py_IS_TYPE(op, &mut PySet_Type) + crate::Py_IS_TYPE(op, addr_of_mut_shim!(PySet_Type)) } extern "C" { @@ -132,5 +133,6 @@ extern "C" { #[inline] #[cfg(not(PyPy))] pub unsafe fn PySet_Check(ob: *mut PyObject) -> c_int { - (Py_TYPE(ob) == &mut PySet_Type || PyType_IsSubtype(Py_TYPE(ob), &mut PySet_Type) != 0) as c_int + (Py_TYPE(ob) == addr_of_mut_shim!(PySet_Type) + || PyType_IsSubtype(Py_TYPE(ob), addr_of_mut_shim!(PySet_Type)) != 0) as c_int } diff --git a/src/ffi/sliceobject.rs b/pyo3-ffi/src/sliceobject.rs similarity index 92% rename from src/ffi/sliceobject.rs rename to pyo3-ffi/src/sliceobject.rs index 31488bb4..5f3138ef 100644 --- a/src/ffi/sliceobject.rs +++ b/pyo3-ffi/src/sliceobject.rs @@ -1,5 +1,5 @@ -use crate::ffi::object::*; -use crate::ffi::pyport::Py_ssize_t; +use crate::object::*; +use crate::pyport::Py_ssize_t; use std::os::raw::c_int; extern "C" { @@ -9,7 +9,7 @@ extern "C" { #[inline] pub unsafe fn Py_Ellipsis() -> *mut PyObject { - &mut _Py_EllipsisObject + addr_of_mut_shim!(_Py_EllipsisObject) } #[cfg(not(Py_LIMITED_API))] @@ -30,7 +30,7 @@ extern "C" { #[inline] pub unsafe fn PySlice_Check(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == &mut PySlice_Type) as c_int + (Py_TYPE(op) == addr_of_mut_shim!(PySlice_Type)) as c_int } extern "C" { diff --git a/src/ffi/structmember.rs b/pyo3-ffi/src/structmember.rs similarity index 96% rename from src/ffi/structmember.rs rename to pyo3-ffi/src/structmember.rs index fc06e6da..4c94a206 100644 --- a/src/ffi/structmember.rs +++ b/pyo3-ffi/src/structmember.rs @@ -1,5 +1,5 @@ -use crate::ffi::object::PyObject; -use crate::ffi::pyport::Py_ssize_t; +use crate::object::PyObject; +use crate::pyport::Py_ssize_t; use std::os::raw::{c_char, c_int}; #[repr(C)] diff --git a/src/ffi/structseq.rs b/pyo3-ffi/src/structseq.rs similarity index 88% rename from src/ffi/structseq.rs rename to pyo3-ffi/src/structseq.rs index d0d45c79..8e66a95e 100644 --- a/src/ffi/structseq.rs +++ b/pyo3-ffi/src/structseq.rs @@ -1,6 +1,6 @@ -use crate::ffi::object::{PyObject, PyTypeObject}; +use crate::object::{PyObject, PyTypeObject}; #[cfg(not(PyPy))] -use crate::ffi::pyport::Py_ssize_t; +use crate::pyport::Py_ssize_t; use std::os::raw::{c_char, c_int}; #[repr(C)] @@ -40,18 +40,18 @@ extern "C" { } #[cfg(not(Py_LIMITED_API))] -pub type PyStructSequence = crate::ffi::PyTupleObject; +pub type PyStructSequence = crate::PyTupleObject; #[cfg(not(any(Py_LIMITED_API, PyPy)))] #[inline] pub unsafe fn PyStructSequence_SET_ITEM(op: *mut PyObject, i: Py_ssize_t, v: *mut PyObject) { - crate::ffi::PyTuple_SET_ITEM(op, i, v) + crate::PyTuple_SET_ITEM(op, i, v) } #[cfg(not(any(Py_LIMITED_API, PyPy)))] #[inline] pub unsafe fn PyStructSequence_GET_ITEM(op: *mut PyObject, i: Py_ssize_t) -> *mut PyObject { - crate::ffi::PyTuple_GET_ITEM(op, i) + crate::PyTuple_GET_ITEM(op, i) } extern "C" { diff --git a/src/ffi/sysmodule.rs b/pyo3-ffi/src/sysmodule.rs similarity index 97% rename from src/ffi/sysmodule.rs rename to pyo3-ffi/src/sysmodule.rs index 24987029..3c552254 100644 --- a/src/ffi/sysmodule.rs +++ b/pyo3-ffi/src/sysmodule.rs @@ -1,4 +1,4 @@ -use crate::ffi::object::PyObject; +use crate::object::PyObject; use libc::wchar_t; use std::os::raw::{c_char, c_int}; diff --git a/src/ffi/traceback.rs b/pyo3-ffi/src/traceback.rs similarity index 79% rename from src/ffi/traceback.rs rename to pyo3-ffi/src/traceback.rs index e08f9c35..b3a25844 100644 --- a/src/ffi/traceback.rs +++ b/pyo3-ffi/src/traceback.rs @@ -1,9 +1,9 @@ -use crate::ffi::object::*; +use crate::object::*; use std::os::raw::c_int; extern "C" { #[cfg_attr(PyPy, link_name = "PyPyTraceBack_Here")] - pub fn PyTraceBack_Here(arg1: *mut crate::ffi::PyFrameObject) -> c_int; + pub fn PyTraceBack_Here(arg1: *mut crate::PyFrameObject) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyTraceBack_Print")] pub fn PyTraceBack_Print(arg1: *mut PyObject, arg2: *mut PyObject) -> c_int; } @@ -21,5 +21,5 @@ extern "C" { #[inline] #[cfg(not(PyPy))] pub unsafe fn PyTraceBack_Check(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == &mut PyTraceBack_Type) as c_int + (Py_TYPE(op) == addr_of_mut_shim!(PyTraceBack_Type)) as c_int } diff --git a/src/ffi/tupleobject.rs b/pyo3-ffi/src/tupleobject.rs similarity index 92% rename from src/ffi/tupleobject.rs rename to pyo3-ffi/src/tupleobject.rs index 17198b39..de3c4c54 100644 --- a/src/ffi/tupleobject.rs +++ b/pyo3-ffi/src/tupleobject.rs @@ -1,5 +1,5 @@ -use crate::ffi::object::*; -use crate::ffi::pyport::Py_ssize_t; +use crate::object::*; +use crate::pyport::Py_ssize_t; use std::os::raw::c_int; #[cfg_attr(windows, link(name = "pythonXY"))] @@ -16,7 +16,7 @@ pub unsafe fn PyTuple_Check(op: *mut PyObject) -> c_int { #[inline] pub unsafe fn PyTuple_CheckExact(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == &mut PyTuple_Type) as c_int + (Py_TYPE(op) == addr_of_mut_shim!(PyTuple_Type)) as c_int } extern "C" { diff --git a/src/ffi/typeslots.rs b/pyo3-ffi/src/typeslots.rs similarity index 98% rename from src/ffi/typeslots.rs rename to pyo3-ffi/src/typeslots.rs index 6356e480..da7c60b3 100644 --- a/src/ffi/typeslots.rs +++ b/pyo3-ffi/src/typeslots.rs @@ -1,8 +1,6 @@ use std::os::raw::c_int; -#[cfg(not(Py_LIMITED_API))] pub const Py_bf_getbuffer: c_int = 1; -#[cfg(not(Py_LIMITED_API))] pub const Py_bf_releasebuffer: c_int = 2; pub const Py_mp_ass_subscript: c_int = 3; pub const Py_mp_length: c_int = 4; diff --git a/src/ffi/unicodeobject.rs b/pyo3-ffi/src/unicodeobject.rs similarity index 99% rename from src/ffi/unicodeobject.rs rename to pyo3-ffi/src/unicodeobject.rs index 10c32ca2..5841fa9e 100644 --- a/src/ffi/unicodeobject.rs +++ b/pyo3-ffi/src/unicodeobject.rs @@ -1,5 +1,5 @@ -use crate::ffi::object::*; -use crate::ffi::pyport::Py_ssize_t; +use crate::object::*; +use crate::pyport::Py_ssize_t; use libc::wchar_t; use std::os::raw::{c_char, c_int, c_void}; @@ -34,7 +34,7 @@ pub unsafe fn PyUnicode_Check(op: *mut PyObject) -> c_int { #[inline] #[cfg(not(PyPy))] pub unsafe fn PyUnicode_CheckExact(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == &mut PyUnicode_Type) as c_int + (Py_TYPE(op) == addr_of_mut_shim!(PyUnicode_Type)) as c_int } pub const Py_UNICODE_REPLACEMENT_CHARACTER: Py_UCS4 = 0xFFFD; diff --git a/src/ffi/warnings.rs b/pyo3-ffi/src/warnings.rs similarity index 92% rename from src/ffi/warnings.rs rename to pyo3-ffi/src/warnings.rs index 2e2731cc..488a23e7 100644 --- a/src/ffi/warnings.rs +++ b/pyo3-ffi/src/warnings.rs @@ -1,5 +1,5 @@ -use crate::ffi::object::PyObject; -use crate::ffi::pyport::Py_ssize_t; +use crate::object::PyObject; +use crate::pyport::Py_ssize_t; use std::os::raw::{c_char, c_int}; extern "C" { diff --git a/src/ffi/weakrefobject.rs b/pyo3-ffi/src/weakrefobject.rs similarity index 82% rename from src/ffi/weakrefobject.rs rename to pyo3-ffi/src/weakrefobject.rs index 19e12da6..b9506115 100644 --- a/src/ffi/weakrefobject.rs +++ b/pyo3-ffi/src/weakrefobject.rs @@ -1,4 +1,4 @@ -use crate::ffi::object::*; +use crate::object::*; use std::os::raw::c_int; opaque_struct!(PyWeakReference); @@ -24,20 +24,20 @@ extern "C" { #[inline] #[cfg(not(PyPy))] pub unsafe fn PyWeakref_CheckRef(op: *mut PyObject) -> c_int { - PyObject_TypeCheck(op, &mut _PyWeakref_RefType) + PyObject_TypeCheck(op, addr_of_mut_shim!(_PyWeakref_RefType)) } #[inline] #[cfg(not(PyPy))] pub unsafe fn PyWeakref_CheckRefExact(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == &mut _PyWeakref_RefType) as c_int + (Py_TYPE(op) == addr_of_mut_shim!(_PyWeakref_RefType)) as c_int } #[inline] #[cfg(not(PyPy))] pub unsafe fn PyWeakref_CheckProxy(op: *mut PyObject) -> c_int { - ((Py_TYPE(op) == &mut _PyWeakref_ProxyType) - || (Py_TYPE(op) == &mut _PyWeakref_CallableProxyType)) as c_int + ((Py_TYPE(op) == addr_of_mut_shim!(_PyWeakref_ProxyType)) + || (Py_TYPE(op) == addr_of_mut_shim!(_PyWeakref_CallableProxyType))) as c_int } #[inline] diff --git a/pyo3-macros-backend/Cargo.toml b/pyo3-macros-backend/Cargo.toml index 5abd99af..f46a0848 100644 --- a/pyo3-macros-backend/Cargo.toml +++ b/pyo3-macros-backend/Cargo.toml @@ -22,3 +22,6 @@ pyo3-build-config = { path = "../pyo3-build-config", version = "0.15.1", feature version = "1" default-features = false features = ["derive", "parsing", "printing", "clone-impls", "full", "extra-traits"] + +[features] +pyproto = [] diff --git a/pyo3-macros-backend/src/defs.rs b/pyo3-macros-backend/src/defs.rs index 258029d6..b3fdaeb4 100644 --- a/pyo3-macros-backend/src/defs.rs +++ b/pyo3-macros-backend/src/defs.rs @@ -64,27 +64,13 @@ impl Proto { }) } - pub(crate) fn slots_trait(&self) -> syn::Ident { - syn::Ident::new(&format!("Py{}ProtocolSlots", self.name), Span::call_site()) + pub(crate) fn items_trait(&self) -> syn::Ident { + syn::Ident::new(&format!("Py{}ProtocolItems", self.name), Span::call_site()) } - pub(crate) fn slots_trait_slots(&self) -> syn::Ident { + pub(crate) fn items_trait_items(&self) -> syn::Ident { syn::Ident::new( - &format!("{}_protocol_slots", self.name.to_ascii_lowercase()), - Span::call_site(), - ) - } - - pub(crate) fn methods_trait(&self) -> syn::Ident { - syn::Ident::new( - &format!("Py{}ProtocolMethods", self.name), - Span::call_site(), - ) - } - - pub(crate) fn methods_trait_methods(&self) -> syn::Ident { - syn::Ident::new( - &format!("{}_protocol_methods", self.name.to_ascii_lowercase()), + &format!("{}_protocol_items", self.name.to_ascii_lowercase()), Span::call_site(), ) } diff --git a/pyo3-macros-backend/src/deprecations.rs b/pyo3-macros-backend/src/deprecations.rs index 3d5fe878..ab164a02 100644 --- a/pyo3-macros-backend/src/deprecations.rs +++ b/pyo3-macros-backend/src/deprecations.rs @@ -3,12 +3,14 @@ use quote::{quote_spanned, ToTokens}; pub enum Deprecation { CallAttribute, + PyClassGcOption, } impl Deprecation { fn ident(&self, span: Span) -> syn::Ident { let string = match self { Deprecation::CallAttribute => "CALL_ATTRIBUTE", + Deprecation::PyClassGcOption => "PYCLASS_GC_OPTION", }; syn::Ident::new(string, span) } diff --git a/pyo3-macros-backend/src/lib.rs b/pyo3-macros-backend/src/lib.rs index c6419427..fd6ec4a4 100644 --- a/pyo3-macros-backend/src/lib.rs +++ b/pyo3-macros-backend/src/lib.rs @@ -9,6 +9,7 @@ mod utils; mod attributes; +#[cfg(feature = "pyproto")] mod defs; mod deprecations; mod frompyobject; @@ -16,11 +17,13 @@ mod konst; mod method; mod module; mod params; +#[cfg(feature = "pyproto")] mod proto_method; mod pyclass; mod pyfunction; mod pyimpl; mod pymethod; +#[cfg(feature = "pyproto")] mod pyproto; mod wrap; @@ -29,6 +32,7 @@ pub use module::{process_functions_in_module, pymodule_impl, PyModuleOptions}; pub use pyclass::{build_py_class, build_py_enum, PyClassArgs}; pub use pyfunction::{build_py_function, PyFunctionOptions}; pub use pyimpl::{build_py_methods, PyClassMethodsType}; +#[cfg(feature = "pyproto")] pub use pyproto::build_py_proto; pub use utils::get_doc; pub use wrap::{wrap_pyfunction_impl, wrap_pymodule_impl, WrapPyFunctionArgs}; diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index 930b005b..bf34070f 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -490,10 +490,12 @@ impl<'a> FnSpec<'a> { { use #krate as _pyo3; #deprecations - _pyo3::callback::handle_panic(|#py| { + let gil = _pyo3::GILPool::new(); + let #py = gil.python(); + _pyo3::callback::panic_result_into_callback_output(#py, ::std::panic::catch_unwind(move || -> _pyo3::PyResult<_> { #self_conversion #rust_call - }) + })) } } } @@ -508,11 +510,13 @@ impl<'a> FnSpec<'a> { { use #krate as _pyo3; #deprecations - _pyo3::callback::handle_panic(|#py| { + let gil = _pyo3::GILPool::new(); + let #py = gil.python(); + _pyo3::callback::panic_result_into_callback_output(#py, ::std::panic::catch_unwind(move || -> _pyo3::PyResult<_> { #self_conversion #arg_convert #rust_call - }) + })) } } } @@ -526,11 +530,13 @@ impl<'a> FnSpec<'a> { { use #krate as _pyo3; #deprecations - _pyo3::callback::handle_panic(|#py| { + let gil = _pyo3::GILPool::new(); + let #py = gil.python(); + _pyo3::callback::panic_result_into_callback_output(#py, ::std::panic::catch_unwind(move || -> _pyo3::PyResult<_> { #self_conversion #arg_convert #rust_call - }) + })) } } } @@ -546,13 +552,15 @@ impl<'a> FnSpec<'a> { use #krate as _pyo3; #deprecations use _pyo3::callback::IntoPyCallbackOutput; - _pyo3::callback::handle_panic(|#py| { + let gil = _pyo3::GILPool::new(); + let #py = gil.python(); + _pyo3::callback::panic_result_into_callback_output(#py, ::std::panic::catch_unwind(move || -> _pyo3::PyResult<_> { #arg_convert let result = #rust_call; let initializer: _pyo3::PyClassInitializer::<#cls> = result.convert(#py)?; let cell = initializer.create_cell_from_subtype(#py, subtype)?; ::std::result::Result::Ok(cell as *mut _pyo3::ffi::PyObject) - }) + })) } } } @@ -566,23 +574,23 @@ impl<'a> FnSpec<'a> { let doc = &self.doc; match self.convention { CallingConvention::Noargs => quote! { - _pyo3::class::methods::PyMethodDef::noargs( + _pyo3::impl_::pymethods::PyMethodDef::noargs( #python_name, - _pyo3::class::methods::PyCFunction(#wrapper), + _pyo3::impl_::pymethods::PyCFunction(#wrapper), #doc, ) }, CallingConvention::Fastcall => quote! { - _pyo3::class::methods::PyMethodDef::fastcall_cfunction_with_keywords( + _pyo3::impl_::pymethods::PyMethodDef::fastcall_cfunction_with_keywords( #python_name, - _pyo3::class::methods::PyCFunctionFastWithKeywords(#wrapper), + _pyo3::impl_::pymethods::PyCFunctionFastWithKeywords(#wrapper), #doc, ) }, CallingConvention::Varargs => quote! { - _pyo3::class::methods::PyMethodDef::cfunction_with_keywords( + _pyo3::impl_::pymethods::PyMethodDef::cfunction_with_keywords( #python_name, - _pyo3::class::methods::PyCFunctionWithKeywords(#wrapper), + _pyo3::impl_::pymethods::PyCFunctionWithKeywords(#wrapper), #doc, ) }, diff --git a/pyo3-macros-backend/src/module.rs b/pyo3-macros-backend/src/module.rs index ecfcc9f4..738b69fb 100644 --- a/pyo3-macros-backend/src/module.rs +++ b/pyo3-macros-backend/src/module.rs @@ -80,8 +80,7 @@ pub fn pymodule_impl( /// This autogenerated function is called by the python interpreter when importing /// the module. pub unsafe extern "C" fn #cb_name() -> *mut #krate::ffi::PyObject { - use #krate::{self as _pyo3, IntoPyPointer}; - _pyo3::callback::handle_panic(|_py| ::std::result::Result::Ok(#module_def_name.make_module(_py)?.into_ptr())) + unsafe { #module_def_name.module_init() } } #[doc(hidden)] diff --git a/pyo3-macros-backend/src/params.rs b/pyo3-macros-backend/src/params.rs index ef70c41d..7aad6207 100644 --- a/pyo3-macros-backend/src/params.rs +++ b/pyo3-macros-backend/src/params.rs @@ -70,7 +70,7 @@ pub fn impl_arg_params( arg_convert.push(impl_arg_param(arg, spec, i, None, &mut 0, py, &args_array)?); } return Ok(quote! { - let _args = ::std::option::Option::Some(#py.from_borrowed_ptr::<_pyo3::types::PyTuple>(_args)); + let _args = #py.from_borrowed_ptr::<_pyo3::types::PyTuple>(_args); let _kwargs: ::std::option::Option<&_pyo3::types::PyDict> = #py.from_borrowed_ptr_or_opt(_kwargs); #(#arg_convert)* }); @@ -126,6 +126,16 @@ pub fn impl_arg_params( } let (accept_args, accept_kwargs) = accept_args_kwargs(&spec.attrs); + let args_handler = if accept_args { + quote! { _pyo3::impl_::extract_argument::TupleVarargs } + } else { + quote! { _pyo3::impl_::extract_argument::NoVarargs } + }; + let kwargs_handler = if accept_kwargs { + quote! { _pyo3::impl_::extract_argument::DictVarkeywords } + } else { + quote! { _pyo3::impl_::extract_argument::NoVarkeywords } + }; let cls_name = if let Some(cls) = self_ { quote! { ::std::option::Option::Some(<#cls as _pyo3::type_object::PyTypeInfo>::NAME) } @@ -136,7 +146,7 @@ pub fn impl_arg_params( let extract_expression = if fastcall { quote! { - DESCRIPTION.extract_arguments_fastcall( + DESCRIPTION.extract_arguments_fastcall::<#args_handler, #kwargs_handler>( #py, _args, _nargs, @@ -146,7 +156,7 @@ pub fn impl_arg_params( } } else { quote! { - DESCRIPTION.extract_arguments_tuple_dict( + DESCRIPTION.extract_arguments_tuple_dict::<#args_handler, #kwargs_handler>( #py, _args, _kwargs, @@ -164,8 +174,6 @@ pub fn impl_arg_params( positional_only_parameters: #positional_only_parameters, required_positional_parameters: #required_positional_parameters, keyword_only_parameters: &[#(#keyword_only_parameters),*], - accept_varargs: #accept_args, - accept_varkeywords: #accept_kwargs, }; let mut #args_array = [::std::option::Option::None; #num_params]; @@ -201,9 +209,6 @@ fn impl_arg_param( let ty = arg.ty; let name = arg.name; let name_str = name.to_string(); - let transform_error = quote! { - |e| _pyo3::impl_::extract_argument::argument_extraction_error(#py, #name_str, e) - }; if is_args(&spec.attrs, name) { ensure_spanned!( @@ -211,7 +216,7 @@ fn impl_arg_param( arg.name.span() => "args cannot be optional" ); return Ok(quote_arg_span! { - let #arg_name = _pyo3::impl_::extract_argument::extract_argument(_args.unwrap(), #name_str)?; + let #arg_name = _pyo3::impl_::extract_argument::extract_argument(_args, #name_str)?; }); } else if is_kwargs(&spec.attrs, name) { ensure_spanned!( @@ -219,43 +224,64 @@ fn impl_arg_param( arg.name.span() => "kwargs must be Option<_>" ); return Ok(quote_arg_span! { - let #arg_name = _kwargs.map(|kwargs| _pyo3::impl_::extract_argument::extract_argument(kwargs, #name_str)) - .transpose()?; + let #arg_name = _pyo3::impl_::extract_argument::extract_optional_argument(_kwargs.map(|kwargs| kwargs.as_ref()), #name_str)?; }); } let arg_value = quote_arg_span!(#args_array[#option_pos]); *option_pos += 1; - let extract = if let Some(FromPyWithAttribute(expr_path)) = &arg.attrs.from_py_with { - quote_arg_span! { #expr_path(_obj).map_err(#transform_error) } + let arg_value_or_default = if let Some(FromPyWithAttribute(expr_path)) = &arg.attrs.from_py_with + { + match (spec.default_value(name), arg.optional.is_some()) { + (Some(default), true) if default.to_string() != "None" => { + quote_arg_span! { + _pyo3::impl_::extract_argument::from_py_with_with_default(#arg_value, #name_str, #expr_path, || Some(#default)) + } + } + (Some(default), _) => { + quote_arg_span! { + _pyo3::impl_::extract_argument::from_py_with_with_default(#arg_value, #name_str, #expr_path, || #default) + } + } + (None, true) => { + quote_arg_span! { + _pyo3::impl_::extract_argument::from_py_with_with_default(#arg_value, #name_str, #expr_path, || Some(None)) + } + } + (None, false) => { + quote_arg_span! { + _pyo3::impl_::extract_argument::from_py_with( + _pyo3::impl_::extract_argument::unwrap_required_argument(#arg_value), + #name_str, + #expr_path, + ) + } + } + } } else { - quote_arg_span! { _pyo3::impl_::extract_argument::extract_argument(_obj, #name_str) } - }; - - let arg_value_or_default = match (spec.default_value(name), arg.optional.is_some()) { - (Some(default), true) if default.to_string() != "None" => { - quote_arg_span! { - #arg_value.map_or_else(|| ::std::result::Result::Ok(::std::option::Option::Some(#default)), - |_obj| #extract)? + match (spec.default_value(name), arg.optional.is_some()) { + (Some(default), true) if default.to_string() != "None" => { + quote_arg_span! { + _pyo3::impl_::extract_argument::extract_argument_with_default(#arg_value, #name_str, || Some(#default)) + } } - } - (Some(default), _) => { - quote_arg_span! { - #arg_value.map_or_else(|| ::std::result::Result::Ok(#default), |_obj| #extract)? + (Some(default), _) => { + quote_arg_span! { + _pyo3::impl_::extract_argument::extract_argument_with_default(#arg_value, #name_str, || #default) + } } - } - (None, true) => { - quote_arg_span! { - #arg_value.map_or(::std::result::Result::Ok(::std::option::Option::None), - |_obj| #extract)? + (None, true) => { + quote_arg_span! { + _pyo3::impl_::extract_argument::extract_optional_argument(#arg_value, #name_str) + } } - } - (None, false) => { - quote_arg_span! { - { - let _obj = #arg_value.expect("Failed to extract required method argument"); - #extract? + (None, false) => { + quote_arg_span! { + _pyo3::impl_::extract_argument::extract_argument( + _pyo3::impl_::extract_argument::unwrap_required_argument(#arg_value), + #name_str + ) } } } @@ -286,13 +312,13 @@ fn impl_arg_param( }; Ok(quote_arg_span! { - let #mut_ _tmp: #target_ty = #arg_value_or_default; + let #mut_ _tmp: #target_ty = #arg_value_or_default?; #[allow(clippy::needless_option_as_deref)] let #arg_name = #borrow_tmp; }) } else { Ok(quote_arg_span! { - let #arg_name = #arg_value_or_default; + let #arg_name = #arg_value_or_default?; }) }; } diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 3a32f4be..ff0085a8 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -3,9 +3,9 @@ use crate::attributes::{ self, take_pyo3_options, CrateAttribute, NameAttribute, TextSignatureAttribute, }; -use crate::deprecations::Deprecations; +use crate::deprecations::{Deprecation, Deprecations}; use crate::konst::{ConstAttributes, ConstSpec}; -use crate::pyimpl::{gen_default_slot_impls, gen_py_const, PyClassMethodsType}; +use crate::pyimpl::{gen_default_items, gen_py_const, PyClassMethodsType}; use crate::pymethod::{impl_py_getter_def, impl_py_setter_def, PropertyType}; use crate::utils::{self, get_pyo3_crate, unwrap_group, PythonDoc}; use proc_macro2::{Span, TokenStream}; @@ -29,12 +29,12 @@ pub struct PyClassArgs { pub base: syn::TypePath, pub has_dict: bool, pub has_weaklist: bool, - pub is_gc: bool, pub is_basetype: bool, pub has_extends: bool, pub has_unsendable: bool, pub module: Option, pub class_kind: PyClassKind, + pub deprecations: Deprecations, } impl PyClassArgs { @@ -63,11 +63,11 @@ impl PyClassArgs { base: parse_quote! { _pyo3::PyAny }, has_dict: false, has_weaklist: false, - is_gc: false, is_basetype: false, has_extends: false, has_unsendable: false, class_kind, + deprecations: Deprecations::new(), } } @@ -158,9 +158,9 @@ impl PyClassArgs { fn add_path(&mut self, exp: &syn::ExprPath) -> syn::Result<()> { let flag = exp.path.segments.first().unwrap().ident.to_string(); match flag.as_str() { - "gc" => { - self.is_gc = true; - } + "gc" => self + .deprecations + .push(Deprecation::PyClassGcOption, exp.span()), "weakref" => { self.has_weaklist = true; } @@ -382,11 +382,15 @@ fn impl_class( ) -> syn::Result { let pytypeinfo_impl = impl_pytypeinfo(cls, attr, Some(&deprecations)); - let py_class_impl = PyClassImplsBuilder::new(cls, attr, methods_type) - .doc(doc) - .impl_all(); - - let descriptors = impl_descriptors(cls, field_options)?; + let py_class_impl = PyClassImplsBuilder::new( + cls, + attr, + methods_type, + descriptors_to_items(cls, field_options)?, + vec![], + ) + .doc(doc) + .impl_all(); Ok(quote! { const _: () = { @@ -395,8 +399,6 @@ fn impl_class( #pytypeinfo_impl #py_class_impl - - #descriptors }; }) } @@ -406,6 +408,54 @@ struct PyClassEnumVariant<'a> { /* currently have no more options */ } +struct PyClassEnum<'a> { + ident: &'a syn::Ident, + // The underlying #[repr] of the enum, used to implement __int__ and __richcmp__. + // This matters when the underlying representation may not fit in `isize`. + repr_type: syn::Ident, + variants: Vec>, +} + +impl<'a> PyClassEnum<'a> { + fn new(enum_: &'a syn::ItemEnum) -> syn::Result { + fn is_numeric_type(t: &syn::Ident) -> bool { + [ + "u8", "i8", "u16", "i16", "u32", "i32", "u64", "i64", "u128", "i128", "usize", + "isize", + ] + .iter() + .any(|&s| t == s) + } + let ident = &enum_.ident; + // According to the [reference](https://doc.rust-lang.org/reference/items/enumerations.html), + // "Under the default representation, the specified discriminant is interpreted as an isize + // value", so `isize` should be enough by default. + let mut repr_type = syn::Ident::new("isize", proc_macro2::Span::call_site()); + if let Some(attr) = enum_.attrs.iter().find(|attr| attr.path.is_ident("repr")) { + let args = + attr.parse_args_with(Punctuated::::parse_terminated)?; + if let Some(ident) = args + .into_iter() + .filter_map(|ts| syn::parse2::(ts).ok()) + .find(is_numeric_type) + { + repr_type = ident; + } + } + + let variants = enum_ + .variants + .iter() + .map(extract_variant_data) + .collect::>()?; + Ok(Self { + ident, + repr_type, + variants, + }) + } +} + pub fn build_py_enum( enum_: &mut syn::ItemEnum, args: &PyClassArgs, @@ -416,22 +466,6 @@ pub fn build_py_enum( if enum_.variants.is_empty() { bail_spanned!(enum_.brace_token.span => "Empty enums can't be #[pyclass]."); } - let variants: Vec = enum_ - .variants - .iter() - .map(extract_variant_data) - .collect::>()?; - impl_enum(enum_, args, variants, method_type, options) -} - -fn impl_enum( - enum_: &syn::ItemEnum, - args: &PyClassArgs, - variants: Vec, - methods_type: PyClassMethodsType, - options: PyClassPyO3Options, -) -> syn::Result { - let enum_name = &enum_.ident; let doc = utils::get_doc( &enum_.attrs, options @@ -439,45 +473,122 @@ fn impl_enum( .as_ref() .map(|attr| (get_class_python_name(&enum_.ident, args), attr)), ); + let enum_ = PyClassEnum::new(enum_)?; + impl_enum(enum_, args, doc, method_type, options) +} + +fn impl_enum( + enum_: PyClassEnum, + args: &PyClassArgs, + doc: PythonDoc, + methods_type: PyClassMethodsType, + options: PyClassPyO3Options, +) -> syn::Result { let krate = get_pyo3_crate(&options.krate); - impl_enum_class(enum_name, args, variants, doc, methods_type, krate) + impl_enum_class(enum_, args, doc, methods_type, krate) } fn impl_enum_class( - cls: &syn::Ident, + enum_: PyClassEnum, args: &PyClassArgs, - variants: Vec, doc: PythonDoc, methods_type: PyClassMethodsType, krate: syn::Path, ) -> syn::Result { + let cls = enum_.ident; + let variants = enum_.variants; let pytypeinfo = impl_pytypeinfo(cls, args, None); - let pyclass_impls = PyClassImplsBuilder::new(cls, args, methods_type) - .doc(doc) - .impl_all(); - let descriptors = unit_variants_as_descriptors(cls, variants.iter().map(|v| v.ident)); - let default_repr_impl = { + let default_repr_impl: syn::ImplItemMethod = { let variants_repr = variants.iter().map(|variant| { let variant_name = variant.ident; // Assuming all variants are unit variants because they are the only type we support. let repr = format!("{}.{}", cls, variant_name); quote! { #cls::#variant_name => #repr, } }); - quote! { + syn::parse_quote! { #[doc(hidden)] #[allow(non_snake_case)] #[pyo3(name = "__repr__")] fn __pyo3__repr__(&self) -> &'static str { match self { #(#variants_repr)* - _ => unreachable!("Unsupported variant type."), } } } }; - let default_impls = gen_default_slot_impls(cls, vec![default_repr_impl]); + let repr_type = &enum_.repr_type; + + let default_int = { + // This implementation allows us to convert &T to #repr_type without implementing `Copy` + let variants_to_int = variants.iter().map(|variant| { + let variant_name = variant.ident; + quote! { #cls::#variant_name => #cls::#variant_name as #repr_type, } + }); + syn::parse_quote! { + #[doc(hidden)] + #[allow(non_snake_case)] + #[pyo3(name = "__int__")] + fn __pyo3__int__(&self) -> #repr_type { + match self { + #(#variants_to_int)* + } + } + } + }; + + let default_richcmp = { + let variants_eq = variants.iter().map(|variant| { + let variant_name = variant.ident; + quote! { + (#cls::#variant_name, #cls::#variant_name) => + Ok(true.to_object(py)), + } + }); + syn::parse_quote! { + #[doc(hidden)] + #[allow(non_snake_case)] + #[pyo3(name = "__richcmp__")] + fn __pyo3__richcmp__( + &self, + py: _pyo3::Python, + other: &_pyo3::PyAny, + op: _pyo3::basic::CompareOp + ) -> _pyo3::PyResult<_pyo3::PyObject> { + use _pyo3::conversion::ToPyObject; + use ::core::result::Result::*; + match op { + _pyo3::basic::CompareOp::Eq => { + if let Ok(i) = other.extract::<#repr_type>() { + let self_val = self.__pyo3__int__(); + return Ok((self_val == i).to_object(py)); + } + let other = other.extract::<_pyo3::PyRef>()?; + let other = &*other; + match (self, other) { + #(#variants_eq)* + _ => Ok(false.to_object(py)), + } + } + _ => Ok(py.NotImplemented()), + } + } + } + }; + + let mut default_methods = vec![default_repr_impl, default_richcmp, default_int]; + + let pyclass_impls = PyClassImplsBuilder::new( + cls, + args, + methods_type, + enum_default_methods(cls, variants.iter().map(|v| v.ident)), + enum_default_slots(cls, &mut default_methods), + ) + .doc(doc) + .impl_all(); + Ok(quote! { const _: () = { use #krate as _pyo3; @@ -486,17 +597,17 @@ fn impl_enum_class( #pyclass_impls - #default_impls - - #descriptors + impl #cls { + #(#default_methods)* + } }; }) } -fn unit_variants_as_descriptors<'a>( +fn enum_default_methods<'a>( cls: &'a syn::Ident, - variant_names: impl IntoIterator, -) -> TokenStream { + unit_variant_names: impl IntoIterator, +) -> Vec { let cls_type = syn::parse_quote!(#cls); let variant_to_attribute = |ident: &syn::Ident| ConstSpec { rust_ident: ident.clone(), @@ -506,20 +617,17 @@ fn unit_variants_as_descriptors<'a>( deprecations: Default::default(), }, }; - let py_methods = variant_names + unit_variant_names .into_iter() - .map(|var| gen_py_const(&cls_type, &variant_to_attribute(var))); + .map(|var| gen_py_const(&cls_type, &variant_to_attribute(var))) + .collect() +} - quote! { - impl _pyo3::class::impl_::PyClassDescriptors<#cls> - for _pyo3::class::impl_::PyClassImplCollector<#cls> - { - fn py_class_descriptors(self) -> &'static [_pyo3::class::methods::PyMethodDefType] { - static METHODS: &[_pyo3::class::methods::PyMethodDefType] = &[#(#py_methods),*]; - METHODS - } - } - } +fn enum_default_slots( + cls: &syn::Ident, + default_items: &mut [syn::ImplItemMethod], +) -> Vec { + gen_default_items(cls, default_items).collect() } fn extract_variant_data(variant: &syn::Variant) -> syn::Result { @@ -528,18 +636,15 @@ fn extract_variant_data(variant: &syn::Variant) -> syn::Result &variant.ident, _ => bail_spanned!(variant.span() => "Currently only support unit variants."), }; - if let Some(discriminant) = variant.discriminant.as_ref() { - bail_spanned!(discriminant.0.span() => "Currently does not support discriminats.") - }; Ok(PyClassEnumVariant { ident }) } -fn impl_descriptors( +fn descriptors_to_items( cls: &syn::Ident, field_options: Vec<(&syn::Field, FieldPyO3Options)>, -) -> syn::Result { +) -> syn::Result> { let ty = syn::parse_quote!(#cls); - let py_methods: Vec = field_options + field_options .into_iter() .enumerate() .flat_map(|(field_index, (field, options))| { @@ -571,18 +676,7 @@ fn impl_descriptors( name_err.into_iter().chain(getter).chain(setter) }) - .collect::>()?; - - Ok(quote! { - impl _pyo3::class::impl_::PyClassDescriptors<#cls> - for _pyo3::class::impl_::PyClassImplCollector<#cls> - { - fn py_class_descriptors(self) -> &'static [_pyo3::class::methods::PyMethodDefType] { - static METHODS: &[_pyo3::class::methods::PyMethodDefType] = &[#(#py_methods),*]; - METHODS - } - } - }) + .collect::>() } fn impl_pytypeinfo( @@ -626,15 +720,25 @@ struct PyClassImplsBuilder<'a> { cls: &'a syn::Ident, attr: &'a PyClassArgs, methods_type: PyClassMethodsType, + default_methods: Vec, + default_slots: Vec, doc: Option, } impl<'a> PyClassImplsBuilder<'a> { - fn new(cls: &'a syn::Ident, attr: &'a PyClassArgs, methods_type: PyClassMethodsType) -> Self { + fn new( + cls: &'a syn::Ident, + attr: &'a PyClassArgs, + methods_type: PyClassMethodsType, + default_methods: Vec, + default_slots: Vec, + ) -> Self { Self { cls, attr, methods_type, + default_methods, + default_slots, doc: None, } } @@ -653,7 +757,6 @@ impl<'a> PyClassImplsBuilder<'a> { self.impl_into_py(), self.impl_pyclassimpl(), self.impl_freelist(), - self.impl_gc(), ] .into_iter() .collect() @@ -676,7 +779,7 @@ impl<'a> PyClassImplsBuilder<'a> { }; let base_nativetype = if attr.has_extends { - quote! { ::BaseNativeType } + quote! { ::BaseNativeType } } else { quote! { _pyo3::PyAny } }; @@ -722,7 +825,6 @@ impl<'a> PyClassImplsBuilder<'a> { fn impl_pyclassimpl(&self) -> TokenStream { let cls = self.cls; let doc = self.doc.as_ref().map_or(quote! {"\0"}, |doc| quote! {#doc}); - let is_gc = self.attr.is_gc; let is_basetype = self.attr.is_basetype; let base = &self.attr.base; let is_subclass = self.attr.has_extends; @@ -749,24 +851,19 @@ impl<'a> PyClassImplsBuilder<'a> { }; let thread_checker = if self.attr.has_unsendable { - quote! { _pyo3::class::impl_::ThreadCheckerImpl<#cls> } + quote! { _pyo3::impl_::pyclass::ThreadCheckerImpl<#cls> } } else if self.attr.has_extends { quote! { - _pyo3::class::impl_::ThreadCheckerInherited<#cls, <#cls as _pyo3::class::impl_::PyClassImpl>::BaseType> + _pyo3::impl_::pyclass::ThreadCheckerInherited<#cls, <#cls as _pyo3::impl_::pyclass::PyClassImpl>::BaseType> } } else { - quote! { _pyo3::class::impl_::ThreadCheckerStub<#cls> } + quote! { _pyo3::impl_::pyclass::ThreadCheckerStub<#cls> } }; - let (for_each_py_method, methods_protos, inventory, inventory_class) = match self - .methods_type - { - PyClassMethodsType::Specialization => ( - quote! { visitor(collector.py_methods()); }, - quote! { visitor(collector.methods_protocol_slots()); }, - None, - None, - ), + let (pymethods_items, inventory, inventory_class) = match self.methods_type { + PyClassMethodsType::Specialization => { + (quote! { visitor(collector.py_methods()); }, None, None) + } PyClassMethodsType::Inventory => { // To allow multiple #[pymethods] block, we define inventory types. let inventory_class_name = syn::Ident::new( @@ -775,13 +872,8 @@ impl<'a> PyClassImplsBuilder<'a> { ); ( quote! { - for inventory in _pyo3::inventory::iter::<::Inventory>() { - visitor(_pyo3::class::impl_::PyClassInventory::methods(inventory)); - } - }, - quote! { - for inventory in _pyo3::inventory::iter::<::Inventory>() { - visitor(_pyo3::class::impl_::PyClassInventory::slots(inventory)); + for inventory in _pyo3::inventory::iter::<::Inventory>() { + visitor(_pyo3::impl_::pyclass::PyClassInventory::items(inventory)); } }, Some(quote! { type Inventory = #inventory_class_name; }), @@ -789,10 +881,32 @@ impl<'a> PyClassImplsBuilder<'a> { ) } }; + + let pyproto_items = if cfg!(feature = "pyproto") { + Some(quote! { + visitor(collector.object_protocol_items()); + visitor(collector.number_protocol_items()); + visitor(collector.iter_protocol_items()); + visitor(collector.gc_protocol_items()); + visitor(collector.descr_protocol_items()); + visitor(collector.mapping_protocol_items()); + visitor(collector.sequence_protocol_items()); + visitor(collector.async_protocol_items()); + visitor(collector.buffer_protocol_items()); + }) + } else { + None + }; + + let default_methods = &self.default_methods; + let default_slots = &self.default_slots; + let freelist_slots = self.freelist_slots(); + + let deprecations = &self.attr.deprecations; + quote! { - impl _pyo3::class::impl_::PyClassImpl for #cls { + impl _pyo3::impl_::pyclass::PyClassImpl for #cls { const DOC: &'static str = #doc; - const IS_GC: bool = #is_gc; const IS_BASETYPE: bool = #is_basetype; const IS_SUBCLASS: bool = #is_subclass; @@ -801,50 +915,17 @@ impl<'a> PyClassImplsBuilder<'a> { type ThreadChecker = #thread_checker; #inventory - fn for_each_method_def(visitor: &mut dyn ::std::ops::FnMut(&[_pyo3::class::PyMethodDefType])) { - use _pyo3::class::impl_::*; + fn for_all_items(visitor: &mut dyn ::std::ops::FnMut(& _pyo3::impl_::pyclass::PyClassItems)) { + use _pyo3::impl_::pyclass::*; let collector = PyClassImplCollector::::new(); - #for_each_py_method; - visitor(collector.py_class_descriptors()); - visitor(collector.object_protocol_methods()); - visitor(collector.async_protocol_methods()); - visitor(collector.descr_protocol_methods()); - visitor(collector.mapping_protocol_methods()); - visitor(collector.number_protocol_methods()); - } - fn get_new() -> ::std::option::Option<_pyo3::ffi::newfunc> { - use _pyo3::class::impl_::*; - let collector = PyClassImplCollector::::new(); - collector.new_impl() - } - fn get_alloc() -> ::std::option::Option<_pyo3::ffi::allocfunc> { - use _pyo3::class::impl_::*; - let collector = PyClassImplCollector::::new(); - collector.alloc_impl() - } - fn get_free() -> ::std::option::Option<_pyo3::ffi::freefunc> { - use _pyo3::class::impl_::*; - let collector = PyClassImplCollector::::new(); - collector.free_impl() - } - - fn for_each_proto_slot(visitor: &mut dyn ::std::ops::FnMut(&[_pyo3::ffi::PyType_Slot])) { - // Implementation which uses dtolnay specialization to load all slots. - use _pyo3::class::impl_::*; - let collector = PyClassImplCollector::::new(); - // This depends on Python implementation detail; - // an old slot entry will be overriden by newer ones. - visitor(collector.py_class_default_slots()); - visitor(collector.object_protocol_slots()); - visitor(collector.number_protocol_slots()); - visitor(collector.iter_protocol_slots()); - visitor(collector.gc_protocol_slots()); - visitor(collector.descr_protocol_slots()); - visitor(collector.mapping_protocol_slots()); - visitor(collector.sequence_protocol_slots()); - visitor(collector.async_protocol_slots()); - visitor(collector.buffer_protocol_slots()); - #methods_protos + #deprecations; + static INTRINSIC_ITEMS: PyClassItems = PyClassItems { + methods: &[#(#default_methods),*], + slots: &[#(#default_slots),* #(#freelist_slots),*], + }; + visitor(&INTRINSIC_ITEMS); + #pymethods_items + #pyproto_items } #dict_offset @@ -861,7 +942,7 @@ impl<'a> PyClassImplsBuilder<'a> { self.attr.freelist.as_ref().map_or(quote!{}, |freelist| { quote! { - impl _pyo3::class::impl_::PyClassWithFreeList for #cls { + impl _pyo3::impl_::pyclass::PyClassWithFreeList for #cls { #[inline] fn get_free_list(_py: _pyo3::Python<'_>) -> &mut _pyo3::impl_::freelist::FreeList<*mut _pyo3::ffi::PyObject> { static mut FREELIST: *mut _pyo3::impl_::freelist::FreeList<*mut _pyo3::ffi::PyObject> = 0 as *mut _; @@ -874,40 +955,30 @@ impl<'a> PyClassImplsBuilder<'a> { } } } - - impl _pyo3::class::impl_::PyClassAllocImpl<#cls> for _pyo3::class::impl_::PyClassImplCollector<#cls> { - #[inline] - fn alloc_impl(self) -> ::std::option::Option<_pyo3::ffi::allocfunc> { - ::std::option::Option::Some(_pyo3::class::impl_::alloc_with_freelist::<#cls>) - } - } - - impl _pyo3::class::impl_::PyClassFreeImpl<#cls> for _pyo3::class::impl_::PyClassImplCollector<#cls> { - #[inline] - fn free_impl(self) -> ::std::option::Option<_pyo3::ffi::freefunc> { - ::std::option::Option::Some(_pyo3::class::impl_::free_with_freelist::<#cls>) - } - } } }) } - /// Enforce at compile time that PyGCProtocol is implemented - fn impl_gc(&self) -> TokenStream { - let cls = self.cls; - let attr = self.attr; - if attr.is_gc { - let closure_name = format!("__assertion_closure_{}", cls); - let closure_token = syn::Ident::new(&closure_name, Span::call_site()); - quote! { - fn #closure_token() { - use _pyo3::class; - fn _assert_implements_protocol<'p, T: _pyo3::class::PyGCProtocol<'p>>() {} - _assert_implements_protocol::<#cls>(); - } - } + fn freelist_slots(&self) -> Vec { + let cls = self.cls; + + if self.attr.freelist.is_some() { + vec![ + quote! { + _pyo3::ffi::PyType_Slot { + slot: _pyo3::ffi::Py_tp_alloc, + pfunc: _pyo3::impl_::pyclass::alloc_with_freelist::<#cls> as *mut _, + } + }, + quote! { + _pyo3::ffi::PyType_Slot { + slot: _pyo3::ffi::Py_tp_free, + pfunc: _pyo3::impl_::pyclass::free_with_freelist::<#cls> as *mut _, + } + }, + ] } else { - quote! {} + Vec::new() } } } @@ -916,31 +987,20 @@ fn define_inventory_class(inventory_class_name: &syn::Ident) -> TokenStream { quote! { #[doc(hidden)] pub struct #inventory_class_name { - methods: &'static [_pyo3::class::PyMethodDefType], - slots: &'static [_pyo3::ffi::PyType_Slot], + items: _pyo3::impl_::pyclass::PyClassItems, } impl #inventory_class_name { - const fn new( - methods: &'static [_pyo3::class::PyMethodDefType], - slots: &'static [_pyo3::ffi::PyType_Slot], - ) -> Self { - Self { methods, slots } + pub const fn new(items: _pyo3::impl_::pyclass::PyClassItems) -> Self { + Self { items } } } - impl _pyo3::class::impl_::PyClassInventory for #inventory_class_name { - fn methods(&'static self) -> &'static [_pyo3::class::PyMethodDefType] { - self.methods - } - fn slots(&'static self) -> &'static [_pyo3::ffi::PyType_Slot] { - self.slots + impl _pyo3::impl_::pyclass::PyClassInventory for #inventory_class_name { + fn items(&self) -> &_pyo3::impl_::pyclass::PyClassItems { + &self.items } } - // inventory requires these bounds - unsafe impl ::std::marker::Send for #inventory_class_name {} - unsafe impl ::std::marker::Sync for #inventory_class_name {} - _pyo3::inventory::collect!(#inventory_class_name); } } diff --git a/pyo3-macros-backend/src/pyimpl.rs b/pyo3-macros-backend/src/pyimpl.rs index 46fe05b9..e5270641 100644 --- a/pyo3-macros-backend/src/pyimpl.rs +++ b/pyo3-macros-backend/src/pyimpl.rs @@ -108,10 +108,6 @@ pub fn impl_methods( let attrs = get_cfg_attributes(&meth.attrs); methods.push(quote!(#(#attrs)* #token_stream)); } - GeneratedPyMethod::TraitImpl(token_stream) => { - let attrs = get_cfg_attributes(&meth.attrs); - trait_impls.push(quote!(#(#attrs)* #token_stream)); - } GeneratedPyMethod::SlotTraitImpl(method_name, token_stream) => { implemented_proto_fragments.insert(method_name); let attrs = get_cfg_attributes(&meth.attrs); @@ -150,35 +146,19 @@ pub fn impl_methods( let krate = get_pyo3_crate(&options.krate); - Ok(match methods_type { - PyClassMethodsType::Specialization => { - let methods_registration = impl_py_methods(ty, methods); - let protos_registration = impl_protos(ty, proto_impls); + let items = match methods_type { + PyClassMethodsType::Specialization => impl_py_methods(ty, methods, proto_impls), + PyClassMethodsType::Inventory => submit_methods_inventory(ty, methods, proto_impls), + }; - quote! { - const _: () = { - use #krate as _pyo3; + Ok(quote! { + const _: () = { + use #krate as _pyo3; - #(#trait_impls)* + #(#trait_impls)* - #protos_registration - - #methods_registration - }; - } - } - PyClassMethodsType::Inventory => { - let inventory = submit_methods_inventory(ty, methods, proto_impls); - quote! { - const _: () = { - use #krate as _pyo3; - - #(#trait_impls)* - - #inventory - }; - } - } + #items + }; }) } @@ -190,7 +170,7 @@ pub fn gen_py_const(cls: &syn::Type, spec: &ConstSpec) -> TokenStream { _pyo3::class::PyMethodDefType::ClassAttribute({ _pyo3::class::PyClassAttributeDef::new( #python_name, - _pyo3::class::methods::PyClassAttributeFactory({ + _pyo3::impl_::pymethods::PyClassAttributeFactory({ fn __wrap(py: _pyo3::Python<'_>) -> _pyo3::PyObject { #deprecations _pyo3::IntoPy::into_py(#cls::#member, py) @@ -202,55 +182,46 @@ pub fn gen_py_const(cls: &syn::Type, spec: &ConstSpec) -> TokenStream { } } -pub fn gen_default_slot_impls(cls: &syn::Ident, method_defs: Vec) -> TokenStream { +pub fn gen_default_items<'a>( + cls: &syn::Ident, + method_defs: &'a mut [syn::ImplItemMethod], +) -> impl Iterator + 'a { // This function uses a lot of `unwrap()`; since method_defs are provided by us, they should // all succeed. let ty: syn::Type = syn::parse_quote!(#cls); - let mut method_defs: Vec<_> = method_defs - .into_iter() - .map(|token| syn::parse2::(token).unwrap()) - .collect(); - - let mut proto_impls = Vec::new(); - - for meth in &mut method_defs { + method_defs.iter_mut().map(move |meth| { let options = PyFunctionOptions::from_attrs(&mut meth.attrs).unwrap(); match pymethod::gen_py_method(&ty, &mut meth.sig, &mut meth.attrs, options).unwrap() { GeneratedPyMethod::Proto(token_stream) => { let attrs = get_cfg_attributes(&meth.attrs); - proto_impls.push(quote!(#(#attrs)* #token_stream)) + quote!(#(#attrs)* #token_stream) } GeneratedPyMethod::SlotTraitImpl(..) => { panic!("SlotFragment methods cannot have default implementation!") } - GeneratedPyMethod::Method(_) | GeneratedPyMethod::TraitImpl(_) => { + GeneratedPyMethod::Method(_) => { panic!("Only protocol methods can have default implementation!") } } - } - - quote! { - impl #cls { - #(#method_defs)* - } - impl ::pyo3::class::impl_::PyClassDefaultSlots<#cls> - for ::pyo3::class::impl_::PyClassImplCollector<#cls> { - fn py_class_default_slots(self) -> &'static [::pyo3::ffi::PyType_Slot] { - &[#(#proto_impls),*] - } - } - } + }) } -fn impl_py_methods(ty: &syn::Type, methods: Vec) -> TokenStream { +fn impl_py_methods( + ty: &syn::Type, + methods: Vec, + proto_impls: Vec, +) -> TokenStream { quote! { - impl _pyo3::class::impl_::PyMethods<#ty> - for _pyo3::class::impl_::PyClassImplCollector<#ty> + impl _pyo3::impl_::pyclass::PyMethods<#ty> + for _pyo3::impl_::pyclass::PyClassImplCollector<#ty> { - fn py_methods(self) -> &'static [_pyo3::class::methods::PyMethodDefType] { - static METHODS: &[_pyo3::class::methods::PyMethodDefType] = &[#(#methods),*]; - METHODS + fn py_methods(self) -> &'static _pyo3::impl_::pyclass::PyClassItems { + static ITEMS: _pyo3::impl_::pyclass::PyClassItems = _pyo3::impl_::pyclass::PyClassItems { + methods: &[#(#methods),*], + slots: &[#(#proto_impls),*] + }; + &ITEMS } } } @@ -266,7 +237,7 @@ fn add_shared_proto_slots( let first_implemented = implemented_proto_fragments.remove($first); let second_implemented = implemented_proto_fragments.remove($second); if first_implemented || second_implemented { - proto_impls.push(quote! { _pyo3::class::impl_::$slot!(#ty) }) + proto_impls.push(quote! { _pyo3::impl_::pyclass::$slot!(#ty) }) } }}; } @@ -293,21 +264,11 @@ fn add_shared_proto_slots( ); try_add_shared_slot!("__pow__", "__rpow__", generate_pyclass_pow_slot); + // if this assertion trips, a slot fragment has been implemented which has not been added in the + // list above assert!(implemented_proto_fragments.is_empty()); } -fn impl_protos(ty: &syn::Type, proto_impls: Vec) -> TokenStream { - quote! { - impl _pyo3::class::impl_::PyMethodsProtocolSlots<#ty> - for _pyo3::class::impl_::PyClassImplCollector<#ty> - { - fn methods_protocol_slots(self) -> &'static [_pyo3::ffi::PyType_Slot] { - &[#(#proto_impls),*] - } - } - } -} - fn submit_methods_inventory( ty: &syn::Type, methods: Vec, @@ -315,8 +276,8 @@ fn submit_methods_inventory( ) -> TokenStream { quote! { _pyo3::inventory::submit! { - type Inventory = <#ty as _pyo3::class::impl_::PyClassImpl>::Inventory; - Inventory::new(&[#(#methods),*], &[#(#proto_impls),*]) + type Inventory = <#ty as _pyo3::impl_::pyclass::PyClassImpl>::Inventory; + Inventory::new(_pyo3::impl_::pyclass::PyClassItems { methods: &[#(#methods),*], slots: &[#(#proto_impls),*] }) } } } diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 31efb6b3..b5687d2f 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -20,7 +20,6 @@ use syn::{ext::IdentExt, spanned::Spanned, Result}; pub enum GeneratedPyMethod { Method(TokenStream), Proto(TokenStream), - TraitImpl(TokenStream), SlotTraitImpl(String, TokenStream), } @@ -37,14 +36,88 @@ enum PyMethodKind { impl PyMethodKind { fn from_name(name: &str) -> Self { - if let Some(slot_def) = pyproto(name) { - PyMethodKind::Proto(PyMethodProtoKind::Slot(slot_def)) - } else if name == "__call__" { - PyMethodKind::Proto(PyMethodProtoKind::Call) - } else if let Some(slot_fragment_def) = pyproto_fragment(name) { - PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(slot_fragment_def)) - } else { - PyMethodKind::Fn + match name { + // Protocol implemented through slots + "__getattr__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__GETATTR__)), + "__str__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__STR__)), + "__repr__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__REPR__)), + "__hash__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__HASH__)), + "__richcmp__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__RICHCMP__)), + "__get__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__GET__)), + "__iter__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__ITER__)), + "__next__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__NEXT__)), + "__await__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__AWAIT__)), + "__aiter__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__AITER__)), + "__anext__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__ANEXT__)), + "__len__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__LEN__)), + "__contains__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__CONTAINS__)), + "__concat__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__CONCAT__)), + "__repeat__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__REPEAT__)), + "__getitem__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__GETITEM__)), + "__pos__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__POS__)), + "__neg__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__NEG__)), + "__abs__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__ABS__)), + "__invert__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__INVERT__)), + "__index__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__INDEX__)), + "__int__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__INT__)), + "__float__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__FLOAT__)), + "__bool__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__BOOL__)), + "__iadd__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__IADD__)), + "__isub__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__ISUB__)), + "__imul__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__IMUL__)), + "__imatmul__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__IMATMUL__)), + "__itruediv__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__ITRUEDIV__)), + "__ifloordiv__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__IFLOORDIV__)), + "__imod__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__IMOD__)), + "__ipow__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__IPOW__)), + "__ilshift__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__ILSHIFT__)), + "__irshift__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__IRSHIFT__)), + "__iand__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__IAND__)), + "__ixor__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__IXOR__)), + "__ior__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__IOR__)), + "__getbuffer__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__GETBUFFER__)), + "__releasebuffer__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__RELEASEBUFFER__)), + "__clear__" => PyMethodKind::Proto(PyMethodProtoKind::Slot(&__CLEAR__)), + // Protocols implemented through traits + "__setattr__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__SETATTR__)), + "__delattr__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__DELATTR__)), + "__set__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__SET__)), + "__delete__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__DELETE__)), + "__setitem__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__SETITEM__)), + "__delitem__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__DELITEM__)), + "__add__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__ADD__)), + "__radd__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__RADD__)), + "__sub__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__SUB__)), + "__rsub__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__RSUB__)), + "__mul__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__MUL__)), + "__rmul__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__RMUL__)), + "__matmul__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__MATMUL__)), + "__rmatmul__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__RMATMUL__)), + "__floordiv__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__FLOORDIV__)), + "__rfloordiv__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__RFLOORDIV__)), + "__truediv__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__TRUEDIV__)), + "__rtruediv__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__RTRUEDIV__)), + "__divmod__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__DIVMOD__)), + "__rdivmod__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__RDIVMOD__)), + "__mod__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__MOD__)), + "__rmod__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__RMOD__)), + "__lshift__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__LSHIFT__)), + "__rlshift__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__RLSHIFT__)), + "__rshift__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__RSHIFT__)), + "__rrshift__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__RRSHIFT__)), + "__and__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__AND__)), + "__rand__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__RAND__)), + "__xor__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__XOR__)), + "__rxor__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__RXOR__)), + "__or__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__OR__)), + "__ror__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__ROR__)), + "__pow__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__POW__)), + "__rpow__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__RPOW__)), + // Some tricky protocols which don't fit the pattern of the rest + "__call__" => PyMethodKind::Proto(PyMethodProtoKind::Call), + "__traverse__" => PyMethodKind::Proto(PyMethodProtoKind::Traverse), + // Not a proto + _ => PyMethodKind::Fn, } } } @@ -52,6 +125,7 @@ impl PyMethodKind { enum PyMethodProtoKind { Slot(&'static SlotDef), Call, + Traverse, SlotFragment(&'static SlotFragmentDef), } @@ -109,6 +183,9 @@ pub fn gen_py_method( PyMethodProtoKind::Call => { GeneratedPyMethod::Proto(impl_call_slot(cls, method.spec)?) } + PyMethodProtoKind::Traverse => { + GeneratedPyMethod::Proto(impl_traverse_slot(cls, method.spec)?) + } PyMethodProtoKind::SlotFragment(slot_fragment_def) => { let proto = slot_fragment_def.generate_pyproto_fragment(cls, spec)?; GeneratedPyMethod::SlotTraitImpl(method.method_name, proto) @@ -128,7 +205,7 @@ pub fn gen_py_method( Some(quote!(_pyo3::ffi::METH_STATIC)), )?), // special prototypes - (_, FnType::FnNew) => GeneratedPyMethod::TraitImpl(impl_py_method_def_new(cls, spec)?), + (_, FnType::FnNew) => GeneratedPyMethod::Proto(impl_py_method_def_new(cls, spec)?), (_, FnType::Getter(self_type)) => GeneratedPyMethod::Method(impl_py_getter_def( cls, @@ -191,18 +268,18 @@ pub fn impl_py_method_def( } fn impl_py_method_def_new(cls: &syn::Type, spec: &FnSpec) -> Result { - let wrapper_ident = syn::Ident::new("__wrap", Span::call_site()); + let wrapper_ident = syn::Ident::new("__pymethod__new__", Span::call_site()); let wrapper = spec.get_wrapper_function(&wrapper_ident, Some(cls))?; - Ok(quote! { - impl _pyo3::class::impl_::PyClassNewImpl<#cls> for _pyo3::class::impl_::PyClassImplCollector<#cls> { - fn new_impl(self) -> ::std::option::Option<_pyo3::ffi::newfunc> { - ::std::option::Option::Some({ - #wrapper - #wrapper_ident - }) - } + Ok(quote! {{ + impl #cls { + #[doc(hidden)] + #wrapper } - }) + _pyo3::ffi::PyType_Slot { + slot: _pyo3::ffi::Py_tp_new, + pfunc: #cls::#wrapper_ident as _pyo3::ffi::newfunc as _ + } + }}) } fn impl_call_slot(cls: &syn::Type, mut spec: FnSpec) -> Result { @@ -221,6 +298,36 @@ fn impl_call_slot(cls: &syn::Type, mut spec: FnSpec) -> Result { }}) } +fn impl_traverse_slot(cls: &syn::Type, spec: FnSpec) -> Result { + let ident = spec.name; + Ok(quote! {{ + pub unsafe extern "C" fn __wrap_( + slf: *mut _pyo3::ffi::PyObject, + visit: _pyo3::ffi::visitproc, + arg: *mut ::std::os::raw::c_void, + ) -> ::std::os::raw::c_int + { + let pool = _pyo3::GILPool::new(); + let py = pool.python(); + _pyo3::callback::abort_on_traverse_panic(::std::panic::catch_unwind(move || { + let slf = py.from_borrowed_ptr::<_pyo3::PyCell<#cls>>(slf); + + let visit = _pyo3::class::gc::PyVisit::from_raw(visit, arg, py); + let borrow = slf.try_borrow(); + if let ::std::result::Result::Ok(borrow) = borrow { + _pyo3::class::gc::unwrap_traverse_result(borrow.#ident(visit)) + } else { + 0 + } + })) + } + _pyo3::ffi::PyType_Slot { + slot: _pyo3::ffi::Py_tp_traverse, + pfunc: __wrap_ as _pyo3::ffi::traverseproc as _ + } + }}) +} + fn impl_py_class_attribute(cls: &syn::Type, spec: &FnSpec) -> TokenStream { let name = &spec.name; let deprecations = &spec.deprecations; @@ -229,7 +336,7 @@ fn impl_py_class_attribute(cls: &syn::Type, spec: &FnSpec) -> TokenStream { _pyo3::class::PyMethodDefType::ClassAttribute({ _pyo3::class::PyClassAttributeDef::new( #python_name, - _pyo3::class::methods::PyClassAttributeFactory({ + _pyo3::impl_::pymethods::PyClassAttributeFactory({ fn __wrap(py: _pyo3::Python<'_>) -> _pyo3::PyObject { #deprecations _pyo3::IntoPy::into_py(#cls::#name(), py) @@ -299,13 +406,15 @@ pub fn impl_py_setter_def(cls: &syn::Type, property_type: PropertyType) -> Resul #deprecations _pyo3::class::PySetterDef::new( #python_name, - _pyo3::class::methods::PySetter({ + _pyo3::impl_::pymethods::PySetter({ unsafe extern "C" fn __wrap( _slf: *mut _pyo3::ffi::PyObject, _value: *mut _pyo3::ffi::PyObject, _: *mut ::std::os::raw::c_void ) -> ::std::os::raw::c_int { - _pyo3::callback::handle_panic(|_py| { + let gil = _pyo3::GILPool::new(); + let _py = gil.python(); + _pyo3::callback::panic_result_into_callback_output(_py, ::std::panic::catch_unwind(move || -> _pyo3::PyResult<_> { #slf let _value = _py .from_borrowed_ptr_or_opt(_value) @@ -315,7 +424,7 @@ pub fn impl_py_setter_def(cls: &syn::Type, property_type: PropertyType) -> Resul let _val = _pyo3::FromPyObject::extract(_value)?; _pyo3::callback::convert(_py, #setter_impl) - }) + })) } __wrap }), @@ -379,15 +488,17 @@ pub fn impl_py_getter_def(cls: &syn::Type, property_type: PropertyType) -> Resul #deprecations _pyo3::class::PyGetterDef::new( #python_name, - _pyo3::class::methods::PyGetter({ + _pyo3::impl_::pymethods::PyGetter({ unsafe extern "C" fn __wrap( _slf: *mut _pyo3::ffi::PyObject, _: *mut ::std::os::raw::c_void ) -> *mut _pyo3::ffi::PyObject { - _pyo3::callback::handle_panic(|_py| { + let gil = _pyo3::GILPool::new(); + let _py = gil.python(); + _pyo3::callback::panic_result_into_callback_output(_py, ::std::panic::catch_unwind(move || -> _pyo3::PyResult<_> { #slf _pyo3::callback::convert(_py, #getter_impl) - }) + })) } __wrap }), @@ -493,6 +604,8 @@ const __LEN__: SlotDef = SlotDef::new("Py_mp_length", "lenfunc").ret_ty(Ty::PySs const __CONTAINS__: SlotDef = SlotDef::new("Py_sq_contains", "objobjproc") .arguments(&[Ty::Object]) .ret_ty(Ty::Int); +const __CONCAT__: SlotDef = SlotDef::new("Py_sq_concat", "binaryfunc").arguments(&[Ty::Object]); +const __REPEAT__: SlotDef = SlotDef::new("Py_sq_repeat", "ssizeargfunc").arguments(&[Ty::PySsizeT]); const __GETITEM__: SlotDef = SlotDef::new("Py_mp_subscript", "binaryfunc").arguments(&[Ty::Object]); const __POS__: SlotDef = SlotDef::new("Py_nb_positive", "unaryfunc"); @@ -564,49 +677,9 @@ const __RELEASEBUFFER__: SlotDef = SlotDef::new("Py_bf_releasebuffer", "releaseb .arguments(&[Ty::PyBuffer]) .ret_ty(Ty::Void) .require_unsafe(); - -fn pyproto(method_name: &str) -> Option<&'static SlotDef> { - match method_name { - "__getattr__" => Some(&__GETATTR__), - "__str__" => Some(&__STR__), - "__repr__" => Some(&__REPR__), - "__hash__" => Some(&__HASH__), - "__richcmp__" => Some(&__RICHCMP__), - "__get__" => Some(&__GET__), - "__iter__" => Some(&__ITER__), - "__next__" => Some(&__NEXT__), - "__await__" => Some(&__AWAIT__), - "__aiter__" => Some(&__AITER__), - "__anext__" => Some(&__ANEXT__), - "__len__" => Some(&__LEN__), - "__contains__" => Some(&__CONTAINS__), - "__getitem__" => Some(&__GETITEM__), - "__pos__" => Some(&__POS__), - "__neg__" => Some(&__NEG__), - "__abs__" => Some(&__ABS__), - "__invert__" => Some(&__INVERT__), - "__index__" => Some(&__INDEX__), - "__int__" => Some(&__INT__), - "__float__" => Some(&__FLOAT__), - "__bool__" => Some(&__BOOL__), - "__iadd__" => Some(&__IADD__), - "__isub__" => Some(&__ISUB__), - "__imul__" => Some(&__IMUL__), - "__imatmul__" => Some(&__IMATMUL__), - "__itruediv__" => Some(&__ITRUEDIV__), - "__ifloordiv__" => Some(&__IFLOORDIV__), - "__imod__" => Some(&__IMOD__), - "__ipow__" => Some(&__IPOW__), - "__ilshift__" => Some(&__ILSHIFT__), - "__irshift__" => Some(&__IRSHIFT__), - "__iand__" => Some(&__IAND__), - "__ixor__" => Some(&__IXOR__), - "__ior__" => Some(&__IOR__), - "__getbuffer__" => Some(&__GETBUFFER__), - "__releasebuffer__" => Some(&__RELEASEBUFFER__), - _ => None, - } -} +const __CLEAR__: SlotDef = SlotDef::new("Py_tp_clear", "inquiry") + .arguments(&[]) + .ret_ty(Ty::Int); #[derive(Clone, Copy)] enum Ty { @@ -704,8 +777,21 @@ impl Ty { let #ident = #extract; } } + Ty::PySsizeT => { + let ty = arg.ty; + let extract = handle_error( + extract_error_mode, + py, + quote! { + ::std::convert::TryInto::<#ty>::try_into(#ident).map_err(|e| _pyo3::exceptions::PyValueError::new_err(e.to_string())) + }, + ); + quote! { + let #ident = #extract; + } + } // Just pass other types through unmodified - Ty::PyBuffer | Ty::Int | Ty::PyHashT | Ty::PySsizeT | Ty::Void => quote! {}, + Ty::PyBuffer | Ty::Int | Ty::PyHashT | Ty::Void => quote! {}, } } } @@ -868,9 +954,11 @@ impl SlotDef { unsafe extern "C" fn __wrap(_raw_slf: *mut _pyo3::ffi::PyObject, #(#method_arguments),*) -> #ret_ty { let _slf = _raw_slf; #before_call_method - _pyo3::callback::handle_panic(|#py| { + let gil = _pyo3::GILPool::new(); + let #py = gil.python(); + _pyo3::callback::panic_result_into_callback_output(#py, ::std::panic::catch_unwind(move || -> _pyo3::PyResult<_> { #body - }) + })) } _pyo3::ffi::PyType_Slot { slot: _pyo3::ffi::#slot, @@ -959,7 +1047,7 @@ impl SlotFragmentDef { let body = generate_method_body(cls, spec, &py, arguments, *extract_error_mode, None)?; let ret_ty = ret_ty.ffi_type(); Ok(quote! { - impl _pyo3::class::impl_::#fragment_trait<#cls> for _pyo3::class::impl_::PyClassImplCollector<#cls> { + impl _pyo3::impl_::pyclass::#fragment_trait<#cls> for _pyo3::impl_::pyclass::PyClassImplCollector<#cls> { #[inline] unsafe fn #method( @@ -1027,46 +1115,6 @@ const __RPOW__: SlotFragmentDef = SlotFragmentDef::new("__rpow__", &[Ty::Object, .extract_error_mode(ExtractErrorMode::NotImplemented) .ret_ty(Ty::Object); -fn pyproto_fragment(method_name: &str) -> Option<&'static SlotFragmentDef> { - match method_name { - "__setattr__" => Some(&__SETATTR__), - "__delattr__" => Some(&__DELATTR__), - "__set__" => Some(&__SET__), - "__delete__" => Some(&__DELETE__), - "__setitem__" => Some(&__SETITEM__), - "__delitem__" => Some(&__DELITEM__), - "__add__" => Some(&__ADD__), - "__radd__" => Some(&__RADD__), - "__sub__" => Some(&__SUB__), - "__rsub__" => Some(&__RSUB__), - "__mul__" => Some(&__MUL__), - "__rmul__" => Some(&__RMUL__), - "__matmul__" => Some(&__MATMUL__), - "__rmatmul__" => Some(&__RMATMUL__), - "__floordiv__" => Some(&__FLOORDIV__), - "__rfloordiv__" => Some(&__RFLOORDIV__), - "__truediv__" => Some(&__TRUEDIV__), - "__rtruediv__" => Some(&__RTRUEDIV__), - "__divmod__" => Some(&__DIVMOD__), - "__rdivmod__" => Some(&__RDIVMOD__), - "__mod__" => Some(&__MOD__), - "__rmod__" => Some(&__RMOD__), - "__lshift__" => Some(&__LSHIFT__), - "__rlshift__" => Some(&__RLSHIFT__), - "__rshift__" => Some(&__RSHIFT__), - "__rrshift__" => Some(&__RRSHIFT__), - "__and__" => Some(&__AND__), - "__rand__" => Some(&__RAND__), - "__xor__" => Some(&__XOR__), - "__rxor__" => Some(&__RXOR__), - "__or__" => Some(&__OR__), - "__ror__" => Some(&__ROR__), - "__pow__" => Some(&__POW__), - "__rpow__" => Some(&__RPOW__), - _ => None, - } -} - fn extract_proto_arguments( cls: &syn::Type, py: &syn::Ident, diff --git a/pyo3-macros-backend/src/pyproto.rs b/pyo3-macros-backend/src/pyproto.rs index 198fb517..67e68ac9 100644 --- a/pyo3-macros-backend/src/pyproto.rs +++ b/pyo3-macros-backend/src/pyproto.rs @@ -84,55 +84,30 @@ fn impl_proto_impl( } } } - let normal_methods = impl_normal_methods(py_methods, ty, proto); - let protocol_methods = impl_proto_methods(method_names, ty, proto); + let items = impl_proto_items(method_names, py_methods, ty, proto); Ok(quote! { const _: () = { use ::pyo3 as _pyo3; // pyproto doesn't support specifying #[pyo3(crate)] #trait_impls - #normal_methods - #protocol_methods + #items }; }) } -fn impl_normal_methods( +fn impl_proto_items( + method_names: HashSet, py_methods: Vec, ty: &syn::Type, proto: &defs::Proto, ) -> TokenStream { - if py_methods.is_empty() { - return TokenStream::default(); - } - - let methods_trait = proto.methods_trait(); - let methods_trait_methods = proto.methods_trait_methods(); - quote! { - impl _pyo3::class::impl_::#methods_trait<#ty> - for _pyo3::class::impl_::PyClassImplCollector<#ty> - { - fn #methods_trait_methods(self) -> &'static [_pyo3::class::methods::PyMethodDefType] { - static METHODS: &[_pyo3::class::methods::PyMethodDefType] = - &[#(#py_methods),*]; - METHODS - } - } - } -} - -fn impl_proto_methods( - method_names: HashSet, - ty: &syn::Type, - proto: &defs::Proto, -) -> TokenStream { - if method_names.is_empty() { + if method_names.is_empty() && py_methods.is_empty() { return TokenStream::default(); } let module = proto.module(); - let slots_trait = proto.slots_trait(); - let slots_trait_slots = proto.slots_trait_slots(); + let items_trait = proto.items_trait(); + let items_trait_items = proto.items_trait_items(); let mut tokens = proto .slot_defs(method_names) @@ -153,11 +128,15 @@ fn impl_proto_methods( } quote! { - impl _pyo3::class::impl_::#slots_trait<#ty> - for _pyo3::class::impl_::PyClassImplCollector<#ty> + impl _pyo3::impl_::pyclass::#items_trait<#ty> + for _pyo3::impl_::pyclass::PyClassImplCollector<#ty> { - fn #slots_trait_slots(self) -> &'static [_pyo3::ffi::PyType_Slot] { - &[#(#tokens),*] + fn #items_trait_items(self) -> &'static _pyo3::impl_::pyclass::PyClassItems { + static ITEMS: _pyo3::impl_::pyclass::PyClassItems = _pyo3::impl_::pyclass::PyClassItems { + methods: &[#(#py_methods),*], + slots: &[#(#tokens),*] + }; + &ITEMS } } } diff --git a/pyo3-macros/Cargo.toml b/pyo3-macros/Cargo.toml index 6f55d41a..a12d7d7c 100644 --- a/pyo3-macros/Cargo.toml +++ b/pyo3-macros/Cargo.toml @@ -16,6 +16,8 @@ proc-macro = true [features] multiple-pymethods = [] +pyproto = ["pyo3-macros-backend/pyproto"] + [dependencies] proc-macro2 = { version = "1", default-features = false } quote = "1" diff --git a/pyo3-macros/src/lib.rs b/pyo3-macros/src/lib.rs index c03d3c86..06738053 100644 --- a/pyo3-macros/src/lib.rs +++ b/pyo3-macros/src/lib.rs @@ -9,9 +9,8 @@ use proc_macro::TokenStream; use proc_macro2::TokenStream as TokenStream2; use pyo3_macros_backend::{ build_derive_from_pyobject, build_py_class, build_py_enum, build_py_function, build_py_methods, - build_py_proto, get_doc, process_functions_in_module, pymodule_impl, wrap_pyfunction_impl, - wrap_pymodule_impl, PyClassArgs, PyClassMethodsType, PyFunctionOptions, PyModuleOptions, - WrapPyFunctionArgs, + get_doc, process_functions_in_module, pymodule_impl, wrap_pyfunction_impl, wrap_pymodule_impl, + PyClassArgs, PyClassMethodsType, PyFunctionOptions, PyModuleOptions, WrapPyFunctionArgs, }; use quote::quote; use syn::{parse::Nothing, parse_macro_input}; @@ -68,9 +67,10 @@ pub fn pymodule(args: TokenStream, input: TokenStream) -> TokenStream { /// [4]: ../class/gc/trait.PyGCProtocol.html /// [5]: ../class/iter/trait.PyIterProtocol.html #[proc_macro_attribute] +#[cfg(feature = "pyproto")] pub fn pyproto(_: TokenStream, input: TokenStream) -> TokenStream { let mut ast = parse_macro_input!(input as syn::ItemImpl); - let expanded = build_py_proto(&mut ast).unwrap_or_compile_error(); + let expanded = pyo3_macros_backend::build_py_proto(&mut ast).unwrap_or_compile_error(); quote!( #ast @@ -79,19 +79,18 @@ pub fn pyproto(_: TokenStream, input: TokenStream) -> TokenStream { .into() } -/// A proc macro used to expose Rust structs as Python objects. +/// A proc macro used to expose Rust structs and fieldless enums as Python objects. /// /// `#[pyclass]` accepts the following [parameters][2]: /// /// | Parameter | Description | /// | :- | :- | /// | `name = "python_name"` | Sets the name that Python sees this class as. Defaults to the name of the Rust struct. | -/// | `freelist = N` | Implements a [free list][10] of size N. This can improve performance for types that are often created and deleted in quick succession. Profile your code to see whether `freelist` is right for you. | -/// | `gc` | Participate in Python's [garbage collection][5]. Required if your type contains references to other Python objects. If you don't (or incorrectly) implement this, contained Python objects may be hidden from Python's garbage collector and you may leak memory. Note that leaking memory, while undesirable, [is safe behavior][7].| +/// | `freelist = N` | Implements a [free list][9] of size N. This can improve performance for types that are often created and deleted in quick succession. Profile your code to see whether `freelist` is right for you. | /// | `weakref` | Allows this class to be [weakly referenceable][6]. | /// | `extends = BaseType` | Use a custom baseclass. Defaults to [`PyAny`][4] | -/// | `subclass` | Allows other Python classes and `#[pyclass]` to inherit from this class. | -/// | `unsendable` | Required if your struct is not [`Send`][3]. Rather than using `unsendable`, consider implementing your struct in a threadsafe way by e.g. substituting [`Rc`][8] with [`Arc`][9]. By using `unsendable`, your class will panic when accessed by another thread.| +/// | `subclass` | Allows other Python classes and `#[pyclass]` to inherit from this class. Enums cannot be subclassed. | +/// | `unsendable` | Required if your struct is not [`Send`][3]. Rather than using `unsendable`, consider implementing your struct in a threadsafe way by e.g. substituting [`Rc`][7] with [`Arc`][8]. By using `unsendable`, your class will panic when accessed by another thread.| /// | `module = "module_name"` | Python code will see the class as being defined in this module. Defaults to `builtins`. | /// /// For more on creating Python classes, @@ -103,10 +102,9 @@ pub fn pyproto(_: TokenStream, input: TokenStream) -> TokenStream { /// [4]: ../prelude/struct.PyAny.html /// [5]: https://pyo3.rs/latest/class/protocols.html#garbage-collector-integration /// [6]: https://docs.python.org/3/library/weakref.html -/// [7]: https://doc.rust-lang.org/nomicon/leaking.html -/// [8]: std::rc::Rc -/// [9]: std::sync::Arc -/// [10]: https://en.wikipedia.org/wiki/Free_list +/// [7]: std::rc::Rc +/// [8]: std::sync::Arc +/// [9]: https://en.wikipedia.org/wiki/Free_list #[proc_macro_attribute] pub fn pyclass(attr: TokenStream, input: TokenStream) -> TokenStream { use syn::Item; diff --git a/pytests/pyo3-pytests/Cargo.toml b/pytests/Cargo.toml similarity index 82% rename from pytests/pyo3-pytests/Cargo.toml rename to pytests/Cargo.toml index 8fdcda88..1e387ca2 100644 --- a/pytests/pyo3-pytests/Cargo.toml +++ b/pytests/Cargo.toml @@ -6,10 +6,10 @@ description = "Python-based tests for PyO3" edition = "2018" [dependencies] -pyo3 = { path = "../../", features = ["extension-module"] } +pyo3 = { path = "../", features = ["extension-module"] } [build-dependencies] -pyo3-build-config = { path = "../../pyo3-build-config" } +pyo3-build-config = { path = "../pyo3-build-config" } [lib] name = "pyo3_pytests" diff --git a/pytests/pyo3-pytests/MANIFEST.in b/pytests/MANIFEST.in similarity index 100% rename from pytests/pyo3-pytests/MANIFEST.in rename to pytests/MANIFEST.in diff --git a/pytests/README.md b/pytests/README.md index 7db3fabe..7ced072a 100644 --- a/pytests/README.md +++ b/pytests/README.md @@ -1,10 +1,34 @@ -# PyO3 Python tests +# pyo3-pytests -These crates are a collection of test extension modules built with PyO3. They are all tested using `nox` in PyO3's CI. +An extension module built using PyO3, used to test and benchmark PyO3 from Python. -Below is a brief description of each of these: +## Testing -| Example | Description | -| ------- | ----------- | -| `pyo3-benchmarks` | A project containing some benchmarks of PyO3 functionality called from Python. | -| `pyo3-pytests` | A project containing some tests of PyO3 functionality called from Python. | +This package is intended to be built using `maturin`. Once built, you can run the tests using `pytest`: + +```shell +pip install maturin +maturin develop +pytest +``` + +Alternatively, install nox and run the tests inside an isolated environment: + +```shell +nox +``` + +## Running benchmarks + +You can install the module in your Python environment and then run the benchmarks with pytest: + +```shell +pip install . +pytest --benchmark-enable +``` + +Or with nox: + +```shell +nox -s bench +``` diff --git a/pytests/build.rs b/pytests/build.rs new file mode 100644 index 00000000..90a1aa32 --- /dev/null +++ b/pytests/build.rs @@ -0,0 +1,4 @@ +fn main() { + pyo3_build_config::use_pyo3_cfgs(); + pyo3_build_config::add_extension_module_link_args(); +} diff --git a/pytests/pyo3-benchmarks/noxfile.py b/pytests/noxfile.py similarity index 79% rename from pytests/pyo3-benchmarks/noxfile.py rename to pytests/noxfile.py index 39230d26..22d0691f 100644 --- a/pytests/pyo3-benchmarks/noxfile.py +++ b/pytests/noxfile.py @@ -15,4 +15,4 @@ def test(session): def bench(session): session.install("-rrequirements-dev.txt") session.install(".") - session.run("pytest", "--benchmark-enable") + session.run("pytest", "--benchmark-enable", "--benchmark-only", *session.posargs) diff --git a/pytests/pyo3-benchmarks/Cargo.toml b/pytests/pyo3-benchmarks/Cargo.toml deleted file mode 100644 index 56305b66..00000000 --- a/pytests/pyo3-benchmarks/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -authors = ["PyO3 Authors"] -name = "pyo3-benchmarks" -version = "0.1.0" -description = "Python-based benchmarks for various PyO3 functionality" -edition = "2018" - -[dependencies] - -[dependencies.pyo3] -path = "../../" -features = ["extension-module"] - -[lib] -name = "pyo3_benchmarks" -crate-type = ["cdylib"] diff --git a/pytests/pyo3-benchmarks/MANIFEST.in b/pytests/pyo3-benchmarks/MANIFEST.in deleted file mode 100644 index 70181be8..00000000 --- a/pytests/pyo3-benchmarks/MANIFEST.in +++ /dev/null @@ -1,3 +0,0 @@ -include Cargo.toml -recursive-include src * -recursive-include tests diff --git a/pytests/pyo3-benchmarks/README.md b/pytests/pyo3-benchmarks/README.md deleted file mode 100644 index 2d7c33ea..00000000 --- a/pytests/pyo3-benchmarks/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# pyo3-benchmarks - -This extension module contains benchmarks for pieces of PyO3's API accessible from Python. - -## Running the benchmarks - -You can install the module in your Python environment and then run the benchmarks with pytest: - -```shell -pip install . -pytest --benchmark-enable -``` - -Or with nox: - -```shell -nox -s bench -``` diff --git a/pytests/pyo3-benchmarks/requirements-dev.txt b/pytests/pyo3-benchmarks/requirements-dev.txt deleted file mode 100644 index 8005ca5e..00000000 --- a/pytests/pyo3-benchmarks/requirements-dev.txt +++ /dev/null @@ -1,4 +0,0 @@ -pytest>=3.5.0 -setuptools_rust~=1.0.0 -pytest-benchmark~=3.2 -pip>=21.3 diff --git a/pytests/pyo3-pytests/README.md b/pytests/pyo3-pytests/README.md deleted file mode 100644 index ddb9573b..00000000 --- a/pytests/pyo3-pytests/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# pyo3-pytests - -An extension module built using PyO3, used to test PyO3 from Python. - -## Testing - -This package is intended to be built using `maturin`. Once built, you can run the tests using `pytest`: - -```shell -pip install maturin -maturin develop -pytest -``` - -Alternatively, install nox and run the tests inside an isolated environment: - -```shell -nox -``` diff --git a/pytests/pyo3-pytests/build.rs b/pytests/pyo3-pytests/build.rs deleted file mode 100644 index 0475124b..00000000 --- a/pytests/pyo3-pytests/build.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn main() { - pyo3_build_config::use_pyo3_cfgs(); -} diff --git a/pytests/pyo3-pytests/noxfile.py b/pytests/pyo3-pytests/noxfile.py deleted file mode 100644 index 17a6b80f..00000000 --- a/pytests/pyo3-pytests/noxfile.py +++ /dev/null @@ -1,9 +0,0 @@ -import nox - - -@nox.session -def python(session): - session.install("-rrequirements-dev.txt") - session.install("maturin") - session.run_always("maturin", "develop") - session.run("pytest") diff --git a/pytests/pyo3-pytests/pyproject.toml b/pytests/pyo3-pytests/pyproject.toml deleted file mode 100644 index 256b3705..00000000 --- a/pytests/pyo3-pytests/pyproject.toml +++ /dev/null @@ -1,3 +0,0 @@ -[build-system] -requires = ["maturin>=0.12,<0.13"] -build-backend = "maturin" diff --git a/pytests/pyo3-pytests/requirements-dev.txt b/pytests/pyo3-pytests/requirements-dev.txt deleted file mode 100644 index cc06de69..00000000 --- a/pytests/pyo3-pytests/requirements-dev.txt +++ /dev/null @@ -1,4 +0,0 @@ -hypothesis>=3.55 -pytest>=3.5.0 -psutil>=5.6 -pip>=21.3 diff --git a/pytests/pyo3-pytests/src/pyclass_iter.rs b/pytests/pyo3-pytests/src/pyclass_iter.rs deleted file mode 100644 index bb09e260..00000000 --- a/pytests/pyo3-pytests/src/pyclass_iter.rs +++ /dev/null @@ -1,34 +0,0 @@ -use pyo3::class::iter::{IterNextOutput, PyIterProtocol}; -use pyo3::prelude::*; - -/// This is for demonstrating how to return a value from __next__ -#[pyclass] -struct PyClassIter { - count: usize, -} - -#[pymethods] -impl PyClassIter { - #[new] - pub fn new() -> Self { - PyClassIter { count: 0 } - } -} - -#[pyproto] -impl PyIterProtocol for PyClassIter { - fn __next__(mut slf: PyRefMut) -> IterNextOutput { - if slf.count < 5 { - slf.count += 1; - IterNextOutput::Yield(slf.count) - } else { - IterNextOutput::Return("Ended") - } - } -} - -#[pymodule] -pub fn pyclass_iter(_py: Python, m: &PyModule) -> PyResult<()> { - m.add_class::()?; - Ok(()) -} diff --git a/pytests/pyo3-pytests/pyo3_pytests/__init__.py b/pytests/pyo3_pytests/__init__.py similarity index 100% rename from pytests/pyo3-pytests/pyo3_pytests/__init__.py rename to pytests/pyo3_pytests/__init__.py diff --git a/pytests/pyo3-benchmarks/pyproject.toml b/pytests/pyproject.toml similarity index 100% rename from pytests/pyo3-benchmarks/pyproject.toml rename to pytests/pyproject.toml diff --git a/pytests/requirements-dev.txt b/pytests/requirements-dev.txt new file mode 100644 index 00000000..d1fa0541 --- /dev/null +++ b/pytests/requirements-dev.txt @@ -0,0 +1,4 @@ +hypothesis>=3.55 +pytest>=6.0 +pytest-benchmark>=3.4 +psutil>=5.6 diff --git a/pytests/pyo3-pytests/src/buf_and_str.rs b/pytests/src/buf_and_str.rs similarity index 100% rename from pytests/pyo3-pytests/src/buf_and_str.rs rename to pytests/src/buf_and_str.rs diff --git a/pytests/pyo3-pytests/src/datetime.rs b/pytests/src/datetime.rs similarity index 100% rename from pytests/pyo3-pytests/src/datetime.rs rename to pytests/src/datetime.rs diff --git a/pytests/pyo3-pytests/src/dict_iter.rs b/pytests/src/dict_iter.rs similarity index 100% rename from pytests/pyo3-pytests/src/dict_iter.rs rename to pytests/src/dict_iter.rs diff --git a/pytests/pyo3-pytests/src/lib.rs b/pytests/src/lib.rs similarity index 83% rename from pytests/pyo3-pytests/src/lib.rs rename to pytests/src/lib.rs index b435e668..09549142 100644 --- a/pytests/pyo3-pytests/src/lib.rs +++ b/pytests/src/lib.rs @@ -9,7 +9,8 @@ pub mod misc; pub mod objstore; pub mod othermod; pub mod path; -pub mod pyclass_iter; +pub mod pyclasses; +pub mod pyfunctions; pub mod subclassing; #[pymodule] @@ -23,7 +24,8 @@ fn pyo3_pytests(py: Python, m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pymodule!(objstore::objstore))?; m.add_wrapped(wrap_pymodule!(othermod::othermod))?; m.add_wrapped(wrap_pymodule!(path::path))?; - m.add_wrapped(wrap_pymodule!(pyclass_iter::pyclass_iter))?; + m.add_wrapped(wrap_pymodule!(pyclasses::pyclasses))?; + m.add_wrapped(wrap_pymodule!(pyfunctions::pyfunctions))?; m.add_wrapped(wrap_pymodule!(subclassing::subclassing))?; // Inserting to sys.modules allows importing submodules nicely from Python @@ -38,7 +40,8 @@ fn pyo3_pytests(py: Python, m: &PyModule) -> PyResult<()> { sys_modules.set_item("pyo3_pytests.objstore", m.getattr("objstore")?)?; sys_modules.set_item("pyo3_pytests.othermod", m.getattr("othermod")?)?; sys_modules.set_item("pyo3_pytests.path", m.getattr("path")?)?; - sys_modules.set_item("pyo3_pytests.pyclass_iter", m.getattr("pyclass_iter")?)?; + sys_modules.set_item("pyo3_pytests.pyclasses", m.getattr("pyclasses")?)?; + sys_modules.set_item("pyo3_pytests.pyfunctions", m.getattr("pyfunctions")?)?; sys_modules.set_item("pyo3_pytests.subclassing", m.getattr("subclassing")?)?; Ok(()) diff --git a/pytests/pyo3-pytests/src/misc.rs b/pytests/src/misc.rs similarity index 100% rename from pytests/pyo3-pytests/src/misc.rs rename to pytests/src/misc.rs diff --git a/pytests/pyo3-pytests/src/objstore.rs b/pytests/src/objstore.rs similarity index 100% rename from pytests/pyo3-pytests/src/objstore.rs rename to pytests/src/objstore.rs diff --git a/pytests/pyo3-pytests/src/othermod.rs b/pytests/src/othermod.rs similarity index 93% rename from pytests/pyo3-pytests/src/othermod.rs rename to pytests/src/othermod.rs index e439cacf..763d38a8 100644 --- a/pytests/pyo3-pytests/src/othermod.rs +++ b/pytests/src/othermod.rs @@ -1,4 +1,4 @@ -//! https://github.com/PyO3/pyo3/issues/233 +//! //! //! The code below just tries to use the most important code generation paths diff --git a/pytests/pyo3-pytests/src/path.rs b/pytests/src/path.rs similarity index 100% rename from pytests/pyo3-pytests/src/path.rs rename to pytests/src/path.rs diff --git a/pytests/src/pyclasses.rs b/pytests/src/pyclasses.rs new file mode 100644 index 00000000..362ce4a7 --- /dev/null +++ b/pytests/src/pyclasses.rs @@ -0,0 +1,43 @@ +use pyo3::iter::IterNextOutput; +use pyo3::prelude::*; + +#[pyclass] +struct EmptyClass {} + +#[pymethods] +impl EmptyClass { + #[new] + fn new() -> Self { + EmptyClass {} + } +} + +/// This is for demonstrating how to return a value from __next__ +#[pyclass] +struct PyClassIter { + count: usize, +} + +#[pymethods] +impl PyClassIter { + #[new] + pub fn new() -> Self { + PyClassIter { count: 0 } + } + + fn __next__(&mut self) -> IterNextOutput { + if self.count < 5 { + self.count += 1; + IterNextOutput::Yield(self.count) + } else { + IterNextOutput::Return("Ended") + } + } +} + +#[pymodule] +pub fn pyclasses(_py: Python<'_>, m: &PyModule) -> PyResult<()> { + m.add_class::()?; + m.add_class::()?; + Ok(()) +} diff --git a/pytests/pyo3-benchmarks/src/lib.rs b/pytests/src/pyfunctions.rs similarity index 87% rename from pytests/pyo3-benchmarks/src/lib.rs rename to pytests/src/pyfunctions.rs index 1f858091..a259aea0 100644 --- a/pytests/pyo3-benchmarks/src/lib.rs +++ b/pytests/src/pyfunctions.rs @@ -54,25 +54,13 @@ fn args_kwargs<'a>( (args, kwargs) } -#[pyclass] -struct EmptyClass {} - -#[pymethods] -impl EmptyClass { - #[new] - fn new() -> Self { - EmptyClass {} - } -} - #[pymodule] -fn pyo3_benchmarks(_py: Python<'_>, m: &PyModule) -> PyResult<()> { +pub fn pyfunctions(_py: Python<'_>, m: &PyModule) -> PyResult<()> { m.add_function(wrap_pyfunction!(none, m)?)?; m.add_function(wrap_pyfunction!(simple, m)?)?; m.add_function(wrap_pyfunction!(simple_args, m)?)?; m.add_function(wrap_pyfunction!(simple_kwargs, m)?)?; m.add_function(wrap_pyfunction!(simple_args_kwargs, m)?)?; m.add_function(wrap_pyfunction!(args_kwargs, m)?)?; - m.add_class::()?; Ok(()) } diff --git a/pytests/pyo3-pytests/src/subclassing.rs b/pytests/src/subclassing.rs similarity index 88% rename from pytests/pyo3-pytests/src/subclassing.rs rename to pytests/src/subclassing.rs index d54bbc4d..8fd111e0 100644 --- a/pytests/pyo3-pytests/src/subclassing.rs +++ b/pytests/src/subclassing.rs @@ -11,10 +11,7 @@ impl Subclassable { fn new() -> Self { Subclassable {} } -} -#[pyproto] -impl pyo3::PyObjectProtocol for Subclassable { fn __str__(&self) -> PyResult<&'static str> { Ok("Subclassable") } diff --git a/pytests/pyo3-pytests/tests/test_buf_and_str.py b/pytests/tests/test_buf_and_str.py similarity index 100% rename from pytests/pyo3-pytests/tests/test_buf_and_str.py rename to pytests/tests/test_buf_and_str.py diff --git a/pytests/pyo3-pytests/tests/test_datetime.py b/pytests/tests/test_datetime.py similarity index 100% rename from pytests/pyo3-pytests/tests/test_datetime.py rename to pytests/tests/test_datetime.py diff --git a/pytests/pyo3-pytests/tests/test_dict_iter.py b/pytests/tests/test_dict_iter.py similarity index 100% rename from pytests/pyo3-pytests/tests/test_dict_iter.py rename to pytests/tests/test_dict_iter.py diff --git a/pytests/pyo3-pytests/tests/test_misc.py b/pytests/tests/test_misc.py similarity index 100% rename from pytests/pyo3-pytests/tests/test_misc.py rename to pytests/tests/test_misc.py diff --git a/pytests/pyo3-pytests/tests/test_objstore.py b/pytests/tests/test_objstore.py similarity index 100% rename from pytests/pyo3-pytests/tests/test_objstore.py rename to pytests/tests/test_objstore.py diff --git a/pytests/pyo3-pytests/tests/test_othermod.py b/pytests/tests/test_othermod.py similarity index 85% rename from pytests/pyo3-pytests/tests/test_othermod.py rename to pytests/tests/test_othermod.py index 08ac367a..ff67bba4 100644 --- a/pytests/pyo3-pytests/tests/test_othermod.py +++ b/pytests/tests/test_othermod.py @@ -3,14 +3,14 @@ from hypothesis import strategies as st from pyo3_pytests import othermod -INTEGER32_ST = st.integers(min_value=(-(2 ** 31)), max_value=(2 ** 31 - 1)) +INTEGER32_ST = st.integers(min_value=(-(2**31)), max_value=(2**31 - 1)) USIZE_ST = st.integers(min_value=othermod.USIZE_MIN, max_value=othermod.USIZE_MAX) @given(x=INTEGER32_ST) def test_double(x): expected = x * 2 - assume(-(2 ** 31) <= expected <= (2 ** 31 - 1)) + assume(-(2**31) <= expected <= (2**31 - 1)) assert othermod.double(x) == expected diff --git a/pytests/pyo3-pytests/tests/test_path.py b/pytests/tests/test_path.py similarity index 100% rename from pytests/pyo3-pytests/tests/test_path.py rename to pytests/tests/test_path.py diff --git a/pytests/pyo3-pytests/tests/test_pyclass_iter.py b/pytests/tests/test_pyclasses.py similarity index 51% rename from pytests/pyo3-pytests/tests/test_pyclass_iter.py rename to pytests/tests/test_pyclasses.py index e7d7d47c..c8697b4e 100644 --- a/pytests/pyo3-pytests/tests/test_pyclass_iter.py +++ b/pytests/tests/test_pyclasses.py @@ -1,9 +1,21 @@ import pytest -from pyo3_pytests import pyclass_iter +from pyo3_pytests import pyclasses + + +def test_empty_class_init(benchmark): + benchmark(pyclasses.EmptyClass) + + +class EmptyClassPy: + pass + + +def test_empty_class_init_py(benchmark): + benchmark(EmptyClassPy) def test_iter(): - i = pyclass_iter.PyClassIter() + i = pyclasses.PyClassIter() assert next(i) == 1 assert next(i) == 2 assert next(i) == 3 diff --git a/pytests/pyo3-benchmarks/tests/test_benchmarks.py b/pytests/tests/test_pyfunctions.py similarity index 63% rename from pytests/pyo3-benchmarks/tests/test_benchmarks.py rename to pytests/tests/test_pyfunctions.py index c00b8583..1dda2528 100644 --- a/pytests/pyo3-benchmarks/tests/test_benchmarks.py +++ b/pytests/tests/test_pyfunctions.py @@ -1,4 +1,4 @@ -import pyo3_benchmarks +from pyo3_pytests import pyfunctions def none_py(): @@ -10,10 +10,10 @@ def test_none_py(benchmark): def test_none_rs(benchmark): - rust = pyo3_benchmarks.none() + rust = pyfunctions.none() py = none_py() assert rust == py - benchmark(pyo3_benchmarks.none) + benchmark(pyfunctions.none) def simple_py(a, b="bar", *, c=None): @@ -25,10 +25,10 @@ def test_simple_py(benchmark): def test_simple_rs(benchmark): - rust = pyo3_benchmarks.simple(1, "foo", c={1: 2}) + rust = pyfunctions.simple(1, "foo", c={1: 2}) py = simple_py(1, "foo", c={1: 2}) assert rust == py - benchmark(pyo3_benchmarks.simple, 1, "foo", c={1: 2}) + benchmark(pyfunctions.simple, 1, "foo", c={1: 2}) def simple_args_py(a, b="bar", *args, c=None): @@ -40,10 +40,10 @@ def test_simple_args_py(benchmark): def test_simple_args_rs(benchmark): - rust = pyo3_benchmarks.simple_args(1, "foo", 4, 5, 6, c={1: 2}) + rust = pyfunctions.simple_args(1, "foo", 4, 5, 6, c={1: 2}) py = simple_args_py(1, "foo", 4, 5, 6, c={1: 2}) assert rust == py - benchmark(pyo3_benchmarks.simple_args, 1, "foo", 4, 5, 6, c={1: 2}) + benchmark(pyfunctions.simple_args, 1, "foo", 4, 5, 6, c={1: 2}) def simple_kwargs_py(a, b="bar", c=None, **kwargs): @@ -55,10 +55,10 @@ def test_simple_kwargs_py(benchmark): def test_simple_kwargs_rs(benchmark): - rust = pyo3_benchmarks.simple_kwargs(1, "foo", c={1: 2}, bar=4, foo=10) + rust = pyfunctions.simple_kwargs(1, "foo", c={1: 2}, bar=4, foo=10) py = simple_kwargs_py(1, "foo", c={1: 2}, bar=4, foo=10) assert rust == py - benchmark(pyo3_benchmarks.simple_kwargs, 1, "foo", c={1: 2}, bar=4, foo=10) + benchmark(pyfunctions.simple_kwargs, 1, "foo", c={1: 2}, bar=4, foo=10) def simple_args_kwargs_py(a, b="bar", *args, c=None, **kwargs): @@ -70,10 +70,10 @@ def test_simple_args_kwargs_py(benchmark): def test_simple_args_kwargs_rs(benchmark): - rust = pyo3_benchmarks.simple_args_kwargs(1, "foo", "baz", bar=4, foo=10) + rust = pyfunctions.simple_args_kwargs(1, "foo", "baz", bar=4, foo=10) py = simple_args_kwargs_py(1, "foo", "baz", bar=4, foo=10) assert rust == py - benchmark(pyo3_benchmarks.simple_args_kwargs, 1, "foo", "baz", bar=4, foo=10) + benchmark(pyfunctions.simple_args_kwargs, 1, "foo", "baz", bar=4, foo=10) def args_kwargs_py(*args, **kwargs): @@ -85,19 +85,7 @@ def test_args_kwargs_py(benchmark): def test_args_kwargs_rs(benchmark): - rust = pyo3_benchmarks.args_kwargs(1, "foo", {1: 2}, bar=4, foo=10) + rust = pyfunctions.args_kwargs(1, "foo", {1: 2}, bar=4, foo=10) py = args_kwargs_py(1, "foo", {1: 2}, bar=4, foo=10) assert rust == py - benchmark(pyo3_benchmarks.args_kwargs, 1, "foo", {1: 2}, a=4, foo=10) - - -def test_empty_class_init(benchmark): - benchmark(pyo3_benchmarks.EmptyClass) - - -class EmptyClassPy: - pass - - -def test_empty_class_init_py(benchmark): - benchmark(EmptyClassPy) + benchmark(pyfunctions.args_kwargs, 1, "foo", {1: 2}, a=4, foo=10) diff --git a/pytests/pyo3-pytests/tests/test_subclassing.py b/pytests/tests/test_subclassing.py similarity index 100% rename from pytests/pyo3-pytests/tests/test_subclassing.py rename to pytests/tests/test_subclassing.py diff --git a/src/buffer.rs b/src/buffer.rs index 0c97865d..dbbf27dc 100644 --- a/src/buffer.rs +++ b/src/buffer.rs @@ -1,4 +1,4 @@ -#![cfg(not(Py_LIMITED_API))] +#![cfg(any(not(Py_LIMITED_API), Py_3_11))] // Copyright (c) 2017 Daniel Grunwald // // Permission is hereby granted, free of charge, to any person obtaining a copy of this @@ -235,8 +235,20 @@ impl PyBuffer { } unsafe { ffi::PyBuffer_GetPointer( - &*self.0 as *const ffi::Py_buffer as *mut ffi::Py_buffer, - indices.as_ptr() as *mut usize as *mut ffi::Py_ssize_t, + #[cfg(Py_3_11)] + &*self.0, + #[cfg(not(Py_3_11))] + { + &*self.0 as *const ffi::Py_buffer as *mut ffi::Py_buffer + }, + #[cfg(Py_3_11)] + { + indices.as_ptr() as *const ffi::Py_ssize_t + }, + #[cfg(not(Py_3_11))] + { + indices.as_ptr() as *mut ffi::Py_ssize_t + }, ) } } @@ -477,7 +489,12 @@ impl PyBuffer { py, ffi::PyBuffer_ToContiguous( target.as_ptr() as *mut raw::c_void, - &*self.0 as *const ffi::Py_buffer as *mut ffi::Py_buffer, + #[cfg(Py_3_11)] + &*self.0, + #[cfg(not(Py_3_11))] + { + &*self.0 as *const ffi::Py_buffer as *mut ffi::Py_buffer + }, self.0.len, fort as std::os::raw::c_char, ), @@ -510,8 +527,13 @@ impl PyBuffer { err::error_on_minusone( py, ffi::PyBuffer_ToContiguous( - vec.as_mut_ptr() as *mut raw::c_void, - &*self.0 as *const ffi::Py_buffer as *mut ffi::Py_buffer, + vec.as_ptr() as *mut raw::c_void, + #[cfg(Py_3_11)] + &*self.0, + #[cfg(not(Py_3_11))] + { + &*self.0 as *const ffi::Py_buffer as *mut ffi::Py_buffer + }, self.0.len, fort as std::os::raw::c_char, ), @@ -564,8 +586,20 @@ impl PyBuffer { err::error_on_minusone( py, ffi::PyBuffer_FromContiguous( - &*self.0 as *const ffi::Py_buffer as *mut ffi::Py_buffer, - source.as_ptr() as *mut raw::c_void, + #[cfg(Py_3_11)] + &*self.0, + #[cfg(not(Py_3_11))] + { + &*self.0 as *const ffi::Py_buffer as *mut ffi::Py_buffer + }, + #[cfg(Py_3_11)] + { + source.as_ptr() as *const raw::c_void + }, + #[cfg(not(Py_3_11))] + { + source.as_ptr() as *mut raw::c_void + }, self.0.len, fort as std::os::raw::c_char, ), @@ -678,6 +712,8 @@ mod tests { assert_eq!(slice[0].get(), b'a'); assert_eq!(slice[2].get(), b'c'); + assert_eq!(unsafe { *(buffer.get_ptr(&[1]) as *mut u8) }, b'b'); + assert!(buffer.copy_to_slice(py, &mut [0u8]).is_err()); let mut arr = [0; 5]; buffer.copy_to_slice(py, &mut arr).unwrap(); diff --git a/src/callback.rs b/src/callback.rs index 4a8598f3..e32def03 100644 --- a/src/callback.rs +++ b/src/callback.rs @@ -245,7 +245,11 @@ where panic_result_into_callback_output(py, panic::catch_unwind(move || -> PyResult<_> { body(py) })) } -fn panic_result_into_callback_output( +/// Converts the output of std::panic::catch_unwind into a Python function output, either by raising a Python +/// exception or by unwrapping the contained success output. +#[doc(hidden)] +#[inline] +pub fn panic_result_into_callback_output( py: Python, panic_result: Result, Box>, ) -> R @@ -262,3 +266,18 @@ where R::ERR_VALUE }) } + +/// Aborts if panic has occurred. Used inside `__traverse__` implementations, where panicking is not possible. +#[doc(hidden)] +#[inline] +pub fn abort_on_traverse_panic( + panic_result: Result>, +) -> c_int { + match panic_result { + Ok(traverse_result) => traverse_result, + Err(_payload) => { + eprintln!("FATAL: panic inside __traverse__ handler; aborting."); + ::std::process::abort() + } + } +} diff --git a/src/class/basic.rs b/src/class/basic.rs index 9d4d569b..b2d0e9ab 100644 --- a/src/class/basic.rs +++ b/src/class/basic.rs @@ -12,37 +12,6 @@ use crate::callback::{HashCallbackOutput, IntoPyCallbackOutput}; use crate::{exceptions, ffi, FromPyObject, PyAny, PyCell, PyClass, PyObject}; use std::os::raw::c_int; -/// Operators for the `__richcmp__` method -#[derive(Debug, Clone, Copy)] -pub enum CompareOp { - /// The *less than* operator. - Lt = ffi::Py_LT as isize, - /// The *less than or equal to* operator. - Le = ffi::Py_LE as isize, - /// The equality operator. - Eq = ffi::Py_EQ as isize, - /// The *not equal to* operator. - Ne = ffi::Py_NE as isize, - /// The *greater than* operator. - Gt = ffi::Py_GT as isize, - /// The *greater than or equal to* operator. - Ge = ffi::Py_GE as isize, -} - -impl CompareOp { - pub fn from_raw(op: c_int) -> Option { - match op { - ffi::Py_LT => Some(CompareOp::Lt), - ffi::Py_LE => Some(CompareOp::Le), - ffi::Py_EQ => Some(CompareOp::Eq), - ffi::Py_NE => Some(CompareOp::Ne), - ffi::Py_GT => Some(CompareOp::Gt), - ffi::Py_GE => Some(CompareOp::Ge), - _ => None, - } - } -} - /// Basic Python class customization #[allow(unused_variables)] pub trait PyObjectProtocol<'p>: PyClass { @@ -202,3 +171,5 @@ py_func_set_del!( __delattr__ ); py_unary_func!(bool, PyObjectBoolProtocol, T::__bool__, c_int); + +pub use crate::pyclass::CompareOp; diff --git a/src/class/gc.rs b/src/class/gc.rs index 2641a2b3..81e761fe 100644 --- a/src/class/gc.rs +++ b/src/class/gc.rs @@ -83,4 +83,20 @@ impl<'p> PyVisit<'p> { Err(PyTraverseError(r)) } } + + /// Creates the PyVisit from the arguments to tp_traverse + #[doc(hidden)] + pub unsafe fn from_raw(visit: ffi::visitproc, arg: *mut c_void, _py: Python<'p>) -> Self { + Self { visit, arg, _py } + } +} + +/// Unwraps the result of __traverse__ for tp_traverse +#[doc(hidden)] +#[inline] +pub fn unwrap_traverse_result(result: Result<(), PyTraverseError>) -> c_int { + match result { + Ok(()) => 0, + Err(PyTraverseError(value)) => value, + } } diff --git a/src/class/impl_.rs b/src/class/impl_.rs deleted file mode 100644 index dc1d74a0..00000000 --- a/src/class/impl_.rs +++ /dev/null @@ -1,782 +0,0 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - -use crate::{ - exceptions::{PyAttributeError, PyNotImplementedError}, - ffi, - impl_::freelist::FreeList, - pycell::PyCellLayout, - pyclass_init::PyObjectInit, - type_object::{PyLayout, PyTypeObject}, - PyClass, PyMethodDefType, PyNativeType, PyResult, PyTypeInfo, Python, -}; -use std::{marker::PhantomData, os::raw::c_void, ptr::NonNull, thread}; - -/// This type is used as a "dummy" type on which dtolnay specializations are -/// applied to apply implementations from `#[pymethods]` & `#[pyproto]` -pub struct PyClassImplCollector(PhantomData); - -impl PyClassImplCollector { - pub fn new() -> Self { - Self(PhantomData) - } -} - -impl Default for PyClassImplCollector { - fn default() -> Self { - Self::new() - } -} - -impl Clone for PyClassImplCollector { - fn clone(&self) -> Self { - Self::new() - } -} - -impl Copy for PyClassImplCollector {} - -/// Implements the underlying functionality of `#[pyclass]`, assembled by various proc macros. -/// -/// Users are discouraged from implementing this trait manually; it is a PyO3 implementation detail -/// and may be changed at any time. -pub trait PyClassImpl: Sized { - /// Class doc string - const DOC: &'static str = "\0"; - - /// #[pyclass(gc)] - const IS_GC: bool = false; - - /// #[pyclass(subclass)] - const IS_BASETYPE: bool = false; - - /// #[pyclass(extends=...)] - const IS_SUBCLASS: bool = false; - - /// Layout - type Layout: PyLayout; - - /// Base class - type BaseType: PyTypeInfo + PyTypeObject + PyClassBaseType; - - /// This handles following two situations: - /// 1. In case `T` is `Send`, stub `ThreadChecker` is used and does nothing. - /// This implementation is used by default. Compile fails if `T: !Send`. - /// 2. In case `T` is `!Send`, `ThreadChecker` panics when `T` is accessed by another thread. - /// This implementation is used when `#[pyclass(unsendable)]` is given. - /// Panicking makes it safe to expose `T: !Send` to the Python interpreter, where all objects - /// can be accessed by multiple threads by `threading` module. - type ThreadChecker: PyClassThreadChecker; - - #[cfg(feature = "multiple-pymethods")] - type Inventory: PyClassInventory; - - fn for_each_method_def(_visitor: &mut dyn FnMut(&[PyMethodDefType])) {} - fn for_each_proto_slot(_visitor: &mut dyn FnMut(&[ffi::PyType_Slot])) {} - - #[inline] - fn get_new() -> Option { - None - } - #[inline] - fn get_alloc() -> Option { - None - } - #[inline] - fn get_free() -> Option { - None - } - #[inline] - fn dict_offset() -> Option { - None - } - #[inline] - fn weaklist_offset() -> Option { - None - } -} - -// Traits describing known special methods. - -pub trait PyClassNewImpl { - fn new_impl(self) -> Option; -} - -impl PyClassNewImpl for &'_ PyClassImplCollector { - fn new_impl(self) -> Option { - None - } -} - -macro_rules! slot_fragment_trait { - ($trait_name:ident, $($default_method:tt)*) => { - #[allow(non_camel_case_types)] - pub trait $trait_name: Sized { - $($default_method)* - } - - impl $trait_name for &'_ PyClassImplCollector {} - } -} - -/// Macro which expands to three items -/// - Trait for a __setitem__ dunder -/// - Trait for the corresponding __delitem__ dunder -/// - A macro which will use dtolnay specialisation to generate the shared slot for the two dunders -macro_rules! define_pyclass_setattr_slot { - ( - $set_trait:ident, - $del_trait:ident, - $set:ident, - $del:ident, - $set_error:expr, - $del_error:expr, - $generate_macro:ident, - $slot:ident, - $func_ty:ident, - ) => { - slot_fragment_trait! { - $set_trait, - - /// # Safety: _slf and _attr must be valid non-null Python objects - #[inline] - unsafe fn $set( - self, - _py: Python, - _slf: *mut ffi::PyObject, - _attr: *mut ffi::PyObject, - _value: NonNull, - ) -> PyResult<()> { - $set_error - } - } - - slot_fragment_trait! { - $del_trait, - - /// # Safety: _slf and _attr must be valid non-null Python objects - #[inline] - unsafe fn $del( - self, - _py: Python, - _slf: *mut ffi::PyObject, - _attr: *mut ffi::PyObject, - ) -> PyResult<()> { - $del_error - } - } - - #[doc(hidden)] - #[macro_export] - macro_rules! $generate_macro { - ($cls:ty) => {{ - unsafe extern "C" fn __wrap( - _slf: *mut $crate::ffi::PyObject, - attr: *mut $crate::ffi::PyObject, - value: *mut $crate::ffi::PyObject, - ) -> ::std::os::raw::c_int { - use ::std::option::Option::*; - use $crate::callback::IntoPyCallbackOutput; - use $crate::class::impl_::*; - $crate::callback::handle_panic(|py| { - let collector = PyClassImplCollector::<$cls>::new(); - if let Some(value) = ::std::ptr::NonNull::new(value) { - collector.$set(py, _slf, attr, value).convert(py) - } else { - collector.$del(py, _slf, attr).convert(py) - } - }) - } - $crate::ffi::PyType_Slot { - slot: $crate::ffi::$slot, - pfunc: __wrap as $crate::ffi::$func_ty as _, - } - }}; - } - pub use $generate_macro; - }; -} - -define_pyclass_setattr_slot! { - PyClass__setattr__SlotFragment, - PyClass__delattr__SlotFragment, - __setattr__, - __delattr__, - Err(PyAttributeError::new_err("can't set attribute")), - Err(PyAttributeError::new_err("can't delete attribute")), - generate_pyclass_setattr_slot, - Py_tp_setattro, - setattrofunc, -} - -define_pyclass_setattr_slot! { - PyClass__set__SlotFragment, - PyClass__delete__SlotFragment, - __set__, - __delete__, - Err(PyNotImplementedError::new_err("can't set descriptor")), - Err(PyNotImplementedError::new_err("can't delete descriptor")), - generate_pyclass_setdescr_slot, - Py_tp_descr_set, - descrsetfunc, -} - -define_pyclass_setattr_slot! { - PyClass__setitem__SlotFragment, - PyClass__delitem__SlotFragment, - __setitem__, - __delitem__, - Err(PyNotImplementedError::new_err("can't set item")), - Err(PyNotImplementedError::new_err("can't delete item")), - generate_pyclass_setitem_slot, - Py_mp_ass_subscript, - objobjargproc, -} - -/// Macro which expands to three items -/// - Trait for a lhs dunder e.g. __add__ -/// - Trait for the corresponding rhs e.g. __radd__ -/// - A macro which will use dtolnay specialisation to generate the shared slot for the two dunders -macro_rules! define_pyclass_binary_operator_slot { - ( - $lhs_trait:ident, - $rhs_trait:ident, - $lhs:ident, - $rhs:ident, - $generate_macro:ident, - $slot:ident, - $func_ty:ident, - ) => { - slot_fragment_trait! { - $lhs_trait, - - /// # Safety: _slf and _other must be valid non-null Python objects - #[inline] - unsafe fn $lhs( - self, - _py: Python, - _slf: *mut ffi::PyObject, - _other: *mut ffi::PyObject, - ) -> PyResult<*mut ffi::PyObject> { - Ok(ffi::_Py_NewRef(ffi::Py_NotImplemented())) - } - } - - slot_fragment_trait! { - $rhs_trait, - - /// # Safety: _slf and _other must be valid non-null Python objects - #[inline] - unsafe fn $rhs( - self, - _py: Python, - _slf: *mut ffi::PyObject, - _other: *mut ffi::PyObject, - ) -> PyResult<*mut ffi::PyObject> { - Ok(ffi::_Py_NewRef(ffi::Py_NotImplemented())) - } - } - - #[doc(hidden)] - #[macro_export] - macro_rules! $generate_macro { - ($cls:ty) => {{ - unsafe extern "C" fn __wrap( - _slf: *mut $crate::ffi::PyObject, - _other: *mut $crate::ffi::PyObject, - ) -> *mut $crate::ffi::PyObject { - $crate::callback::handle_panic(|py| { - use $crate::class::impl_::*; - let collector = PyClassImplCollector::<$cls>::new(); - let lhs_result = collector.$lhs(py, _slf, _other)?; - if lhs_result == $crate::ffi::Py_NotImplemented() { - $crate::ffi::Py_DECREF(lhs_result); - collector.$rhs(py, _other, _slf) - } else { - ::std::result::Result::Ok(lhs_result) - } - }) - } - $crate::ffi::PyType_Slot { - slot: $crate::ffi::$slot, - pfunc: __wrap as $crate::ffi::$func_ty as _, - } - }}; - } - pub use $generate_macro; - }; -} - -define_pyclass_binary_operator_slot! { - PyClass__add__SlotFragment, - PyClass__radd__SlotFragment, - __add__, - __radd__, - generate_pyclass_add_slot, - Py_nb_add, - binaryfunc, -} - -define_pyclass_binary_operator_slot! { - PyClass__sub__SlotFragment, - PyClass__rsub__SlotFragment, - __sub__, - __rsub__, - generate_pyclass_sub_slot, - Py_nb_subtract, - binaryfunc, -} - -define_pyclass_binary_operator_slot! { - PyClass__mul__SlotFragment, - PyClass__rmul__SlotFragment, - __mul__, - __rmul__, - generate_pyclass_mul_slot, - Py_nb_multiply, - binaryfunc, -} - -define_pyclass_binary_operator_slot! { - PyClass__mod__SlotFragment, - PyClass__rmod__SlotFragment, - __mod__, - __rmod__, - generate_pyclass_mod_slot, - Py_nb_remainder, - binaryfunc, -} - -define_pyclass_binary_operator_slot! { - PyClass__divmod__SlotFragment, - PyClass__rdivmod__SlotFragment, - __divmod__, - __rdivmod__, - generate_pyclass_divmod_slot, - Py_nb_divmod, - binaryfunc, -} - -define_pyclass_binary_operator_slot! { - PyClass__lshift__SlotFragment, - PyClass__rlshift__SlotFragment, - __lshift__, - __rlshift__, - generate_pyclass_lshift_slot, - Py_nb_lshift, - binaryfunc, -} - -define_pyclass_binary_operator_slot! { - PyClass__rshift__SlotFragment, - PyClass__rrshift__SlotFragment, - __rshift__, - __rrshift__, - generate_pyclass_rshift_slot, - Py_nb_rshift, - binaryfunc, -} - -define_pyclass_binary_operator_slot! { - PyClass__and__SlotFragment, - PyClass__rand__SlotFragment, - __and__, - __rand__, - generate_pyclass_and_slot, - Py_nb_and, - binaryfunc, -} - -define_pyclass_binary_operator_slot! { - PyClass__or__SlotFragment, - PyClass__ror__SlotFragment, - __or__, - __ror__, - generate_pyclass_or_slot, - Py_nb_or, - binaryfunc, -} - -define_pyclass_binary_operator_slot! { - PyClass__xor__SlotFragment, - PyClass__rxor__SlotFragment, - __xor__, - __rxor__, - generate_pyclass_xor_slot, - Py_nb_xor, - binaryfunc, -} - -define_pyclass_binary_operator_slot! { - PyClass__matmul__SlotFragment, - PyClass__rmatmul__SlotFragment, - __matmul__, - __rmatmul__, - generate_pyclass_matmul_slot, - Py_nb_matrix_multiply, - binaryfunc, -} - -define_pyclass_binary_operator_slot! { - PyClass__truediv__SlotFragment, - PyClass__rtruediv__SlotFragment, - __truediv__, - __rtruediv__, - generate_pyclass_truediv_slot, - Py_nb_true_divide, - binaryfunc, -} - -define_pyclass_binary_operator_slot! { - PyClass__floordiv__SlotFragment, - PyClass__rfloordiv__SlotFragment, - __floordiv__, - __rfloordiv__, - generate_pyclass_floordiv_slot, - Py_nb_floor_divide, - binaryfunc, -} - -slot_fragment_trait! { - PyClass__pow__SlotFragment, - - /// # Safety: _slf and _other must be valid non-null Python objects - #[inline] - unsafe fn __pow__( - self, - _py: Python, - _slf: *mut ffi::PyObject, - _other: *mut ffi::PyObject, - _mod: *mut ffi::PyObject, - ) -> PyResult<*mut ffi::PyObject> { - Ok(ffi::_Py_NewRef(ffi::Py_NotImplemented())) - } -} - -slot_fragment_trait! { - PyClass__rpow__SlotFragment, - - /// # Safety: _slf and _other must be valid non-null Python objects - #[inline] - unsafe fn __rpow__( - self, - _py: Python, - _slf: *mut ffi::PyObject, - _other: *mut ffi::PyObject, - _mod: *mut ffi::PyObject, - ) -> PyResult<*mut ffi::PyObject> { - Ok(ffi::_Py_NewRef(ffi::Py_NotImplemented())) - } -} - -#[doc(hidden)] -#[macro_export] -macro_rules! generate_pyclass_pow_slot { - ($cls:ty) => {{ - unsafe extern "C" fn __wrap( - _slf: *mut $crate::ffi::PyObject, - _other: *mut $crate::ffi::PyObject, - _mod: *mut $crate::ffi::PyObject, - ) -> *mut $crate::ffi::PyObject { - $crate::callback::handle_panic(|py| { - use $crate::class::impl_::*; - let collector = PyClassImplCollector::<$cls>::new(); - let lhs_result = collector.__pow__(py, _slf, _other, _mod)?; - if lhs_result == $crate::ffi::Py_NotImplemented() { - $crate::ffi::Py_DECREF(lhs_result); - collector.__rpow__(py, _other, _slf, _mod) - } else { - ::std::result::Result::Ok(lhs_result) - } - }) - } - $crate::ffi::PyType_Slot { - slot: $crate::ffi::Py_nb_power, - pfunc: __wrap as $crate::ffi::ternaryfunc as _, - } - }}; -} -pub use generate_pyclass_pow_slot; - -pub trait PyClassAllocImpl { - fn alloc_impl(self) -> Option; -} - -impl PyClassAllocImpl for &'_ PyClassImplCollector { - fn alloc_impl(self) -> Option { - None - } -} - -pub trait PyClassFreeImpl { - fn free_impl(self) -> Option; -} - -impl PyClassFreeImpl for &'_ PyClassImplCollector { - fn free_impl(self) -> Option { - None - } -} - -/// Implements a freelist. -/// -/// Do not implement this trait manually. Instead, use `#[pyclass(freelist = N)]` -/// on a Rust struct to implement it. -pub trait PyClassWithFreeList: PyClass { - fn get_free_list(py: Python) -> &mut FreeList<*mut ffi::PyObject>; -} - -/// Implementation of tp_alloc for `freelist` classes. -/// -/// # Safety -/// - `subtype` must be a valid pointer to the type object of T or a subclass. -/// - The GIL must be held. -pub unsafe extern "C" fn alloc_with_freelist( - subtype: *mut ffi::PyTypeObject, - nitems: ffi::Py_ssize_t, -) -> *mut ffi::PyObject { - let py = Python::assume_gil_acquired(); - - #[cfg(not(Py_3_8))] - bpo_35810_workaround(py, subtype); - - let self_type = T::type_object_raw(py); - // If this type is a variable type or the subtype is not equal to this type, we cannot use the - // freelist - if nitems == 0 && subtype == self_type { - if let Some(obj) = T::get_free_list(py).pop() { - ffi::PyObject_Init(obj, subtype); - return obj as _; - } - } - - ffi::PyType_GenericAlloc(subtype, nitems) -} - -/// Implementation of tp_free for `freelist` classes. -/// -/// # Safety -/// - `obj` must be a valid pointer to an instance of T (not a subclass). -/// - The GIL must be held. -pub unsafe extern "C" fn free_with_freelist(obj: *mut c_void) { - let obj = obj as *mut ffi::PyObject; - debug_assert_eq!( - T::type_object_raw(Python::assume_gil_acquired()), - ffi::Py_TYPE(obj) - ); - if let Some(obj) = T::get_free_list(Python::assume_gil_acquired()).insert(obj) { - let ty = ffi::Py_TYPE(obj); - - // Deduce appropriate inverse of PyType_GenericAlloc - let free = if ffi::PyType_IS_GC(ty) != 0 { - ffi::PyObject_GC_Del - } else { - ffi::PyObject_Free - }; - free(obj as *mut c_void); - - #[cfg(Py_3_8)] - if ffi::PyType_HasFeature(ty, ffi::Py_TPFLAGS_HEAPTYPE) != 0 { - ffi::Py_DECREF(ty as *mut ffi::PyObject); - } - } -} - -/// Workaround for Python issue 35810; no longer necessary in Python 3.8 -#[inline] -#[cfg(not(Py_3_8))] -unsafe fn bpo_35810_workaround(_py: Python, ty: *mut ffi::PyTypeObject) { - #[cfg(Py_LIMITED_API)] - { - // Must check version at runtime for abi3 wheels - they could run against a higher version - // than the build config suggests. - use crate::once_cell::GILOnceCell; - static IS_PYTHON_3_8: GILOnceCell = GILOnceCell::new(); - - if *IS_PYTHON_3_8.get_or_init(_py, || _py.version_info() >= (3, 8)) { - // No fix needed - the wheel is running on a sufficiently new interpreter. - return; - } - } - - ffi::Py_INCREF(ty as *mut ffi::PyObject); -} - -// General methods implementation: either dtolnay specialization trait or inventory if -// multiple-pymethods feature is enabled. - -macro_rules! methods_trait { - ($name:ident, $function_name: ident) => { - pub trait $name { - fn $function_name(self) -> &'static [PyMethodDefType]; - } - - impl $name for &'_ PyClassImplCollector { - fn $function_name(self) -> &'static [PyMethodDefType] { - &[] - } - } - }; -} - -/// Implementation detail. Only to be used through our proc macro code. -/// Method storage for `#[pyclass]`. -/// Allows arbitrary `#[pymethod]` blocks to submit their methods, -/// which are eventually collected by `#[pyclass]`. -#[cfg(feature = "multiple-pymethods")] -pub trait PyClassInventory: inventory::Collect { - /// Returns the methods for a single `#[pymethods] impl` block - fn methods(&'static self) -> &'static [PyMethodDefType]; - - /// Returns the slots for a single `#[pymethods] impl` block - fn slots(&'static self) -> &'static [ffi::PyType_Slot]; -} - -// Methods from #[pyo3(get, set)] on struct fields. -methods_trait!(PyClassDescriptors, py_class_descriptors); - -// Methods from #[pymethods] if not using inventory. -#[cfg(not(feature = "multiple-pymethods"))] -methods_trait!(PyMethods, py_methods); - -// All traits describing slots, as well as the fallback implementations for unimplemented protos -// -// Protos which are implemented use dtolnay specialization to implement for PyClassImplCollector. -// -// See https://github.com/dtolnay/case-studies/blob/master/autoref-specialization/README.md - -macro_rules! slots_trait { - ($name:ident, $function_name: ident) => { - pub trait $name { - fn $function_name(self) -> &'static [ffi::PyType_Slot]; - } - - impl $name for &'_ PyClassImplCollector { - fn $function_name(self) -> &'static [ffi::PyType_Slot] { - &[] - } - } - }; -} - -slots_trait!(PyObjectProtocolSlots, object_protocol_slots); -slots_trait!(PyDescrProtocolSlots, descr_protocol_slots); -slots_trait!(PyGCProtocolSlots, gc_protocol_slots); -slots_trait!(PyIterProtocolSlots, iter_protocol_slots); -slots_trait!(PyMappingProtocolSlots, mapping_protocol_slots); -slots_trait!(PyNumberProtocolSlots, number_protocol_slots); -slots_trait!(PyAsyncProtocolSlots, async_protocol_slots); -slots_trait!(PySequenceProtocolSlots, sequence_protocol_slots); -slots_trait!(PyBufferProtocolSlots, buffer_protocol_slots); - -// slots that PyO3 implements by default, but can be overidden by the users. -slots_trait!(PyClassDefaultSlots, py_class_default_slots); - -// Protocol slots from #[pymethods] if not using inventory. -#[cfg(not(feature = "multiple-pymethods"))] -slots_trait!(PyMethodsProtocolSlots, methods_protocol_slots); - -methods_trait!(PyObjectProtocolMethods, object_protocol_methods); -methods_trait!(PyAsyncProtocolMethods, async_protocol_methods); -methods_trait!(PyDescrProtocolMethods, descr_protocol_methods); -methods_trait!(PyMappingProtocolMethods, mapping_protocol_methods); -methods_trait!(PyNumberProtocolMethods, number_protocol_methods); - -// Thread checkers - -#[doc(hidden)] -pub trait PyClassThreadChecker: Sized { - fn ensure(&self); - fn new() -> Self; - private_decl! {} -} - -/// Stub checker for `Send` types. -#[doc(hidden)] -pub struct ThreadCheckerStub(PhantomData); - -impl PyClassThreadChecker for ThreadCheckerStub { - fn ensure(&self) {} - #[inline] - fn new() -> Self { - ThreadCheckerStub(PhantomData) - } - private_impl! {} -} - -impl PyClassThreadChecker for ThreadCheckerStub { - fn ensure(&self) {} - #[inline] - fn new() -> Self { - ThreadCheckerStub(PhantomData) - } - private_impl! {} -} - -/// Thread checker for unsendable types. -/// Panics when the value is accessed by another thread. -#[doc(hidden)] -pub struct ThreadCheckerImpl(thread::ThreadId, PhantomData); - -impl PyClassThreadChecker for ThreadCheckerImpl { - fn ensure(&self) { - if thread::current().id() != self.0 { - panic!( - "{} is unsendable, but sent to another thread!", - std::any::type_name::() - ); - } - } - fn new() -> Self { - ThreadCheckerImpl(thread::current().id(), PhantomData) - } - private_impl! {} -} - -/// Thread checker for types that have `Send` and `extends=...`. -/// Ensures that `T: Send` and the parent is not accessed by another thread. -#[doc(hidden)] -pub struct ThreadCheckerInherited(PhantomData, U::ThreadChecker); - -impl PyClassThreadChecker for ThreadCheckerInherited { - fn ensure(&self) { - self.1.ensure(); - } - fn new() -> Self { - ThreadCheckerInherited(PhantomData, U::ThreadChecker::new()) - } - private_impl! {} -} - -/// Trait denoting that this class is suitable to be used as a base type for PyClass. -pub trait PyClassBaseType: Sized { - type LayoutAsBase: PyCellLayout; - type BaseNativeType; - type ThreadChecker: PyClassThreadChecker; - type Initializer: PyObjectInit; -} - -/// All PyClasses can be used as a base type. -impl PyClassBaseType for T { - type LayoutAsBase = crate::pycell::PyCell; - type BaseNativeType = T::BaseNativeType; - type ThreadChecker = T::ThreadChecker; - type Initializer = crate::pyclass_init::PyClassInitializer; -} - -/// Default new implementation -pub(crate) unsafe extern "C" fn fallback_new( - _subtype: *mut ffi::PyTypeObject, - _args: *mut ffi::PyObject, - _kwds: *mut ffi::PyObject, -) -> *mut ffi::PyObject { - crate::callback_body!(py, { - Err::<(), _>(crate::exceptions::PyTypeError::new_err( - "No constructor defined", - )) - }) -} - -/// Implementation of tp_dealloc for all pyclasses -pub(crate) unsafe extern "C" fn tp_dealloc(obj: *mut ffi::PyObject) { - crate::callback_body!(py, T::Layout::tp_dealloc(obj, py)) -} diff --git a/src/class/iter.rs b/src/class/iter.rs index 97b47fe3..3d980375 100644 --- a/src/class/iter.rs +++ b/src/class/iter.rs @@ -4,8 +4,7 @@ use crate::callback::IntoPyCallbackOutput; use crate::derive_utils::TryFromPyCell; -use crate::err::PyResult; -use crate::{ffi, IntoPy, IntoPyPointer, PyClass, PyObject, Python}; +use crate::{PyClass, PyObject}; /// Python Iterator Interface. /// @@ -41,7 +40,7 @@ use crate::{ffi, IntoPy, IntoPyPointer, PyClass, PyObject, Python}; /// # Python::with_gil(|py| { /// # let inst = Py::new(py, Iter { count: 0 }).unwrap(); /// # pyo3::py_run!(py, inst, "assert next(inst) == 1"); -/// # }); // test of StopIteration is done in examples/pyo3-pytests/pyclass_iter.rs +/// # }); // test of StopIteration is done in pytests/src/pyclasses.rs /// ``` #[allow(unused_variables)] pub trait PyIterProtocol<'p>: PyClass { @@ -73,49 +72,4 @@ pub trait PyIterNextProtocol<'p>: PyIterProtocol<'p> { py_unarys_func!(iter, PyIterIterProtocol, Self::__iter__); py_unarys_func!(iternext, PyIterNextProtocol, Self::__next__); -/// Output of `__next__` which can either `yield` the next value in the iteration, or -/// `return` a value to raise `StopIteration` in Python. -/// -/// See [`PyIterProtocol`](trait.PyIterProtocol.html) for an example. -pub enum IterNextOutput { - /// The value yielded by the iterator. - Yield(T), - /// The `StopIteration` object. - Return(U), -} - -pub type PyIterNextOutput = IterNextOutput; - -impl IntoPyCallbackOutput<*mut ffi::PyObject> for PyIterNextOutput { - fn convert(self, _py: Python) -> PyResult<*mut ffi::PyObject> { - match self { - IterNextOutput::Yield(o) => Ok(o.into_ptr()), - IterNextOutput::Return(opt) => Err(crate::exceptions::PyStopIteration::new_err((opt,))), - } - } -} - -impl IntoPyCallbackOutput for IterNextOutput -where - T: IntoPy, - U: IntoPy, -{ - fn convert(self, py: Python) -> PyResult { - match self { - IterNextOutput::Yield(o) => Ok(IterNextOutput::Yield(o.into_py(py))), - IterNextOutput::Return(o) => Ok(IterNextOutput::Return(o.into_py(py))), - } - } -} - -impl IntoPyCallbackOutput for Option -where - T: IntoPy, -{ - fn convert(self, py: Python) -> PyResult { - match self { - Some(o) => Ok(PyIterNextOutput::Yield(o.into_py(py))), - None => Ok(PyIterNextOutput::Return(py.None())), - } - } -} +pub use crate::pyclass::{IterNextOutput, PyIterNextOutput}; diff --git a/src/class/methods.rs b/src/class/methods.rs index 5f537e3a..e69de29b 100644 --- a/src/class/methods.rs +++ b/src/class/methods.rs @@ -1,220 +0,0 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - -use crate::internal_tricks::{extract_cstr_or_leak_cstring, NulByteInString}; -use crate::{ffi, PyObject, Python}; -use std::ffi::CStr; -use std::fmt; -use std::os::raw::c_int; - -/// `PyMethodDefType` represents different types of Python callable objects. -/// It is used by the `#[pymethods]` and `#[pyproto]` annotations. -#[derive(Debug)] -pub enum PyMethodDefType { - /// Represents class method - Class(PyMethodDef), - /// Represents static method - Static(PyMethodDef), - /// Represents normal method - Method(PyMethodDef), - /// Represents class attribute, used by `#[attribute]` - ClassAttribute(PyClassAttributeDef), - /// Represents getter descriptor, used by `#[getter]` - Getter(PyGetterDef), - /// Represents setter descriptor, used by `#[setter]` - Setter(PySetterDef), -} - -#[derive(Copy, Clone, Debug)] -pub enum PyMethodType { - PyCFunction(PyCFunction), - PyCFunctionWithKeywords(PyCFunctionWithKeywords), - #[cfg(not(Py_LIMITED_API))] - PyCFunctionFastWithKeywords(PyCFunctionFastWithKeywords), -} - -// These newtype structs serve no purpose other than wrapping which are function pointers - because -// function pointers aren't allowed in const fn, but types wrapping them are! -#[derive(Clone, Copy, Debug)] -pub struct PyCFunction(pub ffi::PyCFunction); -#[derive(Clone, Copy, Debug)] -pub struct PyCFunctionWithKeywords(pub ffi::PyCFunctionWithKeywords); -#[cfg(not(Py_LIMITED_API))] -#[derive(Clone, Copy, Debug)] -pub struct PyCFunctionFastWithKeywords(pub ffi::_PyCFunctionFastWithKeywords); -#[derive(Clone, Copy, Debug)] -pub struct PyGetter(pub ffi::getter); -#[derive(Clone, Copy, Debug)] -pub struct PySetter(pub ffi::setter); -#[derive(Clone, Copy)] -pub struct PyClassAttributeFactory(pub for<'p> fn(Python<'p>) -> PyObject); - -// TODO: it would be nice to use CStr in these types, but then the constructors can't be const fn -// until `CStr::from_bytes_with_nul_unchecked` is const fn. - -#[derive(Clone, Debug)] -pub struct PyMethodDef { - pub(crate) ml_name: &'static str, - pub(crate) ml_meth: PyMethodType, - pub(crate) ml_flags: c_int, - pub(crate) ml_doc: &'static str, -} - -#[derive(Copy, Clone)] -pub struct PyClassAttributeDef { - pub(crate) name: &'static str, - pub(crate) meth: PyClassAttributeFactory, -} - -#[derive(Clone, Debug)] -pub struct PyGetterDef { - pub(crate) name: &'static str, - pub(crate) meth: PyGetter, - doc: &'static str, -} - -#[derive(Clone, Debug)] -pub struct PySetterDef { - pub(crate) name: &'static str, - pub(crate) meth: PySetter, - doc: &'static str, -} - -unsafe impl Sync for PyMethodDef {} - -unsafe impl Sync for PyGetterDef {} - -unsafe impl Sync for PySetterDef {} - -impl PyMethodDef { - /// Define a function with no `*args` and `**kwargs`. - pub const fn noargs(name: &'static str, cfunction: PyCFunction, doc: &'static str) -> Self { - Self { - ml_name: name, - ml_meth: PyMethodType::PyCFunction(cfunction), - ml_flags: ffi::METH_NOARGS, - ml_doc: doc, - } - } - - /// Define a function that can take `*args` and `**kwargs`. - pub const fn cfunction_with_keywords( - name: &'static str, - cfunction: PyCFunctionWithKeywords, - doc: &'static str, - ) -> Self { - Self { - ml_name: name, - ml_meth: PyMethodType::PyCFunctionWithKeywords(cfunction), - ml_flags: ffi::METH_VARARGS | ffi::METH_KEYWORDS, - ml_doc: doc, - } - } - - /// Define a function that can take `*args` and `**kwargs`. - #[cfg(not(Py_LIMITED_API))] - pub const fn fastcall_cfunction_with_keywords( - name: &'static str, - cfunction: PyCFunctionFastWithKeywords, - doc: &'static str, - ) -> Self { - Self { - ml_name: name, - ml_meth: PyMethodType::PyCFunctionFastWithKeywords(cfunction), - ml_flags: ffi::METH_FASTCALL | ffi::METH_KEYWORDS, - ml_doc: doc, - } - } - - pub const fn flags(mut self, flags: c_int) -> Self { - self.ml_flags |= flags; - self - } - - /// Convert `PyMethodDef` to Python method definition struct `ffi::PyMethodDef` - pub(crate) fn as_method_def(&self) -> Result { - let meth = match self.ml_meth { - PyMethodType::PyCFunction(meth) => meth.0, - PyMethodType::PyCFunctionWithKeywords(meth) => unsafe { std::mem::transmute(meth.0) }, - #[cfg(not(Py_LIMITED_API))] - PyMethodType::PyCFunctionFastWithKeywords(meth) => unsafe { - std::mem::transmute(meth.0) - }, - }; - - Ok(ffi::PyMethodDef { - ml_name: get_name(self.ml_name)?.as_ptr(), - ml_meth: Some(meth), - ml_flags: self.ml_flags, - ml_doc: get_doc(self.ml_doc)?.as_ptr(), - }) - } -} - -impl PyClassAttributeDef { - /// Define a class attribute. - pub const fn new(name: &'static str, meth: PyClassAttributeFactory) -> Self { - Self { name, meth } - } -} - -// Manual implementation because `Python<'_>` does not implement `Debug` and -// trait bounds on `fn` compiler-generated derive impls are too restrictive. -impl fmt::Debug for PyClassAttributeDef { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("PyClassAttributeDef") - .field("name", &self.name) - .finish() - } -} - -impl PyGetterDef { - /// Define a getter. - pub const fn new(name: &'static str, getter: PyGetter, doc: &'static str) -> Self { - Self { - name, - meth: getter, - doc, - } - } - - /// Copy descriptor information to `ffi::PyGetSetDef` - pub fn copy_to(&self, dst: &mut ffi::PyGetSetDef) { - if dst.name.is_null() { - dst.name = get_name(self.name).unwrap().as_ptr() as _; - } - if dst.doc.is_null() { - dst.doc = get_doc(self.doc).unwrap().as_ptr() as _; - } - dst.get = Some(self.meth.0); - } -} - -impl PySetterDef { - /// Define a setter. - pub const fn new(name: &'static str, setter: PySetter, doc: &'static str) -> Self { - Self { - name, - meth: setter, - doc, - } - } - - /// Copy descriptor information to `ffi::PyGetSetDef` - pub fn copy_to(&self, dst: &mut ffi::PyGetSetDef) { - if dst.name.is_null() { - dst.name = get_name(self.name).unwrap().as_ptr() as _; - } - if dst.doc.is_null() { - dst.doc = get_doc(self.doc).unwrap().as_ptr() as _; - } - dst.set = Some(self.meth.0); - } -} - -fn get_name(name: &'static str) -> Result<&'static CStr, NulByteInString> { - extract_cstr_or_leak_cstring(name, "Function name cannot contain NUL byte.") -} - -fn get_doc(doc: &'static str) -> Result<&'static CStr, NulByteInString> { - extract_cstr_or_leak_cstring(doc, "Document cannot contain NUL byte.") -} diff --git a/src/class/mod.rs b/src/class/mod.rs index 0e889573..a7717abd 100644 --- a/src/class/mod.rs +++ b/src/class/mod.rs @@ -6,22 +6,20 @@ mod macros; pub mod basic; -#[cfg(not(Py_LIMITED_API))] +#[cfg(any(not(Py_LIMITED_API), Py_3_11))] pub mod buffer; pub mod descr; pub mod gc; -#[doc(hidden)] -pub mod impl_; pub mod iter; pub mod mapping; #[doc(hidden)] -pub mod methods; +pub use crate::impl_::pymethods as methods; pub mod number; pub mod pyasync; pub mod sequence; pub use self::basic::PyObjectProtocol; -#[cfg(not(Py_LIMITED_API))] +#[cfg(any(not(Py_LIMITED_API), Py_3_11))] pub use self::buffer::PyBufferProtocol; pub use self::descr::PyDescrProtocol; pub use self::gc::{PyGCProtocol, PyTraverseError, PyVisit}; diff --git a/src/class/pyasync.rs b/src/class/pyasync.rs index f79cd7ac..96053824 100644 --- a/src/class/pyasync.rs +++ b/src/class/pyasync.rs @@ -9,8 +9,7 @@ use crate::callback::IntoPyCallbackOutput; use crate::derive_utils::TryFromPyCell; -use crate::err::PyResult; -use crate::{ffi, IntoPy, IntoPyPointer, PyClass, PyObject, Python}; +use crate::{PyClass, PyObject}; /// Python Async/Await support interface. /// @@ -58,51 +57,4 @@ py_unarys_func!(await_, PyAsyncAwaitProtocol, Self::__await__); py_unarys_func!(aiter, PyAsyncAiterProtocol, Self::__aiter__); py_unarys_func!(anext, PyAsyncAnextProtocol, Self::__anext__); -/// Output of `__anext__`. -/// -/// -pub enum IterANextOutput { - /// An expression which the generator yielded. - Yield(T), - /// A `StopAsyncIteration` object. - Return(U), -} - -/// An [IterANextOutput] of Python objects. -pub type PyIterANextOutput = IterANextOutput; - -impl IntoPyCallbackOutput<*mut ffi::PyObject> for PyIterANextOutput { - fn convert(self, _py: Python) -> PyResult<*mut ffi::PyObject> { - match self { - IterANextOutput::Yield(o) => Ok(o.into_ptr()), - IterANextOutput::Return(opt) => { - Err(crate::exceptions::PyStopAsyncIteration::new_err((opt,))) - } - } - } -} - -impl IntoPyCallbackOutput for IterANextOutput -where - T: IntoPy, - U: IntoPy, -{ - fn convert(self, py: Python) -> PyResult { - match self { - IterANextOutput::Yield(o) => Ok(IterANextOutput::Yield(o.into_py(py))), - IterANextOutput::Return(o) => Ok(IterANextOutput::Return(o.into_py(py))), - } - } -} - -impl IntoPyCallbackOutput for Option -where - T: IntoPy, -{ - fn convert(self, py: Python) -> PyResult { - match self { - Some(o) => Ok(PyIterANextOutput::Yield(o.into_py(py))), - None => Ok(PyIterANextOutput::Return(py.None())), - } - } -} +pub use crate::pyclass::{IterANextOutput, PyIterANextOutput}; diff --git a/src/conversion.rs b/src/conversion.rs index ccd690be..6d9e74b0 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -44,7 +44,7 @@ where #[inline] fn as_ptr(&self) -> *mut ffi::PyObject { self.as_ref() - .map_or_else(std::ptr::null_mut, |t| t.into_ptr()) + .map_or_else(std::ptr::null_mut, |t| t.as_ptr()) } } @@ -97,38 +97,7 @@ pub trait ToBorrowedObject: ToPyObject { } } -impl ToBorrowedObject for T -where - T: ToPyObject, -{ - #[cfg(feature = "nightly")] - #[cfg_attr(docsrs, doc(cfg(feature = "nightly")))] - default fn with_borrowed_ptr(&self, py: Python, f: F) -> R - where - F: FnOnce(*mut ffi::PyObject) -> R, - { - let ptr = self.to_object(py).into_ptr(); - let result = f(ptr); - unsafe { - ffi::Py_XDECREF(ptr); - } - result - } -} - -#[cfg(feature = "nightly")] -#[cfg_attr(docsrs, doc(cfg(feature = "nightly")))] -impl ToBorrowedObject for T -where - T: ToPyObject + AsPyPointer, -{ - fn with_borrowed_ptr(&self, _py: Python, f: F) -> R - where - F: FnOnce(*mut ffi::PyObject) -> R, - { - f(self.as_ptr()) - } -} +impl ToBorrowedObject for T where T: ToPyObject {} /// Defines a conversion from a Rust type to a Python object. /// @@ -555,7 +524,7 @@ where #[cfg(test)] mod tests { use crate::types::{IntoPyDict, PyAny, PyDict, PyList}; - use crate::{Python, ToPyObject}; + use crate::{AsPyPointer, PyObject, Python, ToPyObject}; use super::PyTryFrom; @@ -595,4 +564,21 @@ mod tests { assert_eq!(list, val); }); } + + #[test] + fn test_option_as_ptr() { + Python::with_gil(|py| { + let mut option: Option = None; + assert_eq!(option.as_ptr(), std::ptr::null_mut()); + + let none = py.None(); + option = Some(none.clone()); + + let ref_cnt = none.get_refcnt(py); + assert_eq!(option.as_ptr(), none.as_ptr()); + + // Ensure ref count not changed by as_ptr call + assert_eq!(none.get_refcnt(py), ref_cnt); + }); + } } diff --git a/src/conversions/array.rs b/src/conversions/array.rs index 77a7fde6..0b186fe3 100644 --- a/src/conversions/array.rs +++ b/src/conversions/array.rs @@ -18,37 +18,9 @@ mod min_const_generics { where T: FromPyObject<'a>, { - #[cfg(not(feature = "nightly"))] fn extract(obj: &'a PyAny) -> PyResult { create_array_from_obj(obj) } - - #[cfg(feature = "nightly")] - default fn extract(obj: &'a PyAny) -> PyResult { - create_array_from_obj(obj) - } - } - - #[cfg(all(feature = "nightly", not(Py_LIMITED_API)))] - impl<'source, T, const N: usize> FromPyObject<'source> for [T; N] - where - for<'a> T: Default + FromPyObject<'a> + crate::buffer::Element, - { - fn extract(obj: &'source PyAny) -> PyResult { - use crate::AsPyPointer; - // first try buffer protocol - if unsafe { crate::ffi::PyObject_CheckBuffer(obj.as_ptr()) } == 1 { - if let Ok(buf) = crate::buffer::PyBuffer::get(obj) { - let mut array = [T::default(); N]; - if buf.dimensions() == 1 && buf.copy_to_slice(obj.py(), &mut array).is_ok() { - buf.release(obj.py()); - return Ok(array); - } - buf.release(obj.py()); - } - } - create_array_from_obj(obj) - } } fn create_array_from_obj<'s, T, const N: usize>(obj: &'s PyAny) -> PyResult<[T; N]> @@ -185,42 +157,11 @@ mod array_impls { where T: Copy + Default + FromPyObject<'a>, { - #[cfg(not(feature = "nightly"))] fn extract(obj: &'a PyAny) -> PyResult { let mut array = [T::default(); $N]; extract_sequence_into_slice(obj, &mut array)?; Ok(array) } - - #[cfg(feature = "nightly")] - default fn extract(obj: &'a PyAny) -> PyResult { - let mut array = [T::default(); $N]; - extract_sequence_into_slice(obj, &mut array)?; - Ok(array) - } - } - - #[cfg(feature = "nightly")] - impl<'source, T> FromPyObject<'source> for [T; $N] - where - for<'a> T: Default + FromPyObject<'a> + crate::buffer::Element, - { - fn extract(obj: &'source PyAny) -> PyResult { - let mut array = [T::default(); $N]; - // first try buffer protocol - if unsafe { crate::ffi::PyObject_CheckBuffer(obj.as_ptr()) } == 1 { - if let Ok(buf) = crate::buffer::PyBuffer::get(obj) { - if buf.dimensions() == 1 && buf.copy_to_slice(obj.py(), &mut array).is_ok() { - buf.release(obj.py()); - return Ok(array); - } - buf.release(obj.py()); - } - } - // fall back to sequence protocol - extract_sequence_into_slice(obj, &mut array)?; - Ok(array) - } } )+ } diff --git a/src/conversions/num_bigint.rs b/src/conversions/num_bigint.rs index 197f7ec3..f55dc385 100644 --- a/src/conversions/num_bigint.rs +++ b/src/conversions/num_bigint.rs @@ -111,12 +111,12 @@ macro_rules! bigint_conversion { let num: Py = Py::from_owned_ptr_or_err(py, ffi::PyNumber_Index(ob.as_ptr()))?; let n_bits = ffi::_PyLong_NumBits(num.as_ptr()); - let n_bytes = if n_bits == -1 { + let n_bytes = if n_bits == (-1isize as usize) { return Err(PyErr::fetch(py)); } else if n_bits == 0 { 0 } else { - (n_bits as usize - 1 + $is_signed) / 8 + 1 + (n_bits - 1 + $is_signed) / 8 + 1 }; if n_bytes <= 128 { let mut buffer = [0; 128]; diff --git a/src/err/mod.rs b/src/err/mod.rs index 5e08bb0e..f45f4952 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -38,6 +38,9 @@ pub struct PyErr { state: UnsafeCell>, } +// The inner value is only accessed through ways that require proving the gil is held +#[cfg(feature = "nightly")] +unsafe impl crate::marker::Ungil for PyErr {} unsafe impl Send for PyErr {} unsafe impl Sync for PyErr {} diff --git a/src/ffi/README.md b/src/ffi/README.md deleted file mode 100644 index f65ba028..00000000 --- a/src/ffi/README.md +++ /dev/null @@ -1,15 +0,0 @@ -rust python3 ffi -================ - -[Rust](https://www.rust-lang.org/) FFI declarations for Python 3. -Supports the PEP 384 stable ABI for Python 3.4 or higher. - ---- - -This [cargo -sys package](https://doc.rust-lang.org/cargo/reference/build-scripts.html#-sys-packages) provides `python3` declarations. -Licensed under the Python license (see `LICENSE`). - -For a safe high-level API, see [PyO3](https://github.com/PyO3/PyO3). - -Documentation for the Python API is available on [https://docs.python.org/3/c-api/]. - diff --git a/src/ffi/mod.rs b/src/ffi/mod.rs index 82e59e99..81d6f38e 100644 --- a/src/ffi/mod.rs +++ b/src/ffi/mod.rs @@ -20,190 +20,12 @@ //! for more information. //! //! [capi]: https://docs.python.org/3/c-api/index.html -#![allow( - missing_docs, - non_camel_case_types, - non_snake_case, - non_upper_case_globals, - clippy::upper_case_acronyms, - clippy::missing_safety_doc -)] -// Until `extern type` is stabilized, use the recommended approach to -// model opaque types: -// https://doc.rust-lang.org/nomicon/ffi.html#representing-opaque-structs -macro_rules! opaque_struct { - ($name:ident) => { - #[repr(C)] - pub struct $name([u8; 0]); - }; -} +#[cfg(all(not(Py_LIMITED_API), test))] +mod tests; -pub use self::abstract_::*; -pub use self::bltinmodule::*; -pub use self::boolobject::*; -pub use self::bytearrayobject::*; -pub use self::bytesobject::*; -pub use self::ceval::*; -pub use self::code::*; -pub use self::codecs::*; -pub use self::compile::*; -pub use self::complexobject::*; -#[cfg(all(Py_3_8, not(Py_LIMITED_API)))] -pub use self::context::*; -#[cfg(not(Py_LIMITED_API))] -pub use self::datetime::*; -pub use self::descrobject::*; -pub use self::dictobject::*; -pub use self::enumobject::*; -pub use self::eval::*; -pub use self::fileobject::*; -pub use self::fileutils::*; -pub use self::floatobject::*; -#[cfg(not(Py_LIMITED_API))] -pub use self::funcobject::*; -#[cfg(not(Py_LIMITED_API))] -pub use self::genobject::*; -pub use self::import::*; -pub use self::intrcheck::*; -pub use self::iterobject::*; -pub use self::listobject::*; -pub use self::longobject::*; -pub use self::marshal::*; -pub use self::memoryobject::*; -pub use self::methodobject::*; -pub use self::modsupport::*; -pub use self::moduleobject::*; -pub use self::object::*; -pub use self::objimpl::*; -pub use self::osmodule::*; -#[cfg(not(Py_LIMITED_API))] -pub use self::pyarena::*; -pub use self::pycapsule::*; -pub use self::pyerrors::*; -pub use self::pyframe::*; -pub use self::pyhash::*; -pub use self::pylifecycle::*; -pub use self::pymem::*; -pub use self::pyport::*; -pub use self::pystate::*; -pub use self::pystrtod::*; -pub use self::pythonrun::*; -pub use self::rangeobject::*; -pub use self::setobject::*; -pub use self::sliceobject::*; -pub use self::structseq::*; -pub use self::sysmodule::*; -pub use self::traceback::*; -pub use self::tupleobject::*; -pub use self::typeslots::*; -pub use self::unicodeobject::*; -pub use self::warnings::*; -pub use self::weakrefobject::*; - -mod abstract_; -// skipped asdl.h -// skipped ast.h -mod bltinmodule; -mod boolobject; -mod bytearrayobject; -mod bytesobject; -// skipped cellobject.h -mod ceval; -// skipped classobject.h -mod code; -mod codecs; -mod compile; -mod complexobject; -#[cfg(all(Py_3_8, not(Py_LIMITED_API)))] -mod context; // It's actually 3.7.1, but no cfg for patches. -#[cfg(not(Py_LIMITED_API))] -pub(crate) mod datetime; -mod descrobject; -mod dictobject; -// skipped dynamic_annotations.h -mod enumobject; -// skipped errcode.h -mod eval; -// skipped exports.h -mod fileobject; -mod fileutils; -mod floatobject; -// skipped empty frameobject.h -#[cfg(not(Py_LIMITED_API))] -pub(crate) mod funcobject; -// skipped genericaliasobject.h -#[cfg(not(Py_LIMITED_API))] -mod genobject; -mod import; -// skipped interpreteridobject.h -mod intrcheck; -mod iterobject; -mod listobject; -// skipped longintrepr.h -mod longobject; -pub(crate) mod marshal; -mod memoryobject; -mod methodobject; -mod modsupport; -mod moduleobject; -// skipped namespaceobject.h -mod object; -mod objimpl; -// skipped odictobject.h -// skipped opcode.h -// skipped osdefs.h -mod osmodule; -// skipped parser_interface.h -// skipped patchlevel.h -// skipped picklebufobject.h -// skipped pyctype.h -// skipped py_curses.h -#[cfg(not(Py_LIMITED_API))] -mod pyarena; -mod pycapsule; -// skipped pydecimal.h -// skipped pydtrace.h -mod pyerrors; -// skipped pyexpat.h -// skipped pyfpe.h -mod pyframe; -mod pyhash; -mod pylifecycle; -// skipped pymacconfig.h -// skipped pymacro.h -// skipped pymath.h -mod pymem; -mod pyport; -mod pystate; -mod pythonrun; -// skipped pystrhex.h -// skipped pystrcmp.h -mod pystrtod; -// skipped pythread.h -// skipped pytime.h -mod rangeobject; -mod setobject; -mod sliceobject; -mod structseq; -mod sysmodule; -mod traceback; -// skipped tracemalloc.h -mod tupleobject; -mod typeslots; -mod unicodeobject; -mod warnings; -mod weakrefobject; - -// Additional headers that are not exported by Python.h -pub mod structmember; - -// "Limited API" definitions matching Python's `include/cpython` directory. -#[cfg(not(Py_LIMITED_API))] -mod cpython; - -#[cfg(not(Py_LIMITED_API))] -pub use self::cpython::*; +// reexport raw bindings exposed in pyo3_ffi +pub use pyo3_ffi::*; /// Helper to enable #\[pymethods\] to see the workaround for __ipow__ on Python 3.7 #[doc(hidden)] diff --git a/src/ffi/tests.rs b/src/ffi/tests.rs new file mode 100644 index 00000000..15b1eb36 --- /dev/null +++ b/src/ffi/tests.rs @@ -0,0 +1,183 @@ +use crate::ffi::*; +use crate::{types::PyDict, AsPyPointer, IntoPy, Py, PyAny, Python}; + +#[cfg(target_endian = "little")] +use crate::types::PyString; +#[cfg(target_endian = "little")] +use libc::wchar_t; + +#[test] +fn test_datetime_fromtimestamp() { + Python::with_gil(|py| { + let args: Py = (100,).into_py(py); + unsafe { PyDateTime_IMPORT() }; + let dt: &PyAny = unsafe { py.from_owned_ptr(PyDateTime_FromTimestamp(args.as_ptr())) }; + let locals = PyDict::new(py); + locals.set_item("dt", dt).unwrap(); + py.run( + "import datetime; assert dt == datetime.datetime.fromtimestamp(100)", + None, + Some(locals), + ) + .unwrap(); + }) +} + +#[test] +fn test_date_fromtimestamp() { + Python::with_gil(|py| { + let args: Py = (100,).into_py(py); + unsafe { PyDateTime_IMPORT() }; + let dt: &PyAny = unsafe { py.from_owned_ptr(PyDate_FromTimestamp(args.as_ptr())) }; + let locals = PyDict::new(py); + locals.set_item("dt", dt).unwrap(); + py.run( + "import datetime; assert dt == datetime.date.fromtimestamp(100)", + None, + Some(locals), + ) + .unwrap(); + }) +} + +#[test] +#[cfg(not(all(PyPy, not(Py_3_8))))] +fn test_utc_timezone() { + Python::with_gil(|py| { + let utc_timezone = unsafe { + PyDateTime_IMPORT(); + &*(&PyDateTime_TimeZone_UTC() as *const *mut crate::ffi::PyObject + as *const crate::PyObject) + }; + let locals = PyDict::new(py); + locals.set_item("utc_timezone", utc_timezone).unwrap(); + py.run( + "import datetime; assert utc_timezone is datetime.timezone.utc", + None, + Some(locals), + ) + .unwrap(); + }) +} + +#[cfg(target_endian = "little")] +#[test] +fn ascii_object_bitfield() { + let ob_base: PyObject = unsafe { std::mem::zeroed() }; + + let mut o = PyASCIIObject { + ob_base, + length: 0, + hash: 0, + state: 0, + wstr: std::ptr::null_mut() as *mut wchar_t, + }; + + unsafe { + assert_eq!(o.interned(), 0); + assert_eq!(o.kind(), 0); + assert_eq!(o.compact(), 0); + assert_eq!(o.ascii(), 0); + assert_eq!(o.ready(), 0); + + for i in 0..4 { + o.state = i; + assert_eq!(o.interned(), i); + } + + for i in 0..8 { + o.state = i << 2; + assert_eq!(o.kind(), i); + } + + o.state = 1 << 5; + assert_eq!(o.compact(), 1); + + o.state = 1 << 6; + assert_eq!(o.ascii(), 1); + + o.state = 1 << 7; + assert_eq!(o.ready(), 1); + } +} + +#[cfg(target_endian = "little")] +#[test] +#[cfg_attr(Py_3_10, allow(deprecated))] +fn ascii() { + Python::with_gil(|py| { + // This test relies on implementation details of PyString. + let s = PyString::new(py, "hello, world"); + let ptr = s.as_ptr(); + + unsafe { + let ascii_ptr = ptr as *mut PyASCIIObject; + let ascii = ascii_ptr.as_ref().unwrap(); + + assert_eq!(ascii.interned(), 0); + assert_eq!(ascii.kind(), PyUnicode_1BYTE_KIND); + assert_eq!(ascii.compact(), 1); + assert_eq!(ascii.ascii(), 1); + assert_eq!(ascii.ready(), 1); + + assert_eq!(PyUnicode_IS_ASCII(ptr), 1); + assert_eq!(PyUnicode_IS_COMPACT(ptr), 1); + assert_eq!(PyUnicode_IS_COMPACT_ASCII(ptr), 1); + + assert!(!PyUnicode_1BYTE_DATA(ptr).is_null()); + // 2 and 4 byte macros return nonsense for this string instance. + assert_eq!(PyUnicode_KIND(ptr), PyUnicode_1BYTE_KIND); + + assert!(!_PyUnicode_COMPACT_DATA(ptr).is_null()); + // _PyUnicode_NONCOMPACT_DATA isn't valid for compact strings. + assert!(!PyUnicode_DATA(ptr).is_null()); + + assert_eq!(PyUnicode_GET_LENGTH(ptr), s.len().unwrap() as _); + assert_eq!(PyUnicode_IS_READY(ptr), 1); + + // This has potential to mutate object. But it should be a no-op since + // we're already ready. + assert_eq!(PyUnicode_READY(ptr), 0); + } + }) +} + +#[cfg(target_endian = "little")] +#[test] +#[cfg_attr(Py_3_10, allow(deprecated))] +fn ucs4() { + Python::with_gil(|py| { + let s = "哈哈🐈"; + let py_string = PyString::new(py, s); + let ptr = py_string.as_ptr(); + + unsafe { + let ascii_ptr = ptr as *mut PyASCIIObject; + let ascii = ascii_ptr.as_ref().unwrap(); + + assert_eq!(ascii.interned(), 0); + assert_eq!(ascii.kind(), PyUnicode_4BYTE_KIND); + assert_eq!(ascii.compact(), 1); + assert_eq!(ascii.ascii(), 0); + assert_eq!(ascii.ready(), 1); + + assert_eq!(PyUnicode_IS_ASCII(ptr), 0); + assert_eq!(PyUnicode_IS_COMPACT(ptr), 1); + assert_eq!(PyUnicode_IS_COMPACT_ASCII(ptr), 0); + + assert!(!PyUnicode_4BYTE_DATA(ptr).is_null()); + assert_eq!(PyUnicode_KIND(ptr), PyUnicode_4BYTE_KIND); + + assert!(!_PyUnicode_COMPACT_DATA(ptr).is_null()); + // _PyUnicode_NONCOMPACT_DATA isn't valid for compact strings. + assert!(!PyUnicode_DATA(ptr).is_null()); + + assert_eq!(PyUnicode_GET_LENGTH(ptr), py_string.len().unwrap() as _); + assert_eq!(PyUnicode_IS_READY(ptr), 1); + + // This has potential to mutate object. But it should be a no-op since + // we're already ready. + assert_eq!(PyUnicode_READY(ptr), 0); + } + }) +} diff --git a/src/impl_/deprecations.rs b/src/impl_/deprecations.rs index c0faa0cf..72c8ff99 100644 --- a/src/impl_/deprecations.rs +++ b/src/impl_/deprecations.rs @@ -2,3 +2,9 @@ #[deprecated(since = "0.15.0", note = "use `fn __call__` instead of `#[call]`")] pub const CALL_ATTRIBUTE: () = (); + +#[deprecated( + since = "0.16.0", + note = "implement a `__traverse__` `#[pymethod]` instead of using `gc` option" +)] +pub const PYCLASS_GC_OPTION: () = (); diff --git a/src/impl_/extract_argument.rs b/src/impl_/extract_argument.rs index c5ef8e24..9d5333a7 100644 --- a/src/impl_/extract_argument.rs +++ b/src/impl_/extract_argument.rs @@ -6,6 +6,7 @@ use crate::{ FromPyObject, PyAny, PyErr, PyResult, Python, }; +/// The standard implementation of how PyO3 extracts a `#[pyfunction]` or `#[pymethod]` function argument. #[doc(hidden)] #[inline] pub fn extract_argument<'py, T>(obj: &'py PyAny, arg_name: &str) -> PyResult @@ -13,11 +14,87 @@ where T: FromPyObject<'py>, { match obj.extract() { - Ok(e) => Ok(e), + Ok(value) => Ok(value), Err(e) => Err(argument_extraction_error(obj.py(), arg_name, e)), } } +/// Alternative to [`extract_argument`] used for `Option` arguments (because they are implicitly treated +/// as optional if at the end of the positional parameters). +#[doc(hidden)] +#[inline] +pub fn extract_optional_argument<'py, T>( + obj: Option<&'py PyAny>, + arg_name: &str, +) -> PyResult> +where + T: FromPyObject<'py>, +{ + match obj { + Some(obj) => match obj.extract() { + Ok(value) => Ok(value), + Err(e) => Err(argument_extraction_error(obj.py(), arg_name, e)), + }, + None => Ok(None), + } +} + +/// Alternative to [`extract_argument`] used when the argument has a default value provided by an annotation. +#[doc(hidden)] +#[inline] +pub fn extract_argument_with_default<'py, T>( + obj: Option<&'py PyAny>, + arg_name: &str, + default: impl FnOnce() -> T, +) -> PyResult +where + T: FromPyObject<'py>, +{ + match obj { + Some(obj) => match obj.extract() { + Ok(value) => Ok(value), + Err(e) => Err(argument_extraction_error(obj.py(), arg_name, e)), + }, + None => Ok(default()), + } +} + +/// Alternative to [`extract_argument`] used when the argument has a `#[pyo3(from_py_with)]` annotation. +/// +/// # Safety +/// - `obj` must not be None (this helper is only used for required function arguments). +#[doc(hidden)] +#[inline] +pub fn from_py_with<'py, T>( + obj: &'py PyAny, + arg_name: &str, + extractor: impl FnOnce(&'py PyAny) -> PyResult, +) -> PyResult { + // Safety: obj is not None (see safety + match extractor(obj) { + Ok(value) => Ok(value), + Err(e) => Err(argument_extraction_error(obj.py(), arg_name, e)), + } +} + +/// Alternative to [`extract_argument`] used when the argument has a `#[pyo3(from_py_with)]` annotation and also a default value. +#[doc(hidden)] +#[inline] +pub fn from_py_with_with_default<'py, T>( + obj: Option<&'py PyAny>, + arg_name: &str, + extractor: impl FnOnce(&'py PyAny) -> PyResult, + default: impl FnOnce() -> T, +) -> PyResult { + match obj { + Some(obj) => match extractor(obj) { + Ok(value) => Ok(value), + Err(e) => Err(argument_extraction_error(obj.py(), arg_name, e)), + }, + None => Ok(default()), + } +} + /// Adds the argument name to the error message of an error which occurred during argument extraction. /// /// Only modifies TypeError. (Cannot guarantee all exceptions have constructors from @@ -26,12 +103,32 @@ where #[cold] pub fn argument_extraction_error(py: Python, arg_name: &str, error: PyErr) -> PyErr { if error.get_type(py) == PyTypeError::type_object(py) { - PyTypeError::new_err(format!("argument '{}': {}", arg_name, error.value(py))) + let remapped_error = + PyTypeError::new_err(format!("argument '{}': {}", arg_name, error.value(py))); + remapped_error.set_cause(py, error.cause(py)); + remapped_error } else { error } } +/// Unwraps the Option<&PyAny> produced by the FunctionDescription `extract_arguments_` methods. +/// They check if required methods are all provided. +/// +/// # Safety +/// `argument` must not be `None` +#[doc(hidden)] +#[inline] +pub unsafe fn unwrap_required_argument(argument: Option<&PyAny>) -> &PyAny { + match argument { + Some(value) => value, + #[cfg(debug_assertions)] + None => unreachable!("required method argument was not extracted"), + #[cfg(not(debug_assertions))] + None => std::hint::unreachable_unchecked(), + } +} + pub struct KeywordOnlyParameterDescription { pub name: &'static str, pub required: bool, @@ -45,8 +142,6 @@ pub struct FunctionDescription { pub positional_only_parameters: usize, pub required_positional_parameters: usize, pub keyword_only_parameters: &'static [KeywordOnlyParameterDescription], - pub accept_varargs: bool, - pub accept_varkeywords: bool, } impl FunctionDescription { @@ -58,81 +153,30 @@ impl FunctionDescription { } } - /// Wrapper around `extract_arguments` which uses the Python C-API "fastcall" convention. + /// Equivalent of `extract_arguments_tuple_dict` which uses the Python C-API "fastcall" convention. /// /// # Safety /// - `args` must be a pointer to a C-style array of valid `ffi::PyObject` pointers. /// - `kwnames` must be a pointer to a PyTuple, or NULL. /// - `nargs + kwnames.len()` is the total length of the `args` array. #[cfg(not(Py_LIMITED_API))] - pub unsafe fn extract_arguments_fastcall<'py>( + pub unsafe fn extract_arguments_fastcall<'py, V, K>( &self, py: Python<'py>, args: *const *mut ffi::PyObject, nargs: ffi::Py_ssize_t, kwnames: *mut ffi::PyObject, output: &mut [Option<&'py PyAny>], - ) -> PyResult<(Option<&'py PyTuple>, Option<&'py PyDict>)> { - let kwnames: Option<&PyTuple> = py.from_borrowed_ptr_or_opt(kwnames); - // Safety: &PyAny has the same memory layout as `*mut ffi::PyObject` - let args = args as *const &PyAny; - let kwargs = if let Option::Some(kwnames) = kwnames { - ::std::slice::from_raw_parts(args.offset(nargs), kwnames.len()) - } else { - &[] - }; - let args = std::slice::from_raw_parts(args, nargs as usize); - self.extract_arguments( - py, - args.iter().copied(), - kwnames.map(|kwnames| { - kwnames - .as_slice() - .iter() - .copied() - .zip(kwargs.iter().copied()) - }), - output, - ) - } + ) -> PyResult<(V::Varargs, K::Varkeywords)> + where + V: VarargsHandler<'py>, + K: VarkeywordsHandler<'py>, + { + // Safety: Option<&PyAny> has the same memory layout as `*mut ffi::PyObject` + let args = args as *const Option<&PyAny>; + let positional_args_provided = nargs as usize; + let args_slice = std::slice::from_raw_parts(args, positional_args_provided); - /// Wrapper around `extract_arguments` which uses the - /// tuple-and-dict Python call convention. - /// - /// # Safety - /// - `args` must be a pointer to a PyTuple. - /// - `kwargs` must be a pointer to a PyDict, or NULL. - pub unsafe fn extract_arguments_tuple_dict<'py>( - &self, - py: Python<'py>, - args: *mut ffi::PyObject, - kwargs: *mut ffi::PyObject, - output: &mut [Option<&'py PyAny>], - ) -> PyResult<(Option<&'py PyTuple>, Option<&'py PyDict>)> { - let args = py.from_borrowed_ptr::(args); - let kwargs: ::std::option::Option<&PyDict> = py.from_borrowed_ptr_or_opt(kwargs); - self.extract_arguments(py, args.iter(), kwargs.map(|dict| dict.iter()), output) - } - - /// Extracts the `args` and `kwargs` provided into `output`, according to this function - /// definition. - /// - /// `output` must have the same length as this function has positional and keyword-only - /// parameters (as per the `positional_parameter_names` and `keyword_only_parameters` - /// respectively). - /// - /// If `accept_varargs` or `accept_varkeywords`, then the returned `&PyTuple` and `&PyDict` may - /// be `Some` if there are extra arguments. - /// - /// Unexpected, duplicate or invalid arguments will cause this function to return `TypeError`. - #[inline] - fn extract_arguments<'py>( - &self, - py: Python<'py>, - mut args: impl ExactSizeIterator, - kwargs: Option>, - output: &mut [Option<&'py PyAny>], - ) -> PyResult<(Option<&'py PyTuple>, Option<&'py PyDict>)> { let num_positional_parameters = self.positional_parameter_names.len(); debug_assert!(self.positional_only_parameters <= num_positional_parameters); @@ -142,129 +186,202 @@ impl FunctionDescription { num_positional_parameters + self.keyword_only_parameters.len() ); - // Handle positional arguments - let args_provided = { - let args_provided = args.len(); - if self.accept_varargs { - std::cmp::min(num_positional_parameters, args_provided) - } else if args_provided > num_positional_parameters { - return Err(self.too_many_positional_arguments(args_provided)); - } else { - args_provided - } - }; - - // Copy positional arguments into output - for (out, arg) in output[..args_provided].iter_mut().zip(args.by_ref()) { - *out = Some(arg); - } - - // Collect varargs into tuple - let varargs = if self.accept_varargs { - Some(PyTuple::new(py, args)) + let varargs = if positional_args_provided > num_positional_parameters { + let (positional_parameters, varargs) = args_slice.split_at(num_positional_parameters); + output[..num_positional_parameters].copy_from_slice(positional_parameters); + V::handle_varargs_fastcall(py, varargs, self)? } else { - None + output[..positional_args_provided].copy_from_slice(args_slice); + V::handle_varargs_fastcall(py, &[], self)? }; // Handle keyword arguments - let varkeywords = match (kwargs, self.accept_varkeywords) { - (Some(kwargs), true) => { - let mut varkeywords = None; - self.extract_keyword_arguments(kwargs, output, |name, value| { - varkeywords - .get_or_insert_with(|| PyDict::new(py)) - .set_item(name, value) - })?; - varkeywords - } - (Some(kwargs), false) => { - self.extract_keyword_arguments( - kwargs, - output, - #[cold] - |name, _| Err(self.unexpected_keyword_argument(name)), - )?; - None - } - (None, _) => None, - }; + let mut varkeywords = Default::default(); + if let Some(kwnames) = py.from_borrowed_ptr_or_opt::(kwnames) { + let mut positional_only_keyword_arguments = Vec::new(); - // Check that there's sufficient positional arguments once keyword arguments are specified - if args_provided < self.required_positional_parameters { - for out in &output[..self.required_positional_parameters] { + // Safety: &PyAny has the same memory layout as `*mut ffi::PyObject` + let kwargs = + ::std::slice::from_raw_parts((args as *const &PyAny).offset(nargs), kwnames.len()); + + for (kwarg_name_py, &value) in kwnames.iter().zip(kwargs) { + // All keyword arguments should be UTF8 strings, but we'll check, just in case. + if let Ok(kwarg_name) = kwarg_name_py.downcast::()?.to_str() { + // Try to place parameter in keyword only parameters + if let Some(i) = self.find_keyword_parameter_in_keyword_only(kwarg_name) { + if output[i + num_positional_parameters] + .replace(value) + .is_some() + { + return Err(self.multiple_values_for_argument(kwarg_name)); + } + continue; + } + + // Repeat for positional parameters + if let Some(i) = self.find_keyword_parameter_in_positional(kwarg_name) { + if i < self.positional_only_parameters { + positional_only_keyword_arguments.push(kwarg_name); + } else if output[i].replace(value).is_some() { + return Err(self.multiple_values_for_argument(kwarg_name)); + } + continue; + } + }; + + K::handle_unexpected_keyword(&mut varkeywords, kwarg_name_py, value, self)? + } + + if !positional_only_keyword_arguments.is_empty() { + return Err( + self.positional_only_keyword_arguments(&positional_only_keyword_arguments) + ); + } + } + + // Once all inputs have been processed, check that all required arguments have been provided. + + self.ensure_no_missing_required_positional_arguments(output, positional_args_provided)?; + self.ensure_no_missing_required_keyword_arguments(output)?; + + Ok((varargs, varkeywords)) + } + + /// Extracts the `args` and `kwargs` provided into `output`, according to this function + /// definition. + /// + /// `output` must have the same length as this function has positional and keyword-only + /// parameters (as per the `positional_parameter_names` and `keyword_only_parameters` + /// respectively). + /// + /// Unexpected, duplicate or invalid arguments will cause this function to return `TypeError`. + /// + /// # Safety + /// - `args` must be a pointer to a PyTuple. + /// - `kwargs` must be a pointer to a PyDict, or NULL. + pub unsafe fn extract_arguments_tuple_dict<'py, V, K>( + &self, + py: Python<'py>, + args: *mut ffi::PyObject, + kwargs: *mut ffi::PyObject, + output: &mut [Option<&'py PyAny>], + ) -> PyResult<(V::Varargs, K::Varkeywords)> + where + V: VarargsHandler<'py>, + K: VarkeywordsHandler<'py>, + { + let args = py.from_borrowed_ptr::(args); + let kwargs: ::std::option::Option<&PyDict> = py.from_borrowed_ptr_or_opt(kwargs); + + let num_positional_parameters = self.positional_parameter_names.len(); + + debug_assert!(self.positional_only_parameters <= num_positional_parameters); + debug_assert!(self.required_positional_parameters <= num_positional_parameters); + debug_assert_eq!( + output.len(), + num_positional_parameters + self.keyword_only_parameters.len() + ); + + // Copy positional arguments into output + for (i, arg) in args.iter().take(num_positional_parameters).enumerate() { + output[i] = Some(arg); + } + + // If any arguments remain, push them to varargs (if possible) or error + let varargs = V::handle_varargs_tuple(args, self)?; + + // Handle keyword arguments + let mut varkeywords = Default::default(); + if let Some(kwargs) = kwargs { + let mut positional_only_keyword_arguments = Vec::new(); + for (kwarg_name_py, value) in kwargs { + // All keyword arguments should be UTF8 strings, but we'll check, just in case. + if let Ok(kwarg_name) = kwarg_name_py.downcast::()?.to_str() { + // Try to place parameter in keyword only parameters + if let Some(i) = self.find_keyword_parameter_in_keyword_only(kwarg_name) { + if output[i + num_positional_parameters] + .replace(value) + .is_some() + { + return Err(self.multiple_values_for_argument(kwarg_name)); + } + continue; + } + + // Repeat for positional parameters + if let Some(i) = self.find_keyword_parameter_in_positional(kwarg_name) { + if i < self.positional_only_parameters { + positional_only_keyword_arguments.push(kwarg_name); + } else if output[i].replace(value).is_some() { + return Err(self.multiple_values_for_argument(kwarg_name)); + } + continue; + } + }; + + K::handle_unexpected_keyword(&mut varkeywords, kwarg_name_py, value, self)? + } + + if !positional_only_keyword_arguments.is_empty() { + return Err( + self.positional_only_keyword_arguments(&positional_only_keyword_arguments) + ); + } + } + + // Once all inputs have been processed, check that all required arguments have been provided. + + self.ensure_no_missing_required_positional_arguments(output, args.len())?; + self.ensure_no_missing_required_keyword_arguments(output)?; + + Ok((varargs, varkeywords)) + } + + #[inline] + fn find_keyword_parameter_in_positional(&self, kwarg_name: &str) -> Option { + self.positional_parameter_names + .iter() + .position(|¶m_name| param_name == kwarg_name) + } + + #[inline] + fn find_keyword_parameter_in_keyword_only(&self, kwarg_name: &str) -> Option { + // Compare the keyword name against each parameter in turn. This is exactly the same method + // which CPython uses to map keyword names. Although it's O(num_parameters), the number of + // parameters is expected to be small so it's not worth constructing a mapping. + self.keyword_only_parameters + .iter() + .position(|param_desc| param_desc.name == kwarg_name) + } + + #[inline] + fn ensure_no_missing_required_positional_arguments( + &self, + output: &[Option<&PyAny>], + positional_args_provided: usize, + ) -> PyResult<()> { + if positional_args_provided < self.required_positional_parameters { + for out in &output[positional_args_provided..self.required_positional_parameters] { if out.is_none() { return Err(self.missing_required_positional_arguments(output)); } } } + Ok(()) + } - // Check no missing required keyword arguments - let keyword_output = &output[num_positional_parameters..]; + #[inline] + fn ensure_no_missing_required_keyword_arguments( + &self, + output: &[Option<&PyAny>], + ) -> PyResult<()> { + let keyword_output = &output[self.positional_parameter_names.len()..]; for (param, out) in self.keyword_only_parameters.iter().zip(keyword_output) { if param.required && out.is_none() { return Err(self.missing_required_keyword_arguments(keyword_output)); } } - - Ok((varargs, varkeywords)) - } - - fn extract_keyword_arguments<'py>( - &self, - kwargs: impl Iterator, - output: &mut [Option<&'py PyAny>], - mut unexpected_keyword_handler: impl FnMut(&'py PyAny, &'py PyAny) -> PyResult<()>, - ) -> PyResult<()> { - let positional_args_count = self.positional_parameter_names.len(); - let mut positional_only_keyword_arguments = Vec::new(); - 'for_each_kwarg: for (kwarg_name_py, value) in kwargs { - let kwarg_name = match kwarg_name_py.downcast::()?.to_str() { - Ok(kwarg_name) => kwarg_name, - // This keyword is not a UTF8 string: all PyO3 argument names are guaranteed to be - // UTF8 by construction. - Err(_) => { - unexpected_keyword_handler(kwarg_name_py, value)?; - continue; - } - }; - - // Compare the keyword name against each parameter in turn. This is exactly the same method - // which CPython uses to map keyword names. Although it's O(num_parameters), the number of - // parameters is expected to be small so it's not worth constructing a mapping. - for (i, param) in self.keyword_only_parameters.iter().enumerate() { - if param.name == kwarg_name { - output[positional_args_count + i] = Some(value); - continue 'for_each_kwarg; - } - } - - // Repeat for positional parameters - if let Some(i) = self.find_keyword_parameter_in_positionals(kwarg_name) { - if i < self.positional_only_parameters { - positional_only_keyword_arguments.push(kwarg_name); - } else if output[i].replace(value).is_some() { - return Err(self.multiple_values_for_argument(kwarg_name)); - } - continue; - } - - unexpected_keyword_handler(kwarg_name_py, value)?; - } - - if positional_only_keyword_arguments.is_empty() { - Ok(()) - } else { - Err(self.positional_only_keyword_arguments(&positional_only_keyword_arguments)) - } - } - - fn find_keyword_parameter_in_positionals(&self, kwarg_name: &str) -> Option { - for (i, param_name) in self.positional_parameter_names.iter().enumerate() { - if *param_name == kwarg_name { - return Some(i); - } - } - None + Ok(()) } #[cold] @@ -373,6 +490,129 @@ impl FunctionDescription { } } +/// A trait used to control whether to accept varargs in FunctionDescription::extract_argument_(method) functions. +pub trait VarargsHandler<'py> { + type Varargs; + /// Called by `FunctionDescription::extract_arguments_fastcall` with any additional arguments. + fn handle_varargs_fastcall( + py: Python<'py>, + varargs: &[Option<&PyAny>], + function_description: &FunctionDescription, + ) -> PyResult; + /// Called by `FunctionDescription::extract_arguments_tuple_dict` with the original tuple. + /// + /// Additional arguments are those in the tuple slice starting from `function_description.positional_parameter_names.len()`. + fn handle_varargs_tuple( + args: &'py PyTuple, + function_description: &FunctionDescription, + ) -> PyResult; +} + +/// Marker struct which indicates varargs are not allowed. +pub struct NoVarargs; + +impl<'py> VarargsHandler<'py> for NoVarargs { + type Varargs = (); + + #[inline] + fn handle_varargs_fastcall( + _py: Python<'py>, + varargs: &[Option<&PyAny>], + function_description: &FunctionDescription, + ) -> PyResult { + let extra_arguments = varargs.len(); + if extra_arguments > 0 { + return Err(function_description.too_many_positional_arguments( + function_description.positional_parameter_names.len() + extra_arguments, + )); + } + Ok(()) + } + + #[inline] + fn handle_varargs_tuple( + args: &'py PyTuple, + function_description: &FunctionDescription, + ) -> PyResult { + let positional_parameter_count = function_description.positional_parameter_names.len(); + let provided_args_count = args.len(); + if provided_args_count <= positional_parameter_count { + Ok(()) + } else { + Err(function_description.too_many_positional_arguments(provided_args_count)) + } + } +} + +/// Marker struct which indicates varargs should be collected into a `PyTuple`. +pub struct TupleVarargs; + +impl<'py> VarargsHandler<'py> for TupleVarargs { + type Varargs = &'py PyTuple; + #[inline] + fn handle_varargs_fastcall( + py: Python<'py>, + varargs: &[Option<&PyAny>], + _function_description: &FunctionDescription, + ) -> PyResult { + Ok(PyTuple::new(py, varargs)) + } + + #[inline] + fn handle_varargs_tuple( + args: &'py PyTuple, + function_description: &FunctionDescription, + ) -> PyResult { + let positional_parameters = function_description.positional_parameter_names.len(); + Ok(args.get_slice(positional_parameters, args.len())) + } +} + +/// A trait used to control whether to accept unrecognised keywords in FunctionDescription::extract_argument_(method) functions. +pub trait VarkeywordsHandler<'py> { + type Varkeywords: Default; + fn handle_unexpected_keyword( + varkeywords: &mut Self::Varkeywords, + name: &'py PyAny, + value: &'py PyAny, + function_description: &FunctionDescription, + ) -> PyResult<()>; +} + +/// Marker struct which indicates unknown keywords are not permitted. +pub struct NoVarkeywords; + +impl<'py> VarkeywordsHandler<'py> for NoVarkeywords { + type Varkeywords = (); + #[inline] + fn handle_unexpected_keyword( + _varkeywords: &mut Self::Varkeywords, + name: &'py PyAny, + _value: &'py PyAny, + function_description: &FunctionDescription, + ) -> PyResult<()> { + Err(function_description.unexpected_keyword_argument(name)) + } +} + +/// Marker struct which indicates unknown keywords should be collected into a `PyDict`. +pub struct DictVarkeywords; + +impl<'py> VarkeywordsHandler<'py> for DictVarkeywords { + type Varkeywords = Option<&'py PyDict>; + #[inline] + fn handle_unexpected_keyword( + varkeywords: &mut Self::Varkeywords, + name: &'py PyAny, + value: &'py PyAny, + _function_description: &FunctionDescription, + ) -> PyResult<()> { + varkeywords + .get_or_insert_with(|| PyDict::new(name.py())) + .set_item(name, value) + } +} + fn push_parameter_list(msg: &mut String, parameter_names: &[&str]) { for (i, parameter) in parameter_names.iter().enumerate() { if i != 0 { @@ -395,9 +635,12 @@ fn push_parameter_list(msg: &mut String, parameter_names: &[&str]) { #[cfg(test)] mod tests { - use crate::{types::PyTuple, AsPyPointer, PyAny, Python, ToPyObject}; + use crate::{ + types::{IntoPyDict, PyTuple}, + AsPyPointer, PyAny, Python, ToPyObject, + }; - use super::{push_parameter_list, FunctionDescription}; + use super::{push_parameter_list, FunctionDescription, NoVarargs, NoVarkeywords}; #[test] fn unexpected_keyword_argument() { @@ -408,26 +651,21 @@ mod tests { positional_only_parameters: 0, required_positional_parameters: 0, keyword_only_parameters: &[], - accept_varargs: false, - accept_varkeywords: false, }; Python::with_gil(|py| { - let err = function_description - .extract_arguments( - py, - [].iter().copied(), - Some( - [( - "foo".to_object(py).into_ref(py), - 1u8.to_object(py).into_ref(py), - )] - .iter() - .copied(), - ), - &mut [], - ) - .unwrap_err(); + let err = unsafe { + function_description + .extract_arguments_tuple_dict::( + py, + PyTuple::new(py, Vec::<&PyAny>::new()).as_ptr(), + [("foo".to_object(py).into_ref(py), 0u8)] + .into_py_dict(py) + .as_ptr(), + &mut [], + ) + .unwrap_err() + }; assert_eq!( err.to_string(), "TypeError: example() got an unexpected keyword argument 'foo'" @@ -444,26 +682,21 @@ mod tests { positional_only_parameters: 0, required_positional_parameters: 0, keyword_only_parameters: &[], - accept_varargs: false, - accept_varkeywords: false, }; Python::with_gil(|py| { - let err = function_description - .extract_arguments( - py, - [].iter().copied(), - Some( - [( - 1u8.to_object(py).into_ref(py), - 1u8.to_object(py).into_ref(py), - )] - .iter() - .copied(), - ), - &mut [], - ) - .unwrap_err(); + let err = unsafe { + function_description + .extract_arguments_tuple_dict::( + py, + PyTuple::new(py, Vec::<&PyAny>::new()).as_ptr(), + [(1u8.to_object(py).into_ref(py), 1u8)] + .into_py_dict(py) + .as_ptr(), + &mut [], + ) + .unwrap_err() + }; assert_eq!( err.to_string(), "TypeError: 'int' object cannot be converted to 'PyString'" @@ -480,14 +713,12 @@ mod tests { positional_only_parameters: 0, required_positional_parameters: 2, keyword_only_parameters: &[], - accept_varargs: false, - accept_varkeywords: false, }; Python::with_gil(|py| { let mut output = [None, None]; let err = unsafe { - function_description.extract_arguments_tuple_dict( + function_description.extract_arguments_tuple_dict::( py, PyTuple::new(py, Vec::<&PyAny>::new()).as_ptr(), std::ptr::null_mut(), diff --git a/src/impl_/pyclass.rs b/src/impl_/pyclass.rs index ae534611..abf7fb26 100644 --- a/src/impl_/pyclass.rs +++ b/src/impl_/pyclass.rs @@ -1,4 +1,18 @@ -use crate::{ffi, PyCell, PyClass, Python}; +use crate::{ + exceptions::{PyAttributeError, PyNotImplementedError}, + ffi, + impl_::freelist::FreeList, + pycell::PyCellLayout, + pyclass_init::PyObjectInit, + type_object::{PyLayout, PyTypeObject}, + PyCell, PyClass, PyMethodDefType, PyNativeType, PyResult, PyTypeInfo, Python, +}; +use std::{ + marker::PhantomData, + os::raw::{c_int, c_void}, + ptr::NonNull, + thread, +}; /// Gets the offset of the dictionary from the start of the object in bytes. #[inline] @@ -94,3 +108,743 @@ impl PyClassWeakRef for PyClassWeakRefSlot { } } } + +/// This type is used as a "dummy" type on which dtolnay specializations are +/// applied to apply implementations from `#[pymethods]` & `#[pyproto]` +pub struct PyClassImplCollector(PhantomData); + +impl PyClassImplCollector { + pub fn new() -> Self { + Self(PhantomData) + } +} + +impl Default for PyClassImplCollector { + fn default() -> Self { + Self::new() + } +} + +impl Clone for PyClassImplCollector { + fn clone(&self) -> Self { + Self::new() + } +} + +impl Copy for PyClassImplCollector {} + +pub struct PyClassItems { + pub methods: &'static [PyMethodDefType], + pub slots: &'static [ffi::PyType_Slot], +} + +// Allow PyClassItems in statics +unsafe impl Sync for PyClassItems {} + +/// Implements the underlying functionality of `#[pyclass]`, assembled by various proc macros. +/// +/// Users are discouraged from implementing this trait manually; it is a PyO3 implementation detail +/// and may be changed at any time. +pub trait PyClassImpl: Sized { + /// Class doc string + const DOC: &'static str = "\0"; + + /// #[pyclass(subclass)] + const IS_BASETYPE: bool = false; + + /// #[pyclass(extends=...)] + const IS_SUBCLASS: bool = false; + + /// Layout + type Layout: PyLayout; + + /// Base class + type BaseType: PyTypeInfo + PyTypeObject + PyClassBaseType; + + /// This handles following two situations: + /// 1. In case `T` is `Send`, stub `ThreadChecker` is used and does nothing. + /// This implementation is used by default. Compile fails if `T: !Send`. + /// 2. In case `T` is `!Send`, `ThreadChecker` panics when `T` is accessed by another thread. + /// This implementation is used when `#[pyclass(unsendable)]` is given. + /// Panicking makes it safe to expose `T: !Send` to the Python interpreter, where all objects + /// can be accessed by multiple threads by `threading` module. + type ThreadChecker: PyClassThreadChecker; + + #[cfg(feature = "multiple-pymethods")] + type Inventory: PyClassInventory; + + fn for_all_items(visitor: &mut dyn FnMut(&PyClassItems)); + + #[inline] + fn dict_offset() -> Option { + None + } + #[inline] + fn weaklist_offset() -> Option { + None + } +} + +// Traits describing known special methods. + +macro_rules! slot_fragment_trait { + ($trait_name:ident, $($default_method:tt)*) => { + #[allow(non_camel_case_types)] + pub trait $trait_name: Sized { + $($default_method)* + } + + impl $trait_name for &'_ PyClassImplCollector {} + } +} + +/// Macro which expands to three items +/// - Trait for a __setitem__ dunder +/// - Trait for the corresponding __delitem__ dunder +/// - A macro which will use dtolnay specialisation to generate the shared slot for the two dunders +macro_rules! define_pyclass_setattr_slot { + ( + $set_trait:ident, + $del_trait:ident, + $set:ident, + $del:ident, + $set_error:expr, + $del_error:expr, + $generate_macro:ident, + $slot:ident, + $func_ty:ident, + ) => { + slot_fragment_trait! { + $set_trait, + + /// # Safety: _slf and _attr must be valid non-null Python objects + #[inline] + unsafe fn $set( + self, + _py: Python, + _slf: *mut ffi::PyObject, + _attr: *mut ffi::PyObject, + _value: NonNull, + ) -> PyResult<()> { + $set_error + } + } + + slot_fragment_trait! { + $del_trait, + + /// # Safety: _slf and _attr must be valid non-null Python objects + #[inline] + unsafe fn $del( + self, + _py: Python, + _slf: *mut ffi::PyObject, + _attr: *mut ffi::PyObject, + ) -> PyResult<()> { + $del_error + } + } + + #[doc(hidden)] + #[macro_export] + macro_rules! $generate_macro { + ($cls:ty) => {{ + unsafe extern "C" fn __wrap( + _slf: *mut $crate::ffi::PyObject, + attr: *mut $crate::ffi::PyObject, + value: *mut $crate::ffi::PyObject, + ) -> ::std::os::raw::c_int { + use ::std::option::Option::*; + use $crate::callback::IntoPyCallbackOutput; + use $crate::impl_::pyclass::*; + let gil = $crate::GILPool::new(); + let py = gil.python(); + $crate::callback::panic_result_into_callback_output( + py, + ::std::panic::catch_unwind(move || -> $crate::PyResult<_> { + let collector = PyClassImplCollector::<$cls>::new(); + if let Some(value) = ::std::ptr::NonNull::new(value) { + collector.$set(py, _slf, attr, value).convert(py) + } else { + collector.$del(py, _slf, attr).convert(py) + } + }), + ) + } + $crate::ffi::PyType_Slot { + slot: $crate::ffi::$slot, + pfunc: __wrap as $crate::ffi::$func_ty as _, + } + }}; + } + pub use $generate_macro; + }; +} + +define_pyclass_setattr_slot! { + PyClass__setattr__SlotFragment, + PyClass__delattr__SlotFragment, + __setattr__, + __delattr__, + Err(PyAttributeError::new_err("can't set attribute")), + Err(PyAttributeError::new_err("can't delete attribute")), + generate_pyclass_setattr_slot, + Py_tp_setattro, + setattrofunc, +} + +define_pyclass_setattr_slot! { + PyClass__set__SlotFragment, + PyClass__delete__SlotFragment, + __set__, + __delete__, + Err(PyNotImplementedError::new_err("can't set descriptor")), + Err(PyNotImplementedError::new_err("can't delete descriptor")), + generate_pyclass_setdescr_slot, + Py_tp_descr_set, + descrsetfunc, +} + +define_pyclass_setattr_slot! { + PyClass__setitem__SlotFragment, + PyClass__delitem__SlotFragment, + __setitem__, + __delitem__, + Err(PyNotImplementedError::new_err("can't set item")), + Err(PyNotImplementedError::new_err("can't delete item")), + generate_pyclass_setitem_slot, + Py_mp_ass_subscript, + objobjargproc, +} + +/// Macro which expands to three items +/// - Trait for a lhs dunder e.g. __add__ +/// - Trait for the corresponding rhs e.g. __radd__ +/// - A macro which will use dtolnay specialisation to generate the shared slot for the two dunders +macro_rules! define_pyclass_binary_operator_slot { + ( + $lhs_trait:ident, + $rhs_trait:ident, + $lhs:ident, + $rhs:ident, + $generate_macro:ident, + $slot:ident, + $func_ty:ident, + ) => { + slot_fragment_trait! { + $lhs_trait, + + /// # Safety: _slf and _other must be valid non-null Python objects + #[inline] + unsafe fn $lhs( + self, + _py: Python, + _slf: *mut ffi::PyObject, + _other: *mut ffi::PyObject, + ) -> PyResult<*mut ffi::PyObject> { + Ok(ffi::_Py_NewRef(ffi::Py_NotImplemented())) + } + } + + slot_fragment_trait! { + $rhs_trait, + + /// # Safety: _slf and _other must be valid non-null Python objects + #[inline] + unsafe fn $rhs( + self, + _py: Python, + _slf: *mut ffi::PyObject, + _other: *mut ffi::PyObject, + ) -> PyResult<*mut ffi::PyObject> { + Ok(ffi::_Py_NewRef(ffi::Py_NotImplemented())) + } + } + + #[doc(hidden)] + #[macro_export] + macro_rules! $generate_macro { + ($cls:ty) => {{ + unsafe extern "C" fn __wrap( + _slf: *mut $crate::ffi::PyObject, + _other: *mut $crate::ffi::PyObject, + ) -> *mut $crate::ffi::PyObject { + let gil = $crate::GILPool::new(); + let py = gil.python(); + $crate::callback::panic_result_into_callback_output( + py, + ::std::panic::catch_unwind(move || -> $crate::PyResult<_> { + use $crate::impl_::pyclass::*; + let collector = PyClassImplCollector::<$cls>::new(); + let lhs_result = collector.$lhs(py, _slf, _other)?; + if lhs_result == $crate::ffi::Py_NotImplemented() { + $crate::ffi::Py_DECREF(lhs_result); + collector.$rhs(py, _other, _slf) + } else { + ::std::result::Result::Ok(lhs_result) + } + }), + ) + } + $crate::ffi::PyType_Slot { + slot: $crate::ffi::$slot, + pfunc: __wrap as $crate::ffi::$func_ty as _, + } + }}; + } + pub use $generate_macro; + }; +} + +define_pyclass_binary_operator_slot! { + PyClass__add__SlotFragment, + PyClass__radd__SlotFragment, + __add__, + __radd__, + generate_pyclass_add_slot, + Py_nb_add, + binaryfunc, +} + +define_pyclass_binary_operator_slot! { + PyClass__sub__SlotFragment, + PyClass__rsub__SlotFragment, + __sub__, + __rsub__, + generate_pyclass_sub_slot, + Py_nb_subtract, + binaryfunc, +} + +define_pyclass_binary_operator_slot! { + PyClass__mul__SlotFragment, + PyClass__rmul__SlotFragment, + __mul__, + __rmul__, + generate_pyclass_mul_slot, + Py_nb_multiply, + binaryfunc, +} + +define_pyclass_binary_operator_slot! { + PyClass__mod__SlotFragment, + PyClass__rmod__SlotFragment, + __mod__, + __rmod__, + generate_pyclass_mod_slot, + Py_nb_remainder, + binaryfunc, +} + +define_pyclass_binary_operator_slot! { + PyClass__divmod__SlotFragment, + PyClass__rdivmod__SlotFragment, + __divmod__, + __rdivmod__, + generate_pyclass_divmod_slot, + Py_nb_divmod, + binaryfunc, +} + +define_pyclass_binary_operator_slot! { + PyClass__lshift__SlotFragment, + PyClass__rlshift__SlotFragment, + __lshift__, + __rlshift__, + generate_pyclass_lshift_slot, + Py_nb_lshift, + binaryfunc, +} + +define_pyclass_binary_operator_slot! { + PyClass__rshift__SlotFragment, + PyClass__rrshift__SlotFragment, + __rshift__, + __rrshift__, + generate_pyclass_rshift_slot, + Py_nb_rshift, + binaryfunc, +} + +define_pyclass_binary_operator_slot! { + PyClass__and__SlotFragment, + PyClass__rand__SlotFragment, + __and__, + __rand__, + generate_pyclass_and_slot, + Py_nb_and, + binaryfunc, +} + +define_pyclass_binary_operator_slot! { + PyClass__or__SlotFragment, + PyClass__ror__SlotFragment, + __or__, + __ror__, + generate_pyclass_or_slot, + Py_nb_or, + binaryfunc, +} + +define_pyclass_binary_operator_slot! { + PyClass__xor__SlotFragment, + PyClass__rxor__SlotFragment, + __xor__, + __rxor__, + generate_pyclass_xor_slot, + Py_nb_xor, + binaryfunc, +} + +define_pyclass_binary_operator_slot! { + PyClass__matmul__SlotFragment, + PyClass__rmatmul__SlotFragment, + __matmul__, + __rmatmul__, + generate_pyclass_matmul_slot, + Py_nb_matrix_multiply, + binaryfunc, +} + +define_pyclass_binary_operator_slot! { + PyClass__truediv__SlotFragment, + PyClass__rtruediv__SlotFragment, + __truediv__, + __rtruediv__, + generate_pyclass_truediv_slot, + Py_nb_true_divide, + binaryfunc, +} + +define_pyclass_binary_operator_slot! { + PyClass__floordiv__SlotFragment, + PyClass__rfloordiv__SlotFragment, + __floordiv__, + __rfloordiv__, + generate_pyclass_floordiv_slot, + Py_nb_floor_divide, + binaryfunc, +} + +slot_fragment_trait! { + PyClass__pow__SlotFragment, + + /// # Safety: _slf and _other must be valid non-null Python objects + #[inline] + unsafe fn __pow__( + self, + _py: Python, + _slf: *mut ffi::PyObject, + _other: *mut ffi::PyObject, + _mod: *mut ffi::PyObject, + ) -> PyResult<*mut ffi::PyObject> { + Ok(ffi::_Py_NewRef(ffi::Py_NotImplemented())) + } +} + +slot_fragment_trait! { + PyClass__rpow__SlotFragment, + + /// # Safety: _slf and _other must be valid non-null Python objects + #[inline] + unsafe fn __rpow__( + self, + _py: Python, + _slf: *mut ffi::PyObject, + _other: *mut ffi::PyObject, + _mod: *mut ffi::PyObject, + ) -> PyResult<*mut ffi::PyObject> { + Ok(ffi::_Py_NewRef(ffi::Py_NotImplemented())) + } +} + +#[doc(hidden)] +#[macro_export] +macro_rules! generate_pyclass_pow_slot { + ($cls:ty) => {{ + unsafe extern "C" fn __wrap( + _slf: *mut $crate::ffi::PyObject, + _other: *mut $crate::ffi::PyObject, + _mod: *mut $crate::ffi::PyObject, + ) -> *mut $crate::ffi::PyObject { + let gil = $crate::GILPool::new(); + let py = gil.python(); + $crate::callback::panic_result_into_callback_output( + py, + ::std::panic::catch_unwind(move || -> $crate::PyResult<_> { + use $crate::impl_::pyclass::*; + let collector = PyClassImplCollector::<$cls>::new(); + let lhs_result = collector.__pow__(py, _slf, _other, _mod)?; + if lhs_result == $crate::ffi::Py_NotImplemented() { + $crate::ffi::Py_DECREF(lhs_result); + collector.__rpow__(py, _other, _slf, _mod) + } else { + ::std::result::Result::Ok(lhs_result) + } + }), + ) + } + $crate::ffi::PyType_Slot { + slot: $crate::ffi::Py_nb_power, + pfunc: __wrap as $crate::ffi::ternaryfunc as _, + } + }}; +} +pub use generate_pyclass_pow_slot; + +/// Implements a freelist. +/// +/// Do not implement this trait manually. Instead, use `#[pyclass(freelist = N)]` +/// on a Rust struct to implement it. +pub trait PyClassWithFreeList: PyClass { + fn get_free_list(py: Python) -> &mut FreeList<*mut ffi::PyObject>; +} + +/// Implementation of tp_alloc for `freelist` classes. +/// +/// # Safety +/// - `subtype` must be a valid pointer to the type object of T or a subclass. +/// - The GIL must be held. +pub unsafe extern "C" fn alloc_with_freelist( + subtype: *mut ffi::PyTypeObject, + nitems: ffi::Py_ssize_t, +) -> *mut ffi::PyObject { + let py = Python::assume_gil_acquired(); + + #[cfg(not(Py_3_8))] + bpo_35810_workaround(py, subtype); + + let self_type = T::type_object_raw(py); + // If this type is a variable type or the subtype is not equal to this type, we cannot use the + // freelist + if nitems == 0 && subtype == self_type { + if let Some(obj) = T::get_free_list(py).pop() { + ffi::PyObject_Init(obj, subtype); + return obj as _; + } + } + + ffi::PyType_GenericAlloc(subtype, nitems) +} + +/// Implementation of tp_free for `freelist` classes. +/// +/// # Safety +/// - `obj` must be a valid pointer to an instance of T (not a subclass). +/// - The GIL must be held. +pub unsafe extern "C" fn free_with_freelist(obj: *mut c_void) { + let obj = obj as *mut ffi::PyObject; + debug_assert_eq!( + T::type_object_raw(Python::assume_gil_acquired()), + ffi::Py_TYPE(obj) + ); + if let Some(obj) = T::get_free_list(Python::assume_gil_acquired()).insert(obj) { + let ty = ffi::Py_TYPE(obj); + + // Deduce appropriate inverse of PyType_GenericAlloc + let free = if ffi::PyType_IS_GC(ty) != 0 { + ffi::PyObject_GC_Del + } else { + ffi::PyObject_Free + }; + free(obj as *mut c_void); + + #[cfg(Py_3_8)] + if ffi::PyType_HasFeature(ty, ffi::Py_TPFLAGS_HEAPTYPE) != 0 { + ffi::Py_DECREF(ty as *mut ffi::PyObject); + } + } +} + +/// Workaround for Python issue 35810; no longer necessary in Python 3.8 +#[inline] +#[cfg(not(Py_3_8))] +unsafe fn bpo_35810_workaround(_py: Python, ty: *mut ffi::PyTypeObject) { + #[cfg(Py_LIMITED_API)] + { + // Must check version at runtime for abi3 wheels - they could run against a higher version + // than the build config suggests. + use crate::once_cell::GILOnceCell; + static IS_PYTHON_3_8: GILOnceCell = GILOnceCell::new(); + + if *IS_PYTHON_3_8.get_or_init(_py, || _py.version_info() >= (3, 8)) { + // No fix needed - the wheel is running on a sufficiently new interpreter. + return; + } + } + + ffi::Py_INCREF(ty as *mut ffi::PyObject); +} + +// General methods implementation: either dtolnay specialization trait or inventory if +// multiple-pymethods feature is enabled. + +macro_rules! items_trait { + ($name:ident, $function_name: ident) => { + pub trait $name { + fn $function_name(self) -> &'static PyClassItems; + } + + impl $name for &'_ PyClassImplCollector { + fn $function_name(self) -> &'static PyClassItems { + &PyClassItems { + methods: &[], + slots: &[], + } + } + } + }; +} + +/// Implementation detail. Only to be used through our proc macro code. +/// Method storage for `#[pyclass]`. +/// Allows arbitrary `#[pymethod]` blocks to submit their methods, +/// which are eventually collected by `#[pyclass]`. +#[cfg(feature = "multiple-pymethods")] +pub trait PyClassInventory: inventory::Collect { + /// Returns the items for a single `#[pymethods] impl` block + fn items(&'static self) -> &'static PyClassItems; +} + +// Items from #[pymethods] if not using inventory. +#[cfg(not(feature = "multiple-pymethods"))] +items_trait!(PyMethods, py_methods); + +/// Items from `#[pyproto]` implementations +#[cfg(feature = "pyproto")] +mod pyproto_traits { + use super::*; + items_trait!(PyObjectProtocolItems, object_protocol_items); + items_trait!(PyDescrProtocolItems, descr_protocol_items); + items_trait!(PyGCProtocolItems, gc_protocol_items); + items_trait!(PyIterProtocolItems, iter_protocol_items); + items_trait!(PyMappingProtocolItems, mapping_protocol_items); + items_trait!(PyNumberProtocolItems, number_protocol_items); + items_trait!(PyAsyncProtocolItems, async_protocol_items); + items_trait!(PySequenceProtocolItems, sequence_protocol_items); + items_trait!(PyBufferProtocolItems, buffer_protocol_items); +} +#[cfg(feature = "pyproto")] +pub use pyproto_traits::*; + +// Protocol slots from #[pymethods] if not using inventory. +#[cfg(not(feature = "multiple-pymethods"))] +items_trait!(PyMethodsProtocolItems, methods_protocol_items); + +// Thread checkers + +#[doc(hidden)] +pub trait PyClassThreadChecker: Sized { + fn ensure(&self); + fn new() -> Self; + private_decl! {} +} + +/// Stub checker for `Send` types. +#[doc(hidden)] +pub struct ThreadCheckerStub(PhantomData); + +impl PyClassThreadChecker for ThreadCheckerStub { + fn ensure(&self) {} + #[inline] + fn new() -> Self { + ThreadCheckerStub(PhantomData) + } + private_impl! {} +} + +impl PyClassThreadChecker for ThreadCheckerStub { + fn ensure(&self) {} + #[inline] + fn new() -> Self { + ThreadCheckerStub(PhantomData) + } + private_impl! {} +} + +/// Thread checker for unsendable types. +/// Panics when the value is accessed by another thread. +#[doc(hidden)] +pub struct ThreadCheckerImpl(thread::ThreadId, PhantomData); + +impl PyClassThreadChecker for ThreadCheckerImpl { + fn ensure(&self) { + if thread::current().id() != self.0 { + panic!( + "{} is unsendable, but sent to another thread!", + std::any::type_name::() + ); + } + } + fn new() -> Self { + ThreadCheckerImpl(thread::current().id(), PhantomData) + } + private_impl! {} +} + +/// Thread checker for types that have `Send` and `extends=...`. +/// Ensures that `T: Send` and the parent is not accessed by another thread. +#[doc(hidden)] +pub struct ThreadCheckerInherited(PhantomData, U::ThreadChecker); + +impl PyClassThreadChecker for ThreadCheckerInherited { + fn ensure(&self) { + self.1.ensure(); + } + fn new() -> Self { + ThreadCheckerInherited(PhantomData, U::ThreadChecker::new()) + } + private_impl! {} +} + +/// Trait denoting that this class is suitable to be used as a base type for PyClass. +pub trait PyClassBaseType: Sized { + type LayoutAsBase: PyCellLayout; + type BaseNativeType; + type ThreadChecker: PyClassThreadChecker; + type Initializer: PyObjectInit; +} + +/// All PyClasses can be used as a base type. +impl PyClassBaseType for T { + type LayoutAsBase = crate::pycell::PyCell; + type BaseNativeType = T::BaseNativeType; + type ThreadChecker = T::ThreadChecker; + type Initializer = crate::pyclass_init::PyClassInitializer; +} + +/// Implementation of tp_dealloc for all pyclasses +pub(crate) unsafe extern "C" fn tp_dealloc(obj: *mut ffi::PyObject) { + crate::callback_body!(py, T::Layout::tp_dealloc(obj, py)) +} + +pub(crate) unsafe extern "C" fn get_sequence_item_from_mapping( + obj: *mut ffi::PyObject, + index: ffi::Py_ssize_t, +) -> *mut ffi::PyObject { + let index = ffi::PyLong_FromSsize_t(index); + if index.is_null() { + return std::ptr::null_mut(); + } + let result = ffi::PyObject_GetItem(obj, index); + ffi::Py_DECREF(index); + result +} + +pub(crate) unsafe extern "C" fn assign_sequence_item_from_mapping( + obj: *mut ffi::PyObject, + index: ffi::Py_ssize_t, + value: *mut ffi::PyObject, +) -> c_int { + let index = ffi::PyLong_FromSsize_t(index); + if index.is_null() { + return -1; + } + let result = if value.is_null() { + ffi::PyObject_DelItem(obj, index) + } else { + ffi::PyObject_SetItem(obj, index, value) + }; + ffi::Py_DECREF(index); + result +} diff --git a/src/impl_/pyfunction.rs b/src/impl_/pyfunction.rs index f852e63b..01cd3f08 100644 --- a/src/impl_/pyfunction.rs +++ b/src/impl_/pyfunction.rs @@ -1,5 +1,5 @@ use crate::{ - class::methods::PyMethodDef, derive_utils::PyFunctionArguments, types::PyCFunction, PyResult, + derive_utils::PyFunctionArguments, impl_::pymethods::PyMethodDef, types::PyCFunction, PyResult, }; pub trait PyFunctionDef { diff --git a/src/impl_/pymethods.rs b/src/impl_/pymethods.rs index 2a8d743a..b53c7db4 100644 --- a/src/impl_/pymethods.rs +++ b/src/impl_/pymethods.rs @@ -1,4 +1,8 @@ -use crate::{ffi, FromPyObject, PyAny, PyResult, Python}; +use crate::internal_tricks::{extract_cstr_or_leak_cstring, NulByteInString}; +use crate::{ffi, FromPyObject, PyAny, PyObject, PyResult, Python}; +use std::ffi::CStr; +use std::fmt; +use std::os::raw::c_int; /// Python 3.8 and up - __ipow__ has modulo argument correctly populated. #[cfg(Py_3_8)] @@ -31,3 +35,220 @@ impl IPowModulo { unsafe { py.from_borrowed_ptr::(ffi::Py_None()) }.extract() } } + +/// `PyMethodDefType` represents different types of Python callable objects. +/// It is used by the `#[pymethods]` and `#[pyproto]` annotations. +#[derive(Debug)] +pub enum PyMethodDefType { + /// Represents class method + Class(PyMethodDef), + /// Represents static method + Static(PyMethodDef), + /// Represents normal method + Method(PyMethodDef), + /// Represents class attribute, used by `#[attribute]` + ClassAttribute(PyClassAttributeDef), + /// Represents getter descriptor, used by `#[getter]` + Getter(PyGetterDef), + /// Represents setter descriptor, used by `#[setter]` + Setter(PySetterDef), +} + +#[derive(Copy, Clone, Debug)] +pub enum PyMethodType { + PyCFunction(PyCFunction), + PyCFunctionWithKeywords(PyCFunctionWithKeywords), + #[cfg(not(Py_LIMITED_API))] + PyCFunctionFastWithKeywords(PyCFunctionFastWithKeywords), +} + +// These newtype structs serve no purpose other than wrapping which are function pointers - because +// function pointers aren't allowed in const fn, but types wrapping them are! +#[derive(Clone, Copy, Debug)] +pub struct PyCFunction(pub ffi::PyCFunction); +#[derive(Clone, Copy, Debug)] +pub struct PyCFunctionWithKeywords(pub ffi::PyCFunctionWithKeywords); +#[cfg(not(Py_LIMITED_API))] +#[derive(Clone, Copy, Debug)] +pub struct PyCFunctionFastWithKeywords(pub ffi::_PyCFunctionFastWithKeywords); +#[derive(Clone, Copy, Debug)] +pub struct PyGetter(pub ffi::getter); +#[derive(Clone, Copy, Debug)] +pub struct PySetter(pub ffi::setter); +#[derive(Clone, Copy)] +pub struct PyClassAttributeFactory(pub for<'p> fn(Python<'p>) -> PyObject); + +// TODO: it would be nice to use CStr in these types, but then the constructors can't be const fn +// until `CStr::from_bytes_with_nul_unchecked` is const fn. + +#[derive(Clone, Debug)] +pub struct PyMethodDef { + pub(crate) ml_name: &'static str, + pub(crate) ml_meth: PyMethodType, + pub(crate) ml_flags: c_int, + pub(crate) ml_doc: &'static str, +} + +#[derive(Copy, Clone)] +pub struct PyClassAttributeDef { + pub(crate) name: &'static str, + pub(crate) meth: PyClassAttributeFactory, +} + +#[derive(Clone, Debug)] +pub struct PyGetterDef { + pub(crate) name: &'static str, + pub(crate) meth: PyGetter, + doc: &'static str, +} + +#[derive(Clone, Debug)] +pub struct PySetterDef { + pub(crate) name: &'static str, + pub(crate) meth: PySetter, + doc: &'static str, +} + +unsafe impl Sync for PyMethodDef {} + +unsafe impl Sync for PyGetterDef {} + +unsafe impl Sync for PySetterDef {} + +impl PyMethodDef { + /// Define a function with no `*args` and `**kwargs`. + pub const fn noargs(name: &'static str, cfunction: PyCFunction, doc: &'static str) -> Self { + Self { + ml_name: name, + ml_meth: PyMethodType::PyCFunction(cfunction), + ml_flags: ffi::METH_NOARGS, + ml_doc: doc, + } + } + + /// Define a function that can take `*args` and `**kwargs`. + pub const fn cfunction_with_keywords( + name: &'static str, + cfunction: PyCFunctionWithKeywords, + doc: &'static str, + ) -> Self { + Self { + ml_name: name, + ml_meth: PyMethodType::PyCFunctionWithKeywords(cfunction), + ml_flags: ffi::METH_VARARGS | ffi::METH_KEYWORDS, + ml_doc: doc, + } + } + + /// Define a function that can take `*args` and `**kwargs`. + #[cfg(not(Py_LIMITED_API))] + pub const fn fastcall_cfunction_with_keywords( + name: &'static str, + cfunction: PyCFunctionFastWithKeywords, + doc: &'static str, + ) -> Self { + Self { + ml_name: name, + ml_meth: PyMethodType::PyCFunctionFastWithKeywords(cfunction), + ml_flags: ffi::METH_FASTCALL | ffi::METH_KEYWORDS, + ml_doc: doc, + } + } + + pub const fn flags(mut self, flags: c_int) -> Self { + self.ml_flags |= flags; + self + } + + /// Convert `PyMethodDef` to Python method definition struct `ffi::PyMethodDef` + pub(crate) fn as_method_def(&self) -> Result { + let meth = match self.ml_meth { + PyMethodType::PyCFunction(meth) => ffi::PyMethodDefPointer { + PyCFunction: meth.0, + }, + PyMethodType::PyCFunctionWithKeywords(meth) => ffi::PyMethodDefPointer { + PyCFunctionWithKeywords: meth.0, + }, + #[cfg(not(Py_LIMITED_API))] + PyMethodType::PyCFunctionFastWithKeywords(meth) => ffi::PyMethodDefPointer { + _PyCFunctionFastWithKeywords: meth.0, + }, + }; + + Ok(ffi::PyMethodDef { + ml_name: get_name(self.ml_name)?.as_ptr(), + ml_meth: meth, + ml_flags: self.ml_flags, + ml_doc: get_doc(self.ml_doc)?.as_ptr(), + }) + } +} + +impl PyClassAttributeDef { + /// Define a class attribute. + pub const fn new(name: &'static str, meth: PyClassAttributeFactory) -> Self { + Self { name, meth } + } +} + +// Manual implementation because `Python<'_>` does not implement `Debug` and +// trait bounds on `fn` compiler-generated derive impls are too restrictive. +impl fmt::Debug for PyClassAttributeDef { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PyClassAttributeDef") + .field("name", &self.name) + .finish() + } +} + +impl PyGetterDef { + /// Define a getter. + pub const fn new(name: &'static str, getter: PyGetter, doc: &'static str) -> Self { + Self { + name, + meth: getter, + doc, + } + } + + /// Copy descriptor information to `ffi::PyGetSetDef` + pub fn copy_to(&self, dst: &mut ffi::PyGetSetDef) { + if dst.name.is_null() { + dst.name = get_name(self.name).unwrap().as_ptr() as _; + } + if dst.doc.is_null() { + dst.doc = get_doc(self.doc).unwrap().as_ptr() as _; + } + dst.get = Some(self.meth.0); + } +} + +impl PySetterDef { + /// Define a setter. + pub const fn new(name: &'static str, setter: PySetter, doc: &'static str) -> Self { + Self { + name, + meth: setter, + doc, + } + } + + /// Copy descriptor information to `ffi::PyGetSetDef` + pub fn copy_to(&self, dst: &mut ffi::PyGetSetDef) { + if dst.name.is_null() { + dst.name = get_name(self.name).unwrap().as_ptr() as _; + } + if dst.doc.is_null() { + dst.doc = get_doc(self.doc).unwrap().as_ptr() as _; + } + dst.set = Some(self.meth.0); + } +} + +fn get_name(name: &'static str) -> Result<&'static CStr, NulByteInString> { + extract_cstr_or_leak_cstring(name, "Function name cannot contain NUL byte.") +} + +fn get_doc(doc: &'static str) -> Result<&'static CStr, NulByteInString> { + extract_cstr_or_leak_cstring(doc, "Document cannot contain NUL byte.") +} diff --git a/src/impl_/pymodule.rs b/src/impl_/pymodule.rs index 11ab08e7..8d7a843b 100644 --- a/src/impl_/pymodule.rs +++ b/src/impl_/pymodule.rs @@ -1,9 +1,10 @@ //! Implementation details of `#[pymodule]` which need to be accessible from proc-macro generated code. -use std::{cell::UnsafeCell, panic::AssertUnwindSafe}; +use std::cell::UnsafeCell; use crate::{ - callback::handle_panic, ffi, types::PyModule, IntoPyPointer, Py, PyObject, PyResult, Python, + callback::panic_result_into_callback_output, ffi, types::PyModule, GILPool, IntoPyPointer, Py, + PyObject, PyResult, Python, }; /// `Sync` wrapper of `ffi::PyModuleDef`. @@ -64,8 +65,15 @@ impl ModuleDef { /// # Safety /// The Python GIL must be held. pub unsafe fn module_init(&'static self) -> *mut ffi::PyObject { - let unwind_safe_self = AssertUnwindSafe(self); - handle_panic(|py| Ok(unwind_safe_self.make_module(py)?.into_ptr())) + let pool = GILPool::new(); + let py = pool.python(); + let unwind_safe_self = std::panic::AssertUnwindSafe(self); + panic_result_into_callback_output( + py, + std::panic::catch_unwind(move || -> PyResult<_> { + Ok(unwind_safe_self.make_module(py)?.into_ptr()) + }), + ) } } diff --git a/src/instance.rs b/src/instance.rs index 0037abb8..6fd84098 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -223,6 +223,9 @@ pub unsafe trait PyNativeType: Sized { #[repr(transparent)] pub struct Py(NonNull, PhantomData); +// The inner value is only accessed through ways that require proving the gil is held +#[cfg(feature = "nightly")] +unsafe impl crate::marker::Ungil for Py {} unsafe impl Send for Py {} unsafe impl Sync for Py {} @@ -580,12 +583,10 @@ impl Py { /// This is equivalent to the Python expression `self()`. pub fn call0(&self, py: Python) -> PyResult { cfg_if::cfg_if! { - // TODO: Use PyObject_CallNoArgs instead after https://bugs.python.org/issue42415. - // Once the issue is resolved, we can enable this optimization for limited API. - if #[cfg(all(Py_3_9, not(Py_LIMITED_API)))] { + if #[cfg(Py_3_9)] { // Optimized path on python 3.9+ unsafe { - PyObject::from_owned_ptr_or_err(py, ffi::_PyObject_CallNoArg(self.as_ptr())) + PyObject::from_owned_ptr_or_err(py, ffi::PyObject_CallNoArgs(self.as_ptr())) } } else { self.call(py, (), None) @@ -923,7 +924,23 @@ impl PyObject { mod tests { use super::{Py, PyObject}; use crate::types::PyDict; - use crate::Python; + use crate::{Python, ToPyObject}; + + #[test] + fn test_call0() { + Python::with_gil(|py| { + let obj = py.get_type::().to_object(py); + assert_eq!( + obj.call0(py) + .unwrap() + .as_ref(py) + .repr() + .unwrap() + .to_string_lossy(), + "{}" + ); + }) + } #[test] fn test_call_for_non_existing_method() { diff --git a/src/lib.rs b/src/lib.rs index bb6d6ace..62a2a86b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,4 @@ -#![cfg_attr(feature = "nightly", feature(specialization))] +#![cfg_attr(feature = "nightly", feature(auto_traits, negative_impls))] #![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] #![cfg_attr( docsrs, // rustdoc:: is not supported on msrv @@ -94,9 +94,7 @@ //! //! ## Unstable features //! -//! - `nightly`: Gates some optimizations that rely on -//! [`#![feature(specialization)]`](https://github.com/rust-lang/rfcs/blob/master/text/1210-impl-specialization.md), -//! for which you'd also need nightly Rust. You should not use this feature. +//! - `nightly`: Uses `#![feature(auto_traits, negative_impls)]` to define [`Ungil`] as an auto trait. // //! ## `rustc` environment flags //! @@ -283,6 +281,7 @@ //! [Python from Rust]: https://github.com/PyO3/pyo3#using-python-from-rust //! [Rust from Python]: https://github.com/PyO3/pyo3#using-rust-from-python //! [Features chapter of the guide]: https://pyo3.rs/latest/features.html#features-reference "Features Reference - PyO3 user guide" +//! [`Ungil`]: crate::marker::Ungil pub use crate::class::*; pub use crate::conversion::{ AsPyPointer, FromPyObject, FromPyPointer, IntoPy, IntoPyPointer, PyTryFrom, PyTryInto, @@ -293,12 +292,37 @@ pub use crate::err::{PyDowncastError, PyErr, PyErrArguments, PyResult}; pub use crate::gil::{prepare_freethreaded_python, with_embedded_python_interpreter}; pub use crate::gil::{GILGuard, GILPool}; pub use crate::instance::{Py, PyNativeType, PyObject}; +pub use crate::marker::Python; pub use crate::pycell::{PyCell, PyRef, PyRefMut}; pub use crate::pyclass::PyClass; pub use crate::pyclass_init::PyClassInitializer; -pub use crate::python::{Python, PythonVersionInfo}; pub use crate::type_object::PyTypeInfo; pub use crate::types::PyAny; +pub use crate::version::PythonVersionInfo; + +// Old directory layout, to be rethought? +#[cfg(not(feature = "pyproto"))] +pub mod class { + #[doc(hidden)] + pub use crate::impl_::pymethods as methods; + + #[doc(hidden)] + pub use self::methods::{ + PyClassAttributeDef, PyGetterDef, PyMethodDef, PyMethodDefType, PyMethodType, PySetterDef, + }; + + pub mod basic { + pub use crate::pyclass::CompareOp; + } + + pub mod pyasync { + pub use crate::pyclass::{IterANextOutput, PyIterANextOutput}; + } + + pub mod iter { + pub use crate::pyclass::{IterNextOutput, PyIterNextOutput}; + } +} #[cfg(feature = "macros")] #[doc(hidden)] @@ -317,6 +341,7 @@ mod internal_tricks; pub mod buffer; #[doc(hidden)] pub mod callback; +#[cfg(feature = "pyproto")] pub mod class; pub mod conversion; mod conversions; @@ -330,6 +355,7 @@ mod gil; #[doc(hidden)] pub mod impl_; mod instance; +pub mod marker; pub mod marshal; pub mod once_cell; pub mod panic; @@ -337,9 +363,10 @@ pub mod prelude; pub mod pycell; pub mod pyclass; pub mod pyclass_init; -mod python; + pub mod type_object; pub mod types; +mod version; pub use crate::conversions::*; @@ -350,12 +377,16 @@ pub use crate::conversions::*; )] #[cfg(feature = "macros")] pub mod proc_macro { - pub use pyo3_macros::{pyclass, pyfunction, pymethods, pymodule, pyproto}; + #[cfg(feature = "pyproto")] + pub use pyo3_macros::pyproto; + pub use pyo3_macros::{pyclass, pyfunction, pymethods, pymodule}; } +#[cfg(all(feature = "macros", feature = "pyproto"))] +pub use pyo3_macros::pyproto; #[cfg(feature = "macros")] pub use pyo3_macros::{ - pyclass, pyfunction, pymethods, pymodule, pyproto, wrap_pyfunction, wrap_pymodule, FromPyObject, + pyclass, pyfunction, pymethods, pymodule, wrap_pyfunction, wrap_pymodule, FromPyObject, }; #[cfg(feature = "macros")] @@ -416,6 +447,8 @@ pub mod doc_test { "guide/src/python_typing_hints.md", guide_python_typing_hints ); + doctest!("guide/src/class/object.md", guide_class_object); + doctest!("guide/src/class/numeric.md", guide_class_numeric); // deliberate choice not to test guide/ecosystem because those pages depend on external crates // such as pyo3_asyncio. diff --git a/src/macros.rs b/src/macros.rs index fafd7386..0d2a668f 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -5,7 +5,7 @@ /// This macro internally calls [`Python::run`](crate::Python::run) and panics /// if it returns `Err`, after printing the error to stdout. /// -/// If you need to handle failures, please use [`Python::run`](crate::python::Python::run) instead. +/// If you need to handle failures, please use [`Python::run`](crate::marker::Python::run) instead. /// /// # Examples /// ``` diff --git a/src/python.rs b/src/marker.rs similarity index 81% rename from src/python.rs rename to src/marker.rs index c410d0d1..583dfdec 100644 --- a/src/python.rs +++ b/src/marker.rs @@ -2,102 +2,188 @@ // // based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython +//! Fundamental properties of objects tied to the Python interpreter. +//! +//! The Python interpreter is not threadsafe. To protect the Python interpreter in multithreaded +//! scenarios there is a global lock, the *global interpreter lock* (hereafter referred to as *GIL*) +//! that must be held to safely interact with Python objects. This is why in PyO3 when you acquire +//! the GIL you get a [`Python`] marker token that carries the *lifetime* of holding the GIL and all +//! borrowed references to Python objects carry this lifetime as well. This will statically ensure +//! that you can never use Python objects after dropping the lock - if you mess this up it will be +//! caught at compile time and your program will fail to compile. +//! +//! It also supports this pattern that many extension modules employ: +//! - Drop the GIL, so that other Python threads can acquire it and make progress themselves +//! - Do something independently of the Python interpreter, like IO, a long running calculation or +//! awaiting a future +//! - Once that is done, reacquire the GIL +//! +//! That API is provided by [`Python::allow_threads`] and enforced via the [`Ungil`] bound on the +//! closure and the return type. This is done by relying on the [`Send`] auto trait. `Ungil` is +//! defined as the following: +//! +//! ```rust +//! pub unsafe trait Ungil {} +//! +//! unsafe impl Ungil for T {} +//! ``` +//! +//! We piggy-back off the `Send` auto trait because it is not possible to implement custom auto +//! traits on stable Rust. This is the solution which enables it for as many types as possible while +//! making the API usable. +//! +//! In practice this API works quite well, but it comes with some drawbacks: +//! +//! ## Drawbacks +//! +//! There is no reason to prevent `!Send` types like [`Rc`] from crossing the closure. After all, +//! [`Python::allow_threads`] just lets other Python threads run - it does not itself launch a new +//! thread. +//! +//! ```rust, compile_fail +//! # #[cfg(feature = "nightly")] +//! # compile_error!("this actually works on nightly") +//! use pyo3::prelude::*; +//! use std::rc::Rc; +//! +//! fn main() { +//! Python::with_gil(|py| { +//! let rc = Rc::new(5); +//! +//! py.allow_threads(|| { +//! // This would actually be fine... +//! println!("{:?}", *rc); +//! }); +//! }); +//! } +//! ``` +//! +//! Because we are using `Send` for something it's not quite meant for, other code that +//! (correctly) upholds the invariants of [`Send`] can cause problems. +//! +//! [`SendWrapper`] is one of those. Per its documentation: +//! +//! > A wrapper which allows you to move around non-Send-types between threads, as long as you +//! > access the contained value only from within the original thread and make sure that it is +//! > dropped from within the original thread. +//! +//! This will "work" to smuggle Python references across the closure, because we're not actually +//! doing anything with threads: +//! +//! ```rust, no_run +//! use pyo3::prelude::*; +//! use pyo3::types::PyString; +//! use send_wrapper::SendWrapper; +//! +//! Python::with_gil(|py| { +//! let string = PyString::new(py, "foo"); +//! +//! let wrapped = SendWrapper::new(string); +//! +//! py.allow_threads(|| { +//! # #[cfg(not(feature = "nightly"))] +//! # { +//! // 💥 Unsound! 💥 +//! let smuggled: &PyString = *wrapped; +//! println!("{:?}", smuggled); +//! # } +//! }); +//! }); +//! ``` +//! +//! For now the answer to that is "don't do that". +//! +//! # A proper implementation using an auto trait +//! +//! However on nightly Rust and when PyO3's `nightly` feature is +//! enabled, `Ungil` is defined as the following: +//! +//! ```rust +//! # #[cfg(FALSE)] +//! # { +//! #![feature(auto_traits, negative_impls)] +//! +//! pub unsafe auto trait Ungil {} +//! +//! // It is unimplemented for the `Python` struct and Python objects. +//! impl !Ungil for Python<'_> {} +//! impl !Ungil for ffi::PyObject {} +//! +//! // `Py` wraps it in a safe api, so this is OK +//! unsafe impl Ungil for Py {} +//! # } +//! ``` +//! +//! With this feature enabled, the above two examples will start working and not working, respectively. +//! +//! [`SendWrapper`]: https://docs.rs/send_wrapper/latest/send_wrapper/struct.SendWrapper.html +//! [`Rc`]: std::rc::Rc +//! [`Py`]: crate::Py use crate::err::{self, PyDowncastError, PyErr, PyResult}; use crate::gil::{self, GILGuard, GILPool}; use crate::impl_::not_send::NotSend; use crate::type_object::{PyTypeInfo, PyTypeObject}; use crate::types::{PyAny, PyDict, PyModule, PyType}; +use crate::version::PythonVersionInfo; use crate::{ffi, AsPyPointer, FromPyPointer, IntoPyPointer, PyNativeType, PyObject, PyTryFrom}; use std::ffi::{CStr, CString}; use std::marker::PhantomData; use std::os::raw::c_int; -/// Represents the major, minor, and patch (if any) versions of this interpreter. +/// Types that are safe to access while the GIL is not held. /// -/// See [Python::version]. -#[derive(Debug)] -pub struct PythonVersionInfo<'py> { - /// Python major version (e.g. `3`). - pub major: u8, - /// Python minor version (e.g. `11`). - pub minor: u8, - /// Python patch version (e.g. `0`). - pub patch: u8, - /// Python version suffix, if applicable (e.g. `a0`). - pub suffix: Option<&'py str>, -} +/// # Safety +/// +/// The type must not carry borrowed Python references or, if it does, not allow access to them if +/// the GIL is not held. +/// +/// See the [module-level documentation](self) for more information. +#[cfg_attr(docsrs, doc(cfg(all())))] // Hide the cfg flag +#[cfg(not(feature = "nightly"))] +pub unsafe trait Ungil {} -impl<'py> PythonVersionInfo<'py> { - /// Parses a hard-coded Python interpreter version string (e.g. 3.9.0a4+). - /// - /// Panics if the string is ill-formatted. - fn from_str(version_number_str: &'py str) -> Self { - fn split_and_parse_number(version_part: &str) -> (u8, Option<&str>) { - match version_part.find(|c: char| !c.is_ascii_digit()) { - None => (version_part.parse().unwrap(), None), - Some(version_part_suffix_start) => { - let (version_part, version_part_suffix) = - version_part.split_at(version_part_suffix_start); - (version_part.parse().unwrap(), Some(version_part_suffix)) - } - } - } +#[cfg_attr(docsrs, doc(cfg(all())))] // Hide the cfg flag +#[cfg(not(feature = "nightly"))] +unsafe impl Ungil for T {} - let mut parts = version_number_str.split('.'); - let major_str = parts.next().expect("Python major version missing"); - let minor_str = parts.next().expect("Python minor version missing"); - let patch_str = parts.next(); - assert!( - parts.next().is_none(), - "Python version string has too many parts" - ); +/// Types that are safe to access while the GIL is not held. +/// +/// # Safety +/// +/// The type must not carry borrowed Python references or, if it does, not allow access to them if +/// the GIL is not held. +/// +/// See the [module-level documentation](self) for more information. +#[cfg(feature = "nightly")] +pub unsafe auto trait Ungil {} - let major = major_str - .parse() - .expect("Python major version not an integer"); - let (minor, suffix) = split_and_parse_number(minor_str); - if suffix.is_some() { - assert!(patch_str.is_none()); - return PythonVersionInfo { - major, - minor, - patch: 0, - suffix, - }; - } +#[cfg(feature = "nightly")] +mod negative_impls { + use super::Ungil; - let (patch, suffix) = patch_str.map(split_and_parse_number).unwrap_or_default(); - PythonVersionInfo { - major, - minor, - patch, - suffix, - } - } -} + impl !Ungil for crate::Python<'_> {} -impl PartialEq<(u8, u8)> for PythonVersionInfo<'_> { - fn eq(&self, other: &(u8, u8)) -> bool { - self.major == other.0 && self.minor == other.1 - } -} + // This means that PyString, PyList, etc all inherit !Ungil from this. + impl !Ungil for crate::PyAny {} -impl PartialEq<(u8, u8, u8)> for PythonVersionInfo<'_> { - fn eq(&self, other: &(u8, u8, u8)) -> bool { - self.major == other.0 && self.minor == other.1 && self.patch == other.2 - } -} + // All the borrowing wrappers + impl !Ungil for crate::PyCell {} + impl !Ungil for crate::PyRef<'_, T> {} + impl !Ungil for crate::PyRefMut<'_, T> {} -impl PartialOrd<(u8, u8)> for PythonVersionInfo<'_> { - fn partial_cmp(&self, other: &(u8, u8)) -> Option { - (self.major, self.minor).partial_cmp(other) - } -} + // FFI pointees + impl !Ungil for crate::ffi::PyObject {} + impl !Ungil for crate::ffi::PyLongObject {} -impl PartialOrd<(u8, u8, u8)> for PythonVersionInfo<'_> { - fn partial_cmp(&self, other: &(u8, u8, u8)) -> Option { - (self.major, self.minor, self.patch).partial_cmp(other) - } + impl !Ungil for crate::ffi::PyThreadState {} + impl !Ungil for crate::ffi::PyInterpreterState {} + impl !Ungil for crate::ffi::PyWeakReference {} + impl !Ungil for crate::ffi::PyFrameObject {} + impl !Ungil for crate::ffi::PyCodeObject {} + #[cfg(not(Py_LIMITED_API))] + impl !Ungil for crate::ffi::PyDictKeysObject {} + #[cfg(not(Py_LIMITED_API))] + impl !Ungil for crate::ffi::PyArena {} } /// A marker token that represents holding the GIL. @@ -305,12 +391,8 @@ impl<'py> Python<'py> { /// interpreter for some time and have other Python threads around, this will let you run /// Rust-only code while letting those other Python threads make progress. /// - /// The closure is impermeable to types that are tied to holding the GIL, such as `&`[`PyAny`] - /// and its concrete-typed siblings like `&`[`PyString`]. This is achieved via the [`Send`] - /// bound on the closure and the return type. This is slightly - /// more restrictive than necessary, but it's the most fitting solution available in stable - /// Rust. In the future this bound may be relaxed by using an "auto-trait" instead, if - /// [auto-traits] ever become a stable feature of the Rust language. + /// Only types that implement [`Ungil`] can cross the closure. See the + /// [module level documentation](self) for more information. /// /// If you need to pass Python objects into the closure you can use [`Py`]``to create a /// reference independent of the GIL lifetime. However, you cannot do much with those without a @@ -365,8 +447,8 @@ impl<'py> Python<'py> { /// [Parallelism]: https://pyo3.rs/main/parallelism.html pub fn allow_threads(self, f: F) -> T where - F: Send + FnOnce() -> T, - T: Send, + F: Ungil + FnOnce() -> T, + T: Ungil, { // Use a guard pattern to handle reacquiring the GIL, so that the GIL will be reacquired // even if `f` panics. @@ -532,7 +614,7 @@ impl<'py> Python<'py> { /// # use pyo3::Python; /// Python::with_gil(|py| { /// // The full string could be, for example: - /// // "3.0a5+ (py3k:63103M, May 12 2008, 00:53:55) \n[GCC 4.2.3]" + /// // "3.10.0 (tags/v3.10.0:b494f59, Oct 4 2021, 19:00:18) [MSC v.1929 64 bit (AMD64)]" /// assert!(py.version().starts_with("3.")); /// }); /// ``` @@ -563,7 +645,7 @@ impl<'py> Python<'py> { // version number. let version_number_str = version_str.split(' ').next().unwrap_or(version_str); - PythonVersionInfo::from_str(version_number_str) + PythonVersionInfo::from_str(version_number_str).unwrap() } /// Registers the object in the release pool, and tries to downcast to specific type. @@ -804,6 +886,8 @@ impl<'py> Python<'py> { mod tests { use super::*; use crate::types::{IntoPyDict, PyList}; + use crate::Py; + use std::sync::Arc; #[test] fn test_eval() { @@ -889,36 +973,19 @@ mod tests { } #[test] - fn test_python_version_info() { - Python::with_gil(|py| { - let version = py.version_info(); - #[cfg(Py_3_7)] - assert!(version >= (3, 7)); - #[cfg(Py_3_7)] - assert!(version >= (3, 7, 0)); - #[cfg(Py_3_8)] - assert!(version >= (3, 8)); - #[cfg(Py_3_8)] - assert!(version >= (3, 8, 0)); - #[cfg(Py_3_9)] - assert!(version >= (3, 9)); - #[cfg(Py_3_9)] - assert!(version >= (3, 9, 0)); + fn test_allow_threads_pass_stuff_in() { + let list: Py = Python::with_gil(|py| { + let list = PyList::new(py, vec!["foo", "bar"]); + list.into() }); - } + let mut v = vec![1, 2, 3]; + let a = Arc::new(String::from("foo")); - #[test] - fn test_python_version_info_parse() { - assert!(PythonVersionInfo::from_str("3.5.0a1") >= (3, 5, 0)); - assert!(PythonVersionInfo::from_str("3.5+") >= (3, 5, 0)); - assert!(PythonVersionInfo::from_str("3.5+") == (3, 5, 0)); - assert!(PythonVersionInfo::from_str("3.5+") != (3, 5, 1)); - assert!(PythonVersionInfo::from_str("3.5.2a1+") < (3, 5, 3)); - assert!(PythonVersionInfo::from_str("3.5.2a1+") == (3, 5, 2)); - assert!(PythonVersionInfo::from_str("3.5.2a1+") == (3, 5)); - assert!(PythonVersionInfo::from_str("3.5+") == (3, 5)); - assert!(PythonVersionInfo::from_str("3.5.2a1+") < (3, 6)); - assert!(PythonVersionInfo::from_str("3.5.2a1+") > (3, 4)); + Python::with_gil(|py| { + py.allow_threads(|| { + drop((list, &mut v, a)); + }); + }); } #[test] diff --git a/src/prelude.rs b/src/prelude.rs index 52f79ac6..d7b65902 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -16,12 +16,13 @@ pub use crate::conversion::{ pub use crate::err::{PyErr, PyResult}; pub use crate::gil::GILGuard; pub use crate::instance::{Py, PyObject}; +pub use crate::marker::Python; pub use crate::pycell::{PyCell, PyRef, PyRefMut}; pub use crate::pyclass_init::PyClassInitializer; -pub use crate::python::Python; pub use crate::types::{PyAny, PyModule}; #[cfg(feature = "macros")] -pub use pyo3_macros::{ - pyclass, pyfunction, pymethods, pymodule, pyproto, wrap_pyfunction, FromPyObject, -}; +pub use pyo3_macros::{pyclass, pyfunction, pymethods, pymodule, wrap_pyfunction, FromPyObject}; + +#[cfg(all(feature = "macros", feature = "pyproto"))] +pub use pyo3_macros::pyproto; diff --git a/src/pycell.rs b/src/pycell.rs index a0c9e197..b8b2e395 100644 --- a/src/pycell.rs +++ b/src/pycell.rs @@ -175,12 +175,11 @@ //! [Interior Mutability]: https://doc.rust-lang.org/book/ch15-05-interior-mutability.html "RefCell and the Interior Mutability Pattern - The Rust Programming Language" use crate::exceptions::PyRuntimeError; -use crate::impl_::pyclass::{PyClassDict, PyClassWeakRef}; +use crate::impl_::pyclass::{PyClassBaseType, PyClassDict, PyClassThreadChecker, PyClassWeakRef}; use crate::pyclass::PyClass; use crate::pyclass_init::PyClassInitializer; use crate::type_object::{PyLayout, PySizedLayout}; use crate::types::PyAny; -use crate::{class::impl_::PyClassBaseType, class::impl_::PyClassThreadChecker}; use crate::{ conversion::{AsPyPointer, FromPyPointer, ToPyObject}, ffi::PyBaseObject_Type, diff --git a/src/pyclass.rs b/src/pyclass.rs index b2e1b0fc..a3277295 100644 --- a/src/pyclass.rs +++ b/src/pyclass.rs @@ -1,9 +1,14 @@ //! `PyClass` and related traits. use crate::{ - class::impl_::{fallback_new, tp_dealloc, PyClassImpl}, + callback::IntoPyCallbackOutput, + exceptions::PyTypeError, ffi, - impl_::pyclass::{PyClassDict, PyClassWeakRef}, - PyCell, PyErr, PyMethodDefType, PyNativeType, PyResult, PyTypeInfo, Python, + impl_::pyclass::{ + assign_sequence_item_from_mapping, get_sequence_item_from_mapping, tp_dealloc, PyClassDict, + PyClassImpl, PyClassItems, PyClassWeakRef, + }, + IntoPy, IntoPyPointer, PyCell, PyErr, PyMethodDefType, PyNativeType, PyObject, PyResult, + PyTypeInfo, Python, }; use std::{ convert::TryInto, @@ -45,15 +50,10 @@ where T::NAME, T::BaseType::type_object_raw(py), std::mem::size_of::(), - T::get_new(), tp_dealloc::, - T::get_alloc(), - T::get_free(), T::dict_offset(), T::weaklist_offset(), - &T::for_each_method_def, - &T::for_each_proto_slot, - T::IS_GC, + &T::for_all_items, T::IS_BASETYPE, ) } { @@ -70,15 +70,10 @@ unsafe fn create_type_object_impl( name: &str, base_type_object: *mut ffi::PyTypeObject, basicsize: usize, - tp_new: Option, tp_dealloc: ffi::destructor, - tp_alloc: Option, - tp_free: Option, dict_offset: Option, weaklist_offset: Option, - for_each_method_def: &dyn Fn(&mut dyn FnMut(&[PyMethodDefType])), - for_each_proto_slot: &dyn Fn(&mut dyn FnMut(&[ffi::PyType_Slot])), - is_gc: bool, + for_all_items: &dyn Fn(&mut dyn FnMut(&PyClassItems)), is_basetype: bool, ) -> PyResult<*mut ffi::PyTypeObject> { let mut slots = Vec::new(); @@ -92,20 +87,8 @@ unsafe fn create_type_object_impl( push_slot(&mut slots, ffi::Py_tp_doc, doc as _); } - push_slot( - &mut slots, - ffi::Py_tp_new, - tp_new.unwrap_or(fallback_new) as _, - ); push_slot(&mut slots, ffi::Py_tp_dealloc, tp_dealloc as _); - if let Some(alloc) = tp_alloc { - push_slot(&mut slots, ffi::Py_tp_alloc, alloc as _); - } - if let Some(free) = tp_free { - push_slot(&mut slots, ffi::Py_tp_free, free as _); - } - #[cfg(Py_3_9)] { let members = py_class_members(dict_offset, weaklist_offset); @@ -114,49 +97,97 @@ unsafe fn create_type_object_impl( } } + let PyClassInfo { + method_defs, + property_defs, + } = method_defs_to_pyclass_info(for_all_items, dict_offset.is_none()); + // normal methods - let methods = py_class_method_defs(for_each_method_def); - if !methods.is_empty() { - push_slot(&mut slots, ffi::Py_tp_methods, into_raw(methods)); + if !method_defs.is_empty() { + push_slot(&mut slots, ffi::Py_tp_methods, into_raw(method_defs)); } // properties - let props = py_class_properties(dict_offset.is_none(), for_each_method_def); - if !props.is_empty() { - push_slot(&mut slots, ffi::Py_tp_getset, into_raw(props)); + if !property_defs.is_empty() { + push_slot(&mut slots, ffi::Py_tp_getset, into_raw(property_defs)); } // protocol methods - let mut has_gc_methods = false; + let mut has_new = false; + let mut has_getitem = false; + let mut has_setitem = false; + let mut has_traverse = false; + let mut has_clear = false; + // Before Python 3.9, need to patch in buffer methods manually (they don't work in slots) #[cfg(all(not(Py_3_9), not(Py_LIMITED_API)))] let mut buffer_procs: ffi::PyBufferProcs = Default::default(); - for_each_proto_slot(&mut |proto_slots| { - for slot in proto_slots { - has_gc_methods |= slot.slot == ffi::Py_tp_clear || slot.slot == ffi::Py_tp_traverse; - - #[cfg(all(not(Py_3_9), not(Py_LIMITED_API)))] - if slot.slot == ffi::Py_bf_getbuffer { - // Safety: slot.pfunc is a valid function pointer - buffer_procs.bf_getbuffer = Some(std::mem::transmute(slot.pfunc)); - } - - #[cfg(all(not(Py_3_9), not(Py_LIMITED_API)))] - if slot.slot == ffi::Py_bf_releasebuffer { - // Safety: slot.pfunc is a valid function pointer - buffer_procs.bf_releasebuffer = Some(std::mem::transmute(slot.pfunc)); + for_all_items(&mut |items| { + for slot in items.slots { + match slot.slot { + ffi::Py_tp_new => has_new = true, + ffi::Py_mp_subscript => has_getitem = true, + ffi::Py_mp_ass_subscript => has_setitem = true, + ffi::Py_tp_traverse => has_traverse = true, + ffi::Py_tp_clear => has_clear = true, + #[cfg(all(not(Py_3_9), not(Py_LIMITED_API)))] + ffi::Py_bf_getbuffer => { + // Safety: slot.pfunc is a valid function pointer + buffer_procs.bf_getbuffer = Some(std::mem::transmute(slot.pfunc)); + } + #[cfg(all(not(Py_3_9), not(Py_LIMITED_API)))] + ffi::Py_bf_releasebuffer => { + // Safety: slot.pfunc is a valid function pointer + buffer_procs.bf_releasebuffer = Some(std::mem::transmute(slot.pfunc)); + } + _ => {} } } - slots.extend_from_slice(proto_slots); + slots.extend_from_slice(items.slots); }); + // If mapping methods implemented, define sequence methods get implemented too. + // CPython does the same for Python `class` statements. + + // NB we don't implement sq_length to avoid annoying CPython behaviour of automatically adding + // the length to negative indices. + + if has_getitem { + push_slot( + &mut slots, + ffi::Py_sq_item, + get_sequence_item_from_mapping as _, + ); + } + + if has_setitem { + push_slot( + &mut slots, + ffi::Py_sq_ass_item, + assign_sequence_item_from_mapping as _, + ); + } + + if !has_new { + push_slot(&mut slots, ffi::Py_tp_new, no_constructor_defined as _); + } + + if has_clear && !has_traverse { + return Err(PyTypeError::new_err(format!( + "`#[pyclass]` {} implements __clear__ without __traverse__", + name + ))); + } + + // Add empty sentinel at the end push_slot(&mut slots, 0, ptr::null_mut()); + let mut spec = ffi::PyType_Spec { name: py_class_qualified_name(module_name, name)?, basicsize: basicsize as c_int, itemsize: 0, - flags: py_class_flags(has_gc_methods, is_gc, is_basetype), + flags: py_class_flags(has_traverse, is_basetype), slots: slots.as_mut_ptr(), }; @@ -266,12 +297,13 @@ fn py_class_qualified_name(module_name: Option<&str>, class_name: &str) -> PyRes .into_raw()) } -fn py_class_flags(has_gc_methods: bool, is_gc: bool, is_basetype: bool) -> c_uint { - let mut flags = if has_gc_methods || is_gc { - ffi::Py_TPFLAGS_DEFAULT | ffi::Py_TPFLAGS_HAVE_GC - } else { - ffi::Py_TPFLAGS_DEFAULT - }; +fn py_class_flags(is_gc: bool, is_basetype: bool) -> c_uint { + let mut flags = ffi::Py_TPFLAGS_DEFAULT; + + if is_gc { + flags |= ffi::Py_TPFLAGS_HAVE_GC; + } + if is_basetype { flags |= ffi::Py_TPFLAGS_BASETYPE; } @@ -282,26 +314,76 @@ fn py_class_flags(has_gc_methods: bool, is_gc: bool, is_basetype: bool) -> c_uin flags.try_into().unwrap() } -fn py_class_method_defs( - for_each_method_def: &dyn Fn(&mut dyn FnMut(&[PyMethodDefType])), -) -> Vec { - let mut defs = Vec::new(); +struct PyClassInfo { + method_defs: Vec, + property_defs: Vec, +} - for_each_method_def(&mut |method_defs| { - defs.extend(method_defs.iter().filter_map(|def| match def { - PyMethodDefType::Method(def) - | PyMethodDefType::Class(def) - | PyMethodDefType::Static(def) => Some(def.as_method_def().unwrap()), - _ => None, - })); +fn method_defs_to_pyclass_info( + for_all_items: &dyn Fn(&mut dyn FnMut(&PyClassItems)), + has_dict: bool, +) -> PyClassInfo { + let mut method_defs = Vec::new(); + let mut property_defs_map = std::collections::HashMap::new(); + + for_all_items(&mut |items| { + for def in items.methods { + match def { + PyMethodDefType::Getter(getter) => { + getter.copy_to( + property_defs_map + .entry(getter.name) + .or_insert(PY_GET_SET_DEF_INIT), + ); + } + PyMethodDefType::Setter(setter) => { + setter.copy_to( + property_defs_map + .entry(setter.name) + .or_insert(PY_GET_SET_DEF_INIT), + ); + } + PyMethodDefType::Method(def) + | PyMethodDefType::Class(def) + | PyMethodDefType::Static(def) => method_defs.push(def.as_method_def().unwrap()), + PyMethodDefType::ClassAttribute(_) => {} + } + } }); - if !defs.is_empty() { + // TODO: use into_values when on MSRV Rust >= 1.54 + let mut property_defs: Vec<_> = property_defs_map + .into_iter() + .map(|(_, value)| value) + .collect(); + + if !method_defs.is_empty() { // Safety: Python expects a zeroed entry to mark the end of the defs - defs.push(unsafe { std::mem::zeroed() }); + method_defs.push(unsafe { std::mem::zeroed() }); } - defs + // PyPy doesn't automatically add __dict__ getter / setter. + // PyObject_GenericGetDict not in the limited API until Python 3.10. + if !has_dict { + #[cfg(not(any(PyPy, all(Py_LIMITED_API, not(Py_3_10)))))] + property_defs.push(ffi::PyGetSetDef { + name: "__dict__\0".as_ptr() as *mut c_char, + get: Some(ffi::PyObject_GenericGetDict), + set: Some(ffi::PyObject_GenericSetDict), + doc: ptr::null_mut(), + closure: ptr::null_mut(), + }); + } + + if !property_defs.is_empty() { + // Safety: Python expects a zeroed entry to mark the end of the defs + property_defs.push(unsafe { std::mem::zeroed() }); + } + + PyClassInfo { + method_defs, + property_defs, + } } /// Generates the __dictoffset__ and __weaklistoffset__ members, to set tp_dictoffset and @@ -352,51 +434,142 @@ const PY_GET_SET_DEF_INIT: ffi::PyGetSetDef = ffi::PyGetSetDef { closure: ptr::null_mut(), }; -fn py_class_properties( - is_dummy: bool, - for_each_method_def: &dyn Fn(&mut dyn FnMut(&[PyMethodDefType])), -) -> Vec { - let mut defs = std::collections::HashMap::new(); +/// Operators for the `__richcmp__` method +#[derive(Debug, Clone, Copy)] +pub enum CompareOp { + /// The *less than* operator. + Lt = ffi::Py_LT as isize, + /// The *less than or equal to* operator. + Le = ffi::Py_LE as isize, + /// The equality operator. + Eq = ffi::Py_EQ as isize, + /// The *not equal to* operator. + Ne = ffi::Py_NE as isize, + /// The *greater than* operator. + Gt = ffi::Py_GT as isize, + /// The *greater than or equal to* operator. + Ge = ffi::Py_GE as isize, +} - for_each_method_def(&mut |method_defs| { - for def in method_defs { - match def { - PyMethodDefType::Getter(getter) => { - getter.copy_to(defs.entry(getter.name).or_insert(PY_GET_SET_DEF_INIT)); - } - PyMethodDefType::Setter(setter) => { - setter.copy_to(defs.entry(setter.name).or_insert(PY_GET_SET_DEF_INIT)); - } - _ => (), +impl CompareOp { + pub fn from_raw(op: c_int) -> Option { + match op { + ffi::Py_LT => Some(CompareOp::Lt), + ffi::Py_LE => Some(CompareOp::Le), + ffi::Py_EQ => Some(CompareOp::Eq), + ffi::Py_NE => Some(CompareOp::Ne), + ffi::Py_GT => Some(CompareOp::Gt), + ffi::Py_GE => Some(CompareOp::Ge), + _ => None, + } + } +} + +/// Output of `__next__` which can either `yield` the next value in the iteration, or +/// `return` a value to raise `StopIteration` in Python. +/// +/// See [`PyIterProtocol`](trait.PyIterProtocol.html) for an example. +pub enum IterNextOutput { + /// The value yielded by the iterator. + Yield(T), + /// The `StopIteration` object. + Return(U), +} + +pub type PyIterNextOutput = IterNextOutput; + +impl IntoPyCallbackOutput<*mut ffi::PyObject> for PyIterNextOutput { + fn convert(self, _py: Python) -> PyResult<*mut ffi::PyObject> { + match self { + IterNextOutput::Yield(o) => Ok(o.into_ptr()), + IterNextOutput::Return(opt) => Err(crate::exceptions::PyStopIteration::new_err((opt,))), + } + } +} + +impl IntoPyCallbackOutput for IterNextOutput +where + T: IntoPy, + U: IntoPy, +{ + fn convert(self, py: Python) -> PyResult { + match self { + IterNextOutput::Yield(o) => Ok(IterNextOutput::Yield(o.into_py(py))), + IterNextOutput::Return(o) => Ok(IterNextOutput::Return(o.into_py(py))), + } + } +} + +impl IntoPyCallbackOutput for Option +where + T: IntoPy, +{ + fn convert(self, py: Python) -> PyResult { + match self { + Some(o) => Ok(PyIterNextOutput::Yield(o.into_py(py))), + None => Ok(PyIterNextOutput::Return(py.None())), + } + } +} + +/// Output of `__anext__`. +/// +/// +pub enum IterANextOutput { + /// An expression which the generator yielded. + Yield(T), + /// A `StopAsyncIteration` object. + Return(U), +} + +/// An [IterANextOutput] of Python objects. +pub type PyIterANextOutput = IterANextOutput; + +impl IntoPyCallbackOutput<*mut ffi::PyObject> for PyIterANextOutput { + fn convert(self, _py: Python) -> PyResult<*mut ffi::PyObject> { + match self { + IterANextOutput::Yield(o) => Ok(o.into_ptr()), + IterANextOutput::Return(opt) => { + Err(crate::exceptions::PyStopAsyncIteration::new_err((opt,))) } } - }); - - let mut props: Vec<_> = defs.values().cloned().collect(); - - // PyPy doesn't automatically adds __dict__ getter / setter. - // PyObject_GenericGetDict not in the limited API until Python 3.10. - push_dict_getset(&mut props, is_dummy); - - if !props.is_empty() { - // Safety: Python expects a zeroed entry to mark the end of the defs - props.push(unsafe { std::mem::zeroed() }); - } - props -} - -#[cfg(not(any(PyPy, all(Py_LIMITED_API, not(Py_3_10)))))] -fn push_dict_getset(props: &mut Vec, is_dummy: bool) { - if !is_dummy { - props.push(ffi::PyGetSetDef { - name: "__dict__\0".as_ptr() as *mut c_char, - get: Some(ffi::PyObject_GenericGetDict), - set: Some(ffi::PyObject_GenericSetDict), - doc: ptr::null_mut(), - closure: ptr::null_mut(), - }); } } -#[cfg(any(PyPy, all(Py_LIMITED_API, not(Py_3_10))))] -fn push_dict_getset(_: &mut Vec, _is_dummy: bool) {} +impl IntoPyCallbackOutput for IterANextOutput +where + T: IntoPy, + U: IntoPy, +{ + fn convert(self, py: Python) -> PyResult { + match self { + IterANextOutput::Yield(o) => Ok(IterANextOutput::Yield(o.into_py(py))), + IterANextOutput::Return(o) => Ok(IterANextOutput::Return(o.into_py(py))), + } + } +} + +impl IntoPyCallbackOutput for Option +where + T: IntoPy, +{ + fn convert(self, py: Python) -> PyResult { + match self { + Some(o) => Ok(PyIterANextOutput::Yield(o.into_py(py))), + None => Ok(PyIterANextOutput::Return(py.None())), + } + } +} + +/// Default new implementation +pub(crate) unsafe extern "C" fn no_constructor_defined( + _subtype: *mut ffi::PyTypeObject, + _args: *mut ffi::PyObject, + _kwds: *mut ffi::PyObject, +) -> *mut ffi::PyObject { + crate::callback_body!(py, { + Err::<(), _>(crate::exceptions::PyTypeError::new_err( + "No constructor defined", + )) + }) +} diff --git a/src/pyclass_init.rs b/src/pyclass_init.rs index 5fb84a96..b8b2025b 100644 --- a/src/pyclass_init.rs +++ b/src/pyclass_init.rs @@ -1,7 +1,6 @@ //! Contains initialization utilities for `#[pyclass]`. -use crate::class::impl_::PyClassThreadChecker; -use crate::impl_::pyclass::{PyClassDict, PyClassWeakRef}; -use crate::{callback::IntoPyCallbackOutput, class::impl_::PyClassBaseType}; +use crate::callback::IntoPyCallbackOutput; +use crate::impl_::pyclass::{PyClassBaseType, PyClassDict, PyClassThreadChecker, PyClassWeakRef}; use crate::{ffi, PyCell, PyClass, PyErr, PyResult, Python}; use crate::{ ffi::PyTypeObject, diff --git a/src/test_hygiene/pyclass.rs b/src/test_hygiene/pyclass.rs index adadc08c..d86ea09c 100644 --- a/src/test_hygiene/pyclass.rs +++ b/src/test_hygiene/pyclass.rs @@ -28,3 +28,9 @@ pub struct Bar { #[pyo3(get, set)] c: ::std::option::Option>, } + +#[crate::pyclass] +#[pyo3(crate = "crate")] +pub enum Enum { + Var0, +} diff --git a/src/type_object.rs b/src/type_object.rs index a12d7812..1022a9d8 100644 --- a/src/type_object.rs +++ b/src/type_object.rs @@ -1,6 +1,7 @@ // Copyright (c) 2017-present PyO3 Project and Contributors //! Python type object information +use crate::impl_::pyclass::PyClassItems; use crate::internal_tricks::extract_cstr_or_leak_cstring; use crate::once_cell::GILOnceCell; use crate::pyclass::create_type_object; @@ -108,7 +109,7 @@ impl LazyStaticType { pub fn get_or_init(&self, py: Python) -> *mut ffi::PyTypeObject { let type_object = *self.value.get_or_init(py, || create_type_object::(py)); - self.ensure_init(py, type_object, T::NAME, &T::for_each_method_def); + self.ensure_init(py, type_object, T::NAME, &T::for_all_items); type_object } @@ -117,7 +118,7 @@ impl LazyStaticType { py: Python, type_object: *mut ffi::PyTypeObject, name: &str, - for_each_method_def: &dyn Fn(&mut dyn FnMut(&[PyMethodDefType])), + for_all_items: &dyn Fn(&mut dyn FnMut(&PyClassItems)), ) { // We might want to fill the `tp_dict` with python instances of `T` // itself. In order to do so, we must first initialize the type object @@ -151,8 +152,8 @@ impl LazyStaticType { // means that another thread can continue the initialization in the // meantime: at worst, we'll just make a useless computation. let mut items = vec![]; - for_each_method_def(&mut |method_defs| { - items.extend(method_defs.iter().filter_map(|def| { + for_all_items(&mut |class_items| { + items.extend(class_items.methods.iter().filter_map(|def| { if let PyMethodDefType::ClassAttribute(attr) = def { let key = extract_cstr_or_leak_cstring( attr.name, diff --git a/src/types/any.rs b/src/types/any.rs index 0206cd46..b54aa540 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -333,12 +333,10 @@ impl PyAny { /// This is equivalent to the Python expression `help()`. pub fn call0(&self) -> PyResult<&PyAny> { cfg_if::cfg_if! { - // TODO: Use PyObject_CallNoArgs instead after https://bugs.python.org/issue42415. - // Once the issue is resolved, we can enable this optimization for limited API. - if #[cfg(all(Py_3_9, not(Py_LIMITED_API)))] { + if #[cfg(Py_3_9)] { // Optimized path on python 3.9+ unsafe { - self.py().from_owned_ptr_or_err(ffi::_PyObject_CallNoArg(self.as_ptr())) + self.py().from_owned_ptr_or_err(ffi::PyObject_CallNoArgs(self.as_ptr())) } } else { self.call((), None) diff --git a/src/types/datetime.rs b/src/types/datetime.rs index 990d82ee..f6ca1fda 100644 --- a/src/types/datetime.rs +++ b/src/types/datetime.rs @@ -5,10 +5,9 @@ use crate::err::PyResult; use crate::ffi; -#[cfg(PyPy)] -use crate::ffi::datetime::{PyDateTime_FromTimestamp, PyDate_FromTimestamp}; -use crate::ffi::PyDateTimeAPI; -use crate::ffi::{PyDateTime_Check, PyDate_Check, PyDelta_Check, PyTZInfo_Check, PyTime_Check}; +use crate::ffi::{ + PyDateTime_CAPI, PyDateTime_FromTimestamp, PyDateTime_IMPORT, PyDate_FromTimestamp, +}; #[cfg(not(PyPy))] use crate::ffi::{PyDateTime_DATE_GET_FOLD, PyDateTime_TIME_GET_FOLD}; use crate::ffi::{ @@ -26,8 +25,64 @@ use crate::ffi::{ use crate::types::PyTuple; use crate::{AsPyPointer, PyAny, PyObject, Python, ToPyObject}; use std::os::raw::c_int; -#[cfg(not(PyPy))] -use std::ptr; + +fn ensure_datetime_api(_py: Python) -> &'static PyDateTime_CAPI { + unsafe { + if pyo3_ffi::PyDateTimeAPI().is_null() { + PyDateTime_IMPORT() + } + + &*pyo3_ffi::PyDateTimeAPI() + } +} + +// Type Check macros +// +// These are bindings around the C API typecheck macros, all of them return +// `1` if True and `0` if False. In all type check macros, the argument (`op`) +// must not be `NULL`. The implementations here all call ensure_datetime_api +// to ensure that the PyDateTimeAPI is initalized before use +// +// +// # Safety +// +// These functions must only be called when the GIL is held! + +macro_rules! ffi_fun_with_autoinit { + ($(#[$outer:meta] unsafe fn $name: ident($arg: ident: *mut PyObject) -> $ret: ty;)*) => { + $( + #[$outer] + #[allow(non_snake_case)] + /// # Safety + /// + /// Must only be called while the GIL is held + unsafe fn $name($arg: *mut crate::ffi::PyObject) -> $ret { + + let _ = ensure_datetime_api(Python::assume_gil_acquired()); + crate::ffi::$name($arg) + } + )* + + + }; +} + +ffi_fun_with_autoinit! { + /// Check if `op` is a `PyDateTimeAPI.DateType` or subtype. + unsafe fn PyDate_Check(op: *mut PyObject) -> c_int; + + /// Check if `op` is a `PyDateTimeAPI.DateTimeType` or subtype. + unsafe fn PyDateTime_Check(op: *mut PyObject) -> c_int; + + /// Check if `op` is a `PyDateTimeAPI.TimeType` or subtype. + unsafe fn PyTime_Check(op: *mut PyObject) -> c_int; + + /// Check if `op` is a `PyDateTimeAPI.DetaType` or subtype. + unsafe fn PyDelta_Check(op: *mut PyObject) -> c_int; + + /// Check if `op` is a `PyDateTimeAPI.TZInfoType` or subtype. + unsafe fn PyTZInfo_Check(op: *mut PyObject) -> c_int; +} // Access traits @@ -111,7 +166,7 @@ pub struct PyDate(PyAny); pyobject_native_type!( PyDate, crate::ffi::PyDateTime_Date, - *PyDateTimeAPI.DateType, + *ensure_datetime_api(Python::assume_gil_acquired()).DateType, #module=Some("datetime"), #checkfunction=PyDate_Check ); @@ -120,11 +175,11 @@ impl PyDate { /// Creates a new `datetime.date`. pub fn new(py: Python, year: i32, month: u8, day: u8) -> PyResult<&PyDate> { unsafe { - let ptr = (PyDateTimeAPI.Date_FromDate)( + let ptr = (ensure_datetime_api(py).Date_FromDate)( year, c_int::from(month), c_int::from(day), - PyDateTimeAPI.DateType, + ensure_datetime_api(py).DateType, ); py.from_owned_ptr_or_err(ptr) } @@ -136,14 +191,11 @@ impl PyDate { pub fn from_timestamp(py: Python, timestamp: i64) -> PyResult<&PyDate> { let time_tuple = PyTuple::new(py, &[timestamp]); + // safety ensure that the API is loaded + let _api = ensure_datetime_api(py); + unsafe { - #[cfg(PyPy)] let ptr = PyDate_FromTimestamp(time_tuple.as_ptr()); - - #[cfg(not(PyPy))] - let ptr = - (PyDateTimeAPI.Date_FromTimestamp)(PyDateTimeAPI.DateType, time_tuple.as_ptr()); - py.from_owned_ptr_or_err(ptr) } } @@ -169,7 +221,7 @@ pub struct PyDateTime(PyAny); pyobject_native_type!( PyDateTime, crate::ffi::PyDateTime_DateTime, - *PyDateTimeAPI.DateTimeType, + *ensure_datetime_api(Python::assume_gil_acquired()).DateType, #module=Some("datetime"), #checkfunction=PyDateTime_Check ); @@ -187,8 +239,9 @@ impl PyDateTime { microsecond: u32, tzinfo: Option<&PyObject>, ) -> PyResult<&'p PyDateTime> { + let api = ensure_datetime_api(py); unsafe { - let ptr = (PyDateTimeAPI.DateTime_FromDateAndTime)( + let ptr = (api.DateTime_FromDateAndTime)( year, c_int::from(month), c_int::from(day), @@ -197,7 +250,7 @@ impl PyDateTime { c_int::from(second), microsecond as c_int, opt_to_pyobj(py, tzinfo), - PyDateTimeAPI.DateTimeType, + api.DateTimeType, ); py.from_owned_ptr_or_err(ptr) } @@ -219,8 +272,9 @@ impl PyDateTime { tzinfo: Option<&PyObject>, fold: bool, ) -> PyResult<&'p PyDateTime> { + let api = ensure_datetime_api(py); unsafe { - let ptr = (PyDateTimeAPI.DateTime_FromDateAndTimeAndFold)( + let ptr = (api.DateTime_FromDateAndTimeAndFold)( year, c_int::from(month), c_int::from(day), @@ -230,7 +284,7 @@ impl PyDateTime { microsecond as c_int, opt_to_pyobj(py, tzinfo), c_int::from(fold), - PyDateTimeAPI.DateTimeType, + api.DateTimeType, ); py.from_owned_ptr_or_err(ptr) } @@ -253,19 +307,11 @@ impl PyDateTime { let args = PyTuple::new(py, &[timestamp, time_zone_info]); + // safety ensure API is loaded + let _api = ensure_datetime_api(py); + unsafe { - #[cfg(PyPy)] let ptr = PyDateTime_FromTimestamp(args.as_ptr()); - - #[cfg(not(PyPy))] - let ptr = { - (PyDateTimeAPI.DateTime_FromTimestamp)( - PyDateTimeAPI.DateTimeType, - args.as_ptr(), - ptr::null_mut(), - ) - }; - py.from_owned_ptr_or_err(ptr) } } @@ -314,7 +360,7 @@ pub struct PyTime(PyAny); pyobject_native_type!( PyTime, crate::ffi::PyDateTime_Time, - *PyDateTimeAPI.TimeType, + *ensure_datetime_api(Python::assume_gil_acquired()).TimeType, #module=Some("datetime"), #checkfunction=PyTime_Check ); @@ -329,14 +375,15 @@ impl PyTime { microsecond: u32, tzinfo: Option<&PyObject>, ) -> PyResult<&'p PyTime> { + let api = ensure_datetime_api(py); unsafe { - let ptr = (PyDateTimeAPI.Time_FromTime)( + let ptr = (api.Time_FromTime)( c_int::from(hour), c_int::from(minute), c_int::from(second), microsecond as c_int, opt_to_pyobj(py, tzinfo), - PyDateTimeAPI.TimeType, + api.TimeType, ); py.from_owned_ptr_or_err(ptr) } @@ -353,15 +400,16 @@ impl PyTime { tzinfo: Option<&PyObject>, fold: bool, ) -> PyResult<&'p PyTime> { + let api = ensure_datetime_api(py); unsafe { - let ptr = (PyDateTimeAPI.Time_FromTimeAndFold)( + let ptr = (api.Time_FromTimeAndFold)( c_int::from(hour), c_int::from(minute), c_int::from(second), microsecond as c_int, opt_to_pyobj(py, tzinfo), fold as c_int, - PyDateTimeAPI.TimeType, + api.TimeType, ); py.from_owned_ptr_or_err(ptr) } @@ -399,7 +447,7 @@ pub struct PyTzInfo(PyAny); pyobject_native_type!( PyTzInfo, crate::ffi::PyObject, - *PyDateTimeAPI.TZInfoType, + *ensure_datetime_api(Python::assume_gil_acquired()).TZInfoType, #module=Some("datetime"), #checkfunction=PyTZInfo_Check ); @@ -410,7 +458,7 @@ pub struct PyDelta(PyAny); pyobject_native_type!( PyDelta, crate::ffi::PyDateTime_Delta, - *PyDateTimeAPI.DeltaType, + *ensure_datetime_api(Python::assume_gil_acquired()).DeltaType, #module=Some("datetime"), #checkfunction=PyDelta_Check ); @@ -424,13 +472,14 @@ impl PyDelta { microseconds: i32, normalize: bool, ) -> PyResult<&PyDelta> { + let api = ensure_datetime_api(py); unsafe { - let ptr = (PyDateTimeAPI.Delta_FromDelta)( + let ptr = (api.Delta_FromDelta)( days as c_int, seconds as c_int, microseconds as c_int, normalize as c_int, - PyDateTimeAPI.DeltaType, + api.DeltaType, ); py.from_owned_ptr_or_err(ptr) } diff --git a/src/types/function.rs b/src/types/function.rs index fb31b087..4e72eefe 100644 --- a/src/types/function.rs +++ b/src/types/function.rs @@ -2,8 +2,9 @@ use crate::derive_utils::PyFunctionArguments; use crate::exceptions::PyValueError; use crate::prelude::*; use crate::{ - class::methods::{self, PyMethodDef}, - ffi, types, AsPyPointer, + ffi, + impl_::pymethods::{self, PyMethodDef}, + types, AsPyPointer, }; use std::os::raw::c_void; @@ -64,7 +65,11 @@ impl PyCFunction { py_or_module: PyFunctionArguments<'a>, ) -> PyResult<&'a Self> { Self::internal_new( - PyMethodDef::cfunction_with_keywords(name, methods::PyCFunctionWithKeywords(fun), doc), + PyMethodDef::cfunction_with_keywords( + name, + pymethods::PyCFunctionWithKeywords(fun), + doc, + ), py_or_module, ) } @@ -77,7 +82,7 @@ impl PyCFunction { py_or_module: PyFunctionArguments<'a>, ) -> PyResult<&'a Self> { Self::internal_new( - PyMethodDef::noargs(name, methods::PyCFunction(fun), doc), + PyMethodDef::noargs(name, pymethods::PyCFunction(fun), doc), py_or_module, ) } @@ -115,9 +120,9 @@ impl PyCFunction { ), )? }; - let method_def = methods::PyMethodDef::cfunction_with_keywords( + let method_def = pymethods::PyMethodDef::cfunction_with_keywords( "pyo3-closure", - methods::PyCFunctionWithKeywords(run_closure::), + pymethods::PyCFunctionWithKeywords(run_closure::), "", ); Self::internal_new_from_pointers(method_def, py, capsule.as_ptr(), std::ptr::null_mut()) diff --git a/src/types/mapping.rs b/src/types/mapping.rs index 0ff430a0..6cb6ff66 100644 --- a/src/types/mapping.rs +++ b/src/types/mapping.rs @@ -33,6 +33,16 @@ impl PyMapping { self.len().map(|l| l == 0) } + /// Determines if the mapping contains the specified key. + /// + /// This is equivalent to the Python expression `key in self`. + pub fn contains(&self, key: K) -> PyResult + where + K: ToBorrowedObject, + { + PyAny::contains(self, key) + } + /// Gets the item in self with key `key`. /// /// Returns an `Err` if the item with specified key is not found, usually `KeyError`. @@ -166,6 +176,21 @@ mod tests { }); } + #[test] + fn test_contains() { + Python::with_gil(|py| { + let mut v = HashMap::new(); + v.insert("key0", 1234); + let ob = v.to_object(py); + let mapping = ::try_from(ob.as_ref(py)).unwrap(); + mapping.set_item("key1", "foo").unwrap(); + + assert!(mapping.contains("key0").unwrap()); + assert!(mapping.contains("key1").unwrap()); + assert!(!mapping.contains("key2").unwrap()); + }); + } + #[test] fn test_get_item() { Python::with_gil(|py| { diff --git a/src/types/mod.rs b/src/types/mod.rs index d2765bac..b85c31af 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -148,7 +148,11 @@ macro_rules! pyobject_native_type_info( // Create a very short lived mutable reference and directly // cast it to a pointer: no mutable references can be aliasing // because we hold the GIL. + #[cfg(not(addr_of))] unsafe { &mut $typeobject } + + #[cfg(addr_of)] + unsafe { ::std::ptr::addr_of_mut!($typeobject) } } $( @@ -196,10 +200,10 @@ macro_rules! pyobject_native_type_sized { ($name:ty, $layout:path $(;$generics:ident)*) => { unsafe impl $crate::type_object::PyLayout<$name> for $layout {} impl $crate::type_object::PySizedLayout<$name> for $layout {} - impl<'a, $($generics,)*> $crate::class::impl_::PyClassBaseType for $name { + impl<'a, $($generics,)*> $crate::impl_::pyclass::PyClassBaseType for $name { type LayoutAsBase = $crate::pycell::PyCellBase<$layout>; type BaseNativeType = $name; - type ThreadChecker = $crate::class::impl_::ThreadCheckerStub<$crate::PyObject>; + type ThreadChecker = $crate::impl_::pyclass::ThreadCheckerStub<$crate::PyObject>; type Initializer = $crate::pyclass_init::PyNativeTypeInitializer; } } diff --git a/src/types/num.rs b/src/types/num.rs index 60e4c695..7a49b6b5 100644 --- a/src/types/num.rs +++ b/src/types/num.rs @@ -143,7 +143,7 @@ int_convert_u64_or_i64!( ffi::PyLong_AsUnsignedLongLong ); -#[cfg(not(any(Py_LIMITED_API, PyPy)))] +#[cfg(not(Py_LIMITED_API))] mod fast_128bit_int_conversion { use super::*; @@ -200,8 +200,8 @@ mod fast_128bit_int_conversion { int_convert_128!(u128, 0); } -// For ABI3 and PyPy, we implement the conversion manually. -#[cfg(any(Py_LIMITED_API, PyPy))] +// For ABI3 we implement the conversion manually. +#[cfg(Py_LIMITED_API)] mod slow_128bit_int_conversion { use super::*; const SHIFT: usize = 64; @@ -245,10 +245,10 @@ mod slow_128bit_int_conversion { -1 as _, ffi::PyLong_AsUnsignedLongLongMask(ob.as_ptr()), )? as $rust_type; - let shifted = PyObject::from_owned_ptr( + let shifted = PyObject::from_owned_ptr_or_err( py, ffi::PyNumber_Rshift(ob.as_ptr(), SHIFT.into_py(py).as_ptr()), - ); + )?; let upper: $half_type = shifted.extract(py)?; Ok((<$rust_type>::from(upper) << SHIFT) | lower) } @@ -461,8 +461,6 @@ mod tests { test_common!(u64, u64); test_common!(isize, isize); test_common!(usize, usize); - #[cfg(not(any(Py_LIMITED_API, PyPy)))] test_common!(i128, i128); - #[cfg(not(any(Py_LIMITED_API, PyPy)))] test_common!(u128, u128); } diff --git a/src/types/sequence.rs b/src/types/sequence.rs index 61f48569..844c80b5 100644 --- a/src/types/sequence.rs +++ b/src/types/sequence.rs @@ -269,35 +269,9 @@ impl<'a, T> FromPyObject<'a> for Vec where T: FromPyObject<'a>, { - #[cfg(not(feature = "nightly"))] fn extract(obj: &'a PyAny) -> PyResult { extract_sequence(obj) } - #[cfg(feature = "nightly")] - default fn extract(obj: &'a PyAny) -> PyResult { - extract_sequence(obj) - } -} - -#[cfg(all(feature = "nightly", not(Py_LIMITED_API)))] -impl<'source, T> FromPyObject<'source> for Vec -where - for<'a> T: FromPyObject<'a> + crate::buffer::Element, -{ - fn extract(obj: &'source PyAny) -> PyResult { - // first try buffer protocol - if let Ok(buf) = crate::buffer::PyBuffer::get(obj) { - if buf.dimensions() == 1 { - if let Ok(v) = buf.to_vec(obj.py()) { - buf.release(obj.py()); - return Ok(v); - } - } - buf.release(obj.py()); - } - // fall back to sequence protocol - extract_sequence(obj) - } } fn extract_sequence<'s, T>(obj: &'s PyAny) -> PyResult> diff --git a/src/version.rs b/src/version.rs new file mode 100644 index 00000000..ea9e9d40 --- /dev/null +++ b/src/version.rs @@ -0,0 +1,143 @@ +/// Represents the major, minor, and patch (if any) versions of this interpreter. +/// +/// This struct is usually created with [`Python::version`]. +/// +/// # Examples +/// +/// ```rust +/// # use pyo3::Python; +/// Python::with_gil(|py| { +/// // PyO3 supports Python 3.7 and up. +/// assert!(py.version_info() >= (3, 7)); +/// assert!(py.version_info() >= (3, 7, 0)); +/// }); +/// ``` +/// +/// [`Python::version`]: crate::marker::Python::version +#[derive(Debug)] +pub struct PythonVersionInfo<'py> { + /// Python major version (e.g. `3`). + pub major: u8, + /// Python minor version (e.g. `11`). + pub minor: u8, + /// Python patch version (e.g. `0`). + pub patch: u8, + /// Python version suffix, if applicable (e.g. `a0`). + pub suffix: Option<&'py str>, +} + +impl<'py> PythonVersionInfo<'py> { + /// Parses a hard-coded Python interpreter version string (e.g. 3.9.0a4+). + pub(crate) fn from_str(version_number_str: &'py str) -> Result { + fn split_and_parse_number(version_part: &str) -> (u8, Option<&str>) { + match version_part.find(|c: char| !c.is_ascii_digit()) { + None => (version_part.parse().unwrap(), None), + Some(version_part_suffix_start) => { + let (version_part, version_part_suffix) = + version_part.split_at(version_part_suffix_start); + (version_part.parse().unwrap(), Some(version_part_suffix)) + } + } + } + + let mut parts = version_number_str.split('.'); + let major_str = parts.next().ok_or("Python major version missing")?; + let minor_str = parts.next().ok_or("Python minor version missing")?; + let patch_str = parts.next(); + if parts.next().is_some() { + return Err("Python version string has too many parts"); + }; + + let major = major_str + .parse() + .map_err(|_| "Python major version not an integer")?; + let (minor, suffix) = split_and_parse_number(minor_str); + if suffix.is_some() { + assert!(patch_str.is_none()); + return Ok(PythonVersionInfo { + major, + minor, + patch: 0, + suffix, + }); + } + + let (patch, suffix) = patch_str.map(split_and_parse_number).unwrap_or_default(); + Ok(PythonVersionInfo { + major, + minor, + patch, + suffix, + }) + } +} + +impl PartialEq<(u8, u8)> for PythonVersionInfo<'_> { + fn eq(&self, other: &(u8, u8)) -> bool { + self.major == other.0 && self.minor == other.1 + } +} + +impl PartialEq<(u8, u8, u8)> for PythonVersionInfo<'_> { + fn eq(&self, other: &(u8, u8, u8)) -> bool { + self.major == other.0 && self.minor == other.1 && self.patch == other.2 + } +} + +impl PartialOrd<(u8, u8)> for PythonVersionInfo<'_> { + fn partial_cmp(&self, other: &(u8, u8)) -> Option { + (self.major, self.minor).partial_cmp(other) + } +} + +impl PartialOrd<(u8, u8, u8)> for PythonVersionInfo<'_> { + fn partial_cmp(&self, other: &(u8, u8, u8)) -> Option { + (self.major, self.minor, self.patch).partial_cmp(other) + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::Python; + #[test] + fn test_python_version_info() { + Python::with_gil(|py| { + let version = py.version_info(); + #[cfg(Py_3_7)] + assert!(version >= (3, 7)); + #[cfg(Py_3_7)] + assert!(version >= (3, 7, 0)); + #[cfg(Py_3_8)] + assert!(version >= (3, 8)); + #[cfg(Py_3_8)] + assert!(version >= (3, 8, 0)); + #[cfg(Py_3_9)] + assert!(version >= (3, 9)); + #[cfg(Py_3_9)] + assert!(version >= (3, 9, 0)); + #[cfg(Py_3_10)] + assert!(version >= (3, 10)); + #[cfg(Py_3_10)] + assert!(version >= (3, 10, 0)); + #[cfg(Py_3_11)] + assert!(version >= (3, 11)); + #[cfg(Py_3_11)] + assert!(version >= (3, 11, 0)); + }); + } + + #[test] + fn test_python_version_info_parse() { + assert!(PythonVersionInfo::from_str("3.5.0a1").unwrap() >= (3, 5, 0)); + assert!(PythonVersionInfo::from_str("3.5+").unwrap() >= (3, 5, 0)); + assert!(PythonVersionInfo::from_str("3.5+").unwrap() == (3, 5, 0)); + assert!(PythonVersionInfo::from_str("3.5+").unwrap() != (3, 5, 1)); + assert!(PythonVersionInfo::from_str("3.5.2a1+").unwrap() < (3, 5, 3)); + assert!(PythonVersionInfo::from_str("3.5.2a1+").unwrap() == (3, 5, 2)); + assert!(PythonVersionInfo::from_str("3.5.2a1+").unwrap() == (3, 5)); + assert!(PythonVersionInfo::from_str("3.5+").unwrap() == (3, 5)); + assert!(PythonVersionInfo::from_str("3.5.2a1+").unwrap() < (3, 6)); + assert!(PythonVersionInfo::from_str("3.5.2a1+").unwrap() > (3, 4)); + } +} diff --git a/tests/test_arithmetics.rs b/tests/test_arithmetics.rs index ed1870bb..548ed913 100644 --- a/tests/test_arithmetics.rs +++ b/tests/test_arithmetics.rs @@ -52,6 +52,39 @@ fn unary_arithmetic() { py_run!(py, c, "assert repr(round(c, 1)) == 'UA(3)'"); } +#[pyclass] +struct Indexable(i32); + +#[pymethods] +impl Indexable { + fn __index__(&self) -> i32 { + self.0 + } + + fn __int__(&self) -> i32 { + self.0 + } + + fn __float__(&self) -> f64 { + f64::from(self.0) + } + + fn __invert__(&self) -> Self { + Self(!self.0) + } +} + +#[test] +fn indexable() { + Python::with_gil(|py| { + let i = PyCell::new(py, Indexable(5)).unwrap(); + py_run!(py, i, "assert int(i) == 5"); + py_run!(py, i, "assert [0, 1, 2, 3, 4, 5][i] == 5"); + py_run!(py, i, "assert float(i) == 5.0"); + py_run!(py, i, "assert int(~i) == -6"); + }) +} + #[pyclass] struct InPlaceOperations { value: u32, diff --git a/tests/test_arithmetics_protos.rs b/tests/test_arithmetics_protos.rs index 86b30c13..b991599a 100644 --- a/tests/test_arithmetics_protos.rs +++ b/tests/test_arithmetics_protos.rs @@ -1,5 +1,6 @@ #![allow(deprecated)] // for deprecated protocol methods #![cfg(feature = "macros")] +#![cfg(feature = "pyproto")] use pyo3::class::basic::CompareOp; use pyo3::class::*; diff --git a/tests/test_buffer.rs b/tests/test_buffer.rs index bfdf39fb..86a356db 100644 --- a/tests/test_buffer.rs +++ b/tests/test_buffer.rs @@ -1,10 +1,7 @@ #![cfg(feature = "macros")] -#![cfg(not(Py_LIMITED_API))] +#![cfg(any(not(Py_LIMITED_API), Py_3_11))] -use pyo3::{ - buffer::PyBuffer, class::PyBufferProtocol, exceptions::PyBufferError, ffi, prelude::*, - AsPyPointer, -}; +use pyo3::{buffer::PyBuffer, exceptions::PyBufferError, ffi, prelude::*, AsPyPointer}; use std::{ ffi::CStr, os::raw::{c_int, c_void}, @@ -28,9 +25,13 @@ struct TestBufferErrors { error: Option, } -#[pyproto] -impl PyBufferProtocol for TestBufferErrors { - fn bf_getbuffer(slf: PyRefMut, view: *mut ffi::Py_buffer, flags: c_int) -> PyResult<()> { +#[pymethods] +impl TestBufferErrors { + unsafe fn __getbuffer__( + slf: PyRefMut, + view: *mut ffi::Py_buffer, + flags: c_int, + ) -> PyResult<()> { if view.is_null() { return Err(PyBufferError::new_err("View is null")); } @@ -39,53 +40,47 @@ impl PyBufferProtocol for TestBufferErrors { return Err(PyBufferError::new_err("Object is not writable")); } - unsafe { - (*view).obj = ffi::_Py_NewRef(slf.as_ptr()); - } + (*view).obj = ffi::_Py_NewRef(slf.as_ptr()); let bytes = &slf.buf; - unsafe { - (*view).buf = bytes.as_ptr() as *mut c_void; - (*view).len = bytes.len() as isize; - (*view).readonly = 1; - (*view).itemsize = std::mem::size_of::() as isize; + (*view).buf = bytes.as_ptr() as *mut c_void; + (*view).len = bytes.len() as isize; + (*view).readonly = 1; + (*view).itemsize = std::mem::size_of::() as isize; - let msg = CStr::from_bytes_with_nul(b"I\0").unwrap(); - (*view).format = msg.as_ptr() as *mut _; + let msg = CStr::from_bytes_with_nul(b"I\0").unwrap(); + (*view).format = msg.as_ptr() as *mut _; - (*view).ndim = 1; - (*view).shape = &mut (*view).len; + (*view).ndim = 1; + (*view).shape = &mut (*view).len; - (*view).strides = &mut (*view).itemsize; + (*view).strides = &mut (*view).itemsize; - (*view).suboffsets = ptr::null_mut(); - (*view).internal = ptr::null_mut(); + (*view).suboffsets = ptr::null_mut(); + (*view).internal = ptr::null_mut(); - if let Some(err) = &slf.error { - use TestGetBufferError::*; - match err { - NullShape => { - (*view).shape = std::ptr::null_mut(); - } - NullStrides => { - (*view).strides = std::ptr::null_mut(); - } - IncorrectItemSize => { - (*view).itemsize += 1; - } - IncorrectFormat => { - (*view).format = CStr::from_bytes_with_nul(b"B\0").unwrap().as_ptr() as _; - } - IncorrectAlignment => (*view).buf = (*view).buf.add(1), + if let Some(err) = &slf.error { + use TestGetBufferError::*; + match err { + NullShape => { + (*view).shape = std::ptr::null_mut(); } + NullStrides => { + (*view).strides = std::ptr::null_mut(); + } + IncorrectItemSize => { + (*view).itemsize += 1; + } + IncorrectFormat => { + (*view).format = CStr::from_bytes_with_nul(b"B\0").unwrap().as_ptr() as _; + } + IncorrectAlignment => (*view).buf = (*view).buf.add(1), } } Ok(()) } - - fn bf_releasebuffer(_slf: PyRefMut, _view: *mut ffi::Py_buffer) {} } #[test] diff --git a/tests/test_buffer_protocol.rs b/tests/test_buffer_protocol.rs index bb2cb253..65c32e2d 100644 --- a/tests/test_buffer_protocol.rs +++ b/tests/test_buffer_protocol.rs @@ -1,5 +1,5 @@ #![cfg(feature = "macros")] -#![cfg(not(Py_LIMITED_API))] +#![cfg(any(not(Py_LIMITED_API), Py_3_11))] use pyo3::buffer::PyBuffer; use pyo3::exceptions::PyBufferError; diff --git a/tests/test_buffer_protocol_pyproto.rs b/tests/test_buffer_protocol_pyproto.rs index 81c5a7f0..2e540b4e 100644 --- a/tests/test_buffer_protocol_pyproto.rs +++ b/tests/test_buffer_protocol_pyproto.rs @@ -1,5 +1,6 @@ #![cfg(feature = "macros")] -#![cfg(not(Py_LIMITED_API))] +#![cfg(feature = "pyproto")] +#![cfg(any(not(Py_LIMITED_API), Py_3_11))] use pyo3::buffer::PyBuffer; use pyo3::class::PyBufferProtocol; diff --git a/tests/test_compile_error.rs b/tests/test_compile_error.rs index 2fef7685..951ebfe2 100644 --- a/tests/test_compile_error.rs +++ b/tests/test_compile_error.rs @@ -7,6 +7,7 @@ fn test_compile_errors() { _test_compile_errors() } +#[cfg(not(feature = "nightly"))] #[rustversion::nightly] #[test] fn test_compile_errors() { @@ -15,8 +16,19 @@ fn test_compile_errors() { let _ = std::panic::catch_unwind(_test_compile_errors); } +#[cfg(feature = "nightly")] +#[rustversion::nightly] +#[test] +fn test_compile_errors() { + // nightly - don't care if test output is potentially wrong, to avoid churn in PyO3's CI thanks + // to diagnostics changing on nightly. + _test_compile_errors() +} + +#[cfg(not(feature = "nightly"))] fn _test_compile_errors() { let t = trybuild::TestCases::new(); + t.compile_fail("tests/ui/invalid_macro_args.rs"); t.compile_fail("tests/ui/invalid_need_module_arg_position.rs"); t.compile_fail("tests/ui/invalid_property_args.rs"); @@ -71,6 +83,8 @@ fn _test_compile_errors() { t.compile_fail("tests/ui/invalid_pymethods.rs"); t.compile_fail("tests/ui/missing_clone.rs"); t.compile_fail("tests/ui/not_send.rs"); + t.compile_fail("tests/ui/not_send2.rs"); + t.compile_fail("tests/ui/not_send3.rs"); #[cfg(Py_LIMITED_API)] t.compile_fail("tests/ui/abi3_nativetype_inheritance.rs"); } @@ -78,3 +92,12 @@ fn _test_compile_errors() { #[rustversion::before(1.58)] fn tests_rust_1_58(_t: &trybuild::TestCases) {} } + +#[cfg(feature = "nightly")] +fn _test_compile_errors() { + let t = trybuild::TestCases::new(); + + t.compile_fail("tests/ui/not_send_auto_trait.rs"); + t.compile_fail("tests/ui/not_send_auto_trait2.rs"); + t.compile_fail("tests/ui/send_wrapper.rs"); +} diff --git a/tests/test_datetime.rs b/tests/test_datetime.rs index 0cb5eddf..976c04ef 100644 --- a/tests/test_datetime.rs +++ b/tests/test_datetime.rs @@ -2,6 +2,7 @@ use pyo3::prelude::*; use pyo3::types::IntoPyDict; +use pyo3_ffi::PyDateTime_IMPORT; fn _get_subclasses<'p>( py: &'p Python, @@ -57,7 +58,7 @@ fn test_date_check() { let gil = Python::acquire_gil(); let py = gil.python(); let (obj, sub_obj, sub_sub_obj) = _get_subclasses(&py, "date", "2018, 1, 1").unwrap(); - + unsafe { PyDateTime_IMPORT() } assert_check_exact!(PyDate_Check, PyDate_CheckExact, obj); assert_check_only!(PyDate_Check, PyDate_CheckExact, sub_obj); assert_check_only!(PyDate_Check, PyDate_CheckExact, sub_sub_obj); @@ -68,6 +69,7 @@ fn test_time_check() { let gil = Python::acquire_gil(); let py = gil.python(); let (obj, sub_obj, sub_sub_obj) = _get_subclasses(&py, "time", "12, 30, 15").unwrap(); + unsafe { PyDateTime_IMPORT() } assert_check_exact!(PyTime_Check, PyTime_CheckExact, obj); assert_check_only!(PyTime_Check, PyTime_CheckExact, sub_obj); @@ -81,6 +83,7 @@ fn test_datetime_check() { let (obj, sub_obj, sub_sub_obj) = _get_subclasses(&py, "datetime", "2018, 1, 1, 13, 30, 15") .map_err(|e| e.print(py)) .unwrap(); + unsafe { PyDateTime_IMPORT() } assert_check_only!(PyDate_Check, PyDate_CheckExact, obj); assert_check_exact!(PyDateTime_Check, PyDateTime_CheckExact, obj); @@ -93,6 +96,7 @@ fn test_delta_check() { let gil = Python::acquire_gil(); let py = gil.python(); let (obj, sub_obj, sub_sub_obj) = _get_subclasses(&py, "timedelta", "1, -3").unwrap(); + unsafe { PyDateTime_IMPORT() } assert_check_exact!(PyDelta_Check, PyDelta_CheckExact, obj); assert_check_only!(PyDelta_Check, PyDelta_CheckExact, sub_obj); diff --git a/tests/test_enum.rs b/tests/test_enum.rs index 405e7f01..d1f66a24 100644 --- a/tests/test_enum.rs +++ b/tests/test_enum.rs @@ -14,12 +14,11 @@ pub enum MyEnum { #[test] fn test_enum_class_attr() { - let gil = Python::acquire_gil(); - let py = gil.python(); - let my_enum = py.get_type::(); - py_assert!(py, my_enum, "getattr(my_enum, 'Variant', None) is not None"); - py_assert!(py, my_enum, "getattr(my_enum, 'foobar', None) is None"); - py_run!(py, my_enum, "my_enum.Variant = None"); + Python::with_gil(|py| { + let my_enum = py.get_type::(); + let var = Py::new(py, MyEnum::Variant).unwrap(); + py_assert!(py, my_enum var, "my_enum.Variant == var"); + }) } #[pyfunction] @@ -28,7 +27,6 @@ fn return_enum() -> MyEnum { } #[test] -#[ignore] // need to implement __eq__ fn test_return_enum() { let gil = Python::acquire_gil(); let py = gil.python(); @@ -44,14 +42,24 @@ fn enum_arg(e: MyEnum) { } #[test] -#[ignore] // need to implement __eq__ fn test_enum_arg() { - let gil = Python::acquire_gil(); - let py = gil.python(); - let f = wrap_pyfunction!(enum_arg)(py).unwrap(); - let mynum = py.get_type::(); + Python::with_gil(|py| { + let f = wrap_pyfunction!(enum_arg)(py).unwrap(); + let mynum = py.get_type::(); - py_run!(py, f mynum, "f(mynum.Variant)") + py_run!(py, f mynum, "f(mynum.OtherVariant)") + }) +} + +#[test] +fn test_enum_eq() { + Python::with_gil(|py| { + let var1 = Py::new(py, MyEnum::Variant).unwrap(); + let var2 = Py::new(py, MyEnum::Variant).unwrap(); + let other_var = Py::new(py, MyEnum::OtherVariant).unwrap(); + py_assert!(py, var1 var2, "var1 == var2"); + py_assert!(py, var1 other_var, "var1 != other_var"); + }) } #[test] @@ -63,3 +71,93 @@ fn test_default_repr_correct() { py_assert!(py, var2, "repr(var2) == 'MyEnum.OtherVariant'"); }) } + +#[pyclass] +enum CustomDiscriminant { + One = 1, + Two = 2, +} + +#[test] +fn test_custom_discriminant() { + Python::with_gil(|py| { + #[allow(non_snake_case)] + let CustomDiscriminant = py.get_type::(); + let one = Py::new(py, CustomDiscriminant::One).unwrap(); + let two = Py::new(py, CustomDiscriminant::Two).unwrap(); + py_run!(py, CustomDiscriminant one two, r#" + assert CustomDiscriminant.One == one + assert CustomDiscriminant.Two == two + assert one != two + "#); + }) +} + +#[test] +fn test_enum_to_int() { + Python::with_gil(|py| { + let one = Py::new(py, CustomDiscriminant::One).unwrap(); + py_assert!(py, one, "int(one) == 1"); + let v = Py::new(py, MyEnum::Variant).unwrap(); + let v_value = MyEnum::Variant as isize; + py_run!(py, v v_value, "int(v) == v_value"); + }) +} + +#[test] +fn test_enum_compare_int() { + Python::with_gil(|py| { + let one = Py::new(py, CustomDiscriminant::One).unwrap(); + py_run!( + py, + one, + r#" + assert one == 1 + assert 1 == one + assert one != 2 + "# + ) + }) +} + +#[pyclass] +#[repr(u8)] +enum SmallEnum { + V = 1, +} + +#[test] +fn test_enum_compare_int_no_throw_when_overflow() { + Python::with_gil(|py| { + let v = Py::new(py, SmallEnum::V).unwrap(); + py_assert!(py, v, "v != 1<<30") + }) +} + +#[pyclass] +#[repr(usize)] +#[allow(clippy::enum_clike_unportable_variant)] +enum BigEnum { + V = usize::MAX, +} + +#[test] +fn test_big_enum_no_overflow() { + Python::with_gil(|py| { + let usize_max = usize::MAX; + let v = Py::new(py, BigEnum::V).unwrap(); + py_assert!(py, usize_max v, "v == usize_max"); + py_assert!(py, usize_max v, "int(v) == usize_max"); + }) +} + +#[pyclass] +#[repr(u16, align(8))] +enum TestReprParse { + V, +} + +#[test] +fn test_repr_parse() { + assert_eq!(std::mem::align_of::(), 8); +} diff --git a/tests/test_frompyobject.rs b/tests/test_frompyobject.rs index 0a213acb..17428ced 100644 --- a/tests/test_frompyobject.rs +++ b/tests/test_frompyobject.rs @@ -3,7 +3,6 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use pyo3::types::{PyDict, PyString, PyTuple}; -use pyo3::PyMappingProtocol; #[macro_use] mod common; @@ -39,8 +38,8 @@ pub struct PyA { foo: Option, } -#[pyproto] -impl PyMappingProtocol for PyA { +#[pymethods] +impl PyA { fn __getitem__(&self, key: String) -> pyo3::PyResult { if key == "t" { Ok("bar".into()) diff --git a/tests/test_gc.rs b/tests/test_gc.rs index bf0ca182..839517b9 100644 --- a/tests/test_gc.rs +++ b/tests/test_gc.rs @@ -1,6 +1,5 @@ #![cfg(feature = "macros")] -use pyo3::class::PyGCProtocol; use pyo3::class::PyTraverseError; use pyo3::class::PyVisit; use pyo3::prelude::*; @@ -89,8 +88,8 @@ struct GcIntegration { dropped: TestDropCall, } -#[pyproto] -impl PyGCProtocol for GcIntegration { +#[pymethods] +impl GcIntegration { fn __traverse__(&self, visit: PyVisit) -> Result<(), PyTraverseError> { visit.call(&self.self_ref) } @@ -129,11 +128,11 @@ fn gc_integration() { assert!(drop_called.load(Ordering::Relaxed)); } -#[pyclass(gc)] +#[pyclass] struct GcIntegration2 {} -#[pyproto] -impl PyGCProtocol for GcIntegration2 { +#[pymethods] +impl GcIntegration2 { fn __traverse__(&self, _visit: PyVisit) -> Result<(), PyTraverseError> { Ok(()) } @@ -216,7 +215,7 @@ fn inheritance_with_new_methods_with_drop() { assert!(drop_called2.load(Ordering::Relaxed)); } -#[pyclass(gc)] +#[pyclass] struct TraversableClass { traversed: AtomicBool, } @@ -229,8 +228,8 @@ impl TraversableClass { } } -#[pyproto] -impl PyGCProtocol for TraversableClass { +#[pymethods] +impl TraversableClass { fn __clear__(&mut self) {} fn __traverse__(&self, _visit: PyVisit) -> Result<(), PyTraverseError> { self.traversed.store(true, Ordering::Relaxed); diff --git a/tests/test_gc_pyproto.rs b/tests/test_gc_pyproto.rs new file mode 100644 index 00000000..bf0911d8 --- /dev/null +++ b/tests/test_gc_pyproto.rs @@ -0,0 +1,282 @@ +#![cfg(feature = "macros")] +#![cfg(feature = "pyproto")] + +use pyo3::class::PyGCProtocol; +use pyo3::class::PyTraverseError; +use pyo3::class::PyVisit; +use pyo3::prelude::*; +use pyo3::type_object::PyTypeObject; +use pyo3::{py_run, AsPyPointer, PyCell, PyTryInto}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; + +mod common; + +#[pyclass(freelist = 2)] +struct ClassWithFreelist {} + +#[test] +fn class_with_freelist() { + let ptr; + { + let gil = Python::acquire_gil(); + let py = gil.python(); + + let inst = Py::new(py, ClassWithFreelist {}).unwrap(); + let _inst2 = Py::new(py, ClassWithFreelist {}).unwrap(); + ptr = inst.as_ptr(); + drop(inst); + } + + { + let gil = Python::acquire_gil(); + let py = gil.python(); + + let inst3 = Py::new(py, ClassWithFreelist {}).unwrap(); + assert_eq!(ptr, inst3.as_ptr()); + + let inst4 = Py::new(py, ClassWithFreelist {}).unwrap(); + assert_ne!(ptr, inst4.as_ptr()) + } +} + +struct TestDropCall { + drop_called: Arc, +} + +impl Drop for TestDropCall { + fn drop(&mut self) { + self.drop_called.store(true, Ordering::Relaxed); + } +} + +#[allow(dead_code)] +#[pyclass] +struct DataIsDropped { + member1: TestDropCall, + member2: TestDropCall, +} + +#[test] +fn data_is_dropped() { + let drop_called1 = Arc::new(AtomicBool::new(false)); + let drop_called2 = Arc::new(AtomicBool::new(false)); + + { + let gil = Python::acquire_gil(); + let py = gil.python(); + let data_is_dropped = DataIsDropped { + member1: TestDropCall { + drop_called: Arc::clone(&drop_called1), + }, + member2: TestDropCall { + drop_called: Arc::clone(&drop_called2), + }, + }; + let inst = Py::new(py, data_is_dropped).unwrap(); + assert!(!drop_called1.load(Ordering::Relaxed)); + assert!(!drop_called2.load(Ordering::Relaxed)); + drop(inst); + } + + assert!(drop_called1.load(Ordering::Relaxed)); + assert!(drop_called2.load(Ordering::Relaxed)); +} + +#[allow(dead_code)] +#[pyclass] +struct GcIntegration { + self_ref: PyObject, + dropped: TestDropCall, +} + +#[pyproto] +impl PyGCProtocol for GcIntegration { + fn __traverse__(&self, visit: PyVisit) -> Result<(), PyTraverseError> { + visit.call(&self.self_ref) + } + + fn __clear__(&mut self) { + let gil = Python::acquire_gil(); + self.self_ref = gil.python().None(); + } +} + +#[test] +fn gc_integration() { + let drop_called = Arc::new(AtomicBool::new(false)); + + { + let gil = Python::acquire_gil(); + let py = gil.python(); + let inst = PyCell::new( + py, + GcIntegration { + self_ref: py.None(), + dropped: TestDropCall { + drop_called: Arc::clone(&drop_called), + }, + }, + ) + .unwrap(); + + let mut borrow = inst.borrow_mut(); + borrow.self_ref = inst.to_object(py); + } + + let gil = Python::acquire_gil(); + let py = gil.python(); + py.run("import gc; gc.collect()", None, None).unwrap(); + assert!(drop_called.load(Ordering::Relaxed)); +} + +#[pyclass] +struct GcIntegration2 {} + +#[pyproto] +impl PyGCProtocol for GcIntegration2 { + fn __traverse__(&self, _visit: PyVisit) -> Result<(), PyTraverseError> { + Ok(()) + } + fn __clear__(&mut self) {} +} + +#[test] +fn gc_integration2() { + let gil = Python::acquire_gil(); + let py = gil.python(); + let inst = PyCell::new(py, GcIntegration2 {}).unwrap(); + py_run!(py, inst, "import gc; assert inst in gc.get_objects()"); +} + +#[pyclass(subclass)] +struct BaseClassWithDrop { + data: Option>, +} + +#[pymethods] +impl BaseClassWithDrop { + #[new] + fn new() -> BaseClassWithDrop { + BaseClassWithDrop { data: None } + } +} + +impl Drop for BaseClassWithDrop { + fn drop(&mut self) { + if let Some(data) = &self.data { + data.store(true, Ordering::Relaxed); + } + } +} + +#[pyclass(extends = BaseClassWithDrop)] +struct SubClassWithDrop { + data: Option>, +} + +#[pymethods] +impl SubClassWithDrop { + #[new] + fn new() -> (Self, BaseClassWithDrop) { + ( + SubClassWithDrop { data: None }, + BaseClassWithDrop { data: None }, + ) + } +} + +impl Drop for SubClassWithDrop { + fn drop(&mut self) { + if let Some(data) = &self.data { + data.store(true, Ordering::Relaxed); + } + } +} + +#[test] +fn inheritance_with_new_methods_with_drop() { + let drop_called1 = Arc::new(AtomicBool::new(false)); + let drop_called2 = Arc::new(AtomicBool::new(false)); + + { + let gil = Python::acquire_gil(); + let py = gil.python(); + let _typebase = py.get_type::(); + let typeobj = py.get_type::(); + let inst = typeobj.call((), None).unwrap(); + + let obj: &PyCell = inst.try_into().unwrap(); + let mut obj_ref_mut = obj.borrow_mut(); + obj_ref_mut.data = Some(Arc::clone(&drop_called1)); + let base: &mut BaseClassWithDrop = obj_ref_mut.as_mut(); + base.data = Some(Arc::clone(&drop_called2)); + } + + assert!(drop_called1.load(Ordering::Relaxed)); + assert!(drop_called2.load(Ordering::Relaxed)); +} + +#[pyclass] +struct TraversableClass { + traversed: AtomicBool, +} + +impl TraversableClass { + fn new() -> Self { + Self { + traversed: AtomicBool::new(false), + } + } +} + +#[pyproto] +impl PyGCProtocol for TraversableClass { + fn __clear__(&mut self) {} + fn __traverse__(&self, _visit: PyVisit) -> Result<(), PyTraverseError> { + self.traversed.store(true, Ordering::Relaxed); + Ok(()) + } +} + +unsafe fn get_type_traverse(tp: *mut pyo3::ffi::PyTypeObject) -> Option { + std::mem::transmute(pyo3::ffi::PyType_GetSlot(tp, pyo3::ffi::Py_tp_traverse)) +} + +#[test] +fn gc_during_borrow() { + let gil = Python::acquire_gil(); + let py = gil.python(); + + unsafe { + // declare a dummy visitor function + extern "C" fn novisit( + _object: *mut pyo3::ffi::PyObject, + _arg: *mut core::ffi::c_void, + ) -> std::os::raw::c_int { + 0 + } + + // get the traverse function + let ty = TraversableClass::type_object(py).as_type_ptr(); + let traverse = get_type_traverse(ty).unwrap(); + + // create an object and check that traversing it works normally + // when it's not borrowed + let cell = PyCell::new(py, TraversableClass::new()).unwrap(); + let obj = cell.to_object(py); + assert!(!cell.borrow().traversed.load(Ordering::Relaxed)); + traverse(obj.as_ptr(), novisit, std::ptr::null_mut()); + assert!(cell.borrow().traversed.load(Ordering::Relaxed)); + + // create an object and check that it is not traversed if the GC + // is invoked while it is already borrowed mutably + let cell2 = PyCell::new(py, TraversableClass::new()).unwrap(); + let obj2 = cell2.to_object(py); + let guard = cell2.borrow_mut(); + assert!(!guard.traversed.load(Ordering::Relaxed)); + traverse(obj2.as_ptr(), novisit, std::ptr::null_mut()); + assert!(!guard.traversed.load(Ordering::Relaxed)); + drop(guard); + } +} diff --git a/tests/test_mapping.rs b/tests/test_mapping.rs index 221496b5..0e65f60d 100644 --- a/tests/test_mapping.rs +++ b/tests/test_mapping.rs @@ -7,7 +7,6 @@ use pyo3::prelude::*; use pyo3::py_run; use pyo3::types::IntoPyDict; use pyo3::types::PyList; -use pyo3::PyMappingProtocol; mod common; @@ -33,10 +32,7 @@ impl Mapping { }) } } -} -#[pyproto] -impl PyMappingProtocol for Mapping { fn __len__(&self) -> usize { self.index.len() } diff --git a/tests/test_mapping_pyproto.rs b/tests/test_mapping_pyproto.rs new file mode 100644 index 00000000..f82b876a --- /dev/null +++ b/tests/test_mapping_pyproto.rs @@ -0,0 +1,110 @@ +#![cfg(feature = "macros")] +#![cfg(feature = "pyproto")] + +use std::collections::HashMap; + +use pyo3::exceptions::PyKeyError; +use pyo3::prelude::*; +use pyo3::py_run; +use pyo3::types::IntoPyDict; +use pyo3::types::PyList; +use pyo3::PyMappingProtocol; + +mod common; + +#[pyclass] +struct Mapping { + index: HashMap, +} + +#[pymethods] +impl Mapping { + #[new] + fn new(elements: Option<&PyList>) -> PyResult { + if let Some(pylist) = elements { + let mut elems = HashMap::with_capacity(pylist.len()); + for (i, pyelem) in pylist.into_iter().enumerate() { + let elem = String::extract(pyelem)?; + elems.insert(elem, i); + } + Ok(Self { index: elems }) + } else { + Ok(Self { + index: HashMap::new(), + }) + } + } +} + +#[pyproto] +impl PyMappingProtocol for Mapping { + fn __len__(&self) -> usize { + self.index.len() + } + + fn __getitem__(&self, query: String) -> PyResult { + self.index + .get(&query) + .copied() + .ok_or_else(|| PyKeyError::new_err("unknown key")) + } + + fn __setitem__(&mut self, key: String, value: usize) { + self.index.insert(key, value); + } + + fn __delitem__(&mut self, key: String) -> PyResult<()> { + if self.index.remove(&key).is_none() { + Err(PyKeyError::new_err("unknown key")) + } else { + Ok(()) + } + } +} + +/// Return a dict with `m = Mapping(['1', '2', '3'])`. +fn map_dict(py: Python) -> &pyo3::types::PyDict { + let d = [("Mapping", py.get_type::())].into_py_dict(py); + py_run!(py, *d, "m = Mapping(['1', '2', '3'])"); + d +} + +#[test] +fn test_getitem() { + let gil = Python::acquire_gil(); + let py = gil.python(); + let d = map_dict(py); + + py_assert!(py, *d, "m['1'] == 0"); + py_assert!(py, *d, "m['2'] == 1"); + py_assert!(py, *d, "m['3'] == 2"); + py_expect_exception!(py, *d, "print(m['4'])", PyKeyError); +} + +#[test] +fn test_setitem() { + let gil = Python::acquire_gil(); + let py = gil.python(); + let d = map_dict(py); + + py_run!(py, *d, "m['1'] = 4; assert m['1'] == 4"); + py_run!(py, *d, "m['0'] = 0; assert m['0'] == 0"); + py_assert!(py, *d, "len(m) == 4"); + py_expect_exception!(py, *d, "m[0] = 'hello'", PyTypeError); + py_expect_exception!(py, *d, "m[0] = -1", PyTypeError); +} + +#[test] +fn test_delitem() { + let gil = Python::acquire_gil(); + let py = gil.python(); + + let d = map_dict(py); + py_run!( + py, + *d, + "del m['1']; assert len(m) == 2 and m['2'] == 1 and m['3'] == 2" + ); + py_expect_exception!(py, *d, "del m[-1]", PyTypeError); + py_expect_exception!(py, *d, "del m['4']", PyKeyError); +} diff --git a/tests/test_proto_methods.rs b/tests/test_proto_methods.rs index 6dd1a752..e2c238fc 100644 --- a/tests/test_proto_methods.rs +++ b/tests/test_proto_methods.rs @@ -1,9 +1,9 @@ #![cfg(feature = "macros")] -use pyo3::exceptions::PyValueError; -use pyo3::types::{PyList, PySlice, PyType}; +use pyo3::exceptions::{PyIndexError, PyValueError}; +use pyo3::types::{PyDict, PyList, PyMapping, PySequence, PySlice, PyType}; use pyo3::{exceptions::PyAttributeError, prelude::*}; -use pyo3::{ffi, py_run, AsPyPointer, PyCell}; +use pyo3::{py_run, PyCell}; use std::{isize, iter}; mod common; @@ -166,37 +166,198 @@ fn test_bool() { } #[pyclass] -pub struct Len { - l: usize, -} +pub struct LenOverflow; #[pymethods] -impl Len { +impl LenOverflow { fn __len__(&self) -> usize { - self.l + (isize::MAX as usize) + 1 } } #[test] -fn len() { - let gil = Python::acquire_gil(); - let py = gil.python(); +fn len_overflow() { + Python::with_gil(|py| { + let inst = Py::new(py, LenOverflow).unwrap(); + py_expect_exception!(py, inst, "len(inst)", PyOverflowError); + }); +} - let inst = Py::new(py, Len { l: 10 }).unwrap(); - py_assert!(py, inst, "len(inst) == 10"); - unsafe { - assert_eq!(ffi::PyObject_Size(inst.as_ptr()), 10); - assert_eq!(ffi::PyMapping_Size(inst.as_ptr()), 10); +#[pyclass] +pub struct Mapping { + values: Py, +} + +#[pymethods] +impl Mapping { + fn __len__(&self, py: Python) -> usize { + self.values.as_ref(py).len() } - let inst = Py::new( - py, - Len { - l: (isize::MAX as usize) + 1, - }, - ) - .unwrap(); - py_expect_exception!(py, inst, "len(inst)", PyOverflowError); + fn __getitem__<'a>(&'a self, key: &'a PyAny) -> PyResult<&'a PyAny> { + let any: &PyAny = self.values.as_ref(key.py()).as_ref(); + any.get_item(key) + } + + fn __setitem__(&self, key: &PyAny, value: &PyAny) -> PyResult<()> { + self.values.as_ref(key.py()).set_item(key, value) + } + + fn __delitem__(&self, key: &PyAny) -> PyResult<()> { + self.values.as_ref(key.py()).del_item(key) + } +} + +#[test] +fn mapping() { + Python::with_gil(|py| { + let inst = Py::new( + py, + Mapping { + values: PyDict::new(py).into(), + }, + ) + .unwrap(); + + // + let mapping: &PyMapping = inst.as_ref(py).downcast().unwrap(); + + py_assert!(py, inst, "len(inst) == 0"); + + py_run!(py, inst, "inst['foo'] = 'foo'"); + py_assert!(py, inst, "inst['foo'] == 'foo'"); + py_run!(py, inst, "del inst['foo']"); + py_expect_exception!(py, inst, "inst['foo']", PyKeyError); + + // Default iteration will call __getitem__ with integer indices + // which fails with a KeyError + py_expect_exception!(py, inst, "[*inst] == []", PyKeyError, "0"); + + // check mapping protocol + assert_eq!(mapping.len().unwrap(), 0); + + mapping.set_item(0, 5).unwrap(); + assert_eq!(mapping.len().unwrap(), 1); + + assert_eq!(mapping.get_item(0).unwrap().extract::().unwrap(), 5); + + mapping.del_item(0).unwrap(); + assert_eq!(mapping.len().unwrap(), 0); + }); +} + +#[derive(FromPyObject)] +enum SequenceIndex<'a> { + Integer(isize), + Slice(&'a PySlice), +} + +#[pyclass] +pub struct Sequence { + values: Vec, +} + +#[pymethods] +impl Sequence { + fn __len__(&self) -> usize { + self.values.len() + } + + fn __getitem__(&self, index: SequenceIndex) -> PyResult { + match index { + SequenceIndex::Integer(index) => { + let uindex = self.usize_index(index)?; + self.values + .get(uindex) + .map(Clone::clone) + .ok_or_else(|| PyIndexError::new_err(index)) + } + // Just to prove that slicing can be implemented + SequenceIndex::Slice(s) => Ok(s.into()), + } + } + + fn __setitem__(&mut self, index: isize, value: PyObject) -> PyResult<()> { + let uindex = self.usize_index(index)?; + self.values + .get_mut(uindex) + .map(|place| *place = value) + .ok_or_else(|| PyIndexError::new_err(index)) + } + + fn __delitem__(&mut self, index: isize) -> PyResult<()> { + let uindex = self.usize_index(index)?; + if uindex >= self.values.len() { + Err(PyIndexError::new_err(index)) + } else { + self.values.remove(uindex); + Ok(()) + } + } + + fn append(&mut self, value: PyObject) { + self.values.push(value); + } +} + +impl Sequence { + fn usize_index(&self, index: isize) -> PyResult { + if index < 0 { + let corrected_index = index + self.values.len() as isize; + if corrected_index < 0 { + Err(PyIndexError::new_err(index)) + } else { + Ok(corrected_index as usize) + } + } else { + Ok(index as usize) + } + } +} + +#[test] +fn sequence() { + Python::with_gil(|py| { + let inst = Py::new(py, Sequence { values: vec![] }).unwrap(); + + let sequence: &PySequence = inst.as_ref(py).downcast().unwrap(); + + py_assert!(py, inst, "len(inst) == 0"); + + py_expect_exception!(py, inst, "inst[0]", PyIndexError); + py_run!(py, inst, "inst.append('foo')"); + + py_assert!(py, inst, "inst[0] == 'foo'"); + py_assert!(py, inst, "inst[-1] == 'foo'"); + + py_expect_exception!(py, inst, "inst[1]", PyIndexError); + py_expect_exception!(py, inst, "inst[-2]", PyIndexError); + + py_assert!(py, inst, "[*inst] == ['foo']"); + + py_run!(py, inst, "del inst[0]"); + + py_expect_exception!(py, inst, "inst['foo']", PyTypeError); + + py_assert!(py, inst, "inst[0:2] == slice(0, 2)"); + + // check sequence protocol + + // we don't implement sequence length so that CPython doesn't attempt to correct negative + // indices. + assert!(sequence.len().is_err()); + // however regular python len() works thanks to mp_len slot + assert_eq!(inst.as_ref(py).len().unwrap(), 0); + + py_run!(py, inst, "inst.append(0)"); + sequence.set_item(0, 5).unwrap(); + assert_eq!(inst.as_ref(py).len().unwrap(), 1); + + assert_eq!(sequence.get_item(0).unwrap().extract::().unwrap(), 5); + sequence.del_item(0).unwrap(); + + assert_eq!(inst.as_ref(py).len().unwrap(), 0); + }); } #[pyclass] @@ -447,6 +608,7 @@ fn getattr_doesnt_override_member() { /// Wraps a Python future and yield it once. #[pyclass] +#[derive(Debug)] struct OnceFuture { future: PyObject, polled: bool, @@ -484,24 +646,20 @@ fn test_await() { let gil = Python::acquire_gil(); let py = gil.python(); let once = py.get_type::(); - let source = pyo3::indoc::indoc!( - r#" + let source = r#" import asyncio import sys async def main(): res = await Once(await asyncio.sleep(0.1)) - return res + assert res is None + # For an odd error similar to https://bugs.python.org/issue38563 if sys.platform == "win32" and sys.version_info >= (3, 8, 0): asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) -# get_event_loop can raise an error: https://github.com/PyO3/pyo3/pull/961#issuecomment-645238579 -loop = asyncio.new_event_loop() -asyncio.set_event_loop(loop) -assert loop.run_until_complete(main()) is None -loop.close() -"# - ); + +asyncio.run(main()) +"#; let globals = PyModule::import(py, "__main__").unwrap().dict(); globals.set_item("Once", once).unwrap(); py.run(source, Some(globals), None) @@ -509,6 +667,62 @@ loop.close() .unwrap(); } +#[pyclass] +struct AsyncIterator { + future: Option>, +} + +#[pymethods] +impl AsyncIterator { + #[new] + fn new(future: Py) -> Self { + Self { + future: Some(future), + } + } + + fn __aiter__(slf: PyRef) -> PyRef { + slf + } + + fn __anext__(&mut self) -> Option> { + self.future.take() + } +} + +#[test] +fn test_anext_aiter() { + let gil = Python::acquire_gil(); + let py = gil.python(); + let once = py.get_type::(); + let source = r#" +import asyncio +import sys + +async def main(): + count = 0 + async for result in AsyncIterator(Once(await asyncio.sleep(0.1))): + # The Once is awaited as part of the `async for` and produces None + assert result is None + count +=1 + assert count == 1 + +# For an odd error similar to https://bugs.python.org/issue38563 +if sys.platform == "win32" and sys.version_info >= (3, 8, 0): + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + +asyncio.run(main()) +"#; + let globals = PyModule::import(py, "__main__").unwrap().dict(); + globals.set_item("Once", once).unwrap(); + globals + .set_item("AsyncIterator", py.get_type::()) + .unwrap(); + py.run(source, Some(globals), None) + .map_err(|e| e.print(py)) + .unwrap(); +} + /// Increment the count when `__get__` is called. #[pyclass] struct DescrCounter { diff --git a/tests/test_pyfunction.rs b/tests/test_pyfunction.rs index 930fe780..14920dc1 100644 --- a/tests/test_pyfunction.rs +++ b/tests/test_pyfunction.rs @@ -165,11 +165,24 @@ fn test_function_with_custom_conversion_error() { ); } +#[pyclass] +#[derive(Debug, FromPyObject)] +struct ValueClass { + #[pyo3(get)] + value: usize, +} + #[pyfunction] -fn conversion_error(str_arg: &str, int_arg: i64, tuple_arg: (&str, f64), option_arg: Option) { +fn conversion_error( + str_arg: &str, + int_arg: i64, + tuple_arg: (&str, f64), + option_arg: Option, + struct_arg: Option, +) { println!( - "{:?} {:?} {:?} {:?}", - str_arg, int_arg, tuple_arg, option_arg + "{:?} {:?} {:?} {:?} {:?}", + str_arg, int_arg, tuple_arg, option_arg, struct_arg ); } @@ -182,38 +195,82 @@ fn test_conversion_error() { py_expect_exception!( py, conversion_error, - "conversion_error(None, None, None, None)", + "conversion_error(None, None, None, None, None)", PyTypeError, "argument 'str_arg': 'NoneType' object cannot be converted to 'PyString'" ); py_expect_exception!( py, conversion_error, - "conversion_error(100, None, None, None)", + "conversion_error(100, None, None, None, None)", PyTypeError, "argument 'str_arg': 'int' object cannot be converted to 'PyString'" ); py_expect_exception!( py, conversion_error, - "conversion_error('string1', 'string2', None, None)", + "conversion_error('string1', 'string2', None, None, None)", PyTypeError, "argument 'int_arg': 'str' object cannot be interpreted as an integer" ); py_expect_exception!( py, conversion_error, - "conversion_error('string1', -100, 'string2', None)", + "conversion_error('string1', -100, 'string2', None, None)", PyTypeError, "argument 'tuple_arg': 'str' object cannot be converted to 'PyTuple'" ); py_expect_exception!( py, conversion_error, - "conversion_error('string1', -100, ('string2', 10.), 'string3')", + "conversion_error('string1', -100, ('string2', 10.), 'string3', None)", PyTypeError, "argument 'option_arg': 'str' object cannot be interpreted as an integer" ); + let exception = py_expect_exception!( + py, + conversion_error, + " +class ValueClass: + def __init__(self, value): + self.value = value +conversion_error('string1', -100, ('string2', 10.), None, ValueClass(\"no_expected_type\"))", + PyTypeError + ); + assert_eq!( + extract_traceback(py, exception), + "TypeError: argument 'struct_arg': failed to \ + extract field ValueClass.value: TypeError: 'str' object cannot be interpreted as an integer" + ); + + let exception = py_expect_exception!( + py, + conversion_error, + " +class ValueClass: + def __init__(self, value): + self.value = value +conversion_error('string1', -100, ('string2', 10.), None, ValueClass(-5))", + PyTypeError + ); + assert_eq!( + extract_traceback(py, exception), + "TypeError: argument 'struct_arg': failed to \ + extract field ValueClass.value: OverflowError: can't convert negative int to unsigned" + ); +} + +/// Helper function that concatenates the error message from +/// each error in the traceback into a single string that can +/// be tested. +fn extract_traceback(py: Python, mut error: PyErr) -> String { + let mut error_msg = error.to_string(); + while let Some(cause) = error.cause(py) { + error_msg.push_str(": "); + error_msg.push_str(&cause.to_string()); + error = cause + } + error_msg } #[test] diff --git a/tests/test_pyproto.rs b/tests/test_pyproto.rs index 7e15d73e..0d963c1a 100644 --- a/tests/test_pyproto.rs +++ b/tests/test_pyproto.rs @@ -1,4 +1,5 @@ #![cfg(feature = "macros")] +#![cfg(feature = "pyproto")] use pyo3::class::{ PyAsyncProtocol, PyDescrProtocol, PyIterProtocol, PyMappingProtocol, PyObjectProtocol, diff --git a/tests/test_pyself.rs b/tests/test_pyself.rs index 7e2eb50f..48196300 100644 --- a/tests/test_pyself.rs +++ b/tests/test_pyself.rs @@ -3,7 +3,7 @@ //! Test slf: PyRef/PyMutRef(especially, slf.into::) works use pyo3::prelude::*; use pyo3::types::{PyBytes, PyString}; -use pyo3::{PyCell, PyIterProtocol}; +use pyo3::PyCell; use std::collections::HashMap; mod common; @@ -54,9 +54,10 @@ struct Iter { idx: usize, } -#[pyproto] -impl PyIterProtocol for Iter { - fn __iter__(slf: PyRef) -> PyRef<'p, Self> { +#[pymethods] +impl Iter { + #[allow(clippy::self_named_constructors)] + fn __iter__(slf: PyRef) -> PyRef { slf } diff --git a/tests/test_sequence.rs b/tests/test_sequence.rs index eebd8650..e60cc2ce 100644 --- a/tests/test_sequence.rs +++ b/tests/test_sequence.rs @@ -1,6 +1,5 @@ #![cfg(feature = "macros")] -use pyo3::class::PySequenceProtocol; use pyo3::exceptions::{PyIndexError, PyValueError}; use pyo3::prelude::*; use pyo3::types::{IntoPyDict, PyList}; @@ -31,10 +30,7 @@ impl ByteSequence { }) } } -} -#[pyproto] -impl PySequenceProtocol for ByteSequence { fn __len__(&self) -> usize { self.elements.len() } @@ -50,8 +46,12 @@ impl PySequenceProtocol for ByteSequence { self.elements[idx as usize] = value; } - fn __delitem__(&mut self, idx: isize) -> PyResult<()> { - if (idx < self.elements.len() as isize) && (idx >= 0) { + fn __delitem__(&mut self, mut idx: isize) -> PyResult<()> { + let self_len = self.elements.len() as isize; + if idx < 0 { + idx += self_len; + } + if (idx < self_len) && (idx >= 0) { self.elements.remove(idx as usize); Ok(()) } else { @@ -66,7 +66,7 @@ impl PySequenceProtocol for ByteSequence { } } - fn __concat__(&self, other: PyRef<'p, Self>) -> Self { + fn __concat__(&self, other: PyRef) -> Self { let mut elements = self.elements.clone(); elements.extend_from_slice(&other.elements); Self { elements } @@ -273,8 +273,8 @@ struct OptionList { items: Vec>, } -#[pyproto] -impl PySequenceProtocol for OptionList { +#[pymethods] +impl OptionList { fn __getitem__(&self, idx: isize) -> PyResult> { match self.items.get(idx as usize) { Some(x) => Ok(*x), diff --git a/tests/test_sequence_pyproto.rs b/tests/test_sequence_pyproto.rs new file mode 100644 index 00000000..db4b6d41 --- /dev/null +++ b/tests/test_sequence_pyproto.rs @@ -0,0 +1,304 @@ +#![cfg(feature = "macros")] +#![cfg(feature = "pyproto")] + +use pyo3::class::PySequenceProtocol; +use pyo3::exceptions::{PyIndexError, PyValueError}; +use pyo3::prelude::*; +use pyo3::types::{IntoPyDict, PyList}; + +use pyo3::py_run; + +mod common; + +#[pyclass] +struct ByteSequence { + elements: Vec, +} + +#[pymethods] +impl ByteSequence { + #[new] + fn new(elements: Option<&PyList>) -> PyResult { + if let Some(pylist) = elements { + let mut elems = Vec::with_capacity(pylist.len()); + for pyelem in pylist.into_iter() { + let elem = u8::extract(pyelem)?; + elems.push(elem); + } + Ok(Self { elements: elems }) + } else { + Ok(Self { + elements: Vec::new(), + }) + } + } +} + +#[pyproto] +impl PySequenceProtocol for ByteSequence { + fn __len__(&self) -> usize { + self.elements.len() + } + + fn __getitem__(&self, idx: isize) -> PyResult { + self.elements + .get(idx as usize) + .copied() + .ok_or_else(|| PyIndexError::new_err("list index out of range")) + } + + fn __setitem__(&mut self, idx: isize, value: u8) { + self.elements[idx as usize] = value; + } + + fn __delitem__(&mut self, idx: isize) -> PyResult<()> { + if (idx < self.elements.len() as isize) && (idx >= 0) { + self.elements.remove(idx as usize); + Ok(()) + } else { + Err(PyIndexError::new_err("list index out of range")) + } + } + + fn __contains__(&self, other: &PyAny) -> bool { + match u8::extract(other) { + Ok(x) => self.elements.contains(&x), + Err(_) => false, + } + } + + fn __concat__(&self, other: PyRef<'p, Self>) -> Self { + let mut elements = self.elements.clone(); + elements.extend_from_slice(&other.elements); + Self { elements } + } + + fn __repeat__(&self, count: isize) -> PyResult { + if count >= 0 { + let mut elements = Vec::with_capacity(self.elements.len() * count as usize); + for _ in 0..count { + elements.extend(&self.elements); + } + Ok(Self { elements }) + } else { + Err(PyValueError::new_err("invalid repeat count")) + } + } +} + +/// Return a dict with `s = ByteSequence([1, 2, 3])`. +fn seq_dict(py: Python) -> &pyo3::types::PyDict { + let d = [("ByteSequence", py.get_type::())].into_py_dict(py); + // Though we can construct `s` in Rust, let's test `__new__` works. + py_run!(py, *d, "s = ByteSequence([1, 2, 3])"); + d +} + +#[test] +fn test_getitem() { + let gil = Python::acquire_gil(); + let py = gil.python(); + let d = seq_dict(py); + + py_assert!(py, *d, "s[0] == 1"); + py_assert!(py, *d, "s[1] == 2"); + py_assert!(py, *d, "s[2] == 3"); + py_expect_exception!(py, *d, "print(s[-4])", PyIndexError); + py_expect_exception!(py, *d, "print(s[4])", PyIndexError); +} + +#[test] +fn test_setitem() { + let gil = Python::acquire_gil(); + let py = gil.python(); + let d = seq_dict(py); + + py_run!(py, *d, "s[0] = 4; assert list(s) == [4, 2, 3]"); + py_expect_exception!(py, *d, "s[0] = 'hello'", PyTypeError); +} + +#[test] +fn test_delitem() { + let gil = Python::acquire_gil(); + let py = gil.python(); + + let d = [("ByteSequence", py.get_type::())].into_py_dict(py); + + py_run!( + py, + *d, + "s = ByteSequence([1, 2, 3]); del s[0]; assert list(s) == [2, 3]" + ); + py_run!( + py, + *d, + "s = ByteSequence([1, 2, 3]); del s[1]; assert list(s) == [1, 3]" + ); + py_run!( + py, + *d, + "s = ByteSequence([1, 2, 3]); del s[-1]; assert list(s) == [1, 2]" + ); + py_run!( + py, + *d, + "s = ByteSequence([1, 2, 3]); del s[-2]; assert list(s) == [1, 3]" + ); + py_expect_exception!( + py, + *d, + "s = ByteSequence([1, 2, 3]); del s[-4]; print(list(s))", + PyIndexError + ); + py_expect_exception!( + py, + *d, + "s = ByteSequence([1, 2, 3]); del s[4]", + PyIndexError + ); +} + +#[test] +fn test_contains() { + let gil = Python::acquire_gil(); + let py = gil.python(); + + let d = seq_dict(py); + + py_assert!(py, *d, "1 in s"); + py_assert!(py, *d, "2 in s"); + py_assert!(py, *d, "3 in s"); + py_assert!(py, *d, "4 not in s"); + py_assert!(py, *d, "'hello' not in s"); +} + +#[test] +fn test_concat() { + let gil = Python::acquire_gil(); + let py = gil.python(); + + let d = seq_dict(py); + + py_run!( + py, + *d, + "s1 = ByteSequence([1, 2]); s2 = ByteSequence([3, 4]); assert list(s1 + s2) == [1, 2, 3, 4]" + ); + py_expect_exception!( + py, + *d, + "s1 = ByteSequence([1, 2]); s2 = 'hello'; s1 + s2", + PyTypeError + ); +} + +#[test] +fn test_inplace_concat() { + let gil = Python::acquire_gil(); + let py = gil.python(); + + let d = seq_dict(py); + + py_run!( + py, + *d, + "s += ByteSequence([4, 5]); assert list(s) == [1, 2, 3, 4, 5]" + ); + py_expect_exception!(py, *d, "s += 'hello'", PyTypeError); +} + +#[test] +fn test_repeat() { + let gil = Python::acquire_gil(); + let py = gil.python(); + + let d = seq_dict(py); + + py_run!(py, *d, "s2 = s * 2; assert list(s2) == [1, 2, 3, 1, 2, 3]"); + py_expect_exception!(py, *d, "s2 = s * -1", PyValueError); +} + +#[test] +fn test_inplace_repeat() { + let gil = Python::acquire_gil(); + let py = gil.python(); + + let d = [("ByteSequence", py.get_type::())].into_py_dict(py); + + py_run!( + py, + *d, + "s = ByteSequence([1, 2]); s *= 3; assert list(s) == [1, 2, 1, 2, 1, 2]" + ); + py_expect_exception!(py, *d, "s = ByteSequence([1, 2]); s *= -1", PyValueError); +} + +// Check that #[pyo3(get, set)] works correctly for Vec + +#[pyclass] +struct GenericList { + #[pyo3(get, set)] + items: Vec, +} + +#[test] +fn test_generic_list_get() { + let gil = Python::acquire_gil(); + let py = gil.python(); + + let list: PyObject = GenericList { + items: [1, 2, 3].iter().map(|i| i.to_object(py)).collect(), + } + .into_py(py); + + py_assert!(py, list, "list.items == [1, 2, 3]"); +} + +#[test] +fn test_generic_list_set() { + let gil = Python::acquire_gil(); + let py = gil.python(); + + let list = PyCell::new(py, GenericList { items: vec![] }).unwrap(); + + py_run!(py, list, "list.items = [1, 2, 3]"); + assert_eq!( + list.borrow().items, + vec![1.to_object(py), 2.to_object(py), 3.to_object(py)] + ); +} + +#[pyclass] +struct OptionList { + #[pyo3(get, set)] + items: Vec>, +} + +#[pyproto] +impl PySequenceProtocol for OptionList { + fn __getitem__(&self, idx: isize) -> PyResult> { + match self.items.get(idx as usize) { + Some(x) => Ok(*x), + None => Err(PyIndexError::new_err("Index out of bounds")), + } + } +} + +#[test] +fn test_option_list_get() { + // Regression test for #798 + let gil = Python::acquire_gil(); + let py = gil.python(); + + let list = PyCell::new( + py, + OptionList { + items: vec![Some(1), None], + }, + ) + .unwrap(); + + py_assert!(py, list, "list[0] == 1"); + py_assert!(py, list, "list[1] == None"); + py_expect_exception!(py, list, "list[2]", PyIndexError); +} diff --git a/tests/ui/abi3_nativetype_inheritance.stderr b/tests/ui/abi3_nativetype_inheritance.stderr index 762250a4..360a0a45 100644 --- a/tests/ui/abi3_nativetype_inheritance.stderr +++ b/tests/ui/abi3_nativetype_inheritance.stderr @@ -15,7 +15,7 @@ error[E0277]: the trait bound `PyDict: PyClass` is not satisfied | = note: required because of the requirements on the impl of `PyClassBaseType` for `PyDict` note: required by a bound in `ThreadCheckerInherited` - --> src/class/impl_.rs + --> src/impl_/pyclass.rs | | pub struct ThreadCheckerInherited(PhantomData, U::ThreadChecker); | ^^^^^^^^^^^^^^^ required by this bound in `ThreadCheckerInherited` diff --git a/tests/ui/deprecations.rs b/tests/ui/deprecations.rs index 5ad70f9c..7ad3f98e 100644 --- a/tests/ui/deprecations.rs +++ b/tests/ui/deprecations.rs @@ -11,6 +11,9 @@ impl DeprecatedCall { fn deprecated_call(&self) {} } +#[pyclass(gc)] +struct DeprecatedGc; + fn main() { } diff --git a/tests/ui/deprecations.stderr b/tests/ui/deprecations.stderr index d71c1927..c5d6aba3 100644 --- a/tests/ui/deprecations.stderr +++ b/tests/ui/deprecations.stderr @@ -9,3 +9,9 @@ note: the lint level is defined here | 1 | #![deny(deprecated)] | ^^^^^^^^^^ + +error: use of deprecated constant `pyo3::impl_::deprecations::PYCLASS_GC_OPTION`: implement a `__traverse__` `#[pymethod]` instead of using `gc` option + --> tests/ui/deprecations.rs:14:11 + | +14 | #[pyclass(gc)] + | ^^ diff --git a/tests/ui/invalid_pymethods.rs b/tests/ui/invalid_pymethods.rs index ed45bac0..4fc63dc7 100644 --- a/tests/ui/invalid_pymethods.rs +++ b/tests/ui/invalid_pymethods.rs @@ -119,4 +119,15 @@ impl MyClass { fn default_arg_before_required(&self, has_default: isize, required: isize) {} } +struct TwoNew { } + +#[pymethods] +impl TwoNew { + #[new] + fn new_1() -> Self { Self { } } + + #[new] + fn new_2() -> Self { Self { } } +} + fn main() {} diff --git a/tests/ui/invalid_pymethods.stderr b/tests/ui/invalid_pymethods.stderr index bc11faba..2a851154 100644 --- a/tests/ui/invalid_pymethods.stderr +++ b/tests/ui/invalid_pymethods.stderr @@ -101,3 +101,14 @@ error: `pass_module` cannot be used on Python methods | 112 | #[pyo3(pass_module)] | ^^^^^^^^^^^ + +error[E0592]: duplicate definitions with name `__pymethod__new__` + --> tests/ui/invalid_pymethods.rs:124:1 + | +124 | #[pymethods] + | ^^^^^^^^^^^^ + | | + | duplicate definitions for `__pymethod__new__` + | other definition for `__pymethod__new__` + | + = note: this error originates in the attribute macro `pymethods` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/tests/ui/not_send.stderr b/tests/ui/not_send.stderr index 166a3d6f..69e07ac0 100644 --- a/tests/ui/not_send.stderr +++ b/tests/ui/not_send.stderr @@ -6,14 +6,15 @@ error[E0277]: `*mut pyo3::Python<'static>` cannot be shared between threads safe | = help: within `pyo3::Python<'_>`, the trait `Sync` is not implemented for `*mut pyo3::Python<'static>` = note: required because it appears within the type `PhantomData<*mut pyo3::Python<'static>>` - = note: required because it appears within the type `pyo3::impl_::not_send::NotSend` - = note: required because it appears within the type `(&GILGuard, pyo3::impl_::not_send::NotSend)` - = note: required because it appears within the type `PhantomData<(&GILGuard, pyo3::impl_::not_send::NotSend)>` + = note: required because it appears within the type `impl_::not_send::NotSend` + = note: required because it appears within the type `(&GILGuard, impl_::not_send::NotSend)` + = note: required because it appears within the type `PhantomData<(&GILGuard, impl_::not_send::NotSend)>` = note: required because it appears within the type `pyo3::Python<'_>` = note: required because of the requirements on the impl of `Send` for `&pyo3::Python<'_>` = note: required because it appears within the type `[closure@$DIR/tests/ui/not_send.rs:4:22: 4:38]` + = note: required because of the requirements on the impl of `Ungil` for `[closure@$DIR/tests/ui/not_send.rs:4:22: 4:38]` note: required by a bound in `pyo3::Python::<'py>::allow_threads` - --> src/python.rs + --> src/marker.rs | - | F: Send + FnOnce() -> T, - | ^^^^ required by this bound in `pyo3::Python::<'py>::allow_threads` + | F: Ungil + FnOnce() -> T, + | ^^^^^ required by this bound in `pyo3::Python::<'py>::allow_threads` diff --git a/tests/ui/not_send2.rs b/tests/ui/not_send2.rs new file mode 100644 index 00000000..4eb0a9f0 --- /dev/null +++ b/tests/ui/not_send2.rs @@ -0,0 +1,12 @@ +use pyo3::prelude::*; +use pyo3::types::PyString; + +fn main() { + Python::with_gil(|py| { + let string = PyString::new(py, "foo"); + + py.allow_threads(|| { + println!("{:?}", string); + }); + }); +} \ No newline at end of file diff --git a/tests/ui/not_send2.stderr b/tests/ui/not_send2.stderr new file mode 100644 index 00000000..c33ee4fc --- /dev/null +++ b/tests/ui/not_send2.stderr @@ -0,0 +1,18 @@ +error[E0277]: `UnsafeCell` cannot be shared between threads safely + --> tests/ui/not_send2.rs:8:12 + | +8 | py.allow_threads(|| { + | ^^^^^^^^^^^^^ `UnsafeCell` cannot be shared between threads safely + | + = help: within `&PyString`, the trait `Sync` is not implemented for `UnsafeCell` + = note: required because it appears within the type `PyAny` + = note: required because it appears within the type `PyString` + = note: required because it appears within the type `&PyString` + = note: required because of the requirements on the impl of `Send` for `&&PyString` + = note: required because it appears within the type `[closure@$DIR/tests/ui/not_send2.rs:8:26: 10:10]` + = note: required because of the requirements on the impl of `Ungil` for `[closure@$DIR/tests/ui/not_send2.rs:8:26: 10:10]` +note: required by a bound in `pyo3::Python::<'py>::allow_threads` + --> src/marker.rs + | + | F: Ungil + FnOnce() -> T, + | ^^^^^ required by this bound in `pyo3::Python::<'py>::allow_threads` diff --git a/tests/ui/not_send3.rs b/tests/ui/not_send3.rs new file mode 100644 index 00000000..5941d10a --- /dev/null +++ b/tests/ui/not_send3.rs @@ -0,0 +1,12 @@ +use pyo3::prelude::*; +use std::rc::Rc; + +fn main() { + Python::with_gil(|py| { + let rc = Rc::new(5); + + py.allow_threads(|| { + println!("{:?}", rc); + }); + }); +} \ No newline at end of file diff --git a/tests/ui/not_send3.stderr b/tests/ui/not_send3.stderr new file mode 100644 index 00000000..b152d076 --- /dev/null +++ b/tests/ui/not_send3.stderr @@ -0,0 +1,15 @@ +error[E0277]: `Rc` cannot be shared between threads safely + --> tests/ui/not_send3.rs:8:12 + | +8 | py.allow_threads(|| { + | ^^^^^^^^^^^^^ `Rc` cannot be shared between threads safely + | + = help: the trait `Sync` is not implemented for `Rc` + = note: required because of the requirements on the impl of `Send` for `&Rc` + = note: required because it appears within the type `[closure@$DIR/tests/ui/not_send3.rs:8:26: 10:10]` + = note: required because of the requirements on the impl of `Ungil` for `[closure@$DIR/tests/ui/not_send3.rs:8:26: 10:10]` +note: required by a bound in `pyo3::Python::<'py>::allow_threads` + --> src/marker.rs + | + | F: Ungil + FnOnce() -> T, + | ^^^^^ required by this bound in `pyo3::Python::<'py>::allow_threads` diff --git a/tests/ui/not_send_auto_trait.rs b/tests/ui/not_send_auto_trait.rs new file mode 100644 index 00000000..d7c47dec --- /dev/null +++ b/tests/ui/not_send_auto_trait.rs @@ -0,0 +1,11 @@ +use pyo3::prelude::*; + +fn test_not_send_allow_threads(py: Python) { + py.allow_threads(|| { drop(py); }); +} + +fn main() { + Python::with_gil(|py| { + test_not_send_allow_threads(py); + }) +} diff --git a/tests/ui/not_send_auto_trait.stderr b/tests/ui/not_send_auto_trait.stderr new file mode 100644 index 00000000..c124fd4a --- /dev/null +++ b/tests/ui/not_send_auto_trait.stderr @@ -0,0 +1,15 @@ +error[E0277]: the trait bound `pyo3::Python<'_>: Ungil` is not satisfied in `[closure@$DIR/tests/ui/not_send_auto_trait.rs:4:22: 4:38]` + --> tests/ui/not_send_auto_trait.rs:4:8 + | +4 | py.allow_threads(|| { drop(py); }); + | ^^^^^^^^^^^^^ ---------------- within this `[closure@$DIR/tests/ui/not_send_auto_trait.rs:4:22: 4:38]` + | | + | within `[closure@$DIR/tests/ui/not_send_auto_trait.rs:4:22: 4:38]`, the trait `Ungil` is not implemented for `pyo3::Python<'_>` + | + = note: required because it appears within the type `&pyo3::Python<'_>` + = note: required because it appears within the type `[closure@$DIR/tests/ui/not_send_auto_trait.rs:4:22: 4:38]` +note: required by a bound in `pyo3::Python::<'py>::allow_threads` + --> src/marker.rs + | + | F: Ungil + FnOnce() -> T, + | ^^^^^ required by this bound in `pyo3::Python::<'py>::allow_threads` diff --git a/tests/ui/not_send_auto_trait2.rs b/tests/ui/not_send_auto_trait2.rs new file mode 100644 index 00000000..4eb0a9f0 --- /dev/null +++ b/tests/ui/not_send_auto_trait2.rs @@ -0,0 +1,12 @@ +use pyo3::prelude::*; +use pyo3::types::PyString; + +fn main() { + Python::with_gil(|py| { + let string = PyString::new(py, "foo"); + + py.allow_threads(|| { + println!("{:?}", string); + }); + }); +} \ No newline at end of file diff --git a/tests/ui/not_send_auto_trait2.stderr b/tests/ui/not_send_auto_trait2.stderr new file mode 100644 index 00000000..e549f7e2 --- /dev/null +++ b/tests/ui/not_send_auto_trait2.stderr @@ -0,0 +1,20 @@ +error[E0277]: the trait bound `PyAny: Ungil` is not satisfied in `[closure@$DIR/tests/ui/not_send_auto_trait2.rs:8:26: 10:10]` + --> tests/ui/not_send_auto_trait2.rs:8:12 + | +8 | py.allow_threads(|| { + | ____________^^^^^^^^^^^^^_- + | | | + | | within `[closure@$DIR/tests/ui/not_send_auto_trait2.rs:8:26: 10:10]`, the trait `Ungil` is not implemented for `PyAny` +9 | | println!("{:?}", string); +10 | | }); + | |_________- within this `[closure@$DIR/tests/ui/not_send_auto_trait2.rs:8:26: 10:10]` + | + = note: required because it appears within the type `PyString` + = note: required because it appears within the type `&PyString` + = note: required because it appears within the type `&&PyString` + = note: required because it appears within the type `[closure@$DIR/tests/ui/not_send_auto_trait2.rs:8:26: 10:10]` +note: required by a bound in `pyo3::Python::<'py>::allow_threads` + --> src/marker.rs + | + | F: Ungil + FnOnce() -> T, + | ^^^^^ required by this bound in `pyo3::Python::<'py>::allow_threads` diff --git a/tests/ui/pyclass_send.stderr b/tests/ui/pyclass_send.stderr index 531a4e9d..287430ac 100644 --- a/tests/ui/pyclass_send.stderr +++ b/tests/ui/pyclass_send.stderr @@ -11,7 +11,7 @@ note: required because it appears within the type `NotThreadSafe` 5 | struct NotThreadSafe { | ^^^^^^^^^^^^^ note: required by a bound in `ThreadCheckerStub` - --> src/class/impl_.rs + --> src/impl_/pyclass.rs | | pub struct ThreadCheckerStub(PhantomData); | ^^^^ required by this bound in `ThreadCheckerStub` diff --git a/tests/ui/send_wrapper.rs b/tests/ui/send_wrapper.rs new file mode 100644 index 00000000..7e7f00ea --- /dev/null +++ b/tests/ui/send_wrapper.rs @@ -0,0 +1,16 @@ +use pyo3::prelude::*; +use pyo3::types::PyString; +use send_wrapper::SendWrapper; + +fn main() { + Python::with_gil(|py| { + let string = PyString::new(py, "foo"); + + let wrapped = SendWrapper::new(string); + + py.allow_threads(|| { + let smuggled: &PyString = *wrapped; + println!("{:?}", smuggled); + }); + }); +} \ No newline at end of file diff --git a/tests/ui/send_wrapper.stderr b/tests/ui/send_wrapper.stderr new file mode 100644 index 00000000..2c91e3ba --- /dev/null +++ b/tests/ui/send_wrapper.stderr @@ -0,0 +1,23 @@ +error[E0277]: the trait bound `PyAny: Ungil` is not satisfied in `[closure@$DIR/tests/ui/send_wrapper.rs:11:26: 14:10]` + --> tests/ui/send_wrapper.rs:11:12 + | +11 | py.allow_threads(|| { + | ____________^^^^^^^^^^^^^_- + | | | + | | within `[closure@$DIR/tests/ui/send_wrapper.rs:11:26: 14:10]`, the trait `Ungil` is not implemented for `PyAny` +12 | | let smuggled: &PyString = *wrapped; +13 | | println!("{:?}", smuggled); +14 | | }); + | |_________- within this `[closure@$DIR/tests/ui/send_wrapper.rs:11:26: 14:10]` + | + = note: required because it appears within the type `PyString` + = note: required because it appears within the type `&PyString` + = note: required because it appears within the type `*mut &PyString` + = note: required because it appears within the type `SendWrapper<&PyString>` + = note: required because it appears within the type `&SendWrapper<&PyString>` + = note: required because it appears within the type `[closure@$DIR/tests/ui/send_wrapper.rs:11:26: 14:10]` +note: required by a bound in `pyo3::Python::<'py>::allow_threads` + --> src/marker.rs + | + | F: Ungil + FnOnce() -> T, + | ^^^^^ required by this bound in `pyo3::Python::<'py>::allow_threads` diff --git a/tests/ui/static_ref.stderr b/tests/ui/static_ref.stderr index e264723f..f087904c 100644 --- a/tests/ui/static_ref.stderr +++ b/tests/ui/static_ref.stderr @@ -1,25 +1,11 @@ -error[E0495]: cannot infer an appropriate lifetime for lifetime parameter 'py in function call due to conflicting requirements +error[E0597]: `gil` does not live long enough --> tests/ui/static_ref.rs:4:1 | 4 | #[pyfunction] - | ^^^^^^^^^^^^^ + | ^^^^^^^^^^^^- + | | | + | | `gil` dropped here while still borrowed + | borrowed value does not live long enough + | cast requires that `gil` is borrowed for `'static` | -note: first, the lifetime cannot outlive the anonymous lifetime #1 defined here... - --> tests/ui/static_ref.rs:4:1 - | -4 | #[pyfunction] - | ^^^^^^^^^^^^^ -note: ...so that the expression is assignable - --> tests/ui/static_ref.rs:4:1 - | -4 | #[pyfunction] - | ^^^^^^^^^^^^^ - = note: expected `pyo3::Python<'_>` - found `pyo3::Python<'_>` - = note: but, the lifetime must be valid for the static lifetime... -note: ...so that reference does not outlive borrowed content - --> tests/ui/static_ref.rs:4:1 - | -4 | #[pyfunction] - | ^^^^^^^^^^^^^ = note: this error originates in the attribute macro `pyfunction` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 2b919815..cdbfbf14 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -1,5 +1,5 @@ use anyhow::{ensure, Context, Result}; -use std::{collections::HashMap, process::Command}; +use std::{collections::HashMap, path::Path, process::Command}; use structopt::StructOpt; #[derive(StructOpt)] @@ -107,6 +107,7 @@ fn llvm_cov_command(args: &[&str]) -> Command { "--package=pyo3-build-config", "--package=pyo3-macros-backend", "--package=pyo3-macros", + "--package=pyo3-ffi", ]) .args(args); command @@ -115,16 +116,12 @@ fn llvm_cov_command(args: &[&str]) -> Command { fn run_python_tests<'a>( env: impl IntoIterator + Copy, ) -> Result<()> { - for entry in std::fs::read_dir("pytests")? { - let path = entry?.path(); - if path.is_dir() && path.join("noxfile.py").exists() { - run(Command::new("nox") - .arg("--non-interactive") - .arg("-f") - .arg(path.join("noxfile.py")) - .envs(env))?; - } - } + run(Command::new("nox") + .arg("--non-interactive") + .arg("-f") + .arg(Path::new("pytests").join("noxfile.py")) + .envs(env))?; + for entry in std::fs::read_dir("examples")? { let path = entry?.path(); if path.is_dir() && path.join("noxfile.py").exists() {