luxolus
/
pyo3
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'main' of https://github.com/PyO3/pyo3 into enhanced_extract_type_errors

This commit is contained in:
R2D2 2021-07-31 21:21:10 +02:00
parent a42447bfe2 8e7737d099
commit 403d882d6c
153 changed files with 6061 additions and 3036 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

97
.github/workflows/bench.yml vendored Normal file
View File

@ -0,0 +1,97 @@
on:
push:
branches:
- main
pull_request:
name: Benchmark
jobs:
benchmark:
name: Cargo benchmark
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions-rs/toolchain@v1
with:
profile: minimal
toolchain: stable
override: true
- uses: actions/cache@v2
with:
path: |
~/.cargo/registry
~/.cargo/git
target
key: cargo-${{ runner.os }}-bench-${{ hashFiles('**/Cargo.toml') }}
continue-on-error: true
- name: Run benchmarks
run: |
for bench in call dict gil list pyclass pyobject set tuple; do
cargo bench --features hashbrown --bench "bench_$bench" -- --output-format bencher | tee -a output.txt
done
# Download previous benchmark result from cache (if exists)
- name: Download previous benchmark data
uses: actions/cache@v1
with:
path: ./cache
key: ${{ runner.os }}-benchmark
# Run `github-action-benchmark` action
- name: Store benchmark result
uses: rhysd/github-action-benchmark@v1
with:
name: pyo3-bench
# What benchmark tool the output.txt came from
tool: "cargo"
# Where the output from the benchmark tool is stored
output-file-path: output.txt
# GitHub API token to make a commit comment
github-token: ${{ secrets.GITHUB_TOKEN }}
auto-push: ${{ github.event_name != 'pull_request' }}
pytest-benchmark:
name: pytest benchmark
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
- uses: actions-rs/toolchain@v1
with:
profile: minimal
toolchain: stable
override: true
- uses: actions/cache@v2
with:
path: |
~/.cargo/registry
~/.cargo/git
target
key: cargo-${{ runner.os }}-pytest-bench-${{ hashFiles('**/Cargo.toml') }}
continue-on-error: true
- name: Download previous benchmark data
uses: actions/cache@v1
with:
path: ./cache
key: ${{ runner.os }}-pytest-benchmark
- name: Run benchmarks
run: |
cd examples/pyo3-benchmarks
python -m pip install -r requirements-dev.txt
python setup.py develop
pytest --benchmark-json ../../output.json
- name: Store benchmark result
uses: rhysd/github-action-benchmark@v1
with:
name: pytest-bench
tool: "pytest"
output-file-path: output.json
github-token: ${{ secrets.GITHUB_TOKEN }}
auto-push: ${{ github.event_name != 'pull_request' }}

71
.github/workflows/ci.yml vendored
View File

@ -42,21 +42,33 @@ jobs:
name: python${{ matrix.python-version }}-${{ matrix.platform.python-architecture }} ${{ matrix.platform.os }} ${{ matrix.msrv }}
runs-on: ${{ matrix.platform.os }}
strategy:
fail-fast: false # If one platform fails, allow the rest to keep testing.
fail-fast: false # If one platform fails, allow the rest to keep testing.
matrix:
rust: [stable]
# Github actions has a "bogus" 3.10-beta.1 release which interacts badly
# with pytest (use 3.10-dev again once pytest has released a fix):
# https://github.com/actions/setup-python/issues/207
# https://github.com/pytest-dev/pytest/issues/8539
# python-version: [3.6, 3.7, 3.8, 3.9, 3.10-dev, pypy-3.6, pypy-3.7]
python-version: [3.6, 3.7, 3.8, 3.9, 3.10.0-alpha.7, pypy-3.6, pypy-3.7]
platform: [
{ os: "macos-latest", python-architecture: "x64", rust-target: "x86_64-apple-darwin" },
{ os: "ubuntu-latest", python-architecture: "x64", rust-target: "x86_64-unknown-linux-gnu" },
{ os: "windows-latest", python-architecture: "x64", rust-target: "x86_64-pc-windows-msvc" },
{ os: "windows-latest", python-architecture: "x86", rust-target: "i686-pc-windows-msvc" },
]
python-version: [3.6, 3.7, 3.8, 3.9, 3.10-dev, pypy-3.6, pypy-3.7]
platform:
[
{
os: "macos-latest",
python-architecture: "x64",
rust-target: "x86_64-apple-darwin",
},
{
os: "ubuntu-latest",
python-architecture: "x64",
rust-target: "x86_64-unknown-linux-gnu",
},
{
os: "windows-latest",
python-architecture: "x64",
rust-target: "x86_64-pc-windows-msvc",
},
{
os: "windows-latest",
python-architecture: "x86",
rust-target: "i686-pc-windows-msvc",
},
]
exclude:
# There is no 64-bit pypy on windows for pypy-3.6
- python-version: pypy-3.6
@ -68,7 +80,12 @@ jobs:
# Test minimal supported Rust version
- rust: 1.41.1
python-version: 3.9
platform: { os: "ubuntu-latest", python-architecture: "x64", rust-target: "x86_64-unknown-linux-gnu" }
platform:
{
os: "ubuntu-latest",
python-architecture: "x64",
rust-target: "x86_64-unknown-linux-gnu",
}
msrv: "MSRV"
steps:
@ -113,11 +130,13 @@ jobs:
id: settings
shell: bash
run: |
echo "::set-output name=all_additive_features::macros num-bigint num-complex hashbrown serde multiple-pymethods"
echo "::set-output name=all_additive_features::macros num-bigint num-complex hashbrown indexmap serde multiple-pymethods"
- if: matrix.msrv == 'MSRV'
name: Prepare minimal package versions (MSRV only)
run: cargo update -p hashbrown --precise 0.9.1
run: |
cargo update -p indexmap --precise 1.6.2
cargo update -p hashbrown:0.11.2 --precise 0.9.1
- name: Build docs
run: cargo doc --no-deps --no-default-features --features "${{ steps.settings.outputs.all_additive_features }}"
@ -151,6 +170,9 @@ jobs:
- name: Test proc-macro code
run: cargo test --manifest-path=pyo3-macros-backend/Cargo.toml
- name: Test build config
run: cargo test --manifest-path=pyo3-build-config/Cargo.toml
- name: Install python test dependencies
run: python -m pip install -U pip tox
@ -163,6 +185,16 @@ jobs:
env:
TOX_TESTENV_PASSENV: "CARGO_BUILD_TARGET"
- name: Test cross compilation
if: ${{ matrix.platform.os == 'ubuntu-latest' && matrix.python-version == '3.9' }}
uses: messense/maturin-action@v1
env:
PYO3_CROSS_LIB_DIR: /opt/python/cp39-cp39/lib
with:
target: aarch64-unknown-linux-gnu
manylinux: auto
args: --release -i python3.9 --no-sdist -m examples/maturin-starter/Cargo.toml
env:
CARGO_TERM_VERBOSE: true
CARGO_BUILD_TARGET: ${{ matrix.platform.rust-target }}
@ -198,8 +230,10 @@ jobs:
override: true
profile: minimal
components: llvm-tools-preview
- run: LLVM_PROFILE_FILE="coverage-%p-%m.profraw" cargo test --no-default-features --no-fail-fast
- run: LLVM_PROFILE_FILE="coverage-features-%p-%m.profraw" cargo test --no-default-features --no-fail-fast --features "macros num-bigint num-complex hashbrown serde multiple-pymethods"
- run: cargo test --no-default-features --no-fail-fast
- run: cargo test --no-default-features --no-fail-fast --features "macros num-bigint num-complex hashbrown indexmap serde multiple-pymethods"
- run: cargo test --manifest-path=pyo3-macros-backend/Cargo.toml
- run: cargo test --manifest-path=pyo3-build-config/Cargo.toml
# can't yet use actions-rs/grcov with source-based coverage: https://github.com/actions-rs/grcov/issues/105
# - uses: actions-rs/grcov@v0.1
# id: coverage
@ -215,3 +249,4 @@ jobs:
CARGO_TERM_VERBOSE: true
RUSTFLAGS: "-Zinstrument-coverage"
RUSTDOCFLAGS: "-Zinstrument-coverage"
LLVM_PROFILE_FILE: "coverage-%p-%m.profraw"

15
.github/workflows/guide.yml vendored
View File

@ -21,13 +21,13 @@ jobs:
- uses: actions-rs/toolchain@v1
with:
toolchain: nightly
profile: minimal
toolchain: nightly
profile: minimal
- name: Setup mdBook
uses: peaceiris/actions-mdbook@v1
with:
mdbook-version: '0.4.7'
mdbook-version: "0.4.10"
- name: Prepare tag
id: prepare_tag
@ -44,7 +44,7 @@ jobs:
# This adds the docs to gh-pages-build/doc
- name: Build the doc
run: |
cargo +nightly rustdoc --lib --no-default-features --features="macros num-bigint num-complex hashbrown serde multiple-pymethods" -- --cfg docsrs
cargo +nightly rustdoc --lib --no-default-features --features="macros num-bigint num-complex hashbrown indexmap serde multiple-pymethods" -- --cfg docsrs
cp -r target/doc gh-pages-build/doc
echo "<meta http-equiv=refresh content=0;url=pyo3/index.html>" > gh-pages-build/doc/index.html
@ -55,24 +55,25 @@ jobs:
github_token: ${{ secrets.GITHUB_TOKEN }}
publish_dir: ./gh-pages-build/
destination_dir: ${{ steps.prepare_tag.outputs.tag_name }}
full_commit_message: 'Upload documentation for ${{ steps.prepare_tag.outputs.tag_name }}'
full_commit_message: "Upload documentation for ${{ steps.prepare_tag.outputs.tag_name }}"
release:
needs: build
runs-on: ubuntu-latest
if: ${{ github.event_name == 'release' }}
steps:
- name: Create latest tag redirect
- name: Create latest tag redirects
env:
TAG_NAME: ${{ needs.build.outputs.tag_name }}
run: |
mkdir public
echo "<meta http-equiv=refresh content=0;url='$TAG_NAME/'>" > public/index.html
ln -sfT $TAG_NAME public/latest
- name: Deploy
uses: peaceiris/actions-gh-pages@v3.7.0-8
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
publish_dir: ./public/
full_commit_message: 'Release ${{ needs.build.outputs.tag_name }}'
full_commit_message: "Release ${{ needs.build.outputs.tag_name }}"
keep_files: true

48
Architecture.md
View File

@ -17,20 +17,22 @@ protocol traits (e.g., `PyIterProtocol`) for supporting object protocols (i.e.,
Since implementing `PyClass` requires lots of boilerplate, we have a proc-macro `#[pyclass]`.
To summarize, there are six main parts to the PyO3 codebase.
1. [Low-level bindings of Python/C API.](#1-low-level-bindings-of-python-capi)
- [`src/ffi`]
- [`src/ffi`]
2. [Bindings to Python objects.](#2-bindings-to-python-objects)
- [`src/instance.rs`] and [`src/types`]
- [`src/instance.rs`] and [`src/types`]
3. [`PyClass` and related functionalities.](#3-pyclass-and-related-functionalities)
- [`src/pycell.rs`], [`src/pyclass.rs`], and more
- [`src/pycell.rs`], [`src/pyclass.rs`], and more
4. [Protocol methods like `__getitem__`.](#4-protocol-methods)
- [`src/class`]
- [`src/class`]
5. [Procedural macros to simplify usage for users.](#5-procedural-macros-to-simplify-usage-for-users)
- [`src/derive_utils.rs`], [`pyo3-macros`] and [`pyo3-macros-backend`]
- [`src/derive_utils.rs`], [`pyo3-macros`] and [`pyo3-macros-backend`]
6. [`build.rs`](#6-buildrs)
- [`build.rs`](https://github.com/PyO3/pyo3/tree/master/build.rs)
- [`build.rs`](https://github.com/PyO3/pyo3/tree/master/build.rs)
## 1. Low-level bindings of Python/C API
[`src/ffi`] contains wrappers of [Python/C API].
We aim to provide straight-forward Rust wrappers resembling the file structure of
@ -44,17 +46,19 @@ In the [`src/ffi`] module, there is lots of conditional compilation such as `#[c
`#[cfg(Py_37)]`, and `#[cfg(PyPy)]`.
`Py_LIMITED_API` corresponds to `#define Py_LIMITED_API` macro in Python/C API.
With `Py_LIMITED_API`, we can build a Python-version-agnostic binary called an
[abi3 wheel](https://pyo3.rs/v0.13.2/building_and_distribution.html#py_limited_apiabi3).
[abi3 wheel](https://pyo3.rs/latest/building_and_distribution.html#py_limited_apiabi3).
`Py_37` means that the API is available from Python >= 3.7.
There are also `Py_38`, `Py_39`, and so on.
`PyPy` means that the API definition is for PyPy.
Those flags are set in [`build.rs`](#6-buildrs).
## 2. Bindings to Python objects
[`src/types`] contains bindings to [built-in types](https://docs.python.org/3/library/stdtypes.html)
of Python, such as `dict` and `list`.
For historical reasons, Python's `object` is called `PyAny` in PyO3 and located in [`src/types/any.rs`].
Currently, `PyAny` is a straightforward wrapper of `ffi::PyObject`, defined as:
```rust
#[repr(transparent)]
pub struct PyAny(UnsafeCell<ffi::PyObject>);
@ -62,6 +66,7 @@ pub struct PyAny(UnsafeCell<ffi::PyObject>);
All built-in types are defined as a C struct.
For example, `dict` is defined as:
```c
typedef struct {
/* Base object */
@ -77,6 +82,7 @@ typedef struct {
However, we cannot access such a specific data structure with `#[cfg(Py_LIMITED_API)]` set.
Thus, all builtin objects are implemented as opaque types by wrapping `PyAny`, e.g.,:
```rust
#[repr(transparent)]
pub struct PyDict(PyAny);
@ -92,12 +98,14 @@ Since we need lots of boilerplate for implementing common traits for these types
[`src/types/mod.rs`].
## 3. `PyClass` and related functionalities
[`src/pycell.rs`], [`src/pyclass.rs`], and [`src/type_object.rs`] contain types and
traits to make `#[pyclass]` work.
Also, [`src/pyclass_init.rs`] and [`src/pyclass_slots.rs`] have related functionalities.
To realize object-oriented programming in C, all Python objects must have the following two fields
at the beginning.
```rust
#[repr(C)]
pub struct PyObject {
@ -106,9 +114,11 @@ pub struct PyObject {
...
}
```
Thanks to this guarantee, casting `*mut A` to `*mut PyObject` is valid if `A` is a Python object.
To ensure this guarantee, we have a wrapper struct `PyCell<T>` in [`src/pycell.rs`] which is roughly:
```rust
#[repr(C)]
pub struct PyCell<T: PyClass> {
@ -116,6 +126,7 @@ pub struct PyCell<T: PyClass> {
inner: T,
}
```
Thus, when copying a Rust struct to a Python object, we first allocate `PyCell` on the Python heap and then
move `T` into it.
Also, `PyCell` provides [RefCell](https://doc.rust-lang.org/std/cell/struct.RefCell.html)-like methods
@ -131,6 +142,7 @@ For example, you can see `type({})` shows `dict` and `type(type({}))` shows `typ
`T: PyTypeObject` implies that `T` has a corresponding type object.
## 4. Protocol methods
Python has some built-in special methods called dunder, such as `__iter__`.
They are called [abstract objects layer](https://docs.python.org/3/c-api/abstract.html) in
Python/C API.
@ -147,6 +159,7 @@ Also, [`src/class/methods.rs`] has utilities for `#[pyfunction]` and [`src/class
some internal tricks for making `#[pyproto]` flexible.
## 5. Procedural macros to simplify usage for users.
[`pyo3-macros`] provides six proc-macro APIs: `pymodule`, `pyproto`, `pyfunction`, `pyclass`,
`pymethods`, and `#[derive(FromPyObject)]`.
[`pyo3-macros-backend`] has the actual implementations of these APIs.
@ -154,33 +167,42 @@ some internal tricks for making `#[pyproto]` flexible.
such as parsing function arguments.
## 6. `build.rs`
PyO3's [`build.rs`](https://github.com/PyO3/pyo3/tree/master/build.rs) is relatively long
(about 900 lines) to support multiple architectures, interpreters, and usages.
Below is a non-exhaustive list of its functionality:
- Cross-compiling support.
- If `TARGET` architecture and `HOST` architecture differ, we find cross compile information
from environment variables (`PYO3_CROSS_LIB_DIR` and `PYO3_CROSS_PYTHON_VERSION`) or system files.
from environment variables (`PYO3_CROSS_LIB_DIR` and `PYO3_CROSS_PYTHON_VERSION`) or system files.
- Find the interpreter for build and detect the Python version.
- We have to set some version flags like `Py_37`.
- If the interpreter is PyPy, we set `PyPy`.
- If `PYO3_NO_PYTHON` environment variable is set then the interpreter detection is bypassed
entirely and only abi3 extensions can be built.
entirely and only abi3 extensions can be built.
- Check if we are building a Python extension.
- If we are building an extension (e.g., Python library installable by `pip`),
we don't link `libpython`.
Currently we use the `extension-module` feature for this purpose. This may change in the future.
See [#1123](https://github.com/PyO3/pyo3/pull/1123).
we don't link `libpython`.
Currently we use the `extension-module` feature for this purpose. This may change in the future.
See [#1123](https://github.com/PyO3/pyo3/pull/1123).
<!-- External Links -->
[Python/C API]: https://docs.python.org/3/c-api/
[python/c api]: https://docs.python.org/3/c-api/
<!-- Crates -->
[`pyo3-macros`]: https://github.com/PyO3/pyo3/tree/master/pyo3-macros
[`pyo3-macros-backend`]: https://github.com/PyO3/pyo3/tree/master/pyo3-macros-backend
<!-- Directories -->
[`src/class`]: https://github.com/PyO3/pyo3/tree/master/src/class
[`src/ffi`]: https://github.com/PyO3/pyo3/tree/master/src/ffi
[`src/types`]: https://github.com/PyO3/pyo3/tree/master/src/types
<!-- Files -->
[`src/derive_utils.rs`]: https://github.com/PyO3/pyo3/tree/master/src/derive_utils.rs
[`src/instance.rs`]: https://github.com/PyO3/pyo3/tree/master/src/instance.rs
[`src/pycell.rs`]: https://github.com/PyO3/pyo3/tree/master/src/pycell.rs

252
CHANGELOG.md
View File

@ -1,40 +1,78 @@
# Changelog
All notable changes to this project will be documented in this file. For help with updating to new
PyO3 versions, please see the [migration guide](https://pyo3.rs/main/migration.html).
PyO3 versions, please see the [migration guide](https://pyo3.rs/latest/migration.html).
The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
## [Unreleased]
### Added
- Add `indexmap` feature to add `ToPyObject`, `IntoPy` and `FromPyObject` implementations for `indexmap::IndexMap`. [#1728](https://github.com/PyO3/pyo3/pull/1728)
### Changed
- `PyErr::new` no longer acquires the Python GIL internally. [#1724](https://github.com/PyO3/pyo3/pull/1724)
### Fixed
- Fix regression in 0.14.0 rejecting usage of `#[doc(hidden)]` on structs and functions annotated with PyO3 macros. [#1722](https://github.com/PyO3/pyo3/pull/1722)
- Fix regression in 0.14.0 leading to incorrect code coverage being computed for `#[pyfunction]`s. [#1726](https://github.com/PyO3/pyo3/pull/1726)
- Fix incorrect FFI definition of `Py_Buffer` on PyPy. [#1737](https://github.com/PyO3/pyo3/pull/1737)
- Fix incorrect calculation of `dictoffset` on 32-bit Windows. [#1475](https://github.com/PyO3/pyo3/pull/1475)
## [0.14.1] - 2021-07-04
### Added
- Implement `IntoPy<PyObject>` for `&PathBuf` and `&OsString`. [#1712](https://github.com/PyO3/pyo3/pull/1712)
### Fixed
- Fix crashes on PyPy due to incorrect definitions of `PyList_SET_ITEM`. [#1713](https://github.com/PyO3/pyo3/pull/1713)
## [0.14.0] - 2021-07-03
### Packaging
- Update `num-bigint` optional dependency to 0.4. [#1481](https://github.com/PyO3/pyo3/pull/1481)
- Update `num-complex` optional dependency to 0.4. [#1482](https://github.com/PyO3/pyo3/pull/1482)
- Extend `hashbrown` optional dependency supported versions to include 0.11. [#1496](https://github.com/PyO3/pyo3/pull/1496)
- Support PyPy 3.7. [#1538](https://github.com/PyO3/pyo3/pull/1538)
### Added
- Add conversions for `[T; N]` for all `N` on Rust 1.51 and up. [#1128](https://github.com/PyO3/pyo3/pull/1128)
- Add conversions between `OsStr`/`OsString`/`Path`/`PathBuf` and Python strings. [#1379](https://github.com/PyO3/pyo3/pull/1379)
- Add `#[pyo3(from_py_with = "...")]` attribute for function arguments and struct fields to override the default from-Python conversion. [#1411](https://github.com/PyO3/pyo3/pull/1411)
- Extend conversions for `[T; N]` to all `N` using const generics (on Rust 1.51 and up). [#1128](https://github.com/PyO3/pyo3/pull/1128)
- Add conversions between `OsStr`/ `OsString` and Python strings. [#1379](https://github.com/PyO3/pyo3/pull/1379)
- Add conversions between `Path`/ `PathBuf` and Python strings (and `pathlib.Path` objects). [#1379](https://github.com/PyO3/pyo3/pull/1379) [#1654](https://github.com/PyO3/pyo3/pull/1654)
- Add a new set of `#[pyo3(...)]` attributes to control various PyO3 macro functionality:
- `#[pyo3(from_py_with = "...")]` function arguments and struct fields to override the default from-Python conversion. [#1411](https://github.com/PyO3/pyo3/pull/1411)
- `#[pyo3(name = "...")]` for setting Python names. [#1567](https://github.com/PyO3/pyo3/pull/1567)
- `#[pyo3(text_signature = "...")]` for setting text signature. [#1658](https://github.com/PyO3/pyo3/pull/1658)
- Add FFI definition `PyCFunction_CheckExact` for Python 3.9 and later. [#1425](https://github.com/PyO3/pyo3/pull/1425)
- Add FFI definition `Py_IS_TYPE`. [#1429](https://github.com/PyO3/pyo3/pull/1429)
- Add FFI definition `_Py_InitializeMain`. [#1473](https://github.com/PyO3/pyo3/pull/1473)
- Add FFI definitions from `cpython/import.h`.[#1475](https://github.com/PyO3/pyo3/pull/1475)
- Add tuple and unit struct support for `#[pyclass]` macro. [#1504](https://github.com/PyO3/pyo3/pull/1504)
- Add `#[pyo3(name = "...")]` syntax for setting Python names. [#1567](https://github.com/PyO3/pyo3/pull/1567)
- Add FFI definition `PyDateTime_TimeZone_UTC`. [#1572](https://github.com/PyO3/pyo3/pull/1572)
- Add support for `#[pyclass(extends=Exception)]`. [#1591](https://github.com/PyO3/pyo3/pull/1591)
- Add `PyErr::cause` and `PyErr::set_cause`. [#1679](https://github.com/PyO3/pyo3/pull/1679)
- Add FFI definitions from `cpython/pystate.h`. [#1687](https://github.com/PyO3/pyo3/pull/1687/)
- Add `wrap_pyfunction!` macro to `pyo3::prelude`. [#1695](https://github.com/PyO3/pyo3/pull/1695)
### Changed
- Allow only one `#[pymethods]` block per `#[pyclass]` by default, to simplify the proc macro implementations. Add `multiple-pymethods` feature to opt-in to the more complex full behavior. [#1457](https://github.com/PyO3/pyo3/pull/1457)
- Change `PyTimeAcces::get_fold()` to return a `bool` instead of a `u8`. [#1397](https://github.com/PyO3/pyo3/pull/1397)
- Deprecate FFI definition `PyCFunction_Call` for Python 3.9 and later. [#1425](https://github.com/PyO3/pyo3/pull/1425)
- Deprecate FFI definitions `PyModule_GetFilename`. [#1425](https://github.com/PyO3/pyo3/pull/1425)
- Allow only one `#[pymethods]` block per `#[pyclass]` by default, to remove the dependency on `inventory`. Add a `multiple-pymethods` feature to opt-in the original behavior and dependency on `inventory`. [#1457](https://github.com/PyO3/pyo3/pull/1457)
- Change `PyTimeAccess::get_fold` to return a `bool` instead of a `u8`. [#1397](https://github.com/PyO3/pyo3/pull/1397)
- Deprecate FFI definition `PyCFunction_Call` for Python 3.9 and up. [#1425](https://github.com/PyO3/pyo3/pull/1425)
- Deprecate FFI definition `PyModule_GetFilename`. [#1425](https://github.com/PyO3/pyo3/pull/1425)
- The `auto-initialize` feature is no longer enabled by default. [#1443](https://github.com/PyO3/pyo3/pull/1443)
- Change `PyCFunction::new()` and `PyCFunction::new_with_keywords()` to take `&'static str` arguments rather than implicitly copying (and leaking) them. [#1450](https://github.com/PyO3/pyo3/pull/1450)
- Deprecate `PyModule` methods `call`, `call0`, `call1` and `get`. [#1492](https://github.com/PyO3/pyo3/pull/1492)
- Deprecate `PyModule::call`, `PyModule::call0`, `PyModule::call1` and `PyModule::get`. [#1492](https://github.com/PyO3/pyo3/pull/1492)
- Add length information to `PyBufferError`s raised from `PyBuffer::copy_to_slice` and `PyBuffer::copy_from_slice`. [#1534](https://github.com/PyO3/pyo3/pull/1534)
- Automatically provide `-undefined` and `dynamic_lookup` linker arguments on macOS with `extension-module` feature. [#1539](https://github.com/PyO3/pyo3/pull/1539)
- Automatically set `-undefined` and `dynamic_lookup` linker arguments on macOS with the `extension-module` feature. [#1539](https://github.com/PyO3/pyo3/pull/1539)
- Deprecate `#[pyproto]` methods which are easier to implement as `#[pymethods]`: [#1560](https://github.com/PyO3/pyo3/pull/1560)
- `PyBasicProtocol::__bytes__` and `PyBasicProtocol::__format__`
- `PyContextProtocol::__enter__` and `PyContextProtocol::__exit__`
@ -42,56 +80,76 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
- `PyMappingProtocol::__reversed__`
- `PyNumberProtocol::__complex__` and `PyNumberProtocol::__round__`
- `PyAsyncProtocol::__aenter__` and `PyAsyncProtocol::__aexit__`
- Deprecate `#[name = "..."]` attributes in favor of `#[pyo3(name = "...")]`. [#1567](https://github.com/PyO3/pyo3/pull/1567)
- Deprecate several attributes in favor of the new `#[pyo3(...)]` options:
- `#[name = "..."]`, replaced by `#[pyo3(name = "...")]` [#1567](https://github.com/PyO3/pyo3/pull/1567)
- `#[pyfn(m, "name")]`, replaced by `#[pyfn(m)] #[pyo3(name = "...")]`. [#1610](https://github.com/PyO3/pyo3/pull/1610)
- `#[pymodule(name)]`, replaced by `#[pymodule] #[pyo3(name = "...")]` [#1650](https://github.com/PyO3/pyo3/pull/1650)
- `#[text_signature = "..."]`, replaced by `#[pyo3(text_signature = "...")]`. [#1658](https://github.com/PyO3/pyo3/pull/1658)
- Reduce LLVM line counts to improve compilation times. [#1604](https://github.com/PyO3/pyo3/pull/1604)
- Deprecate string-literal second argument to `#[pyfn(m, "name")]`. [#1610](https://github.com/PyO3/pyo3/pull/1610)
- No longer call `PyEval_InitThreads()` in `#[pymodule]` init code. [#1630](https://github.com/PyO3/pyo3/pull/1630)
- No longer call `PyEval_InitThreads` in `#[pymodule]` init code. [#1630](https://github.com/PyO3/pyo3/pull/1630)
- Use `METH_FASTCALL` argument passing convention, when possible, to improve `#[pyfunction]` and method performance.
[#1619](https://github.com/PyO3/pyo3/pull/1619), [#1660](https://github.com/PyO3/pyo3/pull/1660)
- Filter sysconfigdata candidates by architecture when cross-compiling. [#1626](https://github.com/PyO3/pyo3/pull/1626)
### Removed
- Remove deprecated exception names `BaseException` etc. [#1426](https://github.com/PyO3/pyo3/pull/1426)
- Remove deprecated redundant methods `Python::is_instance`, `Python::is_subclass`, `Python::release`, `Python::xdecref`, and `Py::from_owned_ptr_or_panic`. [#1426](https://github.com/PyO3/pyo3/pull/1426)
- Remove many ffi definitions which never existed in the Python C-API:
- Remove deprecated methods `Python::is_instance`, `Python::is_subclass`, `Python::release`, `Python::xdecref`, and `Py::from_owned_ptr_or_panic`. [#1426](https://github.com/PyO3/pyo3/pull/1426)
- Remove many FFI definitions which never existed in the Python C-API:
- (previously deprecated) `PyGetSetDef_INIT`, `PyGetSetDef_DICT`, `PyCoro_Check`, `PyCoroWrapper_Check`, and `PyAsyncGen_Check` [#1426](https://github.com/PyO3/pyo3/pull/1426)
- `PyMethodDef_INIT` [#1426](https://github.com/PyO3/pyo3/pull/1426)
- `PyTypeObject_INIT` [#1429](https://github.com/PyO3/pyo3/pull/1429)
- `PyObject_Check`, `PySuper_Check`, and `FreeFunc` [#1438](https://github.com/PyO3/pyo3/pull/1438)
- `PyModuleDef_INIT` [#1630](https://github.com/PyO3/pyo3/pull/1630)
- Remove pyclass implementation details from `PyTypeInfo`:
- `Type`, `DESCRIPTION`, and `FLAGS` [#1456](https://github.com/PyO3/pyo3/pull/1456)
- `BaseType`, `BaseLayout`, `Layout`, `Initializer` [#1596](https://github.com/PyO3/pyo3/pull/1596)
- `PyModuleDef_INIT` [#1630](https://github.com/PyO3/pyo3/pull/1630)
- Remove `__doc__` from module's `__all__`. [#1509](https://github.com/PyO3/pyo3/pull/1509)
- Remove `PYO3_CROSS_INCLUDE_DIR` environment variable and the associated C header parsing functionality. [#1521](https://github.com/PyO3/pyo3/pull/1521)
- Remove `raw_pycfunction!` macro. [#1619](https://github.com/PyO3/pyo3/pull/1619)
- Remove `PyClassAlloc` trait. [#1657](https://github.com/PyO3/pyo3/pull/1657)
- Remove `PyList::get_parked_item`. [#1664](https://github.com/PyO3/pyo3/pull/1664)
### Fixed
- Remove FFI definition `PyCFunction_ClearFreeList` for Python 3.9 and later. [#1425](https://github.com/PyO3/pyo3/pull/1425)
- `PYO3_CROSS_LIB_DIR` enviroment variable no long required when compiling for x86-64 Python from macOS arm64 and reverse. [#1428](https://github.com/PyO3/pyo3/pull/1428)
- Fix FFI definition `_PyEval_RequestCodeExtraIndex` which took an argument of the wrong type. [#1429](https://github.com/PyO3/pyo3/pull/1429)
- Fix FFI definition `_PyEval_RequestCodeExtraIndex`, which took an argument of the wrong type. [#1429](https://github.com/PyO3/pyo3/pull/1429)
- Fix FFI definition `PyIndex_Check` missing with the `abi3` feature. [#1436](https://github.com/PyO3/pyo3/pull/1436)
- Fix incorrect `TypeError` raised when keyword-only argument passed along with a positional argument in `*args`. [#1440](https://github.com/PyO3/pyo3/pull/1440)
- Fix inability to use a named lifetime for `&PyTuple` of `*args` in `#[pyfunction]`. [#1440](https://github.com/PyO3/pyo3/pull/1440)
- Fix inability to add `#[text_signature]` to some `#[pyproto]` methods. [#1483](https://github.com/PyO3/pyo3/pull/1483)
- Fix use of Python argument for #[pymethods] inside macro expansions. [#1505](https://github.com/PyO3/pyo3/pull/1505)
- Always use cross-compiling configuration if any of the environment variables are set. [#1514](https://github.com/PyO3/pyo3/pull/1514)
- Fix use of Python argument for `#[pymethods]` inside macro expansions. [#1505](https://github.com/PyO3/pyo3/pull/1505)
- No longer include `__doc__` in `__all__` generated for `#[pymodule]`. [#1509](https://github.com/PyO3/pyo3/pull/1509)
- Always use cross-compiling configuration if any of the `PYO3_CROSS` family of environment variables are set. [#1514](https://github.com/PyO3/pyo3/pull/1514)
- Support `EnvironmentError`, `IOError`, and `WindowsError` on PyPy. [#1533](https://github.com/PyO3/pyo3/pull/1533)
- Fix unneccessary rebuilds when cycling between `cargo check` and `cargo clippy` in a Python virtualenv. [#1557](https://github.com/PyO3/pyo3/pull/1557)
- Fix segfault when dereferencing `ffi::PyDateTimeAPI` without the GIL. [#1563](https://github.com/PyO3/pyo3/pull/1563)
- Fix memory leak when converting to u128 and i128. [#1638](https://github.com/PyO3/pyo3/pull/1638)
- Fix memory leak in `FromPyObject` implementations for `u128` and `i128`. [#1638](https://github.com/PyO3/pyo3/pull/1638)
- Fix `#[pyclass(extends=PyDict)]` leaking the dict contents on drop. [#1657](https://github.com/PyO3/pyo3/pull/1657)
- Fix segfault when calling `PyList::get_item` with negative indices. [#1668](https://github.com/PyO3/pyo3/pull/1668)
- Fix FFI definitions of `PyEval_SetProfile`/`PyEval_SetTrace` to take `Option<Py_tracefunc>` parameters. [#1692](https://github.com/PyO3/pyo3/pull/1692)
- Fix `ToPyObject` impl for `HashSet` to accept non-default hashers. [#1702](https://github.com/PyO3/pyo3/pull/1702)
## [0.13.2] - 2021-02-12
### Packaging
- Lower minimum supported Rust version to 1.41. [#1421](https://github.com/PyO3/pyo3/pull/1421)
### Added
- Add unsafe API `with_embedded_python_interpreter` to initalize a Python interpreter, execute a closure, and finalize the interpreter. [#1355](https://github.com/PyO3/pyo3/pull/1355)
- Add unsafe API `with_embedded_python_interpreter` to initalize a Python interpreter, execute a closure, and finalize the interpreter. [#1355](https://github.com/PyO3/pyo3/pull/1355)
- Add `serde` feature which provides implementations of `Serialize` and `Deserialize` for `Py<T>`. [#1366](https://github.com/PyO3/pyo3/pull/1366)
- Add FFI definition `_PyCFunctionFastWithKeywords` on Python 3.7 and up. [#1384](https://github.com/PyO3/pyo3/pull/1384)
- Add `PyDateTime::new_with_fold()` method. [#1398](https://github.com/PyO3/pyo3/pull/1398)
- Add `size_hint` impls for `{PyDict,PyList,PySet,PyTuple}Iterator`s. [#1699](https://github.com/PyO3/pyo3/pull/1699)
### Changed
- `prepare_freethreaded_python` will no longer register an `atexit` handler to call `Py_Finalize`. This resolves a number of issues with incompatible C extensions causing crashes at finalization. [#1355](https://github.com/PyO3/pyo3/pull/1355)
- Mark `PyLayout::py_init`, `PyClassDict::clear_dict`, and `opt_to_pyobj` safe, as they do not perform any unsafe operations. [#1404](https://github.com/PyO3/pyo3/pull/1404)
### Fixed
- Fix support for using `r#raw_idents` as argument names in pyfunctions. [#1383](https://github.com/PyO3/pyo3/pull/1383)
- Fix typo in FFI definition for `PyFunction_GetCode` (was incorrectly `PyFunction_Code`). [#1387](https://github.com/PyO3/pyo3/pull/1387)
- Fix FFI definitions `PyMarshal_WriteObjectToString` and `PyMarshal_ReadObjectFromString` as available in limited API. [#1387](https://github.com/PyO3/pyo3/pull/1387)
@ -101,13 +159,16 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
- Fix build on mingw / MSYS2. [#1423](https://github.com/PyO3/pyo3/pull/1423)
## [0.13.1] - 2021-01-10
### Added
- Add support for `#[pyclass(dict)]` and `#[pyclass(weakref)]` with the `abi3` feature on Python 3.9 and up. [#1342](https://github.com/PyO3/pyo3/pull/1342)
- Add FFI definitions `PyOS_BeforeFork`, `PyOS_AfterFork_Parent`, `PyOS_AfterFork_Child` for Python 3.7 and up. [#1348](https://github.com/PyO3/pyo3/pull/1348)
- Add an `auto-initialize` feature to control whether PyO3 should automatically initialize an embedded Python interpreter. For compatibility this feature is enabled by default in PyO3 0.13.1, but is planned to become opt-in from PyO3 0.14.0. [#1347](https://github.com/PyO3/pyo3/pull/1347)
- Add support for cross-compiling to Windows without needing `PYO3_CROSS_INCLUDE_DIR`. [#1350](https://github.com/PyO3/pyo3/pull/1350)
### Deprecated
- Deprecate FFI definitions `PyEval_CallObjectWithKeywords`, `PyEval_CallObject`, `PyEval_CallFunction`, `PyEval_CallMethod` when building for Python 3.9. [#1338](https://github.com/PyO3/pyo3/pull/1338)
- Deprecate FFI definitions `PyGetSetDef_DICT` and `PyGetSetDef_INIT` which have never been in the Python API. [#1341](https://github.com/PyO3/pyo3/pull/1341)
- Deprecate FFI definitions `PyGen_NeedsFinalizing`, `PyImport_Cleanup` (removed in 3.9), and `PyOS_InitInterrupts` (3.10). [#1348](https://github.com/PyO3/pyo3/pull/1348)
@ -116,11 +177,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
- Deprecate FFI definitions for `PyUnicode_FromUnicode`, `PyUnicode_AsUnicode` and `PyUnicode_AsUnicodeAndSize`, which will be removed from 3.12 and up due to [PEP 613](https://www.python.org/dev/peps/pep-0623/). [#1370](https://github.com/PyO3/pyo3/pull/1370)
### Removed
- Remove FFI definition `PyFrame_ClearFreeList` when building for Python 3.9. [#1341](https://github.com/PyO3/pyo3/pull/1341)
- Remove FFI definition `_PyDict_Contains` when building for Python 3.10. [#1341](https://github.com/PyO3/pyo3/pull/1341)
- Remove FFI definitions `PyGen_NeedsFinalizing` and `PyImport_Cleanup` (for 3.9 and up), and `PyOS_InitInterrupts` (3.10). [#1348](https://github.com/PyO3/pyo3/pull/1348)
### Fixed
- Stop including `Py_TRACE_REFS` config setting automatically if `Py_DEBUG` is set on Python 3.8 and up. [#1334](https://github.com/PyO3/pyo3/pull/1334)
- Remove `#[deny(warnings)]` attribute (and instead refuse warnings only in CI). [#1340](https://github.com/PyO3/pyo3/pull/1340)
- Fix deprecation warning for missing `__module__` with `#[pyclass]`. [#1343](https://github.com/PyO3/pyo3/pull/1343)
@ -128,7 +191,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
- Fix missing `Py_INCREF` on heap type objects on Python versions before 3.8. [#1365](https://github.com/PyO3/pyo3/pull/1365)
## [0.13.0] - 2020-12-22
### Packaging
- Drop support for Python 3.5 (as it is now end-of-life). [#1250](https://github.com/PyO3/pyo3/pull/1250)
- Bump minimum supported Rust version to 1.45. [#1272](https://github.com/PyO3/pyo3/pull/1272)
- Bump indoc dependency to 1.0. [#1272](https://github.com/PyO3/pyo3/pull/1272)
@ -136,6 +201,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
- Rename internal crates `pyo3cls` and `pyo3-derive-backend` to `pyo3-macros` and `pyo3-macros-backend` respectively. [#1317](https://github.com/PyO3/pyo3/pull/1317)
### Added
- Add support for building for CPython limited API. Opting-in to the limited API enables a single extension wheel built with PyO3 to be installable on multiple Python versions. This required a few minor changes to runtime behaviour of of PyO3 `#[pyclass]` types. See the migration guide for full details. [#1152](https://github.com/PyO3/pyo3/pull/1152)
- Add feature flags `abi3-py36`, `abi3-py37`, `abi3-py38` etc. to set the minimum Python version when using the limited API. [#1263](https://github.com/PyO3/pyo3/pull/1263)
- Add argument names to `TypeError` messages generated by pymethod wrappers. [#1212](https://github.com/PyO3/pyo3/pull/1212)
@ -150,6 +216,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
- Add conversions for tuples of length 10, 11, and 12. [#1454](https://github.com/PyO3/pyo3/pull/1454)
### Changed
- Change return type of `PyType::name()` from `Cow<str>` to `PyResult<&str>`. [#1152](https://github.com/PyO3/pyo3/pull/1152)
- `#[pyclass(subclass)]` is now required for subclassing from Rust (was previously just required for subclassing from Python). [#1152](https://github.com/PyO3/pyo3/pull/1152)
- Change `PyIterator` to be consistent with other native types: it is now used as `&PyIterator` instead of `PyIterator<'a>`. [#1176](https://github.com/PyO3/pyo3/pull/1176)
@ -161,13 +228,16 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
- Require double-quotes for pyclass name argument e.g `#[pyclass(name = "MyClass")]`. [#1303](https://github.com/PyO3/pyo3/pull/1303)
### Deprecated
- Deprecate `Python::is_instance`, `Python::is_subclass`, `Python::release`, and `Python::xdecref`. [#1292](https://github.com/PyO3/pyo3/pull/1292)
### Removed
- Remove deprecated ffi definitions `PyUnicode_AsUnicodeCopy`, `PyUnicode_GetMax`, `_Py_CheckRecursionLimit`, `PyObject_AsCharBuffer`, `PyObject_AsReadBuffer`, `PyObject_CheckReadBuffer` and `PyObject_AsWriteBuffer`, which will be removed in Python 3.10. [#1217](https://github.com/PyO3/pyo3/pull/1217)
- Remove unused `python3` feature. [#1235](https://github.com/PyO3/pyo3/pull/1235)
### Fixed
- Fix missing field in `PyCodeObject` struct (`co_posonlyargcount`) - caused invalid access to other fields in Python >3.7. [#1260](https://github.com/PyO3/pyo3/pull/1260)
- Fix building for `x86_64-unknown-linux-musl` target from `x86_64-unknown-linux-gnu` host. [#1267](https://github.com/PyO3/pyo3/pull/1267)
- Fix `#[text_signature]` interacting badly with rust `r#raw_identifiers`. [#1286](https://github.com/PyO3/pyo3/pull/1286)
@ -177,30 +247,41 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
- Fix using custom error type in pyclass `#[new]` methods. [#1319](https://github.com/PyO3/pyo3/pull/1319)
## [0.12.4] - 2020-11-28
### Fixed
- Fix reference count bug in implementation of `From<Py<T>>` for `PyObject`, a regression introduced in PyO3 0.12. [#1297](https://github.com/PyO3/pyo3/pull/1297)
## [0.12.3] - 2020-10-12
### Fixed
- Fix support for Rust versions 1.39 to 1.44, broken by an incorrect internal update to paste 1.0 which was done in PyO3 0.12.2. [#1234](https://github.com/PyO3/pyo3/pull/1234)
## [0.12.2] - 2020-10-12
### Added
- Add support for keyword-only arguments without default values in `#[pyfunction]`. [#1209](https://github.com/PyO3/pyo3/pull/1209)
- Add `Python::check_signals()` as a safe a wrapper for `PyErr_CheckSignals()`. [#1214](https://github.com/PyO3/pyo3/pull/1214)
### Fixed
- Fix invalid document for protocol methods. [#1169](https://github.com/PyO3/pyo3/pull/1169)
- Hide docs of PyO3 private implementation details in `pyo3::class::methods`. [#1169](https://github.com/PyO3/pyo3/pull/1169)
- Fix unnecessary rebuild on PATH changes when the python interpreter is provided by PYO3_PYTHON. [#1231](https://github.com/PyO3/pyo3/pull/1231)
## [0.12.1] - 2020-09-16
### Fixed
- Fix building for a 32-bit Python on 64-bit Windows with a 64-bit Rust toolchain. [#1179](https://github.com/PyO3/pyo3/pull/1179)
- Fix building on platforms where `c_char` is `u8`. [#1182](https://github.com/PyO3/pyo3/pull/1182)
## [0.12.0] - 2020-09-12
### Added
- Add FFI definitions `Py_FinalizeEx`, `PyOS_getsig`, and `PyOS_setsig`. [#1021](https://github.com/PyO3/pyo3/pull/1021)
- Add `PyString::to_str` for accessing `PyString` as `&str`. [#1023](https://github.com/PyO3/pyo3/pull/1023)
- Add `Python::with_gil` for executing a closure with the Python GIL. [#1037](https://github.com/PyO3/pyo3/pull/1037)
@ -216,6 +297,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
- Add native `PyCFunction` and `PyFunction` types. [#1163](https://github.com/PyO3/pyo3/pull/1163)
### Changed
- Rework exception types: [#1024](https://github.com/PyO3/pyo3/pull/1024) [#1115](https://github.com/PyO3/pyo3/pull/1115)
- Rename exception types from e.g. `RuntimeError` to `PyRuntimeError`. The old names continue to exist but are deprecated.
- Exception objects are now accessible as `&T` or `Py<T>`, just like other Python-native types.
@ -240,6 +322,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
- Change argument to `PyModule::add` to `impl IntoPy<PyObject>` (was `impl ToPyObject`). #[1124](https://github.com/PyO3/pyo3/pull/1124)
### Removed
- Remove many exception and `PyErr` APIs; see the "changed" section above. [#1024](https://github.com/PyO3/pyo3/pull/1024) [#1067](https://github.com/PyO3/pyo3/pull/1067) [#1115](https://github.com/PyO3/pyo3/pull/1115)
- Remove `PyString::to_string` (use new `PyString::to_str`). [#1023](https://github.com/PyO3/pyo3/pull/1023)
- Remove `PyString::as_bytes`. [#1023](https://github.com/PyO3/pyo3/pull/1023)
@ -249,6 +332,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
- Remove the `AsPyRef` trait. [#1098](https://github.com/PyO3/pyo3/pull/1098)
### Fixed
- Correct FFI definitions `Py_SetProgramName` and `Py_SetPythonHome` to take `*const` arguments (was `*mut`). [#1021](https://github.com/PyO3/pyo3/pull/1021)
- Fix `FromPyObject` for `num_bigint::BigInt` for Python objects with an `__index__` method. [#1027](https://github.com/PyO3/pyo3/pull/1027)
- Correct FFI definition `_PyLong_AsByteArray` to take `*mut c_uchar` argument (was `*const c_uchar`). [#1029](https://github.com/PyO3/pyo3/pull/1029)
@ -258,18 +342,23 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
- Fix many cases of lifetime elision in `#[pyproto]` implementations. [#1093](https://github.com/PyO3/pyo3/pull/1093)
- Fix detection of Python build configuration when cross-compiling. [#1095](https://github.com/PyO3/pyo3/pull/1095)
- Always link against libpython on android with the `extension-module` feature. [#1095](https://github.com/PyO3/pyo3/pull/1095)
- Fix the `+` operator not trying `__radd__` when both `__add__` and `__radd__` are defined in `PyNumberProtocol` (and similar for all other reversible operators). [#1107](https://github.com/PyO3/pyo3/pull/1107)
- Fix the `+` operator not trying `__radd__` when both `__add__` and `__radd__` are defined in `PyNumberProtocol` (and similar for all other reversible operators). [#1107](https://github.com/PyO3/pyo3/pull/1107)
- Fix building with Anaconda python. [#1175](https://github.com/PyO3/pyo3/pull/1175)
## [0.11.1] - 2020-06-30
### Added
- `#[pyclass(unsendable)]`. [#1009](https://github.com/PyO3/pyo3/pull/1009)
### Changed
- Update `parking_lot` dependency to `0.11`. [#1010](https://github.com/PyO3/pyo3/pull/1010)
## [0.11.0] - 2020-06-28
### Added
- Support stable versions of Rust (>=1.39). [#969](https://github.com/PyO3/pyo3/pull/969)
- Add FFI definition `PyObject_AsFileDescriptor`. [#938](https://github.com/PyO3/pyo3/pull/938)
- Add `PyByteArray::data`, `PyByteArray::as_bytes`, and `PyByteArray::as_bytes_mut`. [#967](https://github.com/PyO3/pyo3/pull/967)
@ -278,6 +367,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
- Add `IterNextOutput` and `IterANextOutput` for returning from `__next__` / `__anext__`. [#997](https://github.com/PyO3/pyo3/pull/997)
### Changed
- Simplify internals of `#[pyo3(get)]` attribute. (Remove the hidden API `GetPropertyValue`.) [#934](https://github.com/PyO3/pyo3/pull/934)
- Call `Py_Finalize` at exit to flush buffers, etc. [#943](https://github.com/PyO3/pyo3/pull/943)
- Add type parameter to PyBuffer. #[951](https://github.com/PyO3/pyo3/pull/951)
@ -295,20 +385,26 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
- `#[pyproto]` methods can now skip annotating the return type if it is `()`. [#998](https://github.com/PyO3/pyo3/pull/998)
### Removed
- Remove `ManagedPyRef` (unused, and needs specialization) [#930](https://github.com/PyO3/pyo3/pull/930)
### Fixed
- Fix passing explicit `None` to `Option<T>` argument `#[pyfunction]` with a default value. [#936](https://github.com/PyO3/pyo3/pull/936)
- Fix `PyClass.__new__`'s not respecting subclasses when inherited by a Python class. [#990](https://github.com/PyO3/pyo3/pull/990)
- Fix returning `Option<T>` from `#[pyproto]` methods. [#996](https://github.com/PyO3/pyo3/pull/996)
- Fix accepting `PyRef<Self>` and `PyRefMut<Self>` to `#[getter]` and `#[setter]` methods. [#999](https://github.com/PyO3/pyo3/pull/999)
## [0.10.1] - 2020-05-14
### Fixed
- Fix deadlock in `Python::acquire_gil()` after dropping a `PyObject` or `Py<T>`. [#924](https://github.com/PyO3/pyo3/pull/924)
## [0.10.0] - 2020-05-13
### Added
- Add FFI definition `_PyDict_NewPresized`. [#849](https://github.com/PyO3/pyo3/pull/849)
- Implement `IntoPy<PyObject>` for `HashSet` and `BTreeSet`. [#864](https://github.com/PyO3/pyo3/pull/864)
- Add `PyAny::dir` method. [#886](https://github.com/PyO3/pyo3/pull/886)
@ -320,6 +416,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
- Add `#[classattr]` support for associated constants in `#[pymethods]`. [#914](https://github.com/PyO3/pyo3/pull/914)
### Changed
- Panics will now be raised as a Python `PanicException`. [#797](https://github.com/PyO3/pyo3/pull/797)
- Change `PyObject` and `Py<T>` reference counts to decrement immediately upon drop when the GIL is held. [#851](https://github.com/PyO3/pyo3/pull/851)
- Allow `PyIterProtocol` methods to use either `PyRef` or `PyRefMut` as the receiver type. [#856](https://github.com/PyO3/pyo3/pull/856)
@ -328,6 +425,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
- Remove need for `#![feature(specialization)]` in crates depending on PyO3. [#917](https://github.com/PyO3/pyo3/pull/917)
### Removed
- Remove `PyMethodsProtocol` trait. [#889](https://github.com/PyO3/pyo3/pull/889)
- Remove `num-traits` dependency. [#895](https://github.com/PyO3/pyo3/pull/895)
- Remove `ObjectProtocol` trait. [#911](https://github.com/PyO3/pyo3/pull/911)
@ -335,6 +433,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
- Remove all `*ProtocolImpl` traits. [#917](https://github.com/PyO3/pyo3/pull/917)
### Fixed
- Fix support for `__radd__` and other `__r*__` methods as implementations for Python mathematical operators. [#839](https://github.com/PyO3/pyo3/pull/839)
- Fix panics during garbage collection when traversing objects that were already mutably borrowed. [#855](https://github.com/PyO3/pyo3/pull/855)
- Prevent `&'static` references to Python objects as arguments to `#[pyfunction]` and `#[pymethods]`. [#869](https://github.com/PyO3/pyo3/pull/869)
@ -345,19 +444,26 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
- Fix segmentatation faults when a panic occurs during a call to `Python::allow_threads`. [#912](https://github.com/PyO3/pyo3/pull/912)
## [0.9.2] - 2020-04-09
### Added
- `FromPyObject` implementations for `HashSet` and `BTreeSet`. [#842](https://github.com/PyO3/pyo3/pull/842)
### Fixed
- Correctly detect 32bit architecture. [#830](https://github.com/PyO3/pyo3/pull/830)
## [0.9.1] - 2020-03-23
### Fixed
- Error messages for `#[pyclass]`. [#826](https://github.com/PyO3/pyo3/pull/826)
- `FromPyObject` implementation for `PySequence`. [#827](https://github.com/PyO3/pyo3/pull/827)
## [0.9.0] - 2020-03-19
### Added
- `PyCell`, which has RefCell-like features. [#770](https://github.com/PyO3/pyo3/pull/770)
- `PyClass`, `PyLayout`, `PyClassInitializer`. [#683](https://github.com/PyO3/pyo3/pull/683)
- Implemented `IntoIterator` for `PySet` and `PyFrozenSet`. [#716](https://github.com/PyO3/pyo3/pull/716)
@ -371,6 +477,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
- `ffi::{_PyBytes_Resize, _PyDict_Next, _PyDict_Contains, _PyDict_GetDictPtr}`. #[820](https://github.com/PyO3/pyo3/pull/820)
### Changed
- `#[new]` does not take `PyRawObject` and can return `Self`. [#683](https://github.com/PyO3/pyo3/pull/683)
- The blanket implementations for `FromPyObject` for `&T` and `&mut T` are no longer specializable. Implement `PyTryFrom` for your type to control the behavior of `FromPyObject::extract()` for your types. [#713](https://github.com/PyO3/pyo3/pull/713)
- The implementation for `IntoPy<U> for T` where `U: FromPy<T>` is no longer specializable. Control the behavior of this via the implementation of `FromPy`. [#713](https://github.com/PyO3/pyo3/pull/713)
@ -381,6 +488,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
- `PyAny` is now on the top level module and prelude. [#816](https://github.com/PyO3/pyo3/pull/816)
### Removed
- `PyRawObject`. [#683](https://github.com/PyO3/pyo3/pull/683)
- `PyNoArgsFunction`. [#741](https://github.com/PyO3/pyo3/pull/741)
- `initialize_type()`. To set the module name for a `#[pyclass]`, use the `module` argument to the macro. #[751](https://github.com/PyO3/pyo3/pull/751)
@ -390,6 +498,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
- `ObjectProtocol::get_base/get_mut_base`. [#770](https://github.com/PyO3/pyo3/pull/770)
### Fixed
- Fixed unsoundness of subclassing. [#683](https://github.com/PyO3/pyo3/pull/683).
- Clear error indicator when the exception is handled on the Rust side. [#719](https://github.com/PyO3/pyo3/pull/719)
- Usage of raw identifiers with `#[pyo3(set)]`. [#745](https://github.com/PyO3/pyo3/pull/745)
@ -400,67 +509,87 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
- Fix the case where `DESCRIPTION` is not null-terminated. #[822](https://github.com/PyO3/pyo3/pull/822)
## [0.8.5] - 2020-01-05
### Added
- Implemented `FromPyObject` for `HashMap` and `BTreeMap`
- Support for `#[name = "foo"]` attribute for `#[pyfunction]` and in `#[pymethods]`. [#692](https://github.com/PyO3/pyo3/pull/692)
## [0.8.4] - 2019-12-14
### Added
- Support for `#[text_signature]` attribute. [#675](https://github.com/PyO3/pyo3/pull/675)
## [0.8.3] - 2019-11-23
### Removed
- `#[init]` is removed. [#658](https://github.com/PyO3/pyo3/pull/658)
### Fixed
- Now all `&Py~` types have `!Send` bound. [#655](https://github.com/PyO3/pyo3/pull/655)
- Fix a compile error raised by the stabilization of `!` type. [#672](https://github.com/PyO3/pyo3/issues/672).
## [0.8.2] - 2019-10-27
### Added
- FFI compatibility for PEP 590 Vectorcall. [#641](https://github.com/PyO3/pyo3/pull/641)
### Fixed
- Fix PySequenceProtocol::set_item. [#624](https://github.com/PyO3/pyo3/pull/624)
- Fix a corner case of BigInt::FromPyObject. [#630](https://github.com/PyO3/pyo3/pull/630)
- Fix index errors in parameter conversion. [#631](https://github.com/PyO3/pyo3/pull/631)
- Fix handling of invalid utf-8 sequences in `PyString::as_bytes`. [#639](https://github.com/PyO3/pyo3/pull/639)
and `PyString::to_string_lossy` [#642](https://github.com/PyO3/pyo3/pull/642).
and `PyString::to_string_lossy` [#642](https://github.com/PyO3/pyo3/pull/642).
- Remove `__contains__` and `__iter__` from PyMappingProtocol. [#644](https://github.com/PyO3/pyo3/pull/644)
- Fix proc-macro definition of PySetAttrProtocol. [#645](https://github.com/PyO3/pyo3/pull/645)
## [0.8.1] - 2019-10-08
### Added
- Conversion between [num-bigint](https://github.com/rust-num/num-bigint) and Python int. [#608](https://github.com/PyO3/pyo3/pull/608)
### Fixed
- Make sure the right Python interpreter is used in OSX builds. [#604](https://github.com/PyO3/pyo3/pull/604)
- Patch specialization being broken by Rust 1.40. [#614](https://github.com/PyO3/pyo3/issues/614)
- Fix a segfault around PyErr. [#597](https://github.com/PyO3/pyo3/pull/597)
## [0.8.0] - 2019-09-16
### Added
- `module` argument to `pyclass` macro. [#499](https://github.com/PyO3/pyo3/pull/499)
- `py_run!` macro [#512](https://github.com/PyO3/pyo3/pull/512)
- Use existing fields and methods before calling custom __getattr__. [#505](https://github.com/PyO3/pyo3/pull/505)
- Use existing fields and methods before calling custom **getattr**. [#505](https://github.com/PyO3/pyo3/pull/505)
- `PyBytes` can now be indexed just like `Vec<u8>`
- Implement `IntoPy<PyObject>` for `PyRef` and `PyRefMut`.
### Changed
- Implementing the Using the `gc` parameter for `pyclass` (e.g. `#[pyclass(gc)]`) without implementing the `class::PyGCProtocol` trait is now a compile-time error. Failing to implement this trait could lead to segfaults. [#532](https://github.com/PyO3/pyo3/pull/532)
- `PyByteArray::data` has been replaced with `PyDataArray::to_vec` because returning a `&[u8]` is unsound. (See [this comment](https://github.com/PyO3/pyo3/issues/373#issuecomment-512332696) for a great write-up for why that was unsound)
- Replace `mashup` with `paste`.
- `GILPool` gained a `Python` marker to prevent it from being misused to release Python objects without the GIL held.
### Removed
- `IntoPyObject` was replaced with `IntoPy<PyObject>`
- `#[pyclass(subclass)]` is hidden a `unsound-subclass` feature because it's causing segmentation faults.
### Fixed
- More readable error message for generics in pyclass [#503](https://github.com/PyO3/pyo3/pull/503)
## [0.7.0] - 2019-05-26
### Added
- PyPy support by omerbenamram in [#393](https://github.com/PyO3/pyo3/pull/393)
- Have `PyModule` generate an index of its members (`__all__` list).
- Allow `slf: PyRef<T>` for pyclass(#419)
@ -468,25 +597,29 @@ and `PyString::to_string_lossy` [#642](https://github.com/PyO3/pyo3/pull/642).
- Add `marshal` module. [#460](https://github.com/PyO3/pyo3/pull/460)
### Changed
- `Python::run` returns `PyResult<()>` instead of `PyResult<&PyAny>`.
- Methods decorated with `#[getter]` and `#[setter]` can now omit wrapping the
result type in `PyResult` if they don't raise exceptions.
result type in `PyResult` if they don't raise exceptions.
### Fixed
- `type_object::PyTypeObject` has been marked unsafe because breaking the contract `type_object::PyTypeObject::init_type` can lead to UB.
- Fixed automatic derive of `PySequenceProtocol` implementation in [#423](https://github.com/PyO3/pyo3/pull/423).
- Capitalization & better wording to README.md.
- Docstrings of properties is now properly set using the doc of the `#[getter]` method.
- Fixed issues with `pymethods` crashing on doc comments containing double quotes.
- `PySet::new` and `PyFrozenSet::new` now return `PyResult<&Py[Frozen]Set>`; exceptions are raised if
the items are not hashable.
the items are not hashable.
- Fixed building using `venv` on Windows.
- `PyTuple::new` now returns `&PyTuple` instead of `Py<PyTuple>`.
- Fixed several issues with argument parsing; notable, the `*args` and `**kwargs`
tuple/dict now doesn't contain arguments that are otherwise assigned to parameters.
tuple/dict now doesn't contain arguments that are otherwise assigned to parameters.
## [0.6.0] - 2019-03-28
### Regressions
- Currently, [#341](https://github.com/PyO3/pyo3/issues/341) causes `cargo test` to fail with weird linking errors when the `extension-module` feature is activated. For now you can work around this by making the `extension-module` feature optional and running the tests with `cargo test --no-default-features`:
```toml
@ -499,6 +632,7 @@ default = ["extension-module"]
```
### Added
- Added a `wrap_pymodule!` macro similar to the existing `wrap_pyfunction!` macro. Only available on python 3
- Added support for cross compiling (e.g. to arm v7) by mtp401 in [#327](https://github.com/PyO3/pyo3/pull/327). See the "Cross Compiling" section in the "Building and Distribution" chapter of the guide for more details.
- The `PyRef` and `PyRefMut` types, which allow to differentiate between an instance of a rust struct on the rust heap and an instance that is embedded inside a python object. By kngwyu in [#335](https://github.com/PyO3/pyo3/pull/335)
@ -506,6 +640,7 @@ default = ["extension-module"]
- Added `ManagedPyRef`, which should eventually replace `ToBorrowedObject`.
### Changed
- Renamed `PyObjectRef` to `PyAny` in #388
- Renamed `add_function` to `add_wrapped` as it now also supports modules.
- Renamed `#[pymodinit]` to `#[pymodule]`
@ -528,29 +663,38 @@ default = ["extension-module"]
- Renamed the `typeob` module to `type_object`
### Removed
- `PyToken` was removed due to unsoundness (See [#94](https://github.com/PyO3/pyo3/issues/94)).
- Removed the unnecessary type parameter from `PyObjectAlloc`
- `NoArgs`. Just use an empty tuple
- `PyObjectWithGIL`. `PyNativeType` is sufficient now that PyToken is removed.
### Fixed
- A soudness hole where every instances of a `#[pyclass]` struct was considered to be part of a python object, even though you can create instances that are not part of the python heap. This was fixed through `PyRef` and `PyRefMut`.
- Fix kwargs support in [#328](https://github.com/PyO3/pyo3/pull/328).
- Add full support for `__dict__` in [#403](https://github.com/PyO3/pyo3/pull/403).
## [0.5.3] - 2019-01-04
### Fixed
- Fix memory leak in ArrayList by kngwyu [#316](https://github.com/PyO3/pyo3/pull/316)
## [0.5.2] - 2018-11-25
### Fixed
- Fix undeterministic segfaults when creating many objects by kngwyu in [#281](https://github.com/PyO3/pyo3/pull/281)
## [0.5.1] - 2018-11-24
Yanked
## [0.5.0] - 2018-11-11
### Added
- `#[pyclass]` objects can now be returned from rust functions
- `PyComplex` by kngwyu in [#226](https://github.com/PyO3/pyo3/pull/226)
- `PyDict::from_sequence()`, equivalent to `dict([(key, val), ...])`
@ -559,6 +703,7 @@ Yanked
- `PyObjectProtocol::get_type_ptr()` by ijl in [#242](https://github.com/PyO3/pyo3/pull/242)
### Changed
- Removes the types from the root module and the prelude. They now live in `pyo3::types` instead.
- All exceptions are consturcted with `py_err` instead of `new`, as they return `PyErr` and not `Self`.
- `as_mut` and friends take and `&mut self` instead of `&self`
@ -572,13 +717,15 @@ Yanked
- Starting to use `NonNull<*mut PyObject>` for Py and PyObject by ijl [#260](https://github.com/PyO3/pyo3/pull/260)
### Removed
- Removed most entries from the prelude. The new prelude is small and clear.
- Slowly removing specialization uses
- `PyString`, `PyUnicode`, and `PyBytes` no longer have a `data()` method
(replaced by `as_bytes()`) and `PyStringData` has been removed.
(replaced by `as_bytes()`) and `PyStringData` has been removed.
- The pyobject_extract macro
### Fixed
- Added an explanation that the GIL can temporarily be released even while holding a GILGuard.
- Lots of clippy errors
- Fix segfault on calling an unknown method on a PyObject
@ -586,38 +733,52 @@ Yanked
- Fixed a segfault with subclassing pyo3 create classes and using `__class__` by kngwyu [#263](https://github.com/PyO3/pyo3/pull/263)
## [0.4.1] - 2018-08-20
### Changed
- PyTryFrom's error is always to `PyDowncastError`
### Fixed
- Fixed compilation on nightly since `use_extern_macros` was stabilized
### Removed
- The pyobject_downcast macro
## [0.4.0] - 2018-07-30
### Changed
- Merged both examples into one
- Rustfmt all the things :heavy_check_mark:
- Switched to [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
### Removed
- Conversions from tuples to PyDict due to [rust-lang/rust#52050](https://github.com/rust-lang/rust/issues/52050)
## [0.3.2] - 2018-07-22
### Changed
- Replaced `concat_idents` with mashup
## [0.3.1] - 2018-07-18
### Fixed
- Fixed scoping bug in pyobject_native_type that would break rust-numpy
## [0.3.0] - 2018-07-18
### Added
- A few internal macros became part of the public api ([#155](https://github.com/PyO3/pyo3/pull/155), [#186](https://github.com/PyO3/pyo3/pull/186))
- Always clone in getters. This allows using the get-annotation on all Clone-Types
### Changed
- Upgraded to syn 0.14 which means much better error messages :tada:
- 128 bit integer support by [kngwyu](https://github.com/kngwyu) ([#137](https://github.com/PyO3/pyo3/pull/173))
- `proc_macro` has been stabilized on nightly ([rust-lang/rust#52081](https://github.com/rust-lang/rust/pull/52081)). This means that we can remove the `proc_macro` feature, but now we need the `use_extern_macros` from the 2018 edition instead.
@ -627,53 +788,73 @@ Yanked
- The guide is now properly versioned.
## [0.2.7] - 2018-05-18
### Fixed
- Fix nightly breakage with proc_macro_path
## [0.2.6] - 2018-04-03
### Fixed
- Fix compatibility with TryFrom trait #137
## [0.2.5] - 2018-02-21
### Added
- CPython 3.7 support
### Fixed
- Embedded CPython 3.7b1 crashes on initialization #110
- Generated extension functions are weakly typed #108
- call_method*() crashes when the method does not exist #113
- call_method\*() crashes when the method does not exist #113
- Allow importing exceptions from nested modules #116
## [0.2.4] - 2018-01-19
### Added
- Allow to get mutable ref from PyObject #106
- Drop `RefFromPyObject` trait
- Add Python::register_any() method
### Fixed
- Fix impl `FromPyObject` for `Py<T>`
- Mark method that work with raw pointers as unsafe #95
## [0.2.3] - 11-27-2017
### Changed
- Rustup to 1.23.0-nightly 2017-11-07
### Fixed
- Proper `c_char` usage #93
### Removed
- Remove use of now unneeded 'AsciiExt' trait
## [0.2.2] - 09-26-2017
### Changed
- Rustup to 1.22.0-nightly 2017-09-30
## [0.2.1] - 09-26-2017
### Fixed
- Fix rustc const_fn nightly breakage
## [0.2.0] - 08-12-2017
### Added
- Added inheritance support #15
- Added weakref support #56
- Added subclass support #64
@ -683,14 +864,19 @@ Yanked
- Introduce IntoPyDictPointer similar to IntoPyTuple #69
### Changed
- Allow to add gc support without implementing PyGCProtocol #57
- Refactor `PyErr` implementation. Drop `py` parameter from constructor.
## [0.1.0] - 07-23-2017
### Added
- Initial release
[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.13.2...HEAD
[unreleased]: https://github.com/pyo3/pyo3/compare/v0.14.1...HEAD
[0.14.1]: https://github.com/pyo3/pyo3/compare/v0.14.0...v0.14.1
[0.14.0]: https://github.com/pyo3/pyo3/compare/v0.13.2...v0.14.0
[0.13.2]: https://github.com/pyo3/pyo3/compare/v0.13.1...v0.13.2
[0.13.1]: https://github.com/pyo3/pyo3/compare/v0.13.0...v0.13.1
[0.13.0]: https://github.com/pyo3/pyo3/compare/v0.12.4...v0.13.0

16
Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "pyo3"
version = "0.13.2"
version = "0.14.1"
description = "Bindings to Python interpreter"
authors = ["PyO3 Project and Contributors <https://github.com/PyO3>"]
readme = "README.md"
@ -24,14 +24,16 @@ num-bigint = { version = "0.4", optional = true }
num-complex = { version = "0.4", optional = true }
# must stay at 0.1.x for Rust 1.41 compatibility
paste = { version = "0.1.18", optional = true }
pyo3-macros = { path = "pyo3-macros", version = "=0.13.2", optional = true }
pyo3-macros = { path = "pyo3-macros", version = "=0.14.1", optional = true }
unindent = { version = "0.1.4", optional = true }
hashbrown = { version = ">= 0.9, < 0.12", optional = true }
indexmap = { version = ">= 1.6, < 1.8", optional = true }
serde = {version = "1.0", optional = true}
[dev-dependencies]
assert_approx_eq = "1.1.0"
criterion = "0.3"
# O.3.5 uses the matches! macro, which isn't compatible with Rust 1.41
criterion = "=0.3.4"
trybuild = "1.0.23"
rustversion = "1.0"
proptest = { version = "0.10.1", default-features = false, features = ["std"] }
@ -40,7 +42,7 @@ pyo3 = { path = ".", default-features = false, features = ["macros", "auto-initi
serde_json = "1.0.61"
[build-dependencies]
pyo3-build-config = { path = "pyo3-build-config", version = "=0.14.0-alpha.0" }
pyo3-build-config = { path = "pyo3-build-config", version = "0.14.1" }
[features]
default = ["macros"]
@ -76,6 +78,10 @@ nightly = []
name = "bench_call"
harness = false
[[bench]]
name = "bench_err"
harness = false
[[bench]]
name = "bench_dict"
harness = false
@ -117,5 +123,5 @@ members = [
[package.metadata.docs.rs]
no-default-features = true
features = ["macros", "num-bigint", "num-complex", "hashbrown", "serde", "multiple-pymethods"]
features = ["macros", "num-bigint", "num-complex", "hashbrown", "serde", "multiple-pymethods", "indexmap"]
rustdoc-args = ["--cfg", "docsrs"]

17
Contributing.md
View File

@ -30,9 +30,22 @@ Don't be afraid if the solution is not clear to you! The core PyO3 contributors
PyO3 has a user guide (using mdbook) as well as the usual Rust API docs. The aim is for both of these to be detailed, easy to understand, and up-to-date. Pull requests are always welcome to fix typos, change wording, add examples, etc.
There are some specific areas of focus where help is currently needed for the documentation:
- Issues requesting documentation improvements are tracked with the [documentation](https://github.com/PyO3/pyo3/issues?q=is%3Aissue+is%3Aopen+label%3Adocumentation) label.
- Not all APIs had docs or examples when they were made. The goal is to have documentation on all PyO3 APIs ([#306](https://github.com/PyO3/pyo3/issues/306)). If you see an API lacking a doc, please write one and open a PR!
- Not all `unsafe` APIs had safety notes when they made. We'd like to ensure all `unsafe` APIs are carefully explained ([#698](https://github.com/PyO3/pyo3/issues/698)). If you see an `unsafe` function missing safety notes, please write some and open a PR!
#### Doctests
We use lots of code blocks in our docs. Run `cargo test --doc` when making changes to check that
the doctests still work, or `cargo test` to run all the tests including doctests. See
https://doc.rust-lang.org/rustdoc/documentation-tests.html for a guide on doctests.
#### Building the guide
You can preview the user guide by building it locally with `mdbook`.
First, [install `mdbook`](https://rust-lang.github.io/mdBook/cli/index.html). Then, run
`mdbook build -d ../gh-pages-build guide --open`.
### Help design the next PyO3
@ -92,4 +105,4 @@ At the moment there is no official organisation that accepts sponsorship on PyO3
In the meanwhile, some of our maintainers have personal Github sponsorship pages and would be grateful for your support:
* [davidhewitt](https://github.com/sponsors/davidhewitt)
- [davidhewitt](https://github.com/sponsors/davidhewitt)

126
README.md
View File

@ -1,35 +1,32 @@
# PyO3
[![actions status](https://github.com/PyO3/pyo3/workflows/CI/badge.svg)](https://github.com/PyO3/pyo3/actions)
[![benchmark](https://github.com/PyO3/pyo3/actions/workflows/bench.yml/badge.svg)](https://pyo3.rs/dev/bench/)
[![codecov](https://codecov.io/gh/PyO3/pyo3/branch/main/graph/badge.svg)](https://codecov.io/gh/PyO3/pyo3)
[![crates.io](https://meritbadge.herokuapp.com/pyo3)](https://crates.io/crates/pyo3)
[![crates.io](https://img.shields.io/crates/v/pyo3)](https://crates.io/crates/pyo3)
[![minimum rustc 1.41](https://img.shields.io/badge/rustc-1.41+-blue.svg)](https://rust-lang.github.io/rfcs/2495-min-rust-version.html)
[![dev chat](https://img.shields.io/gitter/room/nwjs/nw.js.svg)](https://gitter.im/PyO3/Lobby)
[![contributing notes](https://img.shields.io/badge/contribute-on%20github-Green)](https://github.com/PyO3/pyo3/blob/main/Contributing.md)
[Rust](http://www.rust-lang.org/) bindings for [Python](https://www.python.org/), including tools for creating native Python extension modules. Running and interacting with Python code from a Rust binary is also supported.
* User Guide: [stable](https://pyo3.rs) | [main](https://pyo3.rs/main)
- User Guide: [stable](https://pyo3.rs) | [main](https://pyo3.rs/main)
* API Documentation: [stable](https://docs.rs/pyo3/) | [main](https://pyo3.rs/main/doc)
- API Documentation: [stable](https://docs.rs/pyo3/) | [main](https://pyo3.rs/main/doc)
## Usage
PyO3 supports Python 3.6 and up. The minimum required Rust version is 1.41.
PyPy is also supported. Some minor features are unavailable on PyPy - please refer to the [pypy section in the guide](https://pyo3.rs/main/building_and_distribution/pypy.html) for more information.
PyPy is also supported. Some minor features are unavailable on PyPy - please refer to the [pypy section in the guide](https://pyo3.rs/latest/building_and_distribution/pypy.html) for more information.
You can either write a native Python module in Rust, or use Python from a Rust binary.
You can use PyO3 to write a native Python module in Rust, or to embed Python in a Rust binary. The following sections explain each of these in turn.
However, on some OSs, you need some additional packages. E.g. if you are on *Ubuntu 18.04*, please run
### Using Rust from Python
```bash
sudo apt install python3-dev python-dev
```
PyO3 can be used to generate a native Python module. The easiest way to try this out for the first time is to use [`maturin`](https://github.com/PyO3/maturin). `maturin` is a tool for building and publishing Rust-based Python packages with minimal configuration. The following steps set up some files for an example Python module, install `maturin`, and then show how build and import the Python module.
## Using Rust from Python
PyO3 can be used to generate a native Python module.
First, create a new folder (let's call it `string_sum`) containing the following two files:
**`Cargo.toml`**
@ -49,7 +46,7 @@ name = "string_sum"
crate-type = ["cdylib"]
[dependencies.pyo3]
version = "0.13.2"
version = "0.14.1"
features = ["extension-module"]
```
@ -57,7 +54,6 @@ features = ["extension-module"]
```rust
use pyo3::prelude::*;
use pyo3::wrap_pyfunction;
/// Formats the sum of two numbers as string.
#[pyfunction]
@ -65,27 +61,54 @@ fn sum_as_string(a: usize, b: usize) -> PyResult<String> {
Ok((a + b).to_string())
}
/// A Python module implemented in Rust.
/// A Python module implemented in Rust. The name of this function must match
/// the `lib.name` setting in the `Cargo.toml`, else Python will not be able to
/// import the module.
#[pymodule]
fn string_sum(py: Python, m: &PyModule) -> PyResult<()> {
fn string_sum(_py: Python, m: &PyModule) -> PyResult<()> {
m.add_function(wrap_pyfunction!(sum_as_string, m)?)?;
Ok(())
}
```
While developing, you can symlink (or copy) and rename the shared library from the target folder: On MacOS, rename `libstring_sum.dylib` to `string_sum.so`, on Windows `libstring_sum.dll` to `string_sum.pyd`, and on Linux `libstring_sum.so` to `string_sum.so`. Then open a Python shell in the same folder and you'll be able to `import string_sum`.
With those two files in place, now `maturin` needs to be installed. This can be done using Python's package manager `pip`. First, load up a new Python `virtualenv`, and install `maturin` into it:
To build, test and publish your crate as a Python module, you can use [maturin](https://github.com/PyO3/maturin) or [setuptools-rust](https://github.com/PyO3/setuptools-rust). You can find an example for setuptools-rust in [examples/word-count](https://github.com/PyO3/pyo3/tree/main/examples/word-count), while maturin should work on your crate without any configuration.
```bash
$ cd string_sum
$ python -m venv .env
$ source .env/bin/activate
$ pip install maturin
```
## Using Python from Rust
Now build and execute the module:
If you want your Rust application to create a Python interpreter internally and
use it to run Python code, add `pyo3` to your `Cargo.toml` like this:
```bash
$ maturin develop
# lots of progress output as maturin runs the compilation...
$ python
>>> import string_sum
>>> string_sum.sum_as_string(5, 20)
'25'
```
As well as with `maturin`, it is possible to build using [`setuptools-rust`](https://github.com/PyO3/setuptools-rust) or [manually](https://pyo3.rs/latest/building_and_distribution.html#manual-builds). Both offer more flexibility than `maturin` but require further configuration.
### Using Python from Rust
To embed Python into a Rust binary, you need to ensure that your Python installation contains a shared library. The following steps demonstrate how to ensure this (for Ubuntu), and then give some example code which runs an embedded Python interpreter.
To install the Python shared library, if you are on Ubuntu, you can run:
```bash
sudo apt install python3-dev
```
Start a new project with `cargo new`. Next, add `pyo3` to your `Cargo.toml` like this:
```toml
[dependencies.pyo3]
version = "0.13.2"
version = "0.14.1"
features = ["auto-initialize"]
```
@ -116,43 +139,46 @@ fn main_(py: Python) -> PyResult<()> {
}
```
Our guide has [a section](https://pyo3.rs/main/python_from_rust.html) with lots of examples
The guide has [a section](https://pyo3.rs/latest/python_from_rust.html) with lots of examples
about this topic.
## Tools and libraries
* [maturin](https://github.com/PyO3/maturin) _Zero configuration build tool for Rust-made Python extensions_.
* [setuptools-rust](https://github.com/PyO3/setuptools-rust) _Setuptools plugin for Rust support_.
* [pyo3-built](https://github.com/PyO3/pyo3-built) _Simple macro to expose metadata obtained with the [`built`](https://crates.io/crates/built) crate as a [`PyDict`](https://docs.rs/pyo3/0.12.0/pyo3/types/struct.PyDict.html)_
* [rust-numpy](https://github.com/PyO3/rust-numpy) _Rust binding of NumPy C-API_
* [dict-derive](https://github.com/gperinazzo/dict-derive) _Derive FromPyObject to automatically transform Python dicts into Rust structs_
* [pyo3-log](https://github.com/vorner/pyo3-log) _Bridge from Rust to Python logging_
* [pythonize](https://github.com/davidhewitt/pythonize) _Serde serializer for converting Rust objects to JSON-compatible Python objects_
* [pyo3-asyncio](https://github.com/awestlake87/pyo3-asyncio) Utilities for working with Python's Asyncio library and async functions
- [maturin](https://github.com/PyO3/maturin) _Zero configuration build tool for Rust-made Python extensions_.
- [setuptools-rust](https://github.com/PyO3/setuptools-rust) _Setuptools plugin for Rust support_.
- [pyo3-built](https://github.com/PyO3/pyo3-built) _Simple macro to expose metadata obtained with the [`built`](https://crates.io/crates/built) crate as a [`PyDict`](https://docs.rs/pyo3/0.12.0/pyo3/types/struct.PyDict.html)_
- [rust-numpy](https://github.com/PyO3/rust-numpy) _Rust binding of NumPy C-API_
- [dict-derive](https://github.com/gperinazzo/dict-derive) _Derive FromPyObject to automatically transform Python dicts into Rust structs_
- [pyo3-log](https://github.com/vorner/pyo3-log) _Bridge from Rust to Python logging_
- [pythonize](https://github.com/davidhewitt/pythonize) _Serde serializer for converting Rust objects to JSON-compatible Python objects_
- [pyo3-asyncio](https://github.com/awestlake87/pyo3-asyncio) Utilities for working with Python's Asyncio library and async functions
## Examples
* [hyperjson](https://github.com/mre/hyperjson) _A hyper-fast Python module for reading/writing JSON data using Rust's serde-json_
* [html-py-ever](https://github.com/PyO3/setuptools-rust/tree/main/examples/html-py-ever) _Using [html5ever](https://github.com/servo/html5ever) through [kuchiki](https://github.com/kuchiki-rs/kuchiki) to speed up html parsing and css-selecting._
* [point-process](https://github.com/ManifoldFR/point-process-rust/tree/master/pylib) _High level API for pointprocesses as a Python library_
* [autopy](https://github.com/autopilot-rs/autopy) _A simple, cross-platform GUI automation library for Python and Rust._
* Contains an example of building wheels on TravisCI and appveyor using [cibuildwheel](https://github.com/joerick/cibuildwheel)
* [orjson](https://github.com/ijl/orjson) _Fast Python JSON library_
* [inline-python](https://github.com/dronesforwork/inline-python) _Inline Python code directly in your Rust code_
* [Rogue-Gym](https://github.com/kngwyu/rogue-gym) _Customizable rogue-like game for AI experiments_
* Contains an example of building wheels on Azure Pipelines
* [fastuuid](https://github.com/thedrow/fastuuid/) _Python bindings to Rust's UUID library_
* [wasmer-python](https://github.com/wasmerio/wasmer-python) _Python library to run WebAssembly binaries_
* [mocpy](https://github.com/cds-astro/mocpy) _Astronomical Python library offering data structures for describing any arbitrary coverage regions on the unit sphere_
* [tokenizers](https://github.com/huggingface/tokenizers/tree/master/bindings/python) _Python bindings to the Hugging Face tokenizers (NLP) written in Rust_
* [pyre](https://github.com/Project-Dream-Weaver/Pyre) _Fast Python HTTP server written in Rust_
* [jsonschema-rs](https://github.com/Stranger6667/jsonschema-rs/tree/master/bindings/python) _Fast JSON Schema validation library_
* [css-inline](https://github.com/Stranger6667/css-inline/tree/master/bindings/python) _CSS inlining for Python implemented in Rust_
* [cryptography](https://github.com/pyca/cryptography/tree/main/src/rust) _Python cryptography library with some functionality in Rust_
* [polaroid](https://github.com/daggy1234/polaroid) _Hyper Fast and safe image manipulation library for Python written in Rust_
- [hyperjson](https://github.com/mre/hyperjson) _A hyper-fast Python module for reading/writing JSON data using Rust's serde-json_
- [html-py-ever](https://github.com/PyO3/setuptools-rust/tree/main/examples/html-py-ever) _Using [html5ever](https://github.com/servo/html5ever) through [kuchiki](https://github.com/kuchiki-rs/kuchiki) to speed up html parsing and css-selecting._
- [point-process](https://github.com/ManifoldFR/point-process-rust/tree/master/pylib) _High level API for pointprocesses as a Python library_
- [autopy](https://github.com/autopilot-rs/autopy) _A simple, cross-platform GUI automation library for Python and Rust._
- Contains an example of building wheels on TravisCI and appveyor using [cibuildwheel](https://github.com/joerick/cibuildwheel)
- [orjson](https://github.com/ijl/orjson) _Fast Python JSON library_
- [inline-python](https://github.com/dronesforwork/inline-python) _Inline Python code directly in your Rust code_
- [Rogue-Gym](https://github.com/kngwyu/rogue-gym) _Customizable rogue-like game for AI experiments_
- Contains an example of building wheels on Azure Pipelines
- [fastuuid](https://github.com/thedrow/fastuuid/) _Python bindings to Rust's UUID library_
- [wasmer-python](https://github.com/wasmerio/wasmer-python) _Python library to run WebAssembly binaries_
- [mocpy](https://github.com/cds-astro/mocpy) _Astronomical Python library offering data structures for describing any arbitrary coverage regions on the unit sphere_
- [tokenizers](https://github.com/huggingface/tokenizers/tree/master/bindings/python) _Python bindings to the Hugging Face tokenizers (NLP) written in Rust_
- [pyre](https://github.com/Project-Dream-Weaver/Pyre) _Fast Python HTTP server written in Rust_
- [jsonschema-rs](https://github.com/Stranger6667/jsonschema-rs/tree/master/bindings/python) _Fast JSON Schema validation library_
- [css-inline](https://github.com/Stranger6667/css-inline/tree/master/bindings/python) _CSS inlining for Python implemented in Rust_
- [cryptography](https://github.com/pyca/cryptography/tree/main/src/rust) _Python cryptography library with some functionality in Rust_
- [polaroid](https://github.com/daggy1234/polaroid) _Hyper Fast and safe image manipulation library for Python written in Rust_
- [ormsgpack](https://github.com/aviramha/ormsgpack) _Fast Python msgpack library_
## Contributing
Everyone is welcomed to contribute to PyO3! There are many ways to support the project, such as:
- help PyO3 users with issues on Github and Gitter
- improve documentation
- write features and bugfixes
@ -162,7 +188,7 @@ Our [contributing notes](https://github.com/PyO3/pyo3/blob/main/Contributing.md)
If you don't have time to contribute yourself but still wish to support the project's future success, some of our maintainers have Github sponsorship pages:
* [davidhewitt](https://github.com/sponsors/davidhewitt)
- [davidhewitt](https://github.com/sponsors/davidhewitt)
## License

8
benches/bench_call.rs
View File

@ -18,11 +18,11 @@ fn bench_call_0(b: &mut Bencher) {
"#
);
let foo = module.getattr("foo").unwrap();
let foo_module = module.getattr("foo").unwrap();
b.iter(|| {
for _ in 0..1000 {
foo.call0().unwrap();
foo_module.call0().unwrap();
}
});
})
@ -38,11 +38,11 @@ fn bench_call_method_0(b: &mut Bencher) {
"#
);
let foo = module.getattr("Foo").unwrap().call0().unwrap();
let foo_module = module.getattr("Foo").unwrap().call0().unwrap();
b.iter(|| {
for _ in 0..1000 {
foo.call_method0("foo").unwrap();
foo_module.call_method0("foo").unwrap();
}
});
})

8
benches/bench_dict.rs
View File

@ -18,6 +18,13 @@ fn iter_dict(b: &mut Bencher) {
});
}
fn dict_new(b: &mut Bencher) {
let gil = Python::acquire_gil();
let py = gil.python();
const LEN: usize = 50_000;
b.iter(|| (0..LEN as u64).map(|i| (i, i * 2)).into_py_dict(py));
}
fn dict_get_item(b: &mut Bencher) {
let gil = Python::acquire_gil();
let py = gil.python();
@ -58,6 +65,7 @@ fn extract_hashbrown_map(b: &mut Bencher) {
fn criterion_benchmark(c: &mut Criterion) {
c.bench_function("iter_dict", iter_dict);
c.bench_function("dict_new", dict_new);
c.bench_function("dict_get_item", dict_get_item);
c.bench_function("extract_hashmap", extract_hashmap);
c.bench_function("extract_btreemap", extract_btreemap);

24
benches/bench_err.rs Normal file
View File

@ -0,0 +1,24 @@
use criterion::{criterion_group, criterion_main, Bencher, Criterion};
use pyo3::{exceptions::PyValueError, prelude::*};
fn err_new_restore_and_fetch(b: &mut Bencher) {
Python::with_gil(|py| {
b.iter(|| {
PyValueError::new_err("some exception message").restore(py);
PyErr::fetch(py)
})
})
}
fn err_new_without_gil(b: &mut Bencher) {
b.iter(|| PyValueError::new_err("some exception message"))
}
fn criterion_benchmark(c: &mut Criterion) {
c.bench_function("err_new_restore_and_fetch", err_new_restore_and_fetch);
c.bench_function("err_new_without_gil", err_new_without_gil);
}
criterion_group!(benches, criterion_benchmark);
criterion_main!(benches);

8
benches/bench_list.rs
View File

@ -17,6 +17,13 @@ fn iter_list(b: &mut Bencher) {
});
}
fn list_new(b: &mut Bencher) {
let gil = Python::acquire_gil();
let py = gil.python();
const LEN: usize = 50_000;
b.iter(|| PyList::new(py, 0..LEN));
}
fn list_get_item(b: &mut Bencher) {
let gil = Python::acquire_gil();
let py = gil.python();
@ -32,6 +39,7 @@ fn list_get_item(b: &mut Bencher) {
fn criterion_benchmark(c: &mut Criterion) {
c.bench_function("iter_list", iter_list);
c.bench_function("list_new", list_new);
c.bench_function("list_get_item", list_get_item);
}

8
benches/bench_tuple.rs
View File

@ -17,6 +17,13 @@ fn iter_tuple(b: &mut Bencher) {
});
}
fn tuple_new(b: &mut Bencher) {
let gil = Python::acquire_gil();
let py = gil.python();
const LEN: usize = 50_000;
b.iter(|| PyTuple::new(py, 0..LEN));
}
fn tuple_get_item(b: &mut Bencher) {
let gil = Python::acquire_gil();
let py = gil.python();
@ -32,6 +39,7 @@ fn tuple_get_item(b: &mut Bencher) {
fn criterion_benchmark(c: &mut Criterion) {
c.bench_function("iter_tuple", iter_tuple);
c.bench_function("tuple_new", tuple_new);
c.bench_function("tuple_get_item", tuple_get_item);
}

68
build.rs
View File

@ -1,23 +1,14 @@
use std::{env, process::Command};
use pyo3_build_config::{InterpreterConfig, PythonImplementation, PythonVersion};
type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
use pyo3_build_config::{
bail, ensure,
errors::{Context, Result},
InterpreterConfig, PythonImplementation, PythonVersion,
};
/// Minimum Python version PyO3 supports.
const MINIMUM_SUPPORTED_VERSION: PythonVersion = PythonVersion { major: 3, minor: 6 };
// A simple macro for returning an error. Resembles anyhow::bail.
macro_rules! bail {
($msg: expr) => { return Err($msg.into()); };
($fmt: literal $($args: tt)+) => { return Err(format!($fmt $($args)+).into()); };
}
// A simple macro for checking a condition. Resembles anyhow::ensure.
macro_rules! ensure {
($condition:expr, $($args: tt)+) => { if !($condition) { bail!($($args)+) } };
}
fn ensure_python_version(interpreter_config: &InterpreterConfig) -> Result<()> {
ensure!(
interpreter_config.version >= MINIMUM_SUPPORTED_VERSION,
@ -87,9 +78,7 @@ fn get_rustc_link_lib(config: &InterpreterConfig) -> Result<String> {
match config.implementation {
PythonImplementation::CPython => match &config.ld_version {
Some(ld_version) => format!("python{}", ld_version),
None => {
return Err("failed to configure `ld_version` when compiling for unix".into())
}
None => bail!("failed to configure `ld_version` when compiling for unix"),
},
PythonImplementation::PyPy => format!("pypy{}-c", config.version.major),
}
@ -119,7 +108,7 @@ fn emit_cargo_configuration(interpreter_config: &InterpreterConfig) -> Result<()
match (is_extension_module, target_os.as_str()) {
(_, "windows") => {
// always link on windows, even with extension module
println!("{}", get_rustc_link_lib(&interpreter_config)?);
println!("{}", get_rustc_link_lib(interpreter_config)?);
// Set during cross-compiling.
if let Some(libdir) = &interpreter_config.libdir {
println!("cargo:rustc-link-search=native={}", libdir);
@ -137,7 +126,7 @@ fn emit_cargo_configuration(interpreter_config: &InterpreterConfig) -> Result<()
(false, _) | (_, "android") => {
// other systems, only link libs if not extension module
// android always link.
println!("{}", get_rustc_link_lib(&interpreter_config)?);
println!("{}", get_rustc_link_lib(interpreter_config)?);
if let Some(libdir) = &interpreter_config.libdir {
println!("cargo:rustc-link-search=native={}", libdir);
}
@ -157,7 +146,7 @@ fn emit_cargo_configuration(interpreter_config: &InterpreterConfig) -> Result<()
if env::var_os("CARGO_FEATURE_AUTO_INITIALIZE").is_some() {
if !interpreter_config.shared {
return Err(format!(
bail!(
"The `auto-initialize` feature is enabled, but your python installation only supports \
embedding the Python interpreter statically. If you are attempting to run tests, or a \
binary which is okay to link dynamically, install a Python distribution which ships \
@ -170,40 +159,69 @@ fn emit_cargo_configuration(interpreter_config: &InterpreterConfig) -> Result<()
https://pyo3.rs/v{pyo3_version}/\
building_and_distribution.html#embedding-python-in-rust",
pyo3_version = env::var("CARGO_PKG_VERSION").unwrap()
)
.into());
);
}
// TODO: PYO3_CI env is a hack to workaround CI with PyPy, where the `dev-dependencies`
// currently cause `auto-initialize` to be enabled in CI.
// Once cargo's `resolver = "2"` is stable (~ MSRV Rust 1.52), remove this.
if interpreter_config.is_pypy() && env::var_os("PYO3_CI").is_none() {
return Err("The `auto-initialize` feature is not supported with PyPy.".into());
bail!("The `auto-initialize` feature is not supported with PyPy.");
}
}
Ok(())
}
/// Generates the interpreter config suitable for the host / target / cross-compilation at hand.
///
/// The result is written to pyo3_build_config::PATH, which downstream scripts can read from
/// (including `pyo3-macros-backend` during macro expansion).
fn configure_pyo3() -> Result<()> {
let interpreter_config = pyo3_build_config::get();
let interpreter_config = pyo3_build_config::make_interpreter_config()?;
ensure_python_version(&interpreter_config)?;
ensure_target_architecture(&interpreter_config)?;
emit_cargo_configuration(&interpreter_config)?;
interpreter_config.to_writer(
&mut std::fs::File::create(pyo3_build_config::PATH).with_context(|| {
format!(
"failed to create config file at {}",
pyo3_build_config::PATH
)
})?,
)?;
interpreter_config.emit_pyo3_cfgs();
let rustc_minor_version = rustc_minor_version().unwrap_or(0);
// Enable use of const generics on Rust 1.51 and greater
if rustc_minor_version().unwrap_or(0) >= 51 {
if rustc_minor_version >= 51 {
println!("cargo:rustc-cfg=min_const_generics");
}
// Enable use of std::ptr::addr_of! on Rust 1.51 and greater
if rustc_minor_version >= 51 {
println!("cargo:rustc-cfg=addr_of");
}
Ok(())
}
fn main() {
// Print out error messages using display, to get nicer formatting.
if let Err(e) = configure_pyo3() {
use std::error::Error;
eprintln!("error: {}", e);
let mut source = e.source();
if source.is_some() {
eprintln!("caused by:");
let mut index = 0;
while let Some(some_source) = source {
eprintln!(" - {}: {}", index, some_source);
source = some_source.source();
index += 1;
}
}
std::process::exit(1)
}
}

3
codecov.yml
View File

@ -7,3 +7,6 @@ coverage:
target: auto
# Allow a tiny drop of overall project coverage in PR to reduce spurious failures.
threshold: 0.25%
ignore:
- tests/*.rs

81
examples/pyo3-benchmarks/src/lib.rs
View File

@ -1,39 +1,78 @@
use pyo3::prelude::*;
use pyo3::types::{PyDict, PyTuple};
use pyo3::wrap_pyfunction;
#[pyfunction]
fn none() {}
#[pyfunction(b = "\"bar\"", "*", c = "None")]
fn simple<'a>(a: i32, b: &'a str, c: Option<&'a PyDict>) -> (i32, &'a str, Option<&'a PyDict>) {
(a, b, c)
}
#[pyfunction(b = "\"bar\"", args = "*", c = "None")]
fn simple_args<'a>(
a: i32,
b: &'a str,
c: Option<&'a PyDict>,
args: &'a PyTuple,
) -> (i32, &'a str, &'a PyTuple, Option<&'a PyDict>) {
(a, b, args, c)
}
#[pyfunction(b = "\"bar\"", c = "None", kwargs = "**")]
fn simple_kwargs<'a>(
a: i32,
b: &'a str,
c: Option<&'a PyDict>,
kwargs: Option<&'a PyDict>,
) -> (i32, &'a str, Option<&'a PyDict>, Option<&'a PyDict>) {
(a, b, c, kwargs)
}
#[pyfunction(a, b = "\"bar\"", args = "*", c = "None", kwargs = "**")]
fn simple_args_kwargs<'a>(
a: i32,
b: &'a str,
args: &'a PyTuple,
c: Option<&'a PyDict>,
kwargs: Option<&'a PyDict>,
) -> (
i32,
&'a str,
&'a PyTuple,
Option<&'a PyDict>,
Option<&'a PyDict>,
) {
(a, b, args, c, kwargs)
}
#[pyfunction(args = "*", kwargs = "**")]
fn args_and_kwargs<'a>(
fn args_kwargs<'a>(
args: &'a PyTuple,
kwargs: Option<&'a PyDict>,
) -> (&'a PyTuple, Option<&'a PyDict>) {
(args, kwargs)
}
#[pyfunction(a, b = 2, args = "*", c = 4, kwargs = "**")]
fn mixed_args<'a>(
a: i32,
b: i32,
args: &'a PyTuple,
c: i32,
kwargs: Option<&'a PyDict>,
) -> (i32, i32, &'a PyTuple, i32, Option<&'a PyDict>) {
(a, b, args, c, kwargs)
}
#[pyclass]
struct EmptyClass {}
#[pyfunction]
fn no_args() {}
#[pyfunction(b = 2, "*", c = 4)]
fn simple_args(a: i32, b: i32, c: i32) -> (i32, i32, i32) {
(a, b, c)
#[pymethods]
impl EmptyClass {
#[new]
fn new() -> Self {
EmptyClass {}
}
}
#[pymodule]
fn _pyo3_benchmarks(_py: Python<'_>, m: &PyModule) -> PyResult<()> {
m.add_function(wrap_pyfunction!(args_and_kwargs, m)?)?;
m.add_function(wrap_pyfunction!(mixed_args, m)?)?;
m.add_function(wrap_pyfunction!(no_args, m)?)?;
m.add_function(wrap_pyfunction!(none, m)?)?;
m.add_function(wrap_pyfunction!(simple, m)?)?;
m.add_function(wrap_pyfunction!(simple_args, m)?)?;
m.add_function(wrap_pyfunction!(simple_kwargs, m)?)?;
m.add_function(wrap_pyfunction!(simple_args_kwargs, m)?)?;
m.add_function(wrap_pyfunction!(args_kwargs, m)?)?;
m.add_class::<EmptyClass>()?;
Ok(())
}

138
examples/pyo3-benchmarks/tests/test_benchmarks.py
View File

@ -1,61 +1,103 @@
import pyo3_benchmarks
def test_args_and_kwargs(benchmark):
benchmark(pyo3_benchmarks.args_and_kwargs, 1, 2, 3, a=4, foo=10)
def args_and_kwargs_py(*args, **kwargs):
return (args, kwargs)
def test_args_and_kwargs_py(benchmark):
rust = pyo3_benchmarks.args_and_kwargs(1, 2, 3, bar=4, foo=10)
py = args_and_kwargs_py(1, 2, 3, bar=4, foo=10)
assert rust == py
benchmark(args_and_kwargs_py, 1, 2, 3, bar=4, foo=10)
def test_mixed_args(benchmark):
benchmark(pyo3_benchmarks.mixed_args, 1, 2, 3, bar=4, foo=10)
def mixed_args_py(a, b=2, *args, c=4, **kwargs):
return (a, b, args, c, kwargs)
def test_mixed_args_py(benchmark):
rust = pyo3_benchmarks.mixed_args(1, 2, 3, bar=4, foo=10)
py = mixed_args_py(1, 2, 3, bar=4, foo=10)
assert rust == py
benchmark(mixed_args_py, 1, 2, 3, bar=4, foo=10)
def test_no_args(benchmark):
benchmark(pyo3_benchmarks.no_args)
def no_args_py():
def none_py():
return None
def test_no_args_py(benchmark):
rust = pyo3_benchmarks.no_args()
py = no_args_py()
def test_none_py(benchmark):
benchmark(none_py)
def test_none_rs(benchmark):
rust = pyo3_benchmarks.none()
py = none_py()
assert rust == py
benchmark(no_args_py)
benchmark(pyo3_benchmarks.none)
def test_simple_args(benchmark):
benchmark(pyo3_benchmarks.simple_args, 1, 3, c=5)
def simple_args_py(a, b=2, *, c=4):
def simple_py(a, b="bar", *, c=None):
return a, b, c
def test_simple_args_py(benchmark):
rust = pyo3_benchmarks.simple_args(1, 3, c=5)
py = simple_args_py(1, 3, c=5)
def test_simple_py(benchmark):
benchmark(simple_py, 1, "foo", c={1: 2})
def test_simple_rs(benchmark):
rust = pyo3_benchmarks.simple(1, "foo", c={1: 2})
py = simple_py(1, "foo", c={1: 2})
assert rust == py
benchmark(simple_args_py, 1, 3, c=5)
benchmark(pyo3_benchmarks.simple, 1, "foo", c={1: 2})
def simple_args_py(a, b="bar", *args, c=None):
return a, b, args, c
def test_simple_args_py(benchmark):
benchmark(simple_args_py, 1, "foo", 4, 5, 6, c={1: 2})
def test_simple_args_rs(benchmark):
rust = pyo3_benchmarks.simple_args(1, "foo", 4, 5, 6, c={1: 2})
py = simple_args_py(1, "foo", 4, 5, 6, c={1: 2})
assert rust == py
benchmark(pyo3_benchmarks.simple_args, 1, "foo", 4, 5, 6, c={1: 2})
def simple_kwargs_py(a, b="bar", c=None, **kwargs):
return a, b, c, kwargs
def test_simple_kwargs_py(benchmark):
benchmark(simple_kwargs_py, 1, "foo", c={1: 2}, bar=4, foo=10)
def test_simple_kwargs_rs(benchmark):
rust = pyo3_benchmarks.simple_kwargs(1, "foo", c={1: 2}, bar=4, foo=10)
py = simple_kwargs_py(1, "foo", c={1: 2}, bar=4, foo=10)
assert rust == py
benchmark(pyo3_benchmarks.simple_kwargs, 1, "foo", c={1: 2}, bar=4, foo=10)
def simple_args_kwargs_py(a, b="bar", *args, c=None, **kwargs):
return (a, b, args, c, kwargs)
def test_simple_args_kwargs_py(benchmark):
benchmark(simple_args_kwargs_py, 1, "foo", "baz", bar=4, foo=10)
def test_simple_args_kwargs_rs(benchmark):
rust = pyo3_benchmarks.simple_args_kwargs(1, "foo", "baz", bar=4, foo=10)
py = simple_args_kwargs_py(1, "foo", "baz", bar=4, foo=10)
assert rust == py
benchmark(pyo3_benchmarks.simple_args_kwargs, 1, "foo", "baz", bar=4, foo=10)
def args_kwargs_py(*args, **kwargs):
return (args, kwargs)
def test_args_kwargs_py(benchmark):
benchmark(args_kwargs_py, 1, "foo", {1: 2}, bar=4, foo=10)
def test_args_kwargs_rs(benchmark):
rust = pyo3_benchmarks.args_kwargs(1, "foo", {1: 2}, bar=4, foo=10)
py = args_kwargs_py(1, "foo", {1: 2}, bar=4, foo=10)
assert rust == py
benchmark(pyo3_benchmarks.args_kwargs, 1, "foo", {1: 2}, a=4, foo=10)
def test_empty_class_init(benchmark):
benchmark(pyo3_benchmarks.EmptyClass)
class EmptyClassPy:
pass
def test_empty_class_init_py(benchmark):
benchmark(EmptyClassPy)

2
examples/pyo3-benchmarks/tox.ini
View File

@ -7,4 +7,4 @@ description = Run the unit tests under {basepython}
deps = -rrequirements-dev.txt
commands =
python setup.py install
pytest {posargs}
pytest --benchmark-sort=name {posargs}

10
examples/pyo3-pytests/src/buf_and_str.rs
View File

@ -1,4 +1,5 @@
//! Objects related to PyBuffer and PyStr
use pyo3::buffer::PyBuffer;
use pyo3::prelude::*;
use pyo3::types::{PyBytes, PyString};
@ -14,12 +15,12 @@ impl BytesExtractor {
}
pub fn from_bytes(&mut self, bytes: &PyBytes) -> PyResult<usize> {
let byte_vec: Vec<u8> = bytes.extract().unwrap();
let byte_vec: Vec<u8> = bytes.extract()?;
Ok(byte_vec.len())
}
pub fn from_str(&mut self, string: &PyString) -> PyResult<usize> {
let rust_string: String = string.extract().unwrap();
let rust_string: String = string.extract()?;
Ok(rust_string.len())
}
@ -27,6 +28,11 @@ impl BytesExtractor {
let rust_string_lossy: String = string.to_string_lossy().to_string();
Ok(rust_string_lossy.len())
}
pub fn from_buffer(&mut self, buf: &PyAny) -> PyResult<usize> {
let buf = PyBuffer::<u8>::get(buf)?;
Ok(buf.item_count())
}
}
#[pymodule]

1
examples/pyo3-pytests/src/datetime.rs
View File

@ -3,7 +3,6 @@ use pyo3::types::{
PyDate, PyDateAccess, PyDateTime, PyDelta, PyDeltaAccess, PyTime, PyTimeAccess, PyTuple,
PyTzInfo,
};
use pyo3::wrap_pyfunction;
#[pyfunction]
fn make_date(py: Python, year: i32, month: u8, day: u8) -> PyResult<&PyDate> {

4
examples/pyo3-pytests/src/lib.rs
View File

@ -8,6 +8,7 @@ pub mod dict_iter;
pub mod misc;
pub mod objstore;
pub mod othermod;
pub mod path;
pub mod pyclass_iter;
pub mod subclassing;
@ -17,6 +18,7 @@ use dict_iter::*;
use misc::*;
use objstore::*;
use othermod::*;
use path::*;
use pyclass_iter::*;
use subclassing::*;
@ -28,6 +30,7 @@ fn pyo3_pytests(py: Python, m: &PyModule) -> PyResult<()> {
m.add_wrapped(wrap_pymodule!(misc))?;
m.add_wrapped(wrap_pymodule!(objstore))?;
m.add_wrapped(wrap_pymodule!(othermod))?;
m.add_wrapped(wrap_pymodule!(path))?;
m.add_wrapped(wrap_pymodule!(pyclass_iter))?;
m.add_wrapped(wrap_pymodule!(subclassing))?;
@ -42,6 +45,7 @@ fn pyo3_pytests(py: Python, m: &PyModule) -> PyResult<()> {
sys_modules.set_item("pyo3_pytests.misc", m.getattr("misc")?)?;
sys_modules.set_item("pyo3_pytests.objstore", m.getattr("objstore")?)?;
sys_modules.set_item("pyo3_pytests.othermod", m.getattr("othermod")?)?;
sys_modules.set_item("pyo3_pytests.path", m.getattr("path")?)?;
sys_modules.set_item("pyo3_pytests.pyclass_iter", m.getattr("pyclass_iter")?)?;
sys_modules.set_item("pyo3_pytests.subclassing", m.getattr("subclassing")?)?;

1
examples/pyo3-pytests/src/misc.rs
View File

@ -1,5 +1,4 @@
use pyo3::prelude::*;
use pyo3::wrap_pyfunction;
#[pyfunction]
fn issue_219() {

1
examples/pyo3-pytests/src/othermod.rs
View File

@ -3,7 +3,6 @@
//! The code below just tries to use the most important code generation paths
use pyo3::prelude::*;
use pyo3::wrap_pyfunction;
#[pyclass]
pub struct ModClass {

20
examples/pyo3-pytests/src/path.rs Normal file
View File

@ -0,0 +1,20 @@
use pyo3::prelude::*;
use std::path::{Path, PathBuf};
#[pyfunction]
fn make_path() -> PathBuf {
Path::new("/root").to_owned()
}
#[pyfunction]
fn take_pathbuf(path: PathBuf) -> PathBuf {
path
}
#[pymodule]
fn path(_py: Python<'_>, m: &PyModule) -> PyResult<()> {
m.add_function(wrap_pyfunction!(make_path, m)?)?;
m.add_function(wrap_pyfunction!(take_pathbuf, m)?)?;
Ok(())
}

58
examples/pyo3-pytests/tests/test_buf_and_str.py
View File

@ -1,51 +1,29 @@
import gc
import os
import platform
import psutil
import pytest
from pyo3_pytests.buf_and_str import BytesExtractor
PYPY = platform.python_implementation() == "PyPy"
@pytest.mark.skipif(
PYPY,
reason="PyPy has a segfault bug around this area."
"See https://github.com/PyO3/pyo3/issues/589 for detail.",
)
def test_pybuffer_doesnot_leak_memory():
N = 10000
def test_extract_bytes():
extractor = BytesExtractor()
process = psutil.Process(os.getpid())
message = b'\\(-"-;) A message written in bytes'
assert extractor.from_bytes(message) == len(message)
def memory_diff(f):
before = process.memory_info().rss
gc.collect() # Trigger Garbage collection
for _ in range(N):
f()
gc.collect() # Trigger Garbage collection
after = process.memory_info().rss
return after - before
message_b = b'\\(-"-;) Praying that memory leak would not happen..'
message_s = '\\(-"-;) Praying that memory leak would not happen..'
message_surrogate = '\\(-"-;) Praying that memory leak would not happen.. \ud800'
def test_extract_str():
extractor = BytesExtractor()
message = '\\(-"-;) A message written as a string'
assert extractor.from_str(message) == len(message)
def from_bytes():
extractor.from_bytes(message_b)
def from_str():
extractor.from_str(message_s)
def test_extract_str_lossy():
extractor = BytesExtractor()
message = '\\(-"-;) A message written with a trailing surrogate \ud800'
rust_surrogate_len = extractor.from_str_lossy("\ud800")
assert extractor.from_str_lossy(message) == len(message) - 1 + rust_surrogate_len
def from_str_lossy():
extractor.from_str_lossy(message_surrogate)
# Running the memory_diff to warm-up the garbage collector
memory_diff(from_bytes)
memory_diff(from_str)
memory_diff(from_str_lossy)
def test_extract_buffer():
extractor = BytesExtractor()
message = b'\\(-"-;) A message written in bytes'
assert extractor.from_buffer(message) == len(message)
assert memory_diff(from_bytes) == 0
assert memory_diff(from_str) == 0
assert memory_diff(from_str_lossy) == 0
arr = bytearray(b'\\(-"-;) A message written in bytes')
assert extractor.from_buffer(arr) == len(arr)

13
examples/pyo3-pytests/tests/test_objstore.py
View File

@ -3,22 +3,23 @@ import platform
import sys
import pytest
from pyo3_pytests.objstore import ObjStore
PYPY = platform.python_implementation() == "PyPy"
@pytest.mark.skipif(PYPY, reason="PyPy does not have sys.getrefcount")
def test_objstore_doesnot_leak_memory():
N = 10000
message = b'\\(-"-;) Praying that memory leak would not happen..'
before = sys.getrefcount(message)
# PyPy does not have sys.getrefcount, provide a no-op lambda and don't
# check refcount on PyPy
getrefcount = getattr(sys, "getrefcount", lambda obj: 0)
before = getrefcount(message)
store = ObjStore()
for _ in range(N):
store.push(message)
del store
gc.collect()
after = sys.getrefcount(message)
after = getrefcount(message)
assert after - before == 0

18
examples/pyo3-pytests/tests/test_path.py Normal file
View File

@ -0,0 +1,18 @@
import pathlib
import pyo3_pytests.path as rpath
def test_make_path():
p = rpath.make_path()
assert p == "/root"
def test_take_pathbuf():
p = "/root"
assert rpath.take_pathbuf(p) == p
def test_take_pathlib():
p = pathlib.Path("/root")
assert rpath.take_pathbuf(p) == str(p)

9
examples/pyo3-pytests/tests/test_subclassing.py
View File

@ -2,8 +2,6 @@ import platform
from pyo3_pytests.subclassing import Subclassable
PYPY = platform.python_implementation() == "PyPy"
class SomeSubClass(Subclassable):
def __str__(self):
@ -11,7 +9,6 @@ class SomeSubClass(Subclassable):
def test_subclassing():
if not PYPY:
a = SomeSubClass()
assert str(a) == "SomeSubclass"
assert type(a) is SomeSubClass
a = SomeSubClass()
assert str(a) == "SomeSubclass"
assert type(a) is SomeSubClass

1
examples/word-count/src/lib.rs
View File

@ -2,7 +2,6 @@
// https://github.com/tildeio/helix-website/blob/master/crates/word_count/src/lib.rs
use pyo3::prelude::*;
use pyo3::wrap_pyfunction;
use rayon::prelude::*;
/// Searches for the word, parallelized by rayon

2
guide/pyo3_version.py
View File

@ -6,7 +6,7 @@ It will replace:
- {{#PYO3_CRATE_VERSION}} with a relevant toml snippet (e.g. version = "0.13.2")
Tested against mdbook 0.4.9.
Tested against mdbook 0.4.10.
"""
import json

15
guide/src/building_and_distribution.md
View File

@ -35,9 +35,18 @@ pyo3 = { {{#PYO3_CRATE_VERSION}} }
On Linux/macOS you might have to change `LD_LIBRARY_PATH` to include libpython, while on windows you might need to set `LIB` to include `pythonxy.lib` (where x and y are major and minor version), which is normally either in the `libs` or `Lib` folder of a Python installation.
## Distribution
## Testing, building and distribution
There are two ways to distribute your module as a Python package: [setuptools-rust] and [maturin]. setuptools-rust needs several configuration files (`setup.py`, `MANIFEST.in`, `build-wheels.sh`, etc.). It allows (and sometimes requires) writing custom workflows in python. maturin has only few options and works without any additional configuration, instead it requires a rigid project structure and does not support some functionality of setuptools such as package data ([pyo3/maturin#258](https://github.com/PyO3/maturin/issues/258)), multiple extensions or running python scripts at build time.
There are two main ways to test, build and distribute your module as a Python package: [setuptools-rust] and [maturin]. setuptools-rust needs several configuration files (`setup.py`, `MANIFEST.in`, `build-wheels.sh`, etc.). It allows (and sometimes requires) writing custom workflows in python. maturin has only few options and works without any additional configuration, instead it requires a rigid project structure and does not support some functionality of setuptools such as package data ([pyo3/maturin#258](https://github.com/PyO3/maturin/issues/258)), multiple extensions or running python scripts at build time.
### Manual builds
You can also symlink (or copy) and rename the shared library from the `target` folder:
- on macOS, rename `libyour_module.dylib` to `your_module.so`.
- on Windows, rename `libyour_module.dll` to `your_module.pyd`.
- on Linux, rename `libyour_module.so` to `your_module.so`.
You can then open a Python shell in the same folder and you'll be able to run `import your_module`.
## `Py_LIMITED_API`/`abi3`
@ -77,7 +86,7 @@ As an advanced feature, you can build PyO3 wheel without calling Python interpre
Due to limitations in the Python API, there are a few `pyo3` features that do
not work when compiling for `abi3`. These are:
- `#[text_signature]` does not work on classes until Python 3.10 or greater.
- `#[pyo3(text_signature = "...")]` does not work on classes until Python 3.10 or greater.
- The `dict` and `weakref` options on classes are not supported until Python 3.9 or greater.
- The buffer API is not supported.
- Optimizations which rely on knowledge of the exact Python version compiled against.

72
guide/src/class.md
View File

@ -169,6 +169,23 @@ impl MyClass {
}
```
Alternatively, if your `new` method may fail you can return `PyResult<Self>`.
```rust
# use pyo3::prelude::*;
#[pyclass]
struct MyClass {
num: i32,
}
#[pymethods]
impl MyClass {
#[new]
fn new(num: i32) -> PyResult<Self> {
Ok(MyClass { num })
}
}
```
If no method marked with `#[new]` is declared, object instances can only be
created from Rust, but not from Python.
@ -349,7 +366,7 @@ struct MyClass {
}
```
The above would make the `num` property available for reading and writing from Python code as `self.num`.
The above would make the `num` field available for reading and writing as a `self.num` Python property. To expose the property with a different name to the field, specify this alongside the rest of the options, e.g. `#[pyo3(get, set, name = "custom_name")]`.
Properties can be readonly or writeonly by using just `#[pyo3(get)]` or `#[pyo3(set)]` respectively.
@ -720,8 +737,6 @@ struct MyClass {
debug: bool,
}
impl pyo3::pyclass::PyClassAlloc for MyClass {}
unsafe impl pyo3::PyTypeInfo for MyClass {
type AsRefTarget = PyCell<Self>;
@ -757,44 +772,51 @@ impl pyo3::class::impl_::PyClassImpl for MyClass {
type BaseType = PyAny;
type ThreadChecker = pyo3::class::impl_::ThreadCheckerStub<MyClass>;
fn for_each_method_def(visitor: &mut dyn FnMut(&pyo3::class::PyMethodDefType)) {
fn for_each_method_def(visitor: &mut dyn FnMut(&[pyo3::class::PyMethodDefType])) {
use pyo3::class::impl_::*;
let collector = PyClassImplCollector::<MyClass>::new();
collector.py_methods().iter()
.chain(collector.py_class_descriptors())
.chain(collector.object_protocol_methods())
.chain(collector.async_protocol_methods())
.chain(collector.context_protocol_methods())
.chain(collector.descr_protocol_methods())
.chain(collector.mapping_protocol_methods())
.chain(collector.number_protocol_methods())
.for_each(visitor)
visitor(collector.py_methods());
visitor(collector.py_class_descriptors());
visitor(collector.object_protocol_methods());
visitor(collector.async_protocol_methods());
visitor(collector.context_protocol_methods());
visitor(collector.descr_protocol_methods());
visitor(collector.mapping_protocol_methods());
visitor(collector.number_protocol_methods());
}
fn get_new() -> Option<pyo3::ffi::newfunc> {
use pyo3::class::impl_::*;
let collector = PyClassImplCollector::<Self>::new();
collector.new_impl()
}
fn get_alloc() -> Option<pyo3::ffi::allocfunc> {
use pyo3::class::impl_::*;
let collector = PyClassImplCollector::<Self>::new();
collector.alloc_impl()
}
fn get_free() -> Option<pyo3::ffi::freefunc> {
use pyo3::class::impl_::*;
let collector = PyClassImplCollector::<Self>::new();
collector.free_impl()
}
fn get_call() -> Option<pyo3::ffi::PyCFunctionWithKeywords> {
use pyo3::class::impl_::*;
let collector = PyClassImplCollector::<Self>::new();
collector.call_impl()
}
fn for_each_proto_slot(visitor: &mut dyn FnMut(&pyo3::ffi::PyType_Slot)) {
fn for_each_proto_slot(visitor: &mut dyn FnMut(&[pyo3::ffi::PyType_Slot])) {
// Implementation which uses dtolnay specialization to load all slots.
use pyo3::class::impl_::*;
let collector = PyClassImplCollector::<Self>::new();
collector.object_protocol_slots()
.iter()
.chain(collector.number_protocol_slots())
.chain(collector.iter_protocol_slots())
.chain(collector.gc_protocol_slots())
.chain(collector.descr_protocol_slots())
.chain(collector.mapping_protocol_slots())
.chain(collector.sequence_protocol_slots())
.chain(collector.async_protocol_slots())
.chain(collector.buffer_protocol_slots())
.for_each(visitor);
visitor(collector.object_protocol_slots());
visitor(collector.number_protocol_slots());
visitor(collector.iter_protocol_slots());
visitor(collector.gc_protocol_slots());
visitor(collector.descr_protocol_slots());
visitor(collector.mapping_protocol_slots());
visitor(collector.sequence_protocol_slots());
visitor(collector.async_protocol_slots());
visitor(collector.buffer_protocol_slots());
}
fn get_buffer() -> Option<&'static pyo3::class::impl_::PyBufferProcs> {

4
guide/src/conversions/tables.md
View File

@ -20,7 +20,7 @@ The table below contains the Python type and the corresponding function argument
| `float` | `f32`, `f64` | `&PyFloat` |
| `complex` | `num_complex::Complex`[^1] | `&PyComplex` |
| `list[T]` | `Vec<T>` | `&PyList` |
| `dict[K, V]` | `HashMap<K, V>`, `BTreeMap<K, V>`, `hashbrown::HashMap<K, V>`[^2] | `&PyDict` |
| `dict[K, V]` | `HashMap<K, V>`, `BTreeMap<K, V>`, `hashbrown::HashMap<K, V>`[^2], `indexmap::IndexMap<K, V>`[^3] | `&PyDict` |
| `tuple[T, U]` | `(T, U)`, `Vec<T>` | `&PyTuple` |
| `set[T]` | `HashSet<T>`, `BTreeSet<T>`, `hashbrown::HashSet<T>`[^2] | `&PySet` |
| `frozenset[T]` | `HashSet<T>`, `BTreeSet<T>`, `hashbrown::HashSet<T>`[^2] | `&PyFrozenSet` |
@ -94,3 +94,5 @@ Finally, the following Rust types are also able to convert to Python as return v
[^1]: Requires the `num-complex` optional feature.
[^2]: Requires the `hashbrown` optional feature.
[^3]: Requires the `indexmap` optional feature.

1
guide/src/ecosystem/logging.md
View File

@ -21,7 +21,6 @@ It's also possible to tweak its configuration (mostly to tune its performance).
```rust
use log::info;
use pyo3::prelude::*;
use pyo3::wrap_pyfunction;
#[pyfunction]
fn log_something() {

88
guide/src/faq.md
View File

@ -35,7 +35,7 @@ The Rust book suggests to [put integration tests inside a `tests/` directory](ht
For a PyO3 `extension-module` project where the `crate-type` is set to `"cdylib"` in your `Cargo.toml`,
the compiler won't be able to find your crate and will display errors such as `E0432` or `E0463`:
```
```text
error[E0432]: unresolved import `my_crate`
--> tests/test_my_crate.rs:1:5
|
@ -45,7 +45,7 @@ error[E0432]: unresolved import `my_crate`
The best solution is to make your crate types include both `rlib` and `cdylib`:
```
```toml
# Cargo.toml
[lib]
crate-type = ["cdylib", "rlib"]
@ -56,3 +56,87 @@ crate-type = ["cdylib", "rlib"]
This is because Ctrl-C raises a SIGINT signal, which is handled by the calling Python process by simply setting a flag to action upon later. This flag isn't checked while Rust code called from Python is executing, only once control returns to the Python interpreter.
You can give the Python interpreter a chance to process the signal properly by calling `Python::check_signals`. It's good practice to call this function regularly if you have a long-running Rust function so that your users can cancel it.
## `#[pyo3(get)]` clones my field!
You may have a nested struct similar to this:
```rust
# use pyo3::prelude::*;
#[pyclass]
#[derive(Clone)]
struct Inner { /* fields omitted */ }
#[pyclass]
struct Outer {
#[pyo3(get)]
inner: Inner,
}
#[pymethods]
impl Outer {
#[new]
fn __new__() -> Self {
Self { inner: Inner {} }
}
}
```
When Python code accesses `Outer`'s field, PyO3 will return a new object on every access (note that their addresses are different):
```python
outer = Outer()
a = outer.inner
b = outer.inner
assert a is b, f"a: {a}\nb: {b}"
```
```text
AssertionError: a: <builtins.Inner object at 0x00000238FFB9C7B0>
b: <builtins.Inner object at 0x00000238FFB9C830>
```
This can be especially confusing if the field is mutable, as getting the field and then mutating it won't persist - you'll just get a fresh clone of the original on the next access. Unfortunately Python and Rust don't agree about ownership - if PyO3 gave out references to (possibly) temporary Rust objects to Python code, Python code could then keep that reference alive indefinitely. Therefore returning Rust objects requires cloning.
If you don't want that cloning to happen, a workaround is to allocate the field on the Python heap and store a reference to that, by using [`Py<...>`]({{#PYO3_DOCS_URL}}/pyo3/struct.Py.html):
```rust
# use pyo3::prelude::*;
#[pyclass]
#[derive(Clone)]
struct Inner { /* fields omitted */ }
#[pyclass]
struct Outer {
#[pyo3(get)]
inner: Py<Inner>,
}
#[pymethods]
impl Outer {
#[new]
fn __new__(py: Python) -> PyResult<Self> {
Ok(Self {
inner: Py::new(py, Inner {})?,
})
}
}
```
This time `a` and `b` *are* the same object:
```python
outer = Outer()
a = outer.inner
b = outer.inner
assert a is b, f"a: {a}\nb: {b}"
print(f"a: {a}\nb: {b}")
```
```text
a: <builtins.Inner object at 0x0000020044FCC670>
b: <builtins.Inner object at 0x0000020044FCC670>
```
The downside to this approach is that any Rust code working on the `Outer` struct now has to acquire the GIL to do anything with its field.

8
guide/src/features.md
View File

@ -69,6 +69,14 @@ The `nightly` feature needs the nightly Rust compiler. This allows PyO3 to use R
- `FromPyObject` for `Vec` and `[T;N]` can perform a `memcpy` when the object supports the Python buffer protocol.
- `ToBorrowedObject` can skip a reference count increase when the provided object is a Python native type.
### `num-bigint`
This feature adds a dependency on [num-bigint](https://docs.rs/num-bigint) and enables conversions into its [`BigInt`](https://docs.rs/num-bigint/latest/num_bigint/struct.BigInt.html) and [`BigUint`](https://docs.rs/num-bigint/latest/num_bigint/struct.BigUInt.html) types.
### `num-complex`
This feature adds a dependency on [num-complex](https://docs.rs/num-complex) and enables conversions into its [`Complex`](https://docs.rs/num-complex/latest/num_complex/struct.Complex.html) type.
### `serde`
The `serde` feature enables (de)serialization of Py<T> objects via [serde](https://serde.rs/).

63
guide/src/function.md
View File

@ -6,7 +6,6 @@ One way is annotating a function with `#[pyfunction]` and then adding it to the
```rust
use pyo3::prelude::*;
use pyo3::wrap_pyfunction;
#[pyfunction]
fn double(x: usize) -> usize {
@ -14,52 +13,47 @@ fn double(x: usize) -> usize {
}
#[pymodule]
fn module_with_functions(py: Python, m: &PyModule) -> PyResult<()> {
fn my_extension(py: Python, m: &PyModule) -> PyResult<()> {
m.add_function(wrap_pyfunction!(double, m)?)?;
Ok(())
}
# fn main() {}
```
Alternatively there is a shorthand; the function can be placed inside the module definition and annotated with `#[pyfn]`, as below:
Alternatively, there is a shorthand: the function can be placed inside the module definition and
annotated with `#[pyfn]`, as below:
```rust
use pyo3::prelude::*;
#[pymodule]
fn rust2py(py: Python, m: &PyModule) -> PyResult<()> {
fn my_extension(py: Python, m: &PyModule) -> PyResult<()> {
#[pyfn(m)]
fn sum_as_string(_py: Python, a:i64, b:i64) -> PyResult<String> {
Ok(format!("{}", a + b))
fn double(x: usize) -> usize {
x * 2
}
Ok(())
}
# fn main() {}
```
`#[pyfn(m)]` is just syntax sugar for `#[pyfunction]`, and takes all the same options documented in the rest of this chapter. The code above is expanded to the following:
`#[pyfn(m)]` is just syntactic sugar for `#[pyfunction]`, and takes all the same options
documented in the rest of this chapter. The code above is expanded to the following:
```rust
use pyo3::prelude::*;
#[pymodule]
fn rust2py(py: Python, m: &PyModule) -> PyResult<()> {
fn my_extension(py: Python, m: &PyModule) -> PyResult<()> {
#[pyfunction]
fn sum_as_string(_py: Python, a:i64, b:i64) -> PyResult<String> {
Ok(format!("{}", a + b))
fn double(x: usize) -> usize {
x * 2
}
m.add_function(pyo3::wrap_pyfunction!(sum_as_string, m)?)?;
m.add_function(wrap_pyfunction!(double, m)?)?;
Ok(())
}
# fn main() {}
```
## Function options
@ -72,7 +66,6 @@ The `#[pyo3]` attribute can be used to modify properties of the generated Python
```rust
use pyo3::prelude::*;
use pyo3::wrap_pyfunction;
#[pyfunction]
#[pyo3(name = "no_args")]
@ -97,7 +90,6 @@ The `#[pyfunction]` attribute supports specifying details of argument parsing. T
```rust
use pyo3::prelude::*;
use pyo3::wrap_pyfunction;
use pyo3::types::PyDict;
#[pyfunction(kwds="**")]
@ -117,7 +109,7 @@ fn module_with_functions(py: Python, m: &PyModule) -> PyResult<()> {
## Making the function signature available to Python
In order to make the function signature available to Python to be retrieved via
`inspect.signature`, use the `#[text_signature]` annotation as in the example
`inspect.signature`, use the `#[pyo3(text_signature)]` annotation as in the example
below. The `/` signifies the end of positional-only arguments. (This
is not a feature of this library in particular, but the general format used by
CPython for annotating signatures of built-in functions.)
@ -127,7 +119,7 @@ use pyo3::prelude::*;
/// This function adds two unsigned 64-bit integers.
#[pyfunction]
#[text_signature = "(a, b, /)"]
#[pyo3(text_signature = "(a, b, /)")]
fn add(a: u64, b: u64) -> u64 {
a + b
}
@ -142,7 +134,7 @@ use pyo3::types::PyType;
// it works even if the item is not documented:
#[pyclass]
#[text_signature = "(c, d, /)"]
#[pyo3(text_signature = "(c, d, /)")]
struct MyClass {}
#[pymethods]
@ -154,17 +146,17 @@ impl MyClass {
Self {}
}
// the self argument should be written $self
#[text_signature = "($self, e, f)"]
#[pyo3(text_signature = "($self, e, f)")]
fn my_method(&self, e: i32, f: i32) -> i32 {
e + f
}
#[classmethod]
#[text_signature = "(cls, e, f)"]
#[pyo3(text_signature = "(cls, e, f)")]
fn my_class_method(cls: &PyType, e: i32, f: i32) -> i32 {
e + f
}
#[staticmethod]
#[text_signature = "(e, f)"]
#[pyo3(text_signature = "(e, f)")]
fn my_static_method(e: i32, f: i32) -> i32 {
e + f
}
@ -180,7 +172,7 @@ Alternatively, simply make sure the first line of your docstring is
formatted like in the following example. Please note that the newline after the
`--` is mandatory. The `/` signifies the end of positional-only arguments.
`#[text_signature]` should be preferred, since it will override automatically
`#[pyo3(text_signature)]` should be preferred, since it will override automatically
generated signatures when those are added in a future version of PyO3.
```rust
@ -255,13 +247,13 @@ in Python code.
### Accessing the module of a function
It is possible to access the module of a `#[pyfunction]` in the function body by passing the `pass_module` argument to the attribute:
It is possible to access the module of a `#[pyfunction]` in the function body by using `#[pyo3(pass_module)]` option:
```rust
use pyo3::wrap_pyfunction;
use pyo3::prelude::*;
#[pyfunction(pass_module)]
#[pyfunction]
#[pyo3(pass_module)]
fn pyfunction_with_module(module: &PyModule) -> PyResult<&str> {
module.name()
}
@ -279,12 +271,11 @@ in the function body.
## Accessing the FFI functions
In order to make Rust functions callable from Python, PyO3 generates a
`extern "C" Fn(slf: *mut PyObject, args: *mut PyObject, kwargs: *mut PyObject) -> *mut Pyobject`
function and embeds the call to the Rust function inside this FFI-wrapper function. This
wrapper handles extraction of the regular arguments and the keyword arguments from the input
`PyObjects`. Since this function is not user-defined but required to build a `PyCFunction`, PyO3
offers the `raw_pycfunction!()` macro to get the identifier of this generated wrapper.
In order to make Rust functions callable from Python, PyO3 generates an `extern "C"`
function whose exact signature depends on the Rust signature. (PyO3 chooses the optimal
Python argument passing convention.) It then embeds the call to the Rust function inside this
FFI-wrapper function. This wrapper handles extraction of the regular arguments and the keyword
arguments from the input `PyObject`s.
The `wrap_pyfunction` macro can be used to directly get a `PyCFunction` given a
`#[pyfunction]` and a `PyModule`: `wrap_pyfunction!(rust_fun, module)`.

200
guide/src/module.md
View File

@ -1,99 +1,183 @@
# Python Modules
You can create a module as follows:
You can create a module using `#[pymodule]`:
```rust
use pyo3::prelude::*;
// add bindings to the generated Python module
// N.B: "rust2py" must be the name of the `.so` or `.pyd` file.
#[pyfunction]
fn double(x: usize) -> usize {
x * 2
}
/// This module is implemented in Rust.
#[pymodule]
fn rust2py(py: Python, m: &PyModule) -> PyResult<()> {
// PyO3 aware function. All of our Python interfaces could be declared in a separate module.
// Note that the `#[pyfn()]` annotation automatically converts the arguments from
// Python objects to Rust values, and the Rust return value back into a Python object.
// The `_py` argument represents that we're holding the GIL.
#[pyfn(m)]
#[pyo3(name = "sum_as_string")]
fn sum_as_string_py(_py: Python, a: i64, b: i64) -> PyResult<String> {
let out = sum_as_string(a, b);
Ok(out)
}
fn my_extension(py: Python, m: &PyModule) -> PyResult<()> {
m.add_function(wrap_pyfunction!(double, m)?)?;
Ok(())
}
// logic implemented as a normal Rust function
fn sum_as_string(a: i64, b: i64) -> String {
format!("{}", a + b)
}
# fn main() {}
```
The `#[pymodule]` procedural macro attribute takes care of exporting the initialization function of your
module to Python. It can take as an argument the name of your module, which must be the name of the `.so`
or `.pyd` file; the default is the Rust function's name.
The `#[pymodule]` procedural macro takes care of exporting the initialization function of your
module to Python.
If the name of the module (the default being the function name) does not match the name of the `.so` or
`.pyd` file, you will get an import error in Python with the following message:
The module's name defaults to the name of the Rust function. You can override the module name by
using `#[pyo3(name = "custom_name")]`:
```rust
use pyo3::prelude::*;
#[pyfunction]
fn double(x: usize) -> usize {
x * 2
}
#[pymodule]
#[pyo3(name = "custom_name")]
fn my_extension(py: Python, m: &PyModule) -> PyResult<()> {
m.add_function(wrap_pyfunction!(double, m)?)?;
Ok(())
}
```
The name of the module must match the name of the `.so` or `.pyd`
file. Otherwise, you will get an import error in Python with the following message:
`ImportError: dynamic module does not define module export function (PyInit_name_of_your_module)`
To import the module, either copy the shared library as described in [the README](https://github.com/PyO3/pyo3)
or use a tool, e.g. `maturin develop` with [maturin](https://github.com/PyO3/maturin) or
To import the module, either:
- copy the shared library as described in [Manual builds](building_and_distribution.html#manual-builds), or
- use a tool, e.g. `maturin develop` with [maturin](https://github.com/PyO3/maturin) or
`python setup.py develop` with [setuptools-rust](https://github.com/PyO3/setuptools-rust).
## Documentation
The [Rust doc comments](https://doc.rust-lang.org/stable/book/first-edition/comments.html) of the module
The [Rust doc comments](https://doc.rust-lang.org/stable/book/ch03-04-comments.html) of the module
initialization function will be applied automatically as the Python docstring of your module.
```python
import rust2py
For example, building off of the above code, this will print `This module is implemented in Rust.`:
print(rust2py.__doc__)
```python
import my_extension
print(my_extension.__doc__)
```
Which means that the above Python code will print `This module is implemented in Rust.`.
## Organizing your module registration code
## Modules as objects
For most projects, it's adequate to centralize all your FFI code into a single Rust module.
In Python, modules are first class objects. This means that you can store them as values or add them to
dicts or other modules:
However, for larger projects, it can be helpful to split your Rust code into several Rust modules to keep your code
readable. Unfortunately, though, some of the macros like `wrap_pyfunction!` do not yet work when used on code defined
in other modules ([#1709](https://github.com/PyO3/pyo3/issues/1709)). One way to work around this is to pass
references to the `PyModule` so that each module registers its own FFI code. For example:
```rust
// src/lib.rs
use pyo3::prelude::*;
#[pymodule]
fn my_extension(py: Python, m: &PyModule) -> PyResult<()> {
dirutil::register(py, m)?;
osutil::register(py, m)?;
Ok(())
}
// src/dirutil.rs
# mod dirutil {
use pyo3::prelude::*;
pub(crate) fn register(py: Python, m: &PyModule) -> PyResult<()> {
m.add_class::<SomeClass>()?;
Ok(())
}
#[pyclass]
struct SomeClass {
x: usize,
}
# }
// src/osutil.rs
# mod osutil {
use pyo3::prelude::*;
pub(crate) fn register(py: Python, m: &PyModule) -> PyResult<()> {
m.add_function(wrap_pyfunction!(determine_current_os, m)?)?;
Ok(())
}
#[pyfunction]
fn determine_current_os() -> String {
"linux".to_owned()
}
# }
```
Another workaround for splitting FFI code across multiple modules ([#1709](https://github.com/PyO3/pyo3/issues/1709))
is to add `use module::*`, like this:
```rust
// src/lib.rs
use pyo3::prelude::*;
use osutil::*;
#[pymodule]
fn my_extension(py: Python, m: &PyModule) -> PyResult<()> {
m.add_function(wrap_pyfunction!(determine_current_os, m)?)?;
Ok(())
}
// src/osutil.rs
# mod osutil {
use pyo3::prelude::*;
#[pyfunction]
pub(crate) fn determine_current_os() -> String {
"linux".to_owned()
}
# }
```
## Python submodules
You can create a module hierarchy within a single extension module by using
[`PyModule.add_submodule()`]({{#PYO3_DOCS_URL}}/pyo3/prelude/struct.PyModule.html#method.add_submodule).
For example, you could define the modules `parent_module` and `parent_module.child_module`.
```rust
use pyo3::prelude::*;
use pyo3::{wrap_pyfunction, wrap_pymodule};
use pyo3::types::IntoPyDict;
#[pyfunction]
fn subfunction() -> String {
"Subfunction".to_string()
}
fn init_submodule(module: &PyModule) -> PyResult<()> {
module.add_function(wrap_pyfunction!(subfunction, module)?)?;
Ok(())
}
#[pymodule]
fn supermodule(py: Python, module: &PyModule) -> PyResult<()> {
let submod = PyModule::new(py, "submodule")?;
init_submodule(submod)?;
module.add_submodule(submod)?;
fn parent_module(py: Python, m: &PyModule) -> PyResult<()> {
register_child_module(py, m)?;
Ok(())
}
fn register_child_module(py: Python, parent_module: &PyModule) -> PyResult<()> {
let child_module = PyModule::new(py, "child_module")?;
child_module.add_function(wrap_pyfunction!(func, child_module)?)?;
parent_module.add_submodule(child_module)?;
Ok(())
}
#[pyfunction]
fn func() -> String {
"func".to_string()
}
# Python::with_gil(|py| {
# let supermodule = wrap_pymodule!(supermodule)(py);
# let ctx = [("supermodule", supermodule)].into_py_dict(py);
# use pyo3::wrap_pymodule;
# use pyo3::types::IntoPyDict;
# let parent_module = wrap_pymodule!(parent_module)(py);
# let ctx = [("parent_module", parent_module)].into_py_dict(py);
#
# py.run("assert supermodule.submodule.subfunction() == 'Subfunction'", None, Some(&ctx)).unwrap();
# py.run("assert parent_module.child_module.func() == 'func'", None, Some(&ctx)).unwrap();
# })
```
This way, you can create a module hierarchy within a single extension module.
Note that this does not define a package, so this won’t allow Python code to directly import
submodules by using `from parent_module import child_module`. For more information, see
[#759](https://github.com/PyO3/pyo3/issues/759) and
[#1517](https://github.com/PyO3/pyo3/issues/1517#issuecomment-808664021).
It is not necessary to add `#[pymodule]` on nested modules, this is only required on the top-level module.
It is not necessary to add `#[pymodule]` on nested modules, which is only required on the top-level module.

1
guide/src/trait_bounds.md
View File

@ -461,7 +461,6 @@ It is also required to make the struct public.
```rust
use pyo3::prelude::*;
use pyo3::wrap_pyfunction;
use pyo3::types::PyAny;
pub trait Model {

3
pyo3-build-config/Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "pyo3-build-config"
version = "0.14.0-alpha.0"
version = "0.14.1"
description = "Build configuration for the PyO3 ecosystem"
authors = ["PyO3 Project and Contributors <https://github.com/PyO3>"]
keywords = ["pyo3", "python", "cpython", "ffi"]
@ -11,6 +11,7 @@ license = "Apache-2.0"
edition = "2018"
[dependencies]
once_cell = "1"
[features]
default = []

10
pyo3-build-config/build.rs
View File

@ -1,11 +1,3 @@
#[allow(dead_code)]
#[path = "src/impl_.rs"]
mod impl_;
fn main() {
// Print out error messages using display, to get nicer formatting.
if let Err(e) = impl_::configure() {
eprintln!("error: {}", e);
std::process::exit(1)
}
// Empty build script to force cargo to produce the "OUT_DIR" environment variable.
}

16
pyo3-build-config/src/bin/print-config.rs Normal file
View File

@ -0,0 +1,16 @@
use pyo3_build_config::{find_interpreter, get_config_from_interpreter};
fn main() -> Result<(), Box<dyn std::error::Error>> {
let config = get_config_from_interpreter(&find_interpreter()?)?;
println!("implementation: {}", config.implementation);
println!("interpreter version: {}", config.version);
println!("interpreter path: {:?}", config.executable);
println!("libdir: {:?}", config.libdir);
println!("shared: {}", config.shared);
println!("base prefix: {:?}", config.base_prefix);
println!("ld_version: {:?}", config.ld_version);
println!("pointer width: {:?}", config.calcsize_pointer);
Ok(())
}

86
pyo3-build-config/src/errors.rs Normal file
View File

@ -0,0 +1,86 @@
/// A simple macro for returning an error. Resembles anyhow::bail.
#[macro_export]
macro_rules! bail {
($msg: expr) => { return Err($msg.into()); };
($fmt: literal $($args: tt)+) => { return Err(format!($fmt $($args)+).into()); };
}
/// A simple macro for checking a condition. Resembles anyhow::ensure.
#[macro_export]
macro_rules! ensure {
($condition:expr, $($args: tt)+) => { if !($condition) { bail!($($args)+) } };
}
/// Show warning. If needed, please extend this macro to support arguments.
#[macro_export]
macro_rules! warn {
($msg: literal) => {
println!(concat!("cargo:warning=", $msg));
};
}
/// A simple error implementation which allows chaining of errors, inspired somewhat by anyhow.
#[derive(Debug)]
pub struct Error {
value: String,
source: Option<Box<dyn std::error::Error>>,
}
impl std::fmt::Display for Error {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.value)
}
}
impl std::error::Error for Error {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
self.source.as_deref()
}
}
impl From<String> for Error {
fn from(value: String) -> Self {
Self {
value,
source: None,
}
}
}
impl From<&'_ str> for Error {
fn from(value: &str) -> Self {
value.to_string().into()
}
}
impl From<std::convert::Infallible> for Error {
fn from(_: std::convert::Infallible) -> Self {
unreachable!()
}
}
pub type Result<T, E = Error> = std::result::Result<T, E>;
pub trait Context<T> {
fn context(self, message: impl Into<String>) -> Result<T>;
fn with_context(self, message: impl FnOnce() -> String) -> Result<T>;
}
impl<T, E> Context<T> for Result<T, E>
where
E: std::error::Error + 'static,
{
fn context(self, message: impl Into<String>) -> Result<T> {
self.map_err(|error| Error {
value: message.into(),
source: Some(Box::new(error)),
})
}
fn with_context(self, message: impl FnOnce() -> String) -> Result<T> {
self.map_err(|error| Error {
value: message(),
source: Some(Box::new(error)),
})
}
}

513
pyo3-build-config/src/impl_.rs
View File

@ -4,38 +4,24 @@ use std::{
env,
ffi::OsString,
fmt::Display,
fs::{self, DirEntry, File},
io::Write,
fs::{self, DirEntry},
io::{BufRead, BufReader, Read, Write},
path::{Path, PathBuf},
process::{Command, Stdio},
str::FromStr,
};
use crate::{
bail, ensure,
errors::{Context, Error, Result},
warn,
};
/// Minimum Python version PyO3 supports.
const MINIMUM_SUPPORTED_VERSION: PythonVersion = PythonVersion { major: 3, minor: 6 };
/// Maximum Python version that can be used as minimum required Python version with abi3.
const ABI3_MAX_MINOR: u8 = 9;
type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
// A simple macro for returning an error. Resembles anyhow::bail.
macro_rules! bail {
($msg: expr) => { return Err($msg.into()); };
($fmt: literal $($args: tt)+) => { return Err(format!($fmt $($args)+).into()); };
}
// A simple macro for checking a condition. Resembles anyhow::ensure.
macro_rules! ensure {
($condition:expr, $($args: tt)+) => { if !($condition) { bail!($($args)+) } };
}
// Show warning. If needed, please extend this macro to support arguments.
macro_rules! warn {
($msg: literal) => {
println!(concat!("cargo:warning=", $msg));
};
}
/// Gets an environment variable owned by cargo.
///
/// Environment variables set by cargo are expected to be valid UTF8.
@ -55,7 +41,7 @@ fn env_var(var: &str) -> Option<OsString> {
/// Usually this is queried directly from the Python interpreter. When the `PYO3_NO_PYTHON` variable
/// is set, or during cross compile situations, then alternative strategies are used to populate
/// this type.
#[derive(Debug)]
#[cfg_attr(test, derive(Debug, PartialEq))]
pub struct InterpreterConfig {
pub version: PythonVersion,
pub libdir: Option<String>,
@ -99,6 +85,114 @@ impl InterpreterConfig {
pub fn is_pypy(&self) -> bool {
self.implementation == PythonImplementation::PyPy
}
#[doc(hidden)]
pub fn from_reader(reader: impl Read) -> Result<Self> {
let reader = BufReader::new(reader);
let mut lines = reader.lines();
macro_rules! parse_line {
($value:literal) => {
lines
.next()
.ok_or(concat!("reached end of config when reading ", $value))?
.context(concat!("failed to read ", $value, " from config"))?
.parse()
.context(concat!("failed to parse ", $value, " from config"))
};
}
macro_rules! parse_option_line {
($value:literal) => {
parse_option_string(
lines
.next()
.ok_or(concat!("reached end of config when reading ", $value))?
.context(concat!("failed to read ", $value, " from config"))?,
)
.context(concat!("failed to parse ", $value, "from config"))
};
}
let major = parse_line!("major version")?;
let minor = parse_line!("minor version")?;
let libdir = parse_option_line!("libdir")?;
let shared = parse_line!("shared")?;
let abi3 = parse_line!("abi3")?;
let ld_version = parse_option_line!("ld_version")?;
let base_prefix = parse_option_line!("base_prefix")?;
let executable = parse_option_line!("executable")?;
let calcsize_pointer = parse_option_line!("calcsize_pointer")?;
let implementation = parse_line!("implementation")?;
let mut build_flags = BuildFlags(HashSet::new());
for line in lines {
build_flags
.0
.insert(line.context("failed to read flag from config")?.parse()?);
}
Ok(InterpreterConfig {
version: PythonVersion { major, minor },
libdir,
shared,
abi3,
ld_version,
base_prefix,
executable,
calcsize_pointer,
implementation,
build_flags,
})
}
#[doc(hidden)]
pub fn to_writer(&self, mut writer: impl Write) -> Result<()> {
macro_rules! write_line {
($value:expr) => {
writeln!(writer, "{}", $value).context(concat!(
"failed to write ",
stringify!($value),
" to config"
))
};
}
macro_rules! write_option_line {
($opt:expr) => {
match &$opt {
Some(value) => writeln!(writer, "{}", value),
None => writeln!(writer, "null"),
}
.context(concat!(
"failed to write ",
stringify!($value),
" to config"
))
};
}
write_line!(self.version.major)?;
write_line!(self.version.minor)?;
write_option_line!(self.libdir)?;
write_line!(self.shared)?;
write_line!(self.abi3)?;
write_option_line!(self.ld_version)?;
write_option_line!(self.base_prefix)?;
write_option_line!(self.executable)?;
write_option_line!(self.calcsize_pointer)?;
write_line!(self.implementation)?;
for flag in &self.build_flags.0 {
write_line!(flag)?;
}
Ok(())
}
}
fn parse_option_string<T: FromStr>(string: String) -> Result<Option<T>, <T as FromStr>::Err> {
if string == "null" {
Ok(None)
} else {
string.parse().map(Some)
}
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)]
@ -123,13 +217,22 @@ pub enum PythonImplementation {
PyPy,
}
impl Display for PythonImplementation {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
PythonImplementation::CPython => write!(f, "CPython"),
PythonImplementation::PyPy => write!(f, "PyPy"),
}
}
}
impl FromStr for PythonImplementation {
type Err = Box<dyn std::error::Error>;
type Err = Error;
fn from_str(s: &str) -> Result<Self> {
match s {
"CPython" => Ok(PythonImplementation::CPython),
"PyPy" => Ok(PythonImplementation::PyPy),
_ => bail!("Invalid interpreter: {}", s),
_ => bail!("unknown interpreter: {}", s),
}
}
}
@ -140,7 +243,9 @@ fn is_abi3() -> bool {
trait GetPrimitive {
fn get_bool(&self, key: &str) -> Result<bool>;
fn get_numeric<T: FromStr>(&self, key: &str) -> Result<T>;
fn get_numeric<T: FromStr>(&self, key: &str) -> Result<T>
where
T::Err: std::error::Error + 'static;
}
impl GetPrimitive for HashMap<String, String> {
@ -156,11 +261,14 @@ impl GetPrimitive for HashMap<String, String> {
}
}
fn get_numeric<T: FromStr>(&self, key: &str) -> Result<T> {
fn get_numeric<T: FromStr>(&self, key: &str) -> Result<T>
where
T::Err: std::error::Error + 'static,
{
self.get(key)
.ok_or(format!("{} is not defined", key))?
.parse::<T>()
.map_err(|_| format!("Could not parse value of {}", key).into())
.with_context(|| format!("Could not parse value of {}", key))
}
}
@ -234,34 +342,75 @@ fn cross_compiling() -> Result<Option<CrossCompileConfig>> {
}))
}
#[allow(non_camel_case_types)]
#[derive(Debug, Clone, Hash, PartialEq, Eq)]
pub enum BuildFlag {
WITH_THREAD,
Py_DEBUG,
Py_REF_DEBUG,
Py_TRACE_REFS,
COUNT_ALLOCS,
Other(String),
}
impl Display for BuildFlag {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
BuildFlag::Other(flag) => write!(f, "{}", flag),
_ => write!(f, "{:?}", self),
}
}
}
impl FromStr for BuildFlag {
type Err = std::convert::Infallible;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
"WITH_THREAD" => Ok(BuildFlag::WITH_THREAD),
"Py_DEBUG" => Ok(BuildFlag::Py_DEBUG),
"Py_REF_DEBUG" => Ok(BuildFlag::Py_REF_DEBUG),
"Py_TRACE_REFS" => Ok(BuildFlag::Py_TRACE_REFS),
"COUNT_ALLOCS" => Ok(BuildFlag::COUNT_ALLOCS),
other => Ok(BuildFlag::Other(other.to_owned())),
}
}
}
/// A list of python interpreter compile-time preprocessor defines that
/// we will pick up and pass to rustc via `--cfg=py_sys_config={varname}`;
/// this allows using them conditional cfg attributes in the .rs files, so
///
/// #[cfg(py_sys_config="{varname}"]
/// ```rust
/// #[cfg(py_sys_config="{varname}")]
/// # struct Foo;
/// ```
///
/// is the equivalent of `#ifdef {varname}` in C.
///
/// see Misc/SpecialBuilds.txt in the python source for what these mean.
#[derive(Debug)]
pub struct BuildFlags(pub HashSet<&'static str>);
#[cfg_attr(test, derive(Debug, PartialEq))]
pub struct BuildFlags(pub HashSet<BuildFlag>);
impl BuildFlags {
const ALL: [&'static str; 5] = [
const ALL: [BuildFlag; 5] = [
// TODO: Remove WITH_THREAD once Python 3.6 support dropped (as it's always on).
"WITH_THREAD",
"Py_DEBUG",
"Py_REF_DEBUG",
"Py_TRACE_REFS",
"COUNT_ALLOCS",
BuildFlag::WITH_THREAD,
BuildFlag::Py_DEBUG,
BuildFlag::Py_REF_DEBUG,
BuildFlag::Py_TRACE_REFS,
BuildFlag::COUNT_ALLOCS,
];
fn from_config_map(config_map: &HashMap<String, String>) -> Self {
Self(
BuildFlags::ALL
.iter()
.copied()
.filter(|flag| config_map.get(*flag).map_or(false, |value| value == "1"))
.cloned()
.filter(|flag| {
config_map
.get(&flag.to_string())
.map_or(false, |value| value == "1")
})
.collect(),
)
}
@ -270,7 +419,8 @@ impl BuildFlags {
/// the interpreter and printing variables of interest from
/// sysconfig.get_config_vars.
fn from_interpreter(interpreter: &Path) -> Result<Self> {
if cargo_env_var("CARGO_CFG_TARGET_OS").unwrap() == "windows" {
// If we're on a Windows host, then Python won't have any useful config vars
if cfg!(windows) {
return Ok(Self::windows_hardcoded());
}
@ -281,7 +431,7 @@ impl BuildFlags {
script.push_str(&format!("print(config.get('{}', '0'))\n", k));
}
let stdout = run_python_script(&interpreter, &script)?;
let stdout = run_python_script(interpreter, &script)?;
let split_stdout: Vec<&str> = stdout.trim_end().lines().collect();
ensure!(
split_stdout.len() == BuildFlags::ALL.len(),
@ -292,7 +442,7 @@ impl BuildFlags {
.iter()
.zip(split_stdout)
.filter(|(_, flag_value)| *flag_value == "1")
.map(|(&flag, _)| flag)
.map(|(flag, _)| flag.clone())
.collect();
Ok(Self(flags))
@ -302,36 +452,36 @@ impl BuildFlags {
// sysconfig is missing all the flags on windows, so we can't actually
// query the interpreter directly for its build flags.
let mut flags = HashSet::new();
flags.insert("WITH_THREAD");
flags.insert(BuildFlag::WITH_THREAD);
// Uncomment these manually if your python was built with these and you want
// the cfg flags to be set in rust.
//
// map.insert("Py_DEBUG", "1");
// map.insert("Py_REF_DEBUG", "1");
// map.insert("Py_TRACE_REFS", "1");
// map.insert("COUNT_ALLOCS", 1");
// flags.insert(BuildFlag::Py_DEBUG);
// flags.insert(BuildFlag::Py_REF_DEBUG);
// flags.insert(BuildFlag::Py_TRACE_REFS);
// flags.insert(BuildFlag::COUNT_ALLOCS;
Self(flags)
}
fn abi3() -> Self {
let mut flags = HashSet::new();
flags.insert("WITH_THREAD");
flags.insert(BuildFlag::WITH_THREAD);
Self(flags)
}
fn fixup(mut self, version: PythonVersion, implementation: PythonImplementation) -> Self {
if self.0.contains("Py_DEBUG") {
self.0.insert("Py_REF_DEBUG");
if self.0.contains(&BuildFlag::Py_DEBUG) {
self.0.insert(BuildFlag::Py_REF_DEBUG);
if version <= PythonVersion::PY37 {
// Py_DEBUG only implies Py_TRACE_REFS until Python 3.7
self.0.insert("Py_TRACE_REFS");
self.0.insert(BuildFlag::Py_TRACE_REFS);
}
}
// WITH_THREAD is always on for Python 3.7, and for PyPy.
if implementation == PythonImplementation::PyPy || version >= PythonVersion::PY37 {
self.0.insert("WITH_THREAD");
self.0.insert(BuildFlag::WITH_THREAD);
}
self
@ -354,10 +504,16 @@ fn parse_script_output(output: &str) -> HashMap<String, String> {
/// python executable and library. Here it is read and added to a script to extract only what is
/// necessary. This necessitates a python interpreter for the host machine to work.
fn parse_sysconfigdata(config_path: impl AsRef<Path>) -> Result<HashMap<String, String>> {
let mut script = fs::read_to_string(config_path)?;
let mut script = fs::read_to_string(config_path.as_ref()).with_context(|| {
format!(
"failed to read config from {}",
config_path.as_ref().display()
)
})?;
script += r#"
print("version_major", build_time_vars["VERSION"][0]) # 3
print("version_minor", build_time_vars["VERSION"][2]) # E.g., 8
print("SOABI", build_time_vars.get("SOABI", ""))
KEYS = [
"WITH_THREAD",
"Py_DEBUG",
@ -417,7 +573,7 @@ fn ends_with(entry: &DirEntry, pat: &str) -> bool {
///
/// [1]: https://github.com/python/cpython/blob/3.5/Lib/sysconfig.py#L389
fn find_sysconfigdata(cross: &CrossCompileConfig) -> Result<PathBuf> {
let sysconfig_paths = search_lib_dir(&cross.lib_dir, &cross);
let sysconfig_paths = search_lib_dir(&cross.lib_dir, cross);
let sysconfig_name = env_var("_PYTHON_SYSCONFIGDATA_NAME");
let mut sysconfig_paths = sysconfig_paths
.iter()
@ -484,6 +640,24 @@ fn search_lib_dir(path: impl AsRef<Path>, cross: &CrossCompileConfig) -> Vec<Pat
};
sysconfig_paths.extend(sysc);
}
// If we got more than one file, only take those that contain the arch name.
// For ubuntu 20.04 with host architecture x86_64 and a foreign architecture of armhf
// this reduces the number of candidates to 1:
//
// $ find /usr/lib/python3.8/ -name '_sysconfigdata*.py' -not -lname '*'
// /usr/lib/python3.8/_sysconfigdata__x86_64-linux-gnu.py
// /usr/lib/python3.8/_sysconfigdata__arm-linux-gnueabihf.py
if sysconfig_paths.len() > 1 {
let temp = sysconfig_paths
.iter()
.filter(|p| p.to_string_lossy().contains(&cross.arch))
.cloned()
.collect::<Vec<PathBuf>>();
if !temp.is_empty() {
sysconfig_paths = temp;
}
}
sysconfig_paths
}
@ -506,9 +680,19 @@ fn load_cross_compile_from_sysconfigdata(
None => format!("{}.{}", major, minor),
};
let calcsize_pointer = sysconfig_data.get_numeric("SIZEOF_VOID_P").ok();
let soabi = match sysconfig_data.get("SOABI") {
Some(s) => s,
None => bail!("sysconfigdata did not define SOABI"),
};
let version = PythonVersion { major, minor };
let implementation = PythonImplementation::CPython;
let implementation = if soabi.starts_with("pypy") {
PythonImplementation::PyPy
} else if soabi.starts_with("cpython") {
PythonImplementation::CPython
} else {
bail!("unsupported Python interpreter");
};
Ok(InterpreterConfig {
version,
@ -602,7 +786,8 @@ fn run_python_script(interpreter: &Path, script: &str) -> Result<String> {
err
),
Ok(ok) if !ok.status.success() => bail!("Python script failed"),
Ok(ok) => Ok(String::from_utf8(ok.stdout)?),
Ok(ok) => Ok(String::from_utf8(ok.stdout)
.context("failed to parse Python script output as utf-8")?),
}
}
@ -626,13 +811,16 @@ fn get_venv_path() -> Option<PathBuf> {
/// 2. If in a virtualenv, that environment's interpreter is used.
/// 3. `python`, if this is functional a Python 3.x interpreter
/// 4. `python3`, as above
fn find_interpreter() -> Result<PathBuf> {
pub fn find_interpreter() -> Result<PathBuf> {
if let Some(exe) = env_var("PYO3_PYTHON") {
Ok(exe.into())
} else if let Some(venv_path) = get_venv_path() {
match cargo_env_var("CARGO_CFG_TARGET_OS").unwrap().as_str() {
"windows" => Ok(venv_path.join("Scripts\\python")),
_ => Ok(venv_path.join("bin/python")),
// Use cfg rather can CARGO_TARGET_OS because this affects how files are located on the
// host OS
if cfg!(windows) {
Ok(venv_path.join("Scripts\\python"))
} else {
Ok(venv_path.join("bin/python"))
}
} else {
println!("cargo:rerun-if-env-changed=PATH");
@ -652,7 +840,7 @@ fn find_interpreter() -> Result<PathBuf> {
}
/// Extract compilation vars from the specified interpreter.
fn get_config_from_interpreter(interpreter: &Path) -> Result<InterpreterConfig> {
pub fn get_config_from_interpreter(interpreter: &Path) -> Result<InterpreterConfig> {
let script = r#"
# Allow the script to run on Python 2, so that nicer error can be printed later.
from __future__ import print_function
@ -681,37 +869,36 @@ def print_if_set(varname, value):
if value is not None:
print(varname, value)
libdir = get_config_var("LIBDIR")
# Windows always uses shared linking
WINDOWS = hasattr(platform, "win32_ver")
# macOS framework packages use shared linking
FRAMEWORK = bool(get_config_var("PYTHONFRAMEWORK"))
# unix-style shared library enabled
SHARED = bool(get_config_var("Py_ENABLE_SHARED"))
print("version_major", sys.version_info[0])
print("version_minor", sys.version_info[1])
print("implementation", platform.python_implementation())
print_if_set("libdir", libdir)
print_if_set("libdir", get_config_var("LIBDIR"))
print_if_set("ld_version", get_config_var("LDVERSION"))
print_if_set("base_prefix", base_prefix)
print("framework", bool(get_config_var("PYTHONFRAMEWORK")))
print("shared", PYPY or ANACONDA or bool(get_config_var("Py_ENABLE_SHARED")))
print("shared", PYPY or ANACONDA or WINDOWS or FRAMEWORK or SHARED)
print("executable", sys.executable)
print("calcsize_pointer", struct.calcsize("P"))
"#;
let output = run_python_script(interpreter, script)?;
let map: HashMap<String, String> = parse_script_output(&output);
let shared = match (
cargo_env_var("CARGO_CFG_TARGET_OS").unwrap().as_str(),
map["framework"].as_str(),
map["shared"].as_str(),
) {
(_, _, "True") // Py_ENABLE_SHARED is set
| ("windows", _, _) // Windows always uses shared linking
| ("macos", "True", _) // MacOS framework package uses shared linking
=> true,
(_, _, "False") => false, // Any other platform, Py_ENABLE_SHARED not set
_ => bail!("Unrecognised link model combination")
};
let shared = map["shared"].as_str() == "True";
let version = PythonVersion {
major: map["version_major"].parse()?,
minor: map["version_minor"].parse()?,
major: map["version_major"]
.parse()
.context("failed to parse major version")?,
minor: map["version_minor"]
.parse()
.context("failed to parse minor version")?,
};
let implementation = map["implementation"].parse()?;
@ -725,7 +912,11 @@ print("calcsize_pointer", struct.calcsize("P"))
ld_version: map.get("ld_version").cloned(),
base_prefix: map.get("base_prefix").cloned(),
executable: map.get("executable").cloned(),
calcsize_pointer: Some(map["calcsize_pointer"].parse()?),
calcsize_pointer: Some(
map["calcsize_pointer"]
.parse()
.context("failed to parse calcsize_pointer")?,
),
build_flags: BuildFlags::from_interpreter(interpreter)?.fixup(version, implementation),
})
}
@ -735,7 +926,7 @@ fn get_abi3_minor_version() -> Option<u8> {
.find(|i| cargo_env_var(&format!("CARGO_FEATURE_ABI3_PY3{}", i)).is_some())
}
fn get_interpreter_config() -> Result<InterpreterConfig> {
pub fn make_interpreter_config() -> Result<InterpreterConfig> {
let abi3_version = get_abi3_minor_version();
// If PYO3_NO_PYTHON is set with abi3, we can build PyO3 without calling Python.
@ -780,53 +971,125 @@ fn get_interpreter_config() -> Result<InterpreterConfig> {
Ok(interpreter_config)
}
pub fn configure() -> Result<()> {
let interpreter_config = get_interpreter_config()?;
write_interpreter_config(&interpreter_config)
}
#[cfg(test)]
mod tests {
use std::io::Cursor;
fn write_interpreter_config(interpreter_config: &InterpreterConfig) -> Result<()> {
let out_dir = env::var_os("OUT_DIR").unwrap();
let mut out = File::create(Path::new(&out_dir).join("pyo3-build-config.rs"))?;
use super::*;
writeln!(out, "{{")?;
writeln!(
out,
"let mut build_flags = std::collections::HashSet::new();"
)?;
for flag in &interpreter_config.build_flags.0 {
writeln!(out, "build_flags.insert({:?});", flag)?;
#[test]
fn test_read_write_roundtrip() {
let config = InterpreterConfig {
abi3: true,
base_prefix: Some("base_prefix".into()),
build_flags: BuildFlags::abi3(),
calcsize_pointer: Some(32),
executable: Some("executable".into()),
implementation: PythonImplementation::CPython,
ld_version: Some("ld_version".into()),
libdir: Some("libdir".into()),
shared: true,
version: MINIMUM_SUPPORTED_VERSION,
};
let mut buf: Vec<u8> = Vec::new();
config.to_writer(&mut buf).unwrap();
assert_eq!(
config,
InterpreterConfig::from_reader(Cursor::new(buf)).unwrap()
);
// And some different options, for variety
let config = InterpreterConfig {
abi3: false,
base_prefix: None,
build_flags: {
let mut flags = HashSet::new();
flags.insert(BuildFlag::Py_DEBUG);
flags.insert(BuildFlag::Other(String::from("Py_SOME_FLAG")));
BuildFlags(flags)
},
calcsize_pointer: None,
executable: None,
implementation: PythonImplementation::PyPy,
ld_version: None,
libdir: None,
shared: true,
version: PythonVersion {
major: 3,
minor: 10,
},
};
let mut buf: Vec<u8> = Vec::new();
config.to_writer(&mut buf).unwrap();
assert_eq!(
config,
InterpreterConfig::from_reader(Cursor::new(buf)).unwrap()
);
}
writeln!(
out,
r#"crate::impl_::InterpreterConfig {{
version: crate::impl_::PythonVersion {{
major: {major},
minor: {minor},
}},
implementation: crate::impl_::PythonImplementation::{implementation:?},
libdir: {libdir:?}.map(|str: &str| str.to_string()),
abi3: {abi3},
build_flags: crate::impl_::BuildFlags(build_flags),
base_prefix: {base_prefix:?}.map(|str: &str| str.to_string()),
calcsize_pointer: {calcsize_pointer:?},
executable: {executable:?}.map(|str: &str| str.to_string()),
ld_version: {ld_version:?}.map(|str: &str| str.to_string()),
shared: {shared:?}
}}"#,
major = interpreter_config.version.major,
minor = interpreter_config.version.minor,
implementation = interpreter_config.implementation,
base_prefix = interpreter_config.base_prefix,
calcsize_pointer = interpreter_config.calcsize_pointer,
executable = interpreter_config.executable,
ld_version = interpreter_config.ld_version,
libdir = interpreter_config.libdir,
shared = interpreter_config.shared,
abi3 = interpreter_config.abi3,
)?;
writeln!(out, "}}")?;
#[test]
fn build_flags_from_config_map() {
let mut config_map = HashMap::new();
Ok(())
assert_eq!(BuildFlags::from_config_map(&config_map).0, HashSet::new());
for flag in &BuildFlags::ALL {
config_map.insert(flag.to_string(), "0".into());
}
assert_eq!(BuildFlags::from_config_map(&config_map).0, HashSet::new());
let mut expected_flags = HashSet::new();
for flag in &BuildFlags::ALL {
config_map.insert(flag.to_string(), "1".into());
expected_flags.insert(flag.clone());
}
assert_eq!(BuildFlags::from_config_map(&config_map).0, expected_flags);
}
#[test]
fn build_flags_fixup_py36_debug() {
let mut build_flags = BuildFlags(HashSet::new());
build_flags.0.insert(BuildFlag::Py_DEBUG);
build_flags = build_flags.fixup(
PythonVersion { major: 3, minor: 6 },
PythonImplementation::CPython,
);
// On 3.6, Py_DEBUG implies Py_REF_DEBUG and Py_TRACE_REFS
assert!(build_flags.0.contains(&BuildFlag::Py_REF_DEBUG));
assert!(build_flags.0.contains(&BuildFlag::Py_TRACE_REFS));
}
#[test]
fn build_flags_fixup_py37_debug() {
let mut build_flags = BuildFlags(HashSet::new());
build_flags.0.insert(BuildFlag::Py_DEBUG);
build_flags = build_flags.fixup(PythonVersion::PY37, PythonImplementation::CPython);
// On 3.7, Py_DEBUG implies Py_REF_DEBUG
assert!(build_flags.0.contains(&BuildFlag::Py_REF_DEBUG));
// 3.7 always has WITH_THREAD
assert!(build_flags.0.contains(&BuildFlag::WITH_THREAD));
}
#[test]
fn build_flags_fixup_pypy() {
let mut build_flags = BuildFlags(HashSet::new());
build_flags = build_flags.fixup(
PythonVersion { major: 3, minor: 6 },
PythonImplementation::PyPy,
);
// PyPy always has WITH_THREAD
assert!(build_flags.0.contains(&BuildFlag::WITH_THREAD));
}
}

33
pyo3-build-config/src/lib.rs
View File

@ -12,19 +12,40 @@
//! | `#[cfg(Py_LIMITED_API)]` | This marks code which is run when compiling with PyO3's `abi3` feature enabled. |
//! | `#[cfg(PyPy)]` | This marks code which is run when compiling for PyPy. |
//!
//! For examples of how to use these attributes, [see PyO3's guide](https://pyo3.rs/main/building_and_distribution/multiple_python_versions.html).
//! For examples of how to use these attributes, [see PyO3's guide](https://pyo3.rs/latest/building_and_distribution/multiple_python_versions.html).
#[allow(dead_code)] // TODO cover this using tests
#[doc(hidden)]
pub mod errors;
mod impl_;
#[doc(hidden)]
pub use crate::impl_::{InterpreterConfig, PythonImplementation, PythonVersion};
use once_cell::sync::OnceCell;
pub use impl_::{
find_interpreter, get_config_from_interpreter, InterpreterConfig, PythonImplementation,
PythonVersion,
};
// Used in PyO3's build.rs
#[doc(hidden)]
pub fn get() -> InterpreterConfig {
include!(concat!(env!("OUT_DIR"), "/pyo3-build-config.rs"))
pub use impl_::make_interpreter_config;
/// Reads the configuration written by PyO3's build.rs
///
/// Because this will never change in a given compilation run, this is cached in a `once_cell`.
#[doc(hidden)]
pub fn get() -> &'static InterpreterConfig {
static CONFIG: OnceCell<InterpreterConfig> = OnceCell::new();
CONFIG.get_or_init(|| {
let config_file = std::fs::File::open(PATH).expect("config file missing");
let reader = std::io::BufReader::new(config_file);
InterpreterConfig::from_reader(reader).expect("failed to parse config file")
})
}
/// Path where PyO3's build.rs will write configuration.
#[doc(hidden)]
pub const PATH: &str = concat!(env!("OUT_DIR"), "/pyo3-build-config.txt");
pub fn use_pyo3_cfgs() {
get().emit_pyo3_cfgs();
}

6
pyo3-macros-backend/Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "pyo3-macros-backend"
version = "0.13.2"
version = "0.14.1"
description = "Code generation for PyO3 package"
authors = ["PyO3 Project and Contributors <https://github.com/PyO3>"]
keywords = ["pyo3", "python", "cpython", "ffi"]
@ -16,11 +16,9 @@ edition = "2018"
[dependencies]
quote = { version = "1", default-features = false }
proc-macro2 = { version = "1", default-features = false }
pyo3-build-config = { path = "../pyo3-build-config", version = "0.14.1" }
[dependencies.syn]
version = "1"
default-features = false
features = ["derive", "parsing", "printing", "clone-impls", "full", "extra-traits"]
[build-dependencies]
pyo3-build-config = { path = "../pyo3-build-config", version = "=0.14.0-alpha.0" }

3
pyo3-macros-backend/build.rs
View File

@ -1,3 +0,0 @@
fn main() {
pyo3_build_config::use_pyo3_cfgs();
}

83
pyo3-macros-backend/src/attributes.rs
View File

@ -12,10 +12,13 @@ pub mod kw {
syn::custom_keyword!(annotation);
syn::custom_keyword!(attribute);
syn::custom_keyword!(from_py_with);
syn::custom_keyword!(get);
syn::custom_keyword!(item);
syn::custom_keyword!(pass_module);
syn::custom_keyword!(name);
syn::custom_keyword!(set);
syn::custom_keyword!(signature);
syn::custom_keyword!(text_signature);
syn::custom_keyword!(transparent);
}
@ -43,9 +46,24 @@ impl Parse for NameAttribute {
}
}
pub fn get_pyo3_attributes<T: Parse>(
attr: &syn::Attribute,
) -> Result<Option<Punctuated<T, Comma>>> {
#[derive(Clone, Debug, PartialEq)]
pub struct TextSignatureAttribute {
pub kw: kw::text_signature,
pub eq_token: Token![=],
pub lit: LitStr,
}
impl Parse for TextSignatureAttribute {
fn parse(input: ParseStream) -> Result<Self> {
Ok(TextSignatureAttribute {
kw: input.parse()?,
eq_token: input.parse()?,
lit: input.parse()?,
})
}
}
pub fn get_pyo3_options<T: Parse>(attr: &syn::Attribute) -> Result<Option<Punctuated<T, Comma>>> {
if is_attribute_ident(attr, "pyo3") {
attr.parse_args_with(Punctuated::parse_terminated).map(Some)
} else {
@ -83,6 +101,19 @@ pub fn take_attributes(
Ok(())
}
pub fn take_pyo3_options<T: Parse>(attrs: &mut Vec<syn::Attribute>) -> Result<Vec<T>> {
let mut out = Vec::new();
take_attributes(attrs, |attr| {
if let Some(options) = get_pyo3_options(attr)? {
out.extend(options.into_iter());
Ok(true)
} else {
Ok(false)
}
})?;
Ok(out)
}
pub fn get_deprecated_name_attribute(
attr: &syn::Attribute,
deprecations: &mut Deprecations,
@ -99,3 +130,49 @@ pub fn get_deprecated_name_attribute(
_ => Ok(None),
}
}
pub fn get_deprecated_text_signature_attribute(
attr: &syn::Attribute,
deprecations: &mut Deprecations,
) -> syn::Result<Option<TextSignatureAttribute>> {
match attr.parse_meta() {
Ok(syn::Meta::NameValue(syn::MetaNameValue {
path,
lit: syn::Lit::Str(lit),
..
})) if path.is_ident("text_signature") => {
let text_signature = TextSignatureAttribute {
kw: syn::parse_quote!(text_signature),
eq_token: syn::parse_quote!(=),
lit,
};
deprecations.push(
crate::deprecations::Deprecation::TextSignatureAttribute,
attr.span(),
);
Ok(Some(text_signature))
}
_ => Ok(None),
}
}
pub fn take_deprecated_text_signature_attribute(
attrs: &mut Vec<syn::Attribute>,
deprecations: &mut Deprecations,
) -> syn::Result<Option<TextSignatureAttribute>> {
let mut text_signature = None;
let mut attrs_out = Vec::with_capacity(attrs.len());
for attr in attrs.drain(..) {
if let Some(value) = get_deprecated_text_signature_attribute(&attr, deprecations)? {
ensure_spanned!(
text_signature.is_none(),
attr.span() => "text_signature attribute already specified previously"
);
text_signature = Some(value);
} else {
attrs_out.push(attr);
}
}
*attrs = attrs_out;
Ok(text_signature)
}

4
pyo3-macros-backend/src/deprecations.rs
View File

@ -4,6 +4,8 @@ use quote::{quote_spanned, ToTokens};
pub enum Deprecation {
NameAttribute,
PyfnNameArgument,
PyModuleNameArgument,
TextSignatureAttribute,
}
impl Deprecation {
@ -11,6 +13,8 @@ impl Deprecation {
let string = match self {
Deprecation::NameAttribute => "NAME_ATTRIBUTE",
Deprecation::PyfnNameArgument => "PYFN_NAME_ARGUMENT",
Deprecation::PyModuleNameArgument => "PYMODULE_NAME_ARGUMENT",
Deprecation::TextSignatureAttribute => "TEXT_SIGNATURE_ATTRIBUTE",
};
syn::Ident::new(string, span)
}

8
pyo3-macros-backend/src/from_pyobject.rs
View File

@ -1,4 +1,4 @@
use crate::attributes::{self, get_pyo3_attributes, FromPyWithAttribute};
use crate::attributes::{self, get_pyo3_options, FromPyWithAttribute};
use proc_macro2::TokenStream;
use quote::quote;
use syn::{
@ -195,7 +195,7 @@ impl<'a> Container<'a> {
/// Build derivation body for a struct.
fn build(&self) -> TokenStream {
match &self.ty {
ContainerType::StructNewtype(ident) => self.build_newtype_struct(Some(&ident)),
ContainerType::StructNewtype(ident) => self.build_newtype_struct(Some(ident)),
ContainerType::TupleNewtype => self.build_newtype_struct(None),
ContainerType::Tuple(len) => self.build_tuple_struct(*len),
ContainerType::Struct(tups) => self.build_struct(tups),
@ -353,7 +353,7 @@ impl ContainerOptions {
annotation: None,
};
for attr in attrs {
if let Some(pyo3_attrs) = get_pyo3_attributes(attr)? {
if let Some(pyo3_attrs) = get_pyo3_options(attr)? {
for pyo3_attr in pyo3_attrs {
match pyo3_attr {
ContainerPyO3Attribute::Transparent(kw) => {
@ -451,7 +451,7 @@ impl FieldPyO3Attributes {
let mut from_py_with = None;
for attr in attrs {
if let Some(pyo3_attrs) = get_pyo3_attributes(attr)? {
if let Some(pyo3_attrs) = get_pyo3_options(attr)? {
for pyo3_attr in pyo3_attrs {
match pyo3_attr {
FieldPyO3Attribute::Getter(field_getter) => {

6
pyo3-macros-backend/src/konst.rs
View File

@ -1,7 +1,7 @@
use crate::{
attributes::{
self, get_deprecated_name_attribute, get_pyo3_attributes, is_attribute_ident,
take_attributes, NameAttribute,
self, get_deprecated_name_attribute, get_pyo3_options, is_attribute_ident, take_attributes,
NameAttribute,
},
deprecations::Deprecations,
};
@ -69,7 +69,7 @@ impl ConstAttributes {
);
attributes.is_class_attr = true;
Ok(true)
} else if let Some(pyo3_attributes) = get_pyo3_attributes(attr)? {
} else if let Some(pyo3_attributes) = get_pyo3_options(attr)? {
for pyo3_attr in pyo3_attributes {
match pyo3_attr {
PyO3ConstAttribute::Name(name) => attributes.set_name(name)?,

3
pyo3-macros-backend/src/lib.rs
View File

@ -15,6 +15,7 @@ mod from_pyobject;
mod konst;
mod method;
mod module;
mod params;
mod proto_method;
mod pyclass;
mod pyfunction;
@ -23,7 +24,7 @@ mod pymethod;
mod pyproto;
pub use from_pyobject::build_derive_from_pyobject;
pub use module::{process_functions_in_module, py_init};
pub use module::{process_functions_in_module, py_init, PyModuleOptions};
pub use pyclass::{build_py_class, PyClassArgs};
pub use pyfunction::{build_py_function, PyFunctionOptions};
pub use pyimpl::{build_py_methods, PyClassMethodsType};

342
pyo3-macros-backend/src/method.rs
View File

@ -1,14 +1,17 @@
// Copyright (c) 2017-present PyO3 Project and Contributors
use crate::attributes::TextSignatureAttribute;
use crate::params::{accept_args_kwargs, impl_arg_params};
use crate::pyfunction::PyFunctionOptions;
use crate::pyfunction::{PyFunctionArgPyO3Attributes, PyFunctionSignature};
use crate::utils;
use crate::{deprecations::Deprecations, pyfunction::Argument};
use proc_macro2::TokenStream;
use proc_macro2::{Span, TokenStream};
use quote::ToTokens;
use quote::{quote, quote_spanned};
use syn::ext::IdentExt;
use syn::spanned::Spanned;
use syn::Result;
#[derive(Clone, PartialEq, Debug)]
pub struct FnArg<'a> {
@ -23,7 +26,7 @@ pub struct FnArg<'a> {
impl<'a> FnArg<'a> {
/// Transforms a rust fn arg parsed with syn into a method::FnArg
pub fn parse(arg: &'a mut syn::FnArg) -> syn::Result<Self> {
pub fn parse(arg: &'a mut syn::FnArg) -> Result<Self> {
match arg {
syn::FnArg::Receiver(recv) => {
bail_spanned!(recv.span() => "unexpected receiver")
@ -85,13 +88,44 @@ pub enum FnType {
FnNew,
FnClass,
FnStatic,
FnModule,
ClassAttribute,
}
impl FnType {
pub fn self_conversion(&self, cls: Option<&syn::Type>) -> TokenStream {
match self {
FnType::Getter(st) | FnType::Setter(st) | FnType::Fn(st) | FnType::FnCall(st) => {
st.receiver(cls.expect("no class given for Fn with a \"self\" receiver"))
}
FnType::FnNew | FnType::FnStatic | FnType::ClassAttribute => {
quote!()
}
FnType::FnClass => {
quote! {
let _slf = pyo3::types::PyType::from_type_ptr(_py, _slf as *mut pyo3::ffi::PyTypeObject);
}
}
FnType::FnModule => {
quote! {
let _slf = _py.from_borrowed_ptr::<pyo3::types::PyModule>(_slf);
}
}
}
}
pub fn self_arg(&self) -> TokenStream {
match self {
FnType::FnNew | FnType::FnStatic | FnType::ClassAttribute => quote!(),
_ => quote!(_slf,),
}
}
}
#[derive(Clone, Debug)]
pub enum SelfType {
Receiver { mutable: bool },
TryFromPyCell(proc_macro2::Span),
TryFromPyCell(Span),
}
impl SelfType {
@ -122,6 +156,42 @@ impl SelfType {
}
}
/// Determines which CPython calling convention a given FnSpec uses.
#[derive(Clone, Debug)]
pub enum CallingConvention {
Noargs, // METH_NOARGS
Varargs, // METH_VARARGS | METH_KEYWORDS
Fastcall, // METH_FASTCALL | METH_KEYWORDS (Py3.7+ and !abi3)
TpNew, // special convention for tp_new
}
impl CallingConvention {
/// Determine default calling convention from an argument signature.
///
/// Different other slots (tp_call, tp_new) can have other requirements
/// and are set manually (see `parse_fn_type` below).
pub fn from_args(args: &[FnArg<'_>], attrs: &[Argument]) -> Self {
let (_, accept_kwargs) = accept_args_kwargs(attrs);
if args.is_empty() {
Self::Noargs
} else if accept_kwargs {
// for functions that accept **kwargs, always prefer varargs
Self::Varargs
} else if can_use_fastcall() {
Self::Fastcall
} else {
Self::Varargs
}
}
}
fn can_use_fastcall() -> bool {
const PY37: pyo3_build_config::PythonVersion =
pyo3_build_config::PythonVersion { major: 3, minor: 7 };
let config = pyo3_build_config::get();
config.version >= PY37 && !config.abi3
}
pub struct FnSpec<'a> {
pub tp: FnType,
// Rust function name
@ -134,6 +204,7 @@ pub struct FnSpec<'a> {
pub output: syn::Type,
pub doc: syn::LitStr,
pub deprecations: Deprecations,
pub convention: CallingConvention,
}
pub fn get_return_info(output: &syn::ReturnType) -> syn::Type {
@ -143,7 +214,7 @@ pub fn get_return_info(output: &syn::ReturnType) -> syn::Type {
}
}
pub fn parse_method_receiver(arg: &syn::FnArg) -> syn::Result<SelfType> {
pub fn parse_method_receiver(arg: &syn::FnArg) -> Result<SelfType> {
match arg {
syn::FnArg::Receiver(recv) => Ok(SelfType::Receiver {
mutable: recv.mutability.is_some(),
@ -163,7 +234,7 @@ impl<'a> FnSpec<'a> {
sig: &'a mut syn::Signature,
meth_attrs: &mut Vec<syn::Attribute>,
options: PyFunctionOptions,
) -> syn::Result<FnSpec<'a>> {
) -> Result<FnSpec<'a>> {
let MethodAttributes {
ty: fn_type_attr,
args: fn_attrs,
@ -175,42 +246,53 @@ impl<'a> FnSpec<'a> {
if let Some(name) = &python_name {
bail_spanned!(name.span() => "`name` not allowed with `#[new]`");
}
python_name = Some(syn::Ident::new("__new__", proc_macro2::Span::call_site()))
python_name = Some(syn::Ident::new("__new__", Span::call_site()))
}
Some(MethodTypeAttribute::Call) => {
if let Some(name) = &python_name {
bail_spanned!(name.span() => "`name` not allowed with `#[call]`");
}
python_name = Some(syn::Ident::new("__call__", proc_macro2::Span::call_site()))
python_name = Some(syn::Ident::new("__call__", Span::call_site()))
}
_ => {}
}
let (fn_type, skip_first_arg) = Self::parse_fn_type(sig, fn_type_attr, &mut python_name)?;
let (fn_type, skip_first_arg, fixed_convention) =
Self::parse_fn_type(sig, fn_type_attr, &mut python_name)?;
Self::ensure_text_signature_on_valid_method(&fn_type, options.text_signature.as_ref())?;
let name = &sig.ident;
let ty = get_return_info(&sig.output);
let python_name = python_name.as_ref().unwrap_or(name).unraw();
let text_signature = Self::parse_text_signature(meth_attrs, &fn_type, &python_name)?;
let doc = utils::get_doc(&meth_attrs, text_signature, true)?;
let doc = utils::get_doc(
meth_attrs,
options
.text_signature
.as_ref()
.map(|attr| (&python_name, attr)),
)?;
let arguments = if skip_first_arg {
let arguments: Vec<_> = if skip_first_arg {
sig.inputs
.iter_mut()
.skip(1)
.map(FnArg::parse)
.collect::<syn::Result<_>>()?
.collect::<Result<_>>()?
} else {
sig.inputs
.iter_mut()
.map(FnArg::parse)
.collect::<syn::Result<_>>()?
.collect::<Result<_>>()?
};
let convention =
fixed_convention.unwrap_or_else(|| CallingConvention::from_args(&arguments, &fn_attrs));
Ok(FnSpec {
tp: fn_type,
name,
convention,
python_name,
attrs: fn_attrs,
args: arguments,
@ -220,48 +302,38 @@ impl<'a> FnSpec<'a> {
})
}
pub fn null_terminated_python_name(&self) -> TokenStream {
let name = format!("{}\0", self.python_name);
quote!({#name})
pub fn null_terminated_python_name(&self) -> syn::LitStr {
syn::LitStr::new(&format!("{}\0", self.python_name), self.python_name.span())
}
fn parse_text_signature(
meth_attrs: &mut Vec<syn::Attribute>,
fn ensure_text_signature_on_valid_method(
fn_type: &FnType,
python_name: &syn::Ident,
) -> syn::Result<Option<syn::LitStr>> {
let mut parse_erroneous_text_signature = |error_msg: &str| {
// try to parse anyway to give better error messages
if let Some(text_signature) =
utils::parse_text_signature_attrs(meth_attrs, &python_name)?
{
bail_spanned!(text_signature.span() => error_msg)
} else {
Ok(None)
text_signature: Option<&TextSignatureAttribute>,
) -> syn::Result<()> {
if let Some(text_signature) = text_signature {
match &fn_type {
FnType::FnNew => bail_spanned!(
text_signature.kw.span() =>
"text_signature not allowed on __new__; if you want to add a signature on \
__new__, put it on the struct definition instead"
),
FnType::FnCall(_)
| FnType::Getter(_)
| FnType::Setter(_)
| FnType::ClassAttribute => bail_spanned!(
text_signature.kw.span() => "text_signature not allowed with this method type"
),
_ => {}
}
};
let text_signature = match &fn_type {
FnType::Fn(_) | FnType::FnClass | FnType::FnStatic => {
utils::parse_text_signature_attrs(&mut *meth_attrs, &python_name)?
}
FnType::FnNew => parse_erroneous_text_signature(
"text_signature not allowed on __new__; if you want to add a signature on \
__new__, put it on the struct definition instead",
)?,
FnType::FnCall(_) | FnType::Getter(_) | FnType::Setter(_) | FnType::ClassAttribute => {
parse_erroneous_text_signature("text_signature not allowed with this method type")?
}
};
Ok(text_signature)
}
Ok(())
}
fn parse_fn_type(
sig: &syn::Signature,
fn_type_attr: Option<MethodTypeAttribute>,
python_name: &mut Option<syn::Ident>,
) -> syn::Result<(FnType, bool)> {
) -> Result<(FnType, bool, Option<CallingConvention>)> {
let name = &sig.ident;
let parse_receiver = |msg: &'static str| {
let first_arg = sig
@ -282,20 +354,23 @@ impl<'a> FnSpec<'a> {
}
};
let (fn_type, skip_first_arg) = match fn_type_attr {
Some(MethodTypeAttribute::StaticMethod) => (FnType::FnStatic, false),
let (fn_type, skip_first_arg, fixed_convention) = match fn_type_attr {
Some(MethodTypeAttribute::StaticMethod) => (FnType::FnStatic, false, None),
Some(MethodTypeAttribute::ClassAttribute) => {
ensure_spanned!(
sig.inputs.is_empty(),
sig.inputs.span() => "class attribute methods cannot take arguments"
);
(FnType::ClassAttribute, false)
(FnType::ClassAttribute, false, None)
}
Some(MethodTypeAttribute::New) => (FnType::FnNew, false),
Some(MethodTypeAttribute::ClassMethod) => (FnType::FnClass, true),
Some(MethodTypeAttribute::New) => {
(FnType::FnNew, false, Some(CallingConvention::TpNew))
}
Some(MethodTypeAttribute::ClassMethod) => (FnType::FnClass, true, None),
Some(MethodTypeAttribute::Call) => (
FnType::FnCall(parse_receiver("expected receiver for #[call]")?),
true,
Some(CallingConvention::Varargs),
),
Some(MethodTypeAttribute::Getter) => {
// Strip off "get_" prefix if needed
@ -306,6 +381,7 @@ impl<'a> FnSpec<'a> {
(
FnType::Getter(parse_receiver("expected receiver for #[getter]")?),
true,
None,
)
}
Some(MethodTypeAttribute::Setter) => {
@ -317,6 +393,7 @@ impl<'a> FnSpec<'a> {
(
FnType::Setter(parse_receiver("expected receiver for #[setter]")?),
true,
None,
)
}
None => (
@ -324,27 +401,10 @@ impl<'a> FnSpec<'a> {
"static method needs #[staticmethod] attribute",
)?),
true,
None,
),
};
Ok((fn_type, skip_first_arg))
}
pub fn is_args(&self, name: &syn::Ident) -> bool {
for s in self.attrs.iter() {
if let Argument::VarArgs(path) = s {
return path.is_ident(name);
}
}
false
}
pub fn is_kwargs(&self, name: &syn::Ident) -> bool {
for s in self.attrs.iter() {
if let Argument::KeywordArgs(path) = s {
return path.is_ident(name);
}
}
false
Ok((fn_type, skip_first_arg, fixed_convention))
}
pub fn default_value(&self, name: &syn::Ident) -> Option<TokenStream> {
@ -353,7 +413,7 @@ impl<'a> FnSpec<'a> {
Argument::Arg(path, opt) | Argument::Kwarg(path, opt) => {
if path.is_ident(name) {
if let Some(val) = opt {
let i: syn::Expr = syn::parse_str(&val).unwrap();
let i: syn::Expr = syn::parse_str(val).unwrap();
return Some(i.into_token_stream());
}
}
@ -374,6 +434,146 @@ impl<'a> FnSpec<'a> {
}
false
}
/// Return a C wrapper function for this signature.
pub fn get_wrapper_function(
&self,
ident: &proc_macro2::Ident,
cls: Option<&syn::Type>,
) -> Result<TokenStream> {
let deprecations = &self.deprecations;
let self_conversion = self.tp.self_conversion(cls);
let self_arg = self.tp.self_arg();
let arg_names = (0..self.args.len())
.map(|pos| syn::Ident::new(&format!("arg{}", pos), Span::call_site()))
.collect::<Vec<_>>();
let py = syn::Ident::new("_py", Span::call_site());
let func_name = &self.name;
let rust_name = if let Some(cls) = cls {
quote!(#cls::#func_name)
} else {
quote!(#func_name)
};
let rust_call =
quote! { pyo3::callback::convert(#py, #rust_name(#self_arg #(#arg_names),*)) };
Ok(match self.convention {
CallingConvention::Noargs => {
quote! {
unsafe extern "C" fn #ident (
_slf: *mut pyo3::ffi::PyObject,
_args: *mut pyo3::ffi::PyObject,
) -> *mut pyo3::ffi::PyObject
{
#deprecations
pyo3::callback::handle_panic(|#py| {
#self_conversion
#rust_call
})
}
}
}
CallingConvention::Fastcall => {
let arg_convert_and_rust_call = impl_arg_params(self, cls, rust_call, &py, true)?;
quote! {
unsafe extern "C" fn #ident (
_slf: *mut pyo3::ffi::PyObject,
_args: *const *mut pyo3::ffi::PyObject,
_nargs: pyo3::ffi::Py_ssize_t,
_kwnames: *mut pyo3::ffi::PyObject) -> *mut pyo3::ffi::PyObject
{
#deprecations
pyo3::callback::handle_panic(|#py| {
#self_conversion
let _kwnames: Option<&pyo3::types::PyTuple> = #py.from_borrowed_ptr_or_opt(_kwnames);
// Safety: &PyAny has the same memory layout as `*mut ffi::PyObject`
let _args = _args as *const &pyo3::PyAny;
let _kwargs = if let Some(kwnames) = _kwnames {
std::slice::from_raw_parts(_args.offset(_nargs), kwnames.len())
} else {
&[]
};
let _args = std::slice::from_raw_parts(_args, _nargs as usize);
#arg_convert_and_rust_call
})
}
}
}
CallingConvention::Varargs => {
let arg_convert_and_rust_call = impl_arg_params(self, cls, rust_call, &py, false)?;
quote! {
unsafe extern "C" fn #ident (
_slf: *mut pyo3::ffi::PyObject,
_args: *mut pyo3::ffi::PyObject,
_kwargs: *mut pyo3::ffi::PyObject) -> *mut pyo3::ffi::PyObject
{
#deprecations
pyo3::callback::handle_panic(|#py| {
#self_conversion
let _args = #py.from_borrowed_ptr::<pyo3::types::PyTuple>(_args);
let _kwargs: Option<&pyo3::types::PyDict> = #py.from_borrowed_ptr_or_opt(_kwargs);
#arg_convert_and_rust_call
})
}
}
}
CallingConvention::TpNew => {
let rust_call = quote! { #rust_name(#(#arg_names),*) };
let arg_convert_and_rust_call = impl_arg_params(self, cls, rust_call, &py, false)?;
quote! {
unsafe extern "C" fn #ident (
subtype: *mut pyo3::ffi::PyTypeObject,
_args: *mut pyo3::ffi::PyObject,
_kwargs: *mut pyo3::ffi::PyObject) -> *mut pyo3::ffi::PyObject
{
#deprecations
use pyo3::callback::IntoPyCallbackOutput;
pyo3::callback::handle_panic(|#py| {
let _args = #py.from_borrowed_ptr::<pyo3::types::PyTuple>(_args);
let _kwargs: Option<&pyo3::types::PyDict> = #py.from_borrowed_ptr_or_opt(_kwargs);
let result = #arg_convert_and_rust_call;
let initializer: pyo3::PyClassInitializer::<#cls> = result.convert(#py)?;
let cell = initializer.create_cell_from_subtype(#py, subtype)?;
Ok(cell as *mut pyo3::ffi::PyObject)
})
}
}
}
})
}
/// Return a `PyMethodDef` constructor for this function, matching the selected
/// calling convention.
pub fn get_methoddef(&self, wrapper: impl ToTokens) -> TokenStream {
let python_name = self.null_terminated_python_name();
let doc = &self.doc;
match self.convention {
CallingConvention::Noargs => quote! {
pyo3::class::methods::PyMethodDef::noargs(
#python_name,
pyo3::class::methods::PyCFunction(#wrapper),
#doc,
)
},
CallingConvention::Fastcall => quote! {
pyo3::class::methods::PyMethodDef::fastcall_cfunction_with_keywords(
#python_name,
pyo3::class::methods::PyCFunctionFastWithKeywords(#wrapper),
#doc,
)
},
CallingConvention::Varargs => quote! {
pyo3::class::methods::PyMethodDef::cfunction_with_keywords(
#python_name,
pyo3::class::methods::PyCFunctionWithKeywords(#wrapper),
#doc,
)
},
CallingConvention::TpNew => unreachable!("tp_new cannot get a methoddef"),
}
}
}
#[derive(Clone, PartialEq, Debug)]
@ -386,7 +586,7 @@ struct MethodAttributes {
fn parse_method_attributes(
attrs: &mut Vec<syn::Attribute>,
mut python_name: Option<syn::Ident>,
) -> syn::Result<MethodAttributes> {
) -> Result<MethodAttributes> {
let mut new_attrs = Vec::new();
let mut args = Vec::new();
let mut ty: Option<MethodTypeAttribute> = None;

96
pyo3-macros-backend/src/module.rs
View File

@ -1,19 +1,72 @@
// Copyright (c) 2017-present PyO3 Project and Contributors
//! Code generation for the function that initializes a python module and adds classes and function.
use crate::pyfunction::{impl_wrap_pyfunction, PyFunctionOptions};
use crate::{
attributes::{self, take_pyo3_options},
deprecations::Deprecations,
pyfunction::{impl_wrap_pyfunction, PyFunctionOptions},
};
use crate::{
attributes::{is_attribute_ident, take_attributes, NameAttribute},
deprecations::Deprecation,
};
use proc_macro2::{Span, TokenStream};
use quote::quote;
use syn::{parse::Parse, spanned::Spanned, token::Comma, Ident, Path};
use syn::{
ext::IdentExt,
parse::{Parse, ParseStream},
spanned::Spanned,
token::Comma,
Ident, Path, Result,
};
pub struct PyModuleOptions {
name: Option<syn::Ident>,
deprecations: Deprecations,
}
impl PyModuleOptions {
pub fn from_pymodule_arg_and_attrs(
deprecated_pymodule_name_arg: Option<syn::Ident>,
attrs: &mut Vec<syn::Attribute>,
) -> Result<Self> {
let mut deprecations = Deprecations::new();
if let Some(name) = &deprecated_pymodule_name_arg {
deprecations.push(Deprecation::PyModuleNameArgument, name.span());
}
let mut options: PyModuleOptions = PyModuleOptions {
name: deprecated_pymodule_name_arg,
deprecations,
};
for option in take_pyo3_options(attrs)? {
match option {
PyModulePyO3Option::Name(name) => options.set_name(name.0)?,
}
}
Ok(options)
}
fn set_name(&mut self, name: syn::Ident) -> Result<()> {
ensure_spanned!(
self.name.is_none(),
name.span() => "`name` may only be specified once"
);
self.name = Some(name);
Ok(())
}
}
/// Generates the function that is called by the python interpreter to initialize the native
/// module
pub fn py_init(fnname: &Ident, name: &Ident, doc: syn::LitStr) -> TokenStream {
pub fn py_init(fnname: &Ident, options: PyModuleOptions, doc: syn::LitStr) -> TokenStream {
let name = options.name.unwrap_or_else(|| fnname.unraw());
let deprecations = options.deprecations;
let cb_name = Ident::new(&format!("PyInit_{}", name), Span::call_site());
assert!(doc.value().ends_with('\0'));
quote! {
#[no_mangle]
@ -23,9 +76,11 @@ pub fn py_init(fnname: &Ident, name: &Ident, doc: syn::LitStr) -> TokenStream {
pub unsafe extern "C" fn #cb_name() -> *mut pyo3::ffi::PyObject {
use pyo3::derive_utils::ModuleDef;
static NAME: &str = concat!(stringify!(#name), "\0");
static DOC: &str = concat!(#doc, "\0");
static DOC: &str = #doc;
static MODULE_DEF: ModuleDef = unsafe { ModuleDef::new(NAME, DOC) };
#deprecations
pyo3::callback::handle_panic(|_py| { MODULE_DEF.make_module(_py, #fnname) })
}
}
@ -35,21 +90,19 @@ pub fn py_init(fnname: &Ident, name: &Ident, doc: syn::LitStr) -> TokenStream {
pub fn process_functions_in_module(func: &mut syn::ItemFn) -> syn::Result<()> {
let mut stmts: Vec<syn::Stmt> = Vec::new();
for stmt in func.block.stmts.iter_mut() {
if let syn::Stmt::Item(syn::Item::Fn(func)) = stmt {
for mut stmt in func.block.stmts.drain(..) {
if let syn::Stmt::Item(syn::Item::Fn(func)) = &mut stmt {
if let Some(pyfn_args) = get_pyfn_attr(&mut func.attrs)? {
let module_name = pyfn_args.modname;
let (ident, wrapped_function) = impl_wrap_pyfunction(func, pyfn_args.options)?;
let item: syn::ItemFn = syn::parse_quote! {
fn block_wrapper() {
#wrapped_function
#module_name.add_function(#ident(#module_name)?)?;
}
let statements: Vec<syn::Stmt> = syn::parse_quote! {
#wrapped_function
#module_name.add_function(#ident(#module_name)?)?;
};
stmts.extend(item.block.stmts.into_iter());
stmts.extend(statements);
}
};
stmts.push(stmt.clone());
stmts.push(stmt);
}
func.block.stmts = stmts;
@ -114,8 +167,23 @@ fn get_pyfn_attr(attrs: &mut Vec<syn::Attribute>) -> syn::Result<Option<PyFnArgs
})?;
if let Some(pyfn_args) = &mut pyfn_args {
pyfn_args.options.take_pyo3_attributes(attrs)?;
pyfn_args.options.take_pyo3_options(attrs)?;
}
Ok(pyfn_args)
}
enum PyModulePyO3Option {
Name(NameAttribute),
}
impl Parse for PyModulePyO3Option {
fn parse(input: ParseStream) -> Result<Self> {
let lookahead = input.lookahead1();
if lookahead.peek(attributes::kw::name) {
input.parse().map(PyModulePyO3Option::Name)
} else {
Err(lookahead.error())
}
}
}

312
pyo3-macros-backend/src/params.rs Normal file
View File

@ -0,0 +1,312 @@
// Copyright (c) 2017-present PyO3 Project and Contributors
use crate::{
attributes::FromPyWithAttribute,
method::{FnArg, FnSpec},
pyfunction::Argument,
utils::unwrap_ty_group,
};
use proc_macro2::{Span, TokenStream};
use quote::{quote, quote_spanned};
use syn::ext::IdentExt;
use syn::spanned::Spanned;
use syn::Result;
/// Determine if the function gets passed a *args tuple or **kwargs dict.
pub fn accept_args_kwargs(attrs: &[Argument]) -> (bool, bool) {
let (mut accept_args, mut accept_kwargs) = (false, false);
for s in attrs {
match s {
Argument::VarArgs(_) => accept_args = true,
Argument::KeywordArgs(_) => accept_kwargs = true,
_ => continue,
}
}
(accept_args, accept_kwargs)
}
/// Return true if the argument list is simply (*args, **kwds).
pub fn is_forwarded_args(args: &[FnArg<'_>], attrs: &[Argument]) -> bool {
args.len() == 2 && is_args(attrs, args[0].name) && is_kwargs(attrs, args[1].name)
}
fn is_args(attrs: &[Argument], name: &syn::Ident) -> bool {
for s in attrs.iter() {
if let Argument::VarArgs(path) = s {
return path.is_ident(name);
}
}
false
}
fn is_kwargs(attrs: &[Argument], name: &syn::Ident) -> bool {
for s in attrs.iter() {
if let Argument::KeywordArgs(path) = s {
return path.is_ident(name);
}
}
false
}
pub fn impl_arg_params(
spec: &FnSpec<'_>,
self_: Option<&syn::Type>,
body: TokenStream,
py: &syn::Ident,
fastcall: bool,
) -> Result<TokenStream> {
if spec.args.is_empty() {
return Ok(body);
}
let args_array = syn::Ident::new("output", Span::call_site());
if !fastcall && is_forwarded_args(&spec.args, &spec.attrs) {
// In the varargs convention, we can just pass though if the signature
// is (*args, **kwds).
let mut arg_convert = vec![];
for (i, arg) in spec.args.iter().enumerate() {
arg_convert.push(impl_arg_param(arg, spec, i, None, &mut 0, py, &args_array)?);
}
return Ok(quote! {{
let _args = Some(_args);
#(#arg_convert)*
#body
}});
};
let mut positional_parameter_names = Vec::new();
let mut required_positional_parameters = 0usize;
let mut keyword_only_parameters = Vec::new();
for arg in spec.args.iter() {
if arg.py || is_args(&spec.attrs, arg.name) || is_kwargs(&spec.attrs, arg.name) {
continue;
}
let name = arg.name.unraw().to_string();
let kwonly = spec.is_kw_only(arg.name);
let required = !(arg.optional.is_some() || spec.default_value(arg.name).is_some());
if kwonly {
keyword_only_parameters.push(quote! {
pyo3::derive_utils::KeywordOnlyParameterDescription {
name: #name,
required: #required,
}
});
} else {
if required {
required_positional_parameters += 1;
}
positional_parameter_names.push(name);
}
}
let num_params = positional_parameter_names.len() + keyword_only_parameters.len();
let mut param_conversion = Vec::new();
let mut option_pos = 0;
for (idx, arg) in spec.args.iter().enumerate() {
param_conversion.push(impl_arg_param(
arg,
spec,
idx,
self_,
&mut option_pos,
py,
&args_array,
)?);
}
let (accept_args, accept_kwargs) = accept_args_kwargs(&spec.attrs);
let cls_name = if let Some(cls) = self_ {
quote! { Some(<#cls as pyo3::type_object::PyTypeInfo>::NAME) }
} else {
quote! { None }
};
let python_name = &spec.python_name;
let (args_to_extract, kwargs_to_extract) = if fastcall {
// _args is a &[&PyAny], _kwnames is a Option<&PyTuple> containing the
// keyword names of the keyword args in _kwargs
(
// need copied() for &&PyAny -> &PyAny
quote! { _args.iter().copied() },
quote! { _kwnames.map(|kwnames| {
kwnames.as_slice().iter().copied().zip(_kwargs.iter().copied())
}) },
)
} else {
// _args is a &PyTuple, _kwargs is an Option<&PyDict>
(
quote! { _args.iter() },
quote! { _kwargs.map(|dict| dict.iter()) },
)
};
// create array of arguments, and then parse
Ok(quote! {{
const DESCRIPTION: pyo3::derive_utils::FunctionDescription = pyo3::derive_utils::FunctionDescription {
cls_name: #cls_name,
func_name: stringify!(#python_name),
positional_parameter_names: &[#(#positional_parameter_names),*],
// TODO: https://github.com/PyO3/pyo3/issues/1439 - support specifying these
positional_only_parameters: 0,
required_positional_parameters: #required_positional_parameters,
keyword_only_parameters: &[#(#keyword_only_parameters),*],
accept_varargs: #accept_args,
accept_varkeywords: #accept_kwargs,
};
let mut #args_array = [None; #num_params];
let (_args, _kwargs) = DESCRIPTION.extract_arguments(
#py,
#args_to_extract,
#kwargs_to_extract,
&mut #args_array
)?;
#(#param_conversion)*
#body
}})
}
/// Re option_pos: The option slice doesn't contain the py: Python argument, so the argument
/// index and the index in option diverge when using py: Python
fn impl_arg_param(
arg: &FnArg<'_>,
spec: &FnSpec<'_>,
idx: usize,
self_: Option<&syn::Type>,
option_pos: &mut usize,
py: &syn::Ident,
args_array: &syn::Ident,
) -> Result<TokenStream> {
// Use this macro inside this function, to ensure that all code generated here is associated
// with the function argument
macro_rules! quote_arg_span {
($($tokens:tt)*) => { quote_spanned!(arg.ty.span() => $($tokens)*) }
}
let arg_name = syn::Ident::new(&format!("arg{}", idx), Span::call_site());
if arg.py {
return Ok(quote_arg_span! { let #arg_name = #py; });
}
let ty = arg.ty;
let name = arg.name;
let transform_error = quote! {
|e| pyo3::derive_utils::argument_extraction_error(#py, stringify!(#name), e)
};
if is_args(&spec.attrs, name) {
ensure_spanned!(
arg.optional.is_none(),
arg.name.span() => "args cannot be optional"
);
return Ok(quote_arg_span! {
let #arg_name = _args.unwrap().extract().map_err(#transform_error)?;
});
} else if is_kwargs(&spec.attrs, name) {
ensure_spanned!(
arg.optional.is_some(),
arg.name.span() => "kwargs must be Option<_>"
);
return Ok(quote_arg_span! {
let #arg_name = _kwargs.map(|kwargs| kwargs.extract())
.transpose()
.map_err(#transform_error)?;
});
}
let arg_value = quote_arg_span!(#args_array[#option_pos]);
*option_pos += 1;
let extract = if let Some(FromPyWithAttribute(expr_path)) = &arg.attrs.from_py_with {
quote_arg_span! { #expr_path(_obj).map_err(#transform_error) }
} else {
quote_arg_span! { _obj.extract().map_err(#transform_error) }
};
let arg_value_or_default = match (spec.default_value(name), arg.optional.is_some()) {
(Some(default), true) if default.to_string() != "None" => {
quote_arg_span! { #arg_value.map_or_else(|| Ok(Some(#default)), |_obj| #extract)? }
}
(Some(default), _) => {
quote_arg_span! { #arg_value.map_or_else(|| Ok(#default), |_obj| #extract)? }
}
(None, true) => quote_arg_span! { #arg_value.map_or(Ok(None), |_obj| #extract)? },
(None, false) => {
quote_arg_span! {
{
let _obj = #arg_value.expect("Failed to extract required method argument");
#extract?
}
}
}
};
return if let syn::Type::Reference(tref) = unwrap_ty_group(arg.optional.unwrap_or(ty)) {
let (tref, mut_) = preprocess_tref(tref, self_);
let (target_ty, borrow_tmp) = if arg.optional.is_some() {
// Get Option<&T> from Option<PyRef<T>>
(
quote_arg_span! { Option<<#tref as pyo3::derive_utils::ExtractExt<'_>>::Target> },
if mut_.is_some() {
quote_arg_span! { _tmp.as_deref_mut() }
} else {
quote_arg_span! { _tmp.as_deref() }
},
)
} else {
// Get &T from PyRef<T>
(
quote_arg_span! { <#tref as pyo3::derive_utils::ExtractExt<'_>>::Target },
quote_arg_span! { &#mut_ *_tmp },
)
};
Ok(quote_arg_span! {
let #mut_ _tmp: #target_ty = #arg_value_or_default;
let #arg_name = #borrow_tmp;
})
} else {
Ok(quote_arg_span! {
let #arg_name = #arg_value_or_default;
})
};
/// Replace `Self`, remove lifetime and get mutability from the type
fn preprocess_tref(
tref: &syn::TypeReference,
self_: Option<&syn::Type>,
) -> (syn::TypeReference, Option<syn::token::Mut>) {
let mut tref = tref.to_owned();
if let Some(syn::Type::Path(tpath)) = self_ {
replace_self(&mut tref, &tpath.path);
}
tref.lifetime = None;
let mut_ = tref.mutability;
(tref, mut_)
}
/// Replace `Self` with the exact type name since it is used out of the impl block
fn replace_self(tref: &mut syn::TypeReference, self_path: &syn::Path) {
match &mut *tref.elem {
syn::Type::Reference(tref_inner) => replace_self(tref_inner, self_path),
syn::Type::Path(tpath) => {
if let Some(ident) = tpath.path.get_ident() {
if ident == "Self" {
tpath.path = self_path.to_owned();
}
}
}
_ => {}
}
}
}

400
pyo3-macros-backend/src/pyclass.rs
View File

@ -1,15 +1,19 @@
// Copyright (c) 2017-present PyO3 Project and Contributors
use crate::method::{FnType, SelfType};
use crate::attributes::{
self, take_deprecated_text_signature_attribute, take_pyo3_options, NameAttribute,
TextSignatureAttribute,
};
use crate::deprecations::Deprecations;
use crate::pyimpl::PyClassMethodsType;
use crate::pymethod::{impl_py_getter_def, impl_py_setter_def, PropertyType};
use crate::utils;
use crate::utils::{self, unwrap_group};
use proc_macro2::{Span, TokenStream};
use quote::quote;
use syn::ext::IdentExt;
use syn::parse::{Parse, ParseStream};
use syn::punctuated::Punctuated;
use syn::{parse_quote, spanned::Spanned, Expr, Token};
use syn::{parse_quote, spanned::Spanned, Expr, Result, Token};
/// The parsed arguments of the pyclass macro
pub struct PyClassArgs {
@ -26,7 +30,7 @@ pub struct PyClassArgs {
}
impl Parse for PyClassArgs {
fn parse(input: ParseStream) -> syn::parse::Result<Self> {
fn parse(input: ParseStream) -> Result<Self> {
let mut slf = PyClassArgs::default();
let vars = Punctuated::<Expr, Token![,]>::parse_terminated(input)?;
@ -57,7 +61,7 @@ impl Default for PyClassArgs {
impl PyClassArgs {
/// Adda single expression from the comma separated list in the attribute, which is
/// either a single word or an assignment expression
fn add_expr(&mut self, expr: &Expr) -> syn::parse::Result<()> {
fn add_expr(&mut self, expr: &Expr) -> Result<()> {
match expr {
syn::Expr::Path(exp) if exp.path.segments.len() == 1 => self.add_path(exp),
syn::Expr::Assign(assign) => self.add_assign(assign),
@ -89,7 +93,7 @@ impl PyClassArgs {
// We allow arbitrary expressions here so you can e.g. use `8*64`
self.freelist = Some(syn::Expr::clone(right));
}
"name" => match &**right {
"name" => match unwrap_group(&**right) {
syn::Expr::Lit(syn::ExprLit {
lit: syn::Lit::Str(lit),
..
@ -110,7 +114,7 @@ impl PyClassArgs {
}
_ => expected!("type name (e.g. \"Name\")"),
},
"extends" => match &**right {
"extends" => match unwrap_group(&**right) {
syn::Expr::Path(exp) => {
self.base = syn::TypePath {
path: exp.path.clone(),
@ -120,7 +124,7 @@ impl PyClassArgs {
}
_ => expected!("type path (e.g., my_mod::BaseClass)"),
},
"module" => match &**right {
"module" => match unwrap_group(&**right) {
syn::Expr::Lit(syn::ExprLit {
lit: syn::Lit::Str(lit),
..
@ -162,73 +166,174 @@ impl PyClassArgs {
}
}
#[derive(Default)]
pub struct PyClassPyO3Options {
pub text_signature: Option<TextSignatureAttribute>,
pub deprecations: Deprecations,
}
enum PyClassPyO3Option {
TextSignature(TextSignatureAttribute),
}
impl Parse for PyClassPyO3Option {
fn parse(input: ParseStream) -> Result<Self> {
let lookahead = input.lookahead1();
if lookahead.peek(attributes::kw::text_signature) {
input.parse().map(PyClassPyO3Option::TextSignature)
} else {
Err(lookahead.error())
}
}
}
impl PyClassPyO3Options {
pub fn take_pyo3_options(attrs: &mut Vec<syn::Attribute>) -> syn::Result<Self> {
let mut options: PyClassPyO3Options = Default::default();
for option in take_pyo3_options(attrs)? {
match option {
PyClassPyO3Option::TextSignature(text_signature) => {
options.set_text_signature(text_signature)?;
}
}
}
Ok(options)
}
pub fn set_text_signature(
&mut self,
text_signature: TextSignatureAttribute,
) -> syn::Result<()> {
ensure_spanned!(
self.text_signature.is_none(),
text_signature.kw.span() => "`text_signature` may only be specified once"
);
self.text_signature = Some(text_signature);
Ok(())
}
}
pub fn build_py_class(
class: &mut syn::ItemStruct,
attr: &PyClassArgs,
args: &PyClassArgs,
methods_type: PyClassMethodsType,
) -> syn::Result<TokenStream> {
let text_signature = utils::parse_text_signature_attrs(
&mut class.attrs,
&get_class_python_name(&class.ident, attr),
let mut options = PyClassPyO3Options::take_pyo3_options(&mut class.attrs)?;
if let Some(text_signature) =
take_deprecated_text_signature_attribute(&mut class.attrs, &mut options.deprecations)?
{
options.set_text_signature(text_signature)?;
}
let doc = utils::get_doc(
&class.attrs,
options
.text_signature
.as_ref()
.map(|attr| (get_class_python_name(&class.ident, args), attr)),
)?;
let doc = utils::get_doc(&class.attrs, text_signature, true)?;
let mut descriptors = Vec::new();
ensure_spanned!(
class.generics.params.is_empty(),
class.generics.span() => "#[pyclass] cannot have generic parameters"
);
match &mut class.fields {
syn::Fields::Named(fields) => {
for field in fields.named.iter_mut() {
let field_descs = parse_descriptors(field)?;
if !field_descs.is_empty() {
descriptors.push((field.clone(), field_descs));
}
}
let field_options = match &mut class.fields {
syn::Fields::Named(fields) => fields
.named
.iter_mut()
.map(|field| {
FieldPyO3Options::take_pyo3_options(&mut field.attrs)
.map(move |options| (&*field, options))
})
.collect::<Result<_>>()?,
syn::Fields::Unnamed(fields) => fields
.unnamed
.iter_mut()
.map(|field| {
FieldPyO3Options::take_pyo3_options(&mut field.attrs)
.map(move |options| (&*field, options))
})
.collect::<Result<_>>()?,
syn::Fields::Unit => {
// No fields for unit struct
Vec::new()
}
syn::Fields::Unnamed(fields) => {
for field in fields.unnamed.iter_mut() {
let field_descs = parse_descriptors(field)?;
if !field_descs.is_empty() {
descriptors.push((field.clone(), field_descs));
}
}
}
syn::Fields::Unit => { /* No fields for unit struct */ }
}
};
impl_class(&class.ident, &attr, doc, descriptors, methods_type)
impl_class(
&class.ident,
args,
doc,
field_options,
methods_type,
options.deprecations,
)
}
/// Parses `#[pyo3(get, set)]`
fn parse_descriptors(item: &mut syn::Field) -> syn::Result<Vec<FnType>> {
let mut descs = Vec::new();
let mut new_attrs = Vec::new();
for attr in item.attrs.drain(..) {
if let Ok(syn::Meta::List(list)) = attr.parse_meta() {
if list.path.is_ident("pyo3") {
for meta in list.nested.iter() {
if let syn::NestedMeta::Meta(metaitem) = meta {
if metaitem.path().is_ident("get") {
descs.push(FnType::Getter(SelfType::Receiver { mutable: false }));
} else if metaitem.path().is_ident("set") {
descs.push(FnType::Setter(SelfType::Receiver { mutable: true }));
} else {
bail_spanned!(metaitem.span() => "only get and set are supported");
}
}
}
} else {
new_attrs.push(attr)
}
/// `#[pyo3()]` options for pyclass fields
struct FieldPyO3Options {
get: bool,
set: bool,
name: Option<NameAttribute>,
}
enum FieldPyO3Option {
Get(attributes::kw::get),
Set(attributes::kw::set),
Name(NameAttribute),
}
impl Parse for FieldPyO3Option {
fn parse(input: ParseStream) -> Result<Self> {
let lookahead = input.lookahead1();
if lookahead.peek(attributes::kw::get) {
input.parse().map(FieldPyO3Option::Get)
} else if lookahead.peek(attributes::kw::set) {
input.parse().map(FieldPyO3Option::Set)
} else if lookahead.peek(attributes::kw::name) {
input.parse().map(FieldPyO3Option::Name)
} else {
new_attrs.push(attr);
Err(lookahead.error())
}
}
item.attrs = new_attrs;
Ok(descs)
}
impl FieldPyO3Options {
fn take_pyo3_options(attrs: &mut Vec<syn::Attribute>) -> Result<Self> {
let mut options = FieldPyO3Options {
get: false,
set: false,
name: None,
};
for option in take_pyo3_options(attrs)? {
match option {
FieldPyO3Option::Get(kw) => {
ensure_spanned!(
!options.get,
kw.span() => "`get` may only be specified once"
);
options.get = true;
}
FieldPyO3Option::Set(kw) => {
ensure_spanned!(
!options.set,
kw.span() => "`set` may only be specified once"
);
options.set = true;
}
FieldPyO3Option::Name(name) => {
ensure_spanned!(
options.name.is_none(),
name.0.span() => "`name` may only be specified once"
);
options.name = Some(name);
}
}
}
Ok(options)
}
}
/// To allow multiple #[pymethods] block, we define inventory types.
@ -267,46 +372,45 @@ fn impl_class(
cls: &syn::Ident,
attr: &PyClassArgs,
doc: syn::LitStr,
descriptors: Vec<(syn::Field, Vec<FnType>)>,
field_options: Vec<(&syn::Field, FieldPyO3Options)>,
methods_type: PyClassMethodsType,
deprecations: Deprecations,
) -> syn::Result<TokenStream> {
let cls_name = get_class_python_name(cls, attr).to_string();
let extra = {
if let Some(freelist) = &attr.freelist {
let alloc = attr.freelist.as_ref().map(|freelist| {
quote! {
impl pyo3::freelist::PyClassWithFreeList for #cls {
impl pyo3::class::impl_::PyClassWithFreeList for #cls {
#[inline]
fn get_free_list(_py: pyo3::Python) -> &mut pyo3::freelist::FreeList<*mut pyo3::ffi::PyObject> {
static mut FREELIST: *mut pyo3::freelist::FreeList<*mut pyo3::ffi::PyObject> = 0 as *mut _;
fn get_free_list(_py: pyo3::Python<'_>) -> &mut pyo3::impl_::freelist::FreeList<*mut pyo3::ffi::PyObject> {
static mut FREELIST: *mut pyo3::impl_::freelist::FreeList<*mut pyo3::ffi::PyObject> = 0 as *mut _;
unsafe {
if FREELIST.is_null() {
FREELIST = Box::into_raw(Box::new(
pyo3::freelist::FreeList::with_capacity(#freelist)));
pyo3::impl_::freelist::FreeList::with_capacity(#freelist)));
}
&mut *FREELIST
}
}
}
}
} else {
quote! {
impl pyo3::pyclass::PyClassAlloc for #cls {}
}
}
};
let extra = if !descriptors.is_empty() {
let path = syn::Path::from(syn::PathSegment::from(cls.clone()));
let ty = syn::Type::from(syn::TypePath { path, qself: None });
let desc_impls = impl_descriptors(&ty, descriptors)?;
quote! {
#desc_impls
#extra
}
} else {
extra
};
impl pyo3::class::impl_::PyClassAllocImpl<#cls> for pyo3::class::impl_::PyClassImplCollector<#cls> {
#[inline]
fn alloc_impl(self) -> Option<pyo3::ffi::allocfunc> {
Some(pyo3::class::impl_::alloc_with_freelist::<#cls>)
}
}
impl pyo3::class::impl_::PyClassFreeImpl<#cls> for pyo3::class::impl_::PyClassImplCollector<#cls> {
#[inline]
fn free_impl(self) -> Option<pyo3::ffi::freefunc> {
Some(pyo3::class::impl_::free_with_freelist::<#cls>)
}
}
}
});
let descriptors = impl_descriptors(cls, field_options)?;
// insert space for weak ref
let weakref = if attr.has_weaklist {
@ -345,14 +449,14 @@ fn impl_class(
quote! {}
};
let (impl_inventory, iter_py_methods) = match methods_type {
PyClassMethodsType::Specialization => (None, quote! { collector.py_methods().iter() }),
let (impl_inventory, for_each_py_method) = match methods_type {
PyClassMethodsType::Specialization => (None, quote! { visitor(collector.py_methods()); }),
PyClassMethodsType::Inventory => (
Some(impl_methods_inventory(&cls)),
Some(impl_methods_inventory(cls)),
quote! {
pyo3::inventory::iter::<<Self as pyo3::class::impl_::HasMethodsInventory>::Methods>
.into_iter()
.flat_map(pyo3::class::impl_::PyMethodsInventory::get)
for inventory in pyo3::inventory::iter::<<Self as pyo3::class::impl_::HasMethodsInventory>::Methods>() {
visitor(pyo3::class::impl_::PyMethodsInventory::get(inventory));
}
},
),
};
@ -399,7 +503,9 @@ fn impl_class(
const MODULE: Option<&'static str> = #module;
#[inline]
fn type_object_raw(py: pyo3::Python) -> *mut pyo3::ffi::PyTypeObject {
fn type_object_raw(py: pyo3::Python<'_>) -> *mut pyo3::ffi::PyTypeObject {
#deprecations
use pyo3::type_object::LazyStaticType;
static TYPE_OBJECT: LazyStaticType = LazyStaticType::new();
TYPE_OBJECT.get_or_init::<Self>(py)
@ -432,49 +538,56 @@ fn impl_class(
const IS_BASETYPE: bool = #is_basetype;
const IS_SUBCLASS: bool = #is_subclass;
type Layout = PyCell<Self>;
type Layout = pyo3::PyCell<Self>;
type BaseType = #base;
type ThreadChecker = #thread_checker;
fn for_each_method_def(visitor: &mut dyn FnMut(&pyo3::class::PyMethodDefType)) {
fn for_each_method_def(visitor: &mut dyn FnMut(&[pyo3::class::PyMethodDefType])) {
use pyo3::class::impl_::*;
let collector = PyClassImplCollector::<Self>::new();
#iter_py_methods
.chain(collector.py_class_descriptors())
.chain(collector.object_protocol_methods())
.chain(collector.async_protocol_methods())
.chain(collector.context_protocol_methods())
.chain(collector.descr_protocol_methods())
.chain(collector.mapping_protocol_methods())
.chain(collector.number_protocol_methods())
.for_each(visitor)
#for_each_py_method;
visitor(collector.py_class_descriptors());
visitor(collector.object_protocol_methods());
visitor(collector.async_protocol_methods());
visitor(collector.context_protocol_methods());
visitor(collector.descr_protocol_methods());
visitor(collector.mapping_protocol_methods());
visitor(collector.number_protocol_methods());
}
fn get_new() -> Option<pyo3::ffi::newfunc> {
use pyo3::class::impl_::*;
let collector = PyClassImplCollector::<Self>::new();
collector.new_impl()
}
fn get_alloc() -> Option<pyo3::ffi::allocfunc> {
use pyo3::class::impl_::*;
let collector = PyClassImplCollector::<Self>::new();
collector.alloc_impl()
}
fn get_free() -> Option<pyo3::ffi::freefunc> {
use pyo3::class::impl_::*;
let collector = PyClassImplCollector::<Self>::new();
collector.free_impl()
}
fn get_call() -> Option<pyo3::ffi::PyCFunctionWithKeywords> {
use pyo3::class::impl_::*;
let collector = PyClassImplCollector::<Self>::new();
collector.call_impl()
}
fn for_each_proto_slot(visitor: &mut dyn FnMut(&pyo3::ffi::PyType_Slot)) {
fn for_each_proto_slot(visitor: &mut dyn FnMut(&[pyo3::ffi::PyType_Slot])) {
// Implementation which uses dtolnay specialization to load all slots.
use pyo3::class::impl_::*;
let collector = PyClassImplCollector::<Self>::new();
collector.object_protocol_slots()
.iter()
.chain(collector.number_protocol_slots())
.chain(collector.iter_protocol_slots())
.chain(collector.gc_protocol_slots())
.chain(collector.descr_protocol_slots())
.chain(collector.mapping_protocol_slots())
.chain(collector.sequence_protocol_slots())
.chain(collector.async_protocol_slots())
.chain(collector.buffer_protocol_slots())
.for_each(visitor);
visitor(collector.object_protocol_slots());
visitor(collector.number_protocol_slots());
visitor(collector.iter_protocol_slots());
visitor(collector.gc_protocol_slots());
visitor(collector.descr_protocol_slots());
visitor(collector.mapping_protocol_slots());
visitor(collector.sequence_protocol_slots());
visitor(collector.async_protocol_slots());
visitor(collector.buffer_protocol_slots());
}
fn get_buffer() -> Option<&'static pyo3::class::impl_::PyBufferProcs> {
@ -484,39 +597,50 @@ fn impl_class(
}
}
#extra
#alloc
#descriptors
#gc_impl
})
}
fn impl_descriptors(
cls: &syn::Type,
descriptors: Vec<(syn::Field, Vec<FnType>)>,
cls: &syn::Ident,
field_options: Vec<(&syn::Field, FieldPyO3Options)>,
) -> syn::Result<TokenStream> {
let py_methods: Vec<TokenStream> = descriptors
.iter()
.flat_map(|(field, fns)| {
fns.iter()
.map(|desc| {
let doc = utils::get_doc(&field.attrs, None, true)
.unwrap_or_else(|_| syn::LitStr::new("", Span::call_site()));
let property_type = PropertyType::Descriptor(
field.ident.as_ref().ok_or_else(
|| err_spanned!(field.span() => "`#[pyo3(get, set)]` is not supported on tuple struct fields")
)?
);
match desc {
FnType::Getter(self_ty) => {
impl_py_getter_def(cls, property_type, self_ty, &doc, &Default::default())
}
FnType::Setter(self_ty) => {
impl_py_setter_def(cls, property_type, self_ty, &doc, &Default::default())
}
_ => unreachable!(),
}
})
.collect::<Vec<syn::Result<TokenStream>>>()
let ty = syn::parse_quote!(#cls);
let py_methods: Vec<TokenStream> = field_options
.into_iter()
.enumerate()
.flat_map(|(field_index, (field, options))| {
let name_err = if options.name.is_some() && !options.get && !options.set {
Some(Err(err_spanned!(options.name.as_ref().unwrap().0.span() => "`name` is useless without `get` or `set`")))
} else {
None
};
let getter = if options.get {
Some(impl_py_getter_def(&ty, PropertyType::Descriptor {
field_index,
field,
python_name: options.name.as_ref()
}))
} else {
None
};
let setter = if options.set {
Some(impl_py_setter_def(&ty, PropertyType::Descriptor {
field_index,
field,
python_name: options.name.as_ref()
}))
} else {
None
};
name_err.into_iter().chain(getter).chain(setter)
})
.collect::<syn::Result<_>>()?;

157
pyo3-macros-backend/src/pyfunction.rs
View File

@ -2,12 +2,13 @@
use crate::{
attributes::{
self, get_deprecated_name_attribute, get_pyo3_attributes, take_attributes,
FromPyWithAttribute, NameAttribute,
self, get_deprecated_name_attribute, get_deprecated_text_signature_attribute,
get_pyo3_options, take_attributes, FromPyWithAttribute, NameAttribute,
TextSignatureAttribute,
},
deprecations::Deprecations,
method::{self, FnArg, FnSpec},
pymethod::{check_generic, get_arg_names, impl_arg_params},
method::{self, CallingConvention, FnArg},
pymethod::check_generic,
utils::{self, ensure_not_async_fn},
};
use proc_macro2::{Span, TokenStream};
@ -62,7 +63,7 @@ impl PyFunctionArgPyO3Attributes {
pub fn from_attrs(attrs: &mut Vec<syn::Attribute>) -> syn::Result<Self> {
let mut attributes = PyFunctionArgPyO3Attributes { from_py_with: None };
take_attributes(attrs, |attr| {
if let Some(pyo3_attrs) = get_pyo3_attributes(attr)? {
if let Some(pyo3_attrs) = get_pyo3_options(attr)? {
for attr in pyo3_attrs {
match attr {
PyFunctionArgPyO3Attribute::FromPyWith(from_py_with) => {
@ -208,18 +209,20 @@ impl PyFunctionSignature {
#[derive(Default)]
pub struct PyFunctionOptions {
pub pass_module: bool,
pub pass_module: Option<attributes::kw::pass_module>,
pub name: Option<NameAttribute>,
pub signature: Option<PyFunctionSignature>,
pub text_signature: Option<TextSignatureAttribute>,
pub deprecations: Deprecations,
}
impl Parse for PyFunctionOptions {
fn parse(input: ParseStream) -> Result<Self> {
let mut options = PyFunctionOptions {
pass_module: false,
pass_module: None,
name: None,
signature: None,
text_signature: None,
deprecations: Deprecations::new(),
};
@ -228,6 +231,7 @@ impl Parse for PyFunctionOptions {
if lookahead.peek(attributes::kw::name)
|| lookahead.peek(attributes::kw::pass_module)
|| lookahead.peek(attributes::kw::signature)
|| lookahead.peek(attributes::kw::text_signature)
{
options.add_attributes(std::iter::once(input.parse()?))?;
if !input.is_empty() {
@ -250,6 +254,7 @@ pub enum PyFunctionOption {
Name(NameAttribute),
PassModule(attributes::kw::pass_module),
Signature(PyFunctionSignature),
TextSignature(TextSignatureAttribute),
}
impl Parse for PyFunctionOption {
@ -261,6 +266,8 @@ impl Parse for PyFunctionOption {
input.parse().map(PyFunctionOption::PassModule)
} else if lookahead.peek(attributes::kw::signature) {
input.parse().map(PyFunctionOption::Signature)
} else if lookahead.peek(attributes::kw::text_signature) {
input.parse().map(PyFunctionOption::TextSignature)
} else {
Err(lookahead.error())
}
@ -270,19 +277,26 @@ impl Parse for PyFunctionOption {
impl PyFunctionOptions {
pub fn from_attrs(attrs: &mut Vec<syn::Attribute>) -> syn::Result<Self> {
let mut options = PyFunctionOptions::default();
options.take_pyo3_attributes(attrs)?;
options.take_pyo3_options(attrs)?;
Ok(options)
}
pub fn take_pyo3_attributes(&mut self, attrs: &mut Vec<syn::Attribute>) -> syn::Result<()> {
pub fn take_pyo3_options(&mut self, attrs: &mut Vec<syn::Attribute>) -> syn::Result<()> {
take_attributes(attrs, |attr| {
if let Some(pyo3_attributes) = get_pyo3_attributes(attr)? {
if let Some(pyo3_attributes) = get_pyo3_options(attr)? {
self.add_attributes(pyo3_attributes)?;
Ok(true)
} else if let Some(name) = get_deprecated_name_attribute(attr, &mut self.deprecations)?
{
self.set_name(name)?;
Ok(true)
} else if let Some(text_signature) =
get_deprecated_text_signature_attribute(attr, &mut self.deprecations)?
{
self.add_attributes(std::iter::once(PyFunctionOption::TextSignature(
text_signature,
)))?;
Ok(true)
} else {
Ok(false)
}
@ -300,10 +314,10 @@ impl PyFunctionOptions {
PyFunctionOption::Name(name) => self.set_name(name)?,
PyFunctionOption::PassModule(kw) => {
ensure_spanned!(
!self.pass_module,
self.pass_module.is_none(),
kw.span() => "`pass_module` may only be specified once"
);
self.pass_module = true;
self.pass_module = Some(kw);
}
PyFunctionOption::Signature(signature) => {
ensure_spanned!(
@ -313,6 +327,13 @@ impl PyFunctionOptions {
);
self.signature = Some(signature);
}
PyFunctionOption::TextSignature(text_signature) => {
ensure_spanned!(
self.text_signature.is_none(),
text_signature.kw.span() => "`text_signature` may only be specified once"
);
self.text_signature = Some(text_signature);
}
}
}
Ok(())
@ -332,7 +353,7 @@ pub fn build_py_function(
ast: &mut syn::ItemFn,
mut options: PyFunctionOptions,
) -> syn::Result<TokenStream> {
options.take_pyo3_attributes(&mut ast.attrs)?;
options.take_pyo3_options(&mut ast.attrs)?;
Ok(impl_wrap_pyfunction(ast, options)?.1)
}
@ -364,7 +385,7 @@ pub fn impl_wrap_pyfunction(
.map(FnArg::parse)
.collect::<syn::Result<Vec<_>>>()?;
if options.pass_module {
if options.pass_module.is_some() {
const PASS_MODULE_ERR: &str = "expected &PyModule as first argument with `pass_module`";
ensure_spanned!(
!arguments.is_empty(),
@ -379,14 +400,24 @@ pub fn impl_wrap_pyfunction(
let ty = method::get_return_info(&func.sig.output);
let text_signature = utils::parse_text_signature_attrs(&mut func.attrs, &python_name)?;
let doc = utils::get_doc(&func.attrs, text_signature, true)?;
let doc = utils::get_doc(
&func.attrs,
options
.text_signature
.as_ref()
.map(|attr| (&python_name, attr)),
)?;
let function_wrapper_ident = function_wrapper_ident(&func.sig.ident);
let spec = method::FnSpec {
tp: method::FnType::FnStatic,
name: &function_wrapper_ident,
tp: if options.pass_module.is_some() {
method::FnType::FnModule
} else {
method::FnType::FnStatic
},
name: &func.sig.ident,
convention: CallingConvention::from_args(&arguments, &signature.arguments),
python_name,
attrs: signature.arguments,
args: arguments,
@ -395,101 +426,21 @@ pub fn impl_wrap_pyfunction(
deprecations: options.deprecations,
};
let doc = &spec.doc;
let python_name = spec.null_terminated_python_name();
let wrapper_ident = format_ident!("__pyo3_raw_{}", spec.name);
let wrapper = spec.get_wrapper_function(&wrapper_ident, None)?;
let methoddef = spec.get_methoddef(wrapper_ident);
let name = &func.sig.ident;
let wrapper_ident = format_ident!("__pyo3_raw_{}", name);
let wrapper = function_c_wrapper(name, &wrapper_ident, &spec, options.pass_module)?;
let methoddef = if spec.args.is_empty() {
quote!(noargs)
} else {
quote!(cfunction_with_keywords)
};
let cfunc = if spec.args.is_empty() {
quote!(PyCFunction)
} else {
quote!(PyCFunctionWithKeywords)
};
let wrapped_pyfunction = quote! {
#wrapper
pub(crate) fn #function_wrapper_ident<'a>(
args: impl Into<pyo3::derive_utils::PyFunctionArguments<'a>>
) -> pyo3::PyResult<&'a pyo3::types::PyCFunction> {
pyo3::types::PyCFunction::internal_new(
pyo3::class::methods::PyMethodDef:: #methoddef (
#python_name,
pyo3::class::methods:: #cfunc (#wrapper_ident),
#doc,
),
args.into(),
)
pyo3::types::PyCFunction::internal_new(#methoddef, args.into())
}
};
Ok((function_wrapper_ident, wrapped_pyfunction))
}
/// Generate static function wrapper (PyCFunction, PyCFunctionWithKeywords)
fn function_c_wrapper(
name: &Ident,
wrapper_ident: &Ident,
spec: &FnSpec<'_>,
pass_module: bool,
) -> Result<TokenStream> {
let names: Vec<Ident> = get_arg_names(&spec);
let (cb, slf_module) = if pass_module {
(
quote! {
pyo3::callback::convert(_py, #name(_slf, #(#names),*))
},
Some(quote! {
let _slf = _py.from_borrowed_ptr::<pyo3::types::PyModule>(_slf);
}),
)
} else {
(
quote! {
pyo3::callback::convert(_py, #name(#(#names),*))
},
None,
)
};
let py = syn::Ident::new("_py", Span::call_site());
let deprecations = &spec.deprecations;
if spec.args.is_empty() {
Ok(quote! {
unsafe extern "C" fn #wrapper_ident(
_slf: *mut pyo3::ffi::PyObject,
_unused: *mut pyo3::ffi::PyObject) -> *mut pyo3::ffi::PyObject
{
#deprecations
pyo3::callback::handle_panic(|#py| {
#slf_module
#cb
})
}
})
} else {
let body = impl_arg_params(spec, None, cb, &py)?;
Ok(quote! {
unsafe extern "C" fn #wrapper_ident(
_slf: *mut pyo3::ffi::PyObject,
_args: *mut pyo3::ffi::PyObject,
_kwargs: *mut pyo3::ffi::PyObject) -> *mut pyo3::ffi::PyObject
{
#deprecations
pyo3::callback::handle_panic(|#py| {
#slf_module
let _args = #py.from_borrowed_ptr::<pyo3::types::PyTuple>(_args);
let _kwargs: Option<&pyo3::types::PyDict> = #py.from_borrowed_ptr_or_opt(_kwargs);
#body
})
}
})
}
}
fn type_is_pymodule(ty: &syn::Type) -> bool {
if let syn::Type::Reference(tyref) = ty {
if let syn::Type::Path(typath) = tyref.elem.as_ref() {
@ -508,7 +459,7 @@ fn type_is_pymodule(ty: &syn::Type) -> bool {
}
#[cfg(test)]
mod test {
mod tests {
use super::{Argument, PyFunctionSignature};
use proc_macro2::TokenStream;
use quote::quote;

22
pyo3-macros-backend/src/pyimpl.rs
View File

@ -67,7 +67,7 @@ pub fn impl_methods(
attributes,
};
let attrs = get_cfg_attributes(&konst.attrs);
let meth = pymethod::gen_py_const(ty, &spec);
let meth = gen_py_const(ty, &spec);
methods.push(quote!(#(#attrs)* #meth));
}
}
@ -89,6 +89,26 @@ pub fn impl_methods(
})
}
fn gen_py_const(cls: &syn::Type, spec: &ConstSpec) -> TokenStream {
let member = &spec.rust_ident;
let deprecations = &spec.attributes.deprecations;
let python_name = &spec.null_terminated_python_name();
quote! {
pyo3::class::PyMethodDefType::ClassAttribute({
pyo3::class::PyClassAttributeDef::new(
#python_name,
pyo3::class::methods::PyClassAttributeFactory({
fn __wrap(py: pyo3::Python<'_>) -> pyo3::PyObject {
#deprecations
pyo3::IntoPy::into_py(#cls::#member, py)
}
__wrap
})
)
})
}
}
fn impl_py_methods(ty: &syn::Type, methods: Vec<TokenStream>) -> TokenStream {
quote! {
impl pyo3::class::impl_::PyMethods<#ty>

899
pyo3-macros-backend/src/pymethod.rs
View File

@ -1,21 +1,18 @@
use crate::utils::ensure_not_async_fn;
// Copyright (c) 2017-present PyO3 Project and Contributors
use crate::{attributes::FromPyWithAttribute, konst::ConstSpec};
use std::borrow::Cow;
use crate::attributes::NameAttribute;
use crate::utils::ensure_not_async_fn;
use crate::{deprecations::Deprecations, utils};
use crate::{
method::{FnArg, FnSpec, FnType, SelfType},
pyfunction::PyFunctionOptions,
};
use proc_macro2::{Span, TokenStream};
use quote::{quote, quote_spanned};
use quote::quote;
use syn::{ext::IdentExt, spanned::Spanned, Result};
#[derive(Clone, Copy)]
pub enum PropertyType<'a> {
Descriptor(&'a syn::Ident),
Function(&'a FnSpec<'a>),
}
pub enum GeneratedPyMethod {
Method(TokenStream),
New(TokenStream),
@ -30,39 +27,47 @@ pub fn gen_py_method(
) -> Result<GeneratedPyMethod> {
check_generic(sig)?;
ensure_not_async_fn(sig)?;
ensure_function_options_valid(&options)?;
let spec = FnSpec::parse(sig, &mut *meth_attrs, options)?;
Ok(match &spec.tp {
FnType::Fn(self_ty) => {
GeneratedPyMethod::Method(impl_py_method_def(cls, &spec, self_ty, None)?)
}
// ordinary functions (with some specialties)
FnType::Fn(_) => GeneratedPyMethod::Method(impl_py_method_def(cls, &spec, None)?),
FnType::FnClass => GeneratedPyMethod::Method(impl_py_method_def(
cls,
&spec,
Some(quote!(pyo3::ffi::METH_CLASS)),
)?),
FnType::FnStatic => GeneratedPyMethod::Method(impl_py_method_def(
cls,
&spec,
Some(quote!(pyo3::ffi::METH_STATIC)),
)?),
// special prototypes
FnType::FnNew => GeneratedPyMethod::New(impl_py_method_def_new(cls, &spec)?),
FnType::FnCall(self_ty) => {
GeneratedPyMethod::Call(impl_py_method_def_call(cls, &spec, self_ty)?)
}
FnType::FnClass => GeneratedPyMethod::Method(impl_py_method_def_class(cls, &spec)?),
FnType::FnStatic => GeneratedPyMethod::Method(impl_py_method_def_static(cls, &spec)?),
FnType::ClassAttribute => {
GeneratedPyMethod::Method(impl_py_method_class_attribute(cls, &spec))
}
FnType::Getter(self_ty) => GeneratedPyMethod::Method(impl_py_getter_def(
FnType::FnCall(_) => GeneratedPyMethod::Call(impl_py_method_def_call(cls, &spec)?),
FnType::ClassAttribute => GeneratedPyMethod::Method(impl_py_class_attribute(cls, &spec)),
FnType::Getter(self_type) => GeneratedPyMethod::Method(impl_py_getter_def(
cls,
PropertyType::Function(&spec),
self_ty,
&spec.doc,
&spec.deprecations,
PropertyType::Function {
self_type,
spec: &spec,
},
)?),
FnType::Setter(self_ty) => GeneratedPyMethod::Method(impl_py_setter_def(
FnType::Setter(self_type) => GeneratedPyMethod::Method(impl_py_setter_def(
cls,
PropertyType::Function(&spec),
self_ty,
&spec.doc,
&spec.deprecations,
PropertyType::Function {
self_type,
spec: &spec,
},
)?),
FnType::FnModule => {
unreachable!("methods cannot be FnModule")
}
})
}
pub(crate) fn check_generic(sig: &syn::Signature) -> syn::Result<()> {
pub fn check_generic(sig: &syn::Signature) -> syn::Result<()> {
let err_msg = |typ| format!("Python functions cannot have generic {} parameters", typ);
for param in &sig.generics.params {
match param {
@ -74,214 +79,81 @@ pub(crate) fn check_generic(sig: &syn::Signature) -> syn::Result<()> {
Ok(())
}
pub fn gen_py_const(cls: &syn::Type, spec: &ConstSpec) -> TokenStream {
let member = &spec.rust_ident;
let deprecations = &spec.attributes.deprecations;
let wrapper = quote! {{
fn __wrap(py: pyo3::Python<'_>) -> pyo3::PyObject {
#deprecations
pyo3::IntoPy::into_py(#cls::#member, py)
}
__wrap
}};
impl_py_const_class_attribute(&spec, &wrapper)
fn ensure_function_options_valid(options: &PyFunctionOptions) -> syn::Result<()> {
if let Some(pass_module) = &options.pass_module {
bail_spanned!(pass_module.span() => "`pass_module` cannot be used on Python methods")
}
Ok(())
}
/// Generate function wrapper for PyCFunctionWithKeywords
pub fn impl_wrap_cfunction_with_keywords(
/// Also used by pyfunction.
pub fn impl_py_method_def(
cls: &syn::Type,
spec: &FnSpec<'_>,
self_ty: &SelfType,
spec: &FnSpec,
flags: Option<TokenStream>,
) -> Result<TokenStream> {
let body = impl_call(cls, &spec);
let slf = self_ty.receiver(cls);
let py = syn::Ident::new("_py", Span::call_site());
let body = impl_arg_params(&spec, Some(cls), body, &py)?;
let deprecations = &spec.deprecations;
Ok(quote! {{
unsafe extern "C" fn __wrap(
_slf: *mut pyo3::ffi::PyObject,
_args: *mut pyo3::ffi::PyObject,
_kwargs: *mut pyo3::ffi::PyObject) -> *mut pyo3::ffi::PyObject
{
#deprecations
pyo3::callback::handle_panic(|#py| {
#slf
let _args = #py.from_borrowed_ptr::<pyo3::types::PyTuple>(_args);
let _kwargs: Option<&pyo3::types::PyDict> = #py.from_borrowed_ptr_or_opt(_kwargs);
#body
})
}
__wrap
}})
}
/// Generate function wrapper PyCFunction
pub fn impl_wrap_noargs(cls: &syn::Type, spec: &FnSpec<'_>, self_ty: &SelfType) -> TokenStream {
let body = impl_call(cls, &spec);
let slf = self_ty.receiver(cls);
let deprecations = &spec.deprecations;
assert!(spec.args.is_empty());
quote! {{
unsafe extern "C" fn __wrap(
_slf: *mut pyo3::ffi::PyObject,
_args: *mut pyo3::ffi::PyObject,
) -> *mut pyo3::ffi::PyObject
{
#deprecations
pyo3::callback::handle_panic(|_py| {
#slf
#body
})
}
__wrap
}}
}
/// Generate class method wrapper (PyCFunction, PyCFunctionWithKeywords)
pub fn impl_wrap_new(cls: &syn::Type, spec: &FnSpec<'_>) -> Result<TokenStream> {
let name = &spec.name;
let names: Vec<syn::Ident> = get_arg_names(&spec);
let cb = quote! { #cls::#name(#(#names),*) };
let py = syn::Ident::new("_py", Span::call_site());
let body = impl_arg_params(spec, Some(cls), cb, &py)?;
let deprecations = &spec.deprecations;
Ok(quote! {{
#[allow(unused_mut)]
unsafe extern "C" fn __wrap(
subtype: *mut pyo3::ffi::PyTypeObject,
_args: *mut pyo3::ffi::PyObject,
_kwargs: *mut pyo3::ffi::PyObject) -> *mut pyo3::ffi::PyObject
{
#deprecations
use pyo3::callback::IntoPyCallbackOutput;
pyo3::callback::handle_panic(|#py| {
let _args = #py.from_borrowed_ptr::<pyo3::types::PyTuple>(_args);
let _kwargs: Option<&pyo3::types::PyDict> = #py.from_borrowed_ptr_or_opt(_kwargs);
let initializer: pyo3::PyClassInitializer::<#cls> = #body.convert(#py)?;
let cell = initializer.create_cell_from_subtype(#py, subtype)?;
Ok(cell as *mut pyo3::ffi::PyObject)
})
}
__wrap
}})
}
/// Generate class method wrapper (PyCFunction, PyCFunctionWithKeywords)
pub fn impl_wrap_class(cls: &syn::Type, spec: &FnSpec<'_>) -> Result<TokenStream> {
let name = &spec.name;
let names: Vec<syn::Ident> = get_arg_names(&spec);
let cb = quote! { pyo3::callback::convert(_py, #cls::#name(&_cls, #(#names),*)) };
let py = syn::Ident::new("_py", Span::call_site());
let body = impl_arg_params(spec, Some(cls), cb, &py)?;
let deprecations = &spec.deprecations;
Ok(quote! {{
#[allow(unused_mut)]
unsafe extern "C" fn __wrap(
_cls: *mut pyo3::ffi::PyObject,
_args: *mut pyo3::ffi::PyObject,
_kwargs: *mut pyo3::ffi::PyObject) -> *mut pyo3::ffi::PyObject
{
#deprecations
pyo3::callback::handle_panic(|#py| {
let _cls = pyo3::types::PyType::from_type_ptr(#py, _cls as *mut pyo3::ffi::PyTypeObject);
let _args = #py.from_borrowed_ptr::<pyo3::types::PyTuple>(_args);
let _kwargs: Option<&pyo3::types::PyDict> = #py.from_borrowed_ptr_or_opt(_kwargs);
#body
})
}
__wrap
}})
}
/// Generate static method wrapper (PyCFunction, PyCFunctionWithKeywords)
pub fn impl_wrap_static(cls: &syn::Type, spec: &FnSpec<'_>) -> Result<TokenStream> {
let name = &spec.name;
let names: Vec<syn::Ident> = get_arg_names(&spec);
let cb = quote! { pyo3::callback::convert(_py, #cls::#name(#(#names),*)) };
let py = syn::Ident::new("_py", Span::call_site());
let body = impl_arg_params(spec, Some(cls), cb, &py)?;
let deprecations = &spec.deprecations;
Ok(quote! {{
#[allow(unused_mut)]
unsafe extern "C" fn __wrap(
_slf: *mut pyo3::ffi::PyObject,
_args: *mut pyo3::ffi::PyObject,
_kwargs: *mut pyo3::ffi::PyObject) -> *mut pyo3::ffi::PyObject
{
#deprecations
pyo3::callback::handle_panic(|#py| {
let _args = #py.from_borrowed_ptr::<pyo3::types::PyTuple>(_args);
let _kwargs: Option<&pyo3::types::PyDict> = #py.from_borrowed_ptr_or_opt(_kwargs);
#body
})
}
__wrap
}})
}
/// Generate a wrapper for initialization of a class attribute from a method
/// annotated with `#[classattr]`.
/// To be called in `pyo3::pyclass::initialize_type_object`.
pub fn impl_wrap_class_attribute(cls: &syn::Type, spec: &FnSpec<'_>) -> TokenStream {
let name = &spec.name;
let cb = quote! { #cls::#name() };
let deprecations = &spec.deprecations;
quote! {{
fn __wrap(py: pyo3::Python<'_>) -> pyo3::PyObject {
#deprecations
pyo3::IntoPy::into_py(#cb, py)
}
__wrap
}}
}
fn impl_call_getter(cls: &syn::Type, spec: &FnSpec) -> syn::Result<TokenStream> {
let (py_arg, args) = split_off_python_arg(&spec.args);
ensure_spanned!(
args.is_empty(),
args[0].ty.span() => "getter function can only have one argument (of type pyo3::Python)"
);
let name = &spec.name;
let fncall = if py_arg.is_some() {
quote!(#cls::#name(_slf, _py))
} else {
quote!(#cls::#name(_slf))
let wrapper_ident = syn::Ident::new("__wrap", Span::call_site());
let wrapper_def = spec.get_wrapper_function(&wrapper_ident, Some(cls))?;
let add_flags = flags.map(|flags| quote!(.flags(#flags)));
let methoddef_type = match spec.tp {
FnType::FnStatic => quote!(Static),
FnType::FnClass => quote!(Class),
_ => quote!(Method),
};
Ok(fncall)
let methoddef = spec.get_methoddef(quote! {{ #wrapper_def #wrapper_ident }});
Ok(quote! {
pyo3::class::PyMethodDefType::#methoddef_type(#methoddef #add_flags)
})
}
/// Generate a function wrapper called `__wrap` for a property getter
pub(crate) fn impl_wrap_getter(
cls: &syn::Type,
property_type: PropertyType,
self_ty: &SelfType,
) -> syn::Result<TokenStream> {
let getter_impl = match &property_type {
PropertyType::Descriptor(ident) => {
quote!(_slf.#ident.clone())
fn impl_py_method_def_new(cls: &syn::Type, spec: &FnSpec) -> Result<TokenStream> {
let wrapper_ident = syn::Ident::new("__wrap", Span::call_site());
let wrapper = spec.get_wrapper_function(&wrapper_ident, Some(cls))?;
Ok(quote! {
impl pyo3::class::impl_::PyClassNewImpl<#cls> for pyo3::class::impl_::PyClassImplCollector<#cls> {
fn new_impl(self) -> Option<pyo3::ffi::newfunc> {
Some({
#wrapper
#wrapper_ident
})
}
}
PropertyType::Function(spec) => impl_call_getter(cls, spec)?,
};
})
}
let slf = self_ty.receiver(cls);
Ok(quote! {{
unsafe extern "C" fn __wrap(
_slf: *mut pyo3::ffi::PyObject, _: *mut std::os::raw::c_void) -> *mut pyo3::ffi::PyObject
{
pyo3::callback::handle_panic(|_py| {
#slf
pyo3::callback::convert(_py, #getter_impl)
})
fn impl_py_method_def_call(cls: &syn::Type, spec: &FnSpec) -> Result<TokenStream> {
let wrapper_ident = syn::Ident::new("__wrap", Span::call_site());
let wrapper = spec.get_wrapper_function(&wrapper_ident, Some(cls))?;
Ok(quote! {
impl pyo3::class::impl_::PyClassCallImpl<#cls> for pyo3::class::impl_::PyClassImplCollector<#cls> {
fn call_impl(self) -> Option<pyo3::ffi::PyCFunctionWithKeywords> {
Some({
#wrapper
#wrapper_ident
})
}
}
__wrap
}})
})
}
fn impl_py_class_attribute(cls: &syn::Type, spec: &FnSpec) -> TokenStream {
let name = &spec.name;
let deprecations = &spec.deprecations;
let python_name = spec.null_terminated_python_name();
quote! {
pyo3::class::PyMethodDefType::ClassAttribute({
pyo3::class::PyClassAttributeDef::new(
#python_name,
pyo3::class::methods::PyClassAttributeFactory({
fn __wrap(py: pyo3::Python<'_>) -> pyo3::PyObject {
#deprecations
pyo3::IntoPy::into_py(#cls::#name(), py)
}
__wrap
})
)
})
}
}
fn impl_call_setter(cls: &syn::Type, spec: &FnSpec) -> syn::Result<TokenStream> {
@ -306,452 +178,121 @@ fn impl_call_setter(cls: &syn::Type, spec: &FnSpec) -> syn::Result<TokenStream>
Ok(fncall)
}
/// Generate a function wrapper called `__wrap` for a property setter
pub(crate) fn impl_wrap_setter(
cls: &syn::Type,
property_type: PropertyType,
self_ty: &SelfType,
) -> syn::Result<TokenStream> {
let setter_impl = match &property_type {
PropertyType::Descriptor(ident) => {
// Used here for PropertyType::Function, used in pyclass for descriptors.
pub fn impl_py_setter_def(cls: &syn::Type, property_type: PropertyType) -> Result<TokenStream> {
let python_name = property_type.null_terminated_python_name()?;
let deprecations = property_type.deprecations();
let doc = property_type.doc();
let setter_impl = match property_type {
PropertyType::Descriptor {
field: syn::Field {
ident: Some(ident), ..
},
..
} => {
// named struct field
quote!({ _slf.#ident = _val; })
}
PropertyType::Function(spec) => impl_call_setter(cls, spec)?,
PropertyType::Descriptor { field_index, .. } => {
// tuple struct field
let index = syn::Index::from(field_index);
quote!({ _slf.#index = _val; })
}
PropertyType::Function { spec, .. } => impl_call_setter(cls, spec)?,
};
let slf = self_ty.receiver(cls);
Ok(quote! {{
#[allow(unused_mut)]
unsafe extern "C" fn __wrap(
_slf: *mut pyo3::ffi::PyObject,
_value: *mut pyo3::ffi::PyObject, _: *mut std::os::raw::c_void) -> std::os::raw::c_int
{
pyo3::callback::handle_panic(|_py| {
#slf
let _value = _py.from_borrowed_ptr::<pyo3::types::PyAny>(_value);
let _val = pyo3::FromPyObject::extract(_value)?;
pyo3::callback::convert(_py, #setter_impl)
})
}
__wrap
}})
}
/// This function abstracts away some copied code and can propably be simplified itself
pub fn get_arg_names(spec: &FnSpec) -> Vec<syn::Ident> {
(0..spec.args.len())
.map(|pos| syn::Ident::new(&format!("arg{}", pos), Span::call_site()))
.collect()
}
fn impl_call(cls: &syn::Type, spec: &FnSpec<'_>) -> TokenStream {
let fname = &spec.name;
let names = get_arg_names(spec);
quote! { pyo3::callback::convert(_py, #cls::#fname(_slf, #(#names),*)) }
}
pub fn impl_arg_params(
spec: &FnSpec<'_>,
self_: Option<&syn::Type>,
body: TokenStream,
py: &syn::Ident,
) -> Result<TokenStream> {
if spec.args.is_empty() {
return Ok(body);
}
let mut positional_parameter_names = Vec::new();
let mut required_positional_parameters = 0usize;
let mut keyword_only_parameters = Vec::new();
for arg in spec.args.iter() {
if arg.py || spec.is_args(&arg.name) || spec.is_kwargs(&arg.name) {
continue;
}
let name = arg.name.unraw().to_string();
let kwonly = spec.is_kw_only(&arg.name);
let required = !(arg.optional.is_some() || spec.default_value(&arg.name).is_some());
if kwonly {
keyword_only_parameters.push(quote! {
pyo3::derive_utils::KeywordOnlyParameterDescription {
name: #name,
required: #required,
}
});
} else {
if required {
required_positional_parameters += 1;
}
positional_parameter_names.push(name);
}
}
let num_params = positional_parameter_names.len() + keyword_only_parameters.len();
let args_array = syn::Ident::new("output", Span::call_site());
let mut param_conversion = Vec::new();
let mut option_pos = 0;
for (idx, arg) in spec.args.iter().enumerate() {
param_conversion.push(impl_arg_param(
&arg,
&spec,
idx,
self_,
&mut option_pos,
py,
&args_array,
)?);
}
let (mut accept_args, mut accept_kwargs) = (false, false);
for s in spec.attrs.iter() {
use crate::pyfunction::Argument;
match s {
Argument::VarArgs(_) => accept_args = true,
Argument::KeywordArgs(_) => accept_kwargs = true,
_ => continue,
}
}
let cls_name = if let Some(cls) = self_ {
quote! { Some(<#cls as pyo3::type_object::PyTypeInfo>::NAME) }
} else {
quote! { None }
let slf = match property_type {
PropertyType::Descriptor { .. } => SelfType::Receiver { mutable: true }.receiver(cls),
PropertyType::Function { self_type, .. } => self_type.receiver(cls),
};
let python_name = &spec.python_name;
// create array of arguments, and then parse
Ok(quote! {
{
const DESCRIPTION: pyo3::derive_utils::FunctionDescription = pyo3::derive_utils::FunctionDescription {
cls_name: #cls_name,
func_name: stringify!(#python_name),
positional_parameter_names: &[#(#positional_parameter_names),*],
// TODO: https://github.com/PyO3/pyo3/issues/1439 - support specifying these
positional_only_parameters: 0,
required_positional_parameters: #required_positional_parameters,
keyword_only_parameters: &[#(#keyword_only_parameters),*],
accept_varargs: #accept_args,
accept_varkeywords: #accept_kwargs,
};
let mut #args_array = [None; #num_params];
let (_args, _kwargs) = DESCRIPTION.extract_arguments(_args, _kwargs, &mut #args_array)?;
#(#param_conversion)*
#body
}
})
}
/// Re option_pos: The option slice doesn't contain the py: Python argument, so the argument
/// index and the index in option diverge when using py: Python
fn impl_arg_param(
arg: &FnArg<'_>,
spec: &FnSpec<'_>,
idx: usize,
self_: Option<&syn::Type>,
option_pos: &mut usize,
py: &syn::Ident,
args_array: &syn::Ident,
) -> Result<TokenStream> {
// Use this macro inside this function, to ensure that all code generated here is associated
// with the function argument
macro_rules! quote_arg_span {
($($tokens:tt)*) => { quote_spanned!(arg.ty.span() => $($tokens)*) }
}
let arg_name = syn::Ident::new(&format!("arg{}", idx), Span::call_site());
if arg.py {
return Ok(quote_arg_span! { let #arg_name = #py; });
}
let ty = arg.ty;
let name = arg.name;
let transform_error = quote_arg_span! {
|e| pyo3::derive_utils::argument_extraction_error(#py, stringify!(#name), e)
};
if spec.is_args(&name) {
ensure_spanned!(
arg.optional.is_none(),
arg.name.span() => "args cannot be optional"
);
return Ok(quote_arg_span! {
let #arg_name = _args.unwrap().extract().map_err(#transform_error)?;
});
} else if spec.is_kwargs(&name) {
ensure_spanned!(
arg.optional.is_some(),
arg.name.span() => "kwargs must be Option<_>"
);
return Ok(quote_arg_span! {
let #arg_name = _kwargs.map(|kwargs| kwargs.extract())
.transpose()
.map_err(#transform_error)?;
});
}
let arg_value = quote_arg_span!(#args_array[#option_pos]);
*option_pos += 1;
let extract = if let Some(FromPyWithAttribute(expr_path)) = &arg.attrs.from_py_with {
quote_arg_span! { #expr_path(_obj).map_err(#transform_error) }
} else {
quote_arg_span! { _obj.extract().map_err(#transform_error) }
};
let arg_value_or_default = match (spec.default_value(name), arg.optional.is_some()) {
(Some(default), true) if default.to_string() != "None" => {
quote_arg_span! { #arg_value.map_or_else(|| Ok(Some(#default)), |_obj| #extract)? }
}
(Some(default), _) => {
quote_arg_span! { #arg_value.map_or_else(|| Ok(#default), |_obj| #extract)? }
}
(None, true) => quote_arg_span! { #arg_value.map_or(Ok(None), |_obj| #extract)? },
(None, false) => {
quote_arg_span! {
{
let _obj = #arg_value.expect("Failed to extract required method argument");
#extract?
}
}
}
};
return if let syn::Type::Reference(tref) = arg.optional.as_ref().unwrap_or(&ty) {
let (tref, mut_) = preprocess_tref(tref, self_);
let (target_ty, borrow_tmp) = if arg.optional.is_some() {
// Get Option<&T> from Option<PyRef<T>>
(
quote_arg_span! { Option<<#tref as pyo3::derive_utils::ExtractExt>::Target> },
if mut_.is_some() {
quote_arg_span! { _tmp.as_deref_mut() }
} else {
quote_arg_span! { _tmp.as_deref() }
},
)
} else {
// Get &T from PyRef<T>
(
quote_arg_span! { <#tref as pyo3::derive_utils::ExtractExt>::Target },
quote_arg_span! { &#mut_ *_tmp },
)
};
Ok(quote_arg_span! {
let #mut_ _tmp: #target_ty = #arg_value_or_default;
let #arg_name = #borrow_tmp;
})
} else {
Ok(quote_arg_span! {
let #arg_name = #arg_value_or_default;
})
};
/// Replace `Self`, remove lifetime and get mutability from the type
fn preprocess_tref(
tref: &syn::TypeReference,
self_: Option<&syn::Type>,
) -> (syn::TypeReference, Option<syn::token::Mut>) {
let mut tref = tref.to_owned();
if let Some(syn::Type::Path(tpath)) = self_ {
replace_self(&mut tref, &tpath.path);
}
tref.lifetime = None;
let mut_ = tref.mutability;
(tref, mut_)
}
/// Replace `Self` with the exact type name since it is used out of the impl block
fn replace_self(tref: &mut syn::TypeReference, self_path: &syn::Path) {
match &mut *tref.elem {
syn::Type::Reference(tref_inner) => replace_self(tref_inner, self_path),
syn::Type::Path(tpath) => {
if let Some(ident) = tpath.path.get_ident() {
if ident == "Self" {
tpath.path = self_path.to_owned();
}
}
}
_ => {}
}
}
}
pub fn impl_py_method_def(
cls: &syn::Type,
spec: &FnSpec,
self_ty: &SelfType,
flags: Option<TokenStream>,
) -> Result<TokenStream> {
let add_flags = flags.map(|flags| quote!(.flags(#flags)));
let python_name = spec.null_terminated_python_name();
let doc = &spec.doc;
if spec.args.is_empty() {
let wrapper = impl_wrap_noargs(cls, spec, self_ty);
Ok(quote! {
pyo3::class::PyMethodDefType::Method({
pyo3::class::PyMethodDef::noargs(
#python_name,
pyo3::class::methods::PyCFunction(#wrapper),
#doc
)
#add_flags
})
})
} else {
let wrapper = impl_wrap_cfunction_with_keywords(cls, &spec, self_ty)?;
Ok(quote! {
pyo3::class::PyMethodDefType::Method({
pyo3::class::PyMethodDef::cfunction_with_keywords(
#python_name,
pyo3::class::methods::PyCFunctionWithKeywords(#wrapper),
#doc
)
#add_flags
})
})
}
}
pub fn impl_py_method_def_new(cls: &syn::Type, spec: &FnSpec) -> Result<TokenStream> {
let wrapper = impl_wrap_new(cls, &spec)?;
Ok(quote! {
impl pyo3::class::impl_::PyClassNewImpl<#cls> for pyo3::class::impl_::PyClassImplCollector<#cls> {
fn new_impl(self) -> Option<pyo3::ffi::newfunc> {
Some(#wrapper)
}
}
})
}
pub fn impl_py_method_def_class(cls: &syn::Type, spec: &FnSpec) -> Result<TokenStream> {
let wrapper = impl_wrap_class(cls, &spec)?;
let python_name = spec.null_terminated_python_name();
let doc = &spec.doc;
Ok(quote! {
pyo3::class::PyMethodDefType::Class({
pyo3::class::PyMethodDef::cfunction_with_keywords(
#python_name,
pyo3::class::methods::PyCFunctionWithKeywords(#wrapper),
#doc
).flags(pyo3::ffi::METH_CLASS)
})
})
}
pub fn impl_py_method_def_static(cls: &syn::Type, spec: &FnSpec) -> Result<TokenStream> {
let wrapper = impl_wrap_static(cls, &spec)?;
let python_name = spec.null_terminated_python_name();
let doc = &spec.doc;
Ok(quote! {
pyo3::class::PyMethodDefType::Static({
pyo3::class::PyMethodDef::cfunction_with_keywords(
#python_name,
pyo3::class::methods::PyCFunctionWithKeywords(#wrapper),
#doc
).flags(pyo3::ffi::METH_STATIC)
})
})
}
pub fn impl_py_method_class_attribute(cls: &syn::Type, spec: &FnSpec) -> TokenStream {
let wrapper = impl_wrap_class_attribute(cls, &spec);
let python_name = spec.null_terminated_python_name();
quote! {
pyo3::class::PyMethodDefType::ClassAttribute({
pyo3::class::PyClassAttributeDef::new(
#python_name,
pyo3::class::methods::PyClassAttributeFactory(#wrapper)
)
})
}
}
pub fn impl_py_const_class_attribute(spec: &ConstSpec, wrapper: &TokenStream) -> TokenStream {
let python_name = &spec.null_terminated_python_name();
quote! {
{
pyo3::class::PyMethodDefType::ClassAttribute({
pyo3::class::PyClassAttributeDef::new(
#python_name,
pyo3::class::methods::PyClassAttributeFactory(#wrapper)
)
})
}
}
}
pub fn impl_py_method_def_call(
cls: &syn::Type,
spec: &FnSpec,
self_ty: &SelfType,
) -> Result<TokenStream> {
let wrapper = impl_wrap_cfunction_with_keywords(cls, &spec, self_ty)?;
Ok(quote! {
impl pyo3::class::impl_::PyClassCallImpl<#cls> for pyo3::class::impl_::PyClassImplCollector<#cls> {
fn call_impl(self) -> Option<pyo3::ffi::PyCFunctionWithKeywords> {
Some(#wrapper)
}
}
})
}
pub(crate) fn impl_py_setter_def(
cls: &syn::Type,
property_type: PropertyType,
self_ty: &SelfType,
doc: &syn::LitStr,
deprecations: &Deprecations,
) -> Result<TokenStream> {
let python_name = match property_type {
PropertyType::Descriptor(ident) => {
let formatted_name = format!("{}\0", ident.unraw());
quote!(#formatted_name)
}
PropertyType::Function(spec) => spec.null_terminated_python_name(),
};
let wrapper = impl_wrap_setter(cls, property_type, self_ty)?;
Ok(quote! {
pyo3::class::PyMethodDefType::Setter({
#deprecations
pyo3::class::PySetterDef::new(
#python_name,
pyo3::class::methods::PySetter(#wrapper),
pyo3::class::methods::PySetter({
unsafe extern "C" fn __wrap(
_slf: *mut pyo3::ffi::PyObject,
_value: *mut pyo3::ffi::PyObject,
_: *mut std::os::raw::c_void
) -> std::os::raw::c_int {
pyo3::callback::handle_panic(|_py| {
#slf
let _value = _py.from_borrowed_ptr::<pyo3::types::PyAny>(_value);
let _val = pyo3::FromPyObject::extract(_value)?;
pyo3::callback::convert(_py, #setter_impl)
})
}
__wrap
}),
#doc
)
})
})
}
pub(crate) fn impl_py_getter_def(
cls: &syn::Type,
property_type: PropertyType,
self_ty: &SelfType,
doc: &syn::LitStr,
deprecations: &Deprecations,
) -> Result<TokenStream> {
let python_name = match property_type {
PropertyType::Descriptor(ident) => {
let formatted_name = format!("{}\0", ident.unraw());
quote!(#formatted_name)
}
PropertyType::Function(spec) => spec.null_terminated_python_name(),
fn impl_call_getter(cls: &syn::Type, spec: &FnSpec) -> syn::Result<TokenStream> {
let (py_arg, args) = split_off_python_arg(&spec.args);
ensure_spanned!(
args.is_empty(),
args[0].ty.span() => "getter function can only have one argument (of type pyo3::Python)"
);
let name = &spec.name;
let fncall = if py_arg.is_some() {
quote!(#cls::#name(_slf, _py))
} else {
quote!(#cls::#name(_slf))
};
Ok(fncall)
}
// Used here for PropertyType::Function, used in pyclass for descriptors.
pub fn impl_py_getter_def(cls: &syn::Type, property_type: PropertyType) -> Result<TokenStream> {
let python_name = property_type.null_terminated_python_name()?;
let deprecations = property_type.deprecations();
let doc = property_type.doc();
let getter_impl = match property_type {
PropertyType::Descriptor {
field: syn::Field {
ident: Some(ident), ..
},
..
} => {
// named struct field
quote!(_slf.#ident.clone())
}
PropertyType::Descriptor { field_index, .. } => {
// tuple struct field
let index = syn::Index::from(field_index);
quote!(_slf.#index.clone())
}
PropertyType::Function { spec, .. } => impl_call_getter(cls, spec)?,
};
let slf = match property_type {
PropertyType::Descriptor { .. } => SelfType::Receiver { mutable: false }.receiver(cls),
PropertyType::Function { self_type, .. } => self_type.receiver(cls),
};
let wrapper = impl_wrap_getter(cls, property_type, self_ty)?;
Ok(quote! {
pyo3::class::PyMethodDefType::Getter({
#deprecations
pyo3::class::PyGetterDef::new(
#python_name,
pyo3::class::methods::PyGetter(#wrapper),
pyo3::class::methods::PyGetter({
unsafe extern "C" fn __wrap(
_slf: *mut pyo3::ffi::PyObject,
_: *mut std::os::raw::c_void
) -> *mut pyo3::ffi::PyObject {
pyo3::callback::handle_panic(|_py| {
#slf
pyo3::callback::convert(_py, #getter_impl)
})
}
__wrap
}),
#doc
)
})
@ -762,7 +303,7 @@ pub(crate) fn impl_py_getter_def(
fn split_off_python_arg<'a>(args: &'a [FnArg<'a>]) -> (Option<&FnArg>, &[FnArg]) {
if args
.get(0)
.map(|py| utils::is_python(&py.ty))
.map(|py| utils::is_python(py.ty))
.unwrap_or(false)
{
(Some(&args[0]), &args[1..])
@ -770,3 +311,53 @@ fn split_off_python_arg<'a>(args: &'a [FnArg<'a>]) -> (Option<&FnArg>, &[FnArg])
(None, args)
}
}
pub enum PropertyType<'a> {
Descriptor {
field_index: usize,
field: &'a syn::Field,
python_name: Option<&'a NameAttribute>,
},
Function {
self_type: &'a SelfType,
spec: &'a FnSpec<'a>,
},
}
impl PropertyType<'_> {
fn null_terminated_python_name(&self) -> Result<syn::LitStr> {
match self {
PropertyType::Descriptor {
field, python_name, ..
} => {
let name = match (python_name, &field.ident) {
(Some(name), _) => name.0.to_string(),
(None, Some(field_name)) => format!("{}\0", field_name.unraw()),
(None, None) => {
bail_spanned!(field.span() => "`get` and `set` with tuple struct fields require `name`")
}
};
Ok(syn::LitStr::new(&name, field.span()))
}
PropertyType::Function { spec, .. } => Ok(spec.null_terminated_python_name()),
}
}
fn deprecations(&self) -> Option<&Deprecations> {
match self {
PropertyType::Descriptor { .. } => None,
PropertyType::Function { spec, .. } => Some(&spec.deprecations),
}
}
fn doc(&self) -> Cow<syn::LitStr> {
match self {
PropertyType::Descriptor { field, .. } => {
let doc = utils::get_doc(&field.attrs, None)
.unwrap_or_else(|_| syn::LitStr::new("", Span::call_site()));
Cow::Owned(doc)
}
PropertyType::Function { spec, .. } => Cow::Borrowed(&spec.doc),
}
}
}

10
pyo3-macros-backend/src/pyproto.rs
View File

@ -73,8 +73,8 @@ fn impl_proto_impl(
None
};
let method = if let FnType::Fn(self_ty) = &fn_spec.tp {
pymethod::impl_py_method_def(ty, &fn_spec, &self_ty, flags)?
let method = if let FnType::Fn(_) = &fn_spec.tp {
pymethod::impl_py_method_def(ty, &fn_spec, flags)?
} else {
bail_spanned!(
met.sig.span() => "expected method with receiver for #[pyproto] method"
@ -133,7 +133,11 @@ fn impl_proto_methods(
let mut maybe_buffer_methods = None;
if cfg!(not(Py_3_9)) && proto.name == "Buffer" {
let build_config = pyo3_build_config::get();
const PY39: pyo3_build_config::PythonVersion =
pyo3_build_config::PythonVersion { major: 3, minor: 9 };
if build_config.version <= PY39 && proto.name == "Buffer" {
maybe_buffer_methods = Some(quote! {
impl pyo3::class::impl_::PyBufferProtocolProcs<#ty>
for pyo3::class::impl_::PyClassImplCollector<#ty>

146
pyo3-macros-backend/src/utils.rs
View File

@ -2,6 +2,8 @@
use proc_macro2::Span;
use syn::spanned::Spanned;
use crate::attributes::TextSignatureAttribute;
/// Macro inspired by `anyhow::anyhow!` to create a compiler error with the given span.
macro_rules! err_spanned {
($span:expr => $msg:expr) => {
@ -27,12 +29,8 @@ macro_rules! ensure_spanned {
}
/// Check if the given type `ty` is `pyo3::Python`.
pub fn is_python(mut ty: &syn::Type) -> bool {
while let syn::Type::Group(group) = ty {
// Macros can create invisible delimiters around types.
ty = &*group.elem;
}
match ty {
pub fn is_python(ty: &syn::Type) -> bool {
match unwrap_ty_group(ty) {
syn::Type::Path(typath) => typath
.path
.segments
@ -56,79 +54,19 @@ pub fn option_type_argument(ty: &syn::Type) -> Option<&syn::Type> {
None
}
pub fn is_text_signature_attr(attr: &syn::Attribute) -> bool {
attr.path.is_ident("text_signature")
}
fn parse_text_signature_attr(
attr: &syn::Attribute,
python_name: &syn::Ident,
) -> syn::Result<Option<syn::LitStr>> {
if !is_text_signature_attr(attr) {
return Ok(None);
}
let python_name_str = python_name.to_string();
let python_name_str = python_name_str
.rsplit('.')
.next()
.map(str::trim)
.filter(|v| !v.is_empty())
.ok_or_else(|| err_spanned!(python_name.span() => "failed to parse python name"))?;
match attr.parse_meta()? {
syn::Meta::NameValue(syn::MetaNameValue {
lit: syn::Lit::Str(lit),
..
}) => {
let value = lit.value();
ensure_spanned!(
value.starts_with('(') && value.ends_with(')'),
lit.span() => "text_signature must start with \"(\" and end with \")\""
);
Ok(Some(syn::LitStr::new(
&(python_name_str.to_owned() + &value),
lit.span(),
)))
}
meta => bail_spanned!(
meta.span() => "text_signature must be of the form #[text_signature = \"\"]"
),
}
}
pub fn parse_text_signature_attrs(
attrs: &mut Vec<syn::Attribute>,
python_name: &syn::Ident,
) -> syn::Result<Option<syn::LitStr>> {
let mut text_signature = None;
let mut attrs_out = Vec::with_capacity(attrs.len());
for attr in attrs.drain(..) {
if let Some(value) = parse_text_signature_attr(&attr, python_name)? {
ensure_spanned!(
text_signature.is_none(),
attr.span() => "text_signature attribute already specified previously"
);
text_signature = Some(value);
} else {
attrs_out.push(attr);
}
}
*attrs = attrs_out;
Ok(text_signature)
}
// FIXME(althonos): not sure the docstring formatting is on par here.
// Returns a null-terminated syn::LitStr for use as a Python docstring.
pub fn get_doc(
attrs: &[syn::Attribute],
text_signature: Option<syn::LitStr>,
null_terminated: bool,
text_signature: Option<(&syn::Ident, &TextSignatureAttribute)>,
) -> syn::Result<syn::LitStr> {
let mut doc = String::new();
let mut span = Span::call_site();
if let Some(text_signature) = text_signature {
if let Some((python_name, text_signature)) = text_signature {
// create special doc string lines to set `__text_signature__`
span = text_signature.span();
doc.push_str(&text_signature.value());
doc.push_str(&python_name.to_string());
span = text_signature.lit.span();
doc.push_str(&text_signature.lit.value());
doc.push_str("\n--\n\n");
}
@ -136,35 +74,45 @@ pub fn get_doc(
let mut first = true;
for attr in attrs.iter() {
if let Ok(syn::Meta::NameValue(metanv)) = attr.parse_meta() {
if metanv.path.is_ident("doc") {
if let syn::Lit::Str(litstr) = metanv.lit {
if first {
first = false;
span = litstr.span();
}
let d = litstr.value();
doc.push_str(separator);
if d.starts_with(' ') {
doc.push_str(&d[1..d.len()]);
} else {
doc.push_str(&d);
};
separator = "\n";
} else {
bail_spanned!(metanv.span() => "invalid doc comment")
if attr.path.is_ident("doc") {
if let Ok(DocArgs { _eq_token, lit_str }) = syn::parse2(attr.tokens.clone()) {
if first {
first = false;
span = lit_str.span();
}
let d = lit_str.value();
doc.push_str(separator);
if d.starts_with(' ') {
doc.push_str(&d[1..d.len()]);
} else {
doc.push_str(&d);
};
separator = "\n";
}
}
}
if null_terminated {
doc.push('\0');
}
doc.push('\0');
Ok(syn::LitStr::new(&doc, span))
}
struct DocArgs {
_eq_token: syn::Token![=],
lit_str: syn::LitStr,
}
impl syn::parse::Parse for DocArgs {
fn parse(input: syn::parse::ParseStream) -> syn::Result<Self> {
let this = Self {
_eq_token: input.parse()?,
lit_str: input.parse()?,
};
ensure_spanned!(input.is_empty(), input.span() => "expected end of doc attribute");
Ok(this)
}
}
pub fn ensure_not_async_fn(sig: &syn::Signature) -> syn::Result<()> {
if let Some(asyncness) = &sig.asyncness {
bail_spanned!(
@ -175,3 +123,17 @@ pub fn ensure_not_async_fn(sig: &syn::Signature) -> syn::Result<()> {
};
Ok(())
}
pub fn unwrap_group(mut expr: &syn::Expr) -> &syn::Expr {
while let syn::Expr::Group(g) = expr {
expr = &*g.expr;
}
expr
}
pub fn unwrap_ty_group(mut ty: &syn::Type) -> &syn::Type {
while let syn::Type::Group(g) = ty {
ty = &*g.elem;
}
ty
}

4
pyo3-macros/Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "pyo3-macros"
version = "0.13.2"
version = "0.14.1"
description = "Proc macros for PyO3 package"
authors = ["PyO3 Project and Contributors <https://github.com/PyO3>"]
keywords = ["pyo3", "python", "cpython", "ffi"]
@ -16,4 +16,4 @@ proc-macro = true
[dependencies]
quote = "1"
syn = { version = "1", features = ["full", "extra-traits"] }
pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.13.2" }
pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.14.1" }

98
pyo3-macros/src/lib.rs
View File

@ -9,35 +9,55 @@ use proc_macro::TokenStream;
use pyo3_macros_backend::{
build_derive_from_pyobject, build_py_class, build_py_function, build_py_methods,
build_py_proto, get_doc, process_functions_in_module, py_init, PyClassArgs, PyClassMethodsType,
PyFunctionOptions,
PyFunctionOptions, PyModuleOptions,
};
use quote::quote;
use syn::parse_macro_input;
/// A proc macro used to implement Python modules.
///
/// For more on creating Python modules
/// see the [module section of the guide](https://pyo3.rs/main/module.html).
/// The name of the module will be taken from the function name, unless `#[pyo3(name = "my_name")]`
/// is also annotated on the function to override the name. **Important**: the module name should
/// match the `lib.name` setting in `Cargo.toml`, so that Python is able to import the module
/// without needing a custom import loader.
///
/// Functions annotated with `#[pymodule]` can also be annotated with the following:
///
/// | Annotation | Description |
/// | :- | :- |
/// | `#[pyo3(name = "...")]` | Defines the name of the module in Python. |
///
/// For more on creating Python modules see the [module section of the guide][1].
///
/// [1]: https://pyo3.rs/latest/module.html
#[proc_macro_attribute]
pub fn pymodule(attr: TokenStream, input: TokenStream) -> TokenStream {
let mut ast = parse_macro_input!(input as syn::ItemFn);
let modname = if attr.is_empty() {
ast.sig.ident.clone()
let deprecated_pymodule_name_arg = if attr.is_empty() {
None
} else {
parse_macro_input!(attr as syn::Ident)
Some(parse_macro_input!(attr as syn::Ident))
};
let options = match PyModuleOptions::from_pymodule_arg_and_attrs(
deprecated_pymodule_name_arg,
&mut ast.attrs,
) {
Ok(options) => options,
Err(e) => return e.to_compile_error().into(),
};
if let Err(err) = process_functions_in_module(&mut ast) {
return err.to_compile_error().into();
}
let doc = match get_doc(&ast.attrs, None, false) {
let doc = match get_doc(&ast.attrs, None) {
Ok(doc) => doc,
Err(err) => return err.to_compile_error().into(),
};
let expanded = py_init(&ast.sig.ident, &modname, doc);
let expanded = py_init(&ast.sig.ident, options, doc);
quote!(
#ast
@ -86,11 +106,11 @@ pub fn pyproto(_: TokenStream, input: TokenStream) -> TokenStream {
/// For more on creating Python classes,
/// see the [class section of the guide][1].
///
/// [1]: https://pyo3.rs/main/class.html
/// [2]: https://pyo3.rs/main/class.html#customizing-the-class
/// [1]: https://pyo3.rs/latest/class.html
/// [2]: https://pyo3.rs/latest/class.html#customizing-the-class
/// [3]: std::marker::Send
/// [4]: ../prelude/struct.PyAny.html
/// [5]: https://pyo3.rs/main/class/protocols.html#garbage-collector-integration
/// [5]: https://pyo3.rs/latest/class/protocols.html#garbage-collector-integration
/// [6]: https://docs.python.org/3/library/weakref.html
/// [7]: https://doc.rust-lang.org/nomicon/leaking.html
/// [8]: std::rc::Rc
@ -119,11 +139,11 @@ pub fn pyclass(attr: TokenStream, input: TokenStream) -> TokenStream {
/// For more on creating Python classes,
/// see the [class section of the guide][1].
///
/// [1]: https://pyo3.rs/main/class.html
/// [2]: https://pyo3.rs/main/class.html#customizing-the-class
/// [1]: https://pyo3.rs/latest/class.html
/// [2]: https://pyo3.rs/latest/class.html#customizing-the-class
/// [3]: std::marker::Send
/// [4]: ../prelude/struct.PyAny.html
/// [5]: https://pyo3.rs/main/class/protocols.html#garbage-collector-integration
/// [5]: https://pyo3.rs/latest/class/protocols.html#garbage-collector-integration
/// [6]: https://docs.python.org/3/library/weakref.html
/// [7]: https://doc.rust-lang.org/nomicon/leaking.html
/// [8]: std::rc::Rc
@ -155,17 +175,17 @@ pub fn pyclass_with_inventory(attr: TokenStream, input: TokenStream) -> TokenStr
/// multiple `#[pymethods]` blocks for a single `#[pyclass]`.
/// This will add a transitive dependency on the [`inventory`][3] crate.
///
/// [1]: https://pyo3.rs/main/class.html#instance-methods
/// [2]: https://pyo3.rs/main/features.html#multiple-pymethods
/// [1]: https://pyo3.rs/latest/class.html#instance-methods
/// [2]: https://pyo3.rs/latest/features.html#multiple-pymethods
/// [3]: https://docs.rs/inventory/
/// [4]: https://pyo3.rs/main/class.html#constructor
/// [5]: https://pyo3.rs/main/class.html#object-properties-using-getter-and-setter
/// [6]: https://pyo3.rs/main/class.html#static-methods
/// [7]: https://pyo3.rs/main/class.html#class-methods
/// [8]: https://pyo3.rs/main/class.html#callable-objects
/// [9]: https://pyo3.rs/main/class.html#class-attributes
/// [10]: https://pyo3.rs/main/class.html#method-arguments
/// [11]: https://pyo3.rs/main/class.html#object-properties-using-pyo3get-set
/// [4]: https://pyo3.rs/latest/class.html#constructor
/// [5]: https://pyo3.rs/latest/class.html#object-properties-using-getter-and-setter
/// [6]: https://pyo3.rs/latest/class.html#static-methods
/// [7]: https://pyo3.rs/latest/class.html#class-methods
/// [8]: https://pyo3.rs/latest/class.html#callable-objects
/// [9]: https://pyo3.rs/latest/class.html#class-attributes
/// [10]: https://pyo3.rs/latest/class.html#method-arguments
/// [11]: https://pyo3.rs/latest/class.html#object-properties-using-pyo3get-set
#[proc_macro_attribute]
pub fn pymethods(_: TokenStream, input: TokenStream) -> TokenStream {
pymethods_impl(input, PyClassMethodsType::Specialization)
@ -185,8 +205,8 @@ pub fn pymethods(_: TokenStream, input: TokenStream) -> TokenStream {
/// | [`#[classattr]`][9] | Defines a class variable. |
/// | [`#[args]`][10] | Define a method's default arguments and allows the function to receive `*args` and `**kwargs`. |
///
/// Methods within a `#[pymethods]` block can also be annotated with any of the attributes which can
/// be used with [`#[pyfunction]`][attr.pyfunction.html].
/// Methods within a `#[pymethods]` block can also be annotated with any of the `#[pyo3]` options which can
/// be used with [`#[pyfunction]`][attr.pyfunction.html], except for `pass_module`.
///
/// For more on creating class methods see the [class section of the guide][1].
///
@ -194,17 +214,17 @@ pub fn pymethods(_: TokenStream, input: TokenStream) -> TokenStream {
/// multiple `#[pymethods]` blocks for a single `#[pyclass]`.
/// This will add a transitive dependency on the [`inventory`][3] crate.
///
/// [1]: https://pyo3.rs/main/class.html#instance-methods
/// [2]: https://pyo3.rs/main/features.html#multiple-pymethods
/// [1]: https://pyo3.rs/latest/class.html#instance-methods
/// [2]: https://pyo3.rs/latest/features.html#multiple-pymethods
/// [3]: https://docs.rs/inventory/
/// [4]: https://pyo3.rs/main/class.html#constructor
/// [5]: https://pyo3.rs/main/class.html#object-properties-using-getter-and-setter
/// [6]: https://pyo3.rs/main/class.html#static-methods
/// [7]: https://pyo3.rs/main/class.html#class-methods
/// [8]: https://pyo3.rs/main/class.html#callable-objects
/// [9]: https://pyo3.rs/main/class.html#class-attributes
/// [10]: https://pyo3.rs/main/class.html#method-arguments
/// [11]: https://pyo3.rs/main/class.html#object-properties-using-pyo3get-set
/// [4]: https://pyo3.rs/latest/class.html#constructor
/// [5]: https://pyo3.rs/latest/class.html#object-properties-using-getter-and-setter
/// [6]: https://pyo3.rs/latest/class.html#static-methods
/// [7]: https://pyo3.rs/latest/class.html#class-methods
/// [8]: https://pyo3.rs/latest/class.html#callable-objects
/// [9]: https://pyo3.rs/latest/class.html#class-attributes
/// [10]: https://pyo3.rs/latest/class.html#method-arguments
/// [11]: https://pyo3.rs/latest/class.html#object-properties-using-pyo3get-set
#[proc_macro_attribute]
pub fn pymethods_with_inventory(_: TokenStream, input: TokenStream) -> TokenStream {
pymethods_impl(input, PyClassMethodsType::Inventory)
@ -212,15 +232,17 @@ pub fn pymethods_with_inventory(_: TokenStream, input: TokenStream) -> TokenStre
/// A proc macro used to expose Rust functions to Python.
///
/// Functions annotated with `#[pyfunction]` can also be annotated with the following:
/// Functions annotated with `#[pyfunction]` can also be annotated with the following `#[pyo3]` options:
///
/// | Annotation | Description |
/// | :- | :- |
/// | `#[pyo3(name = "...")]` | Defines the name of the function in Python. |
/// | `#[pyo3(text_signature = "...")]` | Defines the `__text_signature__` attribute of the function in Python. |
/// | `#[pyo3(pass_module)]` | Passes the module containing the function as a `&PyModule` first argument to the function. |
///
/// For more on exposing functions see the [function section of the guide][1].
///
/// [1]: https://pyo3.rs/main/function.html
/// [1]: https://pyo3.rs/latest/function.html
#[proc_macro_attribute]
pub fn pyfunction(attr: TokenStream, input: TokenStream) -> TokenStream {
let mut ast = parse_macro_input!(input as syn::ItemFn);

15
src/buffer.rs
View File

@ -55,16 +55,23 @@ impl<T> Debug for PyBuffer<T> {
}
}
/// Represents the type of a Python buffer element.
#[derive(Copy, Clone, Eq, PartialEq)]
pub enum ElementType {
/// A signed integer type and its width in bytes.
SignedInteger { bytes: usize },
/// An unsigned integer type and its width in bytes.
UnsignedInteger { bytes: usize },
/// A boolean type.
Bool,
/// A float type and its width in bytes.
Float { bytes: usize },
/// An unknown type. This may occur when parsing has failed.
Unknown,
}
impl ElementType {
/// Determines the `ElementType` from a Python `struct` module format string.
pub fn from_format(format: &CStr) -> ElementType {
match format.to_bytes() {
[char] | [b'@', char] => native_element_type_from_type_char(*char),
@ -590,7 +597,7 @@ impl<T> Drop for PyBuffer<T> {
}
}
/// Like `std::mem::cell`, but only provides read-only access to the data.
/// Like [std::cell::Cell], but only provides read-only access to the data.
///
/// `&ReadOnlyCell<T>` is basically a safe version of `*const T`:
/// The data cannot be modified through the reference, but other references may
@ -599,11 +606,13 @@ impl<T> Drop for PyBuffer<T> {
pub struct ReadOnlyCell<T: Element>(cell::UnsafeCell<T>);
impl<T: Element> ReadOnlyCell<T> {
/// Returns a copy of the current value.
#[inline]
pub fn get(&self) -> T {
unsafe { *self.0.get() }
}
/// Returns a pointer to the current value.
#[inline]
pub fn as_ptr(&self) -> *const T {
self.0.get()
@ -638,7 +647,7 @@ impl_element!(f32, Float);
impl_element!(f64, Float);
#[cfg(test)]
mod test {
mod tests {
use super::PyBuffer;
use crate::ffi;
use crate::Python;
@ -657,7 +666,7 @@ mod test {
let gil = Python::acquire_gil();
let py = gil.python();
let bytes = py.eval("b'abcde'", None, None).unwrap();
let buffer = PyBuffer::get(&bytes).unwrap();
let buffer = PyBuffer::get(bytes).unwrap();
assert_eq!(buffer.dimensions(), 1);
assert_eq!(buffer.item_count(), 5);
assert_eq!(buffer.format().to_str().unwrap(), "B");

8
src/class/basic.rs
View File

@ -12,14 +12,20 @@ use crate::callback::{HashCallbackOutput, IntoPyCallbackOutput};
use crate::{exceptions, ffi, FromPyObject, PyAny, PyCell, PyClass, PyObject};
use std::os::raw::c_int;
/// Operators for the __richcmp__ method
/// Operators for the `__richcmp__` method
#[derive(Debug)]
pub enum CompareOp {
/// The *less than* operator.
Lt = ffi::Py_LT as isize,
/// The *less than or equal to* operator.
Le = ffi::Py_LE as isize,
/// The equality operator.
Eq = ffi::Py_EQ as isize,
/// The *not equal to* operator.
Ne = ffi::Py_NE as isize,
/// The *greater than* operator.
Gt = ffi::Py_GT as isize,
/// The *greater than or equal to* operator.
Ge = ffi::Py_GE as isize,
}

143
src/class/impl_.rs
View File

@ -2,12 +2,13 @@
use crate::{
ffi,
impl_::freelist::FreeList,
pycell::PyCellLayout,
pyclass_init::PyObjectInit,
type_object::{PyLayout, PyTypeObject},
PyClass, PyMethodDefType, PyNativeType, PyTypeInfo,
PyClass, PyMethodDefType, PyNativeType, PyTypeInfo, Python,
};
use std::{marker::PhantomData, thread};
use std::{marker::PhantomData, os::raw::c_void, thread};
/// This type is used as a "dummy" type on which dtolnay specializations are
/// applied to apply implementations from `#[pymethods]` & `#[pyproto]`
@ -65,14 +66,20 @@ pub trait PyClassImpl: Sized {
/// can be accessed by multiple threads by `threading` module.
type ThreadChecker: PyClassThreadChecker<Self>;
fn for_each_method_def(_visitor: &mut dyn FnMut(&PyMethodDefType)) {}
fn for_each_method_def(_visitor: &mut dyn FnMut(&[PyMethodDefType])) {}
fn get_new() -> Option<ffi::newfunc> {
None
}
fn get_call() -> Option<ffi::PyCFunctionWithKeywords> {
None
}
fn for_each_proto_slot(_visitor: &mut dyn FnMut(&ffi::PyType_Slot)) {}
fn get_alloc() -> Option<ffi::allocfunc> {
None
}
fn get_free() -> Option<ffi::freefunc> {
None
}
fn for_each_proto_slot(_visitor: &mut dyn FnMut(&[ffi::PyType_Slot])) {}
fn get_buffer() -> Option<&'static PyBufferProcs> {
None
}
@ -100,6 +107,112 @@ impl<T> PyClassCallImpl<T> for &'_ PyClassImplCollector<T> {
}
}
pub trait PyClassAllocImpl<T> {
fn alloc_impl(self) -> Option<ffi::allocfunc>;
}
impl<T> PyClassAllocImpl<T> for &'_ PyClassImplCollector<T> {
fn alloc_impl(self) -> Option<ffi::allocfunc> {
None
}
}
pub trait PyClassFreeImpl<T> {
fn free_impl(self) -> Option<ffi::freefunc>;
}
impl<T> PyClassFreeImpl<T> for &'_ PyClassImplCollector<T> {
fn free_impl(self) -> Option<ffi::freefunc> {
None
}
}
/// Implements a freelist.
///
/// Do not implement this trait manually. Instead, use `#[pyclass(freelist = N)]`
/// on a Rust struct to implement it.
pub trait PyClassWithFreeList: PyClass {
fn get_free_list(py: Python) -> &mut FreeList<*mut ffi::PyObject>;
}
/// Implementation of tp_alloc for `freelist` classes.
///
/// # Safety
/// - `subtype` must be a valid pointer to the type object of T or a subclass.
/// - The GIL must be held.
pub unsafe extern "C" fn alloc_with_freelist<T: PyClassWithFreeList>(
subtype: *mut ffi::PyTypeObject,
nitems: ffi::Py_ssize_t,
) -> *mut ffi::PyObject {
let py = Python::assume_gil_acquired();
#[cfg(not(Py_3_8))]
bpo_35810_workaround(py, subtype);
let self_type = T::type_object_raw(py);
// If this type is a variable type or the subtype is not equal to this type, we cannot use the
// freelist
if nitems == 0 && subtype == self_type {
if let Some(obj) = T::get_free_list(py).pop() {
ffi::PyObject_Init(obj, subtype);
return obj as _;
}
}
ffi::PyType_GenericAlloc(subtype, nitems)
}
/// Implementation of tp_free for `freelist` classes.
///
/// # Safety
/// - `obj` must be a valid pointer to an instance of T (not a subclass).
/// - The GIL must be held.
#[allow(clippy::collapsible_if)] // for if cfg!
pub unsafe extern "C" fn free_with_freelist<T: PyClassWithFreeList>(obj: *mut c_void) {
let obj = obj as *mut ffi::PyObject;
debug_assert_eq!(
T::type_object_raw(Python::assume_gil_acquired()),
ffi::Py_TYPE(obj)
);
if let Some(obj) = T::get_free_list(Python::assume_gil_acquired()).insert(obj) {
let ty = ffi::Py_TYPE(obj);
// Deduce appropriate inverse of PyType_GenericAlloc
let free = if ffi::PyType_IS_GC(ty) != 0 {
ffi::PyObject_GC_Del
} else {
ffi::PyObject_Free
};
free(obj as *mut c_void);
if cfg!(Py_3_8) {
if ffi::PyType_HasFeature(ty, ffi::Py_TPFLAGS_HEAPTYPE) != 0 {
ffi::Py_DECREF(ty as *mut ffi::PyObject);
}
}
}
}
/// Workaround for Python issue 35810; no longer necessary in Python 3.8
#[inline]
#[cfg(not(Py_3_8))]
unsafe fn bpo_35810_workaround(_py: Python, ty: *mut ffi::PyTypeObject) {
#[cfg(Py_LIMITED_API)]
{
// Must check version at runtime for abi3 wheels - they could run against a higher version
// than the build config suggests.
use crate::once_cell::GILOnceCell;
static IS_PYTHON_3_8: GILOnceCell<bool> = GILOnceCell::new();
if *IS_PYTHON_3_8.get_or_init(_py, || _py.version_info() >= (3, 8)) {
// No fix needed - the wheel is running on a sufficiently new interpreter.
return;
}
}
ffi::Py_INCREF(ty as *mut ffi::PyObject);
}
// General methods implementation: either dtolnay specialization trait or inventory if
// multiple-pymethods feature is enabled.
@ -216,6 +329,7 @@ pub struct ThreadCheckerStub<T: Send>(PhantomData<T>);
impl<T: Send> PyClassThreadChecker<T> for ThreadCheckerStub<T> {
fn ensure(&self) {}
#[inline]
fn new() -> Self {
ThreadCheckerStub(PhantomData)
}
@ -224,6 +338,7 @@ impl<T: Send> PyClassThreadChecker<T> for ThreadCheckerStub<T> {
impl<T: PyNativeType> PyClassThreadChecker<T> for ThreadCheckerStub<crate::PyObject> {
fn ensure(&self) {}
#[inline]
fn new() -> Self {
ThreadCheckerStub(PhantomData)
}
@ -279,8 +394,26 @@ pub trait PyClassBaseType: Sized {
impl<T: PyClass> PyClassBaseType for T {
type Dict = T::Dict;
type WeakRef = T::WeakRef;
type LayoutAsBase = crate::pycell::PyCellInner<T>;
type LayoutAsBase = crate::pycell::PyCell<T>;
type BaseNativeType = T::BaseNativeType;
type ThreadChecker = T::ThreadChecker;
type Initializer = crate::pyclass_init::PyClassInitializer<Self>;
}
/// Default new implementation
pub(crate) unsafe extern "C" fn fallback_new(
_subtype: *mut ffi::PyTypeObject,
_args: *mut ffi::PyObject,
_kwds: *mut ffi::PyObject,
) -> *mut ffi::PyObject {
crate::callback_body!(py, {
Err::<(), _>(crate::exceptions::PyTypeError::new_err(
"No constructor defined",
))
})
}
/// Implementation of tp_dealloc for all pyclasses
pub(crate) unsafe extern "C" fn tp_dealloc<T: PyClass>(obj: *mut ffi::PyObject) {
crate::callback_body!(py, T::Layout::tp_dealloc(obj, py))
}

2
src/class/iter.rs
View File

@ -78,7 +78,9 @@ py_unarys_func!(iternext, PyIterNextProtocol, Self::__next__);
///
/// See [`PyIterProtocol`](trait.PyIterProtocol.html) for an example.
pub enum IterNextOutput<T, U> {
/// The value yielded by the iterator.
Yield(T),
/// The `StopIteration` object.
Return(U),
}

24
src/class/methods.rs
View File

@ -28,6 +28,8 @@ pub enum PyMethodDefType {
pub enum PyMethodType {
PyCFunction(PyCFunction),
PyCFunctionWithKeywords(PyCFunctionWithKeywords),
#[cfg(all(Py_3_7, not(Py_LIMITED_API)))]
PyCFunctionFastWithKeywords(PyCFunctionFastWithKeywords),
}
// These newtype structs serve no purpose other than wrapping which are function pointers - because
@ -36,6 +38,9 @@ pub enum PyMethodType {
pub struct PyCFunction(pub ffi::PyCFunction);
#[derive(Clone, Copy, Debug)]
pub struct PyCFunctionWithKeywords(pub ffi::PyCFunctionWithKeywords);
#[cfg(all(Py_3_7, not(Py_LIMITED_API)))]
#[derive(Clone, Copy, Debug)]
pub struct PyCFunctionFastWithKeywords(pub ffi::_PyCFunctionFastWithKeywords);
#[derive(Clone, Copy, Debug)]
pub struct PyGetter(pub ffi::getter);
#[derive(Clone, Copy, Debug)]
@ -105,6 +110,21 @@ impl PyMethodDef {
}
}
/// Define a function that can take `*args` and `**kwargs`.
#[cfg(all(Py_3_7, not(Py_LIMITED_API)))]
pub const fn fastcall_cfunction_with_keywords(
name: &'static str,
cfunction: PyCFunctionFastWithKeywords,
doc: &'static str,
) -> Self {
Self {
ml_name: name,
ml_meth: PyMethodType::PyCFunctionFastWithKeywords(cfunction),
ml_flags: ffi::METH_FASTCALL | ffi::METH_KEYWORDS,
ml_doc: doc,
}
}
pub const fn flags(mut self, flags: c_int) -> Self {
self.ml_flags |= flags;
self
@ -115,6 +135,10 @@ impl PyMethodDef {
let meth = match self.ml_meth {
PyMethodType::PyCFunction(meth) => meth.0,
PyMethodType::PyCFunctionWithKeywords(meth) => unsafe { std::mem::transmute(meth.0) },
#[cfg(all(Py_3_7, not(Py_LIMITED_API)))]
PyMethodType::PyCFunctionFastWithKeywords(meth) => unsafe {
std::mem::transmute(meth.0)
},
};
Ok(ffi::PyMethodDef {

5
src/class/pyasync.rs
View File

@ -98,11 +98,16 @@ py_unarys_func!(aiter, PyAsyncAiterProtocol, Self::__aiter__);
py_unarys_func!(anext, PyAsyncAnextProtocol, Self::__anext__);
/// Output of `__anext__`.
///
/// <https://docs.python.org/3/reference/expressions.html#agen.__anext__>
pub enum IterANextOutput<T, U> {
/// An expression which the generator yielded.
Yield(T),
/// A `StopAsyncIteration` object.
Return(U),
}
/// An [IterANextOutput] of Python objects.
pub type PyIterANextOutput = IterANextOutput<PyObject, PyObject>;
impl IntoPyCallbackOutput<*mut ffi::PyObject> for PyIterANextOutput {

2
src/conversion.rs
View File

@ -543,7 +543,7 @@ where
}
#[cfg(test)]
mod test {
mod tests {
use crate::types::{IntoPyDict, PyAny, PyDict, PyList};
use crate::{Python, ToPyObject};

4
src/conversions/array.rs
View File

@ -107,7 +107,7 @@ mod min_const_generics {
}
#[cfg(test)]
mod test {
mod tests {
use super::*;
use std::{
panic,
@ -257,7 +257,7 @@ fn invalid_sequence_length(expected: usize, actual: usize) -> PyErr {
}
#[cfg(test)]
mod test {
mod tests {
use crate::{PyResult, Python};
#[test]

219
src/conversions/indexmap.rs Normal file
View File

@ -0,0 +1,219 @@
//! Conversions to and from [indexmap](https://docs.rs/indexmap/)’s
//! `IndexMap`.
//!
//! [`indexmap::IndexMap`] is a hash table that is closely compatible with the standard [`std::collections::HashMap`],
//! with the difference that it preserves the insertion order when iterating over keys. It was inspired
//! by Python's 3.6+ dict implementation.
//!
//! Dictionary order is guaranteed to be insertion order in Python, hence IndexMap is a good candidate
//! for maintaining an equivalent behaviour in Rust.
//!
//! # Setup
//!
//! To use this feature, add this to your **`Cargo.toml`**:
//!
//! ```toml
//! [dependencies]
//! # change * to the latest versions
//! indexmap = "*"
// workaround for `extended_key_value_attributes`: https://github.com/rust-lang/rust/issues/82768#issuecomment-803935643
#![cfg_attr(docsrs, cfg_attr(docsrs, doc = concat!("pyo3 = { version = \"", env!("CARGO_PKG_VERSION"), "\", features = [\"indexmap\"] }")))]
#![cfg_attr(
not(docsrs),
doc = "pyo3 = { version = \"*\", features = [\"indexmap\"] }"
)]
//! ```
//!
//! Note that you must use compatible versions of indexmap and PyO3.
//! The required indexmap version may vary based on the version of PyO3.
//!
//! # Examples
//!
//! Using [indexmap](https://docs.rs/indexmap) to return a dictionary with some statistics
//! about a list of numbers. Because of the insertion order guarantees, the Python code will
//! always print the same result, matching users' expectations about Python's dict.
//!
//! ```rust
//! use indexmap::{indexmap, IndexMap};
//! use pyo3::prelude::*;
//!
//! fn median(data: &Vec<i32>) -> f32 {
//! let sorted_data = data.clone().sort();
//! let mid = data.len() / 2;
//! if (data.len() % 2 == 0) {
//! data[mid] as f32
//! }
//! else {
//! (data[mid] + data[mid - 1]) as f32 / 2.0
//! }
//! }
//!
//! fn mean(data: &Vec<i32>) -> f32 {
//! data.iter().sum::<i32>() as f32 / data.len() as f32
//! }
//! fn mode(data: &Vec<i32>) -> f32 {
//! let mut frequency = IndexMap::new(); // we can use IndexMap as any hash table
//!
//! for &element in data {
//! *frequency.entry(element).or_insert(0) += 1;
//! }
//!
//! frequency
//! .iter()
//! .max_by(|a, b| a.1.cmp(&b.1))
//! .map(|(k, _v)| *k)
//! .unwrap() as f32
//! }
//!
//! #[pyfunction]
//! fn calculate_statistics(data: Vec<i32>) -> IndexMap<&'static str, f32> {
//! indexmap!{
//! "median" => median(&data),
//! "mean" => mean(&data),
//! "mode" => mode(&data),
//! }
//! }
//!
//! #[pymodule]
//! fn my_module(_py: Python, m: &PyModule) -> PyResult<()> {
//! m.add_function(wrap_pyfunction!(calculate_statistics, m)?)?;
//! Ok(())
//! }
//! ```
//!
//! Python code:
//! ```python
//! from my_module import calculate_statistics
//!
//! data = [1, 1, 1, 3, 4, 5]
//! print(calculate_statistics(data))
//! # always prints {"median": 2.0, "mean": 2.5, "mode": 1.0} in the same order
//! # if another hash table was used, the order could be random
//! ```
use crate::types::*;
use crate::{FromPyObject, IntoPy, PyErr, PyObject, PyTryFrom, Python, ToPyObject};
use std::{cmp, hash};
impl<K, V, H> ToPyObject for indexmap::IndexMap<K, V, H>
where
K: hash::Hash + cmp::Eq + ToPyObject,
V: ToPyObject,
H: hash::BuildHasher,
{
fn to_object(&self, py: Python) -> PyObject {
IntoPyDict::into_py_dict(self, py).into()
}
}
impl<K, V, H> IntoPy<PyObject> for indexmap::IndexMap<K, V, H>
where
K: hash::Hash + cmp::Eq + IntoPy<PyObject>,
V: IntoPy<PyObject>,
H: hash::BuildHasher,
{
fn into_py(self, py: Python) -> PyObject {
let iter = self
.into_iter()
.map(|(k, v)| (k.into_py(py), v.into_py(py)));
IntoPyDict::into_py_dict(iter, py).into()
}
}
impl<'source, K, V, S> FromPyObject<'source> for indexmap::IndexMap<K, V, S>
where
K: FromPyObject<'source> + cmp::Eq + hash::Hash,
V: FromPyObject<'source>,
S: hash::BuildHasher + Default,
{
fn extract(ob: &'source PyAny) -> Result<Self, PyErr> {
let dict = <PyDict as PyTryFrom>::try_from(ob)?;
let mut ret = indexmap::IndexMap::with_capacity_and_hasher(dict.len(), S::default());
for (k, v) in dict.iter() {
ret.insert(K::extract(k)?, V::extract(v)?);
}
Ok(ret)
}
}
#[cfg(test)]
mod test_indexmap {
use crate::types::*;
use crate::{IntoPy, PyObject, PyTryFrom, Python, ToPyObject};
#[test]
fn test_indexmap_indexmap_to_python() {
Python::with_gil(|py| {
let mut map = indexmap::IndexMap::<i32, i32>::new();
map.insert(1, 1);
let m = map.to_object(py);
let py_map = <PyDict as PyTryFrom>::try_from(m.as_ref(py)).unwrap();
assert!(py_map.len() == 1);
assert!(py_map.get_item(1).unwrap().extract::<i32>().unwrap() == 1);
assert_eq!(
map,
py_map.extract::<indexmap::IndexMap::<i32, i32>>().unwrap()
);
});
}
#[test]
fn test_indexmap_indexmap_into_python() {
Python::with_gil(|py| {
let mut map = indexmap::IndexMap::<i32, i32>::new();
map.insert(1, 1);
let m: PyObject = map.into_py(py);
let py_map = <PyDict as PyTryFrom>::try_from(m.as_ref(py)).unwrap();
assert!(py_map.len() == 1);
assert!(py_map.get_item(1).unwrap().extract::<i32>().unwrap() == 1);
});
}
#[test]
fn test_indexmap_indexmap_into_dict() {
Python::with_gil(|py| {
let mut map = indexmap::IndexMap::<i32, i32>::new();
map.insert(1, 1);
let py_map = map.into_py_dict(py);
assert_eq!(py_map.len(), 1);
assert_eq!(py_map.get_item(1).unwrap().extract::<i32>().unwrap(), 1);
});
}
#[test]
fn test_indexmap_indexmap_insertion_order_round_trip() {
Python::with_gil(|py| {
let n = 20;
let mut map = indexmap::IndexMap::<i32, i32>::new();
for i in 1..=n {
if i % 2 == 1 {
map.insert(i, i);
} else {
map.insert(n - i, i);
}
}
let py_map = map.clone().into_py_dict(py);
let trip_map = py_map.extract::<indexmap::IndexMap<i32, i32>>().unwrap();
for (((k1, v1), (k2, v2)), (k3, v3)) in
map.iter().zip(py_map.iter()).zip(trip_map.iter())
{
let k2 = k2.extract::<i32>().unwrap();
let v2 = v2.extract::<i32>().unwrap();
assert_eq!((k1, v1), (&k2, &v2));
assert_eq!((k1, v1), (k3, v3));
assert_eq!((&k2, &v2), (k3, v3));
}
});
}
}

3
src/conversions/mod.rs
View File

@ -1,5 +1,8 @@
//! This module contains conversions between various Rust object and their representation in Python.
mod array;
#[cfg(feature = "indexmap")]
#[cfg_attr(docsrs, doc(cfg(feature = "indexmap")))]
pub mod indexmap;
mod osstr;
mod path;

13
src/conversions/osstr.rs
View File

@ -121,7 +121,7 @@ impl IntoPy<PyObject> for &'_ OsStr {
impl ToPyObject for Cow<'_, OsStr> {
#[inline]
fn to_object(&self, py: Python) -> PyObject {
(&self as &OsStr).to_object(py)
(self as &OsStr).to_object(py)
}
}
@ -135,7 +135,7 @@ impl IntoPy<PyObject> for Cow<'_, OsStr> {
impl ToPyObject for OsString {
#[inline]
fn to_object(&self, py: Python) -> PyObject {
(&self as &OsStr).to_object(py)
(self as &OsStr).to_object(py)
}
}
@ -145,8 +145,14 @@ impl IntoPy<PyObject> for OsString {
}
}
impl<'a> IntoPy<PyObject> for &'a OsString {
fn into_py(self, py: Python) -> PyObject {
self.to_object(py)
}
}
#[cfg(test)]
mod test {
mod tests {
use crate::{types::PyString, IntoPy, PyObject, Python, ToPyObject};
use std::fmt::Debug;
use std::{
@ -205,6 +211,7 @@ mod test {
let os_str = OsStr::new("Hello\0\n🐍");
test_roundtrip::<&OsStr>(py, os_str);
test_roundtrip::<OsString>(py, os_str.to_os_string());
test_roundtrip::<&OsString>(py, &os_str.to_os_string());
})
}
}

30
src/conversions/path.rs
View File

@ -1,4 +1,5 @@
use crate::{FromPyObject, IntoPy, PyAny, PyObject, PyResult, Python, ToPyObject};
use crate::types::PyType;
use crate::{FromPyObject, IntoPy, PyAny, PyNativeType, PyObject, PyResult, Python, ToPyObject};
use std::borrow::Cow;
use std::ffi::OsString;
use std::path::{Path, PathBuf};
@ -13,7 +14,21 @@ impl ToPyObject for Path {
impl FromPyObject<'_> for PathBuf {
fn extract(ob: &PyAny) -> PyResult<Self> {
Ok(PathBuf::from(OsString::extract(ob)?))
let os_str = match OsString::extract(ob) {
Ok(s) => s,
Err(err) => {
let py = ob.py();
let pathlib = py.import("pathlib")?;
let pathlib_path: &PyType = pathlib.getattr("Path")?.downcast()?;
if pathlib_path.is_instance(ob)? {
let path_str = ob.call_method0("__str__")?;
OsString::extract(path_str)?
} else {
return Err(err);
}
}
};
Ok(PathBuf::from(os_str))
}
}
@ -51,8 +66,14 @@ impl IntoPy<PyObject> for PathBuf {
}
}
impl<'a> IntoPy<PyObject> for &'a PathBuf {
fn into_py(self, py: Python) -> PyObject {
self.as_os_str().to_object(py)
}
}
#[cfg(test)]
mod test {
mod tests {
use crate::{types::PyString, IntoPy, PyObject, Python, ToPyObject};
use std::borrow::Cow;
use std::fmt::Debug;
@ -105,11 +126,12 @@ mod test {
let pystring: &PyString = pyobject.extract(py).unwrap();
assert_eq!(pystring.to_string_lossy(), obj.as_ref().to_string_lossy());
let roundtripped_obj: PathBuf = pystring.extract().unwrap();
assert!(obj.as_ref() == roundtripped_obj.as_path());
assert_eq!(obj.as_ref(), roundtripped_obj.as_path());
}
let path = Path::new("Hello\0\n🐍");
test_roundtrip::<&Path>(py, path);
test_roundtrip::<PathBuf>(py, path.to_path_buf());
test_roundtrip::<&PathBuf>(py, &path.to_path_buf());
})
}
}

31
src/derive_utils.rs
View File

@ -39,6 +39,7 @@ impl FunctionDescription {
format!("{}()", self.func_name)
}
}
/// Extracts the `args` and `kwargs` provided into `output`, according to this function
/// definition.
///
@ -52,8 +53,9 @@ impl FunctionDescription {
/// Unexpected, duplicate or invalid arguments will cause this function to return `TypeError`.
pub fn extract_arguments<'p>(
&self,
args: &'p PyTuple,
kwargs: Option<&'p PyDict>,
py: Python<'p>,
mut args: impl ExactSizeIterator<Item = &'p PyAny>,
kwargs: Option<impl Iterator<Item = (&'p PyAny, &'p PyAny)>>,
output: &mut [Option<&'p PyAny>],
) -> PyResult<(Option<&'p PyTuple>, Option<&'p PyDict>)> {
let num_positional_parameters = self.positional_parameter_names.len();
@ -66,33 +68,36 @@ impl FunctionDescription {
);
// Handle positional arguments
let (args_provided, varargs) = {
let args_provided = {
let args_provided = args.len();
if self.accept_varargs {
(
std::cmp::min(num_positional_parameters, args_provided),
Some(args.slice(num_positional_parameters as isize, args_provided as isize)),
)
std::cmp::min(num_positional_parameters, args_provided)
} else if args_provided > num_positional_parameters {
return Err(self.too_many_positional_arguments(args_provided));
} else {
(args_provided, None)
args_provided
}
};
// Copy positional arguments into output
for (out, arg) in output[..args_provided].iter_mut().zip(args) {
for (out, arg) in output[..args_provided].iter_mut().zip(args.by_ref()) {
*out = Some(arg);
}
// Collect varargs into tuple
let varargs = if self.accept_varargs {
Some(PyTuple::new(py, args))
} else {
None
};
// Handle keyword arguments
let varkeywords = match (kwargs, self.accept_varkeywords) {
(Some(kwargs), true) => {
let mut varkeywords = None;
self.extract_keyword_arguments(kwargs, output, |name, value| {
varkeywords
.get_or_insert_with(|| PyDict::new(kwargs.py()))
.get_or_insert_with(|| PyDict::new(py))
.set_item(name, value)
})?;
varkeywords
@ -146,7 +151,7 @@ impl FunctionDescription {
#[inline]
fn extract_keyword_arguments<'p>(
&self,
kwargs: &'p PyDict,
kwargs: impl Iterator<Item = (&'p PyAny, &'p PyAny)>,
output: &mut [Option<&'p PyAny>],
mut unexpected_keyword_handler: impl FnMut(&'p PyAny, &'p PyAny) -> PyResult<()>,
) -> PyResult<()> {
@ -289,7 +294,7 @@ impl ModuleDef {
/// Make new module defenition with given module name.
///
/// # Safety
/// `name` must be a null-terminated string.
/// `name` and `doc` must be null-terminated strings.
pub const unsafe fn new(name: &'static str, doc: &'static str) -> Self {
const INIT: ffi::PyModuleDef = ffi::PyModuleDef {
m_base: ffi::PyModuleDef_HEAD_INIT,

34
src/err/err_state.rs
View File

@ -1,5 +1,9 @@
use crate::{
exceptions::PyBaseException, ffi, types::PyType, IntoPy, IntoPyPointer, Py, PyObject, Python,
exceptions::{PyBaseException, PyTypeError},
ffi,
type_object::PyTypeObject,
types::PyType,
AsPyPointer, IntoPy, IntoPyPointer, Py, PyObject, Python,
};
#[derive(Clone)]
@ -10,7 +14,11 @@ pub(crate) struct PyErrStateNormalized {
}
pub(crate) enum PyErrState {
Lazy {
LazyTypeAndValue {
ptype: fn(Python) -> &PyType,
pvalue: Box<dyn FnOnce(Python) -> PyObject + Send + Sync>,
},
LazyValue {
ptype: Py<PyType>,
pvalue: Box<dyn FnOnce(Python) -> PyObject + Send + Sync>,
},
@ -49,7 +57,19 @@ impl PyErrState {
py: Python,
) -> (*mut ffi::PyObject, *mut ffi::PyObject, *mut ffi::PyObject) {
match self {
PyErrState::Lazy { ptype, pvalue } => (
PyErrState::LazyTypeAndValue { ptype, pvalue } => {
let ty = ptype(py);
if unsafe { ffi::PyExceptionClass_Check(ty.as_ptr()) } == 0 {
Self::exceptions_must_derive_from_base_exception(py).into_ffi_tuple(py)
} else {
(
ptype(py).into_ptr(),
pvalue(py).into_ptr(),
std::ptr::null_mut(),
)
}
}
PyErrState::LazyValue { ptype, pvalue } => (
ptype.into_ptr(),
pvalue(py).into_ptr(),
std::ptr::null_mut(),
@ -66,4 +86,12 @@ impl PyErrState {
}) => (ptype.into_ptr(), pvalue.into_ptr(), ptraceback.into_ptr()),
}
}
#[inline]
pub(crate) fn exceptions_must_derive_from_base_exception(py: Python) -> Self {
PyErrState::LazyValue {
ptype: PyTypeError::type_object(py).into(),
pvalue: boxed_args("exceptions must derive from BaseException"),
}
}
}

109
src/err/mod.rs
View File

@ -81,7 +81,10 @@ impl PyErr {
T: PyTypeObject,
A: PyErrArguments + Send + Sync + 'static,
{
Python::with_gil(|py| PyErr::from_type(T::type_object(py), args))
PyErr::from_state(PyErrState::LazyTypeAndValue {
ptype: T::type_object,
pvalue: boxed_args(args),
})
}
/// Constructs a new error, with the usual lazy initialization of Python exceptions.
@ -97,7 +100,7 @@ impl PyErr {
return exceptions_must_derive_from_base_exception(ty.py());
}
PyErr::from_state(PyErrState::Lazy {
PyErr::from_state(PyErrState::LazyValue {
ptype: ty.into(),
pvalue: boxed_args(args),
})
@ -319,7 +322,7 @@ impl PyErr {
T: ToBorrowedObject,
{
exc.with_borrowed_ptr(py, |exc| unsafe {
ffi::PyErr_GivenExceptionMatches(self.ptype_ptr(), exc) != 0
ffi::PyErr_GivenExceptionMatches(self.ptype_ptr(py), exc) != 0
})
}
@ -329,7 +332,7 @@ impl PyErr {
T: PyTypeObject,
{
unsafe {
ffi::PyErr_GivenExceptionMatches(self.ptype_ptr(), T::type_object(py).as_ptr()) != 0
ffi::PyErr_GivenExceptionMatches(self.ptype_ptr(py), T::type_object(py).as_ptr()) != 0
}
}
@ -390,6 +393,28 @@ impl PyErr {
PyErr::from_state(PyErrState::Normalized(self.normalized(py).clone()))
}
/// Return the cause (either an exception instance, or None, set by `raise ... from ...`)
/// associated with the exception, as accessible from Python through `__cause__`.
pub fn cause(&self, py: Python) -> Option<PyErr> {
let ptr = unsafe { ffi::PyException_GetCause(self.pvalue(py).as_ptr()) };
let obj = unsafe { py.from_owned_ptr_or_opt::<PyAny>(ptr) };
obj.map(|x| Self::from_instance(x))
}
/// Set the cause associated with the exception, pass `None` to clear it.
pub fn set_cause(&self, py: Python, cause: Option<Self>) {
if let Some(cause) = cause {
let cause = cause.into_instance(py);
unsafe {
ffi::PyException_SetCause(self.pvalue(py).as_ptr(), cause.as_ptr());
}
} else {
unsafe {
ffi::PyException_SetCause(self.pvalue(py).as_ptr(), std::ptr::null_mut());
}
}
}
fn from_state(state: PyErrState) -> PyErr {
PyErr {
state: UnsafeCell::new(Some(state)),
@ -397,9 +422,12 @@ impl PyErr {
}
/// Returns borrowed reference to this Err's type
fn ptype_ptr(&self) -> *mut ffi::PyObject {
fn ptype_ptr(&self, py: Python) -> *mut ffi::PyObject {
match unsafe { &*self.state.get() } {
Some(PyErrState::Lazy { ptype, .. }) => ptype.as_ptr(),
// In lazy type case, normalize before returning ptype in case the type is not a valid
// exception type.
Some(PyErrState::LazyTypeAndValue { .. }) => self.normalized(py).ptype.as_ptr(),
Some(PyErrState::LazyValue { ptype, .. }) => ptype.as_ptr(),
Some(PyErrState::FfiTuple { ptype, .. }) => ptype.as_ptr(),
Some(PyErrState::Normalized(n)) => n.ptype.as_ptr(),
None => panic!("Cannot access exception type while normalizing"),
@ -532,10 +560,7 @@ pub fn error_on_minusone(py: Python, result: c_int) -> PyResult<()> {
#[inline]
fn exceptions_must_derive_from_base_exception(py: Python) -> PyErr {
PyErr::from_state(PyErrState::Lazy {
ptype: exceptions::PyTypeError::type_object(py).into(),
pvalue: boxed_args("exceptions must derive from BaseException"),
})
PyErr::from_state(PyErrState::exceptions_must_derive_from_base_exception(py))
}
#[cfg(test)]
@ -545,13 +570,31 @@ mod tests {
use crate::{PyErr, Python};
#[test]
fn set_typeerror() {
let gil = Python::acquire_gil();
let py = gil.python();
let err: PyErr = exceptions::PyTypeError::new_err(());
err.restore(py);
assert!(PyErr::occurred(py));
drop(PyErr::fetch(py));
fn set_valueerror() {
Python::with_gil(|py| {
let err: PyErr = exceptions::PyValueError::new_err("some exception message");
assert!(err.is_instance::<exceptions::PyValueError>(py));
err.restore(py);
assert!(PyErr::occurred(py));
let err = PyErr::fetch(py);
assert!(err.is_instance::<exceptions::PyValueError>(py));
assert_eq!(err.to_string(), "ValueError: some exception message");
})
}
#[test]
fn invalid_error_type() {
Python::with_gil(|py| {
let err: PyErr = PyErr::new::<crate::types::PyString, _>(());
assert!(err.is_instance::<exceptions::PyTypeError>(py));
err.restore(py);
let err = PyErr::fetch(py);
assert!(err.is_instance::<exceptions::PyTypeError>(py));
assert_eq!(
err.to_string(),
"TypeError: exceptions must derive from BaseException"
);
})
}
#[test]
@ -627,4 +670,36 @@ mod tests {
is_send::<PyErrState>();
is_sync::<PyErrState>();
}
#[test]
fn test_pyerr_cause() {
Python::with_gil(|py| {
let err = py
.run("raise Exception('banana')", None, None)
.expect_err("raising should have given us an error");
assert!(err.cause(py).is_none());
let err = py
.run(
"raise Exception('banana') from Exception('apple')",
None,
None,
)
.expect_err("raising should have given us an error");
let cause = err
.cause(py)
.expect("raising from should have given us a cause");
assert_eq!(cause.to_string(), "Exception: apple");
err.set_cause(py, None);
assert!(err.cause(py).is_none());
let new_cause = exceptions::PyValueError::new_err("orange");
err.set_cause(py, Some(new_cause));
let cause = err
.cause(py)
.expect("set_cause should have given us a cause");
assert_eq!(cause.to_string(), "ValueError: orange");
});
}
}

9
src/exceptions.rs
View File

@ -18,6 +18,7 @@ macro_rules! impl_exception_boilerplate {
}
impl $name {
/// Creates a new [PyErr](crate::PyErr) of this type.
pub fn new_err<A>(args: A) -> $crate::PyErr
where
A: $crate::PyErrArguments + Send + Sync + 'static,
@ -275,6 +276,7 @@ impl_native_exception!(PyIOError, PyExc_IOError);
impl_native_exception!(PyWindowsError, PyExc_WindowsError);
impl PyUnicodeDecodeError {
/// Creates a Python `UnicodeDecodeError`.
pub fn new<'p>(
py: Python<'p>,
encoding: &CStr,
@ -294,6 +296,7 @@ impl PyUnicodeDecodeError {
}
}
/// Creates a Python `UnicodeDecodeError` from a Rust UTF-8 decoding error.
pub fn new_utf8<'p>(
py: Python<'p>,
input: &[u8],
@ -329,7 +332,7 @@ pub mod socket {
}
#[cfg(test)]
mod test {
mod tests {
use super::{PyException, PyUnicodeDecodeError};
use crate::types::{IntoPyDict, PyDict};
use crate::{PyErr, Python};
@ -399,7 +402,7 @@ mod test {
let error_type = py.get_type::<CustomError>();
let ctx = [("CustomError", error_type)].into_py_dict(py);
let type_description: String = py
.eval("str(CustomError)", None, Some(&ctx))
.eval("str(CustomError)", None, Some(ctx))
.unwrap()
.extract()
.unwrap();
@ -407,7 +410,7 @@ mod test {
py.run(
"assert CustomError('oops').args == ('oops',)",
None,
Some(&ctx),
Some(ctx),
)
.unwrap();
}

4
src/ffi/ceval.rs
View File

@ -2,10 +2,6 @@ use crate::ffi::object::PyObject;
use crate::ffi::pystate::PyThreadState;
use std::os::raw::{c_char, c_int, c_void};
// TODO: move to cpython
pub type _PyFrameEvalFunction =
extern "C" fn(*mut crate::ffi::PyFrameObject, c_int) -> *mut PyObject;
extern "C" {
#[cfg_attr(Py_3_9, deprecated(note = "Python 3.9"))]
#[cfg_attr(PyPy, link_name = "PyPyEval_CallObjectWithKeywords")]

4
src/ffi/cpython/ceval.rs
View File

@ -8,6 +8,6 @@ extern "C" {
exc: c_int,
) -> *mut PyObject;
pub fn _PyEval_RequestCodeExtraIndex(func: freefunc) -> c_int;
pub fn PyEval_SetProfile(trace_func: Py_tracefunc, arg1: *mut PyObject);
pub fn PyEval_SetTrace(trace_func: Py_tracefunc, arg1: *mut PyObject);
pub fn PyEval_SetProfile(trace_func: Option<Py_tracefunc>, arg1: *mut PyObject);
pub fn PyEval_SetTrace(trace_func: Option<Py_tracefunc>, arg1: *mut PyObject);
}

3
src/ffi/cpython/listobject.rs
View File

@ -15,17 +15,20 @@ pub struct PyListObject {
/// Macro, trading safety for speed
#[inline]
#[cfg(not(PyPy))]
pub unsafe fn PyList_GET_ITEM(op: *mut PyObject, i: Py_ssize_t) -> *mut PyObject {
*(*(op as *mut PyListObject)).ob_item.offset(i as isize)
}
/// Macro, *only* to be used to fill in brand new lists
#[inline]
#[cfg(not(PyPy))]
pub unsafe fn PyList_SET_ITEM(op: *mut PyObject, i: Py_ssize_t, v: *mut PyObject) {
*(*(op as *mut PyListObject)).ob_item.offset(i as isize) = v;
}
#[inline]
#[cfg(not(PyPy))]
pub unsafe fn PyList_GET_SIZE(op: *mut PyObject) -> Py_ssize_t {
Py_SIZE(op)
}

32
src/ffi/cpython/mod.rs
View File

@ -1,23 +1,26 @@
pub mod abstract_;
pub(crate) mod abstract_;
// skipped bytearrayobject.h
#[cfg(not(PyPy))]
pub mod bytesobject;
pub mod ceval;
pub mod code;
pub mod compile;
pub(crate) mod bytesobject;
pub(crate) mod ceval;
pub(crate) mod code;
pub(crate) mod compile;
#[cfg(not(PyPy))]
pub mod dictobject;
pub(crate) mod dictobject;
// skipped fileobject.h
pub mod frameobject;
pub mod import;
pub(crate) mod frameobject;
pub(crate) mod import;
#[cfg(all(Py_3_8, not(PyPy)))]
pub mod initconfig;
pub(crate) mod initconfig;
// skipped interpreteridobject.h
pub mod listobject;
pub mod object;
pub mod pydebug;
pub(crate) mod listobject;
pub(crate) mod object;
pub(crate) mod pydebug;
#[cfg(all(Py_3_8, not(PyPy)))]
pub mod pylifecycle;
pub(crate) mod pylifecycle;
#[cfg(all(Py_3_8, not(PyPy)))]
pub(crate) mod pystate;
pub use self::abstract_::*;
#[cfg(not(PyPy))]
@ -36,3 +39,6 @@ pub use self::object::*;
pub use self::pydebug::*;
#[cfg(all(Py_3_8, not(PyPy)))]
pub use self::pylifecycle::*;
#[cfg(all(Py_3_8, not(PyPy)))]
pub use self::pystate::*;

15
src/ffi/cpython/object.rs
View File

@ -16,6 +16,9 @@ mod bufferinfo {
use std::os::raw::{c_char, c_int, c_void};
use std::ptr;
#[cfg(PyPy)]
const Py_MAX_NDIMS: usize = 36;
#[repr(C)]
#[derive(Copy, Clone)]
pub struct Py_buffer {
@ -31,6 +34,12 @@ mod bufferinfo {
pub strides: *mut Py_ssize_t,
pub suboffsets: *mut Py_ssize_t,
pub internal: *mut c_void,
#[cfg(PyPy)]
pub flags: c_int,
#[cfg(PyPy)]
pub _strides: [Py_ssize_t; Py_MAX_NDIMS],
#[cfg(PyPy)]
pub _shape: [Py_ssize_t; Py_MAX_NDIMS],
}
impl Py_buffer {
@ -47,6 +56,12 @@ mod bufferinfo {
strides: ptr::null_mut(),
suboffsets: ptr::null_mut(),
internal: ptr::null_mut(),
#[cfg(PyPy)]
flags: 0,
#[cfg(PyPy)]
_strides: [0; Py_MAX_NDIMS],
#[cfg(PyPy)]
_shape: [0; Py_MAX_NDIMS],
}
}
}

46
src/ffi/cpython/pystate.rs Normal file
View File

@ -0,0 +1,46 @@
use crate::ffi::pystate::{PyInterpreterState, PyThreadState};
use std::os::raw::c_int;
// Py_tracefunc is defined in ffi::pystate
pub const PyTrace_CALL: c_int = 0;
pub const PyTrace_EXCEPTION: c_int = 1;
pub const PyTrace_LINE: c_int = 2;
pub const PyTrace_RETURN: c_int = 3;
pub const PyTrace_C_CALL: c_int = 4;
pub const PyTrace_C_EXCEPTION: c_int = 5;
pub const PyTrace_C_RETURN: c_int = 6;
pub const PyTrace_OPCODE: c_int = 7;
extern "C" {
// PyGILState_Check is defined in ffi::pystate
pub fn PyInterpreterState_Main() -> *mut PyInterpreterState;
pub fn PyInterpreterState_Head() -> *mut PyInterpreterState;
pub fn PyInterpreterState_Next(interp: *mut PyInterpreterState) -> *mut PyInterpreterState;
pub fn PyInterpreterState_ThreadHead(interp: *mut PyInterpreterState) -> *mut PyThreadState;
pub fn PyThreadState_Next(tstate: *mut PyThreadState) -> *mut PyThreadState;
}
#[cfg(Py_3_9)]
#[cfg_attr(docsrs, doc(cfg(Py_3_9)))]
pub type _PyFrameEvalFunction = extern "C" fn(
*mut crate::ffi::PyThreadState,
*mut crate::ffi::PyFrameObject,
c_int,
) -> *mut crate::ffi::object::PyObject;
#[cfg(Py_3_9)]
extern "C" {
/// Get the frame evaluation function.
#[cfg_attr(docsrs, doc(cfg(Py_3_9)))]
pub fn _PyInterpreterState_GetEvalFrameFunc(
interp: *mut PyInterpreterState,
) -> _PyFrameEvalFunction;
///Set the frame evaluation function.
#[cfg_attr(docsrs, doc(cfg(Py_3_9)))]
pub fn _PyInterpreterState_SetEvalFrameFunc(
interp: *mut PyInterpreterState,
eval_frame: _PyFrameEvalFunction,
);
}

1
src/ffi/datetime.rs
View File

@ -650,7 +650,6 @@ mod tests {
fn test_date_fromtimestamp() {
Python::with_gil(|py| {
let args: Py<PyAny> = (100,).into_py(py);
dbg!(args.as_ref(py));
unsafe { PyDateTime_IMPORT() };
let dt: &PyAny = unsafe { py.from_owned_ptr(PyDate_FromTimestamp(args.as_ptr())) };
py_run!(

11
src/ffi/listobject.rs
View File

@ -52,4 +52,15 @@ extern "C" {
pub fn PyList_Reverse(arg1: *mut PyObject) -> c_int;
#[cfg_attr(PyPy, link_name = "PyPyList_AsTuple")]
pub fn PyList_AsTuple(arg1: *mut PyObject) -> *mut PyObject;
// CPython macros exported as functions on PyPy
#[cfg(PyPy)]
#[cfg_attr(PyPy, link_name = "PyPyList_GET_ITEM")]
pub fn PyList_GET_ITEM(arg1: *mut PyObject, arg2: Py_ssize_t) -> *mut PyObject;
#[cfg(PyPy)]
#[cfg_attr(PyPy, link_name = "PyPyList_GET_SIZE")]
pub fn PyList_GET_SIZE(arg1: *mut PyObject) -> Py_ssize_t;
#[cfg(PyPy)]
#[cfg_attr(PyPy, link_name = "PyPyList_SET_ITEM")]
pub fn PyList_SET_ITEM(arg1: *mut PyObject, arg2: Py_ssize_t, arg3: *mut PyObject);
}

2
src/ffi/methodobject.rs
View File

@ -45,7 +45,7 @@ pub type PyCFunctionWithKeywords = unsafe extern "C" fn(
kwds: *mut PyObject,
) -> *mut PyObject;
#[cfg(Py_3_7)]
#[cfg(all(Py_3_7, not(Py_LIMITED_API)))]
pub type _PyCFunctionFastWithKeywords = unsafe extern "C" fn(
slf: *mut PyObject,
args: *const *mut PyObject,

28
src/ffi/mod.rs
View File

@ -1,5 +1,27 @@
//! Raw ffi declarations for the C interface of Python.
//! Raw FFI declarations for Python's C API.
//!
//! This module provides low level bindings to the Python interpreter.
//! It is meant for advanced users only - regular PyO3 users shouldn't
//! need to interact with this module at all.
//!
//! The contents of this module are not documented here, as it would entail
//! basically copying the documentation from CPython. Consult the [Python/C API Reference
//! Manual][capi] for up-to-date documentation.
//!
//! # Safety
//!
//! The functions in this module lack individual safety documentation, but
//! generally the following apply:
//! - Pointer arguments have to point to a valid Python object of the correct type,
//! although null pointers are sometimes valid input.
//! - The vast majority can only be used safely while the GIL is held.
//! - Some functions have additional safety requirements, consult the
//! [Python/C API Reference Manual][capi]
//! for more information.
//!
//! [capi]: https://docs.python.org/3/c-api/index.html
#![allow(
missing_docs,
non_camel_case_types,
non_snake_case,
non_upper_case_globals,
@ -27,6 +49,7 @@ pub use self::code::*;
pub use self::codecs::*;
pub use self::compile::*;
pub use self::complexobject::*;
#[cfg(all(Py_3_8, not(Py_LIMITED_API)))]
pub use self::context::*;
#[cfg(not(Py_LIMITED_API))]
pub use self::datetime::*;
@ -78,6 +101,7 @@ pub use self::weakrefobject::*;
#[cfg(not(Py_LIMITED_API))]
pub use self::cpython::*;
mod abstract_;
// skipped asdl.h
// skipped ast.h
@ -95,8 +119,6 @@ mod compile; // TODO: incomplete
mod complexobject; // TODO supports PEP-384 only
#[cfg(all(Py_3_8, not(Py_LIMITED_API)))]
mod context; // It's actually 3.7.1, but no cfg for patches.
#[cfg(not(all(Py_3_8, not(Py_LIMITED_API))))]
mod context {}
#[cfg(not(Py_LIMITED_API))]
pub(crate) mod datetime;
mod descrobject; // TODO supports PEP-384 only

8
src/ffi/pystate.rs
View File

@ -1,4 +1,3 @@
use crate::ffi::ceval::_PyFrameEvalFunction;
use crate::ffi::moduleobject::PyModuleDef;
use crate::ffi::object::PyObject;
use crate::ffi::PyFrameObject;
@ -6,12 +5,7 @@ use std::os::raw::{c_int, c_long};
pub const MAX_CO_EXTRA_USERS: c_int = 255;
#[repr(C)]
#[derive(Copy, Clone)]
pub struct PyInterpreterState {
pub ob_base: PyObject,
pub eval_frame: _PyFrameEvalFunction,
}
opaque_struct!(PyInterpreterState);
#[repr(C)]
#[derive(Copy, Clone)]

6
src/ffi/tupleobject.rs
View File

@ -47,7 +47,7 @@ extern "C" {
/// Macro, trading safety for speed
#[inline]
#[cfg(not(Py_LIMITED_API))]
#[cfg(not(any(Py_LIMITED_API, PyPy)))]
pub unsafe fn PyTuple_GET_ITEM(op: *mut PyObject, i: Py_ssize_t) -> *mut PyObject {
*(*(op as *mut PyTupleObject))
.ob_item
@ -56,14 +56,14 @@ pub unsafe fn PyTuple_GET_ITEM(op: *mut PyObject, i: Py_ssize_t) -> *mut PyObjec
}
#[inline]
#[cfg(not(Py_LIMITED_API))]
#[cfg(not(any(Py_LIMITED_API, PyPy)))]
pub unsafe fn PyTuple_GET_SIZE(op: *mut PyObject) -> Py_ssize_t {
Py_SIZE(op)
}
/// Macro, *only* to be used to fill in brand new tuples
#[inline]
#[cfg(not(Py_LIMITED_API))]
#[cfg(not(any(Py_LIMITED_API, PyPy)))]
pub unsafe fn PyTuple_SET_ITEM(op: *mut PyObject, i: Py_ssize_t, v: *mut PyObject) {
*(*(op as *mut PyTupleObject))
.ob_item

115
src/freelist.rs
View File

@ -1,115 +0,0 @@
// Copyright (c) 2017-present PyO3 Project and Contributors
//! Support for [free allocation lists][1].
//!
//! This can improve performance for types that are often created and deleted in quick succession.
//!
//! Rather than implementing this manually,
//! implement it by annotating a struct with `#[pyclass(freelist = N)]`,
//! where `N` is the size of the freelist.
//!
//! [1]: https://en.wikipedia.org/wiki/Free_list
use crate::class::impl_::PyClassImpl;
use crate::pyclass::{get_type_free, tp_free_fallback, PyClassAlloc};
use crate::type_object::{PyLayout, PyTypeInfo};
use crate::{ffi, AsPyPointer, FromPyPointer, PyAny, Python};
use std::mem;
use std::os::raw::c_void;
/// Implements a freelist.
///
/// Do not implement this trait manually. Instead, use `#[pyclass(freelist = N)]`
/// on a Rust struct to implement it.
pub trait PyClassWithFreeList {
fn get_free_list(py: Python) -> &mut FreeList<*mut ffi::PyObject>;
}
/// Represents a slot of a [`FreeList`].
pub enum Slot<T> {
Empty,
Filled(T),
}
pub struct FreeList<T> {
entries: Vec<Slot<T>>,
split: usize,
capacity: usize,
}
impl<T> FreeList<T> {
/// Creates a new `FreeList` instance with specified capacity.
pub fn with_capacity(capacity: usize) -> FreeList<T> {
let entries = (0..capacity).map(|_| Slot::Empty).collect::<Vec<_>>();
FreeList {
entries,
split: 0,
capacity,
}
}
/// Pops the first non empty item.
pub fn pop(&mut self) -> Option<T> {
let idx = self.split;
if idx == 0 {
None
} else {
match mem::replace(&mut self.entries[idx - 1], Slot::Empty) {
Slot::Filled(v) => {
self.split = idx - 1;
Some(v)
}
_ => panic!("FreeList is corrupt"),
}
}
}
/// Inserts a value into the list. Returns `None` if the `FreeList` is full.
pub fn insert(&mut self, val: T) -> Option<T> {
let next = self.split + 1;
if next < self.capacity {
self.entries[self.split] = Slot::Filled(val);
self.split = next;
None
} else {
Some(val)
}
}
}
impl<T> PyClassAlloc for T
where
T: PyTypeInfo + PyClassImpl + PyClassWithFreeList,
{
unsafe fn new(py: Python, subtype: *mut ffi::PyTypeObject) -> *mut Self::Layout {
// if subtype is not equal to this type, we cannot use the freelist
if subtype == Self::type_object_raw(py) {
if let Some(obj) = <Self as PyClassWithFreeList>::get_free_list(py).pop() {
ffi::PyObject_Init(obj, subtype);
#[cfg(not(Py_3_8))]
crate::pyclass::bpo_35810_workaround(py, subtype);
return obj as _;
}
}
crate::pyclass::default_new::<Self>(py, subtype) as _
}
#[allow(clippy::collapsible_if)] // for if cfg!
unsafe fn dealloc(py: Python, self_: *mut Self::Layout) {
(*self_).py_drop(py);
let obj = PyAny::from_borrowed_ptr_or_panic(py, self_ as _);
if let Some(obj) = <Self as PyClassWithFreeList>::get_free_list(py).insert(obj.as_ptr()) {
let ty = ffi::Py_TYPE(obj);
let free = get_type_free(ty).unwrap_or_else(|| tp_free_fallback(ty));
free(obj as *mut c_void);
if cfg!(Py_3_8) {
if ffi::PyType_HasFeature(ty, ffi::Py_TPFLAGS_HEAPTYPE) != 0 {
ffi::Py_DECREF(ty as *mut ffi::PyObject);
}
}
}
}
}

2
src/gil.rs
View File

@ -495,7 +495,7 @@ impl EnsureGIL {
}
#[cfg(test)]
mod test {
mod tests {
use super::{gil_is_acquired, GILPool, GIL_COUNT, OWNED_OBJECTS, POOL};
use crate::{ffi, gil, AsPyPointer, IntoPyPointer, PyObject, Python, ToPyObject};
use std::ptr::NonNull;

1
src/impl_.rs
View File

@ -3,3 +3,4 @@
//! any time without documentation in the CHANGELOG and without breaking semver guarantees.
pub mod deprecations;
pub mod freelist;

12
src/impl_/deprecations.rs
View File

@ -11,3 +11,15 @@ pub const NAME_ATTRIBUTE: () = ();
note = "use `#[pyfn(m)] #[pyo3(name = \"...\")]` instead of `#[pyfn(m, \"...\")]`"
)]
pub const PYFN_NAME_ARGUMENT: () = ();
#[deprecated(
since = "0.14.0",
note = "use `#[pymodule] #[pyo3(name = \"...\")]` instead of `#[pymodule(...)]`"
)]
pub const PYMODULE_NAME_ARGUMENT: () = ();
#[deprecated(
since = "0.14.0",
note = "use `#[pyo3(text_signature = \"...\")]` instead of `#[text_signature = \"...\"]`"
)]
pub const TEXT_SIGNATURE_ATTRIBUTE: () = ();

71
src/impl_/freelist.rs Normal file
View File

@ -0,0 +1,71 @@
// Copyright (c) 2017-present PyO3 Project and Contributors
//! Support for [free allocation lists][1].
//!
//! This can improve performance for types that are often created and deleted in quick succession.
//!
//! Rather than implementing this manually,
//! implement it by annotating a struct with `#[pyclass(freelist = N)]`,
//! where `N` is the size of the freelist.
//!
//! [1]: https://en.wikipedia.org/wiki/Free_list
use std::mem;
/// Represents a slot of a [`FreeList`].
pub enum Slot<T> {
/// A free slot.
Empty,
/// An allocated slot.
Filled(T),
}
/// A free allocation list.
///
/// See [the parent module](crate::impl_::freelist) for more details.
pub struct FreeList<T> {
entries: Vec<Slot<T>>,
split: usize,
capacity: usize,
}
impl<T> FreeList<T> {
/// Creates a new `FreeList` instance with specified capacity.
pub fn with_capacity(capacity: usize) -> FreeList<T> {
let entries = (0..capacity).map(|_| Slot::Empty).collect::<Vec<_>>();
FreeList {
entries,
split: 0,
capacity,
}
}
/// Pops the first non empty item.
pub fn pop(&mut self) -> Option<T> {
let idx = self.split;
if idx == 0 {
None
} else {
match mem::replace(&mut self.entries[idx - 1], Slot::Empty) {
Slot::Filled(v) => {
self.split = idx - 1;
Some(v)
}
_ => panic!("FreeList is corrupt"),
}
}
}
/// Inserts a value into the list. Returns `None` if the `FreeList` is full.
pub fn insert(&mut self, val: T) -> Option<T> {
let next = self.split + 1;
if next < self.capacity {
self.entries[self.split] = Slot::Filled(val);
self.split = next;
None
} else {
Some(val)
}
}
}

352
src/instance.rs
View File

@ -18,6 +18,7 @@ use std::ptr::NonNull;
/// to the GIL, which is why you can get a token from all references of those
/// types.
pub unsafe trait PyNativeType: Sized {
/// Returns a GIL marker constrained to the lifetime of this type.
fn py(&self) -> Python {
unsafe { Python::assume_gil_acquired() }
}
@ -32,21 +33,145 @@ pub unsafe trait PyNativeType: Sized {
}
}
/// A Python object of known type T.
/// A GIL-independent reference to an object allocated on the Python heap.
///
/// Accessing this object is thread-safe, since any access to its API requires a `Python<'py>` GIL
/// token. There are a few different ways to use the Python object contained:
/// - [`Py::as_ref`](#method.as_ref) to borrow a GIL-bound reference to the contained object.
/// - [`Py::borrow`](#method.borrow), [`Py::try_borrow`](#method.try_borrow),
/// [`Py::borrow_mut`](#method.borrow_mut), or [`Py::try_borrow_mut`](#method.try_borrow_mut),
/// to directly access a `#[pyclass]` value (which has RefCell-like behavior, see
/// [the `PyCell` guide entry](https://pyo3.rs/main/class.html#pycell-and-interior-mutability)
/// ).
/// - Use methods directly on `Py`, such as [`Py::call`](#method.call) and
/// [`Py::call_method`](#method.call_method).
/// This type does not auto-dereference to the inner object because you must prove you hold the GIL to access it.
/// Instead, call one of its methods to access the inner object:
/// - [`Py::as_ref`], to borrow a GIL-bound reference to the contained object.
/// - [`Py::borrow`], [`Py::try_borrow`], [`Py::borrow_mut`], or [`Py::try_borrow_mut`],
/// to get a (mutable) reference to a contained pyclass, using a scheme similar to std's [`RefCell`].
/// See the [`PyCell` guide entry](https://pyo3.rs/latest/class.html#pycell-and-interior-mutability)
/// for more information.
/// - You can call methods directly on `Py` with [`Py::call`], [`Py::call_method`] and friends.
/// These require passing in the [`Python<'py>`](crate::Python) token but are otherwise similar to the corresponding
/// methods on [`PyAny`].
///
/// See [the guide](https://pyo3.rs/main/types.html) for an explanation
/// of the different Python object types.
/// # Example: Storing Python objects in structs
///
/// As all the native Python objects only appear as references, storing them in structs doesn't work well.
/// For example, this won't compile:
///
/// ```compile_fail
/// # use pyo3::prelude::*;
/// # use pyo3::types::PyDict;
/// #
/// #[pyclass]
/// struct Foo<'py> {
/// inner: &'py PyDict,
/// }
///
/// impl Foo {
/// fn new() -> Foo {
/// let foo = Python::with_gil(|py| {
/// // `py` will only last for this scope.
///
/// // `&PyDict` derives its lifetime from `py` and
/// // so won't be able to outlive this closure.
/// let dict: &PyDict = PyDict::new(py);
///
/// // because `Foo` contains `dict` its lifetime
/// // is now also tied to `py`.
/// Foo { inner: dict }
/// });
/// // Foo is no longer valid.
/// // Returning it from this function is a 💥 compiler error 💥
/// foo
/// }
/// }
/// ```
///
/// [`Py`]`<T>` can be used to get around this by converting `dict` into a GIL-independent reference:
///
/// ```rust
/// # use pyo3::prelude::*;
/// # use pyo3::types::PyDict;
/// #
/// #[pyclass]
/// struct Foo {
/// inner: Py<PyDict>,
/// }
///
/// impl Foo {
/// fn new() -> Foo {
/// Python::with_gil(|py| {
/// let dict: Py<PyDict> = PyDict::new(py).into();
/// Foo { inner: dict }
/// })
/// }
/// }
/// ```
///
/// This can also be done with other pyclasses:
/// ```rust
/// # use pyo3::prelude::*;
/// #
/// #[pyclass]
/// struct Bar {/* fields omitted */}
///
/// #[pyclass]
/// struct Foo {
/// inner: Py<Bar>,
/// }
///
/// impl Foo {
/// fn new() -> PyResult<Foo> {
/// Python::with_gil(|py| {
/// let bar: Py<Bar> = Py::new(py, Bar {})?;
/// Ok(Foo { inner: bar })
/// })
/// }
/// }
/// ```
///
/// # Example: Shared ownership of Python objects
///
/// `Py<T>` can be used to share ownership of a Python object, similar to std's [`Rc`]`<T>`.
/// As with [`Rc`]`<T>`, cloning it increases its reference count rather than duplicating
/// the underlying object.
///
/// This can be done using either [`Py::clone_ref`] or [`Py`]`<T>`'s [`Clone`] trait implementation.
/// [`Py::clone_ref`] will be faster if you happen to be already holding the GIL.
///
/// ```rust
/// use pyo3::prelude::*;
/// use pyo3::types::PyDict;
/// use pyo3::conversion::AsPyPointer;
///
/// # fn main() {
/// Python::with_gil(|py| {
/// let first: Py<PyDict> = PyDict::new(py).into();
///
/// // All of these are valid syntax
/// let second = Py::clone_ref(&first, py);
/// let third = first.clone_ref(py);
/// let fourth = Py::clone(&first);
/// let fifth = first.clone();
///
/// // Disposing of our original `Py<PyDict>` just decrements the reference count.
/// drop(first);
///
/// // They all point to the same object
/// assert_eq!(second.as_ptr(), third.as_ptr());
/// assert_eq!(fourth.as_ptr(), fifth.as_ptr());
/// assert_eq!(second.as_ptr(), fourth.as_ptr());
/// });
/// # }
/// ```
///
/// # Preventing reference cycles
///
/// It is easy to accidentally create reference cycles using [`Py`]`<T>`.
/// The Python interpreter can break these reference cycles within pyclasses if they
/// implement the [`PyGCProtocol`](crate::class::gc::PyGCProtocol). If your pyclass
/// contains other Python objects you should implement this protocol to avoid leaking memory.
///
/// # A note on `Send` and `Sync`
///
/// Accessing this object is threadsafe, since any access to its API requires a [`Python<'py>`](crate::Python) token.
/// As you can only get this by acquiring the GIL, `Py<...>` "implements [`Send`] and [`Sync`].
///
/// [`Rc`]: std::rc::Rc
/// [`RefCell`]: std::cell::RefCell
#[repr(transparent)]
pub struct Py<T>(NonNull<ffi::PyObject>, PhantomData<T>);
@ -57,7 +182,24 @@ impl<T> Py<T>
where
T: PyClass,
{
/// Create a new instance `Py<T>` of a `#[pyclass]` on the Python heap.
/// Creates a new instance `Py<T>` of a `#[pyclass]` on the Python heap.
///
/// # Examples
///
/// ```rust
/// use pyo3::prelude::*;
///
/// #[pyclass]
/// struct Foo {/* fields omitted */}
///
/// # fn main() -> PyResult<()> {
/// Python::with_gil(|py| -> PyResult<Py<Foo>> {
/// let foo: Py<Foo> = Py::new(py, Foo {})?;
/// Ok(foo)
/// })?;
/// # Ok(())
/// # }
/// ```
pub fn new(py: Python, value: impl Into<PyClassInitializer<T>>) -> PyResult<Py<T>> {
let initializer = value.into();
let obj = initializer.create_cell(py)?;
@ -70,56 +212,91 @@ impl<T> Py<T>
where
T: PyTypeInfo,
{
/// Borrows a GIL-bound reference to the contained `T`. By binding to the GIL lifetime, this
/// allows the GIL-bound reference to not require `Python` for any of its methods.
/// Borrows a GIL-bound reference to the contained `T`.
///
/// By binding to the GIL lifetime, this allows the GIL-bound reference to not require
/// [`Python<'py>`](crate::Python) for any of its methods, which makes calling methods
/// on it more ergonomic.
///
/// For native types, this reference is `&T`. For pyclasses, this is `&PyCell<T>`.
///
/// Note that the lifetime of the returned reference is the shortest of `&self` and
/// [`Python<'py>`](crate::Python).
/// Consider using [`Py::into_ref`] instead if this poses a problem.
///
/// # Examples
///
/// Get access to `&PyList` from `Py<PyList>`:
///
/// ```
/// # use pyo3::prelude::*;
/// # use pyo3::types::PyList;
/// # Python::with_gil(|py| {
/// let list: Py<PyList> = PyList::empty(py).into();
/// let list: &PyList = list.as_ref(py);
/// assert_eq!(list.len(), 0);
/// # });
/// #
/// Python::with_gil(|py| {
/// let list: Py<PyList> = PyList::empty(py).into();
/// let list: &PyList = list.as_ref(py);
/// assert_eq!(list.len(), 0);
/// });
/// ```
///
/// Get access to `&PyCell<MyClass>` from `Py<MyClass>`:
///
/// ```
/// # use pyo3::prelude::*;
/// #
/// #[pyclass]
/// struct MyClass { }
/// # Python::with_gil(|py| {
/// let my_class: Py<MyClass> = Py::new(py, MyClass { }).unwrap();
/// let my_class_cell: &PyCell<MyClass> = my_class.as_ref(py);
/// assert!(my_class_cell.try_borrow().is_ok());
/// # });
///
/// Python::with_gil(|py| {
/// let my_class: Py<MyClass> = Py::new(py, MyClass { }).unwrap();
/// let my_class_cell: &PyCell<MyClass> = my_class.as_ref(py);
/// assert!(my_class_cell.try_borrow().is_ok());
/// });
/// ```
pub fn as_ref<'py>(&'py self, _py: Python<'py>) -> &'py T::AsRefTarget {
let any = self.as_ptr() as *const PyAny;
unsafe { PyNativeType::unchecked_downcast(&*any) }
}
/// Similar to [`as_ref`](#method.as_ref), and also consumes this `Py` and registers the
/// Borrows a GIL-bound reference to the contained `T` independently of the lifetime of `T`.
///
/// This method is similar to [`as_ref`](#method.as_ref) but consumes `self` and registers the
/// Python object reference in PyO3's object storage. The reference count for the Python
/// object will not be decreased until the GIL lifetime ends.
///
/// You should prefer using [`as_ref`](#method.as_ref) if you can as it'll have less overhead.
///
/// # Examples
///
/// Useful when returning GIL-bound references from functions. In the snippet below, note that
/// the `'py` lifetime of the input GIL lifetime is also given to the returned reference:
/// ```
/// # use pyo3::prelude::*;
/// fn new_py_any<'py>(py: Python<'py>, value: impl IntoPy<PyObject>) -> &'py PyAny {
/// let obj: PyObject = value.into_py(py);
/// [`Py::as_ref`]'s lifetime limitation forbids creating a function that references a
/// variable created inside the function.
///
/// // .as_ref(py) would not be suitable here, because a reference to `obj` may not be
/// // returned from the function.
/// ```rust,compile_fail
/// # use pyo3::prelude::*;
/// #
/// fn new_py_any<'py>(py: Python<'py>, value: impl IntoPy<Py<PyAny>>) -> &'py PyAny {
/// let obj: Py<PyAny> = value.into_py(py);
///
/// // The lifetime of the return value of this function is the shortest
/// // of `obj` and `py`. As `obj` is owned by the current function,
/// // Rust won't let the return value escape this function!
/// obj.as_ref(py)
/// }
/// ```
///
/// This can be solved by using [`Py::into_ref`] instead, which does not suffer from this issue.
/// Note that the lifetime of the [`Python<'py>`](crate::Python) token is transferred to
/// the returned reference.
///
/// ```rust
/// # use pyo3::prelude::*;
/// #
/// fn new_py_any<'py>(py: Python<'py>, value: impl IntoPy<Py<PyAny>>) -> &'py PyAny {
/// let obj: Py<PyAny> = value.into_py(py);
///
/// // This reference's lifetime is determined by `py`'s lifetime.
/// // Because that originates from outside this function,
/// // this return value is allowed.
/// obj.into_ref(py)
/// }
/// ```
@ -132,22 +309,72 @@ impl<T> Py<T>
where
T: PyClass,
{
/// Immutably borrows the value `T`. This borrow lasts untill the returned `PyRef` exists.
/// Immutably borrows the value `T`.
///
/// This borrow lasts while the returned [`PyRef`] exists.
/// Multiple immutable borrows can be taken out at the same time.
///
/// Equivalent to `self.as_ref(py).borrow()` -
/// see [`PyCell::borrow`](../pycell/struct.PyCell.html#method.borrow)
/// see [`PyCell::borrow`](crate::pycell::PyCell::borrow).
///
/// # Examples
///
/// ```rust
/// # use pyo3::prelude::*;
/// #
/// #[pyclass]
/// struct Foo {
/// inner: u8,
/// }
///
/// # fn main() -> PyResult<()> {
/// Python::with_gil(|py| -> PyResult<()> {
/// let foo: Py<Foo> = Py::new(py, Foo { inner: 73 })?;
/// let inner: &u8 = &foo.borrow(py).inner;
///
/// assert_eq!(*inner, 73);
/// Ok(())
/// })?;
/// # Ok(())
/// # }
/// ```
///
/// # Panics
///
/// Panics if the value is currently mutably borrowed. For a non-panicking variant, use
/// [`try_borrow`](#method.try_borrow).
pub fn borrow<'py>(&'py self, py: Python<'py>) -> PyRef<'py, T> {
self.as_ref(py).borrow()
}
/// Mutably borrows the value `T`. This borrow lasts untill the returned `PyRefMut` exists.
/// Mutably borrows the value `T`.
///
/// This borrow lasts while the returned [`PyRefMut`] exists.
///
/// Equivalent to `self.as_ref(py).borrow_mut()` -
/// see [`PyCell::borrow_mut`](../pycell/struct.PyCell.html#method.borrow_mut)
/// see [`PyCell::borrow_mut`](crate::pycell::PyCell::borrow_mut).
///
/// # Examples
///
/// ```
/// # use pyo3::prelude::*;
/// #
/// #[pyclass]
/// struct Foo {
/// inner: u8,
/// }
///
/// # fn main() -> PyResult<()> {
/// Python::with_gil(|py| -> PyResult<()> {
/// let foo: Py<Foo> = Py::new(py, Foo { inner: 73 })?;
/// foo.borrow_mut(py).inner = 35;
///
/// assert_eq!(foo.borrow(py).inner, 35);
/// Ok(())
/// })?;
/// # Ok(())
/// # }
/// ```
///
/// # Panics
/// Panics if the value is currently mutably borrowed. For a non-panicking variant, use
@ -156,24 +383,26 @@ where
self.as_ref(py).borrow_mut()
}
/// Immutably borrows the value `T`, returning an error if the value is currently
/// mutably borrowed. This borrow lasts untill the returned `PyRef` exists.
/// Attempts to immutably borrow the value `T`, returning an error if the value is currently mutably borrowed.
///
/// The borrow lasts while the returned [`PyRef`] exists.
///
/// This is the non-panicking variant of [`borrow`](#method.borrow).
///
/// Equivalent to `self.as_ref(py).try_borrow()` -
/// see [`PyCell::try_borrow`](../pycell/struct.PyCell.html#method.try_borrow)
/// Equivalent to `self.as_ref(py).borrow_mut()` -
/// see [`PyCell::try_borrow`](crate::pycell::PyCell::try_borrow).
pub fn try_borrow<'py>(&'py self, py: Python<'py>) -> Result<PyRef<'py, T>, PyBorrowError> {
self.as_ref(py).try_borrow()
}
/// Mutably borrows the value `T`, returning an error if the value is currently borrowed.
/// This borrow lasts untill the returned `PyRefMut` exists.
/// Attempts to mutably borrow the value `T`, returning an error if the value is currently borrowed.
///
/// The borrow lasts while the returned [`PyRefMut`] exists.
///
/// This is the non-panicking variant of [`borrow_mut`](#method.borrow_mut).
///
/// Equivalent to `self.as_ref(py).try_borrow_mut() -
/// see [`PyCell::try_borrow_mut`](../pycell/struct.PyCell.html#method.try_borrow_mut)
/// Equivalent to `self.as_ref(py).try_borrow_mut()` -
/// see [`PyCell::try_borrow_mut`](crate::pycell::PyCell::try_borrow_mut).
pub fn try_borrow_mut<'py>(
&'py self,
py: Python<'py>,
@ -189,7 +418,29 @@ impl<T> Py<T> {
unsafe { ffi::Py_REFCNT(self.0.as_ptr()) }
}
/// Clones self by calling `Py_INCREF()` on the ptr.
/// Makes a clone of `self`.
///
/// This creates another pointer to the same object, increasing its reference count.
///
/// You should prefer using this method over [`Clone`] if you happen to be holding the GIL already.
///
/// # Examples
///
/// ```rust
/// use pyo3::prelude::*;
/// use pyo3::types::PyDict;
/// use pyo3::conversion::AsPyPointer;
///
/// # fn main() {
/// Python::with_gil(|py| {
/// let first: Py<PyDict> = PyDict::new(py).into();
/// let second = Py::clone_ref(&first, py);
///
/// // Both point to the same object
/// assert_eq!(first.as_ptr(), second.as_ptr());
/// });
/// # }
/// ```
#[inline]
pub fn clone_ref(&self, py: Python) -> Py<T> {
unsafe { Py::from_borrowed_ptr(py, self.0.as_ptr()) }
@ -524,6 +775,9 @@ impl<T> PartialEq for Py<T> {
}
}
/// If the GIL is held this increments `self`'s reference count.
/// Otherwise this registers the [`Py`]`<T>` instance to have its reference count
/// incremented the next time PyO3 acquires the GIL.
impl<T> Clone for Py<T> {
fn clone(&self) -> Self {
unsafe {
@ -608,7 +862,7 @@ impl PyObject {
}
#[cfg(test)]
mod test {
mod tests {
use super::{Py, PyObject};
use crate::types::PyDict;
use crate::{ffi, AsPyPointer, Python};

232
src/lib.rs
View File

@ -1,44 +1,139 @@
#![cfg_attr(feature = "nightly", feature(specialization))]
#![cfg_attr(docsrs, feature(doc_cfg))]
#![allow(clippy::missing_safety_doc)] // FIXME (#698)
//! Rust bindings to the Python interpreter.
//!
//! Look at [the guide](https://pyo3.rs/) for a detailed introduction.
//! PyO3 can be used to write native Python modules or run Python code and modules from Rust.
//!
//! # Ownership and Lifetimes
//! See [the guide](https://pyo3.rs/) for a detailed introduction.
//!
//! Because all Python objects potentially have multiple owners, the concept of
//! Rust mutability does not apply to Python objects. As a result, PyO3 allows
//! mutating Python objects even if they are not stored in a mutable Rust
//! variable.
//! # PyO3's object types
//!
//! In Python, all objects are implicitly reference counted. The Python
//! interpreter uses a global interpreter lock (GIL) to ensure thread-safety.
//! Thus, we use `struct Python<'py>` as a token to indicate that
//! a function can assume that the GIL is held. In Rust, we use different types
//! to represent a reference to a Python object, depending on whether we know
//! the GIL is held, and depending on whether we know the underlying type. See
//! [the guide](https://pyo3.rs/main/types.html) for an explanation of
//! the different Python object types.
//! PyO3 has several core types that you should familiarize yourself with:
//!
//! A `Python` instance is either obtained explicitly by acquiring the GIL,
//! or implicitly by PyO3 when it generates the wrapper code for Rust functions
//! and structs wrapped as Python functions and objects.
//! ## The Python<'py> object
//!
//! # Error Handling
//! Holding the [global interpreter lock](https://docs.python.org/3/glossary.html#term-global-interpreter-lock)
//! (GIL) is modeled with the [`Python<'py>`](crate::Python) token.
//! All APIs that require that the GIL is held require this token as proof
//! that you really are holding the GIL. It can be explicitly acquired and
//! is also implicitly acquired by PyO3 as it wraps Rust functions and structs
//! into Python functions and objects.
//!
//! The vast majority of operations in this library will return `PyResult<...>`.
//! ## The GIL-dependent types
//!
//! For example `&`[`PyAny`](crate::types::PyAny).
//! These are only ever seen as references, with a lifetime that is only valid for as long
//! as the GIL is held, which is why using them doesn't require a [`Python<'py>`](crate::Python) token.
//! The underlying Python object, if mutable, can be mutated through any reference.
//!
//! See the [guide](https://pyo3.rs/latest/types.html) for an explanation of the different Python object types.
//!
//! ## The GIL-independent types
//!
//! When wrapped in [`Py`]`<...>`, like with [`Py`]`<`[`PyAny`](crate::types::PyAny)`>` or [`Py`]`<SomePyClass>`, Python objects
//! no longer have a limited lifetime which makes them easier to store in structs and pass between functions.
//! However, you cannot do much with them without a
//! [`Python<'py>`](crate::Python) token, for which you’d need to reacquire the GIL.
//!
//! ## PyErr
//!
//! The vast majority of operations in this library will return [`PyResult<...>`](PyResult).
//! This is an alias for the type `Result<..., PyErr>`.
//!
//! A `PyErr` represents a Python exception. Errors within the `PyO3` library are
//! also exposed as Python exceptions.
//! A `PyErr` represents a Python exception. A `PyErr` returned to Python code will be raised as a Python exception.
//! Errors from `PyO3` itself are also exposed as Python exceptions.
//!
//! # Examples
//! # Feature flags
//!
//! ## Using Rust from Python
//! PyO3 uses [feature flags](https://doc.rust-lang.org/cargo/reference/features.html)
//! to enable you to opt-in to additional functionality. For a detailed description, see
//! the [Features Reference chapter of the guide](https://pyo3.rs/latest/features.html#features-reference).
//!
//! PyO3 can be used to generate a native Python module.
//! ## Default feature flags
//!
//! The following features are turned on by default:
//! - `macros`: Enables various macros, including all the attribute macros.
//!
//! ## Optional feature flags
//!
//! The following features are optional:
//! - `abi3`: Restricts PyO3's API to a subset of the full Python API which is guaranteed
//! by [PEP 384](https://www.python.org/dev/peps/pep-0384/) to be forward-compatible with future Python versions.
//
//! - `auto-initialize`: Changes [`Python::with_gil`](crate::Python::with_gil) and
//! [`Python::acquire_gil`](crate::Python::acquire_gil) to automatically initialize the
//! Python interpreter if needed.
//
//! - `extension-module`: This will tell the linker to keep the Python symbols unresolved,
//! so that your module can also be used with statically linked Python interpreters.
//! Use this feature when building an extension module.
//
//! - `hashbrown`: Enables conversions between Python objects and
//! [hashbrown](https://docs.rs/hashbrown)'s
//! [`HashMap`](https://docs.rs/hashbrown/latest/hashbrown/struct.HashMap.html) and
//! [`HashSet`](https://docs.rs/hashbrown/latest/hashbrown/struct.HashSet.html) types.
//
//! - [`indexmap`](crate::indexmap): Enables conversions between Python dictionary and
//! [indexmap](https://docs.rs/indexmap)'s
//! [`IndexMap`](https://docs.rs/indexmap/latest/indexmap/map/struct.IndexMap.html).
//
//! - `multiple-pymethods`: Enables the use of multiple
//! [`#[pymethods]`](crate::proc_macro::pymethods) blocks per
//! [`#[pyclass]`](crate::proc_macro::pyclass). This adds a dependency on the
//! [`inventory`](https://docs.rs/inventory) crate, which is not supported on all platforms.
//
//! - [`num-bigint`](./num_bigint/index.html): Enables conversions between Python objects and
//! [num-bigint](https://docs.rs/num-bigint)'s
//! [`BigInt`](https://docs.rs/num-bigint/latest/num_bigint/struct.BigInt.html) and
//! [`BigUint`](https://docs.rs/num-bigint/latest/num_bigint/struct.BigUint.html) types.
//
//! - [`num-complex`](crate::num_complex): Enables conversions between Python objects and
//! [num-complex](https://docs.rs/num-complex)'s
//! [`Complex`](https://docs.rs/num-complex/latest/num_complex/struct.Complex.html) type.
//
//! - `serde`: Allows implementing [serde](https://docs.rs/serde)'s
//! [`Serialize`](https://docs.rs/serde/latest/serde/trait.Serialize.html) and
//! [`Deserialize`](https://docs.rs/serde/latest/serde/trait.Deserialize.html) traits for
//! [`Py`]`<T>` for all `T` that implement
//! [`Serialize`](https://docs.rs/serde/latest/serde/trait.Serialize.html) and
//! [`Deserialize`](https://docs.rs/serde/latest/serde/trait.Deserialize.html).
//!
//! ## Unstable features
//!
//! - `nightly`: Gates some optimizations that rely on
//! [`#![feature(specialization)]`](https://github.com/rust-lang/rfcs/blob/master/text/1210-impl-specialization.md),
//! for which you'd also need nightly Rust. You should not use this feature.
//
//! ## `rustc` environment flags
//!
//! PyO3 uses `rustc`'s `--cfg` flags to enable or disable code used for different Python versions.
//! If you want to do this for your own crate, you can do so with the [`pyo3-build-config`](https://docs.rs/pyo3-build-config) crate.
//!
//! - `Py_3_6`, `Py_3_7`, `Py_3_8`, `Py_3_9`, `Py_3_10`: Marks code that is only enabled when compiling for a given minimum Python version.
//
//! - `Py_LIMITED_API`: Marks code enabled when the `abi3` feature flag is enabled.
//
//! - `PyPy` - Marks code enabled when compiling for PyPy.
//!
//! # Minimum supported Rust and Python versions
//!
//! PyO3 supports Python 3.6+ and Rust 1.41+.
//!
//! Building with PyPy is also possible (via cpyext) for Python 3.6,
//! targeted PyPy version is 7.3+. Please refer to the
//! [pypy section](https://pyo3.rs/latest/building_and_distribution/pypy.html)
//! in the guide for more information.
//!
//! # Example: Building a native Python module
//!
//! To build, test and publish your crate as a Python module, it is recommended that you use
//! [maturin](https://github.com/PyO3/maturin) or
//! [setuptools-rust](https://github.com/PyO3/setuptools-rust). You can also do this manually. See the
//! [Building and Distribution chapter of the guide](https://pyo3.rs/latest/building_and_distribution.html)
//! for more information.
//!
//! Add these files to your crate's root:
//!
//! **`Cargo.toml`**
//!
@ -58,7 +153,9 @@
//! crate-type = ["cdylib"]
//!
//! [dependencies.pyo3]
//! version = "0.13.2"
// workaround for `extended_key_value_attributes`: https://github.com/rust-lang/rust/issues/82768#issuecomment-803935643
#![cfg_attr(docsrs, cfg_attr(docsrs, doc = concat!("version = \"", env!("CARGO_PKG_VERSION"), "\"")))]
#![cfg_attr(not(docsrs), doc = "version = \"*\"")]
//! features = ["extension-module"]
//! ```
//!
@ -66,16 +163,15 @@
//!
//! ```rust
//! use pyo3::prelude::*;
//! use pyo3::wrap_pyfunction;
//!
//! #[pyfunction]
//! /// Formats the sum of two numbers as string.
//! #[pyfunction]
//! fn sum_as_string(a: usize, b: usize) -> PyResult<String> {
//! Ok((a + b).to_string())
//! }
//!
//! #[pymodule]
//! /// A Python module implemented in Rust.
//! #[pymodule]
//! fn string_sum(py: Python, m: &PyModule) -> PyResult<()> {
//! m.add_function(wrap_pyfunction!(sum_as_string, m)?)?;
//!
@ -83,25 +179,37 @@
//! }
//! ```
//!
//! While developing, you symlink (or copy) and rename the shared library from
//! the target folder: On macOS, rename `libstring_sum.dylib` to
//! `string_sum.so`, on Windows `libstring_sum.dll` to `string_sum.pyd` and on
//! Linux `libstring_sum.so` to `string_sum.so`. Then open a Python shell in the
//! same folder and you'll be able to `import string_sum`.
//! **`.cargo/config.toml`**
//! ```toml
//! # These flags must be passed to rustc when compiling for macOS
//! # They can be omitted if you pass the flags yourself
//! # or don't care about macOS
//!
//! To build, test and publish your crate as a Python module, you can use
//! [maturin](https://github.com/PyO3/maturin) or
//! [setuptools-rust](https://github.com/PyO3/setuptools-rust). You can find an
//! example for setuptools-rust in [examples/word-count](examples/word-count),
//! while maturin should work on your crate without any configuration.
//! [target.x86_64-apple-darwin]
//! rustflags = [
//! "-C", "link-arg=-undefined",
//! "-C", "link-arg=dynamic_lookup",
//! ]
//!
//! ## Using Python from Rust
//! [target.aarch64-apple-darwin]
//! rustflags = [
//! "-C", "link-arg=-undefined",
//! "-C", "link-arg=dynamic_lookup",
//! ]
//! ```
//!
//! # Example: Using Python from Rust
//!
//! You can use PyO3 to call Python functions from Rust.
//!
//! Add `pyo3` to your `Cargo.toml`:
//!
//! ```toml
//! [dependencies.pyo3]
//! version = "0.13.2"
// workaround for `extended_key_value_attributes`: https://github.com/rust-lang/rust/issues/82768#issuecomment-803935643
#![cfg_attr(docsrs, cfg_attr(docsrs, doc = concat!("version = \"", env!("CARGO_PKG_VERSION"), "\"")))]
#![cfg_attr(not(docsrs), doc = "version = \"*\"")]
//! # this is necessary to automatically initialize the Python interpreter
//! features = ["auto-initialize"]
//! ```
//!
@ -175,7 +283,6 @@ pub mod derive_utils;
mod err;
pub mod exceptions;
pub mod ffi;
pub mod freelist;
mod gil;
pub mod impl_;
mod instance;
@ -192,15 +299,26 @@ pub mod pyclass;
pub mod pyclass_init;
pub mod pyclass_slots;
mod python;
pub mod type_object;
pub mod types;
pub mod num_bigint;
pub mod num_complex;
#[cfg_attr(docsrs, doc(cfg(feature = "indexmap")))]
#[cfg(feature = "indexmap")]
pub use crate::conversions::indexmap;
#[cfg_attr(docsrs, doc(cfg(feature = "serde")))]
#[cfg(feature = "serde")]
pub mod serde;
/// The proc macros, all of which are part of the prelude.
///
/// Import these with `use pyo3::prelude::*;`
#[cfg_attr(docsrs, doc(cfg(feature = "macros")))]
#[cfg(feature = "macros")]
pub mod proc_macro {
pub use pyo3_macros::pymodule;
@ -230,35 +348,6 @@ macro_rules! wrap_pyfunction {
};
}
/// Returns the function that is called in the C-FFI.
///
/// Use this together with `#[pyfunction]` and [types::PyCFunction].
/// ```
/// use pyo3::prelude::*;
/// use pyo3::types::PyCFunction;
/// use pyo3::raw_pycfunction;
///
/// #[pyfunction]
/// fn some_fun(arg: i32) -> PyResult<()> {
/// Ok(())
/// }
///
/// #[pymodule]
/// fn module(_py: Python, module: &PyModule) -> PyResult<()> {
/// let ffi_wrapper_fun = raw_pycfunction!(some_fun);
/// let docs = "Some documentation string with null-termination\0";
/// let py_cfunction =
/// PyCFunction::new_with_keywords(ffi_wrapper_fun, "function_name", docs, module.into())?;
/// module.add_function(py_cfunction)
/// }
/// ```
#[macro_export]
macro_rules! raw_pycfunction {
($function_name: ident) => {{
pyo3::paste::expr! { [<__pyo3_raw_ $function_name>] }
}};
}
/// Returns a function that takes a [Python] instance and returns a Python module.
///
/// Use this together with `#[pymodule]` and [types::PyModule::add_wrapped].
@ -431,4 +520,5 @@ pub mod doc_test {
doctest!("guide/src/rust_cpython.md", guide_rust_cpython_md);
doctest!("guide/src/trait_bounds.md", guide_trait_bounds_md);
doctest!("guide/src/types.md", guide_types_md);
doctest!("guide/src/faq.md", faq);
}

2
src/marshal.rs
View File

@ -51,7 +51,7 @@ where
}
#[cfg(test)]
mod test {
mod tests {
use super::*;
use crate::types::PyDict;

295
src/num_bigint.rs Normal file
View File

@ -0,0 +1,295 @@
// Copyright (c) 2017-present PyO3 Project and Contributors
//
// based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython
#![cfg(all(feature = "num-bigint", not(any(Py_LIMITED_API, PyPy))))]
#![cfg_attr(
docsrs,
doc(cfg(all(feature = "num-bigint", not(any(Py_LIMITED_API, PyPy)))))
)]
//! Conversions to and from [num-bigint](https://docs.rs/num-bigint)’s [`BigInt`] and [`BigUint`] types.
//!
//! This is useful for converting Python integers when they may not fit in Rust's built-in integer types.
//!
//! # Setup
//!
//! To use this feature, add this to your **`Cargo.toml`**:
//!
//! ```toml
//! [dependencies]
//! num-bigint = "*"
// workaround for `extended_key_value_attributes`: https://github.com/rust-lang/rust/issues/82768#issuecomment-803935643
#![cfg_attr(docsrs, cfg_attr(docsrs, doc = concat!("pyo3 = { version = \"", env!("CARGO_PKG_VERSION"), "\", features = [\"num-bigint\"] }")))]
#![cfg_attr(
not(docsrs),
doc = "pyo3 = { version = \"*\", features = [\"num-bigint\"] }"
)]
//! ```
//!
//! Note that you must use compatible versions of num-bigint and PyO3.
//! The required num-bigint version may vary based on the version of PyO3.
//!
//! ## Examples
//!
//! Using [`BigInt`] to correctly increment an arbitrary precision integer.
//! This is not possible with Rust's native integers if the Python integer is too large,
//! in which case it will fail its conversion and raise `OverflowError`.
//!
//! ```rust
//! use num_bigint::BigInt;
//! use pyo3::prelude::*;
//!
//! #[pyfunction]
//! fn add_one(n: BigInt) -> BigInt {
//! n + 1
//! }
//!
//! #[pymodule]
//! fn my_module(_py: Python, m: &PyModule) -> PyResult<()> {
//! m.add_function(wrap_pyfunction!(add_one, m)?)?;
//! Ok(())
//! }
//! ```
//!
//! Python code:
//! ```python
//! from my_module import add_one
//!
//! n = 1 << 1337
//! value = add_one(n)
//!
//! assert n + 1 == value
//! ```
use crate::{
err, ffi, types::*, AsPyPointer, FromPyObject, IntoPy, Py, PyAny, PyErr, PyNativeType,
PyObject, PyResult, Python, ToPyObject,
};
use num_bigint::{BigInt, BigUint};
use std::os::raw::{c_int, c_uchar};
#[cfg(not(all(windows, PyPy)))]
unsafe fn extract(ob: &PyLong, buffer: &mut [c_uchar], is_signed: c_int) -> PyResult<()> {
err::error_on_minusone(
ob.py(),
ffi::_PyLong_AsByteArray(
ob.as_ptr() as *mut ffi::PyLongObject,
buffer.as_mut_ptr(),
buffer.len(),
1,
is_signed,
),
)
}
macro_rules! bigint_conversion {
($rust_ty: ty, $is_signed: expr, $to_bytes: path, $from_bytes: path) => {
impl ToPyObject for $rust_ty {
fn to_object(&self, py: Python) -> PyObject {
unsafe {
let bytes = $to_bytes(self);
let obj = ffi::_PyLong_FromByteArray(
bytes.as_ptr() as *const c_uchar,
bytes.len(),
1,
$is_signed,
);
PyObject::from_owned_ptr(py, obj)
}
}
}
impl IntoPy<PyObject> for $rust_ty {
fn into_py(self, py: Python) -> PyObject {
self.to_object(py)
}
}
impl<'source> FromPyObject<'source> for $rust_ty {
fn extract(ob: &'source PyAny) -> PyResult<$rust_ty> {
let py = ob.py();
unsafe {
let num = ffi::PyNumber_Index(ob.as_ptr());
if num.is_null() {
return Err(PyErr::fetch(py));
}
let n_bits = ffi::_PyLong_NumBits(num);
let n_bytes = if n_bits < 0 {
return Err(PyErr::fetch(py));
} else if n_bits == 0 {
0
} else {
(n_bits as usize - 1 + $is_signed) / 8 + 1
};
let num: Py<PyLong> = Py::from_owned_ptr(py, num);
if n_bytes <= 128 {
let mut buffer = [0; 128];
extract(num.as_ref(py), &mut buffer[..n_bytes], $is_signed)?;
Ok($from_bytes(&buffer[..n_bytes]))
} else {
let mut buffer = vec![0; n_bytes];
extract(num.as_ref(py), &mut buffer, $is_signed)?;
Ok($from_bytes(&buffer))
}
}
}
}
};
}
bigint_conversion!(BigUint, 0, BigUint::to_bytes_le, BigUint::from_bytes_le);
bigint_conversion!(
BigInt,
1,
BigInt::to_signed_bytes_le,
BigInt::from_signed_bytes_le
);
#[cfg(test)]
mod tests {
use super::*;
use crate::types::{PyDict, PyModule};
use indoc::indoc;
fn python_fib(py: Python) -> &PyModule {
let fib_code = indoc!(
r#"
def fib(n):
f0, f1 = 0, 1
for _ in range(n):
f0, f1 = f1, f0 + f1
return f0
def fib_neg(n):
return -fib(n)
"#
);
PyModule::from_code(py, fib_code, "fib.py", "fib").unwrap()
}
fn rust_fib<T>(n: usize) -> T
where
T: From<u16>,
for<'a> &'a T: std::ops::Add<Output = T>,
{
let mut f0: T = T::from(0);
let mut f1: T = T::from(1);
for _ in 0..n {
let f2 = &f0 + &f1;
f0 = std::mem::replace(&mut f1, f2);
}
f0
}
#[test]
fn convert_biguint() {
Python::with_gil(|py| {
let rs_result: BigUint = rust_fib(400);
let fib = python_fib(py);
let locals = PyDict::new(py);
locals.set_item("rs_result", &rs_result).unwrap();
locals.set_item("fib", fib).unwrap();
// Checks if Rust BigUint -> Python Long conversion is correct
py.run("assert fib.fib(400) == rs_result", None, Some(locals))
.unwrap();
// Checks if Python Long -> Rust BigUint conversion is correct if N is small
let py_result: BigUint =
FromPyObject::extract(fib.getattr("fib").unwrap().call1((400,)).unwrap()).unwrap();
assert_eq!(rs_result, py_result);
// Checks if Python Long -> Rust BigUint conversion is correct if N is large
let rs_result: BigUint = rust_fib(2000);
let py_result: BigUint =
FromPyObject::extract(fib.getattr("fib").unwrap().call1((2000,)).unwrap()).unwrap();
assert_eq!(rs_result, py_result);
});
}
#[test]
fn convert_bigint() {
Python::with_gil(|py| {
let rs_result = rust_fib::<BigInt>(400) * -1;
let fib = python_fib(py);
let locals = PyDict::new(py);
locals.set_item("rs_result", &rs_result).unwrap();
locals.set_item("fib", fib).unwrap();
// Checks if Rust BigInt -> Python Long conversion is correct
py.run("assert fib.fib_neg(400) == rs_result", None, Some(locals))
.unwrap();
// Checks if Python Long -> Rust BigInt conversion is correct if N is small
let py_result: BigInt =
FromPyObject::extract(fib.getattr("fib_neg").unwrap().call1((400,)).unwrap())
.unwrap();
assert_eq!(rs_result, py_result);
// Checks if Python Long -> Rust BigInt conversion is correct if N is large
let rs_result = rust_fib::<BigInt>(2000) * -1;
let py_result: BigInt =
FromPyObject::extract(fib.getattr("fib_neg").unwrap().call1((2000,)).unwrap())
.unwrap();
assert_eq!(rs_result, py_result);
})
}
fn python_index_class(py: Python) -> &PyModule {
let index_code = indoc!(
r#"
class C:
def __init__(self, x):
self.x = x
def __index__(self):
return self.x
"#
);
PyModule::from_code(py, index_code, "index.py", "index").unwrap()
}
#[test]
fn convert_index_class() {
Python::with_gil(|py| {
let index = python_index_class(py);
let locals = PyDict::new(py);
locals.set_item("index", index).unwrap();
let ob = py.eval("index.C(10)", None, Some(locals)).unwrap();
let _: BigInt = FromPyObject::extract(ob).unwrap();
});
}
#[test]
fn handle_zero() {
Python::with_gil(|py| {
let fib = python_fib(py);
let zero: BigInt =
FromPyObject::extract(fib.getattr("fib").unwrap().call1((0,)).unwrap()).unwrap();
assert_eq!(zero, BigInt::from(0));
})
}
/// `OverflowError` on converting Python int to BigInt, see issue #629
#[test]
fn check_overflow() {
Python::with_gil(|py| {
macro_rules! test {
($T:ty, $value:expr, $py:expr) => {
let value = $value;
println!("{}: {}", stringify!($T), value);
let python_value = value.clone().to_object(py);
let roundtrip_value = python_value.extract::<$T>(py).unwrap();
assert_eq!(value, roundtrip_value);
};
}
for i in 0..=256usize {
// test a lot of values to help catch other bugs too
test!(BigInt, BigInt::from(i), py);
test!(BigUint, BigUint::from(i), py);
test!(BigInt, -BigInt::from(i), py);
test!(BigInt, BigInt::from(1) << i, py);
test!(BigUint, BigUint::from(1u32) << i, py);
test!(BigInt, -BigInt::from(1) << i, py);
test!(BigInt, (BigInt::from(1) << i) + 1u32, py);
test!(BigUint, (BigUint::from(1u32) << i) + 1u32, py);
test!(BigInt, (-BigInt::from(1) << i) + 1u32, py);
test!(BigInt, (BigInt::from(1) << i) - 1u32, py);
test!(BigUint, (BigUint::from(1u32) << i) - 1u32, py);
test!(BigInt, (-BigInt::from(1) << i) - 1u32, py);
}
});
}
}

Some files were not shown because too many files have changed in this diff Show More