Merge branch 'immutable' of https://github.com/mejrs/pyo3 into immutable

.cargo/config

@@ -0,0 +1,8 @@
+[alias]
+xtask = "run --package xtask --"
+pyo3_doc = "doc --lib --no-default-features --features=full --no-deps --workspace --open --exclude pyo3-macros --exclude pyo3-macros-backend"
+pyo3_doc_scrape = "doc --lib --no-default-features --features=full --no-deps --workspace --open --exclude pyo3-macros --exclude pyo3-macros-backend -Z unstable-options -Z rustdoc-scrape-examples=examples"
+pyo3_doc_internal = "doc --lib --no-default-features --features=full --no-deps --workspace --open --document-private-items -Z unstable-options -Z rustdoc-scrape-examples=examples"
+
+[build]
+rustdocflags = ["--cfg", "docsrs"]

.github/pull_request_template.md

@@ -10,6 +10,6 @@ Be aware the CI pipeline will check your pull request for the following:
  - Rust lints (`make clippy`)
  - Rust formatting (`cargo fmt`)
  - Python formatting (`black . --check`. You can install black with `pip install black`)
- - Compatibility with all supported Python versions for all examples. This uses `tox`; you can do run it using `make test_py`.
+ - Compatibility with all supported Python versions for all examples. This uses `tox`; you can do run it using `cargo xtask test-py`.
 
 You can run a similar set of checks as the CI pipeline using `make test`.

.github/workflows/bench.yml

@@ -83,10 +83,8 @@ jobs:
 
       - name: Run benchmarks
         run: |
-          cd examples/pyo3-benchmarks
+          pip install nox
-          python -m pip install -r requirements-dev.txt
+          nox -f pytests/noxfile.py -s bench -- --benchmark-json $(pwd)/output.json
-          python setup.py develop
-          pytest --benchmark-json ../../output.json
       - name: Store benchmark result
         uses: rhysd/github-action-benchmark@v1
         with:

.github/workflows/ci.yml

@@ -15,16 +15,16 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
-      - run: pip install black==20.8b1
+      - run: pip install black==21.12b0
       - uses: actions-rs/toolchain@v1
         with:
           toolchain: stable
           profile: minimal
           components: rustfmt
       - name: Check python formatting (black)
-        run: black --check .
+        run: make fmt_py
       - name: Check rust formatting (rustfmt)
-        run: cargo fmt --all -- --check
+        run: make fmt_rust
 
   clippy:
     runs-on: ubuntu-latest
@@ -64,8 +64,8 @@ jobs:
             echo "suppress_build_script_link_lines=true" >> config.txt
             PYO3_BUILD_CONFIG=$(pwd)/config.txt cargo check --all-targets --no-default-features
             PYO3_BUILD_CONFIG=$(pwd)/config.txt cargo check --all-targets --no-default-features --features "abi3"
-            PYO3_BUILD_CONFIG=$(pwd)/config.txt cargo check --all-targets --no-default-features --features "$(make list_all_additive_features)"
+            PYO3_BUILD_CONFIG=$(pwd)/config.txt cargo check --all-targets --features full
-            PYO3_BUILD_CONFIG=$(pwd)/config.txt cargo check --all-targets --no-default-features --features "abi3 $(make list_all_additive_features)"
+            PYO3_BUILD_CONFIG=$(pwd)/config.txt cargo check --all-targets --features "abi3 full"
           done
 
   build:
@@ -76,7 +76,7 @@ jobs:
       fail-fast: false # If one platform fails, allow the rest to keep testing.
       matrix:
         rust: [stable]
-        python-version: [3.7, 3.8, 3.9, "3.10", pypy-3.7, pypy-3.8]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11-dev", "pypy-3.7", "pypy-3.8"]
         platform:
           [
             {
@@ -121,23 +121,6 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      # macos: install gnu-tar because BSD tar is buggy for github actions
-      # https://github.com/actions/cache/issues/403
-      - name: Install GNU tar (macOS only)
-        if: matrix.platform.os == 'macos-latest'
-        run: |
-          brew install gnu-tar
-          echo "/usr/local/opt/gnu-tar/libexec/gnubin" >> $GITHUB_PATH
-
-      - uses: actions/cache@v2
-        with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            target
-          key: cargo-${{ matrix.python-version }}-${{ matrix.platform.python-architecture }}-${{ matrix.platform.os }}-${{ matrix.msrv }}-${{ hashFiles('**/Cargo.toml') }}
-        continue-on-error: true
-
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v2
         with:
@@ -154,16 +137,15 @@ jobs:
           # needed to correctly format errors, see #1865
           components: rust-src
 
+      - uses: Swatinem/rust-cache@v1
+        with:
+          key: cargo-${{ matrix.platform.python-architecture }}-${{ matrix.platform.os }}-${{ matrix.msrv }}
+        continue-on-error: true
+
       - if: matrix.platform.os == 'ubuntu-latest'
         name: Prepare LD_LIBRARY_PATH (Ubuntu only)
         run: echo LD_LIBRARY_PATH=${pythonLocation}/lib >> $GITHUB_ENV
 
-      - name: Prepare workflow settings
-        id: settings
-        shell: bash
-        run: |
-          echo "::set-output name=all_additive_features::$(make list_all_additive_features)"
-
       - if: matrix.msrv == 'MSRV'
         name: Prepare minimal package versions (MSRV only)
         run: |
@@ -171,7 +153,7 @@ jobs:
           cargo update -p hashbrown:0.11.2 --precise 0.9.1
 
       - name: Build docs
-        run: cargo doc --no-deps --no-default-features --features "${{ steps.settings.outputs.all_additive_features }}"
+        run: cargo doc --no-deps --no-default-features --features full
 
       - name: Build (no features)
         run: cargo build --lib --tests --no-default-features
@@ -185,7 +167,7 @@ jobs:
       # Run tests (except on PyPy, because no embedding API).
       - if: ${{ !startsWith(matrix.python-version, 'pypy') }}
         name: Test (no features)
-        run: cargo test --no-default-features
+        run: cargo test --no-default-features --lib --tests
 
       # --no-default-features when used with `cargo build/test -p` doesn't seem to work!
       - name: Test pyo3-build-config (no features)
@@ -194,26 +176,26 @@ jobs:
           cargo test --no-default-features
 
       - name: Build (all additive features)
-        run: cargo build --lib --tests --no-default-features --features "${{ steps.settings.outputs.all_additive_features }}"
+        run: cargo build --lib --tests --no-default-features --features full
 
       - if: ${{ startsWith(matrix.python-version, 'pypy') }}
         name: Build PyPy (abi3-py37)
-        run: cargo build --lib --tests --no-default-features --features "abi3-py37 ${{ steps.settings.outputs.all_additive_features }}"
+        run: cargo build --lib --tests --no-default-features --features "abi3-py37 full"
 
       # Run tests (except on PyPy, because no embedding API).
       - if: ${{ !startsWith(matrix.python-version, 'pypy') }}
         name: Test
-        run: cargo test --no-default-features --features "${{ steps.settings.outputs.all_additive_features }}"
+        run: cargo test --no-default-features --features full
 
       # Run tests again, but in abi3 mode
       - if: ${{ !startsWith(matrix.python-version, 'pypy') }}
         name: Test (abi3)
-        run: cargo test --no-default-features --features "abi3 ${{ steps.settings.outputs.all_additive_features }}"
+        run: cargo test --no-default-features --features "abi3 full"
 
       # Run tests again, for abi3-py37 (the minimal Python version)
       - if: ${{ (!startsWith(matrix.python-version, 'pypy')) && (matrix.python-version != '3.7') }}
         name: Test (abi3-py37)
-        run: cargo test --no-default-features --features "abi3-py37 ${{ steps.settings.outputs.all_additive_features }}"
+        run: cargo test --no-default-features --features "abi3-py37 full"
 
       - name: Test proc-macro code
         run: cargo test --manifest-path=pyo3-macros-backend/Cargo.toml
@@ -221,17 +203,13 @@ jobs:
       - name: Test build config
         run: cargo test --manifest-path=pyo3-build-config/Cargo.toml
 
-      - name: Install python test dependencies
+      - name: Test python examples and tests
-        run: python -m pip install -U pip tox
-
-      - name: Test example extension modules
         shell: bash
         run: |
-          for example_dir in examples/*/; do
+          python -m pip install -U pip nox
-              tox -c $example_dir -e py
+          cargo xtask test-py
-          done
         env:
-          TOX_TESTENV_PASSENV: "CARGO_BUILD_TARGET"
+          CARGO_TARGET_DIR: ${{ github.workspace }}/target
 
       - name: Test cross compilation
         if: ${{ matrix.platform.os == 'ubuntu-latest' && matrix.python-version == '3.9' }}
@@ -252,6 +230,10 @@ jobs:
       # TODO: this is a hack to workaround compile_error! warnings about auto-initialize on PyPy
       # Once cargo's `resolver = "2"` is stable (~ MSRV Rust 1.52), remove this.
       PYO3_CI: 1
+      # This is a hack to make CARGO_PRIMARY_PACKAGE always set even for the
+      # msrv job. MSRV is currently 1.48, but CARGO_PRIMARY_PACKAGE only came in
+      # 1.49.
+      CARGO_PRIMARY_PACKAGE: 1
 
   coverage:
     needs: [fmt]
@@ -265,13 +247,9 @@ jobs:
       - uses: actions/setup-python@v2
         with:
           python-version: "3.10"
-      - uses: actions/cache@v2
+      - uses: Swatinem/rust-cache@v1
         with:
-          path: |
+          key: coverage-cargo-${{ matrix.os }}
-            ~/.cargo/registry
-            ~/.cargo/git
-            target
-          key: coverage-cargo-${{ hashFiles('**/Cargo.toml') }}
         continue-on-error: true
       - uses: actions-rs/toolchain@v1
         with:
@@ -279,29 +257,10 @@ jobs:
           override: true
           profile: minimal
           components: llvm-tools-preview
-      - name: install cargo-llvm-cov
+      - name: Install cargo-llvm-cov
-        shell: bash
+        uses: taiki-e/install-action@cargo-llvm-cov
-        run: |
+      - run: python -m pip install -U pip nox
-          host=$(rustc -Vv | grep host | sed 's/host: //')
+      - run: cargo xtask coverage --output-lcov coverage.lcov
-          curl -fsSL https://github.com/taiki-e/cargo-llvm-cov/releases/download/v${CARGO_LLVM_COV_VERSION}/cargo-llvm-cov-"$host".tar.gz | tar xzf - -C ~/.cargo/bin
-        env:
-          CARGO_LLVM_COV_VERSION: 0.1.11
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: nightly
-          override: true
-          profile: minimal
-          components: llvm-tools-preview
-      - run: |
-          cargo llvm-cov clean --workspace
-          cargo llvm-cov --package $ALL_PACKAGES --no-report
-          cargo llvm-cov --package $ALL_PACKAGES --no-report --features abi3
-          cargo llvm-cov --package $ALL_PACKAGES --no-report --features $(make list_all_additive_features)
-          cargo llvm-cov --package $ALL_PACKAGES --no-report --features abi3 $(make list_all_additive_features)
-          cargo llvm-cov --package $ALL_PACKAGES --no-run --lcov --output-path coverage.lcov
-        shell: bash
-        env:
-          ALL_PACKAGES: pyo3 pyo3-build-config pyo3-macros-backend pyo3-macros
       - uses: codecov/codecov-action@v2
         with:
           file: coverage.lcov

.github/workflows/guide.yml

@@ -35,6 +35,29 @@ jobs:
           TAG_NAME="${GITHUB_REF##*/}"
           echo "::set-output name=tag_name::${TAG_NAME}"
 
+      # Build some internal docs and inject a banner on top of it.
+      - name: Build the internal docs
+        run: |
+          mkdir target
+          mkdir -p gh-pages-build/internal
+          echo "<div class='internal-banner' style='position:fixed; z-index: 99999; color:red;border:3px solid red;margin-left: auto; margin-right: auto; width: 430px;left:0;right: 0;'><div style='display: flex; align-items: center; justify-content: center;'> ⚠️ Internal Docs ⚠️ Not Public API 👉 <a href='https://pyo3.rs/main/doc/pyo3/index.html' style='color:red;text-decoration:underline;'>Official Docs Here</a></div></div>" > target/banner.html
+          cargo +nightly pyo3_doc_internal
+          cp -r target/doc gh-pages-build/internal
+        env:
+          RUSTDOCFLAGS: "--cfg docsrs --Z unstable-options --document-hidden-items --html-before-content target/banner.html"
+
+      - name: Deploy internal docs
+        if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'release' }}
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./gh-pages-build/internal
+          destination_dir: internal
+          full_commit_message: "Upload internal documentation"
+
+      - name: Clear the extra artefacts created earlier 
+        run: rm -rf target
+
       # This builds the book in gh-pages-build. See https://github.com/rust-lang-nursery/mdBook/issues/698
       - name: Build the guide
         run: mdbook build -d ../gh-pages-build guide
@@ -44,11 +67,11 @@ jobs:
         # This adds the docs to gh-pages-build/doc
       - name: Build the doc
         run: |
-          cargo +nightly rustdoc --lib --no-default-features --features="macros num-bigint num-complex hashbrown indexmap serde multiple-pymethods eyre anyhow" -- --cfg docsrs
+          cargo +nightly pyo3_doc_scrape
           cp -r target/doc gh-pages-build/doc
           echo "<meta http-equiv=refresh content=0;url=pyo3/index.html>" > gh-pages-build/doc/index.html
 
-      - name: Deploy
+      - name: Deploy docs and the guide
         if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'release' }}
         uses: peaceiris/actions-gh-pages@v3
         with:
@@ -56,7 +79,6 @@ jobs:
           publish_dir: ./gh-pages-build/
           destination_dir: ${{ steps.prepare_tag.outputs.tag_name }}
           full_commit_message: "Upload documentation for ${{ steps.prepare_tag.outputs.tag_name }}"
-
   release:
     needs: build
     runs-on: ubuntu-latest

.gitignore

@@ -14,7 +14,6 @@ dist/
 .eggs/
 venv*
 guide/book/
-examples/*/py*
 *.so
 *.out
 *.egg-info

Architecture.md

@@ -19,7 +19,7 @@ Since implementing `PyClass` requires lots of boilerplate, we have a proc-macro
 To summarize, there are six main parts to the PyO3 codebase.
 
 1. [Low-level bindings of Python/C API.](#1-low-level-bindings-of-python-capi)
-   - [`src/ffi`]
+   - [`pyo3-ffi`] and [`src/ffi`]
 2. [Bindings to Python objects.](#2-bindings-to-python-objects)
    - [`src/instance.rs`] and [`src/types`]
 3. [`PyClass` and related functionalities.](#3-pyclass-and-related-functionalities)
@@ -34,7 +34,7 @@ To summarize, there are six main parts to the PyO3 codebase.
 
 ## 1. Low-level bindings of Python/C API
 
-[`src/ffi`] contains wrappers of [Python/C API].
+[`pyo3-ffi`] contains wrappers of [Python/C API].
 
 We aim to provide straight-forward Rust wrappers resembling the file structure of
 [`cpython/Include`](https://github.com/python/cpython/tree/v3.9.2/Include).
@@ -43,7 +43,7 @@ However, we still lack some APIs and are continuously updating the module to mat
 the file contents upstream in CPython.
 The tracking issue is [#1289](https://github.com/PyO3/pyo3/issues/1289), and contribution is welcome.
 
-In the [`src/ffi`] module, there is lots of conditional compilation such as `#[cfg(Py_LIMITED_API)]`,
+In the [`pyo3-ffi`] crate, there is lots of conditional compilation such as `#[cfg(Py_LIMITED_API)]`,
 `#[cfg(Py_37)]`, and `#[cfg(PyPy)]`.
 `Py_LIMITED_API` corresponds to `#define Py_LIMITED_API` macro in Python/C API.
 With `Py_LIMITED_API`, we can build a Python-version-agnostic binary called an
@@ -102,7 +102,7 @@ Since we need lots of boilerplate for implementing common traits for these types
 
 [`src/pycell.rs`], [`src/pyclass.rs`], and [`src/type_object.rs`] contain types and
 traits to make `#[pyclass]` work.
-Also, [`src/pyclass_init.rs`] and [`src/pyclass_slots.rs`] have related functionalities.
+Also, [`src/pyclass_init.rs`] and [`src/impl_/pyclass.rs`] have related functionalities.
 
 To realize object-oriented programming in C, all Python objects must have the following two fields
 at the beginning.
@@ -208,6 +208,7 @@ Some of the functionality of `pyo3-build-config`:
 [`pyo3-macros`]: https://github.com/PyO3/pyo3/tree/main/pyo3-macros
 [`pyo3-macros-backend`]: https://github.com/PyO3/pyo3/tree/main/pyo3-macros-backend
 [`pyo3-build-config`]: https://github.com/PyO3/pyo3/tree/main/pyo3-build-config
+[`pyo3-ffi`]: https://github.com/PyO3/pyo3/tree/main/pyo3-ffi
 
 <!-- Directories -->
 

CHANGELOG.md

@@ -12,23 +12,79 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Update MSRV to Rust 1.48. [#2004](https://github.com/PyO3/pyo3/pull/2004)
 - Update `indoc` optional dependency to 1.0. [#2004](https://github.com/PyO3/pyo3/pull/2004)
-- Update `paste` optional dependency to 1.0. [#2004](https://github.com/PyO3/pyo3/pull/2004)
 - Drop support for Python 3.6, remove `abi3-py36` feature. [#2006](https://github.com/PyO3/pyo3/pull/2006)
 - `pyo3-build-config` no longer enables the `resolve-config` feature by default. [#2008](https://github.com/PyO3/pyo3/pull/2008)
+- Update `inventory` optional dependency to 0.2. [#2019](https://github.com/PyO3/pyo3/pull/2019)
+- Drop `paste` dependency. [#2081](https://github.com/PyO3/pyo3/pull/2081)
+- The bindings found `pyo3::ffi` are now a re-export of the new `pyo3-ffi` crate. [#2126](https://github.com/PyO3/pyo3/pull/2126)
 
 ### Added
 
 - Add `Py::setattr` method. [#2009](https://github.com/PyO3/pyo3/pull/2009)
+- Add `PyCapsule`, exposing the [Capsule API](https://docs.python.org/3/c-api/capsule.html#capsules). [#1980](https://github.com/PyO3/pyo3/pull/1980)
+- Expose `pyo3-build-config` APIs for cross-compiling and Python configuration discovery for use in other projects. [#1996](https://github.com/PyO3/pyo3/pull/1996)
+- All PyO3 proc-macros except the deprecated `#[pyproto]` now accept a supplemental attribute `#[pyo3(crate = "some::path")]` that specifies where to find the `pyo3` crate, in case it has been renamed or is re-exported and not found at the crate root. [#2022](https://github.com/PyO3/pyo3/pull/2022)
+- Enable `#[pyclass]` for fieldless (aka C-like) enums. [#2034](https://github.com/PyO3/pyo3/pull/2034)
+- Add buffer magic methods `__getbuffer__` and `__releasebuffer__` to `#[pymethods]`. [#2067](https://github.com/PyO3/pyo3/pull/2067)
+- Accept paths in `wrap_pyfunction` and `wrap_pymodule`. [#2081](https://github.com/PyO3/pyo3/pull/2081)
+- Add check for correct number of arguments on magic methods. [#2083](https://github.com/PyO3/pyo3/pull/2083)
+- `wrap_pyfunction!` can now wrap a `#[pyfunction]` which is implemented in a different Rust module or crate. [#2091](https://github.com/PyO3/pyo3/pull/2091)
+- Add `PyAny::contains` method (`in` operator for `PyAny`). [#2115](https://github.com/PyO3/pyo3/pull/2115)
+- Add `PyMapping::contains` method (`in` operator for `PyMapping`). [#2133](https://github.com/PyO3/pyo3/pull/2133)
 
 ### Changed
 
 - `PyType::is_subclass`, `PyErr::is_instance` and `PyAny::is_instance` now operate run-time type object instead of a type known at compile-time. The old behavior is still available as `PyType::is_subclass_of`, `PyErr::is_instance_of` and `PyAny::is_instance_of`.  [#1985](https://github.com/PyO3/pyo3/pull/1985)
+- Rename some methods on `PyErr` (the old names are just marked deprecated for now): [#2026](https://github.com/PyO3/pyo3/pull/2026)
+  - `pytype` -> `get_type`
+  - `pvalue` -> `value` (and deprecate equivalent `instance`)
+  - `ptraceback` -> `traceback`
+  - `from_instance` -> `from_value`
+  - `into_instance` -> `into_value`
+- Deprecate `PyType::is_instance`; it is inconsistent with other `is_instance` methods in PyO3. Instead of `typ.is_instance(obj)`, use `obj.is_instance(typ)`. [#2031](https://github.com/PyO3/pyo3/pull/2031)
+- `PyErr::new_type` now takes an optional docstring and now returns `PyResult<Py<PyType>>` rather than a `ffi::PyTypeObject` pointer.
+- The `create_exception!` macro can now take an optional docstring. This docstring, if supplied, is visible to users (with `.__doc__` and `help()`) and
+  accompanies your error type in your crate's documentation.
+- `__getitem__`, `__setitem__` and `__delitem__` in `#[pymethods]` now implement both a Python mapping and sequence by default. [#2065](https://github.com/PyO3/pyo3/pull/2065)
+- Improve performance and error messages for `#[derive(FromPyObject)]` for enums. [#2068](https://github.com/PyO3/pyo3/pull/2068)
+- Reduce generated LLVM code size (to improve compile times) for:
+  - internal `handle_panic` helper [#2074](https://github.com/PyO3/pyo3/pull/2074)
+  - `#[pyfunction]` and `#[pymethods]` argument extraction [#2075](https://github.com/PyO3/pyo3/pull/2075) [#2085](https://github.com/PyO3/pyo3/pull/2085)
+  - `#[pyclass]` type object creation [#2076](https://github.com/PyO3/pyo3/pull/2076) [#2081](https://github.com/PyO3/pyo3/pull/2081)
+- `__ipow__` now supports modulo argument on Python 3.8+. [#2083](https://github.com/PyO3/pyo3/pull/2083)
+- `pyo3-macros-backend` is now compiled with PyO3 cfgs to enable different magic method definitions based on version. [#2083](https://github.com/PyO3/pyo3/pull/2083)
+- `_PyCFunctionFast` now correctly reflects the signature defined in the [Python docs](https://docs.python.org/3/c-api/structures.html#c._PyCFunctionFast). [#2126](https://github.com/PyO3/pyo3/pull/2126)
+- `PyDateTimeAPI` and `PyDateTime_TimeZone_UTC` are are now unsafe functions instead of statics. [#2126](https://github.com/PyO3/pyo3/pull/2126)
+- `PyDateTimeAPI` does not implicitly call `PyDateTime_IMPORT` anymore to reflect the original Python API more closely. Before the first call to `PyDateTime_IMPORT` a null pointer is returned. Therefore before calling any of the following FFI functions `PyDateTime_IMPORT` must be called to avoid undefined behaviour: [#2126](https://github.com/PyO3/pyo3/pull/2126)
+  - `PyDateTime_TimeZone_UTC`
+  - `PyDate_Check`
+  - `PyDate_CheckExact`
+  - `PyDateTime_Check`
+  - `PyDateTime_CheckExact`
+  - `PyTime_Check`
+  - `PyTime_CheckExact`
+  - `PyDelta_Check`
+  - `PyDelta_CheckExact`
+  - `PyTZInfo_Check`
+  - `PyTZInfo_CheckExact`
+  - `PyDateTime_FromTimestamp`
+  - `PyDate_FromTimestamp`
 
 ### Removed
 
-- Remove `PyType::is_instance`, which is unintuitive; instead of `typ.is_instance(obj)`, use `obj.is_instance(typ)`. [#1985](https://github.com/PyO3/pyo3/pull/1985)
 - Remove all functionality deprecated in PyO3 0.14. [#2007](https://github.com/PyO3/pyo3/pull/2007)
 
+### Fixed
+
+- Fix undefined symbol for `PyObject_HasAttr` on PyPy. [#2025](https://github.com/PyO3/pyo3/pull/2025)
+- Fix memory leak in `PyErr::into_value`. [#2026](https://github.com/PyO3/pyo3/pull/2026)
+- Fix clippy warning `needless-option-as-deref` in code generated by `#[pyfunction]` and `#[pymethods]`. [#2040](https://github.com/PyO3/pyo3/pull/2040)
+- Fix undefined behavior in `PySlice::indices`. [#2061](https://github.com/PyO3/pyo3/pull/2061)
+- Use the Rust function path for `wrap_pymodule!` of a `#[pymodule]` with a `#[pyo3(name = "..")]` attribute, not the Python name. [#2081](https://github.com/PyO3/pyo3/pull/2081)
+- Fix panic in `#[pyfunction]` generated code when a required argument following an `Option` was not provided.  [#2093](https://github.com/PyO3/pyo3/pull/2093)
+- Fixed undefined behaviour caused by incorrect `ExactSizeIterator` implementations. [#2124](https://github.com/PyO3/pyo3/pull/2124)
+- Add missing FFI definitions `_PyLong_NumBits` and `_PyLong_AsByteArray` on PyPy. [#2146](https://github.com/PyO3/pyo3/pull/2146)
+
 ## [0.15.1] - 2021-11-19
 
 ### Added

Cargo.toml

@@ -10,7 +10,7 @@ repository = "https://github.com/pyo3/pyo3"
 documentation = "https://docs.rs/crate/pyo3/"
 categories = ["api-bindings", "development-tools::ffi"]
 license = "Apache-2.0"
-exclude = ["/.gitignore", ".cargo/config", "/codecov.yml", "/Makefile", "/pyproject.toml", "/tox.ini"]
+exclude = ["/.gitignore", ".cargo/config", "/codecov.yml", "/Makefile", "/pyproject.toml", "/noxfile.py"]
 edition = "2018"
 links = "python"
 
@@ -19,15 +19,16 @@ cfg-if = "1.0"
 libc = "0.2.62"
 parking_lot = "0.11.0"
 
+# ffi bindings to the python interpreter, split into a seperate crate so they can be used independently
+pyo3-ffi = { path = "pyo3-ffi", version = "=0.15.1" }
+
 # support crates for macros feature
 pyo3-macros = { path = "pyo3-macros", version = "=0.15.1", optional = true }
 indoc = { version = "1.0.3", optional = true }
-paste = { version = "1.0.6", optional = true }
 unindent = { version = "0.1.4", optional = true }
 
 # support crate for multiple-pymethods feature
-# must stay at 0.1.x for Rust 1.41 compatibility
+inventory = { version = "0.2.0", optional = true }
-inventory = { version = "0.1.4", optional = true }
 
 # crate integrations that can be added using the eponymous features
 anyhow = { version = "1.0", optional = true }
@@ -47,34 +48,34 @@ rustversion = "1.0"
 proptest = { version = "0.10.1", default-features = false, features = ["std"] }
 serde_json = "1.0.61"
 
-# features needed to run the PyO3 test suite
-pyo3 = { path = ".", default-features = false, features = ["macros", "auto-initialize"] }
-
 [build-dependencies]
 pyo3-build-config = { path = "pyo3-build-config", version = "0.15.1", features = ["resolve-config"] }
 
 [features]
-default = ["macros"]
+default = ["macros", "pyproto"]
 
 # Enables macros: #[pyclass], #[pymodule], #[pyfunction] etc.
-macros = ["pyo3-macros", "indoc", "paste", "unindent"]
+macros = ["pyo3-macros", "indoc", "unindent"]
 
 # Enables multiple #[pymethods] per #[pyclass]
 multiple-pymethods = ["inventory", "pyo3-macros/multiple-pymethods"]
 
+# Enables deprecated #[pyproto] macro
+pyproto = ["pyo3-macros/pyproto"]
+
 # Use this feature when building an extension module.
 # It tells the linker to keep the python symbols unresolved,
 # so that the module can also be used with statically linked python interpreters.
-extension-module = []
+extension-module = ["pyo3-ffi/extension-module"]
 
 # Use the Python limited API. See https://www.python.org/dev/peps/pep-0384/ for more.
-abi3 = ["pyo3-build-config/abi3"]
+abi3 = ["pyo3-build-config/abi3", "pyo3-ffi/abi3"]
 
 # With abi3, we can manually set the minimum Python version.
-abi3-py37 = ["abi3-py38", "pyo3-build-config/abi3-py37"]
+abi3-py37 = ["abi3-py38", "pyo3-build-config/abi3-py37", "pyo3-ffi/abi3-py37"]
-abi3-py38 = ["abi3-py39", "pyo3-build-config/abi3-py38"]
+abi3-py38 = ["abi3-py39", "pyo3-build-config/abi3-py38", "pyo3-ffi/abi3-py38"]
-abi3-py39 = ["abi3-py310", "pyo3-build-config/abi3-py39"]
+abi3-py39 = ["abi3-py310", "pyo3-build-config/abi3-py39", "pyo3-ffi/abi3-py39"]
-abi3-py310 = ["abi3", "pyo3-build-config/abi3-py310"]
+abi3-py310 = ["abi3", "pyo3-build-config/abi3-py310", "pyo3-ffi/abi3-py310"]
 
 # Changes `Python::with_gil` and `Python::acquire_gil` to automatically initialize the
 # Python interpreter if needed.
@@ -83,6 +84,10 @@ auto-initialize = []
 # Optimizes PyObject to Vec conversion and so on.
 nightly = []
 
+# Activates all additional features
+# This is mostly intended for testing purposes - activating *all* of these isn't particularly useful.
+full = ["macros", "pyproto", "multiple-pymethods", "num-bigint", "num-complex", "hashbrown", "serde", "indexmap", "eyre", "anyhow"]
+
 [[bench]]
 name = "bench_call"
 harness = false
@@ -95,6 +100,11 @@ harness = false
 name = "bench_dict"
 harness = false
 
+[[bench]]
+name = "bench_frompyobject"
+harness = false
+required-features = ["macros"]
+
 [[bench]]
 name = "bench_gil"
 harness = false
@@ -106,6 +116,7 @@ harness = false
 [[bench]]
 name = "bench_pyclass"
 harness = false
+required-features = ["macros"]
 
 [[bench]]
 name = "bench_pyobject"
@@ -121,13 +132,13 @@ harness = false
 
 [workspace]
 members = [
+    "pyo3-ffi",
+    "pyo3-build-config",
     "pyo3-macros",
     "pyo3-macros-backend",
-    "examples/pyo3-benchmarks",
+    "pytests",
-    "examples/pyo3-pytests",
+    "examples",
-    "examples/maturin-starter",
+    "xtask"
-    "examples/setuptools-rust-starter",
-    "examples/word-count"
 ]
 
 [package.metadata.docs.rs]

Contributing.md

@@ -11,8 +11,21 @@ If you want to become familiar with the codebase, see
 
 Please join in with any part of PyO3 which interests you. We use GitHub issues to record all bugs and ideas. Feel free to request an issue to be assigned to you if you want to work on it.
 
+You can browse the API of the non-public parts of PyO3 [here](https://pyo3.rs/internal/doc/pyo3/index.html).
+
 The following sections also contain specific ideas on where to start contributing to PyO3.
 
+## Setting up a development environment
+
+To work and develop PyO3, you need Python & Rust installed on your system.
+* We encourage the use of [rustup](https://rustup.rs/) to be able to select and choose specific toolchains based on the project.
+* [Pyenv](https://github.com/pyenv/pyenv) is also highly recommended for being able to choose a specific Python version.
+* [virtualenv](https://virtualenv.pypa.io/en/latest/) can also be used with or without Pyenv to use specific installed Python versions.
+
+### Caveats
+
+* When using pyenv on macOS, installing a Python version using `--enable-shared` is required to make it work. i.e `env PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install 3.7.12`
+
 ### Help users identify bugs
 
 The [PyO3 Gitter channel](https://gitter.im/PyO3/Lobby) is very active with users who are new to PyO3, and often completely new to Rust. Helping them debug is a great way to get experience with the PyO3 codebase.
@@ -34,7 +47,8 @@ There are some specific areas of focus where help is currently needed for the do
 - Issues requesting documentation improvements are tracked with the [documentation](https://github.com/PyO3/pyo3/issues?q=is%3Aissue+is%3Aopen+label%3Adocumentation) label.
 - Not all APIs had docs or examples when they were made. The goal is to have documentation on all PyO3 APIs ([#306](https://github.com/PyO3/pyo3/issues/306)). If you see an API lacking a doc, please write one and open a PR!
 
-You can build the docs (including all features) with `cargo +nightly rustdoc --features="$(make list_all_additive_features)" --open -- --cfg docsrs`.
+You can build the docs (including all features) with
+```cargo +nightly pyo3_doc_scrape```
 
 #### Doctests
 
@@ -47,7 +61,7 @@ https://doc.rust-lang.org/rustdoc/documentation-tests.html for a guide on doctes
 You can preview the user guide by building it locally with `mdbook`.
 
 First, [install `mdbook`](https://rust-lang.github.io/mdBook/cli/index.html). Then, run
-`mdbook build -d ../gh-pages-build guide --open`.
+```mdbook build -d ../gh-pages-build guide --open```
 
 ### Help design the next PyO3
 
@@ -71,7 +85,7 @@ Formatting, linting and tests are checked for all Rust and Python code. In addit
 
 Tests run with all supported Python versions with the latest stable Rust compiler, as well as for Python 3.9 with the minimum supported Rust version.
 
-If you are adding a new feature, you should add it to the `ALL_ADDITIVE_FEATURES` declaration in the `Makefile` so that it is tested in CI.
+If you are adding a new feature, you should add it to the `full` feature in our *Cargo.toml** so that it is tested in CI.
 
 ## Python and Rust version support policy
 
@@ -101,7 +115,7 @@ First, there are Rust-based benchmarks located in the `benches` subdirectory. As
 
     cargo +nightly bench
 
-Second, there is a Python-based benchmark contained in the `pyo3-benchmarks` example. You can read more about it [here](examples/pyo3-benchmarks).
+Second, there is a Python-based benchmark contained in the `pytests` subdirectory. You can read more about it [here](pytests).
 
 ## Sponsor this project
 

Makefile

@@ -1,9 +1,7 @@
-.PHONY: test test_py publish clippy lint fmt
+.PHONY: test test_py publish clippy lint fmt fmt_py fmt_rust
 
 ALL_ADDITIVE_FEATURES = macros multiple-pymethods num-bigint num-complex hashbrown serde indexmap eyre anyhow
-
+COVERAGE_PACKAGES = --package pyo3 --package pyo3-build-config --package pyo3-macros-backend --package pyo3-macros
-list_all_additive_features:
-	@echo $(ALL_ADDITIVE_FEATURES)
 
 test: lint test_py
 	cargo test
@@ -12,21 +10,30 @@ test: lint test_py
 	cargo test --features="abi3 $(ALL_ADDITIVE_FEATURES)"
 
 test_py:
-	for example in examples/*/; do TOX_TESTENV_PASSENV=RUSTUP_HOME tox -e py -c $$example || exit 1; done
+	@for example in examples/*/noxfile.py; do echo "-- Running nox for $$example --"; nox -f $$example/noxfile.py || exit 1; echo ""; done
+	echo "-- Running nox for pytests/noxfile.py --";
+	nox -f pytests/noxfile.py || exit 1;
 
-fmt:
+fmt_py:
-	cargo fmt --all -- --check
 	black . --check
 
+fmt_rust:
+	cargo fmt --all -- --check
+
+fmt: fmt_rust fmt_py
+	@true
+
 clippy:
-	cargo clippy --features="$(ALL_ADDITIVE_FEATURES)" --tests -- -Dwarnings
+	cargo clippy --features="$(ALL_ADDITIVE_FEATURES)" --all-targets --workspace -- -Dwarnings
-	cargo clippy --features="abi3 $(ALL_ADDITIVE_FEATURES)" --tests -- -Dwarnings
+	cargo clippy --features="abi3 $(ALL_ADDITIVE_FEATURES)" --all-targets --workspace -- -Dwarnings
 	for example in examples/*/; do cargo clippy --manifest-path $$example/Cargo.toml -- -Dwarnings || exit 1; done
 
 lint: fmt clippy
 	@true
 
 publish: test
+	cargo publish --manifest-path pyo3-build-config/Cargo.toml
+	sleep 10
 	cargo publish --manifest-path pyo3-macros-backend/Cargo.toml
 	sleep 10  # wait for crates.io to update
 	cargo publish --manifest-path pyo3-macros/Cargo.toml

README.md

@@ -24,15 +24,34 @@ You can use PyO3 to write a native Python module in Rust, or to embed Python in
 
 ### Using Rust from Python
 
-PyO3 can be used to generate a native Python module. The easiest way to try this out for the first time is to use [`maturin`](https://github.com/PyO3/maturin). `maturin` is a tool for building and publishing Rust-based Python packages with minimal configuration. The following steps set up some files for an example Python module, install `maturin`, and then show how build and import the Python module.
+PyO3 can be used to generate a native Python module. The easiest way to try this out for the first time is to use [`maturin`](https://github.com/PyO3/maturin). `maturin` is a tool for building and publishing Rust-based Python packages with minimal configuration. The following steps install `maturin`, use it to generate and build a new Python package, and then launch Python to import and execute a function from the package.
 
-First, create a new folder (let's call it `string_sum`) containing the following two files:
+First, follow the commands below to create a new directory containing a new Python `virtualenv`, and install `maturin` into the virtualenv using Python's package manager, `pip`:
+
+```bash
+# (replace string_sum with the desired package name)
+$ mkdir string_sum
+$ cd string_sum
+$ python -m venv .env
+$ source .env/bin/activate
+$ pip install maturin
+```
+
+Still inside this `string_sum` directory, now run `maturin init`. This will generate the new package source. When given the choice of bindings to use, select pyo3 bindings:
+
+```bash
+$ maturin init
+✔ 🤷 What kind of bindings to use? · pyo3
+  ✨ Done! New project created string_sum
+```
+
+The most important files generated by this command are `Cargo.toml` and `lib.rs`, which will look roughly like the following:
 
 **`Cargo.toml`**
 
 ```toml
 [package]
-name = "string-sum"
+name = "string_sum"
 version = "0.1.0"
 edition = "2018"
 
@@ -45,9 +64,8 @@ name = "string_sum"
 # crate-type = ["cdylib", "rlib"]
 crate-type = ["cdylib"]
 
-[dependencies.pyo3]
+[dependencies]
-version = "0.15.1"
+pyo3 = { version = "0.15.1", features = ["extension-module"] }
-features = ["extension-module"]
 ```
 
 **`src/lib.rs`**
@@ -67,21 +85,11 @@ fn sum_as_string(a: usize, b: usize) -> PyResult<String> {
 #[pymodule]
 fn string_sum(_py: Python, m: &PyModule) -> PyResult<()> {
     m.add_function(wrap_pyfunction!(sum_as_string, m)?)?;
-
     Ok(())
 }
 ```
 
-With those two files in place, now `maturin` needs to be installed. This can be done using Python's package manager `pip`. First, load up a new Python `virtualenv`, and install `maturin` into it:
+Finally, run `maturin develop`. This will build the package and install it into the Python virtualenv previously created and activated. The package is then ready to be used from `python`:
-
-```bash
-$ cd string_sum
-$ python -m venv .env
-$ source .env/bin/activate
-$ pip install maturin
-```
-
-Now build and execute the module:
 
 ```bash
 $ maturin develop
@@ -92,7 +100,20 @@ $ python
 '25'
 ```
 
-As well as with `maturin`, it is possible to build using [`setuptools-rust`](https://github.com/PyO3/setuptools-rust) or [manually](https://pyo3.rs/latest/building_and_distribution.html#manual-builds). Both offer more flexibility than `maturin` but require further configuration.
+To make changes to the package, just edit the Rust source code and then re-run `maturin develop` to recompile.
+
+To run this all as a single copy-and-paste, use the bash script below (replace `string_sum` in the first command with the desired package name):
+
+```bash
+mkdir string_sum && cd "$_"
+python -m venv .env
+source .env/bin/activate
+pip install maturin
+maturin init --bindings pyo3
+maturin develop
+```
+
+As well as with `maturin`, it is possible to build using [`setuptools-rust`](https://github.com/PyO3/setuptools-rust) or [manually](https://pyo3.rs/latest/building_and_distribution.html#manual-builds). Both offer more flexibility than `maturin` but require more configuration to get started.
 
 ### Using Python from Rust
 
@@ -138,7 +159,7 @@ about this topic.
 
 ## Tools and libraries
 
-- [maturin](https://github.com/PyO3/maturin) _Zero configuration build tool for Rust-made Python extensions_.
+- [maturin](https://github.com/PyO3/maturin) _Build and publish crates with pyo3, rust-cpython or cffi bindings as well as rust binaries as python packages_
 - [setuptools-rust](https://github.com/PyO3/setuptools-rust) _Setuptools plugin for Rust support_.
 - [pyo3-built](https://github.com/PyO3/pyo3-built) _Simple macro to expose metadata obtained with the [`built`](https://crates.io/crates/built) crate as a [`PyDict`](https://docs.rs/pyo3/*/pyo3/types/struct.PyDict.html)_
 - [rust-numpy](https://github.com/PyO3/rust-numpy) _Rust binding of NumPy C-API_
@@ -168,9 +189,14 @@ about this topic.
 - [cryptography](https://github.com/pyca/cryptography/tree/main/src/rust) _Python cryptography library with some functionality in Rust_
 - [polaroid](https://github.com/daggy1234/polaroid) _Hyper Fast and safe image manipulation library for Python written in Rust_
 - [ormsgpack](https://github.com/aviramha/ormsgpack) _Fast Python msgpack library_
+- [bed-reader](https://github.com/fastlmm/bed-reader) _Read and write the PLINK BED format, simply and efficiently_
+    - Shows Rayon/ndarray::parallel (including capturing errors, controlling thread num), Python types to Rust generics, Github Actions
+- [pyheck](https://github.com/kevinheavey/pyheck) _Fast case conversion library, built by wrapping [heck](https://github.com/withoutboats/heck)_
+    - Quite easy to follow as there's not much code.
 
 ## Articles and other media
 
+- [Nine Rules for Writing Python Extensions in Rust](https://towardsdatascience.com/nine-rules-for-writing-python-extensions-in-rust-d35ea3a4ec29?sk=f8d808d5f414154fdb811e4137011437) - Dec 31, 2021
 - [Calling Rust from Python using PyO3](https://saidvandeklundert.net/learn/2021-11-18-calling-rust-from-python-using-pyo3/) - Nov 18, 2021
 - [davidhewitt's 2021 talk at Rust Manchester meetup](https://www.youtube.com/watch?v=-XyWG_klSAw&t=320s) - Aug 19, 2021
 - [Incrementally porting a small Python project to Rust](https://blog.waleedkhan.name/port-python-to-rust/) - Apr 29, 2021

benches/bench_call.rs

@@ -4,19 +4,13 @@ use pyo3::prelude::*;
 
 macro_rules! test_module {
     ($py:ident, $code:literal) => {
-        PyModule::from_code($py, indoc::indoc!($code), file!(), "test_module")
+        PyModule::from_code($py, $code, file!(), "test_module").expect("module creation failed")
-            .expect("module creation failed")
     };
 }
 
 fn bench_call_0(b: &mut Bencher) {
     Python::with_gil(|py| {
-        let module = test_module!(
+        let module = test_module!(py, "def foo(): pass");
-            py,
-            r#"
-            def foo(): pass
-        "#
-        );
 
         let foo_module = module.getattr("foo").unwrap();
 
@@ -32,10 +26,11 @@ fn bench_call_method_0(b: &mut Bencher) {
     Python::with_gil(|py| {
         let module = test_module!(
             py,
-            r#"
+            "
-            class Foo:
+class Foo:
-                def foo(self): pass
+    def foo(self):
-        "#
+        pass
+"
         );
 
         let foo_module = module.getattr("Foo").unwrap().call0().unwrap();

benches/bench_frompyobject.rs

@@ -0,0 +1,26 @@
+use criterion::{criterion_group, criterion_main, Bencher, Criterion};
+
+use pyo3::{prelude::*, types::PyString};
+
+#[derive(FromPyObject)]
+enum ManyTypes {
+    Int(i32),
+    Bytes(Vec<u8>),
+    String(String),
+}
+
+fn enum_from_pyobject(b: &mut Bencher) {
+    Python::with_gil(|py| {
+        let obj = PyString::new(py, "hello world");
+        b.iter(|| {
+            let _: ManyTypes = obj.extract().unwrap();
+        });
+    })
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    c.bench_function("enum_from_pyobject", enum_from_pyobject);
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

benches/bench_pyclass.rs

@@ -1,5 +1,4 @@
-use criterion::{criterion_group, criterion_main, Bencher, Criterion};
+use criterion::{criterion_group, criterion_main, Criterion};
-
 use pyo3::{class::PyObjectProtocol, prelude::*, type_object::LazyStaticType};
 
 /// This is a feature-rich class instance used to benchmark various parts of the pyclass lifecycle.
@@ -30,7 +29,7 @@ impl PyObjectProtocol for MyClass {
     }
 }
 
-fn first_time_init(b: &mut Bencher) {
+pub fn first_time_init(b: &mut criterion::Bencher) {
     let gil = Python::acquire_gil();
     let py = gil.python();
     b.iter(|| {
@@ -46,4 +45,5 @@ fn criterion_benchmark(c: &mut Criterion) {
 }
 
 criterion_group!(benches, criterion_benchmark);
+
 criterion_main!(benches);

build.rs

@@ -1,48 +1,7 @@
 use std::{env, process::Command};
 
-use pyo3_build_config::{
+use pyo3_build_config::pyo3_build_script_impl::{cargo_env_var, errors::Result};
-    bail, ensure,
+use pyo3_build_config::{bail, InterpreterConfig};
-    pyo3_build_script_impl::{
-        cargo_env_var, env_var, errors::Result, resolve_interpreter_config, InterpreterConfig,
-        PythonVersion,
-    },
-};
-
-/// Minimum Python version PyO3 supports.
-const MINIMUM_SUPPORTED_VERSION: PythonVersion = PythonVersion { major: 3, minor: 6 };
-
-fn ensure_python_version(interpreter_config: &InterpreterConfig) -> Result<()> {
-    ensure!(
-        interpreter_config.version >= MINIMUM_SUPPORTED_VERSION,
-        "the configured Python interpreter version ({}) is lower than PyO3's minimum supported version ({})",
-        interpreter_config.version,
-        MINIMUM_SUPPORTED_VERSION,
-    );
-
-    Ok(())
-}
-
-fn ensure_target_pointer_width(interpreter_config: &InterpreterConfig) -> Result<()> {
-    if let Some(pointer_width) = interpreter_config.pointer_width {
-        // Try to check whether the target architecture matches the python library
-        let rust_target = match cargo_env_var("CARGO_CFG_TARGET_POINTER_WIDTH")
-            .unwrap()
-            .as_str()
-        {
-            "64" => 64,
-            "32" => 32,
-            x => bail!("unexpected Rust target pointer width: {}", x),
-        };
-
-        ensure!(
-            rust_target == pointer_width,
-            "your Rust target architecture ({}-bit) does not match your python interpreter ({}-bit)",
-            rust_target,
-            pointer_width
-        );
-    }
-    Ok(())
-}
 
 fn ensure_auto_initialize_ok(interpreter_config: &InterpreterConfig) -> Result<()> {
     if cargo_env_var("CARGO_FEATURE_AUTO_INITIALIZE").is_some() {
@@ -84,36 +43,6 @@ fn rustc_minor_version() -> Option<u32> {
     pieces.next()?.parse().ok()
 }
 
-fn emit_link_config(interpreter_config: &InterpreterConfig) -> Result<()> {
-    let target_os = cargo_env_var("CARGO_CFG_TARGET_OS").unwrap();
-    let is_extension_module = cargo_env_var("CARGO_FEATURE_EXTENSION_MODULE").is_some();
-    if target_os == "windows" || target_os == "android" || !is_extension_module {
-        // windows and android - always link
-        // other systems - only link if not extension module
-        println!(
-            "cargo:rustc-link-lib={link_model}{alias}{lib_name}",
-            link_model = if interpreter_config.shared {
-                ""
-            } else {
-                "static="
-            },
-            alias = if target_os == "windows" {
-                "pythonXY:"
-            } else {
-                ""
-            },
-            lib_name = interpreter_config.lib_name.as_ref().ok_or(
-                "attempted to link to Python shared library but config does not contain lib_name"
-            )?,
-        );
-        if let Some(lib_dir) = &interpreter_config.lib_dir {
-            println!("cargo:rustc-link-search=native={}", lib_dir);
-        }
-    }
-
-    Ok(())
-}
-
 /// Prepares the PyO3 crate for compilation.
 ///
 /// This loads the config from pyo3-build-config and then makes some additional checks to improve UX
@@ -122,28 +51,12 @@ fn emit_link_config(interpreter_config: &InterpreterConfig) -> Result<()> {
 /// Emits the cargo configuration based on this config as well as a few checks of the Rust compiler
 /// version to enable features which aren't supported on MSRV.
 fn configure_pyo3() -> Result<()> {
-    let interpreter_config = resolve_interpreter_config()?;
+    let interpreter_config = pyo3_build_config::get();
-
-    if env_var("PYO3_PRINT_CONFIG").map_or(false, |os_str| os_str == "1") {
-        print_config_and_exit(&interpreter_config);
-    }
-
-    ensure_python_version(&interpreter_config)?;
-    ensure_target_pointer_width(&interpreter_config)?;
-    ensure_auto_initialize_ok(&interpreter_config)?;
-
-    if !interpreter_config.suppress_build_script_link_lines {
-        emit_link_config(&interpreter_config)?;
-    }
 
     interpreter_config.emit_pyo3_cfgs();
 
     let rustc_minor_version = rustc_minor_version().unwrap_or(0);
-
+    ensure_auto_initialize_ok(interpreter_config)?;
-    // Enable use of #[track_caller] on Rust 1.46 and greater
-    if rustc_minor_version >= 46 {
-        println!("cargo:rustc-cfg=track_caller");
-    }
 
     // Enable use of const generics on Rust 1.51 and greater
     if rustc_minor_version >= 51 {
@@ -155,22 +68,9 @@ fn configure_pyo3() -> Result<()> {
         println!("cargo:rustc-cfg=addr_of");
     }
 
-    // Extra lines come last, to support last write wins.
-    for line in &interpreter_config.extra_build_script_lines {
-        println!("{}", line);
-    }
-
     Ok(())
 }
 
-fn print_config_and_exit(config: &InterpreterConfig) {
-    println!("\n-- PYO3_PRINT_CONFIG=1 is set, printing configuration and halting compile --");
-    config
-        .to_writer(&mut std::io::stdout())
-        .expect("failed to print config to stdout");
-    std::process::exit(101);
-}
-
 fn main() {
     if let Err(e) = configure_pyo3() {
         eprintln!("error: {}", e.report());

codecov.yml

@@ -10,3 +10,4 @@ coverage:
 
 ignore:
   - tests/*.rs
+  - src/test_hygiene/*.rs

examples/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "examples"
+version = "0.0.0"
+publish = false
+edition = "2018"
+
+[dev-dependencies]
+pyo3 = { version = "0.15.1", path = "..", features = ["auto-initialize", "extension-module"] }
+
+[[example]]
+name = "decorator"
+path = "decorator/src/lib.rs"
+crate_type = ["cdylib"]

examples/README.md

@@ -1,13 +1,23 @@
 # PyO3 Examples
 
-These examples are a collection of toy extension modules built with PyO3. They are all tested using `tox` in PyO3's CI.
+These example crates are a collection of toy extension modules built with PyO3. They are all tested using `nox` in PyO3's CI.
 
 Below is a brief description of each of these:
 
 | Example | Description |
 | ------- | ----------- |
+| `decorator` | A project showcasing the example from the [Emulating callable objects](https://pyo3.rs/latest/class/call.html) chapter of the guide. |
 | `maturin-starter` | A template project which is configured to use [`maturin`](https://github.com/PyO3/maturin) for development. |
 | `setuptools-rust-starter` | A template project which is configured to use [`setuptools_rust`](https://github.com/PyO3/setuptools-rust/) for development. |
 | `word-count` | A quick performance comparison between word counter implementations written in each of Rust and Python. |
-| `pyo3-benchmarks` | A project containing some benchmarks of PyO3 functionality called from Python. |
+
-| `pyo3-pytests` | A project containing some tests of PyO3 functionality called from Python. |
+## Creating new projects from these examples
+
+To copy an example, use [`cargo-generate`](https://crates.io/crates/cargo-generate). Follow the commands below, replacing `<example>` with the example to start from:
+
+```bash
+$ cargo install cargo-generate
+$ cargo generate --git https://github.com/PyO3/pyo3 examples/<example>
+```
+
+(`cargo generate` will take a little while to clone the PyO3 repo first; be patient when waiting for the command to run.)

examples/decorator/.template/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+authors = ["{{authors}}"]
+name = "{{project-name}}"
+version = "0.1.0"
+edition = "2018"
+
+[lib]
+name = "decorator"
+crate-type = ["cdylib"]
+
+[dependencies]
+pyo3 = { version = "{{PYO3_VERSION}}", features = ["extension-module"] }

examples/decorator/.template/pre-script.rhai

@@ -0,0 +1,5 @@
+variable::set("PYO3_VERSION", "0.15.1");
+file::rename(".template/Cargo.toml", "Cargo.toml");
+file::rename(".template/pyproject.toml", "pyproject.toml");
+file::rename(".template/tox.ini", "tox.ini");
+file::delete(".template");

examples/decorator/.template/pyproject.toml

@@ -0,0 +1,7 @@
+[build-system]
+requires = ["maturin>=0.12,<0.13"]
+build-backend = "maturin"
+
+[project]
+name = "{{project-name}}"
+version = "0.1.0"

examples/decorator/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "decorator"
+version = "0.1.0"
+edition = "2018"
+
+[lib]
+name = "decorator"
+crate-type = ["cdylib"]
+
+[dependencies]
+pyo3 = { path = "../../", features = ["extension-module"] }
+
+[workspace]

examples/decorator/README.md

@@ -0,0 +1,36 @@
+# decorator
+
+A project showcasing the example from the [Emulating callable objects](https://pyo3.rs/latest/class/call.html) chapter of the guide.
+
+## Building and Testing
+
+To build this package, first install `maturin`:
+
+```shell
+pip install maturin
+```
+
+To build and test use `maturin develop`:
+
+```shell
+pip install -r requirements-dev.txt
+maturin develop
+pytest
+```
+
+Alternatively, install nox and run the tests inside an isolated environment:
+
+```shell
+nox
+```
+
+## Copying this example
+
+Use [`cargo-generate`](https://crates.io/crates/cargo-generate):
+
+```bash
+$ cargo install cargo-generate
+$ cargo generate --git https://github.com/PyO3/pyo3 examples/decorator
+```
+
+(`cargo generate` will take a little while to clone the PyO3 repo first; be patient when waiting for the command to run.)

examples/decorator/cargo-generate.toml

@@ -0,0 +1,5 @@
+[template]
+ignore = [".nox"]
+
+[hooks]
+pre = [".template/pre-script.rhai"]

examples/decorator/noxfile.py

@@ -0,0 +1,9 @@
+import nox
+
+
+@nox.session
+def python(session):
+    session.install("-rrequirements-dev.txt")
+    session.install("maturin")
+    session.run_always("maturin", "develop")
+    session.run("pytest")

examples/decorator/pyproject.toml

@@ -0,0 +1,16 @@
+[build-system]
+requires = ["maturin>=0.12,<0.13"]
+build-backend = "maturin"
+
+[project]
+name = "decorator"
+version = "0.1.0"
+classifier=[
+    "License :: OSI Approved :: MIT License",
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python",
+    "Programming Language :: Rust",
+    "Operating System :: POSIX",
+    "Operating System :: MacOS :: MacOS X",
+]

examples/decorator/requirements-dev.txt

@@ -0,0 +1,3 @@
+pytest>=3.5.0
+pip>=21.3
+maturin>=0.12,<0.13

examples/decorator/src/lib.rs

@@ -0,0 +1,54 @@
+use pyo3::prelude::*;
+use pyo3::types::{PyDict, PyTuple};
+
+/// A function decorator that keeps track how often it is called.
+///
+/// It otherwise doesn't do anything special.
+#[pyclass(name = "Counter")]
+pub struct PyCounter {
+    // We use `#[pyo3(get)]` so that python can read the count but not mutate it.
+    #[pyo3(get)]
+    count: u64,
+
+    // This is the actual function being wrapped.
+    wraps: Py<PyAny>,
+}
+
+#[pymethods]
+impl PyCounter {
+    // Note that we don't validate whether `wraps` is actually callable.
+    //
+    // While we could use `PyAny::is_callable` for that, it has some flaws:
+    //    1. It doesn't guarantee the object can actually be called successfully
+    //    2. We still need to handle any exceptions that the function might raise
+    #[new]
+    fn __new__(wraps: Py<PyAny>) -> Self {
+        PyCounter { count: 0, wraps }
+    }
+
+    #[args(args = "*", kwargs = "**")]
+    fn __call__(
+        &mut self,
+        py: Python,
+        args: &PyTuple,
+        kwargs: Option<&PyDict>,
+    ) -> PyResult<Py<PyAny>> {
+        self.count += 1;
+        let name = self.wraps.getattr(py, "__name__")?;
+
+        println!("{} has been called {} time(s).", name, self.count);
+
+        // After doing something, we finally forward the call to the wrapped function
+        let ret = self.wraps.call(py, args, kwargs)?;
+
+        // We could do something with the return value of
+        // the function before returning it
+        Ok(ret)
+    }
+}
+
+#[pymodule]
+pub fn decorator(_py: Python, module: &PyModule) -> PyResult<()> {
+    module.add_class::<PyCounter>()?;
+    Ok(())
+}

examples/decorator/tests/example.py

@@ -0,0 +1,11 @@
+@Counter
+def say_hello():
+    print("hello")
+
+
+say_hello()
+say_hello()
+say_hello()
+say_hello()
+
+assert say_hello.count == 4

examples/decorator/tests/test_.py

@@ -0,0 +1,40 @@
+from decorator import Counter
+
+
+def test_no_args():
+    @Counter
+    def say_hello():
+        print("hello")
+
+    say_hello()
+    say_hello()
+    say_hello()
+    say_hello()
+
+    assert say_hello.count == 4
+
+
+def test_arg():
+    @Counter
+    def say_hello(name):
+        print(f"hello {name}")
+
+    say_hello("a")
+    say_hello("b")
+    say_hello("c")
+    say_hello("d")
+
+    assert say_hello.count == 4
+
+
+def test_default_arg():
+    @Counter
+    def say_hello(name="default"):
+        print(f"hello {name}")
+
+    say_hello("a")
+    say_hello()
+    say_hello("c")
+    say_hello()
+
+    assert say_hello.count == 4

examples/maturin-starter/.template/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+authors = ["{{authors}}"]
+name = "{{project-name}}"
+version = "0.1.0"
+edition = "2018"
+
+[lib]
+name = "maturin_starter"
+crate-type = ["cdylib"]
+
+[dependencies]
+pyo3 = { version = "{{PYO3_VERSION}}", features = ["extension-module"] }

examples/maturin-starter/.template/pre-script.rhai

@@ -0,0 +1,5 @@
+variable::set("PYO3_VERSION", "0.15.1");
+file::rename(".template/Cargo.toml", "Cargo.toml");
+file::rename(".template/pyproject.toml", "pyproject.toml");
+file::rename(".template/tox.ini", "tox.ini");
+file::delete(".template");

examples/maturin-starter/.template/pyproject.toml

@@ -0,0 +1,7 @@
+[build-system]
+requires = ["maturin>=0.12,<0.13"]
+build-backend = "maturin"
+
+[project]
+name = "{{project-name}}"
+version = "0.1.0"

examples/maturin-starter/Cargo.toml

@@ -1,27 +1,13 @@
 [package]
-authors = ["PyO3 Authors"]
 name = "maturin-starter"
 version = "0.1.0"
-description = "An example project to get started using PyO3 with maturin"
 edition = "2018"
 
-[dependencies]
-
-[dependencies.pyo3]
-path = "../../"
-features = ["extension-module"]
-
 [lib]
 name = "maturin_starter"
 crate-type = ["cdylib"]
 
-[package.metadata.maturin]
+[dependencies]
-classifier=[
+pyo3 = { path = "../../", features = ["extension-module"] }
-    "License :: OSI Approved :: MIT License",
+
-    "Development Status :: 3 - Alpha",
+[workspace]
-    "Intended Audience :: Developers",
-    "Programming Language :: Python",
-    "Programming Language :: Rust",
-    "Operating System :: POSIX",
-    "Operating System :: MacOS :: MacOS X",
-]

examples/maturin-starter/README.md

@@ -17,8 +17,19 @@ pip install -r requirements-dev.txt
 maturin develop && pytest
 ```
 
-Alternatively, install tox and run the tests inside an isolated environment:
+Alternatively, install nox and run the tests inside an isolated environment:
 
 ```shell
-tox -e py
+nox
 ```
+
+## Copying this example
+
+Use [`cargo-generate`](https://crates.io/crates/cargo-generate):
+
+```bash
+$ cargo install cargo-generate
+$ cargo generate --git https://github.com/PyO3/pyo3 examples/maturin-starter
+```
+
+(`cargo generate` will take a little while to clone the PyO3 repo first; be patient when waiting for the command to run.)

examples/maturin-starter/cargo-generate.toml

@@ -0,0 +1,5 @@
+[template]
+ignore = [".nox"]
+
+[hooks]
+pre = [".template/pre-script.rhai"]

examples/maturin-starter/noxfile.py

@@ -0,0 +1,9 @@
+import nox
+
+
+@nox.session
+def python(session):
+    session.install("-rrequirements-dev.txt")
+    session.install("maturin")
+    session.run_always("maturin", "develop")
+    session.run("pytest")

examples/maturin-starter/pyproject.toml

@@ -1,3 +1,16 @@
 [build-system]
-requires = ["maturin>=0.10,<0.11"]
+requires = ["maturin>=0.12,<0.13"]
 build-backend = "maturin"
+
+[project]
+name = "maturin-starter"
+version = "0.1.0"
+classifier=[
+    "License :: OSI Approved :: MIT License",
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python",
+    "Programming Language :: Rust",
+    "Operating System :: POSIX",
+    "Operating System :: MacOS :: MacOS X",
+]

examples/maturin-starter/requirements-dev.txt

@@ -1,2 +1,3 @@
 pytest>=3.5.0
 pip>=21.3
+maturin>=0.12,<0.13

examples/maturin-starter/tox.ini

@@ -1,10 +0,0 @@
-[tox]
-# can't install from sdist because local pyo3 repo can't be included in the sdist
-skipsdist = true
-
-[testenv]
-description = Run the unit tests under {basepython}
-deps = -rrequirements-dev.txt
-commands =
-    python -m pip install .
-    pytest {posargs}

examples/pyo3-benchmarks/Cargo.toml

@@ -1,16 +0,0 @@
-[package]
-authors = ["PyO3 Authors"]
-name = "pyo3-benchmarks"
-version = "0.1.0"
-description = "Python-based benchmarks for various PyO3 functionality"
-edition = "2018"
-
-[dependencies]
-
-[dependencies.pyo3]
-path = "../../"
-features = ["extension-module"]
-
-[lib]
-name = "_pyo3_benchmarks"
-crate-type = ["cdylib"]

examples/pyo3-benchmarks/MANIFEST.in

@@ -1,3 +0,0 @@
-include Cargo.toml
-recursive-include src *
-recursive-include tests

examples/pyo3-benchmarks/README.md

@@ -1,18 +0,0 @@
-# pyo3-benchmarks
-
-This extension module contains benchmarks for pieces of PyO3's API accessible from Python.
-
-## Running the benchmarks
-
-You can install the module in your Python environment and then run the benchmarks with pytest:
-
-```shell
-python setup.py develop
-pytest
-```
-
-Or with tox:
-
-```shell
-tox -e py
-```

examples/pyo3-benchmarks/pyo3_benchmarks/__init__.py

@@ -1 +0,0 @@
-from ._pyo3_benchmarks import *

examples/pyo3-benchmarks/requirements-dev.txt

@@ -1,4 +0,0 @@
-pytest>=3.5.0
-setuptools_rust~=1.0.0
-pytest-benchmark~=3.2
-pip>=21.3

examples/pyo3-benchmarks/setup.py

@@ -1,26 +0,0 @@
-from setuptools import setup
-from setuptools_rust import RustExtension
-
-
-setup(
-    name="pyo3-benchmarks",
-    version="0.1.0",
-    classifiers=[
-        "License :: OSI Approved :: MIT License",
-        "Development Status :: 3 - Alpha",
-        "Intended Audience :: Developers",
-        "Programming Language :: Python",
-        "Programming Language :: Rust",
-        "Operating System :: POSIX",
-        "Operating System :: MacOS :: MacOS X",
-    ],
-    packages=["pyo3_benchmarks"],
-    rust_extensions=[
-        RustExtension(
-            "pyo3_benchmarks._pyo3_benchmarks",
-            debug=False,
-        ),
-    ],
-    include_package_data=True,
-    zip_safe=False,
-)

examples/pyo3-benchmarks/tox.ini

@@ -1,5 +0,0 @@
-[testenv]
-usedevelop = True
-description = Run the unit tests under {basepython}
-deps = -rrequirements-dev.txt
-commands = pytest --benchmark-sort=name {posargs}

examples/pyo3-pytests/README.md

@@ -1,19 +0,0 @@
-# pyo3-pytests
-
-An extension module built using PyO3, used to test PyO3 from Python.
-
-## Testing
-
-This package is intended to be built using `maturin`. Once built, you can run the tests using `pytest`:
-
-```shell
-pip install maturin
-maturin develop
-pytest
-```
-
-Alternatively, install tox and run the tests inside an isolated environment:
-
-```shell
-tox -e py
-```

examples/pyo3-pytests/build.rs

@@ -1,3 +0,0 @@
-fn main() {
-    pyo3_build_config::use_pyo3_cfgs();
-}

examples/pyo3-pytests/pyproject.toml

@@ -1,3 +0,0 @@
-[build-system]
-requires = ["maturin>=0.10,<0.11"]
-build-backend = "maturin"

examples/pyo3-pytests/requirements-dev.txt

@@ -1,4 +0,0 @@
-hypothesis>=3.55
-pytest>=3.5.0
-psutil>=5.6
-pip>=21.3

examples/pyo3-pytests/src/pyclass_iter.rs

@@ -1,34 +0,0 @@
-use pyo3::class::iter::{IterNextOutput, PyIterProtocol};
-use pyo3::prelude::*;
-
-/// This is for demonstrating how to return a value from __next__
-#[pyclass]
-struct PyClassIter {
-    count: usize,
-}
-
-#[pymethods]
-impl PyClassIter {
-    #[new]
-    pub fn new() -> Self {
-        PyClassIter { count: 0 }
-    }
-}
-
-#[pyproto]
-impl PyIterProtocol for PyClassIter {
-    fn __next__(mut slf: PyRefMut<Self>) -> IterNextOutput<usize, &'static str> {
-        if slf.count < 5 {
-            slf.count += 1;
-            IterNextOutput::Yield(slf.count)
-        } else {
-            IterNextOutput::Return("Ended")
-        }
-    }
-}
-
-#[pymodule]
-pub fn pyclass_iter(_py: Python, m: &PyModule) -> PyResult<()> {
-    m.add_class::<PyClassIter>()?;
-    Ok(())
-}

examples/pyo3-pytests/tox.ini

@@ -1,10 +0,0 @@
-[tox]
-# can't install from sdist because local pyo3 repo can't be included in the sdist
-skipsdist = true
-
-[testenv]
-description = Run the unit tests under {basepython}
-deps = -rrequirements-dev.txt
-commands =
-    python -m pip install .
-    pytest {posargs}

examples/setuptools-rust-starter/.template/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+authors = ["{{authors}}"]
+name = "{{project-name}}"
+version = "0.1.0"
+edition = "2018"
+
+[lib]
+name = "setuptools_rust_starter"
+crate-type = ["cdylib"]
+
+[dependencies]
+pyo3 = { version = "{{PYO3_VERSION}}", features = ["extension-module"] }

examples/setuptools-rust-starter/.template/pre-script.rhai

@@ -0,0 +1,5 @@
+variable::set("PYO3_VERSION", "0.15.1");
+file::rename(".template/Cargo.toml", "Cargo.toml");
+file::rename(".template/setup.cfg", "setup.cfg");
+file::rename(".template/tox.ini", "tox.ini");
+file::delete(".template");

examples/setuptools-rust-starter/.template/setup.cfg

@@ -0,0 +1,9 @@
+[metadata]
+name = {{project-name}}
+version = 0.1.0
+packages =
+    setuptools_rust_starter
+
+[options]
+include_package_data = True
+zip_safe = False

examples/setuptools-rust-starter/Cargo.toml

@@ -1,27 +1,13 @@
 [package]
-authors = ["PyO3 Authors"]
 name = "setuptools-rust-starter"
 version = "0.1.0"
-description = "An example project to get started using PyO3 with maturin"
 edition = "2018"
 
-[dependencies]
-
-[dependencies.pyo3]
-path = "../../"
-features = ["extension-module"]
-
 [lib]
 name = "setuptools_rust_starter"
 crate-type = ["cdylib"]
 
-[package.metadata.maturin]
+[dependencies]
-classifier=[
+pyo3 = { path = "../../", features = ["extension-module"] }
-    "License :: OSI Approved :: MIT License",
+
-    "Development Status :: 3 - Alpha",
+[workspace]
-    "Intended Audience :: Developers",
-    "Programming Language :: Python",
-    "Programming Language :: Rust",
-    "Operating System :: POSIX",
-    "Operating System :: MacOS :: MacOS X",
-]

examples/setuptools-rust-starter/README.md

@@ -17,8 +17,19 @@ pip install -r requirements-dev.txt
 python setup.py develop && pytest
 ```
 
-Alternatively, install tox and run the tests inside an isolated environment:
+Alternatively, install nox and run the tests inside an isolated environment:
 
 ```shell
-tox -e py
+nox
 ```
+
+## Copying this example
+
+Use [`cargo-generate`](https://crates.io/crates/cargo-generate):
+
+```bash
+$ cargo install cargo-generate
+$ cargo generate --git https://github.com/PyO3/pyo3 examples/setuptools-rust-starter
+```
+
+(`cargo generate` will take a little while to clone the PyO3 repo first; be patient when waiting for the command to run.)

examples/setuptools-rust-starter/cargo-generate.toml

@@ -0,0 +1,5 @@
+[template]
+ignore = [".nox"]
+
+[hooks]
+pre = [".template/pre-script.rhai"]

examples/setuptools-rust-starter/noxfile.py

@@ -0,0 +1,10 @@
+import nox
+
+
+@nox.session
+def python(session):
+    session.install("-rrequirements-dev.txt")
+    session.run_always(
+        "pip", "install", "-e", ".", "--no-build-isolation", env={"BUILD_DEBUG": "1"}
+    )
+    session.run("pytest")

examples/setuptools-rust-starter/pyproject.toml

@@ -0,0 +1,2 @@
+[build-system]
+requires = ["setuptools>=41.0.0", "wheel", "setuptools_rust>=1.0.0"]

examples/setuptools-rust-starter/setup.cfg

@@ -0,0 +1,17 @@
+[metadata]
+name = setuptools-rust-starter
+version = 0.1.0
+classifiers =
+    License :: OSI Approved :: MIT License
+    Development Status :: 3 - Alpha
+    Intended Audience :: Developers
+    Programming Language :: Python
+    Programming Language :: Rust
+    Operating System :: POSIX
+    Operating System :: MacOS :: MacOS X
+
+[options]
+packages =
+    setuptools_rust_starter
+include_package_data = True
+zip_safe = False

examples/setuptools-rust-starter/setup.py

@@ -1,26 +1,13 @@
+import os
+
 from setuptools import setup
 from setuptools_rust import RustExtension
 
-
 setup(
-    name="setuptools-rust-starter",
-    version="0.1.0",
-    classifiers=[
-        "License :: OSI Approved :: MIT License",
-        "Development Status :: 3 - Alpha",
-        "Intended Audience :: Developers",
-        "Programming Language :: Python",
-        "Programming Language :: Rust",
-        "Operating System :: POSIX",
-        "Operating System :: MacOS :: MacOS X",
-    ],
-    packages=["setuptools_rust_starter"],
     rust_extensions=[
         RustExtension(
             "setuptools_rust_starter._setuptools_rust_starter",
-            debug=False,
+            debug=os.environ.get("BUILD_DEBUG") == "1",
-        ),
+        )
     ],
-    include_package_data=True,
-    zip_safe=False,
 )

examples/setuptools-rust-starter/tox.ini

@@ -1,5 +0,0 @@
-[testenv]
-usedevelop = true
-description = Run the unit tests under {basepython}
-deps = -rrequirements-dev.txt
-commands = pytest {posargs}

examples/word-count/.template/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+authors = ["{{authors}}"]
+name = "{{project-name}}"
+version = "0.1.0"
+edition = "2018"
+
+[lib]
+name = "word_count"
+crate-type = ["cdylib"]
+
+[dependencies]
+pyo3 = { version = "{{PYO3_VERSION}}", features = ["extension-module"] }
+rayon = "1.0.2"

examples/word-count/.template/pre-script.rhai

@@ -0,0 +1,4 @@
+variable::set("PYO3_VERSION", "0.15.1");
+file::rename(".template/Cargo.toml", "Cargo.toml");
+file::rename(".template/tox.ini", "tox.ini");
+file::delete(".template");

examples/word-count/.template/pyproject.toml

@@ -0,0 +1,9 @@
+[build-system]
+requires = ["setuptools>=41.0.0", "wheel", "setuptools_rust>=1.0.0"]
+
+[project]
+name = "{{project-name}}"
+version = "0.1.0"
+
+[tool.pytest.ini_options]
+addopts = "--benchmark-disable"

examples/word-count/.template/setup.cfg

@@ -0,0 +1,9 @@
+[metadata]
+name = {{project-name}}
+version = 0.1.0
+packages =
+    word_count
+
+[options]
+include_package_data = True
+zip_safe = False

examples/word-count/Cargo.toml

@@ -1,13 +1,14 @@
 [package]
-authors = ["Messense Lv <messense@icloud.com>"]
 name = "word-count"
 version = "0.1.0"
 edition = "2018"
 
-[dependencies]
-rayon = "1.0.2"
-pyo3 = { path = "../..", features = ["extension-module"] }
-
 [lib]
 name = "word_count"
 crate-type = ["cdylib"]
+
+[dependencies]
+pyo3 = { path = "../..", features = ["extension-module"] }
+rayon = "1.0.2"
+
+[workspace]

examples/word-count/README.md

@@ -5,7 +5,7 @@ Demonstrates searching for a file in plain python, with rust singlethreaded and
 ## Build
 
 ```shell
-python setup.py install
+pip install .
 ```
 
 ## Usage
@@ -18,25 +18,29 @@ search("foo bar", "foo")
 search_sequential("foo bar", "foo")
 ```
 
-## Benchmark
-
-Install the depedencies:
-
-```shell
-pip install -r requirements-dev.txt
-```
-
-
-There is a benchmark in `tests/test_word_count.py`:
-
-```shell
-pytest -v tests
-```
-
 ## Testing
 
-To test install tox globally and run
+To test install nox globally and run
 
 ```shell
-tox -e py
+nox
 ```
+
+## Benchmark
+
+To test install nox globally and run
+
+```shell
+nox -s bench
+```
+
+## Copying this example
+
+Use [`cargo-generate`](https://crates.io/crates/cargo-generate):
+
+```bash
+$ cargo install cargo-generate
+$ cargo generate --git https://github.com/PyO3/pyo3 examples/word-count
+```
+
+(`cargo generate` will take a little while to clone the PyO3 repo first; be patient when waiting for the command to run.)

examples/word-count/cargo-generate.toml

@@ -0,0 +1,5 @@
+[template]
+ignore = [".nox"]
+
+[hooks]
+pre = [".template/pre-script.rhai"]

examples/word-count/noxfile.py

@@ -0,0 +1,18 @@
+import nox
+
+nox.options.sessions = ["test"]
+
+
+@nox.session
+def test(session):
+    session.install("-rrequirements-dev.txt")
+    session.install("maturin")
+    session.run_always("maturin", "develop")
+    session.run("pytest")
+
+
+@nox.session
+def bench(session):
+    session.install("-rrequirements-dev.txt")
+    session.install(".")
+    session.run("pytest", "--benchmark-enable")

examples/word-count/pyproject.toml

@@ -1,2 +1,20 @@
 [build-system]
-requires = ["setuptools>=41.0.0", "wheel", "setuptools_rust>=0.10.2"]
+requires = ["maturin>=0.12,<0.13"]
+build-backend = "maturin"
+
+[project]
+name = "word-count"
+version = "0.1.0"
+classifier=[
+    "License :: OSI Approved :: MIT License",
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python",
+    "Programming Language :: Rust",
+    "Operating System :: POSIX",
+    "Operating System :: MacOS :: MacOS X",
+]
+
+
+[tool.pytest.ini_options]
+addopts = "--benchmark-disable"

examples/word-count/requirements-dev.txt

@@ -1,3 +1,2 @@
 pytest>=3.5.0
-setuptools_rust~=1.0.0
 pytest-benchmark>=3.1.1

examples/word-count/setup.py

@@ -1,21 +0,0 @@
-from setuptools import setup
-from setuptools_rust import RustExtension
-
-
-setup(
-    name="word-count",
-    version="0.1.0",
-    classifiers=[
-        "License :: OSI Approved :: MIT License",
-        "Development Status :: 3 - Alpha",
-        "Intended Audience :: Developers",
-        "Programming Language :: Python",
-        "Programming Language :: Rust",
-        "Operating System :: POSIX",
-        "Operating System :: MacOS :: MacOS X",
-    ],
-    packages=["word_count"],
-    rust_extensions=[RustExtension("word_count.word_count", "Cargo.toml", debug=False)],
-    include_package_data=True,
-    zip_safe=False,
-)

examples/word-count/tox.ini

@@ -1,5 +0,0 @@
-[testenv]
-usedevelop = true
-description = Run the unit tests under {basepython}
-deps = -rrequirements-dev.txt
-commands = pytest {posargs}

guide/src/SUMMARY.md

@@ -8,6 +8,7 @@
 - [Python Functions](function.md)
 - [Python Classes](class.md)
   - [Class customizations](class/protocols.md)
+    - [Emulating callable objects](class/call.md)
 - [Type Conversions](conversions.md)
   - [Mapping of Rust types to Python types](conversions/tables.md)]
   - [Conversion traits](conversions/traits.md)]

guide/src/building_and_distribution.md

@@ -163,7 +163,7 @@ not work when compiling for `abi3`. These are:
 
 - `#[pyo3(text_signature = "...")]` does not work on classes until Python 3.10 or greater.
 - The `dict` and `weakref` options on classes are not supported until Python 3.9 or greater.
-- The buffer API is not supported.
+- The buffer API is not supported until Python 3.11 or greater.
 - Optimizations which rely on knowledge of the exact Python version compiled against.
 
 ## Embedding Python in Rust

guide/src/class.md

@@ -2,7 +2,7 @@
 
 PyO3 exposes a group of attributes powered by Rust's proc macro system for defining Python classes as Rust structs.
 
-The main attribute is `#[pyclass]`, which is placed upon a Rust `struct` to generate a Python type for it. A struct will usually also have *one* `#[pymethods]`-annotated `impl` block for the struct, which is used to define Python methods and constants for the generated Python type. (If the [`multiple-pymethods`] feature is enabled each `#[pyclass]` is allowed to have multiple `#[pymethods]` blocks.) Finally, there may be multiple `#[pyproto]` trait implementations for the struct, which are used to define certain python magic methods such as `__str__`.
+The main attribute is `#[pyclass]`, which is placed upon a Rust `struct` or a fieldless `enum` (a.k.a. C-like enum) to generate a Python type for it. They will usually also have *one* `#[pymethods]`-annotated `impl` block for the struct, which is used to define Python methods and constants for the generated Python type. (If the [`multiple-pymethods`] feature is enabled each `#[pyclass]` is allowed to have multiple `#[pymethods]` blocks.) Finally, there may be multiple `#[pyproto]` trait implementations for the struct, which are used to define certain python magic methods such as `__str__`.
 
 This chapter will discuss the functionality and configuration these attributes offer. Below is a list of links to the relevant section of this chapter for each:
 
@@ -20,9 +20,7 @@ This chapter will discuss the functionality and configuration these attributes o
 
 ## Defining a new class
 
-To define a custom Python class, a Rust struct needs to be annotated with the
+To define a custom Python class, add the `#[pyclass]` attribute to a Rust struct or a fieldless enum.
-`#[pyclass]` attribute.
-
 ```rust
 # #![allow(dead_code)]
 # use pyo3::prelude::*;
@@ -31,11 +29,17 @@ struct MyClass {
     # #[pyo3(get)]
     num: i32,
 }
+
+#[pyclass]
+enum MyEnum {
+    Variant,
+    OtherVariant = 30, // PyO3 supports custom discriminants.
+}
 ```
 
-Because Python objects are freely shared between threads by the Python interpreter, all structs annotated with `#[pyclass]` must implement `Send` (unless annotated with [`#[pyclass(unsendable)]`](#customizing-the-class)).
+Because Python objects are freely shared between threads by the Python interpreter, all types annotated with `#[pyclass]` must implement `Send` (unless annotated with [`#[pyclass(unsendable)]`](#customizing-the-class)).
 
-The above example generates implementations for [`PyTypeInfo`], [`PyTypeObject`], and [`PyClass`] for `MyClass`. To see these generated implementations, refer to the [implementation details](#implementation-details) at the end of this chapter.
+The above example generates implementations for [`PyTypeInfo`], [`PyTypeObject`], and [`PyClass`] for `MyClass` and `MyEnum`. To see these generated implementations, refer to the [implementation details](#implementation-details) at the end of this chapter.
 
 ## Adding the class to a module
 
@@ -140,8 +144,8 @@ so that they can benefit from a freelist. `XXX` is a number of items for the fre
 * `gc` - Classes with the `gc` parameter participate in Python garbage collection.
 If a custom class contains references to other Python objects that can be collected, the [`PyGCProtocol`]({{#PYO3_DOCS_URL}}/pyo3/class/gc/trait.PyGCProtocol.html) trait has to be implemented.
 * `weakref` - Adds support for Python weak references.
-* `extends=BaseType` - Use a custom base class. The base `BaseType` must implement `PyTypeInfo`.
+* `extends=BaseType` - Use a custom base class. The base `BaseType` must implement `PyTypeInfo`. `enum` pyclasses can't use a custom base class.
-* `subclass` - Allows Python classes to inherit from this class.
+* `subclass` - Allows Python classes to inherit from this class. `enum` pyclasses can't be inherited from.
 * `dict` - Adds `__dict__` support, so that the instances of this type have a dictionary containing arbitrary instance variables.
 * `unsendable` - Making it safe to expose `!Send` structs to Python, where all object can be accessed
    by multiple threads. A class marked with `unsendable` panics when accessed by another thread.
@@ -351,7 +355,7 @@ impl SubClass {
 ## Object properties
 
 PyO3 supports two ways to add properties to your `#[pyclass]`:
-- For simple fields with no side effects, a `#[pyo3(get, set)]` attribute can be added directly to the field definition in the `#[pyclass]`.
+- For simple struct fields with no side effects, a `#[pyo3(get, set)]` attribute can be added directly to the field definition in the `#[pyclass]`.
 - For properties which require computation you can define `#[getter]` and `#[setter]` functions in the [`#[pymethods]`](#instance-methods) block.
 
 We'll cover each of these in the following sections.
@@ -802,6 +806,118 @@ impl MyClass {
 Note that `text_signature` on classes is not compatible with compilation in
 `abi3` mode until Python 3.10 or greater.
 
+## #[pyclass] enums
+
+Currently PyO3 only supports fieldless enums. PyO3 adds a class attribute for each variant, so you can access them in Python without defining `#[new]`. PyO3 also provides default implementations of `__richcmp__` and `__int__`, so they can be compared using `==`:
+
+```rust
+# use pyo3::prelude::*;
+#[pyclass]
+enum MyEnum {
+    Variant,
+    OtherVariant,
+}
+
+Python::with_gil(|py| {
+    let x = Py::new(py, MyEnum::Variant).unwrap();
+    let y = Py::new(py, MyEnum::OtherVariant).unwrap();
+    let cls = py.get_type::<MyEnum>();
+    pyo3::py_run!(py, x y cls, r#"
+        assert x == cls.Variant
+        assert y == cls.OtherVariant
+        assert x != y
+    "#)
+})
+```
+
+You can also convert your enums into `int`:
+
+```rust
+# use pyo3::prelude::*;
+#[pyclass]
+enum MyEnum {
+    Variant,
+    OtherVariant = 10,
+}
+
+Python::with_gil(|py| {
+    let cls = py.get_type::<MyEnum>();
+    let x = MyEnum::Variant as i32; // The exact value is assigned by the compiler.
+    pyo3::py_run!(py, cls x, r#"
+        assert int(cls.Variant) == x
+        assert int(cls.OtherVariant) == 10
+        assert cls.OtherVariant == 10  # You can also compare against int.
+        assert 10 == cls.OtherVariant
+    "#)
+})
+```
+
+PyO3 also provides `__repr__` for enums:
+
+```rust
+# use pyo3::prelude::*;
+#[pyclass]
+enum MyEnum{
+    Variant,
+    OtherVariant,
+}
+
+Python::with_gil(|py| {
+    let cls = py.get_type::<MyEnum>();
+    let x = Py::new(py, MyEnum::Variant).unwrap();
+    pyo3::py_run!(py, cls x, r#"
+        assert repr(x) == 'MyEnum.Variant'
+        assert repr(cls.OtherVariant) == 'MyEnum.OtherVariant'
+    "#)
+})
+```
+
+All methods defined by PyO3 can be overriden. For example here's how you override `__repr__`:
+
+```rust
+# use pyo3::prelude::*;
+#[pyclass]
+enum MyEnum {
+    Answer = 42,
+}
+
+#[pymethods]
+impl MyEnum {
+    fn __repr__(&self) -> &'static str {
+        "42"
+    }
+}
+
+Python::with_gil(|py| {
+    let cls = py.get_type::<MyEnum>();
+    pyo3::py_run!(py, cls, "assert repr(cls.Answer) == '42'")
+})
+```
+
+You may not use enums as a base class or let enums inherit from other classes.
+
+```rust,compile_fail
+# use pyo3::prelude::*;
+#[pyclass(subclass)]
+enum BadBase{
+    Var1,
+}
+```
+
+```rust,compile_fail
+# use pyo3::prelude::*;
+
+#[pyclass(subclass)]
+struct Base;
+
+#[pyclass(extends=Base)]
+enum BadSubclass{
+    Var1,
+}
+```
+
+`#[pyclass]` enums are currently not interoperable with `IntEnum` in Python.
+
 ## Implementation details
 
 The `#[pyclass]` macros rely on a lot of conditional code generation: each `#[pyclass]` can optionally have a `#[pymethods]` block as well as several different possible `#[pyproto]` trait implementations.
@@ -814,7 +930,92 @@ The `#[pyclass]` macro expands to roughly the code seen below. The `PyClassImplC
 
 ```rust
 # #[cfg(not(feature = "multiple-pymethods"))] {
+# use pyo3::prelude::*;
+// Note: the implementation differs slightly with the `multiple-pymethods` feature enabled.
+struct MyClass {
+    # #[allow(dead_code)]
+    num: i32,
+}
+unsafe impl ::pyo3::type_object::PyTypeInfo for MyClass {
+    type AsRefTarget = ::pyo3::PyCell<Self>;
+    const NAME: &'static str = "MyClass";
+    const MODULE: ::std::option::Option<&'static str> = ::std::option::Option::None;
+    #[inline]
+    fn type_object_raw(py: ::pyo3::Python<'_>) -> *mut ::pyo3::ffi::PyTypeObject {
+        use ::pyo3::type_object::LazyStaticType;
+        static TYPE_OBJECT: LazyStaticType = LazyStaticType::new();
+        TYPE_OBJECT.get_or_init::<Self>(py)
+    }
+}
 
+impl ::pyo3::PyClass for MyClass {
+    type Dict = ::pyo3::impl_::pyclass::PyClassDummySlot;
+    type WeakRef = ::pyo3::impl_::pyclass::PyClassDummySlot;
+    type BaseNativeType = ::pyo3::PyAny;
+}
+
+unsafe impl ::pyo3::pyclass::MutablePyClass for MyClass {}
+
+impl<'a> ::pyo3::derive_utils::ExtractExt<'a> for &'a mut MyClass {
+    type Target = ::pyo3::PyRefMut<'a, MyClass>;
+}
+
+impl<'a> ::pyo3::derive_utils::ExtractExt<'a> for &'a MyClass {
+    type Target = ::pyo3::PyRef<'a, MyClass>;
+}
+
+impl ::pyo3::IntoPy<::pyo3::PyObject> for MyClass {
+    fn into_py(self, py: ::pyo3::Python) -> ::pyo3::PyObject {
+        ::pyo3::IntoPy::into_py(::pyo3::Py::new(py, self).unwrap(), py)
+    }
+}
+
+impl pyo3::impl_::pyclass::PyClassImpl for MyClass {
+    const DOC: &'static str = "Class for demonstration\u{0}";
+    const IS_GC: bool = false;
+    const IS_BASETYPE: bool = false;
+    const IS_SUBCLASS: bool = false;
+    type Layout = PyCell<MyClass>;
+    type BaseType = PyAny;
+    type ThreadChecker = pyo3::impl_::pyclass::ThreadCheckerStub<MyClass>;
+    type Mutabilty = pyo3::pyclass::Mutable;
+
+    fn for_all_items(visitor: &mut dyn FnMut(&pyo3::impl_::pyclass::PyClassItems)) {
+        use pyo3::impl_::pyclass::*;
+        let collector = PyClassImplCollector::<MyClass>::new();
+        static INTRINSIC_ITEMS: PyClassItems = PyClassItems { slots: &[], methods: &[] };
+        visitor(&INTRINSIC_ITEMS);
+        visitor(collector.py_methods());
+    }
+    fn get_new() -> Option<pyo3::ffi::newfunc> {
+        use pyo3::impl_::pyclass::*;
+        let collector = PyClassImplCollector::<Self>::new();
+        collector.new_impl()
+    }
+    fn get_alloc() -> Option<pyo3::ffi::allocfunc> {
+        use pyo3::impl_::pyclass::*;
+        let collector = PyClassImplCollector::<Self>::new();
+        collector.alloc_impl()
+    }
+    fn get_free() -> Option<pyo3::ffi::freefunc> {
+        use pyo3::impl_::pyclass::*;
+        let collector = PyClassImplCollector::<Self>::new();
+        collector.free_impl()
+    }
+}
+
+impl ::pyo3::impl_::pyclass::PyClassDescriptors<MyClass>
+    for ::pyo3::impl_::pyclass::PyClassImplCollector<MyClass>
+{
+    fn py_class_descriptors(self) -> &'static [::pyo3::class::methods::PyMethodDefType] {
+        static METHODS: &[::pyo3::class::methods::PyMethodDefType] = &[];
+        METHODS
+    }
+}
+# Python::with_gil(|py| {
+#     let cls = py.get_type::<MyClass>();
+#     pyo3::py_run!(py, cls, "assert cls.__name__ == 'MyClass'")
+# });
 # }
 ```
 

guide/src/class/call.md

@@ -0,0 +1,67 @@
+# Emulating callable objects
+
+Classes can be callable if they have a `#[pymethod]` named `__call__`.
+This allows instances of a class to behave similar to functions.
+
+This method's signature must look like `__call__(<self>, ...) -> object` - here,
+ any argument list can be defined as for normal pymethods
+
+### Example: Implementing a call counter
+
+The following pyclass is a basic decorator - its constructor takes a Python object
+as argument and calls that object when called. An equivalent Python implementation
+is linked at the end.
+
+An example crate containing this pyclass can be found [here](https://github.com/PyO3/pyo3/tree/main/examples/decorator)
+
+```rust
+{{#include ../../../examples/decorator/src/lib.rs}}
+```
+
+Python code:
+
+```python
+{{#include ../../../examples/decorator/tests/example.py}}
+```
+
+Output:
+
+```text
+say_hello has been called 1 time(s).
+hello
+say_hello has been called 2 time(s).
+hello
+say_hello has been called 3 time(s).
+hello
+say_hello has been called 4 time(s).
+hello
+```
+
+#### Pure Python implementation
+
+A Python implementation of this looks similar to the Rust version:
+
+```python
+class Counter:
+    def __init__(self, wraps):
+        self.count = 0
+        self.wraps = wraps
+
+    def __call__(self, *args, **kwargs):
+        self.count += 1
+        print(f"{self.wraps.__name__} has been called {self.count} time(s)")
+        self.wraps(*args, **kwargs)
+```
+
+Note that it can also be implemented as a higher order function:
+
+```python
+def Counter(wraps):
+    count = 0
+    def call(*args, **kwargs):
+        nonlocal count
+        count += 1
+        print(f"{wraps.__name__} has been called {count} time(s)")
+        return wraps(*args, **kwargs)
+    return call
+```

guide/src/class/protocols.md

@@ -2,14 +2,14 @@
 
 Python's object model defines several protocols for different object behavior, such as the sequence, mapping, and number protocols. You may be familiar with implementing these protocols in Python classes by "magic" methods, such as `__str__` or `__repr__`. Because of the double-underscores surrounding their name, these are also known as "dunder" methods.
 
-In the Python C-API which PyO3 is implemented upon, many of these magic methods have to be placed into special "slots" on the class type object. as already covered in the previous section. There are two ways in which this can be done:
+In the Python C-API which PyO3 is implemented upon, many of these magic methods have to be placed into special "slots" on the class type object, as covered in the previous section. There are two ways in which this can be done:
 
  - [Experimental for PyO3 0.15, may change slightly in PyO3 0.16] In `#[pymethods]`, if the name of the method is a recognised magic method, PyO3 will place it in the type object automatically.
  - [Stable, but expected to be deprecated in PyO3 0.16] In special traits combined with the `#[pyproto]` attribute.
 
 (There are also many magic methods which don't have a special slot, such as `__dir__`. These methods can be implemented as normal in `#[pymethods]`.)
 
-This chapter of the guide has a section on each of these solutions in turn:
+This chapter gives a brief overview of the available methods. An in depth example is given in the following sub-chapters.
 
 ### Magic methods in `#[pymethods]`
 
@@ -18,7 +18,6 @@ In PyO3 0.15, if a function name in `#[pymethods]` is a recognised magic method,
 The magic methods handled by PyO3 are very similar to the standard Python ones on [this page](https://docs.python.org/3/reference/datamodel.html#special-method-names) - in particular they are the the subset which have slots as [defined here](https://docs.python.org/3/c-api/typeobj.html). Some of the slots do not have a magic method in Python, which leads to a few additional magic methods defined only in PyO3:
  - Magic methods for garbage collection
  - Magic methods for the buffer protocol
- - Magic methods for the sequence protocol
 
 When PyO3 handles a magic method, a couple of changes apply compared to other `#[pymethods]`:
  - The `#[pyo3(text_signature = "...")]` attribute is not allowed
@@ -75,71 +74,6 @@ given signatures should be interpreted as follows:
   - `__call__(<self>, ...) -> object` - here, any argument list can be defined
     as for normal `pymethods`
 
-##### Example: Callable objects
-
-Custom classes can be callable if they have a `#[pymethod]` named `__call__`.
-
-The following pyclass is a basic decorator - its constructor takes a Python object
-as argument and calls that object when called.
-
-```rust
-# use pyo3::prelude::*;
-# use pyo3::types::{PyDict, PyTuple};
-#
-#[pyclass(name = "counter")]
-struct PyCounter {
-    count: u64,
-    wraps: Py<PyAny>,
-}
-
-#[pymethods]
-impl PyCounter {
-    #[new]
-    fn __new__(wraps: Py<PyAny>) -> Self {
-        PyCounter { count: 0, wraps }
-    }
-    #[args(args="*", kwargs="**")]
-    fn __call__(
-        &mut self,
-        py: Python,
-        args: &PyTuple,
-        kwargs: Option<&PyDict>,
-    ) -> PyResult<Py<PyAny>> {
-        self.count += 1;
-        let name = self.wraps.getattr(py, "__name__").unwrap();
-
-        println!("{} has been called {} time(s).", name, self.count);
-        self.wraps.call(py, args, kwargs)
-    }
-}
-```
-
-Python code:
-
-```python
-@counter
-def say_hello():
-    print("hello")
-
-say_hello()
-say_hello()
-say_hello()
-say_hello()
-```
-
-Output:
-
-```text
-say_hello has been called 1 time(s).
-hello
-say_hello has been called 2 time(s).
-hello
-say_hello has been called 3 time(s).
-hello
-say_hello has been called 4 time(s).
-hello
-```
-
 #### Iterable objects
 
   - `__iter__(<self>) -> object`
@@ -151,11 +85,7 @@ hello
   - `__aiter__(<self>) -> object`
   - `__anext__(<self>) -> Option<object> or IterANextOutput`
 
-#### Sequence types
+#### Mapping & Sequence types
-
-TODO; see [#1884](https://github.com/PyO3/pyo3/issues/1884)
-
-#### Mapping types
 
   - `__len__(<self>) -> usize`
   - `__contains__(<self>, object) -> bool`
@@ -240,7 +170,8 @@ TODO; see [#1884](https://github.com/PyO3/pyo3/issues/1884)
 
 #### Buffer objects
 
-TODO; see [#1884](https://github.com/PyO3/pyo3/issues/1884)
+  - `__getbuffer__(<self>, *mut ffi::Py_buffer, flags) -> ()`
+  - `__releasebuffer__(<self>, *mut ffi::Py_buffer)` (no return value, not even `PyResult`)
 
 #### Garbage Collector Integration
 
@@ -353,13 +284,15 @@ This trait also has support the augmented arithmetic assignments (`+=`, `-=`,
   * `fn __itruediv__(&'p mut self, other: impl FromPyObject) -> PyResult<()>`
   * `fn __ifloordiv__(&'p mut self, other: impl FromPyObject) -> PyResult<()>`
   * `fn __imod__(&'p mut self, other: impl FromPyObject) -> PyResult<()>`
-  * `fn __ipow__(&'p mut self, other: impl FromPyObject) -> PyResult<()>`
+  * `fn __ipow__(&'p mut self, other: impl FromPyObject, modulo: impl FromPyObject) -> PyResult<()>` on Python 3.8^
+  * `fn __ipow__(&'p mut self, other: impl FromPyObject) -> PyResult<()>` on Python 3.7 see https://bugs.python.org/issue36379
   * `fn __ilshift__(&'p mut self, other: impl FromPyObject) -> PyResult<()>`
   * `fn __irshift__(&'p mut self, other: impl FromPyObject) -> PyResult<()>`
   * `fn __iand__(&'p mut self, other: impl FromPyObject) -> PyResult<()>`
   * `fn __ior__(&'p mut self, other: impl FromPyObject) -> PyResult<()>`
   * `fn __ixor__(&'p mut self, other: impl FromPyObject) -> PyResult<()>`
 
+
 The following methods implement the unary arithmetic operations (`-`, `+`, `abs()` and `~`):
 
   * `fn __neg__(&'p self) -> PyResult<impl ToPyObject>`

guide/src/exception.md

@@ -90,10 +90,10 @@ Python::with_gil(|py| {
 });
 ```
 
-If you already have a Python exception instance, you can simply call [`PyErr::from_instance`].
+If you already have a Python exception object, you can simply call [`PyErr::from_value`].
 
 ```rust,ignore
-PyErr::from_instance(py, err).restore(py);
+PyErr::from_value(py, err).restore(py);
 ```
 
 
@@ -134,7 +134,7 @@ which is an alias for the type `Result<T, PyErr>`.
 A [`PyErr`] represents a Python exception. Errors within the PyO3 library are also exposed as
 Python exceptions.
 
-If your code has a custom error type, adding an implementation of `std::convert::From<MyError> for PyErr` 
+If your code has a custom error type, adding an implementation of `std::convert::From<MyError> for PyErr`
 is usually enough. PyO3 will then automatically convert your error to a Python exception when needed.
 
 The following code snippet defines a Rust error named `CustomIOError`. In its `From<CustomIOError> for PyErr`
@@ -188,7 +188,7 @@ fn main() {
 ```
 
 This has been implemented for most of Rust's standard library errors, so that you can use the `?`
-("try") operator with them. The following code snippet will raise a `ValueError` in Python if 
+("try") operator with them. The following code snippet will raise a `ValueError` in Python if
 `String::parse()` returns an error.
 
 ```rust
@@ -254,6 +254,6 @@ defines exceptions for several standard library modules.
 
 [`PyErr`]: {{#PYO3_DOCS_URL}}/pyo3/struct.PyErr.html
 [`PyResult`]: {{#PYO3_DOCS_URL}}/pyo3/type.PyResult.html
-[`PyErr::from_instance`]: {{#PYO3_DOCS_URL}}/pyo3/struct.PyErr.html#method.from_instance
+[`PyErr::from_value`]: {{#PYO3_DOCS_URL}}/pyo3/struct.PyErr.html#method.from_value
 [`PyAny::is_instance`]: {{#PYO3_DOCS_URL}}/pyo3/struct.PyAny.html#method.is_instance
 [`PyAny::is_instance_of`]: {{#PYO3_DOCS_URL}}/pyo3/struct.PyAny.html#method.is_instance_of

guide/src/faq.md

@@ -139,3 +139,23 @@ a: <builtins.Inner object at 0x0000020044FCC670>
 b: <builtins.Inner object at 0x0000020044FCC670>
 ```
 The downside to this approach is that any Rust code working on the `Outer` struct now has to acquire the GIL to do anything with its field.
+
+## I want to use the `pyo3` crate re-exported from from dependency but the proc-macros fail!
+
+All PyO3 proc-macros (`#[pyclass]`, `#[pyfunction]`, `#[derive(FromPyObject)]`
+and so on) expect the `pyo3` crate to be available under that name in your crate
+root, which is the normal situation when `pyo3` is a direct dependency of your
+crate.
+
+However, when the dependency is renamed, or your crate only indirectly depends
+on `pyo3`, you need to let the macro code know where to find the crate.  This is
+done with the `crate` attribute:
+
+```rust
+# use pyo3::prelude::*;
+# pub extern crate pyo3;
+# mod reexported { pub use ::pyo3; }
+#[pyclass]
+#[pyo3(crate = "reexported::pyo3")]
+struct MyClass;
+```

guide/src/features.md

@@ -65,6 +65,12 @@ Most users should only need a single `#[pymethods]` per `#[pyclass]`. In additio
 
 See [the `#[pyclass]` implementation details](class.md#implementation-details) for more information.
 
+### `pyproto`
+
+This feature enables the `#[pyproto]` macro, which is an alternative (older, soon-to-be-deprecated) to `#[pymethods]` for defining magic methods such as `__eq__`.
+
+> This feature is enabled by default. To disable it, set `default-features = false` for the `pyo3` entry in your Cargo.toml.
+
 ### `nightly`
 
 The `nightly` feature needs the nightly Rust compiler. This allows PyO3 to use Rust's unstable specialization feature to apply the following optimizations:
@@ -83,7 +89,7 @@ These features enable conversions between Python types and types from other Rust
 
 ### `anyhow`
 
-Adds a dependency on [anyhow](https://docs.rs/anyhow). Enables a conversion from [anyhow](https://docs.rs/anyhow)’s [`Error`]https://docs.rs/anyhow/latest/anyhow/struct.Error.html) type to [`PyErr`](https://docs.rs/pyo3/latest/pyo3/struct.PyErr.html), for easy error handling.
+Adds a dependency on [anyhow](https://docs.rs/anyhow). Enables a conversion from [anyhow](https://docs.rs/anyhow)’s [`Error`](https://docs.rs/anyhow/latest/anyhow/struct.Error.html) type to [`PyErr`](https://docs.rs/pyo3/latest/pyo3/struct.PyErr.html), for easy error handling.
 
 ### `eyre`
 

guide/src/migration.md

@@ -5,10 +5,40 @@ For a detailed list of all changes, see the [CHANGELOG](changelog.md).
 
 ## from 0.15.* to 0.16
 
-### Drop support for older technogies
+### Drop support for older technologies
 
 PyO3 0.16 has increased minimum Rust version to 1.48 and minimum Python version to 3.7. This enables ore use of newer language features (enabling some of the other additions in 0.16) and simplifies maintenance of the project.
 
+### Container magic methods now match Python behavior
+
+In PyO3 0.15, `__getitem__`, `__setitem__` and `__delitem__` in `#[pymethods]` would generate only the _mapping_ implementation for a `#[pyclass]`. To match the Python behavior, these methods now generate both the _mapping_ **and** _sequence_ implementations.
+
+This means that classes implementing these `#[pymethods]` will now also be treated as sequences, same as a Python `class` would be. Small differences in behavior may result:
+ - PyO3 will allow instances of these classes to be cast to `PySequence` as well as `PyMapping`.
+ - Python will provide a default implementation of `__iter__` (if the class did not have one) which repeatedly calls `__getitem__` with integers (starting at 0) until an `IndexError` is raised.
+
+To explain this in detail, consider the following Python class:
+
+```python
+class ExampleContainer:
+
+    def __len__(self):
+        return 5
+
+    def __getitem__(self, idx: int) -> int:
+        if idx < 0 or idx > 5:
+            raise IndexError()
+        return idx
+```
+
+This class implements a Python [sequence](https://docs.python.org/3/glossary.html#term-sequence).
+
+The `__len__` and `__getitem__` methods are also used to implement a Python [mapping](https://docs.python.org/3/glossary.html#term-mapping). In the Python C-API, these methods are not shared: the sequence `__len__` and `__getitem__` are defined by the `sq_len` and `sq_item` slots, and the mapping equivalents are `mp_len` and `mp_subscript`. There are similar distinctions for `__setitem__` and `__delitem__`.
+
+Because there is no such distinction from Python, implementing these methods will fill the mapping and sequence slots simultaneously. A Python class with `__len__` implemented, for example, will have both the `sq_len` and `mp_len` slots filled.
+
+The PyO3 behavior in 0.16 has been changed to be closer to this Python behavior by default.
+
 ## from 0.14.* to 0.15
 
 ### Changes in sequence indexing
@@ -170,7 +200,7 @@ let err: PyErr = TypeError::py_err("error message");
 
 After:
 
-```rust
+```rust,ignore
 # use pyo3::{PyErr, PyResult, Python, type_object::PyTypeObject};
 # use pyo3::exceptions::{PyBaseException, PyTypeError};
 # Python::with_gil(|py| -> PyResult<()> {

guide/src/parallelism.md

@@ -56,7 +56,7 @@ We are using `pytest-benchmark` to benchmark four word count functions:
 3. Rust sequential version
 4. Rust sequential version executed twice with two Python threads
 
-The benchmark script can be found [here](https://github.com/PyO3/pyo3/blob/main/examples/word-count/tests/test_word_count.py), and we can run `tox` in the `word-count` folder to benchmark these functions.
+The benchmark script can be found [here](https://github.com/PyO3/pyo3/blob/main/examples/word-count/tests/test_word_count.py), and we can run `nox` in the `word-count` folder to benchmark these functions.
 
 While the results of the benchmark of course depend on your machine, the relative results should be similar to this (mid 2020):
 ```ignore

guide/src/python_from_rust.md

@@ -209,6 +209,9 @@ assert userdata.as_tuple() == userdata_as_tuple
 can be used to generate a Python module which can then be used just as if it was imported with
 `PyModule::import`.
 
+**Warning**: This will compile and execute code. **Never** pass untrusted code
+to this function!
+
 ```rust
 use pyo3::{prelude::*, types::{IntoPyDict, PyModule}};
 
@@ -236,6 +239,105 @@ def leaky_relu(x, slope=0.01):
 # }
 ```
 
+### Include multiple Python files
+
+You can include a file at compile time by using
+[`std::include_str`](https://doc.rust-lang.org/std/macro.include_str.html) macro.
+
+Or you can load a file at runtime by using
+[`std::fs::read_to_string`](https://doc.rust-lang.org/std/fs/fn.read_to_string.html) function.
+
+Many Python files can be included and loaded as modules. If one file depends on
+another you must preserve correct order while declaring `PyModule`.
+
+Example directory structure:
+```text
+.
+├── Cargo.lock
+├── Cargo.toml
+├── python_app
+│   ├── app.py
+│   └── utils
+│       └── foo.py
+└── src
+    └── main.rs
+```
+
+`python_app/app.py`:
+```python
+from utils.foo import bar
+
+
+def run():
+    return bar()
+```
+
+`python_app/utils/foo.py`:
+```python
+def bar():
+    return "baz"
+```
+
+The example below shows:
+* how to include content of `app.py` and `utils/foo.py` into your rust binary
+* how to call function `run()` (declared in `app.py`) that needs function
+  imported from `utils/foo.py`
+
+`src/main.rs`:
+```ignore
+use pyo3::prelude::*;
+
+fn main() -> PyResult<()> {
+    let py_foo = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/python_app/utils/foo.py"));
+    let py_app = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/python_app/app.py"));
+    let from_python = Python::with_gil(|py| -> PyResult<Py<PyAny>> {
+        PyModule::from_code(py, py_foo, "utils.foo", "utils.foo")?;
+        let app: Py<PyAny> = PyModule::from_code(py, py_app, "", "")?
+            .getattr("run")?
+            .into();
+        app.call0(py)
+    });
+
+    println!("py: {}", from_python?);
+    Ok(())
+}
+```
+
+The example below shows:
+* how to load content of `app.py` at runtime so that it sees its dependencies
+  automatically
+* how to call function `run()` (declared in `app.py`) that needs function
+  imported from `utils/foo.py`
+
+It is recommended to use absolute paths because then your binary can be run
+from anywhere as long as your `app.py` is in the expected directory (in this example
+that directory is `/usr/share/python_app`).
+
+`src/main.rs`:
+```ignore
+use pyo3::prelude::*;
+use pyo3::types::PyList;
+use std::fs;
+use std::path::Path;
+
+fn main() -> PyResult<()> {
+    let path = Path::new("/usr/share/python_app");
+    let py_app = fs::read_to_string(path.join("app.py"))?;
+    let from_python = Python::with_gil(|py| -> PyResult<Py<PyAny>> {
+        let syspath: &PyList = py.import("sys")?.getattr("path")?.downcast::<PyList>()?;
+        syspath.insert(0, &path)?;
+        let app: Py<PyAny> = PyModule::from_code(py, &py_app, "", "")?
+            .getattr("run")?
+            .into();
+        app.call0(py)
+    });
+
+    println!("py: {}", from_python?);
+    Ok(())
+}
+```
+
+
 [`Python::run`]: {{#PYO3_DOCS_URL}}/pyo3/struct.Python.html#method.run
 [`py_run!`]: {{#PYO3_DOCS_URL}}/pyo3/macro.py_run.html
 
@@ -280,7 +382,7 @@ class House(object):
             Err(e) => {
                 house.call_method1(
                     "__exit__",
-                    (e.ptype(py), e.pvalue(py), e.ptraceback(py))
+                    (e.get_type(py), e.value(py), e.traceback(py))
                 ).unwrap();
             }
         }

pyo3-build-config/src/impl_.rs

@@ -32,7 +32,9 @@ pub fn cargo_env_var(var: &str) -> Option<String> {
 /// Gets an external environment variable, and registers the build script to rerun if
 /// the variable changes.
 pub fn env_var(var: &str) -> Option<OsString> {
-    println!("cargo:rerun-if-env-changed={}", var);
+    if cfg!(feature = "resolve-config") {
+        println!("cargo:rerun-if-env-changed={}", var);
+    }
     env::var_os(var)
 }
 
@@ -219,6 +221,7 @@ print("mingw", get_platform().startswith("mingw"))
         };
 
         let abi3 = is_abi3();
+
         let implementation = map["implementation"].parse()?;
 
         let lib_name = if cfg!(windows) {
@@ -267,6 +270,61 @@ print("mingw", get_platform().startswith("mingw"))
         })
     }
 
+    /// Generate from parsed sysconfigdata file
+    ///
+    /// Use [`parse_sysconfigdata`] to generate a hash map of configuration values which may be
+    /// used to build an [`InterpreterConfig`].
+    pub fn from_sysconfigdata(sysconfigdata: &Sysconfigdata) -> Result<Self> {
+        macro_rules! get_key {
+            ($sysconfigdata:expr, $key:literal) => {
+                $sysconfigdata
+                    .get_value($key)
+                    .ok_or(concat!($key, " not found in sysconfigdata file"))
+            };
+        }
+
+        macro_rules! parse_key {
+            ($sysconfigdata:expr, $key:literal) => {
+                get_key!($sysconfigdata, $key)?
+                    .parse()
+                    .context(concat!("could not parse value of ", $key))
+            };
+        }
+
+        let soabi = get_key!(sysconfigdata, "SOABI")?;
+        let implementation = PythonImplementation::from_soabi(soabi)?;
+        let version = parse_key!(sysconfigdata, "VERSION")?;
+        let shared = match sysconfigdata.get_value("Py_ENABLE_SHARED") {
+            Some("1") | Some("true") | Some("True") => true,
+            Some("0") | Some("false") | Some("False") => false,
+            _ => bail!("expected a bool (1/true/True or 0/false/False) for Py_ENABLE_SHARED"),
+        };
+        let lib_dir = get_key!(sysconfigdata, "LIBDIR").ok().map(str::to_string);
+        let lib_name = Some(default_lib_name_unix(
+            version,
+            implementation,
+            sysconfigdata.get_value("LDVERSION"),
+        ));
+        let pointer_width = parse_key!(sysconfigdata, "SIZEOF_VOID_P")
+            .map(|bytes_width: u32| bytes_width * 8)
+            .ok();
+        let build_flags = BuildFlags::from_sysconfigdata(sysconfigdata).fixup(version);
+
+        Ok(InterpreterConfig {
+            implementation,
+            version,
+            shared,
+            abi3: is_abi3(),
+            lib_dir,
+            lib_name,
+            executable: None,
+            pointer_width,
+            build_flags,
+            suppress_build_script_link_lines: false,
+            extra_build_script_lines: vec![],
+        })
+    }
+
     #[doc(hidden)]
     pub fn from_path(path: impl AsRef<Path>) -> Result<Self> {
         let path = path.as_ref();
@@ -445,6 +503,17 @@ impl PythonImplementation {
     pub fn is_pypy(self) -> bool {
         self == PythonImplementation::PyPy
     }
+
+    #[doc(hidden)]
+    pub fn from_soabi(soabi: &str) -> Result<Self> {
+        if soabi.starts_with("pypy") {
+            Ok(PythonImplementation::PyPy)
+        } else if soabi.starts_with("cpython") {
+            Ok(PythonImplementation::CPython)
+        } else {
+            bail!("unsupported Python interpreter");
+        }
+    }
 }
 
 impl Display for PythonImplementation {
@@ -471,11 +540,32 @@ fn is_abi3() -> bool {
     cargo_env_var("CARGO_FEATURE_ABI3").is_some()
 }
 
-struct CrossCompileConfig {
+/// Configuration needed by PyO3 to cross-compile for a target platform.
-    lib_dir: PathBuf,
+///
+/// Usually this is collected from the environment (i.e. `PYO3_CROSS_*` and `CARGO_CFG_TARGET_*`)
+/// when a cross-compilation configuration is detected.
+#[derive(Debug, PartialEq)]
+pub struct CrossCompileConfig {
+    /// The directory containing the Python library to link against.
+    pub lib_dir: PathBuf,
+
+    /// The version of the Python library to link against.
     version: Option<PythonVersion>,
-    os: String,
+
+    /// The `arch` component of the compilaton target triple.
+    ///
+    /// e.g. x86_64, i386, arm, thumb, mips, etc.
     arch: String,
+
+    /// The `vendor` component of the compilaton target triple.
+    ///
+    /// e.g. apple, pc, unknown, etc.
+    vendor: String,
+
+    /// The `os` component of the compilaton target triple.
+    ///
+    /// e.g. darwin, freebsd, linux, windows, etc.
+    os: String,
 }
 
 #[allow(unused)]
@@ -485,47 +575,74 @@ pub fn any_cross_compiling_env_vars_set() -> bool {
         || env::var_os("PYO3_CROSS_PYTHON_VERSION").is_some()
 }
 
-fn cross_compiling() -> Result<Option<CrossCompileConfig>> {
+fn cross_compiling_from_cargo_env() -> Result<Option<CrossCompileConfig>> {
+    let host = cargo_env_var("HOST").ok_or("expected HOST env var")?;
+    let target = cargo_env_var("TARGET").ok_or("expected TARGET env var")?;
+
+    if host == target {
+        // Definitely not cross compiling if the host matches the target
+        return Ok(None);
+    }
+
+    if target == "i686-pc-windows-msvc" && host == "x86_64-pc-windows-msvc" {
+        // Not cross-compiling to compile for 32-bit Python from windows 64-bit
+        return Ok(None);
+    }
+
+    let target_arch =
+        cargo_env_var("CARGO_CFG_TARGET_ARCH").ok_or("expected CARGO_CFG_TARGET_ARCH env var")?;
+    let target_vendor = cargo_env_var("CARGO_CFG_TARGET_VENDOR")
+        .ok_or("expected CARGO_CFG_TARGET_VENDOR env var")?;
+    let target_os =
+        cargo_env_var("CARGO_CFG_TARGET_OS").ok_or("expected CARGO_CFG_TARGET_OS env var")?;
+
+    cross_compiling(&host, &target_arch, &target_vendor, &target_os)
+}
+
+/// Detect whether we are cross compiling and return an assembled CrossCompileConfig if so.
+///
+/// This function relies on PyO3 cross-compiling environment variables:
+///
+///   * `PYO3_CROSS`: If present, forces PyO3 to configure as a cross-compilation.
+///   * `PYO3_CROSS_LIB_DIR`: Must be set to the directory containing the target's libpython DSO and
+///   the associated `_sysconfigdata*.py` file for Unix-like targets, or the Python DLL import
+///   libraries for the Windows target.
+///   * `PYO3_CROSS_PYTHON_VERSION`: Major and minor version (e.g. 3.9) of the target Python
+///   installation. This variable is only needed if PyO3 cannnot determine the version to target
+///   from `abi3-py3*` features, or if there are multiple versions of Python present in
+///   `PYO3_CROSS_LIB_DIR`.
+///
+/// See the [PyO3 User Guide](https://pyo3.rs/) for more info on cross-compiling.
+pub fn cross_compiling(
+    host: &str,
+    target_arch: &str,
+    target_vendor: &str,
+    target_os: &str,
+) -> Result<Option<CrossCompileConfig>> {
     let cross = env_var("PYO3_CROSS");
     let cross_lib_dir = env_var("PYO3_CROSS_LIB_DIR");
     let cross_python_version = env_var("PYO3_CROSS_PYTHON_VERSION");
 
-    let target_arch = cargo_env_var("CARGO_CFG_TARGET_ARCH");
+    let target_triple = format!("{}-{}-{}", target_arch, target_vendor, target_os);
-    let target_vendor = cargo_env_var("CARGO_CFG_TARGET_VENDOR");
-    let target_os = cargo_env_var("CARGO_CFG_TARGET_OS");
 
     if cross.is_none() && cross_lib_dir.is_none() && cross_python_version.is_none() {
         // No cross-compiling environment variables set; try to determine if this is a known case
         // which is not cross-compilation.
 
-        let target = cargo_env_var("TARGET").unwrap();
+        if target_triple == "x86_64-apple-darwin" && host == "aarch64-apple-darwin" {
-        let host = cargo_env_var("HOST").unwrap();
-        if target == host {
-            // Not cross-compiling
-            return Ok(None);
-        }
-
-        if target == "i686-pc-windows-msvc" && host == "x86_64-pc-windows-msvc" {
-            // Not cross-compiling to compile for 32-bit Python from windows 64-bit
-            return Ok(None);
-        }
-
-        if target == "x86_64-apple-darwin" && host == "aarch64-apple-darwin" {
             // Not cross-compiling to compile for x86-64 Python from macOS arm64
             return Ok(None);
         }
 
-        if target == "aarch64-apple-darwin" && host == "x86_64-apple-darwin" {
+        if target_triple == "aarch64-apple-darwin" && host == "x86_64-apple-darwin" {
             // Not cross-compiling to compile for arm64 Python from macOS x86_64
             return Ok(None);
         }
 
-        if let (Some(arch), Some(vendor), Some(os)) = (&target_arch, &target_vendor, &target_os) {
+        if host.starts_with(&target_triple) {
-            if host.starts_with(&format!("{}-{}-{}", arch, vendor, os)) {
+            // Not cross-compiling if arch-vendor-os is all the same
-                // Not cross-compiling if arch-vendor-os is all the same
+            // e.g. x86_64-unknown-linux-musl on x86_64-unknown-linux-gnu host
-                // e.g. x86_64-unknown-linux-musl on x86_64-unknown-linux-gnu host
+            return Ok(None);
-                return Ok(None);
-            }
         }
     }
 
@@ -535,8 +652,9 @@ fn cross_compiling() -> Result<Option<CrossCompileConfig>> {
         lib_dir: cross_lib_dir
             .ok_or("The PYO3_CROSS_LIB_DIR environment variable must be set when cross-compiling")?
             .into(),
-        os: target_os.unwrap(),
+        arch: target_arch.into(),
-        arch: target_arch.unwrap(),
+        vendor: target_vendor.into(),
+        os: target_os.into(),
         version: cross_python_version
             .map(|os_string| {
                 let utf8_str = os_string
@@ -610,14 +728,14 @@ impl BuildFlags {
         BuildFlags(HashSet::new())
     }
 
-    fn from_config_map(config_map: &HashMap<String, String>) -> Self {
+    fn from_sysconfigdata(config_map: &Sysconfigdata) -> Self {
         Self(
             BuildFlags::ALL
                 .iter()
                 .cloned()
                 .filter(|flag| {
                     config_map
-                        .get(&flag.to_string())
+                        .get_value(&flag.to_string())
                         .map_or(false, |value| value == "1")
                 })
                 .collect(),
@@ -708,12 +826,35 @@ fn parse_script_output(output: &str) -> HashMap<String, String> {
         .collect()
 }
 
+/// Parsed data from Python sysconfigdata file
+///
+/// A hash map of all values from a sysconfigdata file.
+pub struct Sysconfigdata(HashMap<String, String>);
+
+impl Sysconfigdata {
+    pub fn get_value<S: AsRef<str>>(&self, k: S) -> Option<&str> {
+        self.0.get(k.as_ref()).map(String::as_str)
+    }
+
+    #[allow(dead_code)]
+    fn new() -> Self {
+        Sysconfigdata(HashMap::new())
+    }
+
+    #[allow(dead_code)]
+    fn insert<S: Into<String>>(&mut self, k: S, v: S) {
+        self.0.insert(k.into(), v.into());
+    }
+}
+
 /// Parse sysconfigdata file
 ///
 /// The sysconfigdata is simply a dictionary containing all the build time variables used for the
-/// python executable and library. Here it is read and added to a script to extract only what is
+/// python executable and library. This function necessitates a python interpreter on the host
-/// necessary. This necessitates a python interpreter for the host machine to work.
+/// machine to work. Here it is read into a `Sysconfigdata` (hash map), which can be turned into an
-fn parse_sysconfigdata(sysconfigdata_path: impl AsRef<Path>) -> Result<InterpreterConfig> {
+/// [`InterpreterConfig`](InterpreterConfig) using
+/// [`from_sysconfigdata`](InterpreterConfig::from_sysconfigdata).
+pub fn parse_sysconfigdata(sysconfigdata_path: impl AsRef<Path>) -> Result<Sysconfigdata> {
     let sysconfigdata_path = sysconfigdata_path.as_ref();
     let mut script = fs::read_to_string(&sysconfigdata_path).with_context(|| {
         format!(
@@ -722,83 +863,13 @@ fn parse_sysconfigdata(sysconfigdata_path: impl AsRef<Path>) -> Result<Interpret
         )
     })?;
     script += r#"
-print("version", build_time_vars["VERSION"])
+for key, val in build_time_vars.items():
-print("SOABI", build_time_vars.get("SOABI", ""))
+    print(key, val)
-if "LIBDIR" in build_time_vars:
-    print("LIBDIR", build_time_vars["LIBDIR"])
-KEYS = [
-    "WITH_THREAD",
-    "Py_DEBUG",
-    "Py_REF_DEBUG",
-    "Py_TRACE_REFS",
-    "COUNT_ALLOCS",
-    "Py_ENABLE_SHARED",
-    "LDVERSION",
-    "SIZEOF_VOID_P"
-]
-for key in KEYS:
-    print(key, build_time_vars.get(key, 0))
 "#;
+
     let output = run_python_script(&find_interpreter()?, &script)?;
 
-    let sysconfigdata = parse_script_output(&output);
+    Ok(Sysconfigdata(parse_script_output(&output)))
-
-    macro_rules! get_key {
-        ($sysconfigdata:expr, $key:literal) => {
-            $sysconfigdata
-                .get($key)
-                .ok_or(concat!($key, " not found in sysconfigdata file"))
-        };
-    }
-
-    macro_rules! parse_key {
-        ($sysconfigdata:expr, $key:literal) => {
-            get_key!($sysconfigdata, $key)?
-                .parse()
-                .context(concat!("could not parse value of ", $key))
-        };
-    }
-
-    let version = parse_key!(sysconfigdata, "version")?;
-    let pointer_width = parse_key!(sysconfigdata, "SIZEOF_VOID_P")
-        .map(|bytes_width: u32| bytes_width * 8)
-        .ok();
-
-    let soabi = get_key!(sysconfigdata, "SOABI")?;
-    let implementation = if soabi.starts_with("pypy") {
-        PythonImplementation::PyPy
-    } else if soabi.starts_with("cpython") {
-        PythonImplementation::CPython
-    } else {
-        bail!("unsupported Python interpreter");
-    };
-
-    let shared = match sysconfigdata
-        .get("Py_ENABLE_SHARED")
-        .map(|string| string.as_str())
-    {
-        Some("1") | Some("true") | Some("True") => true,
-        Some("0") | Some("false") | Some("False") | None => false,
-        _ => bail!("expected a bool (1/true/True or 0/false/False) for Py_ENABLE_SHARED"),
-    };
-
-    Ok(InterpreterConfig {
-        implementation,
-        version,
-        shared,
-        abi3: is_abi3(),
-        lib_dir: get_key!(sysconfigdata, "LIBDIR").ok().cloned(),
-        lib_name: Some(default_lib_name_unix(
-            version,
-            implementation,
-            sysconfigdata.get("LDVERSION").map(String::as_str),
-        )),
-        executable: None,
-        pointer_width,
-        build_flags: BuildFlags::from_config_map(&sysconfigdata).fixup(version),
-        suppress_build_script_link_lines: false,
-        extra_build_script_lines: vec![],
-    })
 }
 
 fn starts_with(entry: &DirEntry, pat: &str) -> bool {
@@ -810,52 +881,8 @@ fn ends_with(entry: &DirEntry, pat: &str) -> bool {
     name.to_string_lossy().ends_with(pat)
 }
 
-/// Finds the `_sysconfigdata*.py` file in the library path.
-///
-/// From the python source for `_sysconfigdata*.py` is always going to be located at
-/// `build/lib.{PLATFORM}-{PY_MINOR_VERSION}` when built from source. The [exact line][1] is defined as:
-///
-/// ```py
-/// pybuilddir = 'build/lib.%s-%s' % (get_platform(), sys.version_info[:2])
-/// ```
-///
-/// Where get_platform returns a kebab-case formatted string containing the os, the architecture and
-/// possibly the os' kernel version (not the case on linux). However, when installed using a package
-/// manager, the `_sysconfigdata*.py` file is installed in the `${PREFIX}/lib/python3.Y/` directory.
-/// The `_sysconfigdata*.py` is generally in a sub-directory of the location of `libpython3.Y.so`.
-/// So we must find the file in the following possible locations:
-///
-/// ```sh
-/// # distribution from package manager, lib_dir should include lib/
-/// ${INSTALL_PREFIX}/lib/python3.Y/_sysconfigdata*.py
-/// ${INSTALL_PREFIX}/lib/libpython3.Y.so
-/// ${INSTALL_PREFIX}/lib/python3.Y/config-3.Y-${HOST_TRIPLE}/libpython3.Y.so
-///
-/// # Built from source from host
-/// ${CROSS_COMPILED_LOCATION}/build/lib.linux-x86_64-Y/_sysconfigdata*.py
-/// ${CROSS_COMPILED_LOCATION}/libpython3.Y.so
-///
-/// # if cross compiled, kernel release is only present on certain OS targets.
-/// ${CROSS_COMPILED_LOCATION}/build/lib.{OS}(-{OS-KERNEL-RELEASE})?-{ARCH}-Y/_sysconfigdata*.py
-/// ${CROSS_COMPILED_LOCATION}/libpython3.Y.so
-/// ```
-///
-/// [1]: https://github.com/python/cpython/blob/3.5/Lib/sysconfig.py#L389
 fn find_sysconfigdata(cross: &CrossCompileConfig) -> Result<PathBuf> {
-    let sysconfig_paths = search_lib_dir(&cross.lib_dir, cross);
+    let mut sysconfig_paths = find_all_sysconfigdata(cross);
-    let sysconfig_name = env_var("_PYTHON_SYSCONFIGDATA_NAME");
-    let mut sysconfig_paths = sysconfig_paths
-        .iter()
-        .filter_map(|p| {
-            let canonical = fs::canonicalize(p).ok();
-            match &sysconfig_name {
-                Some(_) => canonical.filter(|p| p.file_stem() == sysconfig_name.as_deref()),
-                None => canonical,
-            }
-        })
-        .collect::<Vec<PathBuf>>();
-    sysconfig_paths.sort();
-    sysconfig_paths.dedup();
     if sysconfig_paths.is_empty() {
         bail!(
             "Could not find either libpython.so or _sysconfigdata*.py in {}",
@@ -877,22 +904,90 @@ fn find_sysconfigdata(cross: &CrossCompileConfig) -> Result<PathBuf> {
     Ok(sysconfig_paths.remove(0))
 }
 
-/// recursive search for _sysconfigdata, returns all possibilities of sysconfigdata paths
+/// Finds `_sysconfigdata*.py` files for detected Python interpreters.
-fn search_lib_dir(path: impl AsRef<Path>, cross: &CrossCompileConfig) -> Vec<PathBuf> {
+///
-    let mut sysconfig_paths = vec![];
+/// From the python source for `_sysconfigdata*.py` is always going to be located at
-    let version_pat = if let Some(v) = &cross.version {
+/// `build/lib.{PLATFORM}-{PY_MINOR_VERSION}` when built from source. The [exact line][1] is defined as:
+///
+/// ```py
+/// pybuilddir = 'build/lib.%s-%s' % (get_platform(), sys.version_info[:2])
+/// ```
+///
+/// Where get_platform returns a kebab-case formatted string containing the os, the architecture and
+/// possibly the os' kernel version (not the case on linux). However, when installed using a package
+/// manager, the `_sysconfigdata*.py` file is installed in the `${PREFIX}/lib/python3.Y/` directory.
+/// The `_sysconfigdata*.py` is generally in a sub-directory of the location of `libpython3.Y.so`.
+/// So we must find the file in the following possible locations:
+///
+/// ```sh
+/// # distribution from package manager, (lib_dir may or may not include lib/)
+/// ${INSTALL_PREFIX}/lib/python3.Y/_sysconfigdata*.py
+/// ${INSTALL_PREFIX}/lib/libpython3.Y.so
+/// ${INSTALL_PREFIX}/lib/python3.Y/config-3.Y-${HOST_TRIPLE}/libpython3.Y.so
+///
+/// # Built from source from host
+/// ${CROSS_COMPILED_LOCATION}/build/lib.linux-x86_64-Y/_sysconfigdata*.py
+/// ${CROSS_COMPILED_LOCATION}/libpython3.Y.so
+///
+/// # if cross compiled, kernel release is only present on certain OS targets.
+/// ${CROSS_COMPILED_LOCATION}/build/lib.{OS}(-{OS-KERNEL-RELEASE})?-{ARCH}-Y/_sysconfigdata*.py
+/// ${CROSS_COMPILED_LOCATION}/libpython3.Y.so
+///
+/// # PyPy includes a similar file since v73
+/// ${INSTALL_PREFIX}/lib/pypy3.Y/_sysconfigdata.py
+/// ${INSTALL_PREFIX}/lib_pypy/_sysconfigdata.py
+/// ```
+///
+/// [1]: https://github.com/python/cpython/blob/3.5/Lib/sysconfig.py#L389
+pub fn find_all_sysconfigdata(cross: &CrossCompileConfig) -> Vec<PathBuf> {
+    let sysconfig_paths = search_lib_dir(&cross.lib_dir, cross);
+    let sysconfig_name = env_var("_PYTHON_SYSCONFIGDATA_NAME");
+    let mut sysconfig_paths = sysconfig_paths
+        .iter()
+        .filter_map(|p| {
+            let canonical = fs::canonicalize(p).ok();
+            match &sysconfig_name {
+                Some(_) => canonical.filter(|p| p.file_stem() == sysconfig_name.as_deref()),
+                None => canonical,
+            }
+        })
+        .collect::<Vec<PathBuf>>();
+
+    sysconfig_paths.sort();
+    sysconfig_paths.dedup();
+
+    sysconfig_paths
+}
+
+fn is_pypy_lib_dir(path: &str, v: &Option<PythonVersion>) -> bool {
+    let pypy_version_pat = if let Some(v) = v {
+        format!("pypy{}", v)
+    } else {
+        "pypy3.".into()
+    };
+    path == "lib_pypy" || path.starts_with(&pypy_version_pat)
+}
+
+fn is_cpython_lib_dir(path: &str, v: &Option<PythonVersion>) -> bool {
+    let cpython_version_pat = if let Some(v) = v {
         format!("python{}", v)
     } else {
         "python3.".into()
     };
-    for f in fs::read_dir(path).expect("Path does not exist").into_iter() {
+    path.starts_with(&cpython_version_pat)
+}
+
+/// recursive search for _sysconfigdata, returns all possibilities of sysconfigdata paths
+fn search_lib_dir(path: impl AsRef<Path>, cross: &CrossCompileConfig) -> Vec<PathBuf> {
+    let mut sysconfig_paths = vec![];
+    for f in fs::read_dir(path).expect("Path does not exist") {
         sysconfig_paths.extend(match &f {
             // Python 3.7+ sysconfigdata with platform specifics
             Ok(f) if starts_with(f, "_sysconfigdata_") && ends_with(f, "py") => vec![f.path()],
             Ok(f) if f.metadata().map_or(false, |metadata| metadata.is_dir()) => {
                 let file_name = f.file_name();
                 let file_name = file_name.to_string_lossy();
-                if file_name.starts_with("build") {
+                if file_name == "build" || file_name == "lib" {
                     search_lib_dir(f.path(), cross)
                 } else if file_name.starts_with("lib.") {
                     // check if right target os
@@ -908,7 +1003,9 @@ fn search_lib_dir(path: impl AsRef<Path>, cross: &CrossCompileConfig) -> Vec<Pat
                         continue;
                     }
                     search_lib_dir(f.path(), cross)
-                } else if file_name.starts_with(&version_pat) {
+                } else if is_cpython_lib_dir(&file_name, &cross.version)
+                    || is_pypy_lib_dir(&file_name, &cross.version)
+                {
                     search_lib_dir(f.path(), cross)
                 } else {
                     continue;
@@ -947,7 +1044,7 @@ fn load_cross_compile_from_sysconfigdata(
     cross_compile_config: CrossCompileConfig,
 ) -> Result<InterpreterConfig> {
     let sysconfigdata_path = find_sysconfigdata(&cross_compile_config)?;
-    parse_sysconfigdata(sysconfigdata_path)
+    InterpreterConfig::from_sysconfigdata(&parse_sysconfigdata(sysconfigdata_path)?)
 }
 
 fn windows_hardcoded_cross_compile(
@@ -957,9 +1054,10 @@ fn windows_hardcoded_cross_compile(
         .ok_or("PYO3_CROSS_PYTHON_VERSION or an abi3-py3* feature must be specified when cross-compiling for Windows.")?;
 
     let abi3 = is_abi3();
+    let implementation = PythonImplementation::CPython;
 
     Ok(InterpreterConfig {
-        implementation: PythonImplementation::CPython,
+        implementation,
         version,
         shared: true,
         abi3,
@@ -1159,7 +1257,7 @@ fn fixup_config_for_abi3(
 /// This must be called from PyO3's build script, because it relies on environment variables such as
 /// CARGO_CFG_TARGET_OS which aren't available at any other time.
 pub fn make_cross_compile_config() -> Result<Option<InterpreterConfig>> {
-    let mut interpreter_config = if let Some(paths) = cross_compiling()? {
+    let mut interpreter_config = if let Some(paths) = cross_compiling_from_cargo_env()? {
         load_cross_compile_config(paths)?
     } else {
         return Ok(None);
@@ -1265,24 +1363,33 @@ mod tests {
     }
 
     #[test]
-    fn build_flags_from_config_map() {
+    fn build_flags_from_sysconfigdata() {
-        let mut config_map = HashMap::new();
+        let mut sysconfigdata = Sysconfigdata::new();
 
-        assert_eq!(BuildFlags::from_config_map(&config_map).0, HashSet::new());
+        assert_eq!(
+            BuildFlags::from_sysconfigdata(&sysconfigdata).0,
+            HashSet::new()
+        );
 
         for flag in &BuildFlags::ALL {
-            config_map.insert(flag.to_string(), "0".into());
+            sysconfigdata.insert(flag.to_string(), "0".into());
         }
 
-        assert_eq!(BuildFlags::from_config_map(&config_map).0, HashSet::new());
+        assert_eq!(
+            BuildFlags::from_sysconfigdata(&sysconfigdata).0,
+            HashSet::new()
+        );
 
         let mut expected_flags = HashSet::new();
         for flag in &BuildFlags::ALL {
-            config_map.insert(flag.to_string(), "1".into());
+            sysconfigdata.insert(flag.to_string(), "1".into());
             expected_flags.insert(flag.clone());
         }
 
-        assert_eq!(BuildFlags::from_config_map(&config_map).0, expected_flags);
+        assert_eq!(
+            BuildFlags::from_sysconfigdata(&sysconfigdata).0,
+            expected_flags
+        );
     }
 
     #[test]
@@ -1325,6 +1432,41 @@ mod tests {
         assert!(make_interpreter_config().is_ok())
     }
 
+    #[test]
+    fn config_from_empty_sysconfigdata() {
+        let sysconfigdata = Sysconfigdata::new();
+        assert!(InterpreterConfig::from_sysconfigdata(&sysconfigdata).is_err());
+    }
+
+    #[test]
+    fn config_from_sysconfigdata() {
+        let mut sysconfigdata = Sysconfigdata::new();
+        // these are the minimal values required such that InterpreterConfig::from_sysconfigdata
+        // does not error
+        sysconfigdata.insert("SOABI", "cpython-37m-x86_64-linux-gnu");
+        sysconfigdata.insert("VERSION", "3.7");
+        sysconfigdata.insert("Py_ENABLE_SHARED", "1");
+        sysconfigdata.insert("LIBDIR", "/usr/lib");
+        sysconfigdata.insert("LDVERSION", "3.7m");
+        sysconfigdata.insert("SIZEOF_VOID_P", "8");
+        assert_eq!(
+            InterpreterConfig::from_sysconfigdata(&sysconfigdata).unwrap(),
+            InterpreterConfig {
+                abi3: false,
+                build_flags: BuildFlags::from_sysconfigdata(&sysconfigdata),
+                pointer_width: Some(64),
+                executable: None,
+                implementation: PythonImplementation::CPython,
+                lib_dir: Some("/usr/lib".into()),
+                lib_name: Some("python3.7m".into()),
+                shared: true,
+                version: PythonVersion::PY37,
+                suppress_build_script_link_lines: false,
+                extra_build_script_lines: vec![],
+            }
+        );
+    }
+
     #[test]
     fn windows_hardcoded_cross_compile() {
         let cross_config = CrossCompileConfig {
@@ -1332,6 +1474,7 @@ mod tests {
             version: Some(PythonVersion { major: 3, minor: 7 }),
             os: "os".into(),
             arch: "arch".into(),
+            vendor: "vendor".into(),
         };
 
         assert_eq!(
@@ -1497,8 +1640,9 @@ mod tests {
         let cross = CrossCompileConfig {
             lib_dir: lib_dir.into(),
             version: Some(interpreter_config.version),
-            os: "linux".into(),
             arch: "x86_64".into(),
+            vendor: "unknown".into(),
+            os: "linux".into(),
         };
 
         let sysconfigdata_path = match find_sysconfigdata(&cross) {
@@ -1506,7 +1650,8 @@ mod tests {
             // Couldn't find a matching sysconfigdata; never mind!
             Err(_) => return,
         };
-        let parsed_config = super::parse_sysconfigdata(sysconfigdata_path).unwrap();
+        let sysconfigdata = super::parse_sysconfigdata(sysconfigdata_path).unwrap();
+        let parsed_config = InterpreterConfig::from_sysconfigdata(&sysconfigdata).unwrap();
 
         assert_eq!(
             parsed_config,
@@ -1530,11 +1675,11 @@ mod tests {
     fn test_venv_interpreter() {
         let base = OsStr::new("base");
         assert_eq!(
-            venv_interpreter(&base, true),
+            venv_interpreter(base, true),
             PathBuf::from_iter(&["base", "Scripts", "python.exe"])
         );
         assert_eq!(
-            venv_interpreter(&base, false),
+            venv_interpreter(base, false),
             PathBuf::from_iter(&["base", "bin", "python"])
         );
     }
@@ -1543,12 +1688,31 @@ mod tests {
     fn test_conda_env_interpreter() {
         let base = OsStr::new("base");
         assert_eq!(
-            conda_env_interpreter(&base, true),
+            conda_env_interpreter(base, true),
             PathBuf::from_iter(&["base", "python.exe"])
         );
         assert_eq!(
-            conda_env_interpreter(&base, false),
+            conda_env_interpreter(base, false),
             PathBuf::from_iter(&["base", "bin", "python"])
         );
     }
+
+    #[test]
+    fn test_not_cross_compiling() {
+        assert!(
+            cross_compiling("aarch64-apple-darwin", "x86_64", "apple", "darwin")
+                .unwrap()
+                .is_none()
+        );
+        assert!(
+            cross_compiling("x86_64-apple-darwin", "aarch64", "apple", "darwin")
+                .unwrap()
+                .is_none()
+        );
+        assert!(
+            cross_compiling("x86_64-unknown-linux-gnu", "x86_64", "unknown", "linux")
+                .unwrap()
+                .is_none()
+        );
+    }
 }

pyo3-build-config/src/lib.rs

@@ -1,8 +1,7 @@
 //! Configuration used by PyO3 for conditional support of varying Python versions.
 //!
-//! This crate exposes two functions, [`use_pyo3_cfgs`] and [`add_extension_module_link_args`],
+//! This crate exposes functionality to be called from build scripts to simplify building crates
-//! which are intended to be called from build scripts to simplify building crates which depend on
+//! which depend on PyO3.
-//! PyO3.
 //!
 //! It used internally by the PyO3 crate's build script to apply the same configuration.
 
@@ -15,7 +14,10 @@ use std::io::Cursor;
 #[cfg(feature = "resolve-config")]
 use once_cell::sync::OnceCell;
 
-pub use impl_::{BuildFlag, BuildFlags, InterpreterConfig, PythonImplementation, PythonVersion};
+pub use impl_::{
+    cross_compiling, find_all_sysconfigdata, parse_sysconfigdata, BuildFlag, BuildFlags,
+    CrossCompileConfig, InterpreterConfig, PythonImplementation, PythonVersion,
+};
 
 /// Adds all the [`#[cfg]` flags](index.html) to the current compilation.
 ///

pyo3-ffi/Cargo.toml

@@ -0,0 +1,39 @@
+[package]
+name = "pyo3-ffi"
+version = "0.15.1"
+description = "Python-API bindings for the PyO3 ecosystem"
+authors = ["PyO3 Project and Contributors <https://github.com/PyO3>"]
+keywords = ["pyo3", "python", "cpython", "ffi"]
+homepage = "https://github.com/pyo3/pyo3"
+repository = "https://github.com/pyo3/pyo3"
+categories = ["api-bindings", "development-tools::ffi"]
+license = "Apache-2.0"
+edition = "2018"
+
+[dependencies]
+libc = "0.2.62"
+
+[features]
+
+default = []
+
+# Use this feature when building an extension module.
+# It tells the linker to keep the python symbols unresolved,
+# so that the module can also be used with statically linked python interpreters.
+extension-module = []
+
+# Use the Python limited API. See https://www.python.org/dev/peps/pep-0384/ for more.
+abi3 = ["pyo3-build-config/abi3"]
+
+# With abi3, we can manually set the minimum Python version.
+abi3-py37 = ["abi3-py38", "pyo3-build-config/abi3-py37"]
+abi3-py38 = ["abi3-py39", "pyo3-build-config/abi3-py38"]
+abi3-py39 = ["abi3-py310", "pyo3-build-config/abi3-py39"]
+abi3-py310 = ["abi3", "pyo3-build-config/abi3-py310"]
+
+
+
+[build-dependencies]
+pyo3-build-config = { path = "../pyo3-build-config", version = "0.15.1", features = ["resolve-config"] }
+
+

pyo3-ffi/README.md

@@ -0,0 +1,201 @@
+# pyo3-ffi
+
+This crate provides [Rust](https://www.rust-lang.org/) FFI declarations for Python 3.
+It supports both the stable and the unstable component of the ABI through the use of cfg flags.
+Python Versions 3.7+ are supported.
+It is meant for advanced users only - regular PyO3 users shouldn't
+need to interact with this crate at all.
+
+The contents of this crate are not documented here, as it would entail
+basically copying the documentation from CPython. Consult the [Python/C API Reference
+Manual][capi] for up-to-date documentation.
+
+# Minimum supported Rust and Python versions
+
+PyO3 supports the following software versions:
+  - Python 3.7 and up (CPython and PyPy)
+  - Rust 1.48 and up
+
+# Example: Building Python Native modules
+
+PyO3 can be used to generate a native Python module. The easiest way to try this out for the
+first time is to use [`maturin`]. `maturin` is a tool for building and publishing Rust-based
+Python packages with minimal configuration. The following steps set up some files for an example
+Python module, install `maturin`, and then show how to build and import the Python module.
+
+First, create a new folder (let's call it `string_sum`) containing the following two files:
+
+**`Cargo.toml`**
+
+```toml
+[lib]
+name = "string_sum"
+# "cdylib" is necessary to produce a shared library for Python to import from.
+#
+# Downstream Rust code (including code in `bin/`, `examples/`, and `tests/`) will not be able
+# to `use string_sum;` unless the "rlib" or "lib" crate type is also included, e.g.:
+# crate-type = ["cdylib", "rlib"]
+crate-type = ["cdylib"]
+
+[dependencies.pyo3-ffi]
+version = "*"
+features = ["extension-module"]
+```
+
+**`src/lib.rs`**
+```rust
+use std::mem::transmute;
+use std::os::raw::c_char;
+
+use pyo3_ffi::*;
+
+#[allow(non_snake_case)]
+#[no_mangle]
+pub unsafe extern "C" fn PyInit_string_sum() -> *mut PyObject {
+    let init = PyModuleDef {
+        m_base: PyModuleDef_HEAD_INIT,
+        m_name: "string_sum\0".as_ptr() as *const c_char,
+        m_doc: std::ptr::null(),
+        m_size: 0,
+        m_methods: std::ptr::null_mut(),
+        m_slots: std::ptr::null_mut(),
+        m_traverse: None,
+        m_clear: None,
+        m_free: None,
+    };
+
+    let mptr = PyModule_Create(Box::into_raw(Box::new(init)));
+    let version = env!("CARGO_PKG_VERSION");
+    PyModule_AddObject(
+        mptr,
+        "__version__\0".as_ptr() as *const c_char,
+        PyUnicode_FromStringAndSize(version.as_ptr() as *const c_char, version.len() as isize),
+    );
+
+    // It's necessary to transmute `sum_as_string` here because functions marked with `METH_FASTCALL`
+    // have a different signature. However the `PyMethodDef` struct currently represents all
+    // functions as a `PyCFunction`. The python interpreter will cast the function pointer back 
+    // to `_PyCFunctionFast`.
+    let wrapped_sum_as_string = PyMethodDef {
+        ml_name: "sum_as_string\0".as_ptr() as *const c_char,
+        ml_meth: Some(transmute::<_PyCFunctionFast, PyCFunction>(sum_as_string)),
+        ml_flags: METH_FASTCALL,
+        ml_doc: "returns the sum of two integers as a string\0".as_ptr() as *const c_char,
+    };
+
+    // PyModule_AddObject can technically fail.
+    // For more involved applications error checking may be necessary
+    PyModule_AddObject(
+        mptr,
+        "sum_as_string\0".as_ptr() as *const c_char,
+        PyCFunction_NewEx(
+            Box::into_raw(Box::new(wrapped_sum_as_string)),
+            std::ptr::null_mut(),
+            PyUnicode_InternFromString("string_sum\0".as_ptr() as *const c_char),
+        ),
+    );
+
+    let all = ["__all__\0", "__version__\0", "sum_as_string\0"];
+
+    let pyall = PyTuple_New(all.len() as isize);
+    for (i, obj) in all.iter().enumerate() {
+        PyTuple_SET_ITEM(
+            pyall,
+            i as isize,
+            PyUnicode_InternFromString(obj.as_ptr() as *const c_char),
+        )
+    }
+
+    PyModule_AddObject(mptr, "__all__\0".as_ptr() as *const c_char, pyall);
+
+    mptr
+}
+
+pub unsafe extern "C" fn sum_as_string(
+    _self: *mut PyObject,
+    args: *mut *mut PyObject,
+    nargs: Py_ssize_t,
+) -> *mut PyObject {
+    if nargs != 2 {
+        return raise_type_error("sum_as_string() expected 2 positional arguments");
+    }
+
+    let arg1 = *args;
+    if PyLong_Check(arg1) == 0 {
+        return raise_type_error("sum_as_string() expected an int for positional argument 1");
+    }
+
+    let arg1 = PyLong_AsLong(arg1);
+    if !PyErr_Occurred().is_null() {
+        return ptr::null()
+    }
+
+    let arg2 = *args.add(1);
+    if PyLong_Check(arg2) == 0 {
+        return raise_type_error("sum_as_string() expected an int for positional argument 2");
+    }
+
+    let arg2 = PyLong_AsLong(arg2);
+    if !PyErr_Occurred().is_null() {
+        return ptr::null()
+    }
+
+
+
+    let res = (arg1 + arg2).to_string();
+    PyUnicode_FromStringAndSize(res.as_ptr() as *const c_char, res.len() as isize)
+}
+
+#[cold]
+#[inline(never)]
+fn raise_type_error(msg: &str) -> *mut PyObject {
+    unsafe {
+        let err_msg =
+            PyUnicode_FromStringAndSize(msg.as_ptr() as *const c_char, msg.len() as isize);
+        PyErr_SetObject(PyExc_TypeError, err_msg);
+        Py_DECREF(err_msg);
+    };
+    std::ptr::null_mut()
+}
+```
+
+With those two files in place, now `maturin` needs to be installed. This can be done using
+Python's package manager `pip`. First, load up a new Python `virtualenv`, and install `maturin`
+into it:
+```bash
+$ cd string_sum
+$ python -m venv .env
+$ source .env/bin/activate
+$ pip install maturin
+```
+
+Now build and execute the module:
+```bash
+$ maturin develop
+# lots of progress output as maturin runs the compilation...
+$ python
+>>> import string_sum
+>>> string_sum.sum_as_string(5, 20)
+'25'
+```
+
+As well as with `maturin`, it is possible to build using [setuptools-rust] or
+[manually][manual_builds]. Both offer more flexibility than `maturin` but require further
+configuration.
+
+
+While most projects use the safe wrapper provided by PyO3,
+you can take a look at the [`orjson`] library as an example on how to use `pyo3-ffi` directly.
+For those well versed in C and Rust the [tutorials] from the CPython documentation
+can be easily converted to rust as well.
+
+[tutorials]: https://docs.python.org/3/extending/
+[`orjson`]: https://github.com/ijl/orjson
+[capi]: https://docs.python.org/3/c-api/index.html
+[`maturin`]: https://github.com/PyO3/maturin "Build and publish crates with pyo3, rust-cpython and cffi bindings as well as rust binaries as python packages"
+[`pyo3-build-config`]: https://docs.rs/pyo3-build-config
+[feature flags]: https://doc.rust-lang.org/cargo/reference/features.html "Features - The Cargo Book"
+[manual_builds]: https://pyo3.rs/latest/building_and_distribution.html#manual-builds "Manual builds - Building and Distribution - PyO3 user guide"
+[setuptools-rust]: https://github.com/PyO3/setuptools-rust "Setuptools plugin for Rust extensions"
+[PEP 384]: https://www.python.org/dev/peps/pep-0384 "PEP 384 -- Defining a Stable ABI"
+[Features chapter of the guide]: https://pyo3.rs/latest/features.html#features-reference "Features Reference - PyO3 user guide"

pyo3-ffi/build.rs

@@ -0,0 +1,119 @@
+use pyo3_build_config::{
+    bail, ensure,
+    pyo3_build_script_impl::{
+        cargo_env_var, env_var, errors::Result, resolve_interpreter_config, InterpreterConfig,
+        PythonVersion,
+    },
+};
+
+/// Minimum Python version PyO3 supports.
+const MINIMUM_SUPPORTED_VERSION: PythonVersion = PythonVersion { major: 3, minor: 7 };
+
+fn ensure_python_version(interpreter_config: &InterpreterConfig) -> Result<()> {
+    ensure!(
+        interpreter_config.version >= MINIMUM_SUPPORTED_VERSION,
+        "the configured Python interpreter version ({}) is lower than PyO3's minimum supported version ({})",
+        interpreter_config.version,
+        MINIMUM_SUPPORTED_VERSION,
+    );
+
+    Ok(())
+}
+
+fn ensure_target_pointer_width(interpreter_config: &InterpreterConfig) -> Result<()> {
+    if let Some(pointer_width) = interpreter_config.pointer_width {
+        // Try to check whether the target architecture matches the python library
+        let rust_target = match cargo_env_var("CARGO_CFG_TARGET_POINTER_WIDTH")
+            .unwrap()
+            .as_str()
+        {
+            "64" => 64,
+            "32" => 32,
+            x => bail!("unexpected Rust target pointer width: {}", x),
+        };
+
+        ensure!(
+            rust_target == pointer_width,
+            "your Rust target architecture ({}-bit) does not match your python interpreter ({}-bit)",
+            rust_target,
+            pointer_width
+        );
+    }
+    Ok(())
+}
+
+fn emit_link_config(interpreter_config: &InterpreterConfig) -> Result<()> {
+    let target_os = cargo_env_var("CARGO_CFG_TARGET_OS").unwrap();
+    let is_extension_module = cargo_env_var("CARGO_FEATURE_EXTENSION_MODULE").is_some();
+    if target_os == "windows" || target_os == "android" || !is_extension_module {
+        // windows and android - always link
+        // other systems - only link if not extension module
+        println!(
+            "cargo:rustc-link-lib={link_model}{alias}{lib_name}",
+            link_model = if interpreter_config.shared {
+                ""
+            } else {
+                "static="
+            },
+            alias = if target_os == "windows" {
+                "pythonXY:"
+            } else {
+                ""
+            },
+            lib_name = interpreter_config.lib_name.as_ref().ok_or(
+                "attempted to link to Python shared library but config does not contain lib_name"
+            )?,
+        );
+        if let Some(lib_dir) = &interpreter_config.lib_dir {
+            println!("cargo:rustc-link-search=native={}", lib_dir);
+        }
+    }
+
+    Ok(())
+}
+
+/// Prepares the PyO3 crate for compilation.
+///
+/// This loads the config from pyo3-build-config and then makes some additional checks to improve UX
+/// for users.
+///
+/// Emits the cargo configuration based on this config as well as a few checks of the Rust compiler
+/// version to enable features which aren't supported on MSRV.
+fn configure_pyo3() -> Result<()> {
+    let interpreter_config = resolve_interpreter_config()?;
+
+    if env_var("PYO3_PRINT_CONFIG").map_or(false, |os_str| os_str == "1") {
+        print_config_and_exit(&interpreter_config);
+    }
+
+    ensure_python_version(&interpreter_config)?;
+    ensure_target_pointer_width(&interpreter_config)?;
+
+    if !interpreter_config.suppress_build_script_link_lines {
+        emit_link_config(&interpreter_config)?;
+    }
+
+    interpreter_config.emit_pyo3_cfgs();
+
+    // Extra lines come last, to support last write wins.
+    for line in &interpreter_config.extra_build_script_lines {
+        println!("{}", line);
+    }
+
+    Ok(())
+}
+
+fn print_config_and_exit(config: &InterpreterConfig) {
+    println!("\n-- PYO3_PRINT_CONFIG=1 is set, printing configuration and halting compile --");
+    config
+        .to_writer(&mut std::io::stdout())
+        .expect("failed to print config to stdout");
+    std::process::exit(101);
+}
+
+fn main() {
+    if let Err(e) = configure_pyo3() {
+        eprintln!("error: {}", e.report());
+        std::process::exit(1)
+    }
+}

pyo3-ffi/src/abstract_.rs

@@ -1,5 +1,5 @@
-use crate::ffi::object::*;
+use crate::object::*;
-use crate::ffi::pyport::Py_ssize_t;
+use crate::pyport::Py_ssize_t;
 use std::os::raw::{c_char, c_int};
 use std::ptr;
 
@@ -96,10 +96,9 @@ extern "C" {
 #[cfg(not(any(Py_LIMITED_API, PyPy)))]
 #[inline]
 pub unsafe fn PyIter_Check(o: *mut PyObject) -> c_int {
-    (match (*crate::ffi::Py_TYPE(o)).tp_iternext {
+    (match (*crate::Py_TYPE(o)).tp_iternext {
         Some(tp_iternext) => {
-            tp_iternext as *const std::os::raw::c_void
+            tp_iternext as *const std::os::raw::c_void != crate::_PyObject_NextNotImplemented as _
-                != crate::ffi::_PyObject_NextNotImplemented as _
         }
         None => false,
     }) as c_int

pyo3-ffi/src/bltinmodule.rs

@@ -1,4 +1,4 @@
-use crate::ffi::object::PyTypeObject;
+use crate::object::PyTypeObject;
 
 #[cfg_attr(windows, link(name = "pythonXY"))]
 extern "C" {

pyo3-ffi/src/boolobject.rs

@@ -1,5 +1,5 @@
-use crate::ffi::longobject::PyLongObject;
+use crate::longobject::PyLongObject;
-use crate::ffi::object::*;
+use crate::object::*;
 use std::os::raw::{c_int, c_long};
 
 #[cfg_attr(windows, link(name = "pythonXY"))]

pyo3-ffi/src/buffer.rs

@@ -0,0 +1,133 @@
+use crate::object::PyObject;
+use crate::pyport::Py_ssize_t;
+use std::os::raw::{c_char, c_int, c_void};
+use std::ptr;
+
+#[cfg(PyPy)]
+const Py_MAX_NDIMS: usize = 36;
+
+#[repr(C)]
+#[derive(Copy, Clone)]
+pub struct Py_buffer {
+    pub buf: *mut c_void,
+    /// Owned reference
+    pub obj: *mut crate::PyObject,
+    pub len: Py_ssize_t,
+    pub itemsize: Py_ssize_t,
+    pub readonly: c_int,
+    pub ndim: c_int,
+    pub format: *mut c_char,
+    pub shape: *mut Py_ssize_t,
+    pub strides: *mut Py_ssize_t,
+    pub suboffsets: *mut Py_ssize_t,
+    pub internal: *mut c_void,
+    #[cfg(PyPy)]
+    pub flags: c_int,
+    #[cfg(PyPy)]
+    pub _strides: [Py_ssize_t; Py_MAX_NDIMS],
+    #[cfg(PyPy)]
+    pub _shape: [Py_ssize_t; Py_MAX_NDIMS],
+}
+
+impl Py_buffer {
+    pub const fn new() -> Self {
+        Py_buffer {
+            buf: ptr::null_mut(),
+            obj: ptr::null_mut(),
+            len: 0,
+            itemsize: 0,
+            readonly: 0,
+            ndim: 0,
+            format: ptr::null_mut(),
+            shape: ptr::null_mut(),
+            strides: ptr::null_mut(),
+            suboffsets: ptr::null_mut(),
+            internal: ptr::null_mut(),
+            #[cfg(PyPy)]
+            flags: 0,
+            #[cfg(PyPy)]
+            _strides: [0; Py_MAX_NDIMS],
+            #[cfg(PyPy)]
+            _shape: [0; Py_MAX_NDIMS],
+        }
+    }
+}
+
+/* Return 1 if the getbuffer function is available, otherwise return 0. */
+extern "C" {
+    #[cfg(not(PyPy))]
+    pub fn PyObject_CheckBuffer(obj: *mut PyObject) -> c_int;
+
+    #[cfg_attr(PyPy, link_name = "PyPyObject_GetBuffer")]
+    pub fn PyObject_GetBuffer(obj: *mut PyObject, view: *mut Py_buffer, flags: c_int) -> c_int;
+    #[cfg_attr(PyPy, link_name = "PyPyBuffer_GetPointer")]
+    pub fn PyBuffer_GetPointer(view: *const Py_buffer, indices: *const Py_ssize_t) -> *mut c_void;
+    #[cfg_attr(PyPy, link_name = "PyPyBuffer_SizeFromFormat")]
+    pub fn PyBuffer_SizeFromFormat(format: *const c_char) -> Py_ssize_t;
+    #[cfg_attr(PyPy, link_name = "PyPyBuffer_ToContiguous")]
+    pub fn PyBuffer_ToContiguous(
+        buf: *mut c_void,
+        view: *const Py_buffer,
+        len: Py_ssize_t,
+        order: c_char,
+    ) -> c_int;
+    #[cfg_attr(PyPy, link_name = "PyPyBuffer_FromContiguous")]
+    pub fn PyBuffer_FromContiguous(
+        view: *const Py_buffer,
+        buf: *const c_void,
+        len: Py_ssize_t,
+        order: c_char,
+    ) -> c_int;
+    pub fn PyObject_CopyData(dest: *mut PyObject, src: *mut PyObject) -> c_int;
+    #[cfg_attr(PyPy, link_name = "PyPyBuffer_IsContiguous")]
+    pub fn PyBuffer_IsContiguous(view: *const Py_buffer, fort: c_char) -> c_int;
+    pub fn PyBuffer_FillContiguousStrides(
+        ndims: c_int,
+        shape: *mut Py_ssize_t,
+        strides: *mut Py_ssize_t,
+        itemsize: c_int,
+        fort: c_char,
+    );
+    #[cfg_attr(PyPy, link_name = "PyPyBuffer_FillInfo")]
+    pub fn PyBuffer_FillInfo(
+        view: *mut Py_buffer,
+        o: *mut PyObject,
+        buf: *mut c_void,
+        len: Py_ssize_t,
+        readonly: c_int,
+        flags: c_int,
+    ) -> c_int;
+    #[cfg_attr(PyPy, link_name = "PyPyBuffer_Release")]
+    pub fn PyBuffer_Release(view: *mut Py_buffer);
+}
+
+/// Maximum number of dimensions
+pub const PyBUF_MAX_NDIM: c_int = 64;
+
+/* Flags for getting buffers */
+pub const PyBUF_SIMPLE: c_int = 0;
+pub const PyBUF_WRITABLE: c_int = 0x0001;
+/* we used to include an E, backwards compatible alias */
+pub const PyBUF_WRITEABLE: c_int = PyBUF_WRITABLE;
+pub const PyBUF_FORMAT: c_int = 0x0004;
+pub const PyBUF_ND: c_int = 0x0008;
+pub const PyBUF_STRIDES: c_int = 0x0010 | PyBUF_ND;
+pub const PyBUF_C_CONTIGUOUS: c_int = 0x0020 | PyBUF_STRIDES;
+pub const PyBUF_F_CONTIGUOUS: c_int = 0x0040 | PyBUF_STRIDES;
+pub const PyBUF_ANY_CONTIGUOUS: c_int = 0x0080 | PyBUF_STRIDES;
+pub const PyBUF_INDIRECT: c_int = 0x0100 | PyBUF_STRIDES;
+
+pub const PyBUF_CONTIG: c_int = PyBUF_ND | PyBUF_WRITABLE;
+pub const PyBUF_CONTIG_RO: c_int = PyBUF_ND;
+
+pub const PyBUF_STRIDED: c_int = PyBUF_STRIDES | PyBUF_WRITABLE;
+pub const PyBUF_STRIDED_RO: c_int = PyBUF_STRIDES;
+
+pub const PyBUF_RECORDS: c_int = PyBUF_STRIDES | PyBUF_WRITABLE | PyBUF_FORMAT;
+pub const PyBUF_RECORDS_RO: c_int = PyBUF_STRIDES | PyBUF_FORMAT;
+
+pub const PyBUF_FULL: c_int = PyBUF_INDIRECT | PyBUF_WRITABLE | PyBUF_FORMAT;
+pub const PyBUF_FULL_RO: c_int = PyBUF_INDIRECT | PyBUF_FORMAT;
+
+pub const PyBUF_READ: c_int = 0x100;
+pub const PyBUF_WRITE: c_int = 0x200;

pyo3-ffi/src/bytearrayobject.rs

@@ -1,5 +1,5 @@
-use crate::ffi::object::*;
+use crate::object::*;
-use crate::ffi::pyport::Py_ssize_t;
+use crate::pyport::Py_ssize_t;
 use std::os::raw::{c_char, c_int};
 
 #[cfg_attr(windows, link(name = "pythonXY"))]