Organize examples + add __call__ example (#2043)

* Add decorator example crate and split off chapter

* Move not-examples to their own folder

* Add some readme's

* Make black happy

* Make clippy happy

* Add decorator example crate and split off chapter

* Fix ci

* Add empty workspace key

* Try fix ci

* fix ci

* reuse target dir for examples CI

* add pytests folder to makefile recipes

* fix ci, try 2

* add missing pyproject.toml

* remove TOX_TESTENV_PASSENV from Makefile

Co-authored-by: David Hewitt <1939362+davidhewitt@users.noreply.github.com>

.github/workflows/bench.yml

@@ -83,7 +83,7 @@ jobs:
 
       - name: Run benchmarks
         run: |
-          cd examples/pyo3-benchmarks
+          cd pytests/pyo3-benchmarks
           python -m pip install -r requirements-dev.txt
           python setup.py develop
           pytest --benchmark-json ../../output.json

.github/workflows/ci.yml

@@ -221,17 +221,14 @@ jobs:
       - name: Test build config
         run: cargo test --manifest-path=pyo3-build-config/Cargo.toml
 
-      - name: Install python test dependencies
-        run: python -m pip install -U pip tox
-
-      - name: Test example extension modules
+      - name: Test python examples and tests
         shell: bash
         run: |
-          for example_dir in examples/*/; do
-              tox -c $example_dir -e py
-          done
+          python -m pip install -U pip tox
+          make test_py
         env:
-          TOX_TESTENV_PASSENV: "CARGO_BUILD_TARGET"
+          CARGO_TARGET_DIR: ${{ github.workspace }}
+          TOX_TESTENV_PASSENV: "CARGO_BUILD_TARGET CARGO_TARGET_DIR"
 
       - name: Test cross compilation
         if: ${{ matrix.platform.os == 'ubuntu-latest' && matrix.python-version == '3.9' }}

.gitignore

@@ -14,7 +14,6 @@ dist/
 .eggs/
 venv*
 guide/book/
-examples/*/py*
 *.so
 *.out
 *.egg-info

Cargo.toml

@@ -119,11 +119,7 @@ harness = false
 members = [
     "pyo3-macros",
     "pyo3-macros-backend",
-    "examples/pyo3-benchmarks",
-    "examples/pyo3-pytests",
-    "examples/maturin-starter",
-    "examples/setuptools-rust-starter",
-    "examples/word-count"
+    "examples"
 ]
 
 [package.metadata.docs.rs]

Makefile

@@ -12,13 +12,15 @@ test: lint test_py
 	cargo test --features="abi3 $(ALL_ADDITIVE_FEATURES)"
 
 test_py:
-	for example in examples/*/; do TOX_TESTENV_PASSENV=RUSTUP_HOME tox -e py -c $$example || exit 1; done
+	@for example in examples/*/tox.ini; do echo "-- Running tox for $$example --"; tox -e py -c $$example || exit 1; echo ""; done
+	@for package in pytests/*/tox.ini; do echo "-- Running tox for $$package --"; tox -e py -c $$package || exit 1; echo ""; done
 
 fmt_py:
 	black . --check
 
 fmt_rust:
 	cargo fmt --all -- --check
+	for package in pytests/*/; do cargo fmt --manifest-path $$package/Cargo.toml -- --check || exit 1; done
 
 fmt: fmt_rust fmt_py
 	@true
@@ -27,11 +29,14 @@ clippy:
 	cargo clippy --features="$(ALL_ADDITIVE_FEATURES)" --all-targets --workspace -- -Dwarnings
 	cargo clippy --features="abi3 $(ALL_ADDITIVE_FEATURES)" --all-targets --workspace -- -Dwarnings
 	for example in examples/*/; do cargo clippy --manifest-path $$example/Cargo.toml -- -Dwarnings || exit 1; done
+	for package in pytests/*/; do cargo clippy --manifest-path $$package/Cargo.toml -- -Dwarnings || exit 1; done
 
 lint: fmt clippy
 	@true
 
 publish: test
+	cargo publish --manifest-path pyo3-build-config/Cargo.toml
+	sleep 10
 	cargo publish --manifest-path pyo3-macros-backend/Cargo.toml
 	sleep 10  # wait for crates.io to update
 	cargo publish --manifest-path pyo3-macros/Cargo.toml

examples/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "examples"
+version = "0.0.0"
+publish = false
+edition = "2018"
+
+[dev-dependencies]
+pyo3 = { version = "0.15.1", path = "..", features = ["auto-initialize", "extension-module"] }
+
+[[example]]
+name = "decorator"
+path = "decorator/src/lib.rs"
+crate_type = ["cdylib"]

examples/README.md

@@ -1,13 +1,12 @@
 # PyO3 Examples
 
-These examples are a collection of toy extension modules built with PyO3. They are all tested using `tox` in PyO3's CI.
+These example crates are a collection of toy extension modules built with PyO3. They are all tested using `tox` in PyO3's CI.
 
 Below is a brief description of each of these:
 
 | Example | Description |
 | ------- | ----------- |
+| `decorator` | A project showcasing the example from the [Emulating callable objects](https://pyo3.rs/latest/class/call.html) chapter of the guide. |
 | `maturin-starter` | A template project which is configured to use [`maturin`](https://github.com/PyO3/maturin) for development. |
 | `setuptools-rust-starter` | A template project which is configured to use [`setuptools_rust`](https://github.com/PyO3/setuptools-rust/) for development. |
 | `word-count` | A quick performance comparison between word counter implementations written in each of Rust and Python. |
-| `pyo3-benchmarks` | A project containing some benchmarks of PyO3 functionality called from Python. |
-| `pyo3-pytests` | A project containing some tests of PyO3 functionality called from Python. |

examples/decorator/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "decorator"
+version = "0.1.0"
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[lib]
+name = "decorator"
+crate-type = ["cdylib"]
+
+[dependencies.pyo3]
+# If you copy this example, you should uncomment this...
+# version = "0.15.1"
+
+# ...and delete this path
+path = "../.."
+features = ["extension-module"]
+
+[workspace]

examples/decorator/README.md

@@ -0,0 +1,25 @@
+# decorator
+
+A project showcasing the example from the [Emulating callable objects](https://pyo3.rs/latest/class/call.html) chapter of the guide.
+
+## Building and Testing
+
+To build this package, first install `maturin`:
+
+```shell
+pip install maturin
+```
+
+To build and test use `maturin develop`:
+
+```shell
+pip install -r requirements-dev.txt
+maturin develop
+pytest
+```
+
+Alternatively, install tox and run the tests inside an isolated environment:
+
+```shell
+tox -e py
+```

examples/decorator/pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["maturin>=0.12,<0.13"]
+build-backend = "maturin"

examples/decorator/requirements-dev.txt

@@ -0,0 +1,2 @@
+pytest>=3.5.0
+pip>=21.3

examples/decorator/src/lib.rs

@@ -0,0 +1,54 @@
+use pyo3::prelude::*;
+use pyo3::types::{PyDict, PyTuple};
+
+/// A function decorator that keeps track how often it is called.
+///
+/// It otherwise doesn't do anything special.
+#[pyclass(name = "Counter")]
+pub struct PyCounter {
+    // We use `#[pyo3(get)]` so that python can read the count but not mutate it.
+    #[pyo3(get)]
+    count: u64,
+
+    // This is the actual function being wrapped.
+    wraps: Py<PyAny>,
+}
+
+#[pymethods]
+impl PyCounter {
+    // Note that we don't validate whether `wraps` is actually callable.
+    //
+    // While we could use `PyAny::is_callable` for that, it has some flaws:
+    //    1. It doesn't guarantee the object can actually be called successfully
+    //    2. We still need to handle any exceptions that the function might raise
+    #[new]
+    fn __new__(wraps: Py<PyAny>) -> Self {
+        PyCounter { count: 0, wraps }
+    }
+
+    #[args(args = "*", kwargs = "**")]
+    fn __call__(
+        &mut self,
+        py: Python,
+        args: &PyTuple,
+        kwargs: Option<&PyDict>,
+    ) -> PyResult<Py<PyAny>> {
+        self.count += 1;
+        let name = self.wraps.getattr(py, "__name__")?;
+
+        println!("{} has been called {} time(s).", name, self.count);
+
+        // After doing something, we finally forward the call to the wrapped function
+        let ret = self.wraps.call(py, args, kwargs)?;
+
+        // We could do something with the return value of
+        // the function before returning it
+        Ok(ret)
+    }
+}
+
+#[pymodule]
+pub fn decorator(_py: Python, module: &PyModule) -> PyResult<()> {
+    module.add_class::<PyCounter>()?;
+    Ok(())
+}

examples/decorator/tests/test_.py

@@ -0,0 +1,40 @@
+from decorator import Counter
+
+
+def test_no_args():
+    @Counter
+    def say_hello():
+        print("hello")
+
+    say_hello()
+    say_hello()
+    say_hello()
+    say_hello()
+
+    assert say_hello.count == 4
+
+
+def test_arg():
+    @Counter
+    def say_hello(name):
+        print(f"hello {name}")
+
+    say_hello("a")
+    say_hello("b")
+    say_hello("c")
+    say_hello("d")
+
+    assert say_hello.count == 4
+
+
+def test_default_arg():
+    @Counter
+    def say_hello(name="default"):
+        print(f"hello {name}")
+
+    say_hello("a")
+    say_hello()
+    say_hello("c")
+    say_hello()
+
+    assert say_hello.count == 4

examples/maturin-starter/Cargo.toml

@@ -11,6 +11,8 @@ edition = "2018"
 path = "../../"
 features = ["extension-module"]
 
+[workspace]
+
 [lib]
 name = "maturin_starter"
 crate-type = ["cdylib"]

examples/maturin-starter/pyproject.toml

@@ -1,3 +1,3 @@
 [build-system]
-requires = ["maturin>=0.10,<0.11"]
+requires = ["maturin>=0.12,<0.13"]
 build-backend = "maturin"

examples/setuptools-rust-starter/Cargo.toml

@@ -11,6 +11,8 @@ edition = "2018"
 path = "../../"
 features = ["extension-module"]
 
+[workspace]
+
 [lib]
 name = "setuptools_rust_starter"
 crate-type = ["cdylib"]
@@ -25,3 +27,4 @@ classifier=[
     "Operating System :: POSIX",
     "Operating System :: MacOS :: MacOS X",
 ]
+

examples/word-count/Cargo.toml

@@ -8,6 +8,8 @@ edition = "2018"
 rayon = "1.0.2"
 pyo3 = { path = "../..", features = ["extension-module"] }
 
+[workspace]
+
 [lib]
 name = "word_count"
 crate-type = ["cdylib"]

guide/src/SUMMARY.md

@@ -8,6 +8,7 @@
 - [Python Functions](function.md)
 - [Python Classes](class.md)
   - [Class customizations](class/protocols.md)
+    - [Emulating callable objects](class/call.md)
 - [Type Conversions](conversions.md)
   - [Mapping of Rust types to Python types](conversions/tables.md)]
   - [Conversion traits](conversions/traits.md)]

guide/src/class/call.md

@@ -0,0 +1,67 @@
+# Emulating callable objects
+
+Classes can be callable if they have a `#[pymethod]` named `__call__`.
+This allows instances of a class to behave similar to functions.
+
+This method's signature must look like `__call__(<self>, ...) -> object` - here,
+ any argument list can be defined as for normal pymethods
+
+### Example: Implementing a call counter
+
+The following pyclass is a basic decorator - its constructor takes a Python object
+as argument and calls that object when called. An equivalent Python implementation
+is linked at the end.
+
+An example crate containing this pyclass can be found [here](https://github.com/PyO3/pyo3/tree/main/examples/decorator)
+
+```rust
+{{#include ../../../examples/decorator/src/lib.rs}}
+```
+
+Python code:
+
+```python
+{{#include ../../../examples/decorator/src/test.py}}
+```
+
+Output:
+
+```text
+say_hello has been called 1 time(s).
+hello
+say_hello has been called 2 time(s).
+hello
+say_hello has been called 3 time(s).
+hello
+say_hello has been called 4 time(s).
+hello
+```
+
+#### Pure Python implementation
+
+A Python implementation of this looks similar to the Rust version:
+
+```python
+class Counter:
+    def __init__(self, wraps):
+        self.count = 0
+        self.wraps = wraps
+
+    def __call__(self, *args, **kwargs):
+        self.count += 1
+        print(f"{self.wraps.__name__} has been called {self.count} time(s)")
+        self.wraps(*args, **kwargs)
+```
+
+Note that it can also be implemented as a higher order function:
+
+```python
+def Counter(wraps):
+    count = 0
+    def call(*args, **kwargs):
+        nonlocal count
+        count += 1
+        print(f"{wraps.__name__} has been called {count} time(s)")
+        return wraps(*args, **kwargs)
+    return call
+```

guide/src/class/protocols.md

@@ -2,14 +2,14 @@
 
 Python's object model defines several protocols for different object behavior, such as the sequence, mapping, and number protocols. You may be familiar with implementing these protocols in Python classes by "magic" methods, such as `__str__` or `__repr__`. Because of the double-underscores surrounding their name, these are also known as "dunder" methods.
 
-In the Python C-API which PyO3 is implemented upon, many of these magic methods have to be placed into special "slots" on the class type object. as already covered in the previous section. There are two ways in which this can be done:
+In the Python C-API which PyO3 is implemented upon, many of these magic methods have to be placed into special "slots" on the class type object, as covered in the previous section. There are two ways in which this can be done:
 
  - [Experimental for PyO3 0.15, may change slightly in PyO3 0.16] In `#[pymethods]`, if the name of the method is a recognised magic method, PyO3 will place it in the type object automatically.
  - [Stable, but expected to be deprecated in PyO3 0.16] In special traits combined with the `#[pyproto]` attribute.
 
 (There are also many magic methods which don't have a special slot, such as `__dir__`. These methods can be implemented as normal in `#[pymethods]`.)
 
-This chapter of the guide has a section on each of these solutions in turn:
+This chapter gives a brief overview of the available methods. An in depth example is given in the following sub-chapters.
 
 ### Magic methods in `#[pymethods]`
 
@@ -75,71 +75,6 @@ given signatures should be interpreted as follows:
   - `__call__(<self>, ...) -> object` - here, any argument list can be defined
     as for normal `pymethods`
 
-##### Example: Callable objects
-
-Custom classes can be callable if they have a `#[pymethod]` named `__call__`.
-
-The following pyclass is a basic decorator - its constructor takes a Python object
-as argument and calls that object when called.
-
-```rust
-# use pyo3::prelude::*;
-# use pyo3::types::{PyDict, PyTuple};
-#
-#[pyclass(name = "counter")]
-struct PyCounter {
-    count: u64,
-    wraps: Py<PyAny>,
-}
-
-#[pymethods]
-impl PyCounter {
-    #[new]
-    fn __new__(wraps: Py<PyAny>) -> Self {
-        PyCounter { count: 0, wraps }
-    }
-    #[args(args="*", kwargs="**")]
-    fn __call__(
-        &mut self,
-        py: Python,
-        args: &PyTuple,
-        kwargs: Option<&PyDict>,
-    ) -> PyResult<Py<PyAny>> {
-        self.count += 1;
-        let name = self.wraps.getattr(py, "__name__").unwrap();
-
-        println!("{} has been called {} time(s).", name, self.count);
-        self.wraps.call(py, args, kwargs)
-    }
-}
-```
-
-Python code:
-
-```python
-@counter
-def say_hello():
-    print("hello")
-
-say_hello()
-say_hello()
-say_hello()
-say_hello()
-```
-
-Output:
-
-```text
-say_hello has been called 1 time(s).
-hello
-say_hello has been called 2 time(s).
-hello
-say_hello has been called 3 time(s).
-hello
-say_hello has been called 4 time(s).
-hello
-```
-
 #### Iterable objects
 
   - `__iter__(<self>) -> object`

pytests/README.md

@@ -0,0 +1,10 @@
+# PyO3 Python tests
+
+These crates are a collection of test extension modules built with PyO3. They are all tested using `tox` in PyO3's CI.
+
+Below is a brief description of each of these:
+
+| Example | Description |
+| ------- | ----------- |
+| `pyo3-benchmarks` | A project containing some benchmarks of PyO3 functionality called from Python. |
+| `pyo3-pytests` | A project containing some tests of PyO3 functionality called from Python. |

pytests/pyo3-benchmarks/Cargo.toml

@@ -14,3 +14,5 @@ features = ["extension-module"]
 [lib]
 name = "_pyo3_benchmarks"
 crate-type = ["cdylib"]
+
+[workspace]

pytests/pyo3-pytests/Cargo.toml

@@ -25,3 +25,5 @@ classifier=[
     "Operating System :: POSIX",
     "Operating System :: MacOS :: MacOS X",
 ]
+
+[workspace]

pytests/pyo3-pytests/MANIFEST.in

@@ -0,0 +1,2 @@
+include pyproject.toml Cargo.toml
+recursive-include src *

pytests/pyo3-pytests/tox.ini

@@ -0,0 +1,10 @@
+[tox]
+# can't install from sdist because local pyo3 repo can't be included in the sdist
+skipsdist = true
+
+[testenv]
+description = Run the unit tests under {basepython}
+deps = -rrequirements-dev.txt
+commands =
+    python -m pip install .
+    pytest {posargs}

src/gil.rs

@@ -185,8 +185,8 @@ impl GILGuard {
                 // to specify `--features auto-initialize` manually. Tests within the crate itself
                 // all depend on the auto-initialize feature for conciseness but Cargo does not
                 // provide a mechanism to specify required features for tests.
+                #[cfg(not(PyPy))]
                 if option_env!("CARGO_PRIMARY_PACKAGE").is_some() {
-                    #[cfg(not(PyPy))]
                     prepare_freethreaded_python();
                 }