Merge #2873

2873: A new example that shows how to integrate Python plugins that use pyclasses into a Rust app r=davidhewitt a=alexpyattaev

Example showing integration of a Python plugin into a Rust app while having option to test pyclass based API without the main app.  This also illustrates some aspects related to import of Python modules into a Rust app while also having an API module available for the Python code to be able to produce Rust objects. 

CI seems to fail on my local machine for reasons unrelated to the example just added:
```
error: unused macro definition: `check_struct`
  --> pyo3-ffi-check/src/main.rs:13:18
   |
13 |     macro_rules! check_struct {
   |                  ^^^^^^^^^^^^
   |
```

Co-authored-by: Alex Pyattaev <alex.pyattaev@gmail.com>
Co-authored-by: David Hewitt <1939362+davidhewitt@users.noreply.github.com>

examples/README.md

@@ -10,6 +10,7 @@ Below is a brief description of each of these:
 | `maturin-starter` | A template project which is configured to use [`maturin`](https://github.com/PyO3/maturin) for development. |
 | `setuptools-rust-starter` | A template project which is configured to use [`setuptools_rust`](https://github.com/PyO3/setuptools-rust/) for development. |
 | `word-count` | A quick performance comparison between word counter implementations written in each of Rust and Python. |
+| `plugin` | Illustrates how to use Python as a scripting language within a Rust application |
 
 ## Creating new projects from these examples
 

examples/plugin/.template/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+authors = ["{{authors}}"]
+name = "{{project-name}}"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+pyo3 = "{{PYO3_VERSION}}"
+plugin_api = { path = "plugin_api" }

examples/plugin/.template/plugin_api/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "plugin_api"
+version = "0.1.0"
+description = "Plugin API example"
+edition = "2021"
+
+[lib]
+name = "plugin_api"
+crate-type = ["cdylib", "rlib"]
+
+[dependencies]
+#!!! Important - DO NOT ENABLE extension-module FEATURE HERE!!!
+pyo3 = "{{PYO3_VERSION}}"
+
+[features]
+# instead extension-module feature for pyo3 is enabled conditionally when we want to build a standalone extension module to test our plugins without "main" program
+extension-module = ["pyo3/extension-module"]

examples/plugin/.template/pre-script.rhai

@@ -0,0 +1,4 @@
+variable::set("PYO3_VERSION", "0.18.0");
+file::rename(".template/Cargo.toml", "Cargo.toml");
+file::rename(".template/plugin_api/Cargo.toml", "plugin_api/Cargo.toml");
+file::delete(".template");

examples/plugin/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "plugin_example"
+version = "0.1.0"
+edition = "2021"
+
+
+[dependencies]
+pyo3={path="../../", features=["macros"]}
+plugin_api={path="plugin_api"}
+
+
+[workspace]

examples/plugin/README.md

@@ -0,0 +1,48 @@
+# plugin
+
+An example of a Rust app that uses Python for a plugin. A Python extension module built using PyO3 and [`maturin`](https://github.com/PyO3/maturin) is used to provide
+interface types that can be used to exchange data between Rust and Python. This also deals with how to separately test and load python modules.
+
+# Building and Testing
+## Host application
+To run the app itself, you only need to run 
+
+```shell
+cargo run
+```
+It will build the app, as well as the plugin API, then run the app, load the plugin and show it working.
+
+## Plugin API testing
+
+The plugin API is in a separate crate `plugin_api`, so you can test it separately from the main app. 
+
+To build the API only package, first install `maturin`:
+
+```shell
+pip install maturin
+```
+
+When building the plugin, simply using `maturin develop` will fail to produce a viable extension module due to the features arrangement of PyO3. 
+Instead, one needs to enable the optional feature as follows:
+
+```shell
+cd plugin_api
+maturin build --features "extension-module"
+```
+
+Alternatively, install nox and run the tests inside an isolated environment:
+
+```shell
+nox
+```
+
+## Copying this example
+
+Use [`cargo-generate`](https://crates.io/crates/cargo-generate):
+
+```bash
+$ cargo install cargo-generate
+$ cargo generate --git https://github.com/PyO3/pyo3 examples/plugin
+```
+
+(`cargo generate` will take a little while to clone the PyO3 repo first; be patient when waiting for the command to run.)

examples/plugin/cargo-generate.toml

@@ -0,0 +1,5 @@
+[template]
+ignore = [".nox"]
+
+[hooks]
+pre = [".template/pre-script.rhai"]

examples/plugin/plugin_api/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "plugin_api"
+version = "0.1.0"
+description = "Plugin API example"
+edition = "2021"
+
+[lib]
+name = "plugin_api"
+crate-type = ["cdylib", "rlib"]
+
+[dependencies]
+#!!! Important - DO NOT ENABLE extension-module FEATURE HERE!!!
+pyo3 = { path = "../../../" }
+
+[features]
+# instead extension-module feature for pyo3 is enabled conditionally when we want to build a standalone extension module to test our plugins without "main" program
+extension-module = ["pyo3/extension-module"]

examples/plugin/plugin_api/noxfile.py

@@ -0,0 +1,9 @@
+import nox
+
+
+@nox.session
+def python(session):
+    session.install("-rrequirements-dev.txt")
+    session.install("maturin")
+    session.run_always("maturin", "develop", "--features", "extension-module")
+    session.run("pytest")

examples/plugin/plugin_api/pyproject.toml

@@ -0,0 +1,14 @@
+[build-system]
+requires = ["maturin>=0.14,<0.15"]
+build-backend = "maturin"
+
+[project]
+name = "plugin_api"
+requires-python = ">=3.7"
+classifiers = [
+    "Programming Language :: Rust",
+    "Programming Language :: Python :: Implementation :: CPython",
+    "Programming Language :: Python :: Implementation :: PyPy",
+]
+
+

examples/plugin/plugin_api/requirements-dev.txt

@@ -0,0 +1,3 @@
+pytest>=3.5.0
+pip>=21.3
+maturin>=0.14

examples/plugin/plugin_api/src/lib.rs

@@ -0,0 +1,32 @@
+use pyo3::prelude::*;
+
+///this is our Gadget that python plugin code can create, and rust app can then access natively.
+#[pyclass]
+pub struct Gadget {
+    #[pyo3(get, set)]
+    pub prop: usize,
+    //this field will only be accessible to rust code
+    pub rustonly: Vec<usize>,
+}
+
+#[pymethods]
+impl Gadget {
+    #[new]
+    fn new() -> Self {
+        Gadget {
+            prop: 777,
+            rustonly: Vec::new(),
+        }
+    }
+
+    fn push(&mut self, v: usize) {
+        self.rustonly.push(v);
+    }
+}
+
+/// A Python module for plugin interface types
+#[pymodule]
+pub fn plugin_api(_py: Python<'_>, m: &PyModule) -> PyResult<()> {
+    m.add_class::<Gadget>()?;
+    Ok(())
+}

examples/plugin/plugin_api/tests/test_Gadget.py

@@ -0,0 +1,22 @@
+import pytest
+
+
+@pytest.fixture
+def gadget():
+    import plugin_api as pa
+
+    g = pa.Gadget()
+    return g
+
+
+def test_creation(gadget):
+    pass
+
+
+def test_property(gadget):
+    gadget.prop = 42
+    assert gadget.prop == 42
+
+
+def test_push(gadget):
+    gadget.push(42)

examples/plugin/plugin_api/tests/test_import.py

@@ -0,0 +1,2 @@
+def test_import():
+    import plugin_api

examples/plugin/python_plugin/gadget_init_plugin.py

@@ -0,0 +1,12 @@
+import plugin_api
+import rng
+
+
+def start():
+    """create an instance of Gadget, configure it and return to Rust"""
+    g = plugin_api.Gadget()
+    g.push(1)
+    g.push(2)
+    g.push(3)
+    g.prop = rng.get_random_number()
+    return g

examples/plugin/python_plugin/rng.py

@@ -0,0 +1,3 @@
+def get_random_number():
+    # verified by the roll of a fair die to be random
+    return 4

examples/plugin/src/main.rs

@@ -0,0 +1,44 @@
+use plugin_api::plugin_api as pylib_module;
+use pyo3::prelude::*;
+use pyo3::types::PyList;
+use std::path::Path;
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    //"export" our API module to the python runtime
+    pyo3::append_to_inittab!(pylib_module);
+    //spawn runtime
+    pyo3::prepare_freethreaded_python();
+    //import path for python
+    let path = Path::new("./python_plugin/");
+    //do useful work
+    Python::with_gil(|py| {
+        //add the current directory to import path of Python (do not use this in production!)
+        let syspath: &PyList = py.import("sys")?.getattr("path")?.extract()?;
+        syspath.insert(0, &path)?;
+        println!("Import path is: {:?}", syspath);
+
+        // Now we can load our python_plugin/gadget_init_plugin.py file.
+        // It can in turn import other stuff as it deems appropriate
+        let plugin = PyModule::import(py, "gadget_init_plugin")?;
+        // and call start function there, which will return a python reference to Gadget.
+        // Gadget here is a "pyclass" object reference
+        let gadget = plugin.getattr("start")?.call0()?;
+
+        //now we extract (i.e. mutably borrow) the rust struct from python object
+        {
+            //this scope will have mutable access to the gadget instance, which will be dropped on
+            //scope exit so Python can access it again.
+            let mut gadget_rs: PyRefMut<'_, plugin_api::Gadget> = gadget.extract()?;
+            // we can now modify it as if it was a native rust struct
+            gadget_rs.prop = 42;
+            //which includes access to rust-only fields that are not visible to python
+            println!("rust-only vec contains {:?}", gadget_rs.rustonly);
+            gadget_rs.rustonly.clear();
+        }
+
+        //any modifications we make to rust object are reflected on Python object as well
+        let res: usize = gadget.getattr("prop")?.extract()?;
+        println!("{res}");
+        Ok(())
+    })
+}