From 9ab7b1fad140bd3fb7c0e0b8f620a27c9576a3b9 Mon Sep 17 00:00:00 2001 From: Eric Arellano <14852634+Eric-Arellano@users.noreply.github.com> Date: Thu, 22 Jul 2021 00:10:32 -0700 Subject: [PATCH] Rewrite `module.md` for clarity and add tip on code organization (#1693) * Rewrite `module.md` for clarity and add tip on code organization * Add section on how to build the guide + add workaround proposed by David * Make more clear references to #1709 --- Contributing.md | 13 +++ guide/src/function.md | 29 +++---- guide/src/module.md | 195 ++++++++++++++++++++++++++++++------------ 3 files changed, 164 insertions(+), 73 deletions(-) diff --git a/Contributing.md b/Contributing.md index d2f8155b..329d570a 100644 --- a/Contributing.md +++ b/Contributing.md @@ -34,6 +34,19 @@ There are some specific areas of focus where help is currently needed for the do - Issues requesting documentation improvements are tracked with the [documentation](https://github.com/PyO3/pyo3/issues?q=is%3Aissue+is%3Aopen+label%3Adocumentation) label. - Not all APIs had docs or examples when they were made. The goal is to have documentation on all PyO3 APIs ([#306](https://github.com/PyO3/pyo3/issues/306)). If you see an API lacking a doc, please write one and open a PR! +#### Doctests + +We use lots of code blocks in our docs. Run `cargo test --doc` when making changes to check that +the doctests still work, or `cargo test` to run all the tests including doctests. See +https://doc.rust-lang.org/rustdoc/documentation-tests.html for a guide on doctests. + +#### Building the guide + +You can preview the user guide by building it locally with `mdbook`. + +First, [install `mdbook`](https://rust-lang.github.io/mdBook/cli/index.html). Then, run +`mdbook build -d ../gh-pages-build guide --open`. + ### Help design the next PyO3 Issues which don't yet have a clear solution use the [needs-design](https://github.com/PyO3/pyo3/issues?q=is%3Aissue+is%3Aopen+label%3Aneeds-design) label. diff --git a/guide/src/function.md b/guide/src/function.md index 2393b946..056384b1 100644 --- a/guide/src/function.md +++ b/guide/src/function.md @@ -13,52 +13,47 @@ fn double(x: usize) -> usize { } #[pymodule] -fn module_with_functions(py: Python, m: &PyModule) -> PyResult<()> { +fn my_extension(py: Python, m: &PyModule) -> PyResult<()> { m.add_function(wrap_pyfunction!(double, m)?)?; Ok(()) } - -# fn main() {} ``` -Alternatively there is a shorthand; the function can be placed inside the module definition and annotated with `#[pyfn]`, as below: +Alternatively, there is a shorthand: the function can be placed inside the module definition and +annotated with `#[pyfn]`, as below: ```rust use pyo3::prelude::*; #[pymodule] -fn rust2py(py: Python, m: &PyModule) -> PyResult<()> { +fn my_extension(py: Python, m: &PyModule) -> PyResult<()> { #[pyfn(m)] - fn sum_as_string(_py: Python, a:i64, b:i64) -> PyResult { - Ok(format!("{}", a + b)) + fn double(x: usize) -> usize { + x * 2 } Ok(()) } - -# fn main() {} ``` -`#[pyfn(m)]` is just syntax sugar for `#[pyfunction]`, and takes all the same options documented in the rest of this chapter. The code above is expanded to the following: +`#[pyfn(m)]` is just syntactic sugar for `#[pyfunction]`, and takes all the same options +documented in the rest of this chapter. The code above is expanded to the following: ```rust use pyo3::prelude::*; #[pymodule] -fn rust2py(py: Python, m: &PyModule) -> PyResult<()> { +fn my_extension(py: Python, m: &PyModule) -> PyResult<()> { #[pyfunction] - fn sum_as_string(_py: Python, a:i64, b:i64) -> PyResult { - Ok(format!("{}", a + b)) + fn double(x: usize) -> usize { + x * 2 } - m.add_function(wrap_pyfunction!(sum_as_string, m)?)?; - + m.add_function(wrap_pyfunction!(double, m)?)?; Ok(()) } - -# fn main() {} ``` ## Function options diff --git a/guide/src/module.md b/guide/src/module.md index 330e1bb5..1ac60155 100644 --- a/guide/src/module.md +++ b/guide/src/module.md @@ -1,44 +1,47 @@ # Python Modules -You can create a module as follows: +You can create a module using `#[pymodule]`: ```rust use pyo3::prelude::*; -// add bindings to the generated Python module -// N.B: "rust2py" must be the name of the `.so` or `.pyd` file. +#[pyfunction] +fn double(x: usize) -> usize { + x * 2 +} /// This module is implemented in Rust. #[pymodule] -fn rust2py(py: Python, m: &PyModule) -> PyResult<()> { - // PyO3 aware function. All of our Python interfaces could be declared in a separate module. - // Note that the `#[pyfn()]` annotation automatically converts the arguments from - // Python objects to Rust values, and the Rust return value back into a Python object. - // The `_py` argument represents that we're holding the GIL. - #[pyfn(m)] - #[pyo3(name = "sum_as_string")] - fn sum_as_string_py(_py: Python, a: i64, b: i64) -> PyResult { - let out = sum_as_string(a, b); - Ok(out) - } - +fn my_extension(py: Python, m: &PyModule) -> PyResult<()> { + m.add_function(wrap_pyfunction!(double, m)?)?; Ok(()) } - -// logic implemented as a normal Rust function -fn sum_as_string(a: i64, b: i64) -> String { - format!("{}", a + b) -} - -# fn main() {} ``` -The `#[pymodule]` procedural macro attribute takes care of exporting the initialization function of your -module to Python. It can take as an argument the name of your module, which must be the name of the `.so` -or `.pyd` file; the default is the Rust function's name. +The `#[pymodule]` procedural macro takes care of exporting the initialization function of your +module to Python. -If the name of the module (the default being the function name) does not match the name of the `.so` or -`.pyd` file, you will get an import error in Python with the following message: +The module's name defaults to the name of the Rust function. You can override the module name by +using `#[pyo3(name = "custom_name")]`: + +```rust +use pyo3::prelude::*; + +#[pyfunction] +fn double(x: usize) -> usize { + x * 2 +} + +#[pymodule] +#[pyo3(name = "custom_name")] +fn my_extension(py: Python, m: &PyModule) -> PyResult<()> { + m.add_function(wrap_pyfunction!(double, m)?)?; + Ok(()) +} +``` + +The name of the module must match the name of the `.so` or `.pyd` +file. Otherwise, you will get an import error in Python with the following message: `ImportError: dynamic module does not define module export function (PyInit_name_of_your_module)` To import the module, either: @@ -48,53 +51,133 @@ To import the module, either: ## Documentation -The [Rust doc comments](https://doc.rust-lang.org/stable/book/first-edition/comments.html) of the module +The [Rust doc comments](https://doc.rust-lang.org/stable/book/ch03-04-comments.html) of the module initialization function will be applied automatically as the Python docstring of your module. -```python -import rust2py +For example, building off of the above code, this will print `This module is implemented in Rust.`: -print(rust2py.__doc__) +```python +import my_extension + +print(my_extension.__doc__) ``` -Which means that the above Python code will print `This module is implemented in Rust.`. +## Organizing your module registration code -## Modules as objects +For most projects, it's adequate to centralize all your FFI code into a single Rust module. -In Python, modules are first class objects. This means that you can store them as values or add them to -dicts or other modules: +However, for larger projects, it can be helpful to split your Rust code into several Rust modules to keep your code +readable. Unfortunately, though, some of the macros like `wrap_pyfunction!` do not yet work when used on code defined +in other modules ([#1709](https://github.com/PyO3/pyo3/issues/1709)). One way to work around this is to pass +references to the `PyModule` so that each module registers its own FFI code. For example: + +```rust +// src/lib.rs +use pyo3::prelude::*; + +#[pymodule] +fn my_extension(py: Python, m: &PyModule) -> PyResult<()> { + dirutil::register(py, m)?; + osutil::register(py, m)?; + Ok(()) +} + +// src/dirutil.rs +# mod dirutil { +use pyo3::prelude::*; + +pub(crate) fn register(py: Python, m: &PyModule) -> PyResult<()> { + m.add_class::()?; + Ok(()) +} + +#[pyclass] +struct SomeClass { + x: usize, +} +# } + +// src/osutil.rs +# mod osutil { +use pyo3::prelude::*; + +pub(crate) fn register(py: Python, m: &PyModule) -> PyResult<()> { + m.add_function(wrap_pyfunction!(determine_current_os, m)?)?; + Ok(()) +} + +#[pyfunction] +fn determine_current_os() -> String { + "linux".to_owned() +} +# } +``` + +Another workaround for splitting FFI code across multiple modules ([#1709](https://github.com/PyO3/pyo3/issues/1709)) +is to add `use module::*`, like this: + +```rust +// src/lib.rs +use pyo3::prelude::*; +use osutil::*; + +#[pymodule] +fn my_extension(py: Python, m: &PyModule) -> PyResult<()> { + m.add_function(wrap_pyfunction!(determine_current_os, m)?)?; + Ok(()) +} + +// src/osutil.rs +# mod osutil { +use pyo3::prelude::*; + +#[pyfunction] +pub(crate) fn determine_current_os() -> String { + "linux".to_owned() +} +# } +``` + +## Python submodules + +You can create a module hierarchy within a single extension module by using +[`PyModule.add_submodule()`]({{#PYO3_DOCS_URL}}/pyo3/prelude/struct.PyModule.html#method.add_submodule). +For example, you could define the modules `parent_module` and `parent_module.child_module`. ```rust use pyo3::prelude::*; -use pyo3::wrap_pymodule; -use pyo3::types::IntoPyDict; - -#[pyfunction] -fn subfunction() -> String { - "Subfunction".to_string() -} - -fn init_submodule(module: &PyModule) -> PyResult<()> { - module.add_function(wrap_pyfunction!(subfunction, module)?)?; - Ok(()) -} #[pymodule] -fn supermodule(py: Python, module: &PyModule) -> PyResult<()> { - let submod = PyModule::new(py, "submodule")?; - init_submodule(submod)?; - module.add_submodule(submod)?; +fn parent_module(py: Python, m: &PyModule) -> PyResult<()> { + register_child_module(py, m)?; Ok(()) } +fn register_child_module(py: Python, parent_module: &PyModule) -> PyResult<()> { + let child_module = PyModule::new(py, "child_module")?; + child_module.add_function(wrap_pyfunction!(func, child_module)?)?; + parent_module.add_submodule(child_module)?; + Ok(()) +} + +#[pyfunction] +fn func() -> String { + "func".to_string() +} + # Python::with_gil(|py| { -# let supermodule = wrap_pymodule!(supermodule)(py); -# let ctx = [("supermodule", supermodule)].into_py_dict(py); +# use pyo3::wrap_pymodule; +# use pyo3::types::IntoPyDict; +# let parent_module = wrap_pymodule!(parent_module)(py); +# let ctx = [("parent_module", parent_module)].into_py_dict(py); # -# py.run("assert supermodule.submodule.subfunction() == 'Subfunction'", None, Some(&ctx)).unwrap(); +# py.run("assert parent_module.child_module.func() == 'func'", None, Some(&ctx)).unwrap(); # }) ``` -This way, you can create a module hierarchy within a single extension module. +Note that this does not define a package, so this won’t allow Python code to directly import +submodules by using `from parent_module import child_module`. For more information, see +[#759](https://github.com/PyO3/pyo3/issues/759) and +[#1517](https://github.com/PyO3/pyo3/issues/1517#issuecomment-808664021). -It is not necessary to add `#[pymodule]` on nested modules, this is only required on the top-level module. +It is not necessary to add `#[pymodule]` on nested modules, which is only required on the top-level module.