Merge pull request #270 from PyO3/module

Treat modules as normal objects

CHANGELOG.md

@@ -6,6 +6,15 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+### Added
+
+ * Added a `wrap_module!` macro similar to the existing `wrap_function!` macro. Only available on python 3
+
+### Changed
+
+ * Renamed `add_function` to `add_wrapped` as it now also supports modules.
+ * Renamed `#[pymodinit]` to `#[pymodule]`.
+
 ### Removed
 
  * `PyToken` was removed due to unsoundness (See [#94](https://github.com/PyO3/pyo3/issues/94)).

README.md

@@ -56,9 +56,9 @@ fn sum_as_string(a: usize, b: usize) -> PyResult<String> {
 }
 
 /// This module is a python module implemented in Rust.
-#[pymodinit]
+#[pymodule]
 fn string_sum(py: Python, m: &PyModule) -> PyResult<()> {
-    m.add_function(wrap_function!(sum_as_string))?;
+    m.add_wrapped(wrap_function!(sum_as_string))?;
 
     Ok(())
 }

examples/rustapi_module/src/datetime.rs

@@ -202,28 +202,28 @@ impl TzClass {
     }
 }
 
-#[pymodinit]
+#[pymodule]
 fn datetime(_py: Python, m: &PyModule) -> PyResult<()> {
-    m.add_function(wrap_function!(make_date))?;
-    m.add_function(wrap_function!(get_date_tuple))?;
-    m.add_function(wrap_function!(date_from_timestamp))?;
-    m.add_function(wrap_function!(make_time))?;
-    m.add_function(wrap_function!(get_time_tuple))?;
-    m.add_function(wrap_function!(make_delta))?;
-    m.add_function(wrap_function!(get_delta_tuple))?;
-    m.add_function(wrap_function!(make_datetime))?;
-    m.add_function(wrap_function!(get_datetime_tuple))?;
-    m.add_function(wrap_function!(datetime_from_timestamp))?;
+    m.add_wrapped(wrap_function!(make_date))?;
+    m.add_wrapped(wrap_function!(get_date_tuple))?;
+    m.add_wrapped(wrap_function!(date_from_timestamp))?;
+    m.add_wrapped(wrap_function!(make_time))?;
+    m.add_wrapped(wrap_function!(get_time_tuple))?;
+    m.add_wrapped(wrap_function!(make_delta))?;
+    m.add_wrapped(wrap_function!(get_delta_tuple))?;
+    m.add_wrapped(wrap_function!(make_datetime))?;
+    m.add_wrapped(wrap_function!(get_datetime_tuple))?;
+    m.add_wrapped(wrap_function!(datetime_from_timestamp))?;
 
     // Python 3.6+ functions
     #[cfg(Py_3_6)]
     {
-        m.add_function(wrap_function!(time_with_fold))?;
-        m.add_function(wrap_function!(get_time_tuple_fold))?;
-        m.add_function(wrap_function!(get_datetime_tuple_fold))?;
+        m.add_wrapped(wrap_function!(time_with_fold))?;
+        m.add_wrapped(wrap_function!(get_time_tuple_fold))?;
+        m.add_wrapped(wrap_function!(get_datetime_tuple_fold))?;
     }
 
-    m.add_function(wrap_function!(issue_219))?;
+    m.add_wrapped(wrap_function!(issue_219))?;
 
     m.add_class::<TzClass>()?;
     Ok(())

examples/rustapi_module/src/dict_iter.rs

@@ -3,7 +3,7 @@ use pyo3::prelude::*;
 use pyo3::exceptions::RuntimeError;
 use pyo3::types::PyDict;
 
-#[pymodinit(test_dict)]
+#[pymodule]
 fn test_dict(_py: Python, m: &PyModule) -> PyResult<()> {
     m.add_class::<DictSize>()?;
     Ok(())

examples/rustapi_module/src/othermod.rs

@@ -28,9 +28,9 @@ fn double(x: i32) -> i32 {
     x * 2
 }
 
-#[pymodinit]
+#[pymodule]
 fn othermod(_py: Python, m: &PyModule) -> PyResult<()> {
-    m.add_function(wrap_function!(double))?;
+    m.add_wrapped(wrap_function!(double))?;
     m.add_class::<ModClass>()?;
 
     m.add("USIZE_MIN", usize::min_value())?;

examples/rustapi_module/src/subclassing.rs

@@ -13,7 +13,7 @@ impl Subclassable {
     }
 }
 
-#[pymodinit]
+#[pymodule]
 fn subclassing(_py: Python, m: &PyModule) -> PyResult<()> {
     m.add_class::<Subclassable>()?;
     Ok(())

examples/word-count/src/lib.rs

@@ -78,9 +78,9 @@ fn count_line(line: &str, needle: &str) -> usize {
     total
 }
 
-#[pymodinit]
+#[pymodule]
 fn word_count(_py: Python, m: &PyModule) -> PyResult<()> {
-    m.add_function(wrap_function!(count_line))?;
+    m.add_wrapped(wrap_function!(count_line))?;
     m.add_class::<WordCounter>()?;
 
     Ok(())

guide/src/debugging.md

@@ -2,7 +2,7 @@
 
 ## Macros
 
-Pyo3's attributes, `#[pyclass]`, `#[pymodinit]`, etc. are [procedural macros](https://doc.rust-lang.org/unstable-book/language-features/proc-macro.html), which means that rewrite the source of the annotated item. You can view the generated source with the following command, which also expands a few other things:
+Pyo3's attributes, `#[pyclass]`, `#[pymodule]`, etc. are [procedural macros](https://doc.rust-lang.org/unstable-book/language-features/proc-macro.html), which means that rewrite the source of the annotated item. You can view the generated source with the following command, which also expands a few other things:
 
 ```bash
 cargo rustc --profile=check -- -Z unstable-options --pretty=expanded > expanded.rs; rustfmt expanded.rs

guide/src/function.md

@@ -12,7 +12,7 @@ extern crate pyo3;
 use pyo3::prelude::*;
 
 
-#[pymodinit]
+#[pymodule]
 fn rust2py(py: Python, m: &PyModule) -> PyResult<()> {
 
     // Note that the `#[pyfn()]` annotation automatically converts the arguments from
@@ -29,7 +29,7 @@ fn rust2py(py: Python, m: &PyModule) -> PyResult<()> {
 ```
 
 The other is annotating a function with `#[py::function]` and then adding it
-to the module using the `add_function_to_module!` macro, which takes the module
+to the module using the `add_wrapped_to_module!` macro, which takes the module
 as first parameter, the function name as second and an instance of `Python`
 as third.
 
@@ -45,9 +45,9 @@ fn double(x: usize) -> usize {
     x * 2
 }
 
-#[pymodinit]
+#[pymodule]
 fn module_with_functions(py: Python, m: &PyModule) -> PyResult<()> {
-    m.add_function(wrap_function!(double)).unwrap();
+    m.add_wrapped(wrap_function!(double)).unwrap();
 
     Ok(())
 }

guide/src/module.md

@@ -8,12 +8,10 @@ As shown in the Getting Started chapter, you can create a module as follows:
 extern crate pyo3;
 use pyo3::{PyResult, Python, PyModule};
 
-
-
 // add bindings to the generated python module
 // N.B: names: "librust2py" must be the name of the `.so` or `.pyd` file
 /// This module is implemented in Rust.
-#[pymodinit]
+#[pymodule]
 fn rust2py(py: Python, m: &PyModule) -> PyResult<()> {
 
     // pyo3 aware function. All of our python interface could be declared in a separate module.
@@ -36,7 +34,11 @@ fn sum_as_string(a:i64, b:i64) -> String {
 # fn main() {}
 ```
 
-The `#[pymodinit}` procedural macro attribute takes care of exporting the initialization function of your module to Python. It takes one argument as the name of your module, it must be the name of the `.so` or `.pyd` file.
+The `#[pymodule]` procedural macro attribute takes care of exporting the initialization function of your module to Python. It takes one argument as the name of your module, which must be the name of the `.so` or `.pyd` file.
+
+To import the module, either copy the shared library as described in [Get Started](./overview.md) or use a tool, e.g. `pyo3-pack develop` with [pyo3-pack](https://github.com/PyO3/pyo3-pack) or `python setup.py develop` with [setuptools-rust](https://github.com/PyO3/setuptools-rust).
+
+## Documentation
 
 The [Rust doc comments](https://doc.rust-lang.org/stable/book/first-edition/comments.html) of the module initialization function will be applied automatically as the Python doc string of your module.
 
@@ -48,9 +50,34 @@ print(rust2py.__doc__)
 
 Which means that the above Python code will print `This module is implemented in Rust.`.
 
-> On macOS, you will need to rename the output from `*.dylib` to `*.so`.
->
-> On Windows, you will need to rename the output from `*.dll` to `*.pyd`.
+## Modules as objects
 
-For `setup.py` integration, You can use [setuptools-rust](https://github.com/PyO3/setuptools-rust),
-learn more about it in [Distribution](./distribution.html).
+In python, modules are first class objects. This means can store them as values or add them to dicts or other modules:
+
+```rust
+#[pyfunction]
+fn subfunction() -> String {
+    "Subfunction".to_string()
+}
+
+#[pymodule]
+fn submodule(_py: Python, module: &PyModule) -> PyResult<()> {
+    module.add_wrapped(wrap_function!(subfunction))?;
+    Ok(())
+}
+
+#[pymodule]
+fn supermodule(_py: Python, module: &PyModule) -> PyResult<()> {
+    module.add_wrapped(wrap_module!(submodule))?;
+    Ok(())
+}
+
+fn nested_call() {
+    let gil = GILGuard::acquire();
+    let py = gil.python();
+    let supermodule = wrap_module!(supermodule)(py);
+    ctx.set_item("supermodule", supermodule);
+
+    py.run("assert supermodule.submodule.subfuntion() == 'Subfunction'", None, Some(&ctx)).unwrap();
+}
+```

guide/src/overview.md

@@ -47,9 +47,9 @@ fn sum_as_string(a: usize, b: usize) -> PyResult<String> {
 }
 
 /// This module is a python moudle implemented in Rust.
-#[pymodinit]
+#[pymodule]
 fn rust_py(py: Python, m: &PyModule) -> PyResult<()> {
-    m.add_function(wrap_function!(sum_as_string))?;
+    m.add_wrapped(wrap_function!(sum_as_string))?;
 
     Ok(())
 }

guide/src/parallelism.md

@@ -28,7 +28,7 @@ Then in the Python bridge, we have a function `search` exposed to Python runtime
 `Python::allow_threads` method to enable true parallelism:
 
 ```rust,ignore
-#[pymodinit]
+#[pymodule]
 fn word_count(py: Python, m: &PyModule) -> PyResult<()> {
 
     #[pyfn(m, "search")]

pyo3-derive-backend/src/module.rs

@@ -12,7 +12,7 @@ use proc_macro2::{Span, TokenStream};
 /// Generates the function that is called by the python interpreter to initialize the native
 /// module
 pub fn py3_init(fnname: &syn::Ident, name: &syn::Ident, doc: syn::Lit) -> TokenStream {
-    let cb_name: syn::Ident = syn::parse_str(&format!("PyInit_{}", name)).unwrap();
+    let cb_name = syn::Ident::new(&format!("PyInit_{}", name), Span::call_site());
 
     quote! {
         #[no_mangle]
@@ -26,7 +26,7 @@ pub fn py3_init(fnname: &syn::Ident, name: &syn::Ident, doc: syn::Lit) -> TokenS
 }
 
 pub fn py2_init(fnname: &syn::Ident, name: &syn::Ident, doc: syn::Lit) -> TokenStream {
-    let cb_name: syn::Ident = syn::parse_str(&format!("init{}", name)).unwrap();
+    let cb_name = syn::Ident::new(&format!("init{}", name), Span::call_site());
 
     quote! {
         #[no_mangle]
@@ -37,7 +37,7 @@ pub fn py2_init(fnname: &syn::Ident, name: &syn::Ident, doc: syn::Lit) -> TokenS
     }
 }
 
-/// Finds and takes care of the #[pyfn(...)] in `#[pymodinit]`
+/// Finds and takes care of the #[pyfn(...)] in `#[pymodule]`
 pub fn process_functions_in_module(func: &mut syn::ItemFn) {
     let mut stmts: Vec<syn::Stmt> = Vec::new();
 
@@ -51,7 +51,7 @@ pub fn process_functions_in_module(func: &mut syn::ItemFn) {
                 let item: syn::ItemFn = parse_quote!{
                     fn block_wrapper() {
                         #function_to_python
-                        #module_name.add_function(&#function_wrapper_ident)?;
+                        #module_name.add_wrapped(&#function_wrapper_ident)?;
                     }
                 };
                 stmts.extend(item.block.stmts.into_iter());

pyo3cls/src/lib.rs

@@ -23,7 +23,7 @@ pub fn mod2init(
     input: proc_macro::TokenStream,
 ) -> proc_macro::TokenStream {
     // Parse the token stream into a syntax tree
-    let mut ast: syn::ItemFn = syn::parse(input).expect("#[pymodinit] must be used on a function");
+    let mut ast: syn::ItemFn = syn::parse(input).expect("#[pymodule] must be used on a function");
 
     let modname: syn::Ident;
     if attr.is_empty() {
@@ -52,7 +52,7 @@ pub fn mod3init(
 ) -> proc_macro::TokenStream {
     // Parse the token stream into a syntax tree
     let mut ast: syn::ItemFn =
-        syn::parse(input).expect("#[pymodinit] must be used on a `fn` block");
+        syn::parse(input).expect("#[pymodule] must be used on a `fn` block");
 
     let modname: syn::Ident;
     if attr.is_empty() {

src/derive_utils.rs

@@ -112,7 +112,7 @@ pub fn parse_fn_args<'p>(
 
 #[cfg(Py_3)]
 #[doc(hidden)]
-/// Builds a module (or null) from a user given initializer. Used for `#[pymodinit]`.
+/// Builds a module (or null) from a user given initializer. Used for `#[pymodule]`.
 pub unsafe fn make_module(
     name: &str,
     doc: &str,
@@ -162,7 +162,7 @@ pub unsafe fn make_module(
 
 #[cfg(not(Py_3))]
 #[doc(hidden)]
-/// Builds a module (or null) from a user given initializer. Used for `#[pymodinit]`.
+/// Builds a module (or null) from a user given initializer. Used for `#[pymodule]`.
 pub unsafe fn make_module(
     name: &str,
     doc: &str,

src/lib.rs

@@ -56,8 +56,8 @@
 //!
 //! To allow Python to load the rust code as a Python extension
 //! module, you need an initialization function with `Fn(Python, &PyModule) -> PyResult<()>`
-//! that is annotates with `#[pymodinit]`. By default the function name will become the module name,
-//! but you can override that with `#[pymodinit(name)]`.
+//! that is annotates with `#[pymodule]`. By default the function name will become the module name,
+//! but you can override that with `#[pymodule(name)]`.
 //!
 //! To creates a Python callable object that invokes a Rust function, specify rust
 //! function and decorate it with `#[pyfn()]` attribute. `pyfn()` accepts three parameters.
@@ -78,7 +78,7 @@
 //! // Add bindings to the generated python module
 //! // N.B: names: "librust2py" must be the name of the `.so` or `.pyd` file
 //! /// This module is implemented in Rust.
-//! #[pymodinit]
+//! #[pymodule]
 //! fn rust2py(py: Python, m: &PyModule) -> PyResult<()> {
 //!
 //!     #[pyfn(m, "sum_as_string")]
@@ -192,16 +192,16 @@ pub mod types;
 /// The proc macros, which are also part of the prelude
 pub mod proc_macro {
     #[cfg(not(Py_3))]
-    pub use pyo3cls::mod2init as pymodinit;
+    pub use pyo3cls::mod2init as pymodule;
     #[cfg(Py_3)]
-    pub use pyo3cls::mod3init as pymodinit;
+    pub use pyo3cls::mod3init as pymodule;
     /// The proc macro attributes
     pub use pyo3cls::{pyclass, pyfunction, pymethods, pyproto};
 }
 
 /// Returns a function that takes a [Python] instance and returns a python function.
 ///
-/// Use this together with `#[function]` and [types::PyModule::add_function].
+/// Use this together with `#[pyfunction]` and [types::PyModule::add_wrapped].
 #[macro_export]
 macro_rules! wrap_function {
     ($function_name:ident) => {{
@@ -218,3 +218,22 @@ macro_rules! wrap_function {
         }
     }};
 }
+
+/// Returns a function that takes a [Python] instance and returns a python module.
+///
+/// Use this together with `#[pymodule]` and [types::PyModule::add_wrapped].
+#[cfg(Py_3)]
+#[macro_export]
+macro_rules! wrap_module {
+    ($module_name:ident) => {{
+        use $crate::mashup::*;
+
+        mashup! {
+            m["method"] = PyInit_ $module_name;
+        }
+
+        m! {
+            &|py| unsafe { crate::PyObject::from_owned_ptr(py, "method"()) }
+        }
+    }};
+}

src/prelude.rs

@@ -18,7 +18,7 @@ pub use crate::object::PyObject;
 pub use crate::objectprotocol::ObjectProtocol;
 pub use crate::python::Python;
 pub use crate::pythonrun::GILGuard;
-// This is only part of the prelude because we need it for the pymodinit function
+// This is only part of the prelude because we need it for the pymodule function
 pub use crate::types::PyModule;
 // This is required for the constructor
 pub use crate::PyRawObject;
@@ -26,7 +26,7 @@ pub use crate::PyRawObject;
 pub use pyo3cls::{pyclass, pyfunction, pymethods, pyproto};
 
 #[cfg(Py_3)]
-pub use pyo3cls::mod3init as pymodinit;
+pub use pyo3cls::mod3init as pymodule;
 
 #[cfg(not(Py_3))]
-pub use pyo3cls::mod2init as pymodinit;
+pub use pyo3cls::mod2init as pymodule;

src/pythonrun.rs

@@ -27,7 +27,7 @@ static START_PYO3: sync::Once = sync::ONCE_INIT;
 /// thread (the thread which originally initialized Python) also initializes
 /// threading.
 ///
-/// When writing an extension module, the `#[pymodinit]` macro
+/// When writing an extension module, the `#[pymodule]` macro
 /// will ensure that Python threading is initialized.
 ///
 pub fn prepare_freethreaded_python() {

src/types/module.rs

@@ -169,12 +169,14 @@ impl PyModule {
         self.setattr(T::NAME, ty)
     }
 
-    /// Adds a function to a module, using the functions __name__ as name.
+    /// Adds a function or a (sub)module to a module, using the functions __name__ as name.
     ///
-    /// Use this together with the`#[pyfunction]` and [wrap_function!] macro.
+    /// Use this together with the`#[pyfunction]` and [wrap_function!] or `#[pymodule]` and
+    /// [wrap_module!].
     ///
     /// ```rust,ignore
-    /// m.add_function(wrap_function!(double));
+    /// m.add_wrapped(wrap_function!(double));
+    /// m.add_wrapped(wrap_module!(utils));
     /// ```
     ///
     /// You can also add a function with a custom name using [add](PyModule::add):
@@ -182,11 +184,11 @@ impl PyModule {
     /// ```rust,ignore
     /// m.add("also_double", wrap_function!(double)(py));
     /// ```
-    pub fn add_function(&self, wrapper: &Fn(Python) -> PyObject) -> PyResult<()> {
+    pub fn add_wrapped(&self, wrapper: &Fn(Python) -> PyObject) -> PyResult<()> {
         let function = wrapper(self.py());
         let name = function
             .getattr(self.py(), "__name__")
-            .expect("A function must have a __name__");
+            .expect("A function or module must have a __name__");
         self.add(name.extract(self.py()).unwrap(), function)
     }
 }

tests/test_module.rs

@@ -4,26 +4,33 @@
 extern crate pyo3;
 
 use pyo3::prelude::*;
+
+#[cfg(Py_3)]
 use pyo3::types::PyDict;
 
+#[cfg(Py_3)]
 #[macro_use]
 mod common;
 
 #[pyclass]
+#[cfg(Py_3)]
 struct EmptyClass {}
 
+#[cfg(Py_3)]
 fn sum_as_string(a: i64, b: i64) -> String {
     format!("{}", a + b).to_string()
 }
 
 #[pyfunction]
+#[cfg(Py_3)]
 /// Doubles the given value
 fn double(x: usize) -> usize {
     x * 2
 }
 
 /// This module is implemented in Rust.
-#[pymodinit]
+#[pymodule]
+#[cfg(Py_3)]
 fn module_with_functions(py: Python, m: &PyModule) -> PyResult<()> {
     #[pyfn(m, "sum_as_string")]
     fn sum_as_string_py(_py: Python, a: i64, b: i64) -> PyResult<String> {
@@ -40,7 +47,7 @@ fn module_with_functions(py: Python, m: &PyModule) -> PyResult<()> {
 
     m.add("foo", "bar").unwrap();
 
-    m.add_function(wrap_function!(double)).unwrap();
+    m.add_wrapped(wrap_function!(double)).unwrap();
     m.add("also_double", wrap_function!(double)(py)).unwrap();
 
     Ok(())
@@ -53,9 +60,10 @@ fn test_module_with_functions() {
     let py = gil.python();
 
     let d = PyDict::new(py);
-    d.set_item("module_with_functions", unsafe {
-        PyObject::from_owned_ptr(py, PyInit_module_with_functions())
-    })
+    d.set_item(
+        "module_with_functions",
+        wrap_module!(module_with_functions)(py),
+    )
     .unwrap();
 
     let run = |code| py.run(code, None, Some(d)).unwrap();
@@ -71,7 +79,7 @@ fn test_module_with_functions() {
     run("assert module_with_functions.also_double.__doc__ == 'Doubles the given value'");
 }
 
-#[pymodinit(other_name)]
+#[pymodule(other_name)]
 fn some_name(_: Python, _: &PyModule) -> PyResult<()> {
     Ok(())
 }
@@ -125,13 +133,15 @@ fn test_module_from_code() {
 }
 
 #[pyfunction]
+#[cfg(Py_3)]
 fn r#move() -> usize {
     42
 }
 
-#[pymodinit]
+#[pymodule]
+#[cfg(Py_3)]
 fn raw_ident_module(_py: Python, module: &PyModule) -> PyResult<()> {
-    module.add_function(wrap_function!(r#move))
+    module.add_wrapped(wrap_function!(r#move))
 }
 
 #[test]
@@ -140,7 +150,53 @@ fn test_raw_idents() {
     let gil = Python::acquire_gil();
     let py = gil.python();
 
-    let module = unsafe { PyObject::from_owned_ptr(py, PyInit_raw_ident_module()) };
+    let module = wrap_module!(raw_ident_module)(py);
 
     py_assert!(py, module, "module.move() == 42");
 }
+
+#[pyfunction]
+#[cfg(Py_3)]
+fn subfunction() -> String {
+    "Subfunction".to_string()
+}
+
+#[cfg(Py_3)]
+#[pymodule]
+fn submodule(_py: Python, module: &PyModule) -> PyResult<()> {
+    module.add_wrapped(wrap_function!(subfunction))?;
+    Ok(())
+}
+
+#[cfg(Py_3)]
+#[pyfunction]
+fn superfunction() -> String {
+    "Superfunction".to_string()
+}
+
+#[cfg(Py_3)]
+#[pymodule]
+fn supermodule(_py: Python, module: &PyModule) -> PyResult<()> {
+    module.add_wrapped(wrap_function!(superfunction))?;
+    module.add_wrapped(wrap_module!(submodule))?;
+    Ok(())
+}
+
+#[test]
+#[cfg(Py_3)]
+fn test_module_nesting() {
+    let gil = GILGuard::acquire();
+    let py = gil.python();
+    let supermodule = wrap_module!(supermodule)(py);
+
+    py_assert!(
+        py,
+        supermodule,
+        "supermodule.superfunction() == 'Superfunction'"
+    );
+    py_assert!(
+        py,
+        supermodule,
+        "supermodule.submodule.subfunction() == 'Subfunction'"
+    );
+}