From 16ad15e04fd4dd283ef3ffb7e45d68c12637de96 Mon Sep 17 00:00:00 2001 From: Bruno Kolenbrander <59372212+mejrs@users.noreply.github.com> Date: Fri, 18 Mar 2022 23:13:23 +0100 Subject: [PATCH 1/9] Expand on xtask (#2176) * Fix Windows OSError * Ignore .pyd files * Put things in modules * Rename functions to `run` * Expand on cargo xtask * Try to work around missing installs * Run all things by default, but not llvm-cov * Test msrv * Fix more OSErrors * Various refinements and docs * Various refinements * Various refinements --- .cargo/config | 8 +- .github/pull_request_template.md | 9 +- .github/workflows/guide.yml | 4 +- .gitignore | 1 + Contributing.md | 6 +- pytests/tests/test_datetime.py | 5 +- xtask/Cargo.toml | 4 + xtask/README.md | 23 ++++ xtask/src/cli.rs | 201 ++++++++++++++++++++++++++++++ xtask/src/clippy.rs | 25 ++++ xtask/src/doc.rs | 47 +++++++ xtask/src/fmt.rs | 23 ++++ xtask/src/llvm_cov.rs | 100 +++++++++++++++ xtask/src/main.rs | 206 +++---------------------------- xtask/src/pytests.rs | 27 ++++ xtask/src/test.rs | 73 +++++++++++ xtask/src/utils.rs | 65 ++++++++++ 17 files changed, 620 insertions(+), 207 deletions(-) create mode 100644 xtask/README.md create mode 100644 xtask/src/cli.rs create mode 100644 xtask/src/clippy.rs create mode 100644 xtask/src/doc.rs create mode 100644 xtask/src/fmt.rs create mode 100644 xtask/src/llvm_cov.rs create mode 100644 xtask/src/pytests.rs create mode 100644 xtask/src/test.rs create mode 100644 xtask/src/utils.rs diff --git a/.cargo/config b/.cargo/config index 83ef5882..68542cc9 100644 --- a/.cargo/config +++ b/.cargo/config @@ -1,11 +1,5 @@ [alias] xtask = "run --package xtask --" -pyo3_doc = "doc --lib --no-default-features --features=full --no-deps --workspace --open --exclude pyo3-macros --exclude pyo3-macros-backend" -pyo3_doc_scrape = "doc --lib --no-default-features --features=full --no-deps --workspace --open --exclude pyo3-macros --exclude pyo3-macros-backend -Z unstable-options -Z rustdoc-scrape-examples=examples" -pyo3_doc_internal = "doc --lib --no-default-features --features=full --no-deps --workspace --open --document-private-items -Z unstable-options -Z rustdoc-scrape-examples=examples" - -[build] -rustdocflags = ["--cfg", "docsrs"] [target.'cfg(feature = "cargo-clippy")'] rustflags = [ @@ -21,4 +15,4 @@ rustflags = [ "-Dclippy::todo", "-Dclippy::unnecessary_wraps", "-Dclippy::useless_transmute", -] +] \ No newline at end of file diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 8a3782ca..077f8ec1 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -5,9 +5,6 @@ Please consider adding the following to your pull request: - docs to all new functions and / or detail in the guide - tests for all new or changed functions -Be aware the CI pipeline will check your pull request for the following. This is done using `nox` (you can install with `pip install nox`): - - Rust tests (`cargo test` or `nox -s test-rust`) - - Examples (`nox -s test-py`) - - Rust lints (`nox -s clippy`) - - Rust formatting (`nox -s fmt-rust`) - - Python formatting (`nox -s fmt-py`) +PyO3's CI pipeline will check your pull request. To run its tests +locally, you can run ```cargo xtask ci```. See its documentation + [here](https://github.com/PyO3/pyo3/tree/main/xtask#readme). diff --git a/.github/workflows/guide.yml b/.github/workflows/guide.yml index 75216cde..531abd96 100644 --- a/.github/workflows/guide.yml +++ b/.github/workflows/guide.yml @@ -45,7 +45,7 @@ jobs: mkdir target mkdir -p gh-pages-build/internal echo "
⚠️ Internal Docs ⚠️ Not Public API 👉 Official Docs Here
" > target/banner.html - cargo +nightly pyo3_doc_internal + cargo xtask doc --internal cp -r target/doc gh-pages-build/internal env: RUSTDOCFLAGS: "--cfg docsrs --Z unstable-options --document-hidden-items --html-before-content target/banner.html" @@ -71,7 +71,7 @@ jobs: # This adds the docs to gh-pages-build/doc - name: Build the doc run: | - cargo +nightly pyo3_doc_scrape + cargo xtask doc cp -r target/doc gh-pages-build/doc echo "" > gh-pages-build/doc/index.html diff --git a/.gitignore b/.gitignore index 7098c348..9e87d2a7 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ guide/book/ extensions/stamps/ pip-wheel-metadata valgrind-python.supp +*.pyd diff --git a/Contributing.md b/Contributing.md index 4fc2c246..a48be238 100644 --- a/Contributing.md +++ b/Contributing.md @@ -48,7 +48,7 @@ There are some specific areas of focus where help is currently needed for the do - Not all APIs had docs or examples when they were made. The goal is to have documentation on all PyO3 APIs ([#306](https://github.com/PyO3/pyo3/issues/306)). If you see an API lacking a doc, please write one and open a PR! You can build the docs (including all features) with -```cargo +nightly pyo3_doc_scrape``` +```cargo xtask doc --open``` #### Doctests @@ -87,6 +87,10 @@ Tests run with all supported Python versions with the latest stable Rust compile If you are adding a new feature, you should add it to the `full` feature in our *Cargo.toml** so that it is tested in CI. +You can run these tests yourself with +```cargo xtask ci``` +See [it's documentation](https://github.com/PyO3/pyo3/tree/main/xtask#readme)for more commands you can run. + ## Python and Rust version support policy PyO3 aims to keep sufficient compatibility to make packaging Python extensions built with PyO3 feasible on most common package managers. diff --git a/pytests/tests/test_datetime.py b/pytests/tests/test_datetime.py index 67d2da16..e70504e7 100644 --- a/pytests/tests/test_datetime.py +++ b/pytests/tests/test_datetime.py @@ -54,8 +54,9 @@ elif _pointer_size == 4: else: raise RuntimeError("unexpected pointer size: " + repr(_pointer_size)) IS_WINDOWS = sys.platform == "win32" + if IS_WINDOWS: - MIN_DATETIME = pdt.datetime(1970, 1, 2, 0, 0) + MIN_DATETIME = pdt.datetime(1971, 1, 2, 0, 0) if IS_32_BIT: MAX_DATETIME = pdt.datetime(3001, 1, 19, 4, 59, 59) else: @@ -227,7 +228,7 @@ def test_datetime_typeerror(): @given(dt=st.datetimes(MIN_DATETIME, MAX_DATETIME)) -@example(dt=pdt.datetime(1970, 1, 2, 0, 0)) +@example(dt=pdt.datetime(1971, 1, 2, 0, 0)) def test_datetime_from_timestamp(dt): if PYPY and dt < pdt.datetime(1900, 1, 1): pytest.xfail("pdt.datetime.timestamp will raise on PyPy with dates before 1900") diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index f6008412..b3560ae1 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -3,9 +3,13 @@ name = "xtask" version = "0.1.0" edition = "2018" +[[bin]] +name = "xtask" + [dependencies] anyhow = "1.0.51" # Clap 3 requires MSRV 1.54 rustversion = "1.0" structopt = { version = "0.3", default-features = false } +clap = { version = "2" } diff --git a/xtask/README.md b/xtask/README.md new file mode 100644 index 00000000..68d078e5 --- /dev/null +++ b/xtask/README.md @@ -0,0 +1,23 @@ +## Commands to test PyO3. + +To run these commands, you should be in PyO3's root directory, and run (for example) `cargo xtask ci`. + +``` +USAGE: + xtask.exe + +FLAGS: + -h, --help Prints help information + -V, --version Prints version information + +SUBCOMMANDS: + ci Runs everything + clippy Runs `clippy`, denying all warnings + coverage Runs `cargo llvm-cov` for the PyO3 codebase + default Only runs the fast things (this is used if no command is specified) + doc Attempts to render the documentation + fmt Checks Rust and Python code formatting with `rustfmt` and `black` + help Prints this message or the help of the given subcommand(s) + test Runs various variations on `cargo test` + test-py Runs the tests in examples/ and pytests/ +``` \ No newline at end of file diff --git a/xtask/src/cli.rs b/xtask/src/cli.rs new file mode 100644 index 00000000..42b384ce --- /dev/null +++ b/xtask/src/cli.rs @@ -0,0 +1,201 @@ +use crate::utils::*; +use anyhow::{ensure, Result}; +use std::io; +use std::process::{Command, Stdio}; +use std::time::Instant; +use structopt::StructOpt; + +pub const MSRV: &str = "1.48"; + +#[derive(StructOpt)] +pub enum Subcommand { + /// Only runs the fast things (this is used if no command is specified) + Default, + /// Runs everything + Ci, + /// Checks Rust and Python code formatting with `rustfmt` and `black` + Fmt, + /// Runs `clippy`, denying all warnings. + Clippy, + /// Runs `cargo llvm-cov` for the PyO3 codebase. + Coverage(CoverageOpts), + /// Attempts to render the documentation. + Doc(DocOpts), + /// Runs various variations on `cargo test` + Test, + /// Runs the tests in examples/ and pytests/ + TestPy, +} + +impl Default for Subcommand { + fn default() -> Self { + Self::Default + } +} + +#[derive(StructOpt, Default)] +pub struct CoverageOpts { + /// Creates an lcov output instead of printing to the terminal. + #[structopt(long)] + pub output_lcov: Option, +} + +#[derive(StructOpt)] +pub struct DocOpts { + /// Whether to run the docs using nightly rustdoc + #[structopt(long)] + pub stable: bool, + /// Whether to open the docs after rendering. + #[structopt(long)] + pub open: bool, + /// Whether to show the private and hidden API. + #[structopt(long)] + pub internal: bool, +} + +impl Default for DocOpts { + fn default() -> Self { + Self { + stable: true, + open: false, + internal: false, + } + } +} + +impl Subcommand { + pub fn execute(self) -> Result<()> { + print_metadata()?; + + let start = Instant::now(); + + match self { + Subcommand::Default => { + crate::fmt::rust::run()?; + crate::clippy::run()?; + crate::test::run()?; + crate::doc::run(DocOpts::default())?; + } + Subcommand::Ci => { + let installed = Installed::new()?; + crate::fmt::rust::run()?; + if installed.black { + crate::fmt::python::run()?; + } else { + Installed::warn_black() + }; + crate::clippy::run()?; + crate::test::run()?; + crate::doc::run(DocOpts::default())?; + if installed.nox { + crate::pytests::run(None)?; + } else { + Installed::warn_nox() + }; + crate::llvm_cov::run(CoverageOpts::default())?; + installed.assert()? + } + + Subcommand::Doc(opts) => crate::doc::run(opts)?, + Subcommand::Fmt => { + crate::fmt::rust::run()?; + crate::fmt::python::run()?; + } + Subcommand::Clippy => crate::clippy::run()?, + Subcommand::Coverage(opts) => crate::llvm_cov::run(opts)?, + Subcommand::TestPy => crate::pytests::run(None)?, + Subcommand::Test => crate::test::run()?, + }; + + let dt = start.elapsed().as_secs(); + let minutes = dt / 60; + let seconds = dt % 60; + println!("\nxtask finished in {}m {}s.", minutes, seconds); + + Ok(()) + } +} + +pub fn run(command: &mut Command) -> Result<()> { + println!("Running: {}", format_command(command)); + + let output = command + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()? + .wait_with_output()?; + + ensure! { + output.status.success(), + "process did not run successfully ({exit}): {command}/n {out} {err}", + exit = match output.status.code() { + Some(code) => format!("exit code {}", code), + None => "terminated by signal".into(), + }, + command = format_command(command), + out = String::from_utf8_lossy(&output.stdout), + err = String::from_utf8_lossy(&output.stderr) + + }; + Ok(()) +} + +#[derive(Copy, Clone, Debug)] +pub struct Installed { + pub nox: bool, + pub black: bool, +} + +impl Installed { + pub fn new() -> anyhow::Result { + Ok(Self { + nox: Self::nox()?, + black: Self::black()?, + }) + } + + pub fn nox() -> anyhow::Result { + let output = std::process::Command::new("nox").arg("--version").output(); + match output { + Ok(_) => Ok(true), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(false), + Err(other) => Err(other.into()), + } + } + + pub fn warn_nox() { + eprintln!("Skipping: formatting Python code, because `nox` was not found"); + } + + pub fn black() -> anyhow::Result { + let output = std::process::Command::new("black") + .arg("--version") + .output(); + match output { + Ok(_) => Ok(true), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(false), + Err(other) => Err(other.into()), + } + } + + pub fn warn_black() { + eprintln!("Skipping: Python code formatting, because `black` was not found."); + } + + pub fn assert(&self) -> anyhow::Result<()> { + if self.nox && self.black { + Ok(()) + } else { + let mut err = + String::from("\n\nxtask was unable to run all tests due to some missing programs:"); + if !self.black { + err.push_str("\n`black` was not installed. (`pip install black`)"); + } + if !self.nox { + err.push_str("\n`nox` was not installed. (`pip install nox`)"); + } + + Err(anyhow::anyhow!(err)) + } + } +} diff --git a/xtask/src/clippy.rs b/xtask/src/clippy.rs new file mode 100644 index 00000000..eec5f5fd --- /dev/null +++ b/xtask/src/clippy.rs @@ -0,0 +1,25 @@ +use crate::cli; +use std::process::Command; + +pub fn run() -> anyhow::Result<()> { + cli::run( + Command::new("cargo") + .arg("clippy") + .arg("--features=full") + .arg("--all-targets") + .arg("--workspace") + .arg("--") + .arg("-Dwarnings"), + )?; + cli::run( + Command::new("cargo") + .arg("clippy") + .arg("--all-targets") + .arg("--workspace") + .arg("--features=abi3,full") + .arg("--") + .arg("-Dwarnings"), + )?; + + Ok(()) +} diff --git a/xtask/src/doc.rs b/xtask/src/doc.rs new file mode 100644 index 00000000..9fe49f30 --- /dev/null +++ b/xtask/src/doc.rs @@ -0,0 +1,47 @@ +use crate::cli; +use crate::cli::DocOpts; +use std::process::Command; +//--cfg docsrs --Z unstable-options --document-hidden-items + +pub fn run(opts: DocOpts) -> anyhow::Result<()> { + let mut flags = Vec::new(); + + if !opts.stable { + flags.push("--cfg docsrs"); + } + if opts.internal { + flags.push("--Z unstable-options"); + flags.push("--document-hidden-items"); + } + flags.push("-Dwarnings"); + + std::env::set_var("RUSTDOCFLAGS", flags.join(" ")); + cli::run( + Command::new("cargo") + .args(if opts.stable { None } else { Some("+nightly") }) + .arg("doc") + .arg("--lib") + .arg("--no-default-features") + .arg("--features=full") + .arg("--no-deps") + .arg("--workspace") + .args(if opts.internal { + &["--document-private-items"][..] + } else { + &["--exclude=pyo3-macros", "--exclude=pyo3-macros-backend"][..] + }) + .args(if opts.stable { + &[][..] + } else { + &[ + "-Z", + "unstable-options", + "-Z", + "rustdoc-scrape-examples=examples", + ] + }) + .args(if opts.open { Some("--open") } else { None }), + )?; + + Ok(()) +} diff --git a/xtask/src/fmt.rs b/xtask/src/fmt.rs new file mode 100644 index 00000000..8bc74524 --- /dev/null +++ b/xtask/src/fmt.rs @@ -0,0 +1,23 @@ +pub mod rust { + use crate::cli; + use std::process::Command; + pub fn run() -> anyhow::Result<()> { + cli::run( + Command::new("cargo") + .arg("fmt") + .arg("--all") + .arg("--") + .arg("--check"), + )?; + Ok(()) + } +} + +pub mod python { + use crate::cli; + use std::process::Command; + pub fn run() -> anyhow::Result<()> { + cli::run(Command::new("black").arg(".").arg("--check"))?; + Ok(()) + } +} diff --git a/xtask/src/llvm_cov.rs b/xtask/src/llvm_cov.rs new file mode 100644 index 00000000..50ea70e6 --- /dev/null +++ b/xtask/src/llvm_cov.rs @@ -0,0 +1,100 @@ +use crate::cli; +use crate::cli::CoverageOpts; +use crate::utils::*; +use anyhow::{Context, Result}; +use std::{collections::HashMap, process::Command}; + +/// Runs `cargo llvm-cov` for the PyO3 codebase. +pub fn run(opts: CoverageOpts) -> Result<()> { + let env = get_coverage_env()?; + + cli::run(llvm_cov_command(&["clean", "--workspace"]).envs(&env))?; + + cli::run( + Command::new("cargo") + .args(&["test", "--manifest-path", "pyo3-build-config/Cargo.toml"]) + .envs(&env), + )?; + cli::run( + Command::new("cargo") + .args(&["test", "--manifest-path", "pyo3-macros-backend/Cargo.toml"]) + .envs(&env), + )?; + cli::run( + Command::new("cargo") + .args(&["test", "--manifest-path", "pyo3-macros/Cargo.toml"]) + .envs(&env), + )?; + + cli::run(Command::new("cargo").arg("test").envs(&env))?; + cli::run( + Command::new("cargo") + .args(&["test", "--features", "abi3"]) + .envs(&env), + )?; + cli::run( + Command::new("cargo") + .args(&["test", "--features", "full"]) + .envs(&env), + )?; + cli::run( + Command::new("cargo") + .args(&["test", "--features", "abi3 full"]) + .envs(&env), + )?; + + crate::pytests::run(&env)?; + + match opts.output_lcov { + Some(path) => { + cli::run(llvm_cov_command(&["--no-run", "--lcov", "--output-path", &path]).envs(&env))? + } + None => cli::run(llvm_cov_command(&["--no-run", "--summary-only"]).envs(&env))?, + } + + Ok(()) +} + +fn llvm_cov_command(args: &[&str]) -> Command { + let mut command = Command::new("cargo"); + command + .args(&[ + "llvm-cov", + "--package=pyo3", + "--package=pyo3-build-config", + "--package=pyo3-macros-backend", + "--package=pyo3-macros", + "--package=pyo3-ffi", + ]) + .args(args); + command +} + +fn get_coverage_env() -> Result> { + let mut env = HashMap::new(); + + let output = String::from_utf8(llvm_cov_command(&["show-env"]).output()?.stdout)?; + + for line in output.trim().split('\n') { + let (key, value) = split_once(line, '=') + .context("expected '=' in each line of output from llvm-cov show-env")?; + env.insert(key.to_owned(), value.trim_matches('"').to_owned()); + } + + // Ensure that examples/ and pytests/ all build to the correct target directory to collect + // coverage artifacts. + env.insert( + "CARGO_TARGET_DIR".to_owned(), + env.get("CARGO_LLVM_COV_TARGET_DIR").unwrap().to_owned(), + ); + + // Coverage only works on nightly. + let rustc_version = + String::from_utf8(get_output(Command::new("rustc").arg("--version"))?.stdout) + .context("failed to parse rust version as utf8")?; + if !rustc_version.contains("nightly") { + env.insert("RUSTUP_TOOLCHAIN".to_owned(), "nightly".to_owned()); + } + + Ok(env) +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index cdbfbf14..ab7afafa 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -1,196 +1,24 @@ -use anyhow::{ensure, Context, Result}; -use std::{collections::HashMap, path::Path, process::Command}; +use clap::ErrorKind::MissingArgumentOrSubcommand; use structopt::StructOpt; -#[derive(StructOpt)] -enum Subcommand { - /// Runs `cargo llvm-cov` for the PyO3 codebase. - Coverage(CoverageOpts), - /// Runs tests in examples/ and pytests/ - TestPy, -} +pub mod cli; +pub mod clippy; +pub mod doc; +pub mod fmt; +pub mod llvm_cov; +pub mod pytests; +pub mod test; +pub mod utils; -#[derive(StructOpt)] -struct CoverageOpts { - /// Creates an lcov output instead of printing to the terminal. - #[structopt(long)] - output_lcov: Option, -} +fn main() -> anyhow::Result<()> { + // Avoid spewing backtraces all over the command line + // For some reason this is automatically enabled on nightly compilers... + std::env::set_var("RUST_LIB_BACKTRACE", "0"); -impl Subcommand { - fn execute(self) -> Result<()> { - match self { - Subcommand::Coverage(opts) => subcommand_coverage(opts), - Subcommand::TestPy => run_python_tests(None), - } - } -} - -fn main() -> Result<()> { - Subcommand::from_args().execute() -} - -/// Runs `cargo llvm-cov` for the PyO3 codebase. -fn subcommand_coverage(opts: CoverageOpts) -> Result<()> { - let env = get_coverage_env()?; - - run(llvm_cov_command(&["clean", "--workspace"]).envs(&env))?; - - run(Command::new("cargo") - .args(&["test", "--manifest-path", "pyo3-build-config/Cargo.toml"]) - .envs(&env))?; - run(Command::new("cargo") - .args(&["test", "--manifest-path", "pyo3-macros-backend/Cargo.toml"]) - .envs(&env))?; - run(Command::new("cargo") - .args(&["test", "--manifest-path", "pyo3-macros/Cargo.toml"]) - .envs(&env))?; - - run(Command::new("cargo").arg("test").envs(&env))?; - run(Command::new("cargo") - .args(&["test", "--features", "abi3"]) - .envs(&env))?; - run(Command::new("cargo") - .args(&["test", "--features", "full"]) - .envs(&env))?; - run(Command::new("cargo") - .args(&["test", "--features", "abi3 full"]) - .envs(&env))?; - - run_python_tests(&env)?; - - match opts.output_lcov { - Some(path) => { - run(llvm_cov_command(&["--no-run", "--lcov", "--output-path", &path]).envs(&env))? - } - None => run(llvm_cov_command(&["--no-run", "--summary-only"]).envs(&env))?, - } - - Ok(()) -} - -fn run(command: &mut Command) -> Result<()> { - println!("running: {}", format_command(command)); - let status = command.spawn()?.wait()?; - ensure! { - status.success(), - "process did not run successfully ({exit}): {command}", - exit = match status.code() { - Some(code) => format!("exit code {}", code), - None => "terminated by signal".into(), - }, - command = format_command(command), - }; - Ok(()) -} - -fn get_output(command: &mut Command) -> Result { - let output = command.output()?; - ensure! { - output.status.success(), - "process did not run successfully ({exit}): {command}", - exit = match output.status.code() { - Some(code) => format!("exit code {}", code), - None => "terminated by signal".into(), - }, - command = format_command(command), - }; - Ok(output) -} - -fn llvm_cov_command(args: &[&str]) -> Command { - let mut command = Command::new("cargo"); - command - .args(&[ - "llvm-cov", - "--package=pyo3", - "--package=pyo3-build-config", - "--package=pyo3-macros-backend", - "--package=pyo3-macros", - "--package=pyo3-ffi", - ]) - .args(args); - command -} - -fn run_python_tests<'a>( - env: impl IntoIterator + Copy, -) -> Result<()> { - run(Command::new("nox") - .arg("--non-interactive") - .arg("-f") - .arg(Path::new("pytests").join("noxfile.py")) - .envs(env))?; - - for entry in std::fs::read_dir("examples")? { - let path = entry?.path(); - if path.is_dir() && path.join("noxfile.py").exists() { - run(Command::new("nox") - .arg("--non-interactive") - .arg("-f") - .arg(path.join("noxfile.py")) - .envs(env))?; - } + match cli::Subcommand::from_args_safe() { + Ok(c) => c.execute()?, + Err(e) if e.kind == MissingArgumentOrSubcommand => cli::Subcommand::default().execute()?, + Err(e) => return Err(e.into()), } Ok(()) } - -fn get_coverage_env() -> Result> { - let mut env = HashMap::new(); - - let output = String::from_utf8(llvm_cov_command(&["show-env"]).output()?.stdout)?; - - for line in output.trim().split('\n') { - let (key, value) = split_once(line, '=') - .context("expected '=' in each line of output from llvm-cov show-env")?; - env.insert(key.to_owned(), value.trim_matches('"').to_owned()); - } - - // Ensure that examples/ and pytests/ all build to the correct target directory to collect - // coverage artifacts. - env.insert( - "CARGO_TARGET_DIR".to_owned(), - env.get("CARGO_LLVM_COV_TARGET_DIR").unwrap().to_owned(), - ); - - // Coverage only works on nightly. - let rustc_version = - String::from_utf8(get_output(Command::new("rustc").arg("--version"))?.stdout) - .context("failed to parse rust version as utf8")?; - if !rustc_version.contains("nightly") { - env.insert("RUSTUP_TOOLCHAIN".to_owned(), "nightly".to_owned()); - } - - Ok(env) -} - -// Replacement for str.split_once() on Rust older than 1.52 -#[rustversion::before(1.52)] -fn split_once(s: &str, pat: char) -> Option<(&str, &str)> { - let mut iter = s.splitn(2, pat); - Some((iter.next()?, iter.next()?)) -} - -#[rustversion::since(1.52)] -fn split_once(s: &str, pat: char) -> Option<(&str, &str)> { - s.split_once(pat) -} - -#[rustversion::since(1.57)] -fn format_command(command: &Command) -> String { - let mut buf = String::new(); - buf.push('`'); - buf.push_str(&command.get_program().to_string_lossy()); - for arg in command.get_args() { - buf.push(' '); - buf.push_str(&arg.to_string_lossy()); - } - buf.push('`'); - buf -} - -#[rustversion::before(1.57)] -fn format_command(command: &Command) -> String { - // Debug impl isn't as nice as the above, but will do on < 1.57 - format!("{:?}", command) -} diff --git a/xtask/src/pytests.rs b/xtask/src/pytests.rs new file mode 100644 index 00000000..78744c69 --- /dev/null +++ b/xtask/src/pytests.rs @@ -0,0 +1,27 @@ +use crate::cli; +use anyhow::Result; +use std::{path::Path, process::Command}; + +pub fn run<'a>(env: impl IntoIterator + Copy) -> Result<()> { + cli::run( + Command::new("nox") + .arg("--non-interactive") + .arg("-f") + .arg(Path::new("pytests").join("noxfile.py")) + .envs(env), + )?; + + for entry in std::fs::read_dir("examples")? { + let path = entry?.path(); + if path.is_dir() && path.join("noxfile.py").exists() { + cli::run( + Command::new("nox") + .arg("--non-interactive") + .arg("-f") + .arg(path.join("noxfile.py")) + .envs(env), + )?; + } + } + Ok(()) +} diff --git a/xtask/src/test.rs b/xtask/src/test.rs new file mode 100644 index 00000000..c383140a --- /dev/null +++ b/xtask/src/test.rs @@ -0,0 +1,73 @@ +use crate::cli::{self, MSRV}; +use std::process::Command; + +pub fn run() -> anyhow::Result<()> { + cli::run( + Command::new("cargo") + .arg("test") + .arg("--lib") + .arg("--no-default-features") + .arg("--tests") + .arg("--quiet"), + )?; + + cli::run( + Command::new("cargo") + .arg("test") + .arg("--no-default-features") + .arg("--features=full") + .arg("--quiet"), + )?; + + cli::run( + Command::new("cargo") + .arg("test") + .arg("--no-default-features") + .arg("--features=abi3,full") + .arg("--quiet"), + )?; + + // If the MSRV toolchain is not installed, this will install it + cli::run( + Command::new("rustup") + .arg("toolchain") + .arg("install") + .arg(MSRV), + )?; + + // Test MSRV + cli::run( + Command::new("cargo") + .arg(format!("+{}", MSRV)) + .arg("test") + .arg("--no-default-features") + .arg("--features=full,auto-initialize") + .arg("--quiet"), + )?; + + cli::run( + Command::new("cargo") + .arg("+nightly") + .arg("test") + .arg("--no-default-features") + .arg("--features=full,nightly") + .arg("--quiet"), + )?; + + cli::run( + Command::new("cargo") + .arg("test") + .arg("--manifest-path=pyo3-ffi/Cargo.toml") + .arg("--quiet"), + )?; + + cli::run( + Command::new("cargo") + .arg("test") + .arg("--no-default-features") + .arg("--manifest-path=pyo3-build-config/Cargo.toml") + .arg("--quiet"), + )?; + + Ok(()) +} diff --git a/xtask/src/utils.rs b/xtask/src/utils.rs new file mode 100644 index 00000000..045697e7 --- /dev/null +++ b/xtask/src/utils.rs @@ -0,0 +1,65 @@ +use anyhow::ensure; +use std::process::Command; + +// Replacement for str.split_once() on Rust older than 1.52 +#[rustversion::before(1.52)] +pub fn split_once(s: &str, pat: char) -> Option<(&str, &str)> { + let mut iter = s.splitn(2, pat); + Some((iter.next()?, iter.next()?)) +} + +#[rustversion::since(1.52)] +pub fn split_once(s: &str, pat: char) -> Option<(&str, &str)> { + s.split_once(pat) +} + +#[rustversion::since(1.57)] +pub fn format_command(command: &Command) -> String { + let mut buf = String::new(); + buf.push('`'); + buf.push_str(&command.get_program().to_string_lossy()); + for arg in command.get_args() { + buf.push(' '); + buf.push_str(&arg.to_string_lossy()); + } + buf.push('`'); + buf +} + +#[rustversion::before(1.57)] +pub fn format_command(command: &Command) -> String { + // Debug impl isn't as nice as the above, but will do on < 1.57 + format!("{:?}", command) +} + +pub fn get_output(command: &mut Command) -> anyhow::Result { + let output = command.output()?; + ensure! { + output.status.success(), + "process did not run successfully ({exit}): {command}", + exit = match output.status.code() { + Some(code) => format!("exit code {}", code), + None => "terminated by signal".into(), + }, + command = format_command(command), + }; + Ok(output) +} + +pub fn print_metadata() -> anyhow::Result<()> { + let rustc_output = std::process::Command::new("rustc") + .arg("--version") + .arg("--verbose") + .output()?; + let rustc_version = core::str::from_utf8(&rustc_output.stdout).unwrap(); + println!("Metadata: \n\n{}", rustc_version); + + let py_output = std::process::Command::new("python") + .arg("--version") + .arg("-V") + .output()?; + let py_version = core::str::from_utf8(&py_output.stdout).unwrap(); + println!("{}", py_version); + + Ok(()) +} From 69655454c146cd7c965a874fca9907a316636651 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Sat, 19 Mar 2022 11:46:40 -0400 Subject: [PATCH 2/9] Added an as_bytes method for Py This allows for obtaining a slice that's not lexically bound to the GIL which can be helpful to avoid copying. --- CHANGELOG.md | 4 ++++ src/types/bytes.rs | 18 ++++++++++++++++++ tests/test_bytes.rs | 15 +++++++++++++++ 3 files changed, 37 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bded739..9615c5a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Panic during compilation when `PYO3_CROSS_LIB_DIR` is set for some host/target combinations. [#2232](https://github.com/PyO3/pyo3/pull/2232) +### Added + +- Added `as_bytes` on `Py`. [#2235](https://github.com/PyO3/pyo3/pull/2235) + ## [0.16.2] - 2022-03-15 ### Packaging diff --git a/src/types/bytes.rs b/src/types/bytes.rs index e733582b..96a49960 100644 --- a/src/types/bytes.rs +++ b/src/types/bytes.rs @@ -97,6 +97,24 @@ impl PyBytes { } } +impl Py { + /// Gets the Python bytes as a byte slice. Because Python bytes are + /// immutable, the result may be used for as long as the reference to + /// `self` is held, including when the GIL is released. + pub fn as_bytes<'a>(&'a self, _py: Python<'_>) -> &'a [u8] { + // py is required here because `PyBytes_AsString` and `PyBytes_Size` + // can both technically raise exceptions which require the GIL to be + // held. The only circumstance in which they raise is if the value + // isn't really a `PyBytes`, but better safe than sorry. + unsafe { + let buffer = ffi::PyBytes_AsString(self.as_ptr()) as *const u8; + let length = ffi::PyBytes_Size(self.as_ptr()) as usize; + debug_assert!(!buffer.is_null()); + std::slice::from_raw_parts(buffer, length) + } + } +} + /// This is the same way [Vec] is indexed. impl> Index for PyBytes { type Output = I::Output; diff --git a/tests/test_bytes.rs b/tests/test_bytes.rs index 710677dd..36ae41e4 100644 --- a/tests/test_bytes.rs +++ b/tests/test_bytes.rs @@ -41,3 +41,18 @@ fn test_bytearray_vec_conversion() { let f = wrap_pyfunction!(bytes_vec_conversion)(py).unwrap(); py_assert!(py, f, "f(bytearray(b'Hello World')) == b'Hello World'"); } + +#[test] +fn test_py_as_bytes() { + let pyobj: pyo3::Py; + let data: &[u8]; + + { + let gil = Python::acquire_gil(); + let py = gil.python(); + pyobj = pyo3::types::PyBytes::new(py, b"abc").into_py(py); + data = pyobj.as_bytes(py); + } + + assert_eq!(data, b"abc"); +} From 5cc3ce99f144be3331f7600d3665958cff264410 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Fri, 18 Mar 2022 14:58:44 +0000 Subject: [PATCH 3/9] pyclass: unify pyclass with its pyo3 arguments --- pyo3-macros-backend/src/attributes.rs | 122 +++++--- pyo3-macros-backend/src/frompyobject.rs | 10 +- pyo3-macros-backend/src/konst.rs | 4 +- pyo3-macros-backend/src/method.rs | 4 +- pyo3-macros-backend/src/module.rs | 4 +- pyo3-macros-backend/src/params.rs | 4 +- pyo3-macros-backend/src/pyclass.rs | 372 ++++++++++-------------- pyo3-macros-backend/src/pyfunction.rs | 10 +- pyo3-macros-backend/src/pyimpl.rs | 2 +- pyo3-macros-backend/src/pymethod.rs | 2 +- pyo3-macros-backend/src/utils.rs | 12 +- pyo3-macros/src/lib.rs | 31 +- tests/ui/invalid_property_args.stderr | 8 +- tests/ui/invalid_pyclass_args.stderr | 14 +- tests/ui/invalid_pyclass_enum.rs | 8 +- tests/ui/invalid_pyclass_enum.stderr | 4 +- tests/ui/invalid_pymethod_names.stderr | 4 +- 17 files changed, 292 insertions(+), 323 deletions(-) diff --git a/pyo3-macros-backend/src/attributes.rs b/pyo3-macros-backend/src/attributes.rs index e0566076..d6801d32 100644 --- a/pyo3-macros-backend/src/attributes.rs +++ b/pyo3-macros-backend/src/attributes.rs @@ -1,77 +1,107 @@ +use proc_macro2::TokenStream; +use quote::ToTokens; use syn::{ parse::{Parse, ParseStream}, punctuated::Punctuated, + spanned::Spanned, token::Comma, - Attribute, ExprPath, Ident, LitStr, Path, Result, Token, + Attribute, Expr, ExprPath, Ident, LitStr, Path, Result, Token, }; pub mod kw { syn::custom_keyword!(annotation); syn::custom_keyword!(attribute); + syn::custom_keyword!(dict); + syn::custom_keyword!(extends); + syn::custom_keyword!(freelist); syn::custom_keyword!(from_py_with); + syn::custom_keyword!(gc); syn::custom_keyword!(get); syn::custom_keyword!(item); - syn::custom_keyword!(pass_module); + syn::custom_keyword!(module); syn::custom_keyword!(name); + syn::custom_keyword!(pass_module); syn::custom_keyword!(set); syn::custom_keyword!(signature); + syn::custom_keyword!(subclass); syn::custom_keyword!(text_signature); syn::custom_keyword!(transparent); + syn::custom_keyword!(unsendable); + syn::custom_keyword!(weakref); } -#[derive(Clone, Debug, PartialEq)] -pub struct FromPyWithAttribute(pub ExprPath); +#[derive(Clone, Debug)] +pub struct KeywordAttribute { + pub kw: K, + pub value: V, +} -impl Parse for FromPyWithAttribute { +/// A helper type which parses the inner type via a literal string +/// e.g. LitStrValue -> parses "some::path" in quotes. +#[derive(Clone, Debug, PartialEq)] +pub struct LitStrValue(pub T); + +impl Parse for LitStrValue { fn parse(input: ParseStream) -> Result { - let _: kw::from_py_with = input.parse()?; - let _: Token![=] = input.parse()?; - let string_literal: LitStr = input.parse()?; - string_literal.parse().map(FromPyWithAttribute) + let lit_str: LitStr = input.parse()?; + lit_str.parse().map(LitStrValue) } } -#[derive(Clone, Debug, PartialEq)] -pub struct NameAttribute(pub Ident); - -impl Parse for NameAttribute { - fn parse(input: ParseStream) -> Result { - let _: kw::name = input.parse()?; - let _: Token![=] = input.parse()?; - let string_literal: LitStr = input.parse()?; - string_literal.parse().map(NameAttribute) +impl ToTokens for LitStrValue { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.0.to_tokens(tokens) } } +/// A helper type which parses a name via a literal string +#[derive(Clone, Debug, PartialEq)] +pub struct NameLitStr(pub Ident); + +impl Parse for NameLitStr { + fn parse(input: ParseStream) -> Result { + let string_literal: LitStr = input.parse()?; + if let Ok(ident) = string_literal.parse() { + Ok(NameLitStr(ident)) + } else { + bail_spanned!(string_literal.span() => "expected a single identifier in double quotes") + } + } +} + +impl ToTokens for NameLitStr { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.0.to_tokens(tokens) + } +} + +pub type ExtendsAttribute = KeywordAttribute; +pub type FreelistAttribute = KeywordAttribute>; +pub type ModuleAttribute = KeywordAttribute; +pub type NameAttribute = KeywordAttribute; +pub type TextSignatureAttribute = KeywordAttribute; + +impl Parse for KeywordAttribute { + fn parse(input: ParseStream) -> Result { + let kw: K = input.parse()?; + let _: Token![=] = input.parse()?; + let value = input.parse()?; + Ok(KeywordAttribute { kw, value }) + } +} + +impl ToTokens for KeywordAttribute { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.kw.to_tokens(tokens); + Token![=](self.kw.span()).to_tokens(tokens); + self.value.to_tokens(tokens); + } +} + +pub type FromPyWithAttribute = KeywordAttribute>; + /// For specifying the path to the pyo3 crate. -#[derive(Clone, Debug, PartialEq)] -pub struct CrateAttribute(pub Path); - -impl Parse for CrateAttribute { - fn parse(input: ParseStream) -> Result { - let _: Token![crate] = input.parse()?; - let _: Token![=] = input.parse()?; - let string_literal: LitStr = input.parse()?; - string_literal.parse().map(CrateAttribute) - } -} - -#[derive(Clone, Debug, PartialEq)] -pub struct TextSignatureAttribute { - pub kw: kw::text_signature, - pub eq_token: Token![=], - pub lit: LitStr, -} - -impl Parse for TextSignatureAttribute { - fn parse(input: ParseStream) -> Result { - Ok(TextSignatureAttribute { - kw: input.parse()?, - eq_token: input.parse()?, - lit: input.parse()?, - }) - } -} +pub type CrateAttribute = KeywordAttribute>; pub fn get_pyo3_options(attr: &syn::Attribute) -> Result>> { if is_attribute_ident(attr, "pyo3") { diff --git a/pyo3-macros-backend/src/frompyobject.rs b/pyo3-macros-backend/src/frompyobject.rs index f262df3e..9f7dd40a 100644 --- a/pyo3-macros-backend/src/frompyobject.rs +++ b/pyo3-macros-backend/src/frompyobject.rs @@ -252,7 +252,9 @@ impl<'a> Container<'a> { None => quote!( obj.get_item(#index)?.extract() ), - Some(FromPyWithAttribute(expr_path)) => quote! ( + Some(FromPyWithAttribute { + value: expr_path, .. + }) => quote! ( #expr_path(obj.get_item(#index)?) ), }; @@ -308,7 +310,9 @@ impl<'a> Container<'a> { new_err.set_cause(py, ::std::option::Option::Some(inner)); new_err })?), - Some(FromPyWithAttribute(expr_path)) => quote! ( + Some(FromPyWithAttribute { + value: expr_path, .. + }) => quote! ( #expr_path(#get_field).map_err(|inner| { let py = _pyo3::PyNativeType::py(obj); let new_err = _pyo3::exceptions::PyTypeError::new_err(#conversion_error_msg); @@ -388,7 +392,7 @@ impl ContainerOptions { ContainerPyO3Attribute::Crate(path) => { ensure_spanned!( options.krate.is_none(), - path.0.span() => "`crate` may only be provided once" + path.span() => "`crate` may only be provided once" ); options.krate = Some(path); } diff --git a/pyo3-macros-backend/src/konst.rs b/pyo3-macros-backend/src/konst.rs index c11c828d..61d1ff14 100644 --- a/pyo3-macros-backend/src/konst.rs +++ b/pyo3-macros-backend/src/konst.rs @@ -21,7 +21,7 @@ pub struct ConstSpec { impl ConstSpec { pub fn python_name(&self) -> Cow { if let Some(name) = &self.attributes.name { - Cow::Borrowed(&name.0) + Cow::Borrowed(&name.value.0) } else { Cow::Owned(self.rust_ident.unraw()) } @@ -89,7 +89,7 @@ impl ConstAttributes { fn set_name(&mut self, name: NameAttribute) -> Result<()> { ensure_spanned!( self.name.is_none(), - name.0.span() => "`name` may only be specified once" + name.span() => "`name` may only be specified once" ); self.name = Some(name); Ok(()) diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index bde4b079..a8d46efb 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -14,7 +14,7 @@ use syn::ext::IdentExt; use syn::spanned::Spanned; use syn::Result; -#[derive(Clone, PartialEq, Debug)] +#[derive(Clone, Debug)] pub struct FnArg<'a> { pub name: &'a syn::Ident, pub by_ref: &'a Option, @@ -273,7 +273,7 @@ impl<'a> FnSpec<'a> { ty: fn_type_attr, args: fn_attrs, mut python_name, - } = parse_method_attributes(meth_attrs, name.map(|name| name.0), &mut deprecations)?; + } = parse_method_attributes(meth_attrs, name.map(|name| name.value.0), &mut deprecations)?; let (fn_type, skip_first_arg, fixed_convention) = Self::parse_fn_type(sig, fn_type_attr, &mut python_name)?; diff --git a/pyo3-macros-backend/src/module.rs b/pyo3-macros-backend/src/module.rs index 738b69fb..aacc9bef 100644 --- a/pyo3-macros-backend/src/module.rs +++ b/pyo3-macros-backend/src/module.rs @@ -31,7 +31,7 @@ impl PyModuleOptions { for option in take_pyo3_options(attrs)? { match option { - PyModulePyO3Option::Name(name) => options.set_name(name.0)?, + PyModulePyO3Option::Name(name) => options.set_name(name.value.0)?, PyModulePyO3Option::Crate(path) => options.set_crate(path)?, } } @@ -52,7 +52,7 @@ impl PyModuleOptions { fn set_crate(&mut self, path: CrateAttribute) -> Result<()> { ensure_spanned!( self.krate.is_none(), - path.0.span() => "`crate` may only be specified once" + path.span() => "`crate` may only be specified once" ); self.krate = Some(path); diff --git a/pyo3-macros-backend/src/params.rs b/pyo3-macros-backend/src/params.rs index 01fb6325..219c1e19 100644 --- a/pyo3-macros-backend/src/params.rs +++ b/pyo3-macros-backend/src/params.rs @@ -231,7 +231,9 @@ fn impl_arg_param( let arg_value = quote_arg_span!(#args_array[#option_pos]); *option_pos += 1; - let arg_value_or_default = if let Some(FromPyWithAttribute(expr_path)) = &arg.attrs.from_py_with + let arg_value_or_default = if let Some(FromPyWithAttribute { + value: expr_path, .. + }) = &arg.attrs.from_py_with { match (spec.default_value(name), arg.optional.is_some()) { (Some(default), true) if default.to_string() != "None" => { diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index e827f8a0..4950844c 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -1,19 +1,20 @@ // Copyright (c) 2017-present PyO3 Project and Contributors use crate::attributes::{ - self, take_pyo3_options, CrateAttribute, NameAttribute, TextSignatureAttribute, + self, kw, take_pyo3_options, CrateAttribute, ExtendsAttribute, FreelistAttribute, + ModuleAttribute, NameAttribute, NameLitStr, TextSignatureAttribute, }; use crate::deprecations::{Deprecation, Deprecations}; use crate::konst::{ConstAttributes, ConstSpec}; use crate::pyimpl::{gen_default_items, gen_py_const, PyClassMethodsType}; use crate::pymethod::{impl_py_getter_def, impl_py_setter_def, PropertyType}; -use crate::utils::{self, get_pyo3_crate, unwrap_group, PythonDoc}; +use crate::utils::{self, get_pyo3_crate, PythonDoc}; use proc_macro2::{Span, TokenStream}; use quote::quote; use syn::ext::IdentExt; use syn::parse::{Parse, ParseStream}; use syn::punctuated::Punctuated; -use syn::{parse_quote, spanned::Spanned, Expr, Result, Token}; //unraw +use syn::{parse_quote, spanned::Spanned, Result, Token}; /// If the class is derived from a Rust `struct` or `enum`. #[derive(Copy, Clone, Debug, PartialEq, Eq)] @@ -24,27 +25,18 @@ pub enum PyClassKind { /// The parsed arguments of the pyclass macro pub struct PyClassArgs { - pub freelist: Option, - pub name: Option, - pub base: syn::TypePath, - pub has_dict: bool, - pub has_weaklist: bool, - pub is_basetype: bool, - pub has_extends: bool, - pub has_unsendable: bool, - pub module: Option, pub class_kind: PyClassKind, + pub options: PyClassPyO3Options, pub deprecations: Deprecations, } impl PyClassArgs { fn parse(input: ParseStream, kind: PyClassKind) -> Result { - let mut slf = PyClassArgs::new(kind); - let vars = Punctuated::::parse_terminated(input)?; - for expr in vars { - slf.add_expr(&expr)?; - } - Ok(slf) + Ok(PyClassArgs { + class_kind: kind, + options: PyClassPyO3Options::parse(input)?, + deprecations: Deprecations::new(), + }) } pub fn parse_stuct_args(input: ParseStream) -> syn::Result { @@ -54,155 +46,64 @@ impl PyClassArgs { pub fn parse_enum_args(input: ParseStream) -> syn::Result { Self::parse(input, PyClassKind::Enum) } - - fn new(class_kind: PyClassKind) -> Self { - PyClassArgs { - freelist: None, - name: None, - module: None, - base: parse_quote! { _pyo3::PyAny }, - has_dict: false, - has_weaklist: false, - is_basetype: false, - has_extends: false, - has_unsendable: false, - class_kind, - deprecations: Deprecations::new(), - } - } - - /// Add a single expression from the comma separated list in the attribute, which is - /// either a single word or an assignment expression - fn add_expr(&mut self, expr: &Expr) -> Result<()> { - match expr { - syn::Expr::Path(exp) if exp.path.segments.len() == 1 => self.add_path(exp), - syn::Expr::Assign(assign) => self.add_assign(assign), - _ => bail_spanned!(expr.span() => "failed to parse arguments"), - } - } - - /// Match a key/value flag - fn add_assign(&mut self, assign: &syn::ExprAssign) -> syn::Result<()> { - let syn::ExprAssign { left, right, .. } = assign; - let key = match &**left { - syn::Expr::Path(exp) if exp.path.segments.len() == 1 => { - exp.path.segments.first().unwrap().ident.to_string() - } - _ => bail_spanned!(assign.span() => "failed to parse arguments"), - }; - - macro_rules! expected { - ($expected: literal) => { - expected!($expected, right.span()) - }; - ($expected: literal, $span: expr) => { - bail_spanned!($span => concat!("expected ", $expected)) - }; - } - - match key.as_str() { - "freelist" => { - // We allow arbitrary expressions here so you can e.g. use `8*64` - self.freelist = Some(syn::Expr::clone(right)); - } - "name" => match unwrap_group(&**right) { - syn::Expr::Lit(syn::ExprLit { - lit: syn::Lit::Str(lit), - .. - }) => { - self.name = Some(lit.parse().map_err(|_| { - err_spanned!( - lit.span() => "expected a single identifier in double-quotes") - })?); - } - syn::Expr::Path(exp) if exp.path.segments.len() == 1 => { - bail_spanned!( - exp.span() => format!( - "since PyO3 0.13 a pyclass name should be in double-quotes, \ - e.g. \"{}\"", - exp.path.get_ident().expect("path has 1 segment") - ) - ); - } - _ => expected!("type name (e.g. \"Name\")"), - }, - "extends" => match unwrap_group(&**right) { - syn::Expr::Path(exp) => { - if self.class_kind == PyClassKind::Enum { - bail_spanned!( assign.span() => "enums cannot extend from other classes" ); - } - self.base = syn::TypePath { - path: exp.path.clone(), - qself: None, - }; - self.has_extends = true; - } - _ => expected!("type path (e.g., my_mod::BaseClass)"), - }, - "module" => match unwrap_group(&**right) { - syn::Expr::Lit(syn::ExprLit { - lit: syn::Lit::Str(lit), - .. - }) => { - self.module = Some(lit.clone()); - } - _ => expected!(r#"string literal (e.g., "my_mod")"#), - }, - _ => expected!("one of freelist/name/extends/module", left.span()), - }; - - Ok(()) - } - - /// Match a single flag - fn add_path(&mut self, exp: &syn::ExprPath) -> syn::Result<()> { - let flag = exp.path.segments.first().unwrap().ident.to_string(); - match flag.as_str() { - "gc" => self - .deprecations - .push(Deprecation::PyClassGcOption, exp.span()), - "weakref" => { - self.has_weaklist = true; - } - "subclass" => { - if self.class_kind == PyClassKind::Enum { - bail_spanned!(exp.span() => "enums can't be inherited by other classes"); - } - self.is_basetype = true; - } - "dict" => { - self.has_dict = true; - } - "unsendable" => { - self.has_unsendable = true; - } - _ => bail_spanned!( - exp.path.span() => "expected one of gc/weakref/subclass/dict/unsendable" - ), - }; - Ok(()) - } } #[derive(Default)] pub struct PyClassPyO3Options { - pub text_signature: Option, - pub deprecations: Deprecations, pub krate: Option, + pub dict: Option, + pub extends: Option, + pub freelist: Option, + pub module: Option, + pub name: Option, + pub subclass: Option, + pub text_signature: Option, + pub unsendable: Option, + pub weakref: Option, + + pub deprecations: Deprecations, } enum PyClassPyO3Option { - TextSignature(TextSignatureAttribute), Crate(CrateAttribute), + Dict(kw::dict), + Extends(ExtendsAttribute), + Freelist(FreelistAttribute), + Module(ModuleAttribute), + Name(NameAttribute), + Subclass(kw::subclass), + TextSignature(TextSignatureAttribute), + Unsendable(kw::unsendable), + Weakref(kw::weakref), + + DeprecatedGC(kw::gc), } impl Parse for PyClassPyO3Option { fn parse(input: ParseStream) -> Result { let lookahead = input.lookahead1(); - if lookahead.peek(attributes::kw::text_signature) { - input.parse().map(PyClassPyO3Option::TextSignature) - } else if lookahead.peek(Token![crate]) { + if lookahead.peek(Token![crate]) { input.parse().map(PyClassPyO3Option::Crate) + } else if lookahead.peek(kw::dict) { + input.parse().map(PyClassPyO3Option::Dict) + } else if lookahead.peek(kw::extends) { + input.parse().map(PyClassPyO3Option::Extends) + } else if lookahead.peek(attributes::kw::freelist) { + input.parse().map(PyClassPyO3Option::Freelist) + } else if lookahead.peek(attributes::kw::module) { + input.parse().map(PyClassPyO3Option::Module) + } else if lookahead.peek(kw::name) { + input.parse().map(PyClassPyO3Option::Name) + } else if lookahead.peek(attributes::kw::subclass) { + input.parse().map(PyClassPyO3Option::Subclass) + } else if lookahead.peek(attributes::kw::text_signature) { + input.parse().map(PyClassPyO3Option::TextSignature) + } else if lookahead.peek(attributes::kw::unsendable) { + input.parse().map(PyClassPyO3Option::Unsendable) + } else if lookahead.peek(attributes::kw::weakref) { + input.parse().map(PyClassPyO3Option::Weakref) + } else if lookahead.peek(attributes::kw::gc) { + input.parse().map(PyClassPyO3Option::DeprecatedGC) } else { Err(lookahead.error()) } @@ -210,57 +111,69 @@ impl Parse for PyClassPyO3Option { } impl PyClassPyO3Options { - pub fn take_pyo3_options(attrs: &mut Vec) -> syn::Result { + fn parse(input: ParseStream) -> syn::Result { let mut options: PyClassPyO3Options = Default::default(); - for option in take_pyo3_options(attrs)? { - match option { - PyClassPyO3Option::TextSignature(text_signature) => { - options.set_text_signature(text_signature)?; - } - PyClassPyO3Option::Crate(path) => { - options.set_crate(path)?; - } - } + + for option in Punctuated::::parse_terminated(input)? { + options.set_option(option)?; } + Ok(options) } - pub fn set_text_signature( - &mut self, - text_signature: TextSignatureAttribute, - ) -> syn::Result<()> { - ensure_spanned!( - self.text_signature.is_none(), - text_signature.kw.span() => "`text_signature` may only be specified once" - ); - self.text_signature = Some(text_signature); - Ok(()) + pub fn take_pyo3_options(&mut self, attrs: &mut Vec) -> syn::Result<()> { + take_pyo3_options(attrs)? + .into_iter() + .try_for_each(|option| self.set_option(option)) } - pub fn set_crate(&mut self, path: CrateAttribute) -> syn::Result<()> { - ensure_spanned!( - self.krate.is_none(), - path.0.span() => "`text_signature` may only be specified once" - ); - self.krate = Some(path); + fn set_option(&mut self, option: PyClassPyO3Option) -> syn::Result<()> { + macro_rules! set_option { + ($key:ident) => { + { + ensure_spanned!( + self.$key.is_none(), + $key.span() => concat!("`", stringify!($key), "` may only be specified once") + ); + self.$key = Some($key); + } + }; + } + + match option { + PyClassPyO3Option::Crate(krate) => set_option!(krate), + PyClassPyO3Option::Dict(dict) => set_option!(dict), + PyClassPyO3Option::Extends(extends) => set_option!(extends), + PyClassPyO3Option::Freelist(freelist) => set_option!(freelist), + PyClassPyO3Option::Module(module) => set_option!(module), + PyClassPyO3Option::Name(name) => set_option!(name), + PyClassPyO3Option::Subclass(subclass) => set_option!(subclass), + PyClassPyO3Option::TextSignature(text_signature) => set_option!(text_signature), + PyClassPyO3Option::Unsendable(unsendable) => set_option!(unsendable), + PyClassPyO3Option::Weakref(weakref) => set_option!(weakref), + + PyClassPyO3Option::DeprecatedGC(gc) => self + .deprecations + .push(Deprecation::PyClassGcOption, gc.span()), + } Ok(()) } } pub fn build_py_class( class: &mut syn::ItemStruct, - args: &PyClassArgs, + mut args: PyClassArgs, methods_type: PyClassMethodsType, ) -> syn::Result { - let options = PyClassPyO3Options::take_pyo3_options(&mut class.attrs)?; + args.options.take_pyo3_options(&mut class.attrs)?; let doc = utils::get_doc( &class.attrs, - options + args.options .text_signature .as_ref() - .map(|attr| (get_class_python_name(&class.ident, args), attr)), + .map(|attr| (get_class_python_name(&class.ident, &args), attr)), ); - let krate = get_pyo3_crate(&options.krate); + let krate = get_pyo3_crate(&args.options.krate); ensure_spanned!( class.generics.params.is_empty(), @@ -290,15 +203,7 @@ pub fn build_py_class( } }; - impl_class( - &class.ident, - args, - doc, - field_options, - methods_type, - options.deprecations, - krate, - ) + impl_class(&class.ident, &args, doc, field_options, methods_type, krate) } /// `#[pyo3()]` options for pyclass fields @@ -356,7 +261,7 @@ impl FieldPyO3Options { FieldPyO3Option::Name(name) => { ensure_spanned!( options.name.is_none(), - name.0.span() => "`name` may only be specified once" + name.span() => "`name` may only be specified once" ); options.name = Some(name); } @@ -367,24 +272,27 @@ impl FieldPyO3Options { } } -fn get_class_python_name<'a>(cls: &'a syn::Ident, attr: &'a PyClassArgs) -> &'a syn::Ident { - attr.name.as_ref().unwrap_or(cls) +fn get_class_python_name<'a>(cls: &'a syn::Ident, args: &'a PyClassArgs) -> &'a syn::Ident { + args.options + .name + .as_ref() + .map(|name_attr| &name_attr.value.0) + .unwrap_or(cls) } fn impl_class( cls: &syn::Ident, - attr: &PyClassArgs, + args: &PyClassArgs, doc: PythonDoc, field_options: Vec<(&syn::Field, FieldPyO3Options)>, methods_type: PyClassMethodsType, - deprecations: Deprecations, krate: syn::Path, ) -> syn::Result { - let pytypeinfo_impl = impl_pytypeinfo(cls, attr, Some(&deprecations)); + let pytypeinfo_impl = impl_pytypeinfo(cls, args, Some(&args.options.deprecations)); let py_class_impl = PyClassImplsBuilder::new( cls, - attr, + args, methods_type, descriptors_to_items(cls, field_options)?, vec![], @@ -458,23 +366,28 @@ impl<'a> PyClassEnum<'a> { pub fn build_py_enum( enum_: &mut syn::ItemEnum, - args: &PyClassArgs, + mut args: PyClassArgs, method_type: PyClassMethodsType, ) -> syn::Result { - let options = PyClassPyO3Options::take_pyo3_options(&mut enum_.attrs)?; + args.options.take_pyo3_options(&mut enum_.attrs)?; - if enum_.variants.is_empty() { - bail_spanned!(enum_.brace_token.span => "Empty enums can't be #[pyclass]."); + if let Some(extends) = &args.options.extends { + bail_spanned!(extends.span() => "enums can't extend from other classes"); + } else if let Some(subclass) = &args.options.subclass { + bail_spanned!(subclass.span() => "enums can't be inherited by other classes"); + } else if enum_.variants.is_empty() { + bail_spanned!(enum_.brace_token.span => "#[pyclass] can't be used on enums without any variants"); } + let doc = utils::get_doc( &enum_.attrs, - options + args.options .text_signature .as_ref() - .map(|attr| (get_class_python_name(&enum_.ident, args), attr)), + .map(|attr| (get_class_python_name(&enum_.ident, &args), attr)), ); let enum_ = PyClassEnum::new(enum_)?; - Ok(impl_enum(enum_, args, doc, method_type, options)) + Ok(impl_enum(enum_, &args, doc, method_type)) } fn impl_enum( @@ -482,9 +395,8 @@ fn impl_enum( args: &PyClassArgs, doc: PythonDoc, methods_type: PyClassMethodsType, - options: PyClassPyO3Options, ) -> TokenStream { - let krate = get_pyo3_crate(&options.krate); + let krate = get_pyo3_crate(&args.options.krate); impl_enum_class(enum_, args, doc, methods_type, krate) } @@ -613,7 +525,10 @@ fn enum_default_methods<'a>( rust_ident: ident.clone(), attributes: ConstAttributes { is_class_attr: true, - name: Some(NameAttribute(ident.clone())), + name: Some(NameAttribute { + kw: syn::parse_quote! { name }, + value: NameLitStr(ident.clone()), + }), deprecations: Default::default(), }, }; @@ -649,7 +564,7 @@ fn descriptors_to_items( .enumerate() .flat_map(|(field_index, (field, options))| { let name_err = if options.name.is_some() && !options.get && !options.set { - Some(Err(err_spanned!(options.name.as_ref().unwrap().0.span() => "`name` is useless without `get` or `set`"))) + Some(Err(err_spanned!(options.name.as_ref().unwrap().span() => "`name` is useless without `get` or `set`"))) } else { None }; @@ -686,8 +601,8 @@ fn impl_pytypeinfo( ) -> TokenStream { let cls_name = get_class_python_name(cls, attr).to_string(); - let module = if let Some(m) = &attr.module { - quote! { ::core::option::Option::Some(#m) } + let module = if let Some(ModuleAttribute { value, .. }) = &attr.options.module { + quote! { ::core::option::Option::Some(#value) } } else { quote! { ::core::option::Option::None } }; @@ -765,20 +680,20 @@ impl<'a> PyClassImplsBuilder<'a> { fn impl_pyclass(&self) -> TokenStream { let cls = self.cls; let attr = self.attr; - let dict = if attr.has_dict { + let dict = if attr.options.dict.is_some() { quote! { _pyo3::impl_::pyclass::PyClassDictSlot } } else { quote! { _pyo3::impl_::pyclass::PyClassDummySlot } }; // insert space for weak ref - let weakref = if attr.has_weaklist { + let weakref = if attr.options.weakref.is_some() { quote! { _pyo3::impl_::pyclass::PyClassWeakRefSlot } } else { quote! { _pyo3::impl_::pyclass::PyClassDummySlot } }; - let base_nativetype = if attr.has_extends { + let base_nativetype = if attr.options.extends.is_some() { quote! { ::BaseNativeType } } else { quote! { _pyo3::PyAny } @@ -810,7 +725,7 @@ impl<'a> PyClassImplsBuilder<'a> { let cls = self.cls; let attr = self.attr; // If #cls is not extended type, we allow Self->PyObject conversion - if !attr.has_extends { + if attr.options.extends.is_none() { quote! { impl _pyo3::IntoPy<_pyo3::PyObject> for #cls { fn into_py(self, py: _pyo3::Python) -> _pyo3::PyObject { @@ -825,11 +740,17 @@ impl<'a> PyClassImplsBuilder<'a> { fn impl_pyclassimpl(&self) -> TokenStream { let cls = self.cls; let doc = self.doc.as_ref().map_or(quote! {"\0"}, |doc| quote! {#doc}); - let is_basetype = self.attr.is_basetype; - let base = &self.attr.base; - let is_subclass = self.attr.has_extends; + let is_basetype = self.attr.options.subclass.is_some(); + let base = self + .attr + .options + .extends + .as_ref() + .map(|extends_attr| extends_attr.value.clone()) + .unwrap_or_else(|| parse_quote! { _pyo3::PyAny }); + let is_subclass = self.attr.options.extends.is_some(); - let dict_offset = if self.attr.has_dict { + let dict_offset = if self.attr.options.dict.is_some() { quote! { fn dict_offset() -> ::std::option::Option<_pyo3::ffi::Py_ssize_t> { ::std::option::Option::Some(_pyo3::impl_::pyclass::dict_offset::()) @@ -840,7 +761,7 @@ impl<'a> PyClassImplsBuilder<'a> { }; // insert space for weak ref - let weaklist_offset = if self.attr.has_weaklist { + let weaklist_offset = if self.attr.options.weakref.is_some() { quote! { fn weaklist_offset() -> ::std::option::Option<_pyo3::ffi::Py_ssize_t> { ::std::option::Option::Some(_pyo3::impl_::pyclass::weaklist_offset::()) @@ -850,9 +771,9 @@ impl<'a> PyClassImplsBuilder<'a> { TokenStream::new() }; - let thread_checker = if self.attr.has_unsendable { + let thread_checker = if self.attr.options.unsendable.is_some() { quote! { _pyo3::impl_::pyclass::ThreadCheckerImpl<#cls> } - } else if self.attr.has_extends { + } else if self.attr.options.extends.is_some() { quote! { _pyo3::impl_::pyclass::ThreadCheckerInherited<#cls, <#cls as _pyo3::impl_::pyclass::PyClassImpl>::BaseType> } @@ -940,7 +861,8 @@ impl<'a> PyClassImplsBuilder<'a> { fn impl_freelist(&self) -> TokenStream { let cls = self.cls; - self.attr.freelist.as_ref().map_or(quote!{}, |freelist| { + self.attr.options.freelist.as_ref().map_or(quote!{}, |freelist| { + let freelist = &freelist.value; quote! { impl _pyo3::impl_::pyclass::PyClassWithFreeList for #cls { #[inline] @@ -962,7 +884,7 @@ impl<'a> PyClassImplsBuilder<'a> { fn freelist_slots(&self) -> Vec { let cls = self.cls; - if self.attr.freelist.is_some() { + if self.attr.options.freelist.is_some() { vec![ quote! { _pyo3::ffi::PyType_Slot { diff --git a/pyo3-macros-backend/src/pyfunction.rs b/pyo3-macros-backend/src/pyfunction.rs index a45c6a9a..3da0044e 100644 --- a/pyo3-macros-backend/src/pyfunction.rs +++ b/pyo3-macros-backend/src/pyfunction.rs @@ -40,7 +40,7 @@ pub struct PyFunctionSignature { has_kwargs: bool, } -#[derive(Clone, PartialEq, Debug)] +#[derive(Clone, Debug)] pub struct PyFunctionArgPyO3Attributes { pub from_py_with: Option, } @@ -71,7 +71,7 @@ impl PyFunctionArgPyO3Attributes { PyFunctionArgPyO3Attribute::FromPyWith(from_py_with) => { ensure_spanned!( attributes.from_py_with.is_none(), - from_py_with.0.span() => "`from_py_with` may only be specified once per argument" + from_py_with.span() => "`from_py_with` may only be specified once per argument" ); attributes.from_py_with = Some(from_py_with); } @@ -339,7 +339,7 @@ impl PyFunctionOptions { PyFunctionOption::Crate(path) => { ensure_spanned!( self.krate.is_none(), - path.0.span() => "`crate` may only be specified once" + path.span() => "`crate` may only be specified once" ); self.krate = Some(path); } @@ -351,7 +351,7 @@ impl PyFunctionOptions { pub fn set_name(&mut self, name: NameAttribute) -> Result<()> { ensure_spanned!( self.name.is_none(), - name.0.span() => "`name` may only be specified once" + name.span() => "`name` may only be specified once" ); self.name = Some(name); Ok(()) @@ -377,7 +377,7 @@ pub fn impl_wrap_pyfunction( let python_name = options .name - .map_or_else(|| func.sig.ident.unraw(), |name| name.0); + .map_or_else(|| func.sig.ident.unraw(), |name| name.value.0); let signature = options.signature.unwrap_or_default(); diff --git a/pyo3-macros-backend/src/pyimpl.rs b/pyo3-macros-backend/src/pyimpl.rs index e8f23819..ebc05a45 100644 --- a/pyo3-macros-backend/src/pyimpl.rs +++ b/pyo3-macros-backend/src/pyimpl.rs @@ -61,7 +61,7 @@ impl PyImplOptions { fn set_crate(&mut self, path: CrateAttribute) -> Result<()> { ensure_spanned!( self.krate.is_none(), - path.0.span() => "`crate` may only be specified once" + path.span() => "`crate` may only be specified once" ); self.krate = Some(path); diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index ef43419e..5fd66c9d 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -544,7 +544,7 @@ impl PropertyType<'_> { field, python_name, .. } => { let name = match (python_name, &field.ident) { - (Some(name), _) => name.0.to_string(), + (Some(name), _) => name.value.0.to_string(), (None, Some(field_name)) => format!("{}\0", field_name.unraw()), (None, None) => { bail_spanned!(field.span() => "`get` and `set` with tuple struct fields require `name`"); diff --git a/pyo3-macros-backend/src/utils.rs b/pyo3-macros-backend/src/utils.rs index 9c75239e..39475d9b 100644 --- a/pyo3-macros-backend/src/utils.rs +++ b/pyo3-macros-backend/src/utils.rs @@ -77,7 +77,8 @@ pub fn get_doc( syn::token::Bracket(Span::call_site()).surround(&mut tokens, |tokens| { if let Some((python_name, text_signature)) = text_signature { // create special doc string lines to set `__text_signature__` - let signature_lines = format!("{}{}\n--\n\n", python_name, text_signature.lit.value()); + let signature_lines = + format!("{}{}\n--\n\n", python_name, text_signature.value.value()); signature_lines.to_tokens(tokens); comma.to_tokens(tokens); } @@ -154,13 +155,6 @@ pub fn ensure_not_async_fn(sig: &syn::Signature) -> syn::Result<()> { Ok(()) } -pub fn unwrap_group(mut expr: &syn::Expr) -> &syn::Expr { - while let syn::Expr::Group(g) = expr { - expr = &*g.expr; - } - expr -} - pub fn unwrap_ty_group(mut ty: &syn::Type) -> &syn::Type { while let syn::Type::Group(g) = ty { ty = &*g.elem; @@ -193,6 +187,6 @@ pub(crate) fn replace_self(ty: &mut syn::Type, cls: &syn::Type) { /// Extract the path to the pyo3 crate, or use the default (`::pyo3`). pub(crate) fn get_pyo3_crate(attr: &Option) -> syn::Path { attr.as_ref() - .map(|p| p.0.clone()) + .map(|p| p.value.0.clone()) .unwrap_or_else(|| syn::parse_str("::pyo3").unwrap()) } diff --git a/pyo3-macros/src/lib.rs b/pyo3-macros/src/lib.rs index 06738053..367b37fa 100644 --- a/pyo3-macros/src/lib.rs +++ b/pyo3-macros/src/lib.rs @@ -81,17 +81,34 @@ pub fn pyproto(_: TokenStream, input: TokenStream) -> TokenStream { /// A proc macro used to expose Rust structs and fieldless enums as Python objects. /// -/// `#[pyclass]` accepts the following [parameters][2]: +/// `#[pyclass]` can be used the following [parameters][2]: /// /// | Parameter | Description | /// | :- | :- | -/// | `name = "python_name"` | Sets the name that Python sees this class as. Defaults to the name of the Rust struct. | -/// | `freelist = N` | Implements a [free list][9] of size N. This can improve performance for types that are often created and deleted in quick succession. Profile your code to see whether `freelist` is right for you. | -/// | `weakref` | Allows this class to be [weakly referenceable][6]. | +/// | `crate = "some::path"` | Path to import the `pyo3` crate, if it's not accessible at `::pyo3`. | +/// | `dict` | Gives instances of this class an empty `__dict__` to store custom attributes. | /// | `extends = BaseType` | Use a custom baseclass. Defaults to [`PyAny`][4] | +/// | `freelist = N` | Implements a [free list][9] of size N. This can improve performance for types that are often created and deleted in quick succession. Profile your code to see whether `freelist` is right for you. | +/// | `module = "module_name"` | Python code will see the class as being defined in this module. Defaults to `builtins`. | +/// | `name = "python_name"` | Sets the name that Python sees this class as. Defaults to the name of the Rust struct. | +/// | `text_signature = "(arg1, arg2, ...)"` | Sets the text signature for the Python class' `__new__` method. | /// | `subclass` | Allows other Python classes and `#[pyclass]` to inherit from this class. Enums cannot be subclassed. | /// | `unsendable` | Required if your struct is not [`Send`][3]. Rather than using `unsendable`, consider implementing your struct in a threadsafe way by e.g. substituting [`Rc`][7] with [`Arc`][8]. By using `unsendable`, your class will panic when accessed by another thread.| -/// | `module = "module_name"` | Python code will see the class as being defined in this module. Defaults to `builtins`. | +/// | `weakref` | Allows this class to be [weakly referenceable][6]. | +/// +/// All of these parameters can either be passed directly on the `#[pyclass(...)]` annotation, or as one or +/// more accompanying `#[pyo3(...)]` annotations, e.g.: +/// +/// ```rust,ignore +/// // Argument supplied directly to the `#[pyclass]` annotation. +/// #[pyclass(name = "SomeName", subclass)] +/// struct MyClass { } +/// +/// // Argument supplied as a separate annotation. +/// #[pyclass] +/// #[pyo3(name = "SomeName", subclass)] +/// struct MyClass { } +/// ``` /// /// For more on creating Python classes, /// see the [class section of the guide][1]. @@ -230,7 +247,7 @@ fn pyclass_impl( methods_type: PyClassMethodsType, ) -> TokenStream { let args = parse_macro_input!(attrs with PyClassArgs::parse_stuct_args); - let expanded = build_py_class(&mut ast, &args, methods_type).unwrap_or_compile_error(); + let expanded = build_py_class(&mut ast, args, methods_type).unwrap_or_compile_error(); quote!( #ast @@ -245,7 +262,7 @@ fn pyclass_enum_impl( methods_type: PyClassMethodsType, ) -> TokenStream { let args = parse_macro_input!(attrs with PyClassArgs::parse_enum_args); - let expanded = build_py_enum(&mut ast, &args, methods_type).unwrap_or_compile_error(); + let expanded = build_py_enum(&mut ast, args, methods_type).unwrap_or_compile_error(); quote!( #ast diff --git a/tests/ui/invalid_property_args.stderr b/tests/ui/invalid_property_args.stderr index 2147682c..a13e40f1 100644 --- a/tests/ui/invalid_property_args.stderr +++ b/tests/ui/invalid_property_args.stderr @@ -35,13 +35,13 @@ error: `set` may only be specified once | ^^^ error: `name` may only be specified once - --> tests/ui/invalid_property_args.rs:37:49 + --> tests/ui/invalid_property_args.rs:37:42 | 37 | struct MultipleName(#[pyo3(name = "foo", name = "bar")] i32); - | ^^^^^ + | ^^^^ error: `name` is useless without `get` or `set` - --> tests/ui/invalid_property_args.rs:40:40 + --> tests/ui/invalid_property_args.rs:40:33 | 40 | struct NameWithoutGetSet(#[pyo3(name = "value")] i32); - | ^^^^^^^ + | ^^^^ diff --git a/tests/ui/invalid_pyclass_args.stderr b/tests/ui/invalid_pyclass_args.stderr index d166cc81..fd64e06e 100644 --- a/tests/ui/invalid_pyclass_args.stderr +++ b/tests/ui/invalid_pyclass_args.stderr @@ -1,40 +1,40 @@ -error: expected one of freelist/name/extends/module +error: expected one of: `crate`, `dict`, `extends`, `freelist`, `module`, `name`, `subclass`, `text_signature`, `unsendable`, `weakref`, `gc` --> tests/ui/invalid_pyclass_args.rs:3:11 | 3 | #[pyclass(extend=pyo3::types::PyDict)] | ^^^^^^ -error: expected type path (e.g., my_mod::BaseClass) +error: expected identifier --> tests/ui/invalid_pyclass_args.rs:6:21 | 6 | #[pyclass(extends = "PyDict")] | ^^^^^^^^ -error: expected type name (e.g. "Name") +error: expected string literal --> tests/ui/invalid_pyclass_args.rs:9:18 | 9 | #[pyclass(name = m::MyClass)] | ^ -error: expected a single identifier in double-quotes +error: expected a single identifier in double quotes --> tests/ui/invalid_pyclass_args.rs:12:18 | 12 | #[pyclass(name = "Custom Name")] | ^^^^^^^^^^^^^ -error: since PyO3 0.13 a pyclass name should be in double-quotes, e.g. "CustomName" +error: expected string literal --> tests/ui/invalid_pyclass_args.rs:15:18 | 15 | #[pyclass(name = CustomName)] | ^^^^^^^^^^ -error: expected string literal (e.g., "my_mod") +error: expected string literal --> tests/ui/invalid_pyclass_args.rs:18:20 | 18 | #[pyclass(module = my_module)] | ^^^^^^^^^ -error: expected one of gc/weakref/subclass/dict/unsendable +error: expected one of: `crate`, `dict`, `extends`, `freelist`, `module`, `name`, `subclass`, `text_signature`, `unsendable`, `weakref`, `gc` --> tests/ui/invalid_pyclass_args.rs:21:11 | 21 | #[pyclass(weakrev)] diff --git a/tests/ui/invalid_pyclass_enum.rs b/tests/ui/invalid_pyclass_enum.rs index a12accbf..4bc53238 100644 --- a/tests/ui/invalid_pyclass_enum.rs +++ b/tests/ui/invalid_pyclass_enum.rs @@ -2,14 +2,14 @@ use pyo3::prelude::*; #[pyclass(subclass)] enum NotBaseClass { - x, - y, + X, + Y, } #[pyclass(extends = PyList)] enum NotDrivedClass { - x, - y, + X, + Y, } #[pyclass] diff --git a/tests/ui/invalid_pyclass_enum.stderr b/tests/ui/invalid_pyclass_enum.stderr index eea36e5c..8f340a76 100644 --- a/tests/ui/invalid_pyclass_enum.stderr +++ b/tests/ui/invalid_pyclass_enum.stderr @@ -4,13 +4,13 @@ error: enums can't be inherited by other classes 3 | #[pyclass(subclass)] | ^^^^^^^^ -error: enums cannot extend from other classes +error: enums can't extend from other classes --> tests/ui/invalid_pyclass_enum.rs:9:11 | 9 | #[pyclass(extends = PyList)] | ^^^^^^^ -error: Empty enums can't be #[pyclass]. +error: #[pyclass] can't be used on enums without any variants --> tests/ui/invalid_pyclass_enum.rs:16:18 | 16 | enum NoEmptyEnum {} diff --git a/tests/ui/invalid_pymethod_names.stderr b/tests/ui/invalid_pymethod_names.stderr index 8aed1d41..c99c692c 100644 --- a/tests/ui/invalid_pymethod_names.stderr +++ b/tests/ui/invalid_pymethod_names.stderr @@ -5,10 +5,10 @@ error: `name` may only be specified once | ^^^^^ error: `name` may only be specified once - --> tests/ui/invalid_pymethod_names.rs:18:19 + --> tests/ui/invalid_pymethod_names.rs:18:12 | 18 | #[pyo3(name = "bar")] - | ^^^^^ + | ^^^^ error: `name` not allowed with `#[new]` --> tests/ui/invalid_pymethod_names.rs:24:19 From 143b7d368fd0fe3f18339a3bc1da22c352fd9909 Mon Sep 17 00:00:00 2001 From: Mo Mirza Date: Sun, 20 Mar 2022 15:45:33 +0000 Subject: [PATCH 4/9] Replace nbsp with space (#2237) This fixes markdown heading rendering --- guide/src/migration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/guide/src/migration.md b/guide/src/migration.md index 6996f9a9..5031fcec 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -9,7 +9,7 @@ For a detailed list of all changes, see the [CHANGELOG](changelog.md). PyO3 0.16 has increased minimum Rust version to 1.48 and minimum Python version to 3.7. This enables use of newer language features (enabling some of the other additions in 0.16) and simplifies maintenance of the project. -### `#[pyproto]` has been deprecated +### `#[pyproto]` has been deprecated In PyO3 0.15, the `#[pymethods]` attribute macro gained support for implementing "magic methods" such as `__str__` (aka "dunder" methods). This implementation was not quite finalized at the time, with a few edge cases to be decided upon. The existing `#[pyproto]` attribute macro was left untouched, because it covered these edge cases. From c734b116f9921944667b5d53991f66fb4fd1161f Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Mon, 21 Mar 2022 23:53:08 +0000 Subject: [PATCH 5/9] macros: fix syn patch version --- CHANGELOG.md | 2 +- pyo3-macros-backend/Cargo.toml | 2 +- pyo3-macros/Cargo.toml | 2 +- pyo3-macros/src/lib.rs | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9615c5a1..e56e57d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,8 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Considered `PYTHONFRAMEWORK` when cross compiling in order that on macos cross compiling against a [Framework bundle](https://developer.apple.com/library/archive/documentation/MacOSX/Conceptual/BPFrameworks/Concepts/FrameworkAnatomy.html) is considered shared. [#2233](https://github.com/PyO3/pyo3/pull/2233) - - Panic during compilation when `PYO3_CROSS_LIB_DIR` is set for some host/target combinations. [#2232](https://github.com/PyO3/pyo3/pull/2232) +- Correct dependency version for `syn` to require correct minimal patch version 1.0.56. [#2240](https://github.com/PyO3/pyo3/pull/2240) ### Added diff --git a/pyo3-macros-backend/Cargo.toml b/pyo3-macros-backend/Cargo.toml index ee59dc8a..8d6ff323 100644 --- a/pyo3-macros-backend/Cargo.toml +++ b/pyo3-macros-backend/Cargo.toml @@ -19,7 +19,7 @@ proc-macro2 = { version = "1", default-features = false } pyo3-build-config = { path = "../pyo3-build-config", version = "0.16.2", features = ["resolve-config"] } [dependencies.syn] -version = "1" +version = "1.0.56" default-features = false features = ["derive", "parsing", "printing", "clone-impls", "full", "extra-traits"] diff --git a/pyo3-macros/Cargo.toml b/pyo3-macros/Cargo.toml index 927b2ad3..08bd5c5f 100644 --- a/pyo3-macros/Cargo.toml +++ b/pyo3-macros/Cargo.toml @@ -21,5 +21,5 @@ pyproto = ["pyo3-macros-backend/pyproto"] [dependencies] proc-macro2 = { version = "1", default-features = false } quote = "1" -syn = { version = "1", features = ["full", "extra-traits"] } +syn = { version = "1.0.56", features = ["full", "extra-traits"] } pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.16.2" } diff --git a/pyo3-macros/src/lib.rs b/pyo3-macros/src/lib.rs index 06738053..d21db179 100644 --- a/pyo3-macros/src/lib.rs +++ b/pyo3-macros/src/lib.rs @@ -38,11 +38,11 @@ pub fn pymodule(args: TokenStream, input: TokenStream) -> TokenStream { let mut ast = parse_macro_input!(input as syn::ItemFn); let options = match PyModuleOptions::from_attrs(&mut ast.attrs) { Ok(options) => options, - Err(e) => return e.to_compile_error().into(), + Err(e) => return e.into_compile_error().into(), }; if let Err(err) = process_functions_in_module(&mut ast) { - return err.to_compile_error().into(); + return err.into_compile_error().into(); } let doc = get_doc(&ast.attrs, None); @@ -114,7 +114,7 @@ pub fn pyclass(attr: TokenStream, input: TokenStream) -> TokenStream { Item::Enum(enum_) => pyclass_enum_impl(attr, enum_, methods_type()), unsupported => { syn::Error::new_spanned(unsupported, "#[pyclass] only supports structs and enums.") - .to_compile_error() + .into_compile_error() .into() } } From 847ffe563cd33843c9e2d12bb9b86fff4d54a037 Mon Sep 17 00:00:00 2001 From: zhangjingqiang Date: Mon, 21 Mar 2022 08:58:26 +0800 Subject: [PATCH 6/9] allow to compile with parking_lot 0.12 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 0c83a75c..6b66c39d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ edition = "2018" [dependencies] cfg-if = "1.0" libc = "0.2.62" -parking_lot = "0.11.0" +parking_lot = ">= 0.11, < 0.13" # ffi bindings to the python interpreter, split into a seperate crate so they can be used independently pyo3-ffi = { path = "pyo3-ffi", version = "=0.16.2" } From 0aead58fcd88d6e92c7d0f7e0a228022bbac1165 Mon Sep 17 00:00:00 2001 From: messense Date: Mon, 21 Mar 2022 11:23:27 +0800 Subject: [PATCH 7/9] Fix minimal package version for parking_lot --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e0cd74a..4a127175 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -175,8 +175,13 @@ jobs: - if: matrix.msrv == 'MSRV' name: Prepare minimal package versions (MSRV only) run: | + set -x cargo update -p indexmap --precise 1.6.2 cargo update -p hashbrown:0.12.0 --precise 0.9.1 + PROJECTS=("." "examples/decorator" "examples/maturin-starter" "examples/setuptools-rust-starter" "examples/word-count") + for PROJ in ${PROJECTS[@]}; do + cargo update --manifest-path "$PROJ/Cargo.toml" -p parking_lot --precise 0.11.0 + done - name: Build docs run: cargo doc --no-deps --no-default-features --features "full ${{ matrix.extra_features }}" From ad8ffaad2e25f2266cb77004bfbf9fccdec06028 Mon Sep 17 00:00:00 2001 From: messense Date: Tue, 22 Mar 2022 13:51:58 +0800 Subject: [PATCH 8/9] Update changelog for parking_lot supported versions --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e56e57d2..0fcd857a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `as_bytes` on `Py`. [#2235](https://github.com/PyO3/pyo3/pull/2235) +### Packaging + +- Extend `parking_lot` dependency supported versions to include 0.12. [#2239](https://github.com/PyO3/pyo3/pull/2239) + ## [0.16.2] - 2022-03-15 ### Packaging From 49c1d22606be647d651d6e908953569d6fa65a12 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Tue, 22 Mar 2022 10:38:36 +0000 Subject: [PATCH 9/9] docs: for #2234 --- CHANGELOG.md | 4 +++ guide/src/class.md | 29 +++++++---------- pyo3-macros/docs/pyclass_parameters.md | 28 +++++++++++++++++ pyo3-macros/src/lib.rs | 43 +++++--------------------- 4 files changed, 50 insertions(+), 54 deletions(-) create mode 100644 pyo3-macros/docs/pyclass_parameters.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bded739..37c09d50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- Allow `#[pyo3(crate = "...", text_signature = "...")]` options to be used directly in `#[pyclass(crate = "...", text_signature = "...")]`. [#2234](https://github.com/PyO3/pyo3/pull/2234) + ### Fixed - Considered `PYTHONFRAMEWORK` when cross compiling in order that on macos cross compiling against a [Framework bundle](https://developer.apple.com/library/archive/documentation/MacOSX/Conceptual/BPFrameworks/Concepts/FrameworkAnatomy.html) is considered shared. [#2233](https://github.com/PyO3/pyo3/pull/2233) diff --git a/guide/src/class.md b/guide/src/class.md index e40e4a5f..3b6ad618 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -195,22 +195,16 @@ Python::with_gil(|py|{ ## Customizing the class -The `#[pyclass]` macro accepts the following parameters: +{{#include ../../pyo3-macros/docs/pyclass_parameters.md}} -* `name="XXX"` - Set the class name shown in Python code. By default, the struct name is used as the class name. -* `freelist=XXX` - The `freelist` parameter adds support of free allocation list to custom class. -The performance improvement applies to types that are often created and deleted in a row, -so that they can benefit from a freelist. `XXX` is a number of items for the free list. -* `gc` - Classes with the `gc` parameter participate in Python garbage collection. -If a custom class contains references to other Python objects that can be collected, the [`PyGCProtocol`]({{#PYO3_DOCS_URL}}/pyo3/class/gc/trait.PyGCProtocol.html) trait has to be implemented. -* `weakref` - Adds support for Python weak references. -* `extends=BaseType` - Use a custom base class. The base `BaseType` must implement `PyTypeInfo`. `enum` pyclasses can't use a custom base class. -* `subclass` - Allows Python classes to inherit from this class. `enum` pyclasses can't be inherited from. -* `dict` - Adds `__dict__` support, so that the instances of this type have a dictionary containing arbitrary instance variables. -* `unsendable` - Making it safe to expose `!Send` structs to Python, where all object can be accessed - by multiple threads. A class marked with `unsendable` panics when accessed by another thread. -* `module="XXX"` - Set the name of the module the class will be shown as defined in. If not given, the class - will be a virtual member of the `builtins` module. +[params-1]: {{#PYO3_DOCS_URL}}/pyo3/prelude/struct.PyAny.html +[params-2]: https://en.wikipedia.org/wiki/Free_list +[params-3]: https://doc.rust-lang.org/stable/std/marker/trait.Send.html +[params-4]: https://doc.rust-lang.org/stable/std/rc/struct.Rc.html +[params-5]: https://doc.rust-lang.org/stable/std/sync/struct.Rc.html +[params-6]: https://docs.python.org/3/library/weakref.html + +These parameters are covered in various sections of this guide. ### Return type @@ -716,7 +710,7 @@ num=-1 ## Making class method signatures available to Python -The [`#[pyo3(text_signature = "...")]`](./function.md#text_signature) option for `#[pyfunction]` also works for classes and methods: +The [`text_signature = "..."`](./function.md#text_signature) option for `#[pyfunction]` also works for classes and methods: ```rust # #![allow(dead_code)] @@ -724,8 +718,7 @@ use pyo3::prelude::*; use pyo3::types::PyType; // it works even if the item is not documented: -#[pyclass] -#[pyo3(text_signature = "(c, d, /)")] +#[pyclass(text_signature = "(c, d, /)")] struct MyClass {} #[pymethods] diff --git a/pyo3-macros/docs/pyclass_parameters.md b/pyo3-macros/docs/pyclass_parameters.md new file mode 100644 index 00000000..ae5ef9dd --- /dev/null +++ b/pyo3-macros/docs/pyclass_parameters.md @@ -0,0 +1,28 @@ +`#[pyclass]` can be used with the following parameters: + +| Parameter | Description | +| :- | :- | +| `crate = "some::path"` | Path to import the `pyo3` crate, if it's not accessible at `::pyo3`. | +| `dict` | Gives instances of this class an empty `__dict__` to store custom attributes. | +| `extends = BaseType` | Use a custom baseclass. Defaults to [`PyAny`][params-1] | +| `freelist = N` | Implements a [free list][params-2] of size N. This can improve performance for types that are often created and deleted in quick succession. Profile your code to see whether `freelist` is right for you. | +| `module = "module_name"` | Python code will see the class as being defined in this module. Defaults to `builtins`. | +| `name = "python_name"` | Sets the name that Python sees this class as. Defaults to the name of the Rust struct. | +| `text_signature = "(arg1, arg2, ...)"` | Sets the text signature for the Python class' `__new__` method. | +| `subclass` | Allows other Python classes and `#[pyclass]` to inherit from this class. Enums cannot be subclassed. | +| `unsendable` | Required if your struct is not [`Send`][params-3]. Rather than using `unsendable`, consider implementing your struct in a threadsafe way by e.g. substituting [`Rc`][params-4] with [`Arc`][params-5]. By using `unsendable`, your class will panic when accessed by another thread.| +| `weakref` | Allows this class to be [weakly referenceable][params-6]. | + +All of these parameters can either be passed directly on the `#[pyclass(...)]` annotation, or as one or +more accompanying `#[pyo3(...)]` annotations, e.g.: + +```rust,ignore +// Argument supplied directly to the `#[pyclass]` annotation. +#[pyclass(name = "SomeName", subclass)] +struct MyClass { } + +// Argument supplied as a separate annotation. +#[pyclass] +#[pyo3(name = "SomeName", subclass)] +struct MyClass { } +``` diff --git a/pyo3-macros/src/lib.rs b/pyo3-macros/src/lib.rs index 367b37fa..a5f7faa7 100644 --- a/pyo3-macros/src/lib.rs +++ b/pyo3-macros/src/lib.rs @@ -81,47 +81,18 @@ pub fn pyproto(_: TokenStream, input: TokenStream) -> TokenStream { /// A proc macro used to expose Rust structs and fieldless enums as Python objects. /// -/// `#[pyclass]` can be used the following [parameters][2]: -/// -/// | Parameter | Description | -/// | :- | :- | -/// | `crate = "some::path"` | Path to import the `pyo3` crate, if it's not accessible at `::pyo3`. | -/// | `dict` | Gives instances of this class an empty `__dict__` to store custom attributes. | -/// | `extends = BaseType` | Use a custom baseclass. Defaults to [`PyAny`][4] | -/// | `freelist = N` | Implements a [free list][9] of size N. This can improve performance for types that are often created and deleted in quick succession. Profile your code to see whether `freelist` is right for you. | -/// | `module = "module_name"` | Python code will see the class as being defined in this module. Defaults to `builtins`. | -/// | `name = "python_name"` | Sets the name that Python sees this class as. Defaults to the name of the Rust struct. | -/// | `text_signature = "(arg1, arg2, ...)"` | Sets the text signature for the Python class' `__new__` method. | -/// | `subclass` | Allows other Python classes and `#[pyclass]` to inherit from this class. Enums cannot be subclassed. | -/// | `unsendable` | Required if your struct is not [`Send`][3]. Rather than using `unsendable`, consider implementing your struct in a threadsafe way by e.g. substituting [`Rc`][7] with [`Arc`][8]. By using `unsendable`, your class will panic when accessed by another thread.| -/// | `weakref` | Allows this class to be [weakly referenceable][6]. | -/// -/// All of these parameters can either be passed directly on the `#[pyclass(...)]` annotation, or as one or -/// more accompanying `#[pyo3(...)]` annotations, e.g.: -/// -/// ```rust,ignore -/// // Argument supplied directly to the `#[pyclass]` annotation. -/// #[pyclass(name = "SomeName", subclass)] -/// struct MyClass { } -/// -/// // Argument supplied as a separate annotation. -/// #[pyclass] -/// #[pyo3(name = "SomeName", subclass)] -/// struct MyClass { } -/// ``` +#[cfg_attr(docsrs, cfg_attr(docsrs, doc = include_str!("../docs/pyclass_parameters.md")))] /// /// For more on creating Python classes, /// see the [class section of the guide][1]. /// /// [1]: https://pyo3.rs/latest/class.html -/// [2]: https://pyo3.rs/latest/class.html#customizing-the-class -/// [3]: std::marker::Send -/// [4]: ../prelude/struct.PyAny.html -/// [5]: https://pyo3.rs/latest/class/protocols.html#garbage-collector-integration -/// [6]: https://docs.python.org/3/library/weakref.html -/// [7]: std::rc::Rc -/// [8]: std::sync::Arc -/// [9]: https://en.wikipedia.org/wiki/Free_list +/// [params-1]: ../prelude/struct.PyAny.html +/// [params-2]: https://en.wikipedia.org/wiki/Free_list +/// [params-3]: std::marker::Send +/// [params-4]: std::rc::Rc +/// [params-5]: std::sync::Arc +/// [params-6]: https://docs.python.org/3/library/weakref.html #[proc_macro_attribute] pub fn pyclass(attr: TokenStream, input: TokenStream) -> TokenStream { use syn::Item;