Expand on xtask (#2176)

* Fix Windows OSError * Ignore .pyd files * Put things in modules * Rename functions to `run` * Expand on cargo xtask * Try to work around missing installs * Run all things by default, but not llvm-cov * Test msrv * Fix more OSErrors * Various refinements and docs * Various refinements * Various refinements
2022-03-18 23:13:23 +01:00 · 2022-03-18 23:13:23 +01:00 · 16ad15e04f
parent 3eb654c81c
commit 16ad15e04f
17 changed files with 620 additions and 207 deletions
--- a/.cargo/config
+++ b/.cargo/config
@ -1,11 +1,5 @@
 [alias]
 xtask = "run --package xtask --"
-pyo3_doc = "doc --lib --no-default-features --features=full --no-deps --workspace --open --exclude pyo3-macros --exclude pyo3-macros-backend"
-pyo3_doc_scrape = "doc --lib --no-default-features --features=full --no-deps --workspace --open --exclude pyo3-macros --exclude pyo3-macros-backend -Z unstable-options -Z rustdoc-scrape-examples=examples"
-pyo3_doc_internal = "doc --lib --no-default-features --features=full --no-deps --workspace --open --document-private-items -Z unstable-options -Z rustdoc-scrape-examples=examples"
-
-[build]
-rustdocflags = ["--cfg", "docsrs"]

 [target.'cfg(feature = "cargo-clippy")']
 rustflags = [
@ -21,4 +15,4 @@ rustflags = [
    "-Dclippy::todo",
    "-Dclippy::unnecessary_wraps",
    "-Dclippy::useless_transmute",
-]
+]
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@ -5,9 +5,6 @@ Please consider adding the following to your pull request:
 - docs to all new functions and / or detail in the guide
 - tests for all new or changed functions

-Be aware the CI pipeline will check your pull request for the following. This is done using `nox` (you can install with `pip install nox`):
- - Rust tests (`cargo test` or `nox -s test-rust`)
- - Examples (`nox -s test-py`)
- - Rust lints (`nox -s clippy`)
- - Rust formatting (`nox -s fmt-rust`)
- - Python formatting (`nox -s fmt-py`)
+PyO3's CI pipeline will check your pull request. To run its tests
+locally, you can run ```cargo xtask ci```. See its documentation
+ [here](https://github.com/PyO3/pyo3/tree/main/xtask#readme).
--- a/.github/workflows/guide.yml
+++ b/.github/workflows/guide.yml
@ -45,7 +45,7 @@ jobs:
          mkdir target
          mkdir -p gh-pages-build/internal
          echo "<div class='internal-banner' style='position:fixed; z-index: 99999; color:red;border:3px solid red;margin-left: auto; margin-right: auto; width: 430px;left:0;right: 0;'><div style='display: flex; align-items: center; justify-content: center;'> ⚠️ Internal Docs ⚠️ Not Public API 👉 <a href='https://pyo3.rs/main/doc/pyo3/index.html' style='color:red;text-decoration:underline;'>Official Docs Here</a></div></div>" > target/banner.html
-          cargo +nightly pyo3_doc_internal
+          cargo xtask doc --internal
          cp -r target/doc gh-pages-build/internal
        env:
          RUSTDOCFLAGS: "--cfg docsrs --Z unstable-options --document-hidden-items --html-before-content target/banner.html"
@ -71,7 +71,7 @@ jobs:
        # This adds the docs to gh-pages-build/doc
      - name: Build the doc
        run: |
-          cargo +nightly pyo3_doc_scrape
+          cargo xtask doc
          cp -r target/doc gh-pages-build/doc
          echo "<meta http-equiv=refresh content=0;url=pyo3/index.html>" > gh-pages-build/doc/index.html

--- a/.gitignore
+++ b/.gitignore
@ -20,3 +20,4 @@ guide/book/
 extensions/stamps/
 pip-wheel-metadata
 valgrind-python.supp
+*.pyd
--- a/Contributing.md
+++ b/Contributing.md
@ -48,7 +48,7 @@ There are some specific areas of focus where help is currently needed for the do
 - Not all APIs had docs or examples when they were made. The goal is to have documentation on all PyO3 APIs ([#306](https://github.com/PyO3/pyo3/issues/306)). If you see an API lacking a doc, please write one and open a PR!

 You can build the docs (including all features) with
-```cargo +nightly pyo3_doc_scrape```
+```cargo xtask doc --open```

 #### Doctests

@ -87,6 +87,10 @@ Tests run with all supported Python versions with the latest stable Rust compile

 If you are adding a new feature, you should add it to the `full` feature in our *Cargo.toml** so that it is tested in CI.

+You can run these tests yourself with
+```cargo xtask ci```
+See [it's documentation](https://github.com/PyO3/pyo3/tree/main/xtask#readme)for more commands you can run.
+
 ## Python and Rust version support policy

 PyO3 aims to keep sufficient compatibility to make packaging Python extensions built with PyO3 feasible on most common package managers.
--- a/pytests/tests/test_datetime.py
+++ b/pytests/tests/test_datetime.py
@ -54,8 +54,9 @@ elif _pointer_size == 4:
 else:
    raise RuntimeError("unexpected pointer size: " + repr(_pointer_size))
 IS_WINDOWS = sys.platform == "win32"
+
 if IS_WINDOWS:
-    MIN_DATETIME = pdt.datetime(1970, 1, 2, 0, 0)
+    MIN_DATETIME = pdt.datetime(1971, 1, 2, 0, 0)
    if IS_32_BIT:
        MAX_DATETIME = pdt.datetime(3001, 1, 19, 4, 59, 59)
    else:
@ -227,7 +228,7 @@ def test_datetime_typeerror():


@given(dt=st.datetimes(MIN_DATETIME, MAX_DATETIME))
-@example(dt=pdt.datetime(1970, 1, 2, 0, 0))
+@example(dt=pdt.datetime(1971, 1, 2, 0, 0))
 def test_datetime_from_timestamp(dt):
    if PYPY and dt < pdt.datetime(1900, 1, 1):
        pytest.xfail("pdt.datetime.timestamp will raise on PyPy with dates before 1900")
--- a/xtask/Cargo.toml
+++ b/xtask/Cargo.toml
@ -3,9 +3,13 @@ name = "xtask"
 version = "0.1.0"
 edition = "2018"

+[[bin]]
+name = "xtask"
+
 [dependencies]
 anyhow = "1.0.51"

 # Clap 3 requires MSRV 1.54
 rustversion = "1.0"
 structopt = { version = "0.3", default-features = false }
+clap = { version = "2" }
--- a/xtask/README.md
+++ b/xtask/README.md
@ -0,0 +1,23 @@
+## Commands to test PyO3.
+
+To run these commands, you should be in PyO3's root directory, and run (for example) `cargo xtask ci`.
+
+```
+USAGE:
+    xtask.exe <SUBCOMMAND>
+
+FLAGS:
+    -h, --help       Prints help information
+    -V, --version    Prints version information
+
+SUBCOMMANDS:
+    ci          Runs everything
+    clippy      Runs `clippy`, denying all warnings
+    coverage    Runs `cargo llvm-cov` for the PyO3 codebase
+    default     Only runs the fast things (this is used if no command is specified)
+    doc         Attempts to render the documentation
+    fmt         Checks Rust and Python code formatting with `rustfmt` and `black`
+    help        Prints this message or the help of the given subcommand(s)
+    test        Runs various variations on `cargo test`
+    test-py     Runs the tests in examples/ and pytests/
+```
--- a/xtask/src/cli.rs
+++ b/xtask/src/cli.rs
@ -0,0 +1,201 @@
+use crate::utils::*;
+use anyhow::{ensure, Result};
+use std::io;
+use std::process::{Command, Stdio};
+use std::time::Instant;
+use structopt::StructOpt;
+
+pub const MSRV: &str = "1.48";
+
+#[derive(StructOpt)]
+pub enum Subcommand {
+    /// Only runs the fast things (this is used if no command is specified)
+    Default,
+    /// Runs everything
+    Ci,
+    /// Checks Rust and Python code formatting with `rustfmt` and `black`
+    Fmt,
+    /// Runs `clippy`, denying all warnings.
+    Clippy,
+    /// Runs `cargo llvm-cov` for the PyO3 codebase.
+    Coverage(CoverageOpts),
+    /// Attempts to render the documentation.
+    Doc(DocOpts),
+    /// Runs various variations on `cargo test`
+    Test,
+    /// Runs the tests in examples/ and pytests/
+    TestPy,
+}
+
+impl Default for Subcommand {
+    fn default() -> Self {
+        Self::Default
+    }
+}
+
+#[derive(StructOpt, Default)]
+pub struct CoverageOpts {
+    /// Creates an lcov output instead of printing to the terminal.
+    #[structopt(long)]
+    pub output_lcov: Option<String>,
+}
+
+#[derive(StructOpt)]
+pub struct DocOpts {
+    /// Whether to run the docs using nightly rustdoc
+    #[structopt(long)]
+    pub stable: bool,
+    /// Whether to open the docs after rendering.
+    #[structopt(long)]
+    pub open: bool,
+    /// Whether to show the private and hidden API.
+    #[structopt(long)]
+    pub internal: bool,
+}
+
+impl Default for DocOpts {
+    fn default() -> Self {
+        Self {
+            stable: true,
+            open: false,
+            internal: false,
+        }
+    }
+}
+
+impl Subcommand {
+    pub fn execute(self) -> Result<()> {
+        print_metadata()?;
+
+        let start = Instant::now();
+
+        match self {
+            Subcommand::Default => {
+                crate::fmt::rust::run()?;
+                crate::clippy::run()?;
+                crate::test::run()?;
+                crate::doc::run(DocOpts::default())?;
+            }
+            Subcommand::Ci => {
+                let installed = Installed::new()?;
+                crate::fmt::rust::run()?;
+                if installed.black {
+                    crate::fmt::python::run()?;
+                } else {
+                    Installed::warn_black()
+                };
+                crate::clippy::run()?;
+                crate::test::run()?;
+                crate::doc::run(DocOpts::default())?;
+                if installed.nox {
+                    crate::pytests::run(None)?;
+                } else {
+                    Installed::warn_nox()
+                };
+                crate::llvm_cov::run(CoverageOpts::default())?;
+                installed.assert()?
+            }
+
+            Subcommand::Doc(opts) => crate::doc::run(opts)?,
+            Subcommand::Fmt => {
+                crate::fmt::rust::run()?;
+                crate::fmt::python::run()?;
+            }
+            Subcommand::Clippy => crate::clippy::run()?,
+            Subcommand::Coverage(opts) => crate::llvm_cov::run(opts)?,
+            Subcommand::TestPy => crate::pytests::run(None)?,
+            Subcommand::Test => crate::test::run()?,
+        };
+
+        let dt = start.elapsed().as_secs();
+        let minutes = dt / 60;
+        let seconds = dt % 60;
+        println!("\nxtask finished in {}m {}s.", minutes, seconds);
+
+        Ok(())
+    }
+}
+
+pub fn run(command: &mut Command) -> Result<()> {
+    println!("Running: {}", format_command(command));
+
+    let output = command
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .spawn()?
+        .wait_with_output()?;
+
+    ensure! {
+        output.status.success(),
+        "process did not run successfully ({exit}): {command}/n {out} {err}",
+        exit = match output.status.code() {
+            Some(code) => format!("exit code {}", code),
+            None => "terminated by signal".into(),
+        },
+        command = format_command(command),
+        out = String::from_utf8_lossy(&output.stdout),
+        err = String::from_utf8_lossy(&output.stderr)
+
+    };
+    Ok(())
+}
+
+#[derive(Copy, Clone, Debug)]
+pub struct Installed {
+    pub nox: bool,
+    pub black: bool,
+}
+
+impl Installed {
+    pub fn new() -> anyhow::Result<Self> {
+        Ok(Self {
+            nox: Self::nox()?,
+            black: Self::black()?,
+        })
+    }
+
+    pub fn nox() -> anyhow::Result<bool> {
+        let output = std::process::Command::new("nox").arg("--version").output();
+        match output {
+            Ok(_) => Ok(true),
+            Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(false),
+            Err(other) => Err(other.into()),
+        }
+    }
+
+    pub fn warn_nox() {
+        eprintln!("Skipping: formatting Python code, because `nox` was not found");
+    }
+
+    pub fn black() -> anyhow::Result<bool> {
+        let output = std::process::Command::new("black")
+            .arg("--version")
+            .output();
+        match output {
+            Ok(_) => Ok(true),
+            Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(false),
+            Err(other) => Err(other.into()),
+        }
+    }
+
+    pub fn warn_black() {
+        eprintln!("Skipping: Python code formatting, because `black` was not found.");
+    }
+
+    pub fn assert(&self) -> anyhow::Result<()> {
+        if self.nox && self.black {
+            Ok(())
+        } else {
+            let mut err =
+                String::from("\n\nxtask was unable to run all tests due to some missing programs:");
+            if !self.black {
+                err.push_str("\n`black` was not installed. (`pip install black`)");
+            }
+            if !self.nox {
+                err.push_str("\n`nox` was not installed. (`pip install nox`)");
+            }
+
+            Err(anyhow::anyhow!(err))
+        }
+    }
+}
--- a/xtask/src/clippy.rs
+++ b/xtask/src/clippy.rs
@ -0,0 +1,25 @@
+use crate::cli;
+use std::process::Command;
+
+pub fn run() -> anyhow::Result<()> {
+    cli::run(
+        Command::new("cargo")
+            .arg("clippy")
+            .arg("--features=full")
+            .arg("--all-targets")
+            .arg("--workspace")
+            .arg("--")
+            .arg("-Dwarnings"),
+    )?;
+    cli::run(
+        Command::new("cargo")
+            .arg("clippy")
+            .arg("--all-targets")
+            .arg("--workspace")
+            .arg("--features=abi3,full")
+            .arg("--")
+            .arg("-Dwarnings"),
+    )?;
+
+    Ok(())
+}
--- a/xtask/src/doc.rs
+++ b/xtask/src/doc.rs
@ -0,0 +1,47 @@
+use crate::cli;
+use crate::cli::DocOpts;
+use std::process::Command;
+//--cfg docsrs --Z unstable-options --document-hidden-items
+
+pub fn run(opts: DocOpts) -> anyhow::Result<()> {
+    let mut flags = Vec::new();
+
+    if !opts.stable {
+        flags.push("--cfg docsrs");
+    }
+    if opts.internal {
+        flags.push("--Z unstable-options");
+        flags.push("--document-hidden-items");
+    }
+    flags.push("-Dwarnings");
+
+    std::env::set_var("RUSTDOCFLAGS", flags.join(" "));
+    cli::run(
+        Command::new("cargo")
+            .args(if opts.stable { None } else { Some("+nightly") })
+            .arg("doc")
+            .arg("--lib")
+            .arg("--no-default-features")
+            .arg("--features=full")
+            .arg("--no-deps")
+            .arg("--workspace")
+            .args(if opts.internal {
+                &["--document-private-items"][..]
+            } else {
+                &["--exclude=pyo3-macros", "--exclude=pyo3-macros-backend"][..]
+            })
+            .args(if opts.stable {
+                &[][..]
+            } else {
+                &[
+                    "-Z",
+                    "unstable-options",
+                    "-Z",
+                    "rustdoc-scrape-examples=examples",
+                ]
+            })
+            .args(if opts.open { Some("--open") } else { None }),
+    )?;
+
+    Ok(())
+}
--- a/xtask/src/fmt.rs
+++ b/xtask/src/fmt.rs
@ -0,0 +1,23 @@
+pub mod rust {
+    use crate::cli;
+    use std::process::Command;
+    pub fn run() -> anyhow::Result<()> {
+        cli::run(
+            Command::new("cargo")
+                .arg("fmt")
+                .arg("--all")
+                .arg("--")
+                .arg("--check"),
+        )?;
+        Ok(())
+    }
+}
+
+pub mod python {
+    use crate::cli;
+    use std::process::Command;
+    pub fn run() -> anyhow::Result<()> {
+        cli::run(Command::new("black").arg(".").arg("--check"))?;
+        Ok(())
+    }
+}
--- a/xtask/src/llvm_cov.rs
+++ b/xtask/src/llvm_cov.rs
@ -0,0 +1,100 @@
+use crate::cli;
+use crate::cli::CoverageOpts;
+use crate::utils::*;
+use anyhow::{Context, Result};
+use std::{collections::HashMap, process::Command};
+
+/// Runs `cargo llvm-cov` for the PyO3 codebase.
+pub fn run(opts: CoverageOpts) -> Result<()> {
+    let env = get_coverage_env()?;
+
+    cli::run(llvm_cov_command(&["clean", "--workspace"]).envs(&env))?;
+
+    cli::run(
+        Command::new("cargo")
+            .args(&["test", "--manifest-path", "pyo3-build-config/Cargo.toml"])
+            .envs(&env),
+    )?;
+    cli::run(
+        Command::new("cargo")
+            .args(&["test", "--manifest-path", "pyo3-macros-backend/Cargo.toml"])
+            .envs(&env),
+    )?;
+    cli::run(
+        Command::new("cargo")
+            .args(&["test", "--manifest-path", "pyo3-macros/Cargo.toml"])
+            .envs(&env),
+    )?;
+
+    cli::run(Command::new("cargo").arg("test").envs(&env))?;
+    cli::run(
+        Command::new("cargo")
+            .args(&["test", "--features", "abi3"])
+            .envs(&env),
+    )?;
+    cli::run(
+        Command::new("cargo")
+            .args(&["test", "--features", "full"])
+            .envs(&env),
+    )?;
+    cli::run(
+        Command::new("cargo")
+            .args(&["test", "--features", "abi3 full"])
+            .envs(&env),
+    )?;
+
+    crate::pytests::run(&env)?;
+
+    match opts.output_lcov {
+        Some(path) => {
+            cli::run(llvm_cov_command(&["--no-run", "--lcov", "--output-path", &path]).envs(&env))?
+        }
+        None => cli::run(llvm_cov_command(&["--no-run", "--summary-only"]).envs(&env))?,
+    }
+
+    Ok(())
+}
+
+fn llvm_cov_command(args: &[&str]) -> Command {
+    let mut command = Command::new("cargo");
+    command
+        .args(&[
+            "llvm-cov",
+            "--package=pyo3",
+            "--package=pyo3-build-config",
+            "--package=pyo3-macros-backend",
+            "--package=pyo3-macros",
+            "--package=pyo3-ffi",
+        ])
+        .args(args);
+    command
+}
+
+fn get_coverage_env() -> Result<HashMap<String, String>> {
+    let mut env = HashMap::new();
+
+    let output = String::from_utf8(llvm_cov_command(&["show-env"]).output()?.stdout)?;
+
+    for line in output.trim().split('\n') {
+        let (key, value) = split_once(line, '=')
+            .context("expected '=' in each line of output from llvm-cov show-env")?;
+        env.insert(key.to_owned(), value.trim_matches('"').to_owned());
+    }
+
+    // Ensure that examples/ and pytests/ all build to the correct target directory to collect
+    // coverage artifacts.
+    env.insert(
+        "CARGO_TARGET_DIR".to_owned(),
+        env.get("CARGO_LLVM_COV_TARGET_DIR").unwrap().to_owned(),
+    );
+
+    // Coverage only works on nightly.
+    let rustc_version =
+        String::from_utf8(get_output(Command::new("rustc").arg("--version"))?.stdout)
+            .context("failed to parse rust version as utf8")?;
+    if !rustc_version.contains("nightly") {
+        env.insert("RUSTUP_TOOLCHAIN".to_owned(), "nightly".to_owned());
+    }
+
+    Ok(env)
+}
--- a/xtask/src/main.rs
+++ b/xtask/src/main.rs
@ -1,196 +1,24 @@
-use anyhow::{ensure, Context, Result};
-use std::{collections::HashMap, path::Path, process::Command};
+use clap::ErrorKind::MissingArgumentOrSubcommand;
 use structopt::StructOpt;

-#[derive(StructOpt)]
-enum Subcommand {
-    /// Runs `cargo llvm-cov` for the PyO3 codebase.
-    Coverage(CoverageOpts),
-    /// Runs tests in examples/ and pytests/
-    TestPy,
-}
+pub mod cli;
+pub mod clippy;
+pub mod doc;
+pub mod fmt;
+pub mod llvm_cov;
+pub mod pytests;
+pub mod test;
+pub mod utils;

-#[derive(StructOpt)]
-struct CoverageOpts {
-    /// Creates an lcov output instead of printing to the terminal.
-    #[structopt(long)]
-    output_lcov: Option<String>,
-}
+fn main() -> anyhow::Result<()> {
+    // Avoid spewing backtraces all over the command line
+    // For some reason this is automatically enabled on nightly compilers...
+    std::env::set_var("RUST_LIB_BACKTRACE", "0");

-impl Subcommand {
-    fn execute(self) -> Result<()> {
-        match self {
-            Subcommand::Coverage(opts) => subcommand_coverage(opts),
-            Subcommand::TestPy => run_python_tests(None),
-        }
-    }
-}
-
-fn main() -> Result<()> {
-    Subcommand::from_args().execute()
-}
-
-/// Runs `cargo llvm-cov` for the PyO3 codebase.
-fn subcommand_coverage(opts: CoverageOpts) -> Result<()> {
-    let env = get_coverage_env()?;
-
-    run(llvm_cov_command(&["clean", "--workspace"]).envs(&env))?;
-
-    run(Command::new("cargo")
-        .args(&["test", "--manifest-path", "pyo3-build-config/Cargo.toml"])
-        .envs(&env))?;
-    run(Command::new("cargo")
-        .args(&["test", "--manifest-path", "pyo3-macros-backend/Cargo.toml"])
-        .envs(&env))?;
-    run(Command::new("cargo")
-        .args(&["test", "--manifest-path", "pyo3-macros/Cargo.toml"])
-        .envs(&env))?;
-
-    run(Command::new("cargo").arg("test").envs(&env))?;
-    run(Command::new("cargo")
-        .args(&["test", "--features", "abi3"])
-        .envs(&env))?;
-    run(Command::new("cargo")
-        .args(&["test", "--features", "full"])
-        .envs(&env))?;
-    run(Command::new("cargo")
-        .args(&["test", "--features", "abi3 full"])
-        .envs(&env))?;
-
-    run_python_tests(&env)?;
-
-    match opts.output_lcov {
-        Some(path) => {
-            run(llvm_cov_command(&["--no-run", "--lcov", "--output-path", &path]).envs(&env))?
-        }
-        None => run(llvm_cov_command(&["--no-run", "--summary-only"]).envs(&env))?,
-    }
-
-    Ok(())
-}
-
-fn run(command: &mut Command) -> Result<()> {
-    println!("running: {}", format_command(command));
-    let status = command.spawn()?.wait()?;
-    ensure! {
-        status.success(),
-        "process did not run successfully ({exit}): {command}",
-        exit = match status.code() {
-            Some(code) => format!("exit code {}", code),
-            None => "terminated by signal".into(),
-        },
-        command = format_command(command),
-    };
-    Ok(())
-}
-
-fn get_output(command: &mut Command) -> Result<std::process::Output> {
-    let output = command.output()?;
-    ensure! {
-        output.status.success(),
-        "process did not run successfully ({exit}): {command}",
-        exit = match output.status.code() {
-            Some(code) => format!("exit code {}", code),
-            None => "terminated by signal".into(),
-        },
-        command = format_command(command),
-    };
-    Ok(output)
-}
-
-fn llvm_cov_command(args: &[&str]) -> Command {
-    let mut command = Command::new("cargo");
-    command
-        .args(&[
-            "llvm-cov",
-            "--package=pyo3",
-            "--package=pyo3-build-config",
-            "--package=pyo3-macros-backend",
-            "--package=pyo3-macros",
-            "--package=pyo3-ffi",
-        ])
-        .args(args);
-    command
-}
-
-fn run_python_tests<'a>(
-    env: impl IntoIterator<Item = (&'a String, &'a String)> + Copy,
-) -> Result<()> {
-    run(Command::new("nox")
-        .arg("--non-interactive")
-        .arg("-f")
-        .arg(Path::new("pytests").join("noxfile.py"))
-        .envs(env))?;
-
-    for entry in std::fs::read_dir("examples")? {
-        let path = entry?.path();
-        if path.is_dir() && path.join("noxfile.py").exists() {
-            run(Command::new("nox")
-                .arg("--non-interactive")
-                .arg("-f")
-                .arg(path.join("noxfile.py"))
-                .envs(env))?;
-        }
+    match cli::Subcommand::from_args_safe() {
+        Ok(c) => c.execute()?,
+        Err(e) if e.kind == MissingArgumentOrSubcommand => cli::Subcommand::default().execute()?,
+        Err(e) => return Err(e.into()),
    }
    Ok(())
 }
-
-fn get_coverage_env() -> Result<HashMap<String, String>> {
-    let mut env = HashMap::new();
-
-    let output = String::from_utf8(llvm_cov_command(&["show-env"]).output()?.stdout)?;
-
-    for line in output.trim().split('\n') {
-        let (key, value) = split_once(line, '=')
-            .context("expected '=' in each line of output from llvm-cov show-env")?;
-        env.insert(key.to_owned(), value.trim_matches('"').to_owned());
-    }
-
-    // Ensure that examples/ and pytests/ all build to the correct target directory to collect
-    // coverage artifacts.
-    env.insert(
-        "CARGO_TARGET_DIR".to_owned(),
-        env.get("CARGO_LLVM_COV_TARGET_DIR").unwrap().to_owned(),
-    );
-
-    // Coverage only works on nightly.
-    let rustc_version =
-        String::from_utf8(get_output(Command::new("rustc").arg("--version"))?.stdout)
-            .context("failed to parse rust version as utf8")?;
-    if !rustc_version.contains("nightly") {
-        env.insert("RUSTUP_TOOLCHAIN".to_owned(), "nightly".to_owned());
-    }
-
-    Ok(env)
-}
-
-// Replacement for str.split_once() on Rust older than 1.52
-#[rustversion::before(1.52)]
-fn split_once(s: &str, pat: char) -> Option<(&str, &str)> {
-    let mut iter = s.splitn(2, pat);
-    Some((iter.next()?, iter.next()?))
-}
-
-#[rustversion::since(1.52)]
-fn split_once(s: &str, pat: char) -> Option<(&str, &str)> {
-    s.split_once(pat)
-}
-
-#[rustversion::since(1.57)]
-fn format_command(command: &Command) -> String {
-    let mut buf = String::new();
-    buf.push('`');
-    buf.push_str(&command.get_program().to_string_lossy());
-    for arg in command.get_args() {
-        buf.push(' ');
-        buf.push_str(&arg.to_string_lossy());
-    }
-    buf.push('`');
-    buf
-}
-
-#[rustversion::before(1.57)]
-fn format_command(command: &Command) -> String {
-    // Debug impl isn't as nice as the above, but will do on < 1.57
-    format!("{:?}", command)
-}
--- a/xtask/src/pytests.rs
+++ b/xtask/src/pytests.rs
@ -0,0 +1,27 @@
+use crate::cli;
+use anyhow::Result;
+use std::{path::Path, process::Command};
+
+pub fn run<'a>(env: impl IntoIterator<Item = (&'a String, &'a String)> + Copy) -> Result<()> {
+    cli::run(
+        Command::new("nox")
+            .arg("--non-interactive")
+            .arg("-f")
+            .arg(Path::new("pytests").join("noxfile.py"))
+            .envs(env),
+    )?;
+
+    for entry in std::fs::read_dir("examples")? {
+        let path = entry?.path();
+        if path.is_dir() && path.join("noxfile.py").exists() {
+            cli::run(
+                Command::new("nox")
+                    .arg("--non-interactive")
+                    .arg("-f")
+                    .arg(path.join("noxfile.py"))
+                    .envs(env),
+            )?;
+        }
+    }
+    Ok(())
+}
--- a/xtask/src/test.rs
+++ b/xtask/src/test.rs
@ -0,0 +1,73 @@
+use crate::cli::{self, MSRV};
+use std::process::Command;
+
+pub fn run() -> anyhow::Result<()> {
+    cli::run(
+        Command::new("cargo")
+            .arg("test")
+            .arg("--lib")
+            .arg("--no-default-features")
+            .arg("--tests")
+            .arg("--quiet"),
+    )?;
+
+    cli::run(
+        Command::new("cargo")
+            .arg("test")
+            .arg("--no-default-features")
+            .arg("--features=full")
+            .arg("--quiet"),
+    )?;
+
+    cli::run(
+        Command::new("cargo")
+            .arg("test")
+            .arg("--no-default-features")
+            .arg("--features=abi3,full")
+            .arg("--quiet"),
+    )?;
+
+    // If the MSRV toolchain is not installed, this will install it
+    cli::run(
+        Command::new("rustup")
+            .arg("toolchain")
+            .arg("install")
+            .arg(MSRV),
+    )?;
+
+    // Test MSRV
+    cli::run(
+        Command::new("cargo")
+            .arg(format!("+{}", MSRV))
+            .arg("test")
+            .arg("--no-default-features")
+            .arg("--features=full,auto-initialize")
+            .arg("--quiet"),
+    )?;
+
+    cli::run(
+        Command::new("cargo")
+            .arg("+nightly")
+            .arg("test")
+            .arg("--no-default-features")
+            .arg("--features=full,nightly")
+            .arg("--quiet"),
+    )?;
+
+    cli::run(
+        Command::new("cargo")
+            .arg("test")
+            .arg("--manifest-path=pyo3-ffi/Cargo.toml")
+            .arg("--quiet"),
+    )?;
+
+    cli::run(
+        Command::new("cargo")
+            .arg("test")
+            .arg("--no-default-features")
+            .arg("--manifest-path=pyo3-build-config/Cargo.toml")
+            .arg("--quiet"),
+    )?;
+
+    Ok(())
+}
--- a/xtask/src/utils.rs
+++ b/xtask/src/utils.rs
@ -0,0 +1,65 @@
+use anyhow::ensure;
+use std::process::Command;
+
+// Replacement for str.split_once() on Rust older than 1.52
+#[rustversion::before(1.52)]
+pub fn split_once(s: &str, pat: char) -> Option<(&str, &str)> {
+    let mut iter = s.splitn(2, pat);
+    Some((iter.next()?, iter.next()?))
+}
+
+#[rustversion::since(1.52)]
+pub fn split_once(s: &str, pat: char) -> Option<(&str, &str)> {
+    s.split_once(pat)
+}
+
+#[rustversion::since(1.57)]
+pub fn format_command(command: &Command) -> String {
+    let mut buf = String::new();
+    buf.push('`');
+    buf.push_str(&command.get_program().to_string_lossy());
+    for arg in command.get_args() {
+        buf.push(' ');
+        buf.push_str(&arg.to_string_lossy());
+    }
+    buf.push('`');
+    buf
+}
+
+#[rustversion::before(1.57)]
+pub fn format_command(command: &Command) -> String {
+    // Debug impl isn't as nice as the above, but will do on < 1.57
+    format!("{:?}", command)
+}
+
+pub fn get_output(command: &mut Command) -> anyhow::Result<std::process::Output> {
+    let output = command.output()?;
+    ensure! {
+        output.status.success(),
+        "process did not run successfully ({exit}): {command}",
+        exit = match output.status.code() {
+            Some(code) => format!("exit code {}", code),
+            None => "terminated by signal".into(),
+        },
+        command = format_command(command),
+    };
+    Ok(output)
+}
+
+pub fn print_metadata() -> anyhow::Result<()> {
+    let rustc_output = std::process::Command::new("rustc")
+        .arg("--version")
+        .arg("--verbose")
+        .output()?;
+    let rustc_version = core::str::from_utf8(&rustc_output.stdout).unwrap();
+    println!("Metadata: \n\n{}", rustc_version);
+
+    let py_output = std::process::Command::new("python")
+        .arg("--version")
+        .arg("-V")
+        .output()?;
+    let py_version = core::str::from_utf8(&py_output.stdout).unwrap();
+    println!("{}", py_version);
+
+    Ok(())
+}