From 0991aa9f379a17d7cea9632d4403fd2140e27aa7 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Sat, 15 Jan 2022 22:42:20 +0000 Subject: [PATCH] ci: add coverage for pytests --- .github/workflows/ci.yml | 12 +-- Cargo.toml | 2 + pytests/pyo3-benchmarks/Cargo.toml | 2 - pytests/pyo3-benchmarks/setup.py | 6 +- pytests/pyo3-pytests/Cargo.toml | 2 - pytests/pyo3-pytests/noxfile.py | 3 +- pytests/pyo3-pytests/pyproject.toml | 3 +- pytests/pyo3-pytests/requirements-dev.txt | 1 - xtask/src/main.rs | 111 ++++++++++++++++++---- 9 files changed, 106 insertions(+), 36 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd3b294d..ab242f38 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -209,7 +209,7 @@ jobs: python -m pip install -U pip nox cargo xtask test-py env: - CARGO_TARGET_DIR: ${{ github.workspace }} + CARGO_TARGET_DIR: ${{ github.workspace }}/target - name: Test cross compilation if: ${{ matrix.platform.os == 'ubuntu-latest' && matrix.python-version == '3.9' }} @@ -265,14 +265,10 @@ jobs: host=$(rustc -Vv | grep host | sed 's/host: //') curl -fsSL https://github.com/taiki-e/cargo-llvm-cov/releases/download/v${CARGO_LLVM_COV_VERSION}/cargo-llvm-cov-"$host".tar.gz | tar xzf - -C ~/.cargo/bin env: - CARGO_LLVM_COV_VERSION: 0.1.11 + CARGO_LLVM_COV_VERSION: 0.1.15 + - run: pip install -U pip nox - run: | - cargo llvm-cov clean --workspace - cargo llvm-cov --package $ALL_PACKAGES --no-report - cargo llvm-cov --package $ALL_PACKAGES --no-report --features abi3 - cargo llvm-cov --package $ALL_PACKAGES --no-report --features full - cargo llvm-cov --package $ALL_PACKAGES --no-report --features "abi3 full" - cargo llvm-cov --package $ALL_PACKAGES --no-run --lcov --output-path coverage.lcov + cargo xtask coverage --output-lcov coverage.lcov shell: bash env: ALL_PACKAGES: pyo3 pyo3-build-config pyo3-macros-backend pyo3-macros diff --git a/Cargo.toml b/Cargo.toml index d88464fb..fc69989c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -128,6 +128,8 @@ harness = false members = [ "pyo3-macros", "pyo3-macros-backend", + "pytests/pyo3-benchmarks", + "pytests/pyo3-pytests", "examples", "xtask" ] diff --git a/pytests/pyo3-benchmarks/Cargo.toml b/pytests/pyo3-benchmarks/Cargo.toml index 70798099..f931e19a 100644 --- a/pytests/pyo3-benchmarks/Cargo.toml +++ b/pytests/pyo3-benchmarks/Cargo.toml @@ -14,5 +14,3 @@ features = ["extension-module"] [lib] name = "_pyo3_benchmarks" crate-type = ["cdylib"] - -[workspace] \ No newline at end of file diff --git a/pytests/pyo3-benchmarks/setup.py b/pytests/pyo3-benchmarks/setup.py index b16f1ed5..f38bebdf 100644 --- a/pytests/pyo3-benchmarks/setup.py +++ b/pytests/pyo3-benchmarks/setup.py @@ -1,7 +1,8 @@ +import os + from setuptools import setup from setuptools_rust import RustExtension - setup( name="pyo3-benchmarks", version="0.1.0", @@ -18,7 +19,8 @@ setup( rust_extensions=[ RustExtension( "pyo3_benchmarks._pyo3_benchmarks", - debug=False, + # build debug when measuring coverage, otherwise release + debug="CARGO_LLVM_COV_TARGET_DIR" in os.environ, ), ], include_package_data=True, diff --git a/pytests/pyo3-pytests/Cargo.toml b/pytests/pyo3-pytests/Cargo.toml index 61f76891..8fdcda88 100644 --- a/pytests/pyo3-pytests/Cargo.toml +++ b/pytests/pyo3-pytests/Cargo.toml @@ -25,5 +25,3 @@ classifier=[ "Operating System :: POSIX", "Operating System :: MacOS :: MacOS X", ] - -[workspace] \ No newline at end of file diff --git a/pytests/pyo3-pytests/noxfile.py b/pytests/pyo3-pytests/noxfile.py index e6d1b300..17a6b80f 100644 --- a/pytests/pyo3-pytests/noxfile.py +++ b/pytests/pyo3-pytests/noxfile.py @@ -4,5 +4,6 @@ import nox @nox.session def python(session): session.install("-rrequirements-dev.txt") - session.install(".", "--no-build-isolation") + session.install("maturin") + session.run_always("maturin", "develop") session.run("pytest") diff --git a/pytests/pyo3-pytests/pyproject.toml b/pytests/pyo3-pytests/pyproject.toml index 9a2f56d9..5f0a84ee 100644 --- a/pytests/pyo3-pytests/pyproject.toml +++ b/pytests/pyo3-pytests/pyproject.toml @@ -1,3 +1,4 @@ [build-system] -requires = ["maturin>=0.10,<0.11"] +# FIXME: branch is necessary for coverage to work +requires = ["maturin @ git+https://github.com/davidhewitt/maturin@profile-arg"] build-backend = "maturin" diff --git a/pytests/pyo3-pytests/requirements-dev.txt b/pytests/pyo3-pytests/requirements-dev.txt index d70f6391..cc06de69 100644 --- a/pytests/pyo3-pytests/requirements-dev.txt +++ b/pytests/pyo3-pytests/requirements-dev.txt @@ -2,4 +2,3 @@ hypothesis>=3.55 pytest>=3.5.0 psutil>=5.6 pip>=21.3 -maturin>=0.12,<0.13 diff --git a/xtask/src/main.rs b/xtask/src/main.rs index e9fa71a2..2b919815 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -1,19 +1,26 @@ -use anyhow::{Context, Result}; +use anyhow::{ensure, Context, Result}; use std::{collections::HashMap, process::Command}; use structopt::StructOpt; #[derive(StructOpt)] enum Subcommand { /// Runs `cargo llvm-cov` for the PyO3 codebase. - Coverage, + Coverage(CoverageOpts), /// Runs tests in examples/ and pytests/ TestPy, } +#[derive(StructOpt)] +struct CoverageOpts { + /// Creates an lcov output instead of printing to the terminal. + #[structopt(long)] + output_lcov: Option, +} + impl Subcommand { fn execute(self) -> Result<()> { match self { - Subcommand::Coverage => subcommand_coverage(), + Subcommand::Coverage(opts) => subcommand_coverage(opts), Subcommand::TestPy => run_python_tests(None), } } @@ -24,32 +31,84 @@ fn main() -> Result<()> { } /// Runs `cargo llvm-cov` for the PyO3 codebase. -fn subcommand_coverage() -> Result<()> { - run(&mut llvm_cov_command(&["clean", "--workspace"]))?; - run(&mut llvm_cov_command(&["--no-report"]))?; - - // FIXME: add various feature combinations using 'full' feature. - // run(&mut llvm_cov_command(&["--no-report"]))?; - - // XXX: the following block doesn't work until https://github.com/taiki-e/cargo-llvm-cov/pull/115 is merged +fn subcommand_coverage(opts: CoverageOpts) -> Result<()> { let env = get_coverage_env()?; - run_python_tests(&env)?; - // (after here works with stable llvm-cov) - // TODO: add an argument to make it possible to generate lcov report & use this in CI. - run(&mut llvm_cov_command(&["--no-run", "--summary-only"]))?; + run(llvm_cov_command(&["clean", "--workspace"]).envs(&env))?; + + run(Command::new("cargo") + .args(&["test", "--manifest-path", "pyo3-build-config/Cargo.toml"]) + .envs(&env))?; + run(Command::new("cargo") + .args(&["test", "--manifest-path", "pyo3-macros-backend/Cargo.toml"]) + .envs(&env))?; + run(Command::new("cargo") + .args(&["test", "--manifest-path", "pyo3-macros/Cargo.toml"]) + .envs(&env))?; + + run(Command::new("cargo").arg("test").envs(&env))?; + run(Command::new("cargo") + .args(&["test", "--features", "abi3"]) + .envs(&env))?; + run(Command::new("cargo") + .args(&["test", "--features", "full"]) + .envs(&env))?; + run(Command::new("cargo") + .args(&["test", "--features", "abi3 full"]) + .envs(&env))?; + + run_python_tests(&env)?; + + match opts.output_lcov { + Some(path) => { + run(llvm_cov_command(&["--no-run", "--lcov", "--output-path", &path]).envs(&env))? + } + None => run(llvm_cov_command(&["--no-run", "--summary-only"]).envs(&env))?, + } + Ok(()) } fn run(command: &mut Command) -> Result<()> { println!("running: {}", format_command(command)); - command.spawn()?.wait()?; + let status = command.spawn()?.wait()?; + ensure! { + status.success(), + "process did not run successfully ({exit}): {command}", + exit = match status.code() { + Some(code) => format!("exit code {}", code), + None => "terminated by signal".into(), + }, + command = format_command(command), + }; Ok(()) } +fn get_output(command: &mut Command) -> Result { + let output = command.output()?; + ensure! { + output.status.success(), + "process did not run successfully ({exit}): {command}", + exit = match output.status.code() { + Some(code) => format!("exit code {}", code), + None => "terminated by signal".into(), + }, + command = format_command(command), + }; + Ok(output) +} + fn llvm_cov_command(args: &[&str]) -> Command { let mut command = Command::new("cargo"); - command.args(&["llvm-cov", "--package=pyo3"]).args(args); + command + .args(&[ + "llvm-cov", + "--package=pyo3", + "--package=pyo3-build-config", + "--package=pyo3-macros-backend", + "--package=pyo3-macros", + ]) + .args(args); command } @@ -85,11 +144,25 @@ fn get_coverage_env() -> Result> { let output = String::from_utf8(llvm_cov_command(&["show-env"]).output()?.stdout)?; for line in output.trim().split('\n') { - let (key, value) = split_once(line, '=').context("expected '=' in each output line")?; + let (key, value) = split_once(line, '=') + .context("expected '=' in each line of output from llvm-cov show-env")?; env.insert(key.to_owned(), value.trim_matches('"').to_owned()); } - env.insert("RUSTUP_TOOLCHAIN".to_owned(), "nightly".to_owned()); + // Ensure that examples/ and pytests/ all build to the correct target directory to collect + // coverage artifacts. + env.insert( + "CARGO_TARGET_DIR".to_owned(), + env.get("CARGO_LLVM_COV_TARGET_DIR").unwrap().to_owned(), + ); + + // Coverage only works on nightly. + let rustc_version = + String::from_utf8(get_output(Command::new("rustc").arg("--version"))?.stdout) + .context("failed to parse rust version as utf8")?; + if !rustc_version.contains("nightly") { + env.insert("RUSTUP_TOOLCHAIN".to_owned(), "nightly".to_owned()); + } Ok(env) }