ci: add coverage for pytests

This commit is contained in:
David Hewitt 2022-01-15 22:42:20 +00:00
parent 91648b2315
commit 0991aa9f37
9 changed files with 106 additions and 36 deletions

View File

@ -209,7 +209,7 @@ jobs:
python -m pip install -U pip nox
cargo xtask test-py
env:
CARGO_TARGET_DIR: ${{ github.workspace }}
CARGO_TARGET_DIR: ${{ github.workspace }}/target
- name: Test cross compilation
if: ${{ matrix.platform.os == 'ubuntu-latest' && matrix.python-version == '3.9' }}
@ -265,14 +265,10 @@ jobs:
host=$(rustc -Vv | grep host | sed 's/host: //')
curl -fsSL https://github.com/taiki-e/cargo-llvm-cov/releases/download/v${CARGO_LLVM_COV_VERSION}/cargo-llvm-cov-"$host".tar.gz | tar xzf - -C ~/.cargo/bin
env:
CARGO_LLVM_COV_VERSION: 0.1.11
CARGO_LLVM_COV_VERSION: 0.1.15
- run: pip install -U pip nox
- run: |
cargo llvm-cov clean --workspace
cargo llvm-cov --package $ALL_PACKAGES --no-report
cargo llvm-cov --package $ALL_PACKAGES --no-report --features abi3
cargo llvm-cov --package $ALL_PACKAGES --no-report --features full
cargo llvm-cov --package $ALL_PACKAGES --no-report --features "abi3 full"
cargo llvm-cov --package $ALL_PACKAGES --no-run --lcov --output-path coverage.lcov
cargo xtask coverage --output-lcov coverage.lcov
shell: bash
env:
ALL_PACKAGES: pyo3 pyo3-build-config pyo3-macros-backend pyo3-macros

View File

@ -128,6 +128,8 @@ harness = false
members = [
"pyo3-macros",
"pyo3-macros-backend",
"pytests/pyo3-benchmarks",
"pytests/pyo3-pytests",
"examples",
"xtask"
]

View File

@ -14,5 +14,3 @@ features = ["extension-module"]
[lib]
name = "_pyo3_benchmarks"
crate-type = ["cdylib"]
[workspace]

View File

@ -1,7 +1,8 @@
import os
from setuptools import setup
from setuptools_rust import RustExtension
setup(
name="pyo3-benchmarks",
version="0.1.0",
@ -18,7 +19,8 @@ setup(
rust_extensions=[
RustExtension(
"pyo3_benchmarks._pyo3_benchmarks",
debug=False,
# build debug when measuring coverage, otherwise release
debug="CARGO_LLVM_COV_TARGET_DIR" in os.environ,
),
],
include_package_data=True,

View File

@ -25,5 +25,3 @@ classifier=[
"Operating System :: POSIX",
"Operating System :: MacOS :: MacOS X",
]
[workspace]

View File

@ -4,5 +4,6 @@ import nox
@nox.session
def python(session):
session.install("-rrequirements-dev.txt")
session.install(".", "--no-build-isolation")
session.install("maturin")
session.run_always("maturin", "develop")
session.run("pytest")

View File

@ -1,3 +1,4 @@
[build-system]
requires = ["maturin>=0.10,<0.11"]
# FIXME: branch is necessary for coverage to work
requires = ["maturin @ git+https://github.com/davidhewitt/maturin@profile-arg"]
build-backend = "maturin"

View File

@ -2,4 +2,3 @@ hypothesis>=3.55
pytest>=3.5.0
psutil>=5.6
pip>=21.3
maturin>=0.12,<0.13

View File

@ -1,19 +1,26 @@
use anyhow::{Context, Result};
use anyhow::{ensure, Context, Result};
use std::{collections::HashMap, process::Command};
use structopt::StructOpt;
#[derive(StructOpt)]
enum Subcommand {
/// Runs `cargo llvm-cov` for the PyO3 codebase.
Coverage,
Coverage(CoverageOpts),
/// Runs tests in examples/ and pytests/
TestPy,
}
#[derive(StructOpt)]
struct CoverageOpts {
/// Creates an lcov output instead of printing to the terminal.
#[structopt(long)]
output_lcov: Option<String>,
}
impl Subcommand {
fn execute(self) -> Result<()> {
match self {
Subcommand::Coverage => subcommand_coverage(),
Subcommand::Coverage(opts) => subcommand_coverage(opts),
Subcommand::TestPy => run_python_tests(None),
}
}
@ -24,32 +31,84 @@ fn main() -> Result<()> {
}
/// Runs `cargo llvm-cov` for the PyO3 codebase.
fn subcommand_coverage() -> Result<()> {
run(&mut llvm_cov_command(&["clean", "--workspace"]))?;
run(&mut llvm_cov_command(&["--no-report"]))?;
// FIXME: add various feature combinations using 'full' feature.
// run(&mut llvm_cov_command(&["--no-report"]))?;
// XXX: the following block doesn't work until https://github.com/taiki-e/cargo-llvm-cov/pull/115 is merged
fn subcommand_coverage(opts: CoverageOpts) -> Result<()> {
let env = get_coverage_env()?;
run_python_tests(&env)?;
// (after here works with stable llvm-cov)
// TODO: add an argument to make it possible to generate lcov report & use this in CI.
run(&mut llvm_cov_command(&["--no-run", "--summary-only"]))?;
run(llvm_cov_command(&["clean", "--workspace"]).envs(&env))?;
run(Command::new("cargo")
.args(&["test", "--manifest-path", "pyo3-build-config/Cargo.toml"])
.envs(&env))?;
run(Command::new("cargo")
.args(&["test", "--manifest-path", "pyo3-macros-backend/Cargo.toml"])
.envs(&env))?;
run(Command::new("cargo")
.args(&["test", "--manifest-path", "pyo3-macros/Cargo.toml"])
.envs(&env))?;
run(Command::new("cargo").arg("test").envs(&env))?;
run(Command::new("cargo")
.args(&["test", "--features", "abi3"])
.envs(&env))?;
run(Command::new("cargo")
.args(&["test", "--features", "full"])
.envs(&env))?;
run(Command::new("cargo")
.args(&["test", "--features", "abi3 full"])
.envs(&env))?;
run_python_tests(&env)?;
match opts.output_lcov {
Some(path) => {
run(llvm_cov_command(&["--no-run", "--lcov", "--output-path", &path]).envs(&env))?
}
None => run(llvm_cov_command(&["--no-run", "--summary-only"]).envs(&env))?,
}
Ok(())
}
fn run(command: &mut Command) -> Result<()> {
println!("running: {}", format_command(command));
command.spawn()?.wait()?;
let status = command.spawn()?.wait()?;
ensure! {
status.success(),
"process did not run successfully ({exit}): {command}",
exit = match status.code() {
Some(code) => format!("exit code {}", code),
None => "terminated by signal".into(),
},
command = format_command(command),
};
Ok(())
}
fn get_output(command: &mut Command) -> Result<std::process::Output> {
let output = command.output()?;
ensure! {
output.status.success(),
"process did not run successfully ({exit}): {command}",
exit = match output.status.code() {
Some(code) => format!("exit code {}", code),
None => "terminated by signal".into(),
},
command = format_command(command),
};
Ok(output)
}
fn llvm_cov_command(args: &[&str]) -> Command {
let mut command = Command::new("cargo");
command.args(&["llvm-cov", "--package=pyo3"]).args(args);
command
.args(&[
"llvm-cov",
"--package=pyo3",
"--package=pyo3-build-config",
"--package=pyo3-macros-backend",
"--package=pyo3-macros",
])
.args(args);
command
}
@ -85,11 +144,25 @@ fn get_coverage_env() -> Result<HashMap<String, String>> {
let output = String::from_utf8(llvm_cov_command(&["show-env"]).output()?.stdout)?;
for line in output.trim().split('\n') {
let (key, value) = split_once(line, '=').context("expected '=' in each output line")?;
let (key, value) = split_once(line, '=')
.context("expected '=' in each line of output from llvm-cov show-env")?;
env.insert(key.to_owned(), value.trim_matches('"').to_owned());
}
env.insert("RUSTUP_TOOLCHAIN".to_owned(), "nightly".to_owned());
// Ensure that examples/ and pytests/ all build to the correct target directory to collect
// coverage artifacts.
env.insert(
"CARGO_TARGET_DIR".to_owned(),
env.get("CARGO_LLVM_COV_TARGET_DIR").unwrap().to_owned(),
);
// Coverage only works on nightly.
let rustc_version =
String::from_utf8(get_output(Command::new("rustc").arg("--version"))?.stdout)
.context("failed to parse rust version as utf8")?;
if !rustc_version.contains("nightly") {
env.insert("RUSTUP_TOOLCHAIN".to_owned(), "nightly".to_owned());
}
Ok(env)
}