xtask: expand on coverage (#2449)

* Install nightly if it is not installed

* Expand on coverage

* Don't swallow stdout

* Apply suggestions from code review

Co-authored-by: mejrs <>
Co-authored-by: David Hewitt <1939362+davidhewitt@users.noreply.github.com>

.gitignore

@@ -21,3 +21,4 @@ extensions/stamps/
 pip-wheel-metadata
 valgrind-python.supp
 *.pyd
+lcov.info

Contributing.md

@@ -128,6 +128,29 @@ First, there are Rust-based benchmarks located in the `benches` subdirectory. As
 
 Second, there is a Python-based benchmark contained in the `pytests` subdirectory. You can read more about it [here](pytests).
 
+## Code coverage
+
+You can view what code is and isn't covered by PyO3's tests. We aim to have 100% coverage - please check coverage and add tests if you notice a lack of coverage!
+
+- First, generate a `lcov.info` file with 
+```shell
+cargo xtask coverage
+```
+You can install an IDE plugin to view the coverage. For example, if you use VSCode:
+- Add the [coverage-gutters](https://marketplace.visualstudio.com/items?itemName=ryanluker.vscode-coverage-gutters) plugin.
+- Add these settings to VSCode's `settings.json`:
+```json
+{    
+    "coverage-gutters.coverageFileNames": [
+        "lcov.info",
+        "cov.xml",
+        "coverage.xml",
+    ],
+    "coverage-gutters.showLineCoverage": true
+}
+```
+- You should now be able to see green highlights for code that is tested, and red highlights for code that is not tested.
+
 ## Sponsor this project
 
 At the moment there is no official organisation that accepts sponsorship on PyO3's behalf. If you're seeking to provide significant funding to the PyO3 ecosystem, please reach out to us on [GitHub](https://github.com/PyO3/pyo3/issues/new) or [Gitter](https://gitter.im/PyO3/Lobby) and we can discuss.

xtask/src/cli.rs

@@ -1,7 +1,7 @@
 use crate::utils::*;
 use anyhow::{ensure, Result};
 use std::io;
-use std::process::Command;
+use std::process::{Command, Output};
 use std::time::Instant;
 use structopt::StructOpt;
 
@@ -33,11 +33,19 @@ impl Default for Subcommand {
     }
 }
 
-#[derive(StructOpt, Default)]
+#[derive(StructOpt)]
 pub struct CoverageOpts {
-    /// Creates an lcov output instead of printing to the terminal.
-    #[structopt(long)]
-    pub output_lcov: Option<String>,
+    /// Creates an lcov output file.
+    #[structopt(long, default_value = "lcov.info")]
+    pub output_lcov: String,
+}
+
+impl Default for CoverageOpts {
+    fn default() -> Self {
+        Self {
+            output_lcov: String::from("lcov.info"),
+        }
+    }
 }
 
 #[derive(StructOpt)]
@@ -116,6 +124,7 @@ impl Subcommand {
     }
 }
 
+/// Run a command as a child process, inheriting stdin, stdout and stderr.
 pub fn run(command: &mut Command) -> Result<()> {
     let command_str = format_command(command);
     let github_actions = std::env::var_os("GITHUB_ACTIONS").is_some();
@@ -143,6 +152,28 @@ pub fn run(command: &mut Command) -> Result<()> {
     Ok(())
 }
 
+/// Like `run`, but does not inherit stdin, stdout and stderr.
+pub fn run_with_output(command: &mut Command) -> Result<Output> {
+    let command_str = format_command(command);
+
+    println!("Running: {}", command_str);
+
+    let output = command.output()?;
+
+    ensure! {
+        output.status.success(),
+        "process did not run successfully ({exit}): {command}:\n{stderr}",
+        exit = match output.status.code() {
+            Some(code) => format!("exit code {}", code),
+            None => "terminated by signal".into(),
+        },
+        command = command_str,
+        stderr = String::from_utf8_lossy(&output.stderr)
+    };
+
+    Ok(output)
+}
+
 #[derive(Copy, Clone, Debug)]
 pub struct Installed {
     pub nox: bool,

xtask/src/llvm_cov.rs

@@ -45,12 +45,9 @@ pub fn run(opts: CoverageOpts) -> Result<()> {
 
     crate::pytests::run(&env)?;
 
-    match opts.output_lcov {
-        Some(path) => {
-            cli::run(llvm_cov_command(&["--no-run", "--lcov", "--output-path", &path]).envs(&env))?
-        }
-        None => cli::run(llvm_cov_command(&["--no-run", "--summary-only"]).envs(&env))?,
-    }
+    cli::run(
+        llvm_cov_command(&["--no-run", "--lcov", "--output-path", &opts.output_lcov]).envs(&env),
+    )?;
 
     Ok(())
 }
@@ -73,7 +70,9 @@ fn llvm_cov_command(args: &[&str]) -> Command {
 fn get_coverage_env() -> Result<HashMap<String, String>> {
     let mut env = HashMap::new();
 
-    let output = String::from_utf8(llvm_cov_command(&["show-env"]).output()?.stdout)?;
+    let output = cli::run_with_output(&mut llvm_cov_command(&["show-env"])).context("Unable to run llvm-cov. If it is not installed, you can install it with `cargo install cargo-llvm-cov`.")?;
+
+    let output = std::str::from_utf8(&output.stdout)?;
 
     for line in output.trim().split('\n') {
         let (key, value) = split_once(line, '=')

xtask/src/test.rs

@@ -45,6 +45,14 @@ pub fn run() -> anyhow::Result<()> {
             .arg("--quiet"),
     )?;
 
+    // If the nightly toolchain is not installed, this will install it
+    cli::run(
+        Command::new("rustup")
+            .arg("toolchain")
+            .arg("install")
+            .arg("nightly"),
+    )?;
+
     cli::run(
         Command::new("cargo")
             .arg("+nightly")