From bf2c5a896ba84fca501ca0c5011bc7163d308846 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Tue, 29 Aug 2023 07:26:36 +0100 Subject: [PATCH] add codspeed benchmarks --- .github/workflows/benches.yml | 40 ++++++++++++++++++++++ noxfile.py | 13 +++++++ pyo3-benches/Cargo.toml | 10 ++---- pyo3-benches/benches/bench_any.rs | 2 +- pyo3-benches/benches/bench_bigint.rs | 2 +- pyo3-benches/benches/bench_call.rs | 2 +- pyo3-benches/benches/bench_comparisons.rs | 2 +- pyo3-benches/benches/bench_decimal.rs | 2 +- pyo3-benches/benches/bench_dict.rs | 2 +- pyo3-benches/benches/bench_err.rs | 2 +- pyo3-benches/benches/bench_extract.rs | 2 +- pyo3-benches/benches/bench_frompyobject.rs | 2 +- pyo3-benches/benches/bench_gil.rs | 2 +- pyo3-benches/benches/bench_intern.rs | 2 +- pyo3-benches/benches/bench_list.rs | 2 +- pyo3-benches/benches/bench_pyclass.rs | 4 +-- pyo3-benches/benches/bench_pyobject.rs | 2 +- pyo3-benches/benches/bench_set.rs | 2 +- pyo3-benches/benches/bench_tuple.rs | 2 +- 19 files changed, 73 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/benches.yml diff --git a/.github/workflows/benches.yml b/.github/workflows/benches.yml new file mode 100644 index 00000000..0e90da7e --- /dev/null +++ b/.github/workflows/benches.yml @@ -0,0 +1,40 @@ +name: benches + +on: + push: + branches: + - "main" + pull_request: + # `workflow_dispatch` allows CodSpeed to trigger backtest + # performance analysis in order to generate initial data. + workflow_dispatch: + +jobs: + benchmarks: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + - uses: dtolnay/rust-toolchain@stable + + - uses: actions/cache@v3 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + pyo3-benches/target + target + key: cargo-${{ runner.os }}-bench-${{ hashFiles('**/Cargo.toml') }} + continue-on-error: true + + - name: Install cargo-codspeed + run: cargo install cargo-codspeed + + - name: Install nox + run: pip install nox + + - name: Run the benchmarks + uses: CodSpeedHQ/action@v1 + with: + run: nox -s codspeed + token: ${{ secrets.CODSPEED_TOKEN }} diff --git a/noxfile.py b/noxfile.py index ee24a26f..c49df1d8 100644 --- a/noxfile.py +++ b/noxfile.py @@ -137,6 +137,19 @@ def bench(session: nox.Session) -> bool: _run_cargo(session, "bench", _BENCHES, *session.posargs) +@nox.session() +def codspeed(session: nox.Session) -> bool: + # rust benchmarks + os.chdir(PYO3_DIR / "pyo3-benches") + _run_cargo(session, "codspeed", "build") + _run_cargo(session, "codspeed", "run") + # python benchmarks + os.chdir(PYO3_DIR / "pytests") + session.install("-r", "requirements-dev.txt", "pytest-codspeed") + session.install(".") + _run(session, "pytest", "--codspeed", external=True) + + @nox.session(name="clippy-all", venv_backend="none") def clippy_all(session: nox.Session) -> None: success = True diff --git a/pyo3-benches/Cargo.toml b/pyo3-benches/Cargo.toml index cc82fb00..e99ef09e 100644 --- a/pyo3-benches/Cargo.toml +++ b/pyo3-benches/Cargo.toml @@ -7,11 +7,13 @@ edition = "2021" publish = false [dependencies] -pyo3 = { path = "../", features = ["auto-initialize"] } +pyo3 = { path = "../", features = ["auto-initialize", "full"] } [dev-dependencies] +codspeed-criterion-compat = "2.3" criterion = "0.5.1" num-bigint = "0.4.3" +rust_decimal = { version = "1.0.0", default-features = false } [[bench]] name = "bench_any" @@ -32,17 +34,14 @@ harness = false [[bench]] name = "bench_decimal" harness = false -required-features = ["pyo3/rust_decimal"] [[bench]] name = "bench_dict" harness = false -required-features = ["pyo3/hashbrown"] [[bench]] name = "bench_frompyobject" harness = false -required-features = ["pyo3/macros"] [[bench]] name = "bench_gil" @@ -55,7 +54,6 @@ harness = false [[bench]] name = "bench_pyclass" harness = false -required-features = ["pyo3/macros"] [[bench]] name = "bench_pyobject" @@ -64,7 +62,6 @@ harness = false [[bench]] name = "bench_set" harness = false -required-features = ["pyo3/hashbrown"] [[bench]] name = "bench_tuple" @@ -81,6 +78,5 @@ harness = false [[bench]] name = "bench_bigint" harness = false -required-features = ["pyo3/num-bigint"] [workspace] diff --git a/pyo3-benches/benches/bench_any.rs b/pyo3-benches/benches/bench_any.rs index 765497fa..bfd010ef 100644 --- a/pyo3-benches/benches/bench_any.rs +++ b/pyo3-benches/benches/bench_any.rs @@ -1,4 +1,4 @@ -use criterion::{criterion_group, criterion_main, Bencher, Criterion}; +use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; use pyo3::{ types::{ diff --git a/pyo3-benches/benches/bench_bigint.rs b/pyo3-benches/benches/bench_bigint.rs index 3df24d21..d3c71629 100644 --- a/pyo3-benches/benches/bench_bigint.rs +++ b/pyo3-benches/benches/bench_bigint.rs @@ -1,4 +1,4 @@ -use criterion::{black_box, criterion_group, criterion_main, Bencher, Criterion}; +use codspeed_criterion_compat::{black_box, criterion_group, criterion_main, Bencher, Criterion}; use pyo3::{types::PyDict, PyAny, Python}; diff --git a/pyo3-benches/benches/bench_call.rs b/pyo3-benches/benches/bench_call.rs index dd98c159..50772097 100644 --- a/pyo3-benches/benches/bench_call.rs +++ b/pyo3-benches/benches/bench_call.rs @@ -1,4 +1,4 @@ -use criterion::{criterion_group, criterion_main, Bencher, Criterion}; +use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; use pyo3::prelude::*; diff --git a/pyo3-benches/benches/bench_comparisons.rs b/pyo3-benches/benches/bench_comparisons.rs index bfa4ac63..ffd4c1a4 100644 --- a/pyo3-benches/benches/bench_comparisons.rs +++ b/pyo3-benches/benches/bench_comparisons.rs @@ -1,4 +1,4 @@ -use criterion::{criterion_group, criterion_main, Bencher, Criterion}; +use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; use pyo3::{prelude::*, pyclass::CompareOp, Python}; diff --git a/pyo3-benches/benches/bench_decimal.rs b/pyo3-benches/benches/bench_decimal.rs index c412c4b1..203756b5 100644 --- a/pyo3-benches/benches/bench_decimal.rs +++ b/pyo3-benches/benches/bench_decimal.rs @@ -1,4 +1,4 @@ -use criterion::{black_box, criterion_group, criterion_main, Bencher, Criterion}; +use codspeed_criterion_compat::{black_box, criterion_group, criterion_main, Bencher, Criterion}; use pyo3::prelude::*; use pyo3::types::PyDict; diff --git a/pyo3-benches/benches/bench_dict.rs b/pyo3-benches/benches/bench_dict.rs index 62fd8820..d1c23466 100644 --- a/pyo3-benches/benches/bench_dict.rs +++ b/pyo3-benches/benches/bench_dict.rs @@ -1,4 +1,4 @@ -use criterion::{criterion_group, criterion_main, Bencher, Criterion}; +use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; use pyo3::types::IntoPyDict; use pyo3::{prelude::*, types::PyMapping}; diff --git a/pyo3-benches/benches/bench_err.rs b/pyo3-benches/benches/bench_err.rs index e3194512..998ed697 100644 --- a/pyo3-benches/benches/bench_err.rs +++ b/pyo3-benches/benches/bench_err.rs @@ -1,4 +1,4 @@ -use criterion::{criterion_group, criterion_main, Bencher, Criterion}; +use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; use pyo3::{exceptions::PyValueError, prelude::*}; diff --git a/pyo3-benches/benches/bench_extract.rs b/pyo3-benches/benches/bench_extract.rs index af454930..6ff22d38 100644 --- a/pyo3-benches/benches/bench_extract.rs +++ b/pyo3-benches/benches/bench_extract.rs @@ -1,4 +1,4 @@ -use criterion::{black_box, criterion_group, criterion_main, Bencher, Criterion}; +use codspeed_criterion_compat::{black_box, criterion_group, criterion_main, Bencher, Criterion}; use pyo3::{ types::{PyDict, PyFloat, PyInt, PyString}, diff --git a/pyo3-benches/benches/bench_frompyobject.rs b/pyo3-benches/benches/bench_frompyobject.rs index c2dfbc0e..1ad53c71 100644 --- a/pyo3-benches/benches/bench_frompyobject.rs +++ b/pyo3-benches/benches/bench_frompyobject.rs @@ -1,4 +1,4 @@ -use criterion::{black_box, criterion_group, criterion_main, Bencher, Criterion}; +use codspeed_criterion_compat::{black_box, criterion_group, criterion_main, Bencher, Criterion}; use pyo3::{ prelude::*, diff --git a/pyo3-benches/benches/bench_gil.rs b/pyo3-benches/benches/bench_gil.rs index e7290644..283e9149 100644 --- a/pyo3-benches/benches/bench_gil.rs +++ b/pyo3-benches/benches/bench_gil.rs @@ -1,4 +1,4 @@ -use criterion::{criterion_group, criterion_main, BatchSize, Bencher, Criterion}; +use codspeed_criterion_compat::{criterion_group, criterion_main, BatchSize, Bencher, Criterion}; use pyo3::{prelude::*, GILPool}; diff --git a/pyo3-benches/benches/bench_intern.rs b/pyo3-benches/benches/bench_intern.rs index b70edf92..d8dd1b8f 100644 --- a/pyo3-benches/benches/bench_intern.rs +++ b/pyo3-benches/benches/bench_intern.rs @@ -1,4 +1,4 @@ -use criterion::{criterion_group, criterion_main, Bencher, Criterion}; +use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; use pyo3::prelude::*; diff --git a/pyo3-benches/benches/bench_list.rs b/pyo3-benches/benches/bench_list.rs index dd2e3db1..34bf9d85 100644 --- a/pyo3-benches/benches/bench_list.rs +++ b/pyo3-benches/benches/bench_list.rs @@ -1,4 +1,4 @@ -use criterion::{criterion_group, criterion_main, Bencher, Criterion}; +use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; use pyo3::prelude::*; use pyo3::types::{PyList, PySequence}; diff --git a/pyo3-benches/benches/bench_pyclass.rs b/pyo3-benches/benches/bench_pyclass.rs index 66cca044..b917a4ac 100644 --- a/pyo3-benches/benches/bench_pyclass.rs +++ b/pyo3-benches/benches/bench_pyclass.rs @@ -1,4 +1,4 @@ -use criterion::{criterion_group, criterion_main, Criterion}; +use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; use pyo3::{impl_::pyclass::LazyTypeObject, prelude::*}; /// This is a feature-rich class instance used to benchmark various parts of the pyclass lifecycle. @@ -26,7 +26,7 @@ impl MyClass { } } -pub fn first_time_init(b: &mut criterion::Bencher<'_>) { +pub fn first_time_init(b: &mut Bencher<'_>) { Python::with_gil(|py| { b.iter(|| { // This is using an undocumented internal PyO3 API to measure pyclass performance; please diff --git a/pyo3-benches/benches/bench_pyobject.rs b/pyo3-benches/benches/bench_pyobject.rs index 3d98e079..af25d61c 100644 --- a/pyo3-benches/benches/bench_pyobject.rs +++ b/pyo3-benches/benches/bench_pyobject.rs @@ -1,4 +1,4 @@ -use criterion::{criterion_group, criterion_main, Bencher, Criterion}; +use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; use pyo3::prelude::*; diff --git a/pyo3-benches/benches/bench_set.rs b/pyo3-benches/benches/bench_set.rs index 1bc81599..0753a2f9 100644 --- a/pyo3-benches/benches/bench_set.rs +++ b/pyo3-benches/benches/bench_set.rs @@ -1,4 +1,4 @@ -use criterion::{criterion_group, criterion_main, Bencher, Criterion}; +use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; use pyo3::prelude::*; use pyo3::types::PySet; diff --git a/pyo3-benches/benches/bench_tuple.rs b/pyo3-benches/benches/bench_tuple.rs index a2988b15..f224ee1b 100644 --- a/pyo3-benches/benches/bench_tuple.rs +++ b/pyo3-benches/benches/bench_tuple.rs @@ -1,4 +1,4 @@ -use criterion::{criterion_group, criterion_main, Bencher, Criterion}; +use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; use pyo3::prelude::*; use pyo3::types::{PyList, PySequence, PyTuple};