From bf2c5a896ba84fca501ca0c5011bc7163d308846 Mon Sep 17 00:00:00 2001
From: David Hewitt <1939362+davidhewitt@users.noreply.github.com>
Date: Tue, 29 Aug 2023 07:26:36 +0100
Subject: [PATCH] add codspeed benchmarks

---
 .github/workflows/benches.yml              | 40 ++++++++++++++++++++++
 noxfile.py                                 | 13 +++++++
 pyo3-benches/Cargo.toml                    | 10 ++----
 pyo3-benches/benches/bench_any.rs          |  2 +-
 pyo3-benches/benches/bench_bigint.rs       |  2 +-
 pyo3-benches/benches/bench_call.rs         |  2 +-
 pyo3-benches/benches/bench_comparisons.rs  |  2 +-
 pyo3-benches/benches/bench_decimal.rs      |  2 +-
 pyo3-benches/benches/bench_dict.rs         |  2 +-
 pyo3-benches/benches/bench_err.rs          |  2 +-
 pyo3-benches/benches/bench_extract.rs      |  2 +-
 pyo3-benches/benches/bench_frompyobject.rs |  2 +-
 pyo3-benches/benches/bench_gil.rs          |  2 +-
 pyo3-benches/benches/bench_intern.rs       |  2 +-
 pyo3-benches/benches/bench_list.rs         |  2 +-
 pyo3-benches/benches/bench_pyclass.rs      |  4 +--
 pyo3-benches/benches/bench_pyobject.rs     |  2 +-
 pyo3-benches/benches/bench_set.rs          |  2 +-
 pyo3-benches/benches/bench_tuple.rs        |  2 +-
 19 files changed, 73 insertions(+), 24 deletions(-)
 create mode 100644 .github/workflows/benches.yml

diff --git a/.github/workflows/benches.yml b/.github/workflows/benches.yml
new file mode 100644
index 00000000..0e90da7e
--- /dev/null
+++ b/.github/workflows/benches.yml
@@ -0,0 +1,40 @@
+name: benches
+
+on:
+  push:
+    branches:
+      - "main"
+  pull_request:
+  # `workflow_dispatch` allows CodSpeed to trigger backtest
+  # performance analysis in order to generate initial data.
+  workflow_dispatch:
+
+jobs:
+  benchmarks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+      - uses: dtolnay/rust-toolchain@stable
+
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            pyo3-benches/target
+            target
+          key: cargo-${{ runner.os }}-bench-${{ hashFiles('**/Cargo.toml') }}
+        continue-on-error: true
+
+      - name: Install cargo-codspeed
+        run: cargo install cargo-codspeed
+
+      - name: Install nox
+        run: pip install nox
+
+      - name: Run the benchmarks
+        uses: CodSpeedHQ/action@v1
+        with:
+          run: nox -s codspeed
+          token: ${{ secrets.CODSPEED_TOKEN }}
diff --git a/noxfile.py b/noxfile.py
index ee24a26f..c49df1d8 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -137,6 +137,19 @@ def bench(session: nox.Session) -> bool:
     _run_cargo(session, "bench", _BENCHES, *session.posargs)
 
 
+@nox.session()
+def codspeed(session: nox.Session) -> bool:
+    # rust benchmarks
+    os.chdir(PYO3_DIR / "pyo3-benches")
+    _run_cargo(session, "codspeed", "build")
+    _run_cargo(session, "codspeed", "run")
+    # python benchmarks
+    os.chdir(PYO3_DIR / "pytests")
+    session.install("-r", "requirements-dev.txt", "pytest-codspeed")
+    session.install(".")
+    _run(session, "pytest", "--codspeed", external=True)
+
+
 @nox.session(name="clippy-all", venv_backend="none")
 def clippy_all(session: nox.Session) -> None:
     success = True
diff --git a/pyo3-benches/Cargo.toml b/pyo3-benches/Cargo.toml
index cc82fb00..e99ef09e 100644
--- a/pyo3-benches/Cargo.toml
+++ b/pyo3-benches/Cargo.toml
@@ -7,11 +7,13 @@ edition = "2021"
 publish = false
 
 [dependencies]
-pyo3 = { path = "../", features = ["auto-initialize"] }
+pyo3 = { path = "../", features = ["auto-initialize", "full"] }
 
 [dev-dependencies]
+codspeed-criterion-compat = "2.3"
 criterion = "0.5.1"
 num-bigint = "0.4.3"
+rust_decimal = { version = "1.0.0", default-features = false }
 
 [[bench]]
 name = "bench_any"
@@ -32,17 +34,14 @@ harness = false
 [[bench]]
 name = "bench_decimal"
 harness = false
-required-features = ["pyo3/rust_decimal"]
 
 [[bench]]
 name = "bench_dict"
 harness = false
-required-features = ["pyo3/hashbrown"]
 
 [[bench]]
 name = "bench_frompyobject"
 harness = false
-required-features = ["pyo3/macros"]
 
 [[bench]]
 name = "bench_gil"
@@ -55,7 +54,6 @@ harness = false
 [[bench]]
 name = "bench_pyclass"
 harness = false
-required-features = ["pyo3/macros"]
 
 [[bench]]
 name = "bench_pyobject"
@@ -64,7 +62,6 @@ harness = false
 [[bench]]
 name = "bench_set"
 harness = false
-required-features = ["pyo3/hashbrown"]
 
 [[bench]]
 name = "bench_tuple"
@@ -81,6 +78,5 @@ harness = false
 [[bench]]
 name = "bench_bigint"
 harness = false
-required-features = ["pyo3/num-bigint"]
 
 [workspace]
diff --git a/pyo3-benches/benches/bench_any.rs b/pyo3-benches/benches/bench_any.rs
index 765497fa..bfd010ef 100644
--- a/pyo3-benches/benches/bench_any.rs
+++ b/pyo3-benches/benches/bench_any.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::{
     types::{
diff --git a/pyo3-benches/benches/bench_bigint.rs b/pyo3-benches/benches/bench_bigint.rs
index 3df24d21..d3c71629 100644
--- a/pyo3-benches/benches/bench_bigint.rs
+++ b/pyo3-benches/benches/bench_bigint.rs
@@ -1,4 +1,4 @@
-use criterion::{black_box, criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{black_box, criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::{types::PyDict, PyAny, Python};
 
diff --git a/pyo3-benches/benches/bench_call.rs b/pyo3-benches/benches/bench_call.rs
index dd98c159..50772097 100644
--- a/pyo3-benches/benches/bench_call.rs
+++ b/pyo3-benches/benches/bench_call.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::prelude::*;
 
diff --git a/pyo3-benches/benches/bench_comparisons.rs b/pyo3-benches/benches/bench_comparisons.rs
index bfa4ac63..ffd4c1a4 100644
--- a/pyo3-benches/benches/bench_comparisons.rs
+++ b/pyo3-benches/benches/bench_comparisons.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::{prelude::*, pyclass::CompareOp, Python};
 
diff --git a/pyo3-benches/benches/bench_decimal.rs b/pyo3-benches/benches/bench_decimal.rs
index c412c4b1..203756b5 100644
--- a/pyo3-benches/benches/bench_decimal.rs
+++ b/pyo3-benches/benches/bench_decimal.rs
@@ -1,4 +1,4 @@
-use criterion::{black_box, criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{black_box, criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::prelude::*;
 use pyo3::types::PyDict;
diff --git a/pyo3-benches/benches/bench_dict.rs b/pyo3-benches/benches/bench_dict.rs
index 62fd8820..d1c23466 100644
--- a/pyo3-benches/benches/bench_dict.rs
+++ b/pyo3-benches/benches/bench_dict.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::types::IntoPyDict;
 use pyo3::{prelude::*, types::PyMapping};
diff --git a/pyo3-benches/benches/bench_err.rs b/pyo3-benches/benches/bench_err.rs
index e3194512..998ed697 100644
--- a/pyo3-benches/benches/bench_err.rs
+++ b/pyo3-benches/benches/bench_err.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::{exceptions::PyValueError, prelude::*};
 
diff --git a/pyo3-benches/benches/bench_extract.rs b/pyo3-benches/benches/bench_extract.rs
index af454930..6ff22d38 100644
--- a/pyo3-benches/benches/bench_extract.rs
+++ b/pyo3-benches/benches/bench_extract.rs
@@ -1,4 +1,4 @@
-use criterion::{black_box, criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{black_box, criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::{
     types::{PyDict, PyFloat, PyInt, PyString},
diff --git a/pyo3-benches/benches/bench_frompyobject.rs b/pyo3-benches/benches/bench_frompyobject.rs
index c2dfbc0e..1ad53c71 100644
--- a/pyo3-benches/benches/bench_frompyobject.rs
+++ b/pyo3-benches/benches/bench_frompyobject.rs
@@ -1,4 +1,4 @@
-use criterion::{black_box, criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{black_box, criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::{
     prelude::*,
diff --git a/pyo3-benches/benches/bench_gil.rs b/pyo3-benches/benches/bench_gil.rs
index e7290644..283e9149 100644
--- a/pyo3-benches/benches/bench_gil.rs
+++ b/pyo3-benches/benches/bench_gil.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, BatchSize, Bencher, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, BatchSize, Bencher, Criterion};
 
 use pyo3::{prelude::*, GILPool};
 
diff --git a/pyo3-benches/benches/bench_intern.rs b/pyo3-benches/benches/bench_intern.rs
index b70edf92..d8dd1b8f 100644
--- a/pyo3-benches/benches/bench_intern.rs
+++ b/pyo3-benches/benches/bench_intern.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::prelude::*;
 
diff --git a/pyo3-benches/benches/bench_list.rs b/pyo3-benches/benches/bench_list.rs
index dd2e3db1..34bf9d85 100644
--- a/pyo3-benches/benches/bench_list.rs
+++ b/pyo3-benches/benches/bench_list.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::prelude::*;
 use pyo3::types::{PyList, PySequence};
diff --git a/pyo3-benches/benches/bench_pyclass.rs b/pyo3-benches/benches/bench_pyclass.rs
index 66cca044..b917a4ac 100644
--- a/pyo3-benches/benches/bench_pyclass.rs
+++ b/pyo3-benches/benches/bench_pyclass.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion};
 use pyo3::{impl_::pyclass::LazyTypeObject, prelude::*};
 
 /// This is a feature-rich class instance used to benchmark various parts of the pyclass lifecycle.
@@ -26,7 +26,7 @@ impl MyClass {
     }
 }
 
-pub fn first_time_init(b: &mut criterion::Bencher<'_>) {
+pub fn first_time_init(b: &mut Bencher<'_>) {
     Python::with_gil(|py| {
         b.iter(|| {
             // This is using an undocumented internal PyO3 API to measure pyclass performance; please
diff --git a/pyo3-benches/benches/bench_pyobject.rs b/pyo3-benches/benches/bench_pyobject.rs
index 3d98e079..af25d61c 100644
--- a/pyo3-benches/benches/bench_pyobject.rs
+++ b/pyo3-benches/benches/bench_pyobject.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::prelude::*;
 
diff --git a/pyo3-benches/benches/bench_set.rs b/pyo3-benches/benches/bench_set.rs
index 1bc81599..0753a2f9 100644
--- a/pyo3-benches/benches/bench_set.rs
+++ b/pyo3-benches/benches/bench_set.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::prelude::*;
 use pyo3::types::PySet;
diff --git a/pyo3-benches/benches/bench_tuple.rs b/pyo3-benches/benches/bench_tuple.rs
index a2988b15..f224ee1b 100644
--- a/pyo3-benches/benches/bench_tuple.rs
+++ b/pyo3-benches/benches/bench_tuple.rs
@@ -1,4 +1,4 @@
-use criterion::{criterion_group, criterion_main, Bencher, Criterion};
+use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion};
 
 use pyo3::prelude::*;
 use pyo3::types::{PyList, PySequence, PyTuple};