Run pytest pyo3-benchmarks on CI

.github/workflows/bench.yml

@@ -8,7 +8,7 @@ name: Benchmark
 
 jobs:
   benchmark:
-    name: Benchmark
+    name: Cargo benchmark
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -60,3 +60,48 @@ jobs:
           comment-on-alert: true
           alert-comment-cc-users: '@PyO3/pyo3'
           auto-push: ${{ github.event_name != 'pull_request' }}
+
+  pytest-benchmark:
+    name: pytest benchmark
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: cargo-${{ runner.os }}-pytest-bench-${{ hashFiles('**/Cargo.toml') }}
+        continue-on-error: true
+
+      - name: Download previous benchmark data
+        uses: actions/cache@v1
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-pytest-benchmark
+
+      - name: Run benchmarks
+        run: |
+          cd examples/pyo3-benchmarks
+          python -m pip install -r requirements-dev.txt
+          python setup.py develop
+          pytest --benchmark-json ../../output.json
+      - name: Store benchmark result
+        uses: rhysd/github-action-benchmark@v1
+        with:
+          name: pytest-bench
+          tool: 'pytest'
+          output-file-path: output.json
+          fail-on-alert: true
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          comment-on-alert: true
+          alert-comment-cc-users: '@PyO3/pyo3'
+          auto-push: ${{ github.event_name != 'pull_request' }}