From 46fa1b2b80429367302c4b17db56cfb1888a0129 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Tue, 19 Dec 2023 16:16:13 +0000 Subject: [PATCH 1/2] add test which is broken with gevent --- pytests/pyproject.toml | 1 + pytests/src/misc.rs | 12 +++++++++++- pytests/tests/test_misc.py | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/pytests/pyproject.toml b/pytests/pyproject.toml index fb1ac3db..4d5b42bf 100644 --- a/pytests/pyproject.toml +++ b/pytests/pyproject.toml @@ -20,6 +20,7 @@ classifiers = [ [project.optional-dependencies] dev = [ + "gevent>=23.9.1", "hypothesis>=3.55", "pytest-asyncio>=0.21", "pytest-benchmark>=3.4", diff --git a/pytests/src/misc.rs b/pytests/src/misc.rs index 029e8b16..548c5ceb 100644 --- a/pytests/src/misc.rs +++ b/pytests/src/misc.rs @@ -1,4 +1,4 @@ -use pyo3::prelude::*; +use pyo3::{prelude::*, types::PyDict}; use std::borrow::Cow; #[pyfunction] @@ -17,10 +17,20 @@ fn accepts_bool(val: bool) -> bool { val } +#[pyfunction] +fn get_item_and_run_callback(dict: &PyDict, callback: &PyAny) -> PyResult<()> { + let item = dict.get_item("key")?.expect("key not found in dict"); + let string = item.to_string(); + callback.call0()?; + assert_eq!(item.to_string(), string); + Ok(()) +} + #[pymodule] pub fn misc(_py: Python<'_>, m: &PyModule) -> PyResult<()> { m.add_function(wrap_pyfunction!(issue_219, m)?)?; m.add_function(wrap_pyfunction!(get_type_full_name, m)?)?; m.add_function(wrap_pyfunction!(accepts_bool, m)?)?; + m.add_function(wrap_pyfunction!(get_item_and_run_callback, m)?)?; Ok(()) } diff --git a/pytests/tests/test_misc.py b/pytests/tests/test_misc.py index 06b2ce73..2f6cee63 100644 --- a/pytests/tests/test_misc.py +++ b/pytests/tests/test_misc.py @@ -2,6 +2,7 @@ import importlib import platform import sys +import gevent import pyo3_pytests.misc import pytest @@ -64,3 +65,39 @@ def test_accepts_numpy_bool(): assert pyo3_pytests.misc.accepts_bool(False) is False assert pyo3_pytests.misc.accepts_bool(numpy.bool_(True)) is True assert pyo3_pytests.misc.accepts_bool(numpy.bool_(False)) is False + + +class ArbitraryClass: + worker_id: int + iteration: int + + def __init__(self, worker_id: int, iteration: int): + self.worker_id = worker_id + self.iteration = iteration + + def __repr__(self): + return f"ArbitraryClass({self.worker_id}, {self.iteration})" + + def __del__(self): + print("del", self.worker_id, self.iteration) + + +def test_gevent(): + def worker(worker_id: int) -> None: + for iteration in range(2): + d = {"key": ArbitraryClass(worker_id, iteration)} + + def arbitrary_python_code(): + # remove the dictionary entry so that the class value can be + # garbage collected + del d["key"] + print("gevent sleep", worker_id, iteration) + gevent.sleep(0) + print("after gevent sleep", worker_id, iteration) + + print("start", worker_id, iteration) + pyo3_pytests.misc.get_item_and_run_callback(d, arbitrary_python_code) + print("end", worker_id, iteration) + + workers = [gevent.spawn(worker, i) for i in range(2)] + gevent.joinall(workers) From 49d7718823b6206d81b56fceac0087705350f009 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 23 Dec 2023 22:10:01 +0000 Subject: [PATCH 2/2] demonstrate switching to `Bound` API resolves `gevent` crash --- pytests/src/misc.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pytests/src/misc.rs b/pytests/src/misc.rs index 548c5ceb..bd941461 100644 --- a/pytests/src/misc.rs +++ b/pytests/src/misc.rs @@ -18,7 +18,11 @@ fn accepts_bool(val: bool) -> bool { } #[pyfunction] -fn get_item_and_run_callback(dict: &PyDict, callback: &PyAny) -> PyResult<()> { +fn get_item_and_run_callback(dict: Bound<'_, PyDict>, callback: Bound<'_, PyAny>) -> PyResult<()> { + // This function gives the opportunity to run a pure-Python callback so that + // gevent can instigate a context switch. This had problematic interactions + // with PyO3's removed "GIL Pool". + // For context, see https://github.com/PyO3/pyo3/issues/3668 let item = dict.get_item("key")?.expect("key not found in dict"); let string = item.to_string(); callback.call0()?;