add maximum Python version check (#3821)

* add maximum Python version check

* restore dependency of `pyo3-macros-backend` on `pyo3-build-config`

* fix clippy-all noxfile job

.github/workflows/ci.yml

@@ -468,6 +468,18 @@ jobs:
           echo PYO3_CONFIG_FILE=$PYO3_CONFIG_FILE >> $GITHUB_ENV
       - run: python3 -m nox -s test
 
+  test-version-limits:
+    needs: [fmt]
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || (github.event_name != 'pull_request' && github.ref != 'refs/heads/main') }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: Swatinem/rust-cache@v2
+        continue-on-error: true
+      - uses: dtolnay/rust-toolchain@stable
+      - run: python3 -m pip install --upgrade pip && pip install nox
+      - run: python3 -m nox -s test-version-limits
+
   conclusion:
     needs:
       - fmt
@@ -480,6 +492,8 @@ jobs:
       - docsrs
       - coverage
       - emscripten
+      - test-debug
+      - test-version-limits
     if: always()
     runs-on: ubuntu-latest
     steps:

Cargo.toml

@@ -81,7 +81,7 @@ multiple-pymethods = ["inventory", "pyo3-macros/multiple-pymethods"]
 extension-module = ["pyo3-ffi/extension-module"]
 
 # Use the Python limited API. See https://www.python.org/dev/peps/pep-0384/ for more.
-abi3 = ["pyo3-build-config/abi3", "pyo3-ffi/abi3", "pyo3-macros/abi3"]
+abi3 = ["pyo3-build-config/abi3", "pyo3-ffi/abi3"]
 
 # With abi3, we can manually set the minimum Python version.
 abi3-py37 = ["abi3-py38", "pyo3-build-config/abi3-py37", "pyo3-ffi/abi3-py37"]

newsfragments/3821.packaging.md

@@ -0,0 +1 @@
+Check maximum version of Python at build time and for versions not yet supported require opt-in to the `abi3` stable ABI by the environment variable `PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1`.

noxfile.py

@@ -1,3 +1,4 @@
+from contextlib import contextmanager
 import json
 import os
 import re
@@ -7,9 +8,10 @@ import tempfile
 from functools import lru_cache
 from glob import glob
 from pathlib import Path
-from typing import Any, Callable, Dict, List, Optional, Tuple
+from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple
 
 import nox
+import nox.command
 
 nox.options.sessions = ["test", "clippy", "rustfmt", "ruff", "docs"]
 
@@ -100,7 +102,7 @@ def _clippy(session: nox.Session, *, env: Dict[str, str] = None) -> bool:
                 "--deny=warnings",
                 env=env,
             )
-        except Exception:
+        except nox.command.CommandFailed:
             success = False
     return success
 
@@ -564,6 +566,33 @@ def ffi_check(session: nox.Session):
     _run_cargo(session, "run", _FFI_CHECK)
 
 
+@nox.session(name="test-version-limits")
+def test_version_limits(session: nox.Session):
+    env = os.environ.copy()
+    with _config_file() as config_file:
+        env["PYO3_CONFIG_FILE"] = config_file.name
+
+        assert "3.6" not in PY_VERSIONS
+        config_file.set("CPython", "3.6")
+        _run_cargo(session, "check", env=env, expect_error=True)
+
+        assert "3.13" not in PY_VERSIONS
+        config_file.set("CPython", "3.13")
+        _run_cargo(session, "check", env=env, expect_error=True)
+
+        # 3.13 CPython should build with forward compatibility
+        env["PYO3_USE_ABI3_FORWARD_COMPATIBILITY"] = "1"
+        _run_cargo(session, "check", env=env)
+
+        assert "3.6" not in PYPY_VERSIONS
+        config_file.set("PyPy", "3.6")
+        _run_cargo(session, "check", env=env, expect_error=True)
+
+        assert "3.11" not in PYPY_VERSIONS
+        config_file.set("PyPy", "3.11")
+        _run_cargo(session, "check", env=env, expect_error=True)
+
+
 def _build_docs_for_ffi_check(session: nox.Session) -> None:
     # pyo3-ffi-check needs to scrape docs of pyo3-ffi
     _run_cargo(session, "doc", _FFI_CHECK, "-p", "pyo3-ffi", "--no-deps")
@@ -652,7 +681,13 @@ def _run(session: nox.Session, *args: str, **kwargs: Any) -> None:
         print("::endgroup::", file=sys.stderr)
 
 
-def _run_cargo(session: nox.Session, *args: str, **kwargs: Any) -> None:
+def _run_cargo(
+    session: nox.Session, *args: str, expect_error: bool = False, **kwargs: Any
+) -> None:
+    if expect_error:
+        if "success_codes" in kwargs:
+            raise ValueError("expect_error overrides success_codes")
+        kwargs["success_codes"] = [101]
     _run(session, "cargo", *args, **kwargs, external=True)
 
 
@@ -700,24 +735,14 @@ def _get_output(*args: str) -> str:
 def _for_all_version_configs(
     session: nox.Session, job: Callable[[Dict[str, str]], None]
 ) -> None:
-    with tempfile.NamedTemporaryFile("r+") as config:
-        env = os.environ.copy()
-        env["PYO3_CONFIG_FILE"] = config.name
-
-        def _job_with_config(implementation, version) -> bool:
-            config.seek(0)
-            config.truncate(0)
-            config.write(
-                f"""\
-implementation={implementation}
-version={version}
-suppress_build_script_link_lines=true
-"""
-            )
-            config.flush()
+    env = os.environ.copy()
+    with _config_file() as config_file:
+        env["PYO3_CONFIG_FILE"] = config_file.name
 
+        def _job_with_config(implementation, version):
             session.log(f"{implementation} {version}")
-            return job(env)
+            config_file.set(implementation, version)
+            job(env)
 
         for version in PY_VERSIONS:
             _job_with_config("CPython", version)
@@ -726,5 +751,34 @@ suppress_build_script_link_lines=true
             _job_with_config("PyPy", version)
 
 
+class _ConfigFile:
+    def __init__(self, config_file) -> None:
+        self._config_file = config_file
+
+    def set(self, implementation: str, version: str) -> None:
+        """Set the contents of this config file to the given implementation and version."""
+        self._config_file.seek(0)
+        self._config_file.truncate(0)
+        self._config_file.write(
+            f"""\
+implementation={implementation}
+version={version}
+suppress_build_script_link_lines=true
+"""
+        )
+        self._config_file.flush()
+
+    @property
+    def name(self) -> str:
+        return self._config_file.name
+
+
+@contextmanager
+def _config_file() -> Iterator[_ConfigFile]:
+    """Creates a temporary config file which can be repeatedly set to different values."""
+    with tempfile.NamedTemporaryFile("r+") as config:
+        yield _ConfigFile(config)
+
+
 _BENCHES = "--manifest-path=pyo3-benches/Cargo.toml"
 _FFI_CHECK = "--manifest-path=pyo3-ffi-check/Cargo.toml"

pyo3-build-config/src/impl_.rs

@@ -701,6 +701,7 @@ fn have_python_interpreter() -> bool {
 /// Must be called from a PyO3 crate build script.
 fn is_abi3() -> bool {
     cargo_env_var("CARGO_FEATURE_ABI3").is_some()
+        || env_var("PYO3_USE_ABI3_FORWARD_COMPATIBILITY").map_or(false, |os_str| os_str == "1")
 }
 
 /// Gets the minimum supported Python version from PyO3 `abi3-py*` features.

pyo3-ffi/build.rs

@@ -4,18 +4,76 @@ use pyo3_build_config::{
         cargo_env_var, env_var, errors::Result, is_linking_libpython, resolve_interpreter_config,
         InterpreterConfig, PythonVersion,
     },
+    PythonImplementation,
 };
 
 /// Minimum Python version PyO3 supports.
-const MINIMUM_SUPPORTED_VERSION: PythonVersion = PythonVersion { major: 3, minor: 7 };
+struct SupportedVersions {
+    min: PythonVersion,
+    max: PythonVersion,
+}
+
+const SUPPORTED_VERSIONS_CPYTHON: SupportedVersions = SupportedVersions {
+    min: PythonVersion { major: 3, minor: 7 },
+    max: PythonVersion {
+        major: 3,
+        minor: 12,
+    },
+};
+
+const SUPPORTED_VERSIONS_PYPY: SupportedVersions = SupportedVersions {
+    min: PythonVersion { major: 3, minor: 7 },
+    max: PythonVersion {
+        major: 3,
+        minor: 10,
+    },
+};
 
 fn ensure_python_version(interpreter_config: &InterpreterConfig) -> Result<()> {
-    ensure!(
-        interpreter_config.version >= MINIMUM_SUPPORTED_VERSION,
-        "the configured Python interpreter version ({}) is lower than PyO3's minimum supported version ({})",
-        interpreter_config.version,
-        MINIMUM_SUPPORTED_VERSION,
-    );
+    // This is an undocumented env var which is only really intended to be used in CI / for testing
+    // and development.
+    if std::env::var("UNSAFE_PYO3_SKIP_VERSION_CHECK").as_deref() == Ok("1") {
+        return Ok(());
+    }
+
+    match interpreter_config.implementation {
+        PythonImplementation::CPython => {
+            let versions = SUPPORTED_VERSIONS_CPYTHON;
+            ensure!(
+                interpreter_config.version >= versions.min,
+                "the configured Python interpreter version ({}) is lower than PyO3's minimum supported version ({})",
+                interpreter_config.version,
+                versions.min,
+            );
+            ensure!(
+                interpreter_config.version <= versions.max || env_var("PYO3_USE_ABI3_FORWARD_COMPATIBILITY").map_or(false, |os_str| os_str == "1"),
+                "the configured Python interpreter version ({}) is newer than PyO3's maximum supported version ({})\n\
+                 = help: please check if an updated version of PyO3 is available. Current version: {}\n\
+                 = help: set PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 to suppress this check and build anyway using the stable ABI",
+                interpreter_config.version,
+                versions.max,
+                std::env::var("CARGO_PKG_VERSION").unwrap(),
+            );
+        }
+        PythonImplementation::PyPy => {
+            let versions = SUPPORTED_VERSIONS_PYPY;
+            ensure!(
+                interpreter_config.version >= versions.min,
+                "the configured PyPy interpreter version ({}) is lower than PyO3's minimum supported version ({})",
+                interpreter_config.version,
+                versions.min,
+            );
+            // PyO3 does not support abi3, so we cannot offer forward compatibility
+            ensure!(
+                interpreter_config.version <= versions.max,
+                "the configured PyPy interpreter version ({}) is newer than PyO3's maximum supported version ({})\n\
+                 = help: please check if an updated version of PyO3 is available. Current version: {}",
+                interpreter_config.version,
+                versions.max,
+                std::env::var("CARGO_PKG_VERSION").unwrap()
+            );
+        }
+    }
 
     Ok(())
 }

pyo3-macros-backend/Cargo.toml

@@ -14,17 +14,15 @@ edition = "2021"
 # not to depend on proc-macro itself.
 # See https://github.com/PyO3/pyo3/pull/810 for more.
 [dependencies]
-quote = { version = "1", default-features = false }
-proc-macro2 = { version = "1", default-features = false }
 heck = "0.4"
+proc-macro2 = { version = "1", default-features = false }
+pyo3-build-config = { path = "../pyo3-build-config", version = "0.21.0-dev", features = ["resolve-config"] }
+quote = { version = "1", default-features = false }
 
 [dependencies.syn]
 version = "2"
 default-features = false
 features = ["derive", "parsing", "printing", "clone-impls", "full", "extra-traits"]
 
-[features]
-abi3 = []
-
 [lints]
 workspace = true

pyo3-macros-backend/src/method.rs

@@ -12,7 +12,7 @@ use crate::{
         FunctionSignature, PyFunctionArgPyO3Attributes, PyFunctionOptions, SignatureAttribute,
     },
     quotes,
-    utils::{self, PythonDoc},
+    utils::{self, is_abi3, PythonDoc},
 };
 
 #[derive(Clone, Debug)]
@@ -234,8 +234,8 @@ impl CallingConvention {
         } else if signature.python_signature.kwargs.is_some() {
             // for functions that accept **kwargs, always prefer varargs
             Self::Varargs
-        } else if cfg!(not(feature = "abi3")) {
-            // Not available in the Stable ABI as of Python 3.10
+        } else if !is_abi3() {
+            // FIXME: available in the stable ABI since 3.10
             Self::Fastcall
         } else {
             Self::Varargs

pyo3-macros-backend/src/utils.rs

@@ -166,3 +166,7 @@ pub fn apply_renaming_rule(rule: RenamingRule, name: &str) -> String {
         RenamingRule::Uppercase => name.to_uppercase(),
     }
 }
+
+pub(crate) fn is_abi3() -> bool {
+    pyo3_build_config::get().abi3
+}

pyo3-macros/Cargo.toml

@@ -16,8 +16,6 @@ proc-macro = true
 [features]
 multiple-pymethods = []
 
-abi3 = ["pyo3-macros-backend/abi3"]
-
 [dependencies]
 proc-macro2 = { version = "1", default-features = false }
 quote = "1"