Merge pull request #2709 from davidhewitt/py3.12

ci: add Python 3.12-dev jobs

.github/workflows/build.yml

@@ -25,6 +25,7 @@ on:
 
 jobs:
   build:
+    continue-on-error: ${{ inputs.python-version == '3.12-dev' }}
     runs-on: ${{ inputs.os }}
     steps:
       - uses: actions/checkout@v3

.github/workflows/ci.yml

@@ -194,6 +194,7 @@ jobs:
           "3.9",
           "3.10",
           "3.11",
+          "3.12-dev",
           "pypy3.7",
           "pypy3.8",
           "pypy3.9",

pyo3-ffi-check/src/main.rs

@@ -40,9 +40,9 @@ fn main() {
                     pyo3_ffi_align,
                     bindgen_align
                 );
-            } else {
-                pyo3_ffi_check_macro::for_all_fields!($name, check_field);
             }
+
+            pyo3_ffi_check_macro::for_all_fields!($name, check_field);
         }};
     }
 

pyo3-ffi/src/cpython/code.rs

@@ -2,7 +2,7 @@ use crate::object::*;
 use crate::pyport::Py_ssize_t;
 
 #[allow(unused_imports)]
-use std::os::raw::{c_char, c_int, c_uchar, c_void};
+use std::os::raw::{c_char, c_int, c_short, c_uchar, c_void};
 #[cfg(not(PyPy))]
 use std::ptr::addr_of_mut;
 
@@ -13,6 +13,16 @@ use std::ptr::addr_of_mut;
 #[cfg(all(Py_3_8, not(PyPy), not(Py_3_11)))]
 opaque_struct!(_PyOpcache);
 
+#[cfg(Py_3_12)]
+#[repr(C)]
+#[derive(Copy, Clone)]
+pub struct _PyCoCached {
+    pub _co_code: *mut PyObject,
+    pub _co_varnames: *mut PyObject,
+    pub _co_cellvars: *mut PyObject,
+    pub _co_freevars: *mut PyObject,
+}
+
 #[cfg(all(not(PyPy), not(Py_3_7)))]
 opaque_struct!(PyCodeObject);
 
@@ -85,17 +95,24 @@ pub struct PyCodeObject {
     pub co_names: *mut PyObject,
     pub co_exceptiontable: *mut PyObject,
     pub co_flags: c_int,
+    #[cfg(not(Py_3_12))]
     pub co_warmup: c_int,
+    #[cfg(Py_3_12)]
+    pub _co_linearray_entry_size: c_short,
     pub co_argcount: c_int,
     pub co_posonlyargcount: c_int,
     pub co_kwonlyargcount: c_int,
     pub co_stacksize: c_int,
     pub co_firstlineno: c_int,
+
     pub co_nlocalsplus: c_int,
+    #[cfg(Py_3_12)]
+    pub co_framesize: c_int,
     pub co_nlocals: c_int,
     pub co_nplaincellvars: c_int,
     pub co_ncellvars: c_int,
     pub co_nfreevars: c_int,
+
     pub co_localsplusnames: *mut PyObject,
     pub co_localspluskinds: *mut PyObject,
     pub co_filename: *mut PyObject,
@@ -103,9 +120,15 @@ pub struct PyCodeObject {
     pub co_qualname: *mut PyObject,
     pub co_linetable: *mut PyObject,
     pub co_weakreflist: *mut PyObject,
+    #[cfg(not(Py_3_12))]
     pub _co_code: *mut PyObject,
+    #[cfg(Py_3_12)]
+    pub _co_cached: *mut _PyCoCached,
+    #[cfg(not(Py_3_12))]
     pub _co_linearray: *mut c_char,
     pub _co_firsttraceable: c_int,
+    #[cfg(Py_3_12)]
+    pub _co_linearray: *mut c_char,
     pub co_extra: *mut c_void,
     pub co_code_adaptive: [c_char; 1],
 }

pyo3-ffi/src/cpython/compile.rs

@@ -30,12 +30,27 @@ pub struct PyCompilerFlags {
 
 // skipped non-limited _PyCompilerFlags_INIT
 
+#[cfg(all(Py_3_12, not(PyPy)))]
+#[repr(C)]
+#[derive(Copy, Clone)]
+pub struct _PyCompilerSrcLocation {
+    pub lineno: c_int,
+    pub end_lineno: c_int,
+    pub col_offset: c_int,
+    pub end_col_offset: c_int,
+}
+
+// skipped SRC_LOCATION_FROM_AST
+
 #[cfg(not(PyPy))]
 #[repr(C)]
 #[derive(Copy, Clone)]
 pub struct PyFutureFeatures {
     pub ff_features: c_int,
+    #[cfg(not(Py_3_12))]
     pub ff_lineno: c_int,
+    #[cfg(Py_3_12)]
+    pub ff_location: _PyCompilerSrcLocation,
 }
 
 pub const FUTURE_NESTED_SCOPES: &str = "nested_scopes";

pyo3-ffi/src/cpython/initconfig.rs

@@ -91,6 +91,8 @@ pub struct PyConfig {
     #[cfg(all(Py_3_9, not(Py_3_10)))]
     pub _use_peg_parser: c_int,
     pub tracemalloc: c_int,
+    #[cfg(Py_3_12)]
+    pub perf_profiling: c_int,
     pub import_time: c_int,
     #[cfg(Py_3_11)]
     pub code_debug_ranges: c_int,
@@ -137,6 +139,8 @@ pub struct PyConfig {
     pub use_frozen_modules: c_int,
     #[cfg(Py_3_11)]
     pub safe_path: c_int,
+    #[cfg(Py_3_12)]
+    pub int_max_str_digits: c_int,
     pub pathconfig_warnings: c_int,
     #[cfg(Py_3_10)]
     pub program_name: *mut wchar_t,
@@ -163,7 +167,7 @@ pub struct PyConfig {
     pub run_filename: *mut wchar_t,
     pub _install_importlib: c_int,
     pub _init_main: c_int,
-    #[cfg(Py_3_9)]
+    #[cfg(all(Py_3_9, not(Py_3_12)))]
     pub _isolated_interpreter: c_int,
     #[cfg(Py_3_11)]
     pub _is_python_build: c_int,

pyo3-ffi/src/cpython/object.rs

@@ -276,6 +276,8 @@ pub struct PyTypeObject {
     pub tp_finalize: Option<object::destructor>,
     #[cfg(Py_3_8)]
     pub tp_vectorcall: Option<super::vectorcallfunc>,
+    #[cfg(Py_3_12)]
+    pub tp_watched: c_char,
     #[cfg(any(all(PyPy, Py_3_8, not(Py_3_10)), all(not(PyPy), Py_3_8, not(Py_3_9))))]
     pub tp_print: Option<printfunc>,
     #[cfg(all(PyPy, not(Py_3_10)))]

pyo3-ffi/src/cpython/pyerrors.rs

@@ -65,6 +65,8 @@ pub struct PyImportErrorObject {
     pub msg: *mut PyObject,
     pub name: *mut PyObject,
     pub path: *mut PyObject,
+    #[cfg(Py_3_12)]
+    pub name_from: *mut PyObject,
 }
 
 #[cfg(not(PyPy))]

pyo3-ffi/src/cpython/unicodeobject.rs

@@ -118,10 +118,7 @@ where
 }
 
 const STATE_INTERNED_INDEX: usize = 0;
-#[cfg(not(Py_3_12))]
 const STATE_INTERNED_WIDTH: u8 = 2;
-#[cfg(Py_3_12)]
-const STATE_INTERNED_WIDTH: u8 = 1;
 
 const STATE_KIND_INDEX: usize = STATE_INTERNED_WIDTH as usize;
 const STATE_KIND_WIDTH: u8 = 3;
@@ -265,20 +262,22 @@ pub struct PyASCIIObject {
 
 /// Interacting with the bitfield is not actually well-defined, so we mark these APIs unsafe.
 impl PyASCIIObject {
+    #[cfg_attr(not(Py_3_12), allow(rustdoc::broken_intra_doc_links))] // SSTATE_INTERNED_IMMORTAL_STATIC requires 3.12
     /// Get the `interned` field of the [`PyASCIIObject`] state bitfield.
     ///
     /// Returns one of: [`SSTATE_NOT_INTERNED`], [`SSTATE_INTERNED_MORTAL`],
-    /// or on CPython earlier than 3.12, [`SSTATE_INTERNED_IMMORTAL`]
+    /// [`SSTATE_INTERNED_IMMORTAL`], or [`SSTATE_INTERNED_IMMORTAL_STATIC`].
     #[inline]
     pub unsafe fn interned(&self) -> c_uint {
         PyASCIIObjectState::from(self.state).interned()
     }
 
+    #[cfg_attr(not(Py_3_12), allow(rustdoc::broken_intra_doc_links))] // SSTATE_INTERNED_IMMORTAL_STATIC requires 3.12
     /// Set the `interned` field of the [`PyASCIIObject`] state bitfield.
     ///
     /// Calling this function with an argument that is not [`SSTATE_NOT_INTERNED`],
-    /// [`SSTATE_INTERNED_MORTAL`], or on CPython earlier than 3.12,
-    /// [`SSTATE_INTERNED_IMMORTAL`] is invalid.
+    /// [`SSTATE_INTERNED_MORTAL`], [`SSTATE_INTERNED_IMMORTAL`], or
+    /// [`SSTATE_INTERNED_IMMORTAL_STATIC`] is invalid.
     #[inline]
     pub unsafe fn set_interned(&mut self, val: c_uint) {
         let mut state = PyASCIIObjectState::from(self.state);
@@ -398,12 +397,14 @@ extern "C" {
 
 pub const SSTATE_NOT_INTERNED: c_uint = 0;
 pub const SSTATE_INTERNED_MORTAL: c_uint = 1;
-#[cfg(not(Py_3_12))]
 pub const SSTATE_INTERNED_IMMORTAL: c_uint = 2;
+#[cfg(Py_3_12)]
+pub const SSTATE_INTERNED_IMMORTAL_STATIC: c_uint = 3;
 
 #[inline]
 pub unsafe fn PyUnicode_IS_ASCII(op: *mut PyObject) -> c_uint {
     debug_assert!(crate::PyUnicode_Check(op) != 0);
+    #[cfg(not(Py_3_12))]
     debug_assert!(PyUnicode_IS_READY(op) != 0);
 
     (*(op as *mut PyASCIIObject)).ascii()
@@ -420,7 +421,7 @@ pub unsafe fn PyUnicode_IS_COMPACT_ASCII(op: *mut PyObject) -> c_uint {
 }
 
 #[cfg(not(Py_3_12))]
-#[cfg_attr(Py_3_10, deprecated(note = "Python 3.10"))]
+#[deprecated(note = "Removed in Python 3.12")]
 pub const PyUnicode_WCHAR_KIND: c_uint = 0;
 
 pub const PyUnicode_1BYTE_KIND: c_uint = 1;
@@ -445,6 +446,7 @@ pub unsafe fn PyUnicode_4BYTE_DATA(op: *mut PyObject) -> *mut Py_UCS4 {
 #[inline]
 pub unsafe fn PyUnicode_KIND(op: *mut PyObject) -> c_uint {
     debug_assert!(crate::PyUnicode_Check(op) != 0);
+    #[cfg(not(Py_3_12))]
     debug_assert!(PyUnicode_IS_READY(op) != 0);
 
     (*(op as *mut PyASCIIObject)).kind()
@@ -484,6 +486,7 @@ pub unsafe fn PyUnicode_DATA(op: *mut PyObject) -> *mut c_void {
 #[inline]
 pub unsafe fn PyUnicode_GET_LENGTH(op: *mut PyObject) -> Py_ssize_t {
     debug_assert!(crate::PyUnicode_Check(op) != 0);
+    #[cfg(not(Py_3_12))]
     debug_assert!(PyUnicode_IS_READY(op) != 0);
 
     (*(op as *mut PyASCIIObject)).length
@@ -502,8 +505,13 @@ pub unsafe fn PyUnicode_IS_READY(op: *mut PyObject) -> c_uint {
     (*(op as *mut PyASCIIObject)).ready()
 }
 
+#[cfg(Py_3_12)]
+#[inline]
+pub unsafe fn PyUnicode_READY(_op: *mut PyObject) -> c_int {
+    0
+}
+
 #[cfg(not(Py_3_12))]
-#[cfg_attr(Py_3_10, deprecated(note = "Python 3.10"))]
 #[inline]
 pub unsafe fn PyUnicode_READY(op: *mut PyObject) -> c_int {
     debug_assert!(crate::PyUnicode_Check(op) != 0);

pyo3-ffi/src/unicodeobject.rs

@@ -61,6 +61,8 @@ extern "C" {
     pub fn PyUnicode_AsUCS4Copy(unicode: *mut PyObject) -> *mut Py_UCS4;
     #[cfg_attr(PyPy, link_name = "PyPyUnicode_GetLength")]
     pub fn PyUnicode_GetLength(unicode: *mut PyObject) -> Py_ssize_t;
+    #[cfg(not(Py_3_12))]
+    #[deprecated(note = "Removed in Python 3.12")]
     #[cfg_attr(PyPy, link_name = "PyPyUnicode_GetSize")]
     pub fn PyUnicode_GetSize(unicode: *mut PyObject) -> Py_ssize_t;
     pub fn PyUnicode_ReadChar(unicode: *mut PyObject, index: Py_ssize_t) -> Py_UCS4;

src/ffi/tests.rs

@@ -2,6 +2,7 @@ use crate::ffi::*;
 use crate::{types::PyDict, AsPyPointer, IntoPy, Py, PyAny, Python};
 
 use crate::types::PyString;
+#[cfg(not(Py_3_12))]
 use libc::wchar_t;
 
 #[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons
@@ -116,6 +117,7 @@ fn ascii_object_bitfield() {
         #[cfg(not(PyPy))]
         hash: 0,
         state: 0u32,
+        #[cfg(not(Py_3_12))]
         wstr: std::ptr::null_mut() as *mut wchar_t,
     };
 
@@ -124,9 +126,12 @@ fn ascii_object_bitfield() {
         assert_eq!(o.kind(), 0);
         assert_eq!(o.compact(), 0);
         assert_eq!(o.ascii(), 0);
+        #[cfg(not(Py_3_12))]
         assert_eq!(o.ready(), 0);
 
-        for i in 0..4 {
+        let interned_count = if cfg!(Py_3_12) { 2 } else { 4 };
+
+        for i in 0..interned_count {
             o.set_interned(i);
             assert_eq!(o.interned(), i);
         }
@@ -142,7 +147,9 @@ fn ascii_object_bitfield() {
         o.set_ascii(1);
         assert_eq!(o.ascii(), 1);
 
+        #[cfg(not(Py_3_12))]
         o.set_ready(1);
+        #[cfg(not(Py_3_12))]
         assert_eq!(o.ready(), 1);
     }
 }
@@ -163,6 +170,7 @@ fn ascii() {
             assert_eq!(ascii.kind(), PyUnicode_1BYTE_KIND);
             assert_eq!(ascii.compact(), 1);
             assert_eq!(ascii.ascii(), 1);
+            #[cfg(not(Py_3_12))]
             assert_eq!(ascii.ready(), 1);
 
             assert_eq!(PyUnicode_IS_ASCII(ptr), 1);
@@ -203,6 +211,7 @@ fn ucs4() {
             assert_eq!(ascii.kind(), PyUnicode_4BYTE_KIND);
             assert_eq!(ascii.compact(), 1);
             assert_eq!(ascii.ascii(), 0);
+            #[cfg(not(Py_3_12))]
             assert_eq!(ascii.ready(), 1);
 
             assert_eq!(PyUnicode_IS_ASCII(ptr), 0);