fix ffi check failures for 3.12.0b4

newsfragments/3342.changed.md

@@ -0,0 +1 @@
+Update `pyo3::ffi` struct definitions to be compatible with 3.12.0b4.

pyo3-ffi-check/macro/Cargo.toml

@@ -12,3 +12,4 @@ glob = "0.3"
 quote = "1"
 proc-macro2 = "1"
 scraper = "0.17"
+pyo3-build-config = { path = "../../pyo3-build-config" }

pyo3-ffi-check/macro/src/lib.rs

@@ -1,8 +1,14 @@
 use std::{env, fs, path::PathBuf};
 
 use proc_macro2::{Ident, Span, TokenStream, TokenTree};
+use pyo3_build_config::PythonVersion;
 use quote::quote;
 
+const PY_3_12: PythonVersion = PythonVersion {
+    major: 3,
+    minor: 12,
+};
+
 /// Macro which expands to multiple macro calls, one per pyo3-ffi struct.
 #[proc_macro]
 pub fn for_all_structs(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
@@ -130,16 +136,27 @@ pub fn for_all_fields(input: proc_macro::TokenStream) -> proc_macro::TokenStream
     let mut output = TokenStream::new();
 
     for el in html.select(&selector) {
-        let id = el
+        let field_name = el
             .value()
             .id()
             .unwrap()
             .strip_prefix("structfield.")
             .unwrap();
 
-        let field_ident = Ident::new(id, Span::call_site());
+        let field_ident = Ident::new(field_name, Span::call_site());
 
-        output.extend(quote!(#macro_name!(#struct_name, #field_ident);));
+        let bindgen_field_ident = if (pyo3_build_config::get().version >= PY_3_12)
+            && struct_name == "PyObject"
+            && field_name == "ob_refcnt"
+        {
+            // PyObject since 3.12 implements ob_refcnt as a union; bindgen creates
+            // an anonymous name for the field
+            Ident::new("__bindgen_anon_1", Span::call_site())
+        } else {
+            field_ident.clone()
+        };
+
+        output.extend(quote!(#macro_name!(#struct_name, #field_ident, #bindgen_field_ident);));
     }
 
     output.into()

pyo3-ffi-check/src/main.rs

@@ -47,9 +47,11 @@ fn main() {
     }
 
     macro_rules! check_field {
-        ($struct_name:ident, $field:ident) => {{
+        ($struct_name:ident, $field:ident, $bindgen_field:ident) => {{
+            #[allow(clippy::used_underscore_binding)]
             let pyo3_ffi_offset = memoffset::offset_of!(pyo3_ffi::$struct_name, $field);
-            let bindgen_offset = memoffset::offset_of!(bindings::$struct_name, $field);
+            #[allow(clippy::used_underscore_binding)]
+            let bindgen_offset = memoffset::offset_of!(bindings::$struct_name, $bindgen_field);
 
             if pyo3_ffi_offset != bindgen_offset {
                 failed = true;
@@ -79,7 +81,9 @@ fn main() {
     non_upper_case_globals,
     dead_code,
     improper_ctypes,
-    clippy::all
+    clippy::all,
+    // clippy fails with lots of errors if this is not set specifically
+    clippy::used_underscore_binding
 )]
 mod bindings {
     include!(concat!(env!("OUT_DIR"), "/bindings.rs"));

pyo3-ffi/src/cpython/code.rs

@@ -6,12 +6,30 @@ use std::os::raw::{c_char, c_int, c_short, c_uchar, c_void};
 #[cfg(not(PyPy))]
 use std::ptr::addr_of_mut;
 
+#[cfg(all(Py_3_8, not(PyPy), not(Py_3_11)))]
+opaque_struct!(_PyOpcache);
+
+pub const _PY_MONITORING_UNGROUPED_EVENTS: usize = 14;
+pub const _PY_MONITORING_EVENTS: usize = 16;
+
+#[cfg(Py_3_12)]
+#[repr(C)]
+#[derive(Copy, Clone)]
+pub struct _Py_Monitors {
+    pub tools: [u8; _PY_MONITORING_UNGROUPED_EVENTS],
+}
+
 // skipped _Py_CODEUNIT
+
 // skipped _Py_OPCODE
 // skipped _Py_OPARG
 
-#[cfg(all(Py_3_8, not(PyPy), not(Py_3_11)))]
+// skipped _py_make_codeunit
-opaque_struct!(_PyOpcache);
+
+// skipped _py_set_opcode
+
+// skipped _Py_MAKE_CODEUNIT
+// skipped _Py_SET_OPCODE
 
 #[cfg(Py_3_12)]
 #[repr(C)]
@@ -23,6 +41,27 @@ pub struct _PyCoCached {
     pub _co_freevars: *mut PyObject,
 }
 
+#[cfg(Py_3_12)]
+#[repr(C)]
+#[derive(Copy, Clone)]
+pub struct _PyCoLineInstrumentationData {
+    pub original_opcode: u8,
+    pub line_delta: i8,
+}
+
+#[cfg(Py_3_12)]
+#[repr(C)]
+#[derive(Copy, Clone)]
+pub struct _PyCoMonitoringData {
+    pub local_monitors: _Py_Monitors,
+    pub active_monitors: _Py_Monitors,
+    pub tools: *mut u8,
+    pub lines: *mut _PyCoLineInstrumentationData,
+    pub line_tools: *mut u8,
+    pub per_instruction_opcodes: *mut u8,
+    pub per_instruction_tools: *mut u8,
+}
+
 #[cfg(all(not(PyPy), not(Py_3_7)))]
 opaque_struct!(PyCodeObject);
 
@@ -97,8 +136,7 @@ pub struct PyCodeObject {
     pub co_flags: c_int,
     #[cfg(not(Py_3_12))]
     pub co_warmup: c_int,
-    #[cfg(Py_3_12)]
+
-    pub _co_linearray_entry_size: c_short,
     pub co_argcount: c_int,
     pub co_posonlyargcount: c_int,
     pub co_kwonlyargcount: c_int,
@@ -109,9 +147,12 @@ pub struct PyCodeObject {
     #[cfg(Py_3_12)]
     pub co_framesize: c_int,
     pub co_nlocals: c_int,
+    #[cfg(not(Py_3_12))]
     pub co_nplaincellvars: c_int,
     pub co_ncellvars: c_int,
     pub co_nfreevars: c_int,
+    #[cfg(Py_3_12)]
+    pub co_version: u32,
 
     pub co_localsplusnames: *mut PyObject,
     pub co_localspluskinds: *mut PyObject,
@@ -122,13 +163,15 @@ pub struct PyCodeObject {
     pub co_weakreflist: *mut PyObject,
     #[cfg(not(Py_3_12))]
     pub _co_code: *mut PyObject,
-    #[cfg(Py_3_12)]
-    pub _co_cached: *mut _PyCoCached,
     #[cfg(not(Py_3_12))]
     pub _co_linearray: *mut c_char,
-    pub _co_firsttraceable: c_int,
     #[cfg(Py_3_12)]
-    pub _co_linearray: *mut c_char,
+    pub _co_cached: *mut _PyCoCached,
+    #[cfg(Py_3_12)]
+    pub _co_instrumentation_version: u64,
+    #[cfg(Py_3_12)]
+    pub _co_monitoring: *mut _PyCoMonitoringData,
+    pub _co_firsttraceable: c_int,
     pub co_extra: *mut c_void,
     pub co_code_adaptive: [c_char; 1],
 }

pyo3-ffi/src/cpython/funcobject.rs

@@ -41,6 +41,8 @@ pub struct PyFunctionObject {
     pub func_weakreflist: *mut PyObject,
     pub func_module: *mut PyObject,
     pub func_annotations: *mut PyObject,
+    #[cfg(Py_3_12)]
+    pub func_typeparams: *mut PyObject,
     pub vectorcall: Option<crate::vectorcallfunc>,
     #[cfg(Py_3_11)]
     pub func_version: u32,

pyo3-ffi/src/cpython/genobject.rs

@@ -16,6 +16,7 @@ pub struct PyGenObject {
     pub gi_frame: *mut PyFrameObject,
     #[cfg(not(Py_3_10))]
     pub gi_running: c_int,
+    #[cfg(not(Py_3_12))]
     pub gi_code: *mut PyObject,
     pub gi_weakreflist: *mut PyObject,
     pub gi_name: *mut PyObject,

pyo3-ffi/src/cpython/object.rs

@@ -299,6 +299,8 @@ pub struct PyTypeObject {
 #[derive(Clone)]
 pub struct _specialization_cache {
     pub getitem: *mut PyObject,
+    #[cfg(Py_3_12)]
+    pub getitem_version: u32,
 }
 
 #[repr(C)]