From 655de94749a1b0eb9113c6b655161820620980e4 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Tue, 25 Jul 2023 05:26:38 +0100 Subject: [PATCH] fix ffi check failures for 3.12.0b4 --- newsfragments/3342.changed.md | 1 + pyo3-ffi-check/macro/Cargo.toml | 1 + pyo3-ffi-check/macro/src/lib.rs | 23 ++++++++++-- pyo3-ffi-check/src/main.rs | 10 +++-- pyo3-ffi/src/cpython/code.rs | 59 ++++++++++++++++++++++++++---- pyo3-ffi/src/cpython/funcobject.rs | 2 + pyo3-ffi/src/cpython/genobject.rs | 1 + pyo3-ffi/src/cpython/object.rs | 2 + 8 files changed, 85 insertions(+), 14 deletions(-) create mode 100644 newsfragments/3342.changed.md diff --git a/newsfragments/3342.changed.md b/newsfragments/3342.changed.md new file mode 100644 index 00000000..71435df0 --- /dev/null +++ b/newsfragments/3342.changed.md @@ -0,0 +1 @@ +Update `pyo3::ffi` struct definitions to be compatible with 3.12.0b4. diff --git a/pyo3-ffi-check/macro/Cargo.toml b/pyo3-ffi-check/macro/Cargo.toml index 359e7337..46395fc8 100644 --- a/pyo3-ffi-check/macro/Cargo.toml +++ b/pyo3-ffi-check/macro/Cargo.toml @@ -12,3 +12,4 @@ glob = "0.3" quote = "1" proc-macro2 = "1" scraper = "0.17" +pyo3-build-config = { path = "../../pyo3-build-config" } diff --git a/pyo3-ffi-check/macro/src/lib.rs b/pyo3-ffi-check/macro/src/lib.rs index 5d036062..e572a9ad 100644 --- a/pyo3-ffi-check/macro/src/lib.rs +++ b/pyo3-ffi-check/macro/src/lib.rs @@ -1,8 +1,14 @@ use std::{env, fs, path::PathBuf}; use proc_macro2::{Ident, Span, TokenStream, TokenTree}; +use pyo3_build_config::PythonVersion; use quote::quote; +const PY_3_12: PythonVersion = PythonVersion { + major: 3, + minor: 12, +}; + /// Macro which expands to multiple macro calls, one per pyo3-ffi struct. #[proc_macro] pub fn for_all_structs(input: proc_macro::TokenStream) -> proc_macro::TokenStream { @@ -130,16 +136,27 @@ pub fn for_all_fields(input: proc_macro::TokenStream) -> proc_macro::TokenStream let mut output = TokenStream::new(); for el in html.select(&selector) { - let id = el + let field_name = el .value() .id() .unwrap() .strip_prefix("structfield.") .unwrap(); - let field_ident = Ident::new(id, Span::call_site()); + let field_ident = Ident::new(field_name, Span::call_site()); - output.extend(quote!(#macro_name!(#struct_name, #field_ident);)); + let bindgen_field_ident = if (pyo3_build_config::get().version >= PY_3_12) + && struct_name == "PyObject" + && field_name == "ob_refcnt" + { + // PyObject since 3.12 implements ob_refcnt as a union; bindgen creates + // an anonymous name for the field + Ident::new("__bindgen_anon_1", Span::call_site()) + } else { + field_ident.clone() + }; + + output.extend(quote!(#macro_name!(#struct_name, #field_ident, #bindgen_field_ident);)); } output.into() diff --git a/pyo3-ffi-check/src/main.rs b/pyo3-ffi-check/src/main.rs index 91a7dca6..99713524 100644 --- a/pyo3-ffi-check/src/main.rs +++ b/pyo3-ffi-check/src/main.rs @@ -47,9 +47,11 @@ fn main() { } macro_rules! check_field { - ($struct_name:ident, $field:ident) => {{ + ($struct_name:ident, $field:ident, $bindgen_field:ident) => {{ + #[allow(clippy::used_underscore_binding)] let pyo3_ffi_offset = memoffset::offset_of!(pyo3_ffi::$struct_name, $field); - let bindgen_offset = memoffset::offset_of!(bindings::$struct_name, $field); + #[allow(clippy::used_underscore_binding)] + let bindgen_offset = memoffset::offset_of!(bindings::$struct_name, $bindgen_field); if pyo3_ffi_offset != bindgen_offset { failed = true; @@ -79,7 +81,9 @@ fn main() { non_upper_case_globals, dead_code, improper_ctypes, - clippy::all + clippy::all, + // clippy fails with lots of errors if this is not set specifically + clippy::used_underscore_binding )] mod bindings { include!(concat!(env!("OUT_DIR"), "/bindings.rs")); diff --git a/pyo3-ffi/src/cpython/code.rs b/pyo3-ffi/src/cpython/code.rs index c3a2fe08..4f667259 100644 --- a/pyo3-ffi/src/cpython/code.rs +++ b/pyo3-ffi/src/cpython/code.rs @@ -6,12 +6,30 @@ use std::os::raw::{c_char, c_int, c_short, c_uchar, c_void}; #[cfg(not(PyPy))] use std::ptr::addr_of_mut; +#[cfg(all(Py_3_8, not(PyPy), not(Py_3_11)))] +opaque_struct!(_PyOpcache); + +pub const _PY_MONITORING_UNGROUPED_EVENTS: usize = 14; +pub const _PY_MONITORING_EVENTS: usize = 16; + +#[cfg(Py_3_12)] +#[repr(C)] +#[derive(Copy, Clone)] +pub struct _Py_Monitors { + pub tools: [u8; _PY_MONITORING_UNGROUPED_EVENTS], +} + // skipped _Py_CODEUNIT + // skipped _Py_OPCODE // skipped _Py_OPARG -#[cfg(all(Py_3_8, not(PyPy), not(Py_3_11)))] -opaque_struct!(_PyOpcache); +// skipped _py_make_codeunit + +// skipped _py_set_opcode + +// skipped _Py_MAKE_CODEUNIT +// skipped _Py_SET_OPCODE #[cfg(Py_3_12)] #[repr(C)] @@ -23,6 +41,27 @@ pub struct _PyCoCached { pub _co_freevars: *mut PyObject, } +#[cfg(Py_3_12)] +#[repr(C)] +#[derive(Copy, Clone)] +pub struct _PyCoLineInstrumentationData { + pub original_opcode: u8, + pub line_delta: i8, +} + +#[cfg(Py_3_12)] +#[repr(C)] +#[derive(Copy, Clone)] +pub struct _PyCoMonitoringData { + pub local_monitors: _Py_Monitors, + pub active_monitors: _Py_Monitors, + pub tools: *mut u8, + pub lines: *mut _PyCoLineInstrumentationData, + pub line_tools: *mut u8, + pub per_instruction_opcodes: *mut u8, + pub per_instruction_tools: *mut u8, +} + #[cfg(all(not(PyPy), not(Py_3_7)))] opaque_struct!(PyCodeObject); @@ -97,8 +136,7 @@ pub struct PyCodeObject { pub co_flags: c_int, #[cfg(not(Py_3_12))] pub co_warmup: c_int, - #[cfg(Py_3_12)] - pub _co_linearray_entry_size: c_short, + pub co_argcount: c_int, pub co_posonlyargcount: c_int, pub co_kwonlyargcount: c_int, @@ -109,9 +147,12 @@ pub struct PyCodeObject { #[cfg(Py_3_12)] pub co_framesize: c_int, pub co_nlocals: c_int, + #[cfg(not(Py_3_12))] pub co_nplaincellvars: c_int, pub co_ncellvars: c_int, pub co_nfreevars: c_int, + #[cfg(Py_3_12)] + pub co_version: u32, pub co_localsplusnames: *mut PyObject, pub co_localspluskinds: *mut PyObject, @@ -122,13 +163,15 @@ pub struct PyCodeObject { pub co_weakreflist: *mut PyObject, #[cfg(not(Py_3_12))] pub _co_code: *mut PyObject, - #[cfg(Py_3_12)] - pub _co_cached: *mut _PyCoCached, #[cfg(not(Py_3_12))] pub _co_linearray: *mut c_char, - pub _co_firsttraceable: c_int, #[cfg(Py_3_12)] - pub _co_linearray: *mut c_char, + pub _co_cached: *mut _PyCoCached, + #[cfg(Py_3_12)] + pub _co_instrumentation_version: u64, + #[cfg(Py_3_12)] + pub _co_monitoring: *mut _PyCoMonitoringData, + pub _co_firsttraceable: c_int, pub co_extra: *mut c_void, pub co_code_adaptive: [c_char; 1], } diff --git a/pyo3-ffi/src/cpython/funcobject.rs b/pyo3-ffi/src/cpython/funcobject.rs index 9a01e4c1..1e9ee0cc 100644 --- a/pyo3-ffi/src/cpython/funcobject.rs +++ b/pyo3-ffi/src/cpython/funcobject.rs @@ -41,6 +41,8 @@ pub struct PyFunctionObject { pub func_weakreflist: *mut PyObject, pub func_module: *mut PyObject, pub func_annotations: *mut PyObject, + #[cfg(Py_3_12)] + pub func_typeparams: *mut PyObject, pub vectorcall: Option, #[cfg(Py_3_11)] pub func_version: u32, diff --git a/pyo3-ffi/src/cpython/genobject.rs b/pyo3-ffi/src/cpython/genobject.rs index bc0986a0..aaa03f82 100644 --- a/pyo3-ffi/src/cpython/genobject.rs +++ b/pyo3-ffi/src/cpython/genobject.rs @@ -16,6 +16,7 @@ pub struct PyGenObject { pub gi_frame: *mut PyFrameObject, #[cfg(not(Py_3_10))] pub gi_running: c_int, + #[cfg(not(Py_3_12))] pub gi_code: *mut PyObject, pub gi_weakreflist: *mut PyObject, pub gi_name: *mut PyObject, diff --git a/pyo3-ffi/src/cpython/object.rs b/pyo3-ffi/src/cpython/object.rs index 8209524f..c2519adc 100644 --- a/pyo3-ffi/src/cpython/object.rs +++ b/pyo3-ffi/src/cpython/object.rs @@ -299,6 +299,8 @@ pub struct PyTypeObject { #[derive(Clone)] pub struct _specialization_cache { pub getitem: *mut PyObject, + #[cfg(Py_3_12)] + pub getitem_version: u32, } #[repr(C)]