From b1a37afa8cc8b46a9749b8faa133385e517c736f Mon Sep 17 00:00:00 2001
From: David Hewitt <1939362+davidhewitt@users.noreply.github.com>
Date: Thu, 9 Jul 2020 09:29:10 +0100
Subject: [PATCH] Fix definition of _PyLong_AsByteArray

---
 CHANGELOG.md          |  3 ++-
 src/ffi/longobject.rs |  2 +-
 src/types/num.rs      | 58 ++++++++++++++++---------------------------
 3 files changed, 24 insertions(+), 39 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d893c292..4cd89a94 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,8 +9,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Add FFI definitions `Py_FinalizeEx`, `PyOS_getsig`, `PyOS_setsig`. [#1021](https://github.com/PyO3/pyo3/pull/1021)
 
 ### Changed
-- Change FFI definitions `Py_SetProgramName` and `Py_SetPythonHome` to take `*const` argument instead of `*mut`. [#1021](https://github.com/PyO3/pyo3/pull/1021)
+- Correct FFI definitions `Py_SetProgramName` and `Py_SetPythonHome` to take `*const` argument instead of `*mut`. [#1021](https://github.com/PyO3/pyo3/pull/1021)
 - Rename `PyString::to_string` to `to_str`, change return type `Cow<str>` to `&str`. [#1023](https://github.com/PyO3/pyo3/pull/1023)
+- Correct FFI definition `_PyLong_AsByteArray` `*mut c_uchar` argument instead of `*const c_uchar`. [#1029](https://github.com/PyO3/pyo3/pull/1029)
 
 ### Removed
 - Remove `PyString::as_bytes`. [#1023](https://github.com/PyO3/pyo3/pull/1023)
diff --git a/src/ffi/longobject.rs b/src/ffi/longobject.rs
index ba260b6b..0e87842a 100644
--- a/src/ffi/longobject.rs
+++ b/src/ffi/longobject.rs
@@ -95,7 +95,7 @@ extern "C" {
 
     pub fn _PyLong_AsByteArray(
         v: *mut PyLongObject,
-        bytes: *const c_uchar,
+        bytes: *mut c_uchar,
         n: size_t,
         little_endian: c_int,
         is_signed: c_int,
diff --git a/src/types/num.rs b/src/types/num.rs
index cf65e110..254953c7 100644
--- a/src/types/num.rs
+++ b/src/types/num.rs
@@ -81,10 +81,10 @@ macro_rules! int_convert_128 {
                     if num.is_null() {
                         return Err(PyErr::fetch(ob.py()));
                     }
-                    let buffer: [c_uchar; $byte_size] = [0; $byte_size];
+                    let mut buffer = [0; $byte_size];
                     let ok = ffi::_PyLong_AsByteArray(
                         num as *mut ffi::PyLongObject,
-                        buffer.as_ptr() as *const c_uchar,
+                        buffer.as_mut_ptr(),
                         $byte_size,
                         IS_LITTLE_ENDIAN,
                         $is_signed,
@@ -222,38 +222,20 @@ int_convert_128!(u128, 16, 0);
 #[cfg(all(feature = "num-bigint", not(Py_LIMITED_API)))]
 mod bigint_conversion {
     use super::*;
+    use crate::{err, Py};
     use num_bigint::{BigInt, BigUint};
 
-    unsafe fn extract_small(ob: &PyAny, n: usize, is_signed: c_int) -> PyResult<[c_uchar; 128]> {
-        let buffer = [0; 128];
-        let ok = ffi::_PyLong_AsByteArray(
-            ob.as_ptr() as *mut ffi::PyLongObject,
-            buffer.as_ptr() as *const c_uchar,
-            n,
-            1,
-            is_signed,
-        );
-        if ok == -1 {
-            Err(PyErr::fetch(ob.py()))
-        } else {
-            Ok(buffer)
-        }
-    }
-
-    unsafe fn extract_large(ob: &PyAny, n: usize, is_signed: c_int) -> PyResult<Vec<c_uchar>> {
-        let buffer = vec![0; n];
-        let ok = ffi::_PyLong_AsByteArray(
-            ob.as_ptr() as *mut ffi::PyLongObject,
-            buffer.as_ptr() as *const c_uchar,
-            n,
-            1,
-            is_signed,
-        );
-        if ok == -1 {
-            Err(PyErr::fetch(ob.py()))
-        } else {
-            Ok(buffer)
-        }
+    unsafe fn extract(ob: &PyLong, buffer: &mut [c_uchar], is_signed: c_int) -> PyResult<()> {
+        err::error_on_minusone(
+            ob.py(),
+            ffi::_PyLong_AsByteArray(
+                ob.as_ptr() as *mut ffi::PyLongObject,
+                buffer.as_mut_ptr(),
+                buffer.len(),
+                1,
+                is_signed,
+            ),
+        )
     }
 
     macro_rules! bigint_conversion {
@@ -294,13 +276,15 @@ mod bigint_conversion {
                         } else {
                             (n_bits as usize - 1 + $is_signed) / 8 + 1
                         };
-                        let ob = PyObject::from_owned_ptr(py, num);
+                        let num: Py<PyLong> = Py::from_owned_ptr(py, num);
                         if n_bytes <= 128 {
-                            extract_small(ob.as_ref(py), n_bytes, $is_signed)
-                                .map(|b| $from_bytes(&b[..n_bytes]))
+                            let mut buffer = [0; 128];
+                            extract(num.as_ref(py), &mut buffer[..n_bytes], $is_signed)?;
+                            Ok($from_bytes(&buffer[..n_bytes]))
                         } else {
-                            extract_large(ob.as_ref(py), n_bytes, $is_signed)
-                                .map(|b| $from_bytes(&b))
+                            let mut buffer = vec![0; n_bytes];
+                            extract(num.as_ref(py), &mut buffer, $is_signed)?;
+                            Ok($from_bytes(&buffer))
                         }
                     }
                 }