diff --git a/CHANGELOG.md b/CHANGELOG.md
index 73f76289..c232fae3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Update `num-complex` optional dependency to 0.4. [#1482](https://github.com/PyO3/pyo3/pull/1482)
 - Extend `hashbrown` optional dependency supported versions to include 0.11. [#1496](https://github.com/PyO3/pyo3/pull/1496)
 - Support PyPy 3.7. [#1538](https://github.com/PyO3/pyo3/pull/1538)
+- Try harder to filter sysconfigdata candidates on arch.config
 
 ### Added
 - Add conversions for `[T; N]` for all `N` on Rust 1.51 and up. [#1128](https://github.com/PyO3/pyo3/pull/1128)
diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs
index b80d2d9f..9146a55f 100644
--- a/pyo3-build-config/src/impl_.rs
+++ b/pyo3-build-config/src/impl_.rs
@@ -484,6 +484,24 @@ fn search_lib_dir(path: impl AsRef<Path>, cross: &CrossCompileConfig) -> Vec<Pat
         };
         sysconfig_paths.extend(sysc);
     }
+    // If we got more than one file, only take those that contain the arch name.
+    // For ubuntu 20.04 with host architecture x86_64 and a foreign architecture of armhf
+    // this reduces the number of candidates to 1:
+    //
+    // $ find /usr/lib/python3.8/ -name '_sysconfigdata*.py' -not -lname '*'
+    //  /usr/lib/python3.8/_sysconfigdata__x86_64-linux-gnu.py
+    //  /usr/lib/python3.8/_sysconfigdata__arm-linux-gnueabihf.py
+    if sysconfig_paths.len() > 1 {
+        let temp = sysconfig_paths
+            .iter()
+            .filter(|p| p.to_string_lossy().contains(&cross.arch))
+            .cloned()
+            .collect::<Vec<PathBuf>>();
+        if !temp.is_empty() {
+            sysconfig_paths = temp;
+        }
+    }
+
     sysconfig_paths
 }