Set up CI for wasm32-emscripten target (#2436)

* ci: test on emscripten target

This adds CI to build libpython3.11 for wasm32-emscripten and
running tests against it. We need to patch instant to work
around the emscripten_get_now:
https://github.com/sebcrozet/instant/pull/47

We also have to patch emscripten to work aroung the "undefined
symbol gxx_personality_v0" error:
https://github.com/emscripten-core/emscripten/issues/17128

I set up a nox file to download and install emscripten,
download and build cpython, set appropriate environment variables
then run cargo test. The workflow just installs python, rust,
node, and nox and runs the nox session.

I xfailed all the test failures. There are problems with datetime.
iter_dict_nosegv and test_filenotfounderror should probably be
fixable. The tests that involve threads or asyncio probably can't
be fixed.

* Some cleanup

* Remove instant patch

* Add explanations for xfails
This commit is contained in:
Hood Chatham 2022-06-07 21:59:18 -07:00 committed by GitHub
parent 5603fa06b3
commit da5b9814cc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 246 additions and 0 deletions

View file

@ -339,3 +339,35 @@ jobs:
with:
file: coverage.lcov
name: ${{ matrix.os }}
emscripten:
name: emscripten
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
with:
python-version: 3.11.0-beta.1
- name: Install Rust toolchain
uses: actions-rs/toolchain@v1
with:
toolchain: stable
target: wasm32-unknown-emscripten
- uses: actions/setup-node@v3
with:
node-version: 14
- run: pip install nox
- uses: actions/cache@v3
id: cache
with:
path: |
.nox/emscripten
key: ${{ hashFiles('emscripten/*') }} - ${{ hashFiles('noxfile.py') }}
- uses: Swatinem/rust-cache@v1
with:
key: cargo-emscripten-wasm32
- name: Build
if: steps.cache.outputs.cache-hit != 'true'
run: nox -s build_emscripten
- name: Test
run: nox -s test_emscripten

88
emscripten/Makefile Normal file
View file

@ -0,0 +1,88 @@
CURDIR=$(abspath .)
# These three are passed in from nox.
BUILDROOT ?= $(CURDIR)/builddir
PYMAJORMINORMICRO ?= 3.11.0
PYPRERELEASE ?= b1 # I'm not sure how to split 3.11.0b1 in Make.
EMSCRIPTEN_VERSION=3.1.13
export EMSDKDIR = $(BUILDROOT)/emsdk
PLATFORM=wasm32_emscripten
SYSCONFIGDATA_NAME=_sysconfigdata__$(PLATFORM)
# BASH_ENV tells bash to source emsdk_env.sh on startup.
export BASH_ENV := $(CURDIR)/env.sh
# Use bash to run each command so that env.sh will be used.
SHELL := /bin/bash
# Set version variables.
version_tuple := $(subst ., ,$(PYMAJORMINORMICRO:v%=%))
PYMAJOR=$(word 1,$(version_tuple))
PYMINOR=$(word 2,$(version_tuple))
PYMICRO=$(word 3,$(version_tuple))
PYVERSION=$(PYMAJORMINORMICRO)$(PYPRERELEASE)
PYMAJORMINOR=$(PYMAJOR).$(PYMINOR)
PYTHONURL=https://www.python.org/ftp/python/$(PYMAJORMINORMICRO)/Python-$(PYVERSION).tgz
PYTHONTARBALL=$(BUILDROOT)/downloads/Python-$(PYVERSION).tgz
PYTHONBUILD=$(BUILDROOT)/build/Python-$(PYVERSION)
PYTHONLIBDIR=$(BUILDROOT)/install/Python-$(PYVERSION)/lib
all: $(PYTHONLIBDIR)/libpython$(PYMAJORMINOR).a
$(BUILDROOT)/.exists:
mkdir -p $(BUILDROOT)
touch $@
# Install emscripten
$(EMSDKDIR): $(CURDIR)/emscripten_patches/* $(BUILDROOT)/.exists
git clone https://github.com/emscripten-core/emsdk.git --depth 1 --branch $(EMSCRIPTEN_VERSION) $(EMSDKDIR)
$(EMSDKDIR)/emsdk install $(EMSCRIPTEN_VERSION)
cd $(EMSDKDIR)/upstream/emscripten && cat $(CURDIR)/emscripten_patches/* | patch -p1
$(EMSDKDIR)/emsdk activate $(EMSCRIPTEN_VERSION)
$(PYTHONTARBALL):
[ -d $(BUILDROOT)/downloads ] || mkdir -p $(BUILDROOT)/downloads
wget -q -O $@ $(PYTHONURL)
$(PYTHONBUILD)/.patched: $(PYTHONTARBALL)
[ -d $(PYTHONBUILD) ] || ( \
mkdir -p $(dir $(PYTHONBUILD));\
tar -C $(dir $(PYTHONBUILD)) -xf $(PYTHONTARBALL) \
)
touch $@
$(PYTHONBUILD)/Makefile: $(PYTHONBUILD)/.patched $(BUILDROOT)/emsdk
cd $(PYTHONBUILD) && \
CONFIG_SITE=Tools/wasm/config.site-wasm32-emscripten \
emconfigure ./configure -C \
--host=wasm32-unknown-emscripten \
--build=$(shell $(PYTHONBUILD)/config.guess) \
--with-emscripten-target=browser \
--enable-wasm-dynamic-linking \
--with-build-python=python3.11
$(PYTHONLIBDIR)/libpython$(PYMAJORMINOR).a : $(PYTHONBUILD)/Makefile
cd $(PYTHONBUILD) && \
emmake make -j3 libpython$(PYMAJORMINOR).a
# Generate sysconfigdata
_PYTHON_SYSCONFIGDATA_NAME=$(SYSCONFIGDATA_NAME) _PYTHON_PROJECT_BASE=$(PYTHONBUILD) python3.11 -m sysconfig --generate-posix-vars
cp `cat pybuilddir.txt`/$(SYSCONFIGDATA_NAME).py $(PYTHONBUILD)/Lib
mkdir -p $(PYTHONLIBDIR)
# Copy libexpat.a, libmpdec.a, and libpython3.11.a
# In noxfile, we explicitly link libexpat and libmpdec via RUSTFLAGS
find $(PYTHONBUILD) -name '*.a' -exec cp {} $(PYTHONLIBDIR) \;
# Install Python stdlib
cp -r $(PYTHONBUILD)/Lib $(PYTHONLIBDIR)/python$(PYMAJORMINOR)
clean:
rm -rf $(BUILDROOT)

View file

@ -0,0 +1,28 @@
From 4b56f37c3dc9185a235a8314086c4d7a6239b2f8 Mon Sep 17 00:00:00 2001
From: Hood Chatham <roberthoodchatham@gmail.com>
Date: Sat, 4 Jun 2022 19:19:47 -0700
Subject: [PATCH] Add _gxx_personality_v0 stub to library.js
Mitigation for an incompatibility between Rust and Emscripten:
https://github.com/rust-lang/rust/issues/85821
https://github.com/emscripten-core/emscripten/issues/17128
---
src/library.js | 2 ++
1 file changed, 2 insertions(+)
diff --git a/src/library.js b/src/library.js
index e7bb4c38e..7d01744df 100644
--- a/src/library.js
+++ b/src/library.js
@@ -403,6 +403,8 @@ mergeInto(LibraryManager.library, {
abort('Assertion failed: ' + UTF8ToString(condition) + ', at: ' + [filename ? UTF8ToString(filename) : 'unknown filename', line, func ? UTF8ToString(func) : 'unknown function']);
},
+ __gxx_personality_v0: function() {},
+
// ==========================================================================
// time.h
// ==========================================================================
--
2.25.1

6
emscripten/env.sh Normal file
View file

@ -0,0 +1,6 @@
#!/bin/bash
# Activate emsdk environment. emsdk_env.sh writes a lot to stderr so we suppress
# the output. This also prevents it from complaining when emscripten isn't yet
# installed.
source "$EMSDKDIR/emsdk_env.sh" 2> /dev/null || true

View file

@ -0,0 +1 @@
build/lib.linux-x86_64-3.11

8
emscripten/runner.py Executable file
View file

@ -0,0 +1,8 @@
#!/usr/local/bin/python
import pathlib
import sys
import subprocess
p = pathlib.Path(sys.argv[1])
sys.exit(subprocess.call(["node", p.name], cwd=p.parent))

View file

@ -1,5 +1,7 @@
import time
from glob import glob
from pathlib import Path
import re
import nox
@ -128,3 +130,66 @@ def contributors(session: nox.Session) -> None:
for author in authors:
print(f"@{author}")
class EmscriptenInfo:
def __init__(self):
rootdir = Path(__file__).parent
self.emscripten_dir = rootdir / "emscripten"
self.builddir = rootdir / ".nox/emscripten"
self.builddir.mkdir(exist_ok=True, parents=True)
self.pyversion = "3.11.0b1"
self.pymajor, self.pyminor, self.pymicro = self.pyversion.split(".")
self.pymicro, self.pydev = re.match(
"([0-9]*)([^0-9].*)?", self.pymicro
).groups()
if self.pydev is None:
self.pydev = ""
self.pymajorminor = f"{self.pymajor}.{self.pyminor}"
self.pymajorminormicro = f"{self.pymajorminor}.{self.pymicro}"
@nox.session(venv_backend="none")
def build_emscripten(session: nox.Session):
info = EmscriptenInfo()
session.run(
"make",
"-C",
str(info.emscripten_dir),
f"BUILDROOT={info.builddir}",
f"PYMAJORMINORMICRO={info.pymajorminormicro}",
f"PYPRERELEASE={info.pydev}",
external=True,
)
@nox.session(venv_backend="none")
def test_emscripten(session: nox.Session):
info = EmscriptenInfo()
libdir = info.builddir / f"install/Python-{info.pyversion}/lib"
pythonlibdir = libdir / f"python{info.pymajorminor}"
target = "wasm32-unknown-emscripten"
session.env["CARGO_TARGET_WASM32_UNKNOWN_EMSCRIPTEN_RUNNER"] = "python " + str(
info.emscripten_dir / "runner.py"
)
session.env["RUSTFLAGS"] = " ".join(
[
f"-L native={libdir}",
"-C link-arg=--preload-file",
f"-C link-arg={pythonlibdir}@/lib/python{info.pymajorminor}",
f"-C link-arg=-lpython{info.pymajorminor}",
"-C link-arg=-lexpat",
"-C link-arg=-lmpdec",
]
)
session.env["CARGO_BUILD_TARGET"] = target
session.env["PYO3_CROSS_LIB_DIR"] = pythonlibdir
session.run("rustup", "target", "add", target, "--toolchain", "stable")
session.run(
"bash", "-c", f"source {info.builddir/'emsdk/emsdk_env.sh'} && cargo test"
)

View file

@ -6,6 +6,7 @@ use crate::types::PyString;
#[cfg(target_endian = "little")]
use libc::wchar_t;
#[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons
#[test]
fn test_datetime_fromtimestamp() {
Python::with_gil(|py| {
@ -23,6 +24,7 @@ fn test_datetime_fromtimestamp() {
})
}
#[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons
#[test]
fn test_date_fromtimestamp() {
Python::with_gil(|py| {
@ -40,6 +42,7 @@ fn test_date_fromtimestamp() {
})
}
#[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons
#[test]
fn test_utc_timezone() {
Python::with_gil(|py| {
@ -183,6 +186,7 @@ fn ucs4() {
}
#[test]
#[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons
#[cfg(not(PyPy))]
fn test_get_tzinfo() {
crate::Python::with_gil(|py| {

View file

@ -729,6 +729,7 @@ mod tests {
}
#[test]
#[cfg(not(target_arch = "wasm32"))] // We are building wasm Python with pthreads disabled
fn test_clone_without_gil() {
use crate::{Py, PyAny};
use std::{sync::Arc, thread};
@ -799,6 +800,7 @@ mod tests {
}
#[test]
#[cfg(not(target_arch = "wasm32"))] // We are building wasm Python with pthreads disabled
fn test_clone_in_other_thread() {
use crate::Py;
use std::{sync::Arc, thread};

View file

@ -942,6 +942,7 @@ mod tests {
}
#[test]
#[cfg(not(target_arch = "wasm32"))] // We are building wasm Python with pthreads disabled
fn test_allow_threads_releases_and_acquires_gil() {
Python::with_gil(|py| {
let b = std::sync::Arc::new(std::sync::Barrier::new(2));

View file

@ -546,6 +546,7 @@ fn opt_to_pyobj(py: Python<'_>, opt: Option<&PyObject>) -> *mut ffi::PyObject {
#[cfg(test)]
mod tests {
#[test]
#[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons
fn test_new_with_fold() {
crate::Python::with_gil(|py| {
use crate::types::{PyDateTime, PyTimeAccess};
@ -560,6 +561,7 @@ mod tests {
#[cfg(not(PyPy))]
#[test]
#[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons
fn test_get_tzinfo() {
crate::Python::with_gil(|py| {
use crate::conversion::ToPyObject;

View file

@ -259,6 +259,7 @@ fn test_unsendable<T: PyClass + 'static>() -> PyResult<()> {
/// If a class is marked as `unsendable`, it panics when accessed by another thread.
#[test]
#[cfg_attr(target_arch = "wasm32", ignore)]
#[should_panic(
expected = "test_class_basics::UnsendableBase is unsendable, but sent to another thread!"
)]
@ -267,6 +268,7 @@ fn panic_unsendable_base() {
}
#[test]
#[cfg_attr(target_arch = "wasm32", ignore)]
#[should_panic(
expected = "test_class_basics::UnsendableBase is unsendable, but sent to another thread!"
)]

View file

@ -1,6 +1,7 @@
#![cfg(feature = "macros")]
#[rustversion::stable]
#[cfg(not(target_arch = "wasm32"))] // Not possible to invoke compiler from wasm
#[test]
fn test_compile_errors() {
// stable - require all tests to pass
@ -8,6 +9,7 @@ fn test_compile_errors() {
}
#[cfg(not(feature = "nightly"))]
#[cfg(not(target_arch = "wasm32"))] // We are building wasm Python with pthreads disabled
#[rustversion::nightly]
#[test]
fn test_compile_errors() {
@ -17,6 +19,7 @@ fn test_compile_errors() {
}
#[cfg(feature = "nightly")]
#[cfg(not(target_arch = "wasm32"))] // Not possible to invoke compiler from wasm
#[rustversion::nightly]
#[test]
fn test_compile_errors() {

View file

@ -2,6 +2,7 @@ use pyo3::prelude::*;
use pyo3::types::IntoPyDict;
#[test]
#[cfg_attr(target_arch = "wasm32", ignore)] // Not sure why this fails.
fn iter_dict_nosegv() {
let gil = Python::acquire_gil();
let py = gil.python();

View file

@ -17,6 +17,7 @@ fn fail_to_open_file() -> PyResult<()> {
}
#[test]
#[cfg_attr(target_arch = "wasm32", ignore)] // Not sure why this fails.
#[cfg(not(target_os = "windows"))]
fn test_filenotfounderror() {
let gil = Python::acquire_gil();

View file

@ -698,6 +698,7 @@ impl OnceFuture {
}
#[test]
#[cfg(not(target_arch = "wasm32"))] // Won't work without wasm32 event loop (e.g., Pyodide has WebLoop)
fn test_await() {
let gil = Python::acquire_gil();
let py = gil.python();
@ -747,6 +748,7 @@ impl AsyncIterator {
}
#[test]
#[cfg(not(target_arch = "wasm32"))] // Won't work without wasm32 event loop (e.g., Pyodide has WebLoop)
fn test_anext_aiter() {
let gil = Python::acquire_gil();
let py = gil.python();