Set up CI for wasm32-emscripten target (#2436)
* ci: test on emscripten target This adds CI to build libpython3.11 for wasm32-emscripten and running tests against it. We need to patch instant to work around the emscripten_get_now: https://github.com/sebcrozet/instant/pull/47 We also have to patch emscripten to work aroung the "undefined symbol gxx_personality_v0" error: https://github.com/emscripten-core/emscripten/issues/17128 I set up a nox file to download and install emscripten, download and build cpython, set appropriate environment variables then run cargo test. The workflow just installs python, rust, node, and nox and runs the nox session. I xfailed all the test failures. There are problems with datetime. iter_dict_nosegv and test_filenotfounderror should probably be fixable. The tests that involve threads or asyncio probably can't be fixed. * Some cleanup * Remove instant patch * Add explanations for xfails
This commit is contained in:
parent
5603fa06b3
commit
da5b9814cc
32
.github/workflows/ci.yml
vendored
32
.github/workflows/ci.yml
vendored
|
@ -339,3 +339,35 @@ jobs:
|
|||
with:
|
||||
file: coverage.lcov
|
||||
name: ${{ matrix.os }}
|
||||
|
||||
emscripten:
|
||||
name: emscripten
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.11.0-beta.1
|
||||
- name: Install Rust toolchain
|
||||
uses: actions-rs/toolchain@v1
|
||||
with:
|
||||
toolchain: stable
|
||||
target: wasm32-unknown-emscripten
|
||||
- uses: actions/setup-node@v3
|
||||
with:
|
||||
node-version: 14
|
||||
- run: pip install nox
|
||||
- uses: actions/cache@v3
|
||||
id: cache
|
||||
with:
|
||||
path: |
|
||||
.nox/emscripten
|
||||
key: ${{ hashFiles('emscripten/*') }} - ${{ hashFiles('noxfile.py') }}
|
||||
- uses: Swatinem/rust-cache@v1
|
||||
with:
|
||||
key: cargo-emscripten-wasm32
|
||||
- name: Build
|
||||
if: steps.cache.outputs.cache-hit != 'true'
|
||||
run: nox -s build_emscripten
|
||||
- name: Test
|
||||
run: nox -s test_emscripten
|
||||
|
|
88
emscripten/Makefile
Normal file
88
emscripten/Makefile
Normal file
|
@ -0,0 +1,88 @@
|
|||
CURDIR=$(abspath .)
|
||||
|
||||
# These three are passed in from nox.
|
||||
BUILDROOT ?= $(CURDIR)/builddir
|
||||
PYMAJORMINORMICRO ?= 3.11.0
|
||||
PYPRERELEASE ?= b1 # I'm not sure how to split 3.11.0b1 in Make.
|
||||
|
||||
EMSCRIPTEN_VERSION=3.1.13
|
||||
|
||||
export EMSDKDIR = $(BUILDROOT)/emsdk
|
||||
|
||||
PLATFORM=wasm32_emscripten
|
||||
SYSCONFIGDATA_NAME=_sysconfigdata__$(PLATFORM)
|
||||
|
||||
# BASH_ENV tells bash to source emsdk_env.sh on startup.
|
||||
export BASH_ENV := $(CURDIR)/env.sh
|
||||
# Use bash to run each command so that env.sh will be used.
|
||||
SHELL := /bin/bash
|
||||
|
||||
|
||||
# Set version variables.
|
||||
version_tuple := $(subst ., ,$(PYMAJORMINORMICRO:v%=%))
|
||||
PYMAJOR=$(word 1,$(version_tuple))
|
||||
PYMINOR=$(word 2,$(version_tuple))
|
||||
PYMICRO=$(word 3,$(version_tuple))
|
||||
PYVERSION=$(PYMAJORMINORMICRO)$(PYPRERELEASE)
|
||||
PYMAJORMINOR=$(PYMAJOR).$(PYMINOR)
|
||||
|
||||
|
||||
PYTHONURL=https://www.python.org/ftp/python/$(PYMAJORMINORMICRO)/Python-$(PYVERSION).tgz
|
||||
PYTHONTARBALL=$(BUILDROOT)/downloads/Python-$(PYVERSION).tgz
|
||||
PYTHONBUILD=$(BUILDROOT)/build/Python-$(PYVERSION)
|
||||
|
||||
PYTHONLIBDIR=$(BUILDROOT)/install/Python-$(PYVERSION)/lib
|
||||
|
||||
all: $(PYTHONLIBDIR)/libpython$(PYMAJORMINOR).a
|
||||
|
||||
$(BUILDROOT)/.exists:
|
||||
mkdir -p $(BUILDROOT)
|
||||
touch $@
|
||||
|
||||
|
||||
# Install emscripten
|
||||
$(EMSDKDIR): $(CURDIR)/emscripten_patches/* $(BUILDROOT)/.exists
|
||||
git clone https://github.com/emscripten-core/emsdk.git --depth 1 --branch $(EMSCRIPTEN_VERSION) $(EMSDKDIR)
|
||||
$(EMSDKDIR)/emsdk install $(EMSCRIPTEN_VERSION)
|
||||
cd $(EMSDKDIR)/upstream/emscripten && cat $(CURDIR)/emscripten_patches/* | patch -p1
|
||||
$(EMSDKDIR)/emsdk activate $(EMSCRIPTEN_VERSION)
|
||||
|
||||
|
||||
$(PYTHONTARBALL):
|
||||
[ -d $(BUILDROOT)/downloads ] || mkdir -p $(BUILDROOT)/downloads
|
||||
wget -q -O $@ $(PYTHONURL)
|
||||
|
||||
$(PYTHONBUILD)/.patched: $(PYTHONTARBALL)
|
||||
[ -d $(PYTHONBUILD) ] || ( \
|
||||
mkdir -p $(dir $(PYTHONBUILD));\
|
||||
tar -C $(dir $(PYTHONBUILD)) -xf $(PYTHONTARBALL) \
|
||||
)
|
||||
touch $@
|
||||
|
||||
$(PYTHONBUILD)/Makefile: $(PYTHONBUILD)/.patched $(BUILDROOT)/emsdk
|
||||
cd $(PYTHONBUILD) && \
|
||||
CONFIG_SITE=Tools/wasm/config.site-wasm32-emscripten \
|
||||
emconfigure ./configure -C \
|
||||
--host=wasm32-unknown-emscripten \
|
||||
--build=$(shell $(PYTHONBUILD)/config.guess) \
|
||||
--with-emscripten-target=browser \
|
||||
--enable-wasm-dynamic-linking \
|
||||
--with-build-python=python3.11
|
||||
|
||||
$(PYTHONLIBDIR)/libpython$(PYMAJORMINOR).a : $(PYTHONBUILD)/Makefile
|
||||
cd $(PYTHONBUILD) && \
|
||||
emmake make -j3 libpython$(PYMAJORMINOR).a
|
||||
|
||||
# Generate sysconfigdata
|
||||
_PYTHON_SYSCONFIGDATA_NAME=$(SYSCONFIGDATA_NAME) _PYTHON_PROJECT_BASE=$(PYTHONBUILD) python3.11 -m sysconfig --generate-posix-vars
|
||||
cp `cat pybuilddir.txt`/$(SYSCONFIGDATA_NAME).py $(PYTHONBUILD)/Lib
|
||||
|
||||
mkdir -p $(PYTHONLIBDIR)
|
||||
# Copy libexpat.a, libmpdec.a, and libpython3.11.a
|
||||
# In noxfile, we explicitly link libexpat and libmpdec via RUSTFLAGS
|
||||
find $(PYTHONBUILD) -name '*.a' -exec cp {} $(PYTHONLIBDIR) \;
|
||||
# Install Python stdlib
|
||||
cp -r $(PYTHONBUILD)/Lib $(PYTHONLIBDIR)/python$(PYMAJORMINOR)
|
||||
|
||||
clean:
|
||||
rm -rf $(BUILDROOT)
|
|
@ -0,0 +1,28 @@
|
|||
From 4b56f37c3dc9185a235a8314086c4d7a6239b2f8 Mon Sep 17 00:00:00 2001
|
||||
From: Hood Chatham <roberthoodchatham@gmail.com>
|
||||
Date: Sat, 4 Jun 2022 19:19:47 -0700
|
||||
Subject: [PATCH] Add _gxx_personality_v0 stub to library.js
|
||||
|
||||
Mitigation for an incompatibility between Rust and Emscripten:
|
||||
https://github.com/rust-lang/rust/issues/85821
|
||||
https://github.com/emscripten-core/emscripten/issues/17128
|
||||
---
|
||||
src/library.js | 2 ++
|
||||
1 file changed, 2 insertions(+)
|
||||
|
||||
diff --git a/src/library.js b/src/library.js
|
||||
index e7bb4c38e..7d01744df 100644
|
||||
--- a/src/library.js
|
||||
+++ b/src/library.js
|
||||
@@ -403,6 +403,8 @@ mergeInto(LibraryManager.library, {
|
||||
abort('Assertion failed: ' + UTF8ToString(condition) + ', at: ' + [filename ? UTF8ToString(filename) : 'unknown filename', line, func ? UTF8ToString(func) : 'unknown function']);
|
||||
},
|
||||
|
||||
+ __gxx_personality_v0: function() {},
|
||||
+
|
||||
// ==========================================================================
|
||||
// time.h
|
||||
// ==========================================================================
|
||||
--
|
||||
2.25.1
|
||||
|
6
emscripten/env.sh
Normal file
6
emscripten/env.sh
Normal file
|
@ -0,0 +1,6 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Activate emsdk environment. emsdk_env.sh writes a lot to stderr so we suppress
|
||||
# the output. This also prevents it from complaining when emscripten isn't yet
|
||||
# installed.
|
||||
source "$EMSDKDIR/emsdk_env.sh" 2> /dev/null || true
|
1
emscripten/pybuilddir.txt
Normal file
1
emscripten/pybuilddir.txt
Normal file
|
@ -0,0 +1 @@
|
|||
build/lib.linux-x86_64-3.11
|
8
emscripten/runner.py
Executable file
8
emscripten/runner.py
Executable file
|
@ -0,0 +1,8 @@
|
|||
#!/usr/local/bin/python
|
||||
import pathlib
|
||||
import sys
|
||||
import subprocess
|
||||
|
||||
p = pathlib.Path(sys.argv[1])
|
||||
|
||||
sys.exit(subprocess.call(["node", p.name], cwd=p.parent))
|
65
noxfile.py
65
noxfile.py
|
@ -1,5 +1,7 @@
|
|||
import time
|
||||
from glob import glob
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
import nox
|
||||
|
||||
|
@ -128,3 +130,66 @@ def contributors(session: nox.Session) -> None:
|
|||
|
||||
for author in authors:
|
||||
print(f"@{author}")
|
||||
|
||||
|
||||
class EmscriptenInfo:
|
||||
def __init__(self):
|
||||
rootdir = Path(__file__).parent
|
||||
self.emscripten_dir = rootdir / "emscripten"
|
||||
self.builddir = rootdir / ".nox/emscripten"
|
||||
self.builddir.mkdir(exist_ok=True, parents=True)
|
||||
|
||||
self.pyversion = "3.11.0b1"
|
||||
self.pymajor, self.pyminor, self.pymicro = self.pyversion.split(".")
|
||||
self.pymicro, self.pydev = re.match(
|
||||
"([0-9]*)([^0-9].*)?", self.pymicro
|
||||
).groups()
|
||||
if self.pydev is None:
|
||||
self.pydev = ""
|
||||
|
||||
self.pymajorminor = f"{self.pymajor}.{self.pyminor}"
|
||||
self.pymajorminormicro = f"{self.pymajorminor}.{self.pymicro}"
|
||||
|
||||
|
||||
@nox.session(venv_backend="none")
|
||||
def build_emscripten(session: nox.Session):
|
||||
info = EmscriptenInfo()
|
||||
session.run(
|
||||
"make",
|
||||
"-C",
|
||||
str(info.emscripten_dir),
|
||||
f"BUILDROOT={info.builddir}",
|
||||
f"PYMAJORMINORMICRO={info.pymajorminormicro}",
|
||||
f"PYPRERELEASE={info.pydev}",
|
||||
external=True,
|
||||
)
|
||||
|
||||
|
||||
@nox.session(venv_backend="none")
|
||||
def test_emscripten(session: nox.Session):
|
||||
info = EmscriptenInfo()
|
||||
|
||||
libdir = info.builddir / f"install/Python-{info.pyversion}/lib"
|
||||
pythonlibdir = libdir / f"python{info.pymajorminor}"
|
||||
|
||||
target = "wasm32-unknown-emscripten"
|
||||
|
||||
session.env["CARGO_TARGET_WASM32_UNKNOWN_EMSCRIPTEN_RUNNER"] = "python " + str(
|
||||
info.emscripten_dir / "runner.py"
|
||||
)
|
||||
session.env["RUSTFLAGS"] = " ".join(
|
||||
[
|
||||
f"-L native={libdir}",
|
||||
"-C link-arg=--preload-file",
|
||||
f"-C link-arg={pythonlibdir}@/lib/python{info.pymajorminor}",
|
||||
f"-C link-arg=-lpython{info.pymajorminor}",
|
||||
"-C link-arg=-lexpat",
|
||||
"-C link-arg=-lmpdec",
|
||||
]
|
||||
)
|
||||
session.env["CARGO_BUILD_TARGET"] = target
|
||||
session.env["PYO3_CROSS_LIB_DIR"] = pythonlibdir
|
||||
session.run("rustup", "target", "add", target, "--toolchain", "stable")
|
||||
session.run(
|
||||
"bash", "-c", f"source {info.builddir/'emsdk/emsdk_env.sh'} && cargo test"
|
||||
)
|
||||
|
|
|
@ -6,6 +6,7 @@ use crate::types::PyString;
|
|||
#[cfg(target_endian = "little")]
|
||||
use libc::wchar_t;
|
||||
|
||||
#[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons
|
||||
#[test]
|
||||
fn test_datetime_fromtimestamp() {
|
||||
Python::with_gil(|py| {
|
||||
|
@ -23,6 +24,7 @@ fn test_datetime_fromtimestamp() {
|
|||
})
|
||||
}
|
||||
|
||||
#[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons
|
||||
#[test]
|
||||
fn test_date_fromtimestamp() {
|
||||
Python::with_gil(|py| {
|
||||
|
@ -40,6 +42,7 @@ fn test_date_fromtimestamp() {
|
|||
})
|
||||
}
|
||||
|
||||
#[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons
|
||||
#[test]
|
||||
fn test_utc_timezone() {
|
||||
Python::with_gil(|py| {
|
||||
|
@ -183,6 +186,7 @@ fn ucs4() {
|
|||
}
|
||||
|
||||
#[test]
|
||||
#[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons
|
||||
#[cfg(not(PyPy))]
|
||||
fn test_get_tzinfo() {
|
||||
crate::Python::with_gil(|py| {
|
||||
|
|
|
@ -729,6 +729,7 @@ mod tests {
|
|||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(not(target_arch = "wasm32"))] // We are building wasm Python with pthreads disabled
|
||||
fn test_clone_without_gil() {
|
||||
use crate::{Py, PyAny};
|
||||
use std::{sync::Arc, thread};
|
||||
|
@ -799,6 +800,7 @@ mod tests {
|
|||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(not(target_arch = "wasm32"))] // We are building wasm Python with pthreads disabled
|
||||
fn test_clone_in_other_thread() {
|
||||
use crate::Py;
|
||||
use std::{sync::Arc, thread};
|
||||
|
|
|
@ -942,6 +942,7 @@ mod tests {
|
|||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(not(target_arch = "wasm32"))] // We are building wasm Python with pthreads disabled
|
||||
fn test_allow_threads_releases_and_acquires_gil() {
|
||||
Python::with_gil(|py| {
|
||||
let b = std::sync::Arc::new(std::sync::Barrier::new(2));
|
||||
|
|
|
@ -546,6 +546,7 @@ fn opt_to_pyobj(py: Python<'_>, opt: Option<&PyObject>) -> *mut ffi::PyObject {
|
|||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
#[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons
|
||||
fn test_new_with_fold() {
|
||||
crate::Python::with_gil(|py| {
|
||||
use crate::types::{PyDateTime, PyTimeAccess};
|
||||
|
@ -560,6 +561,7 @@ mod tests {
|
|||
|
||||
#[cfg(not(PyPy))]
|
||||
#[test]
|
||||
#[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons
|
||||
fn test_get_tzinfo() {
|
||||
crate::Python::with_gil(|py| {
|
||||
use crate::conversion::ToPyObject;
|
||||
|
|
|
@ -259,6 +259,7 @@ fn test_unsendable<T: PyClass + 'static>() -> PyResult<()> {
|
|||
|
||||
/// If a class is marked as `unsendable`, it panics when accessed by another thread.
|
||||
#[test]
|
||||
#[cfg_attr(target_arch = "wasm32", ignore)]
|
||||
#[should_panic(
|
||||
expected = "test_class_basics::UnsendableBase is unsendable, but sent to another thread!"
|
||||
)]
|
||||
|
@ -267,6 +268,7 @@ fn panic_unsendable_base() {
|
|||
}
|
||||
|
||||
#[test]
|
||||
#[cfg_attr(target_arch = "wasm32", ignore)]
|
||||
#[should_panic(
|
||||
expected = "test_class_basics::UnsendableBase is unsendable, but sent to another thread!"
|
||||
)]
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
#![cfg(feature = "macros")]
|
||||
|
||||
#[rustversion::stable]
|
||||
#[cfg(not(target_arch = "wasm32"))] // Not possible to invoke compiler from wasm
|
||||
#[test]
|
||||
fn test_compile_errors() {
|
||||
// stable - require all tests to pass
|
||||
|
@ -8,6 +9,7 @@ fn test_compile_errors() {
|
|||
}
|
||||
|
||||
#[cfg(not(feature = "nightly"))]
|
||||
#[cfg(not(target_arch = "wasm32"))] // We are building wasm Python with pthreads disabled
|
||||
#[rustversion::nightly]
|
||||
#[test]
|
||||
fn test_compile_errors() {
|
||||
|
@ -17,6 +19,7 @@ fn test_compile_errors() {
|
|||
}
|
||||
|
||||
#[cfg(feature = "nightly")]
|
||||
#[cfg(not(target_arch = "wasm32"))] // Not possible to invoke compiler from wasm
|
||||
#[rustversion::nightly]
|
||||
#[test]
|
||||
fn test_compile_errors() {
|
||||
|
|
|
@ -2,6 +2,7 @@ use pyo3::prelude::*;
|
|||
use pyo3::types::IntoPyDict;
|
||||
|
||||
#[test]
|
||||
#[cfg_attr(target_arch = "wasm32", ignore)] // Not sure why this fails.
|
||||
fn iter_dict_nosegv() {
|
||||
let gil = Python::acquire_gil();
|
||||
let py = gil.python();
|
||||
|
|
|
@ -17,6 +17,7 @@ fn fail_to_open_file() -> PyResult<()> {
|
|||
}
|
||||
|
||||
#[test]
|
||||
#[cfg_attr(target_arch = "wasm32", ignore)] // Not sure why this fails.
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
fn test_filenotfounderror() {
|
||||
let gil = Python::acquire_gil();
|
||||
|
|
|
@ -698,6 +698,7 @@ impl OnceFuture {
|
|||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(not(target_arch = "wasm32"))] // Won't work without wasm32 event loop (e.g., Pyodide has WebLoop)
|
||||
fn test_await() {
|
||||
let gil = Python::acquire_gil();
|
||||
let py = gil.python();
|
||||
|
@ -747,6 +748,7 @@ impl AsyncIterator {
|
|||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(not(target_arch = "wasm32"))] // Won't work without wasm32 event loop (e.g., Pyodide has WebLoop)
|
||||
fn test_anext_aiter() {
|
||||
let gil = Python::acquire_gil();
|
||||
let py = gil.python();
|
||||
|
|
Loading…
Reference in a new issue