Support WebAssembly target platforms `wasm{32,64}-unknown-unknown` (#405)

This commit is contained in:
John Millikin 2024-10-18 18:04:41 +09:00 committed by GitHub
parent a80f2b4219
commit bda1c9fbf2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
20 changed files with 325 additions and 17 deletions

View File

@ -45,3 +45,19 @@ platform(
"@platforms//cpu:aarch64",
],
)
platform(
name = "wasm32",
constraint_values = [
"@platforms//os:none",
"@platforms//cpu:wasm32",
],
)
platform(
name = "wasm64",
constraint_values = [
"@platforms//os:none",
"@platforms//cpu:wasm64",
],
)

View File

@ -179,3 +179,40 @@ llvm.toolchain(
exec_arch = "amd64",
)
use_repo(llvm, "llvm_toolchain_linux_exec")
# Toolchain example for WebAssembly.
llvm.toolchain(
name = "llvm_toolchain_wasm",
# WebAssembly tests use a separate (newer) version of LLVM to exercise
# support for experimental features such as wasm64.
llvm_versions = {
# The most recent LLVM as of 2024-10-17
"": "19.1.0",
},
stdlib = {
"wasm32": "libc",
"wasm64": "none",
},
libclang_rt = {
"@libclang_rt_wasm32//:libclang_rt.builtins-wasm32.a": "wasm32-unknown-unknown/libclang_rt.builtins.a",
},
)
llvm.sysroot(
name = "llvm_toolchain_wasm",
label = "@wasi_sdk_sysroots//wasm32-wasip2",
targets = ["wasm32"],
)
llvm.sysroot(
name = "llvm_toolchain_wasm",
label = "@wasi_sdk_sysroots//empty",
targets = ["wasm64"],
)
use_repo(llvm, "llvm_toolchain_wasm")
register_toolchains("@llvm_toolchain_wasm//:all")
wasi_sdk_sysroots = use_repo_rule("//wasm:wasi_sdk.bzl", "wasi_sdk_sysroots")
wasi_sdk_sysroots(name = "wasi_sdk_sysroots")
libclang_rt_wasm32 = use_repo_rule("//wasm:wasi_sdk.bzl", "libclang_rt_wasm32")
libclang_rt_wasm32(name = "libclang_rt_wasm32")

View File

@ -156,6 +156,41 @@ llvm_toolchain(
urls = {"": ["https://github.com/llvm/llvm-project/releases/download/llvmorg-17.0.6/clang+llvm-17.0.6-x86_64-linux-gnu-ubuntu-22.04.tar.xz"]},
)
# Toolchain example for WebAssembly.
llvm_toolchain(
name = "llvm_toolchain_wasm",
libclang_rt = {
"@libclang_rt_wasm32//:libclang_rt.builtins-wasm32.a": "wasm32-unknown-unknown/libclang_rt.builtins.a",
},
# WebAssembly tests use a separate (newer) version of LLVM to exercise
# support for experimental features such as wasm64.
llvm_versions = {
# The most recent LLVM as of 2024-10-17
"": "19.1.0",
},
stdlib = {
"wasm32": "libc",
"wasm64": "none",
},
sysroot = {
"wasm32": "@wasi_sdk_sysroots//wasm32-wasip2",
"wasm64": "@wasi_sdk_sysroots//empty",
},
)
load(
"@llvm_toolchain_wasm//:toolchains.bzl",
llvm_register_toolchains_wasm = "llvm_register_toolchains",
)
llvm_register_toolchains_wasm()
load("//wasm:wasi_sdk.bzl", "libclang_rt_wasm32", "wasi_sdk_sysroots")
libclang_rt_wasm32(name = "libclang_rt_wasm32")
wasi_sdk_sysroots(name = "wasi_sdk_sysroots")
## Test dependencies.
# Well known repos; present here only for testing.

View File

@ -30,7 +30,7 @@ set -exuo pipefail
# Common setup
export DEBIAN_FRONTEND=noninteractive
apt-get -qq update
apt-get -qq -y install curl libtinfo5 zlib1g-dev >/dev/null
apt-get -qq -y install curl libtinfo5 libxml2 zlib1g-dev >/dev/null
# The above command gives some verbose output that can not be silenced easily.
# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=288778

View File

@ -30,7 +30,7 @@ set -exuo pipefail
# Common setup
export DEBIAN_FRONTEND=noninteractive
apt-get -qq update
apt-get -qq -y install curl libtinfo5 zlib1g-dev >/dev/null
apt-get -qq -y install curl libtinfo5 libxml2 zlib1g-dev >/dev/null
# The above command gives some verbose output that can not be silenced easily.
# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=288778

View File

@ -40,7 +40,7 @@ docker build --platform=linux/amd64 --pull --tag=bazel-docker-sandbox - <<-EOF
FROM ${base_image}
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get -qq update && \
apt-get -qq -y install libtinfo5 zlib1g-dev libxml2
apt-get -qq -y install libtinfo5 libxml2 zlib1g-dev libxml2
EOF
build_args=(

View File

@ -16,8 +16,9 @@
set -euo pipefail
toolchain_name=""
disable_wasm_tests=""
while getopts "t:h" opt; do
while getopts "t:hW" opt; do
case "${opt}" in
"t") toolchain_name="${OPTARG}" ;;
"h")
@ -25,6 +26,9 @@ while getopts "t:h" opt; do
echo "-t - Toolchain name to use for testing; default is llvm_toolchain"
exit 2
;;
"W")
disable_wasm_tests="yes"
;;
*)
echo "invalid option: -${OPTARG}"
exit 1
@ -62,3 +66,24 @@ fi
# Note that the following flags are currently known to cause issues in migration tests:
# --incompatible_disallow_struct_provider_syntax # https://github.com/bazelbuild/bazel/issues/7347
# --incompatible_no_rule_outputs_param # from rules_rust
# WebAssembly tests use a separate (newer) version of LLVM to exercise support
# for experimental features such as wasm64, which can cause the CI environment
# to run out of disk space.
#
# Mitigate this by expunging the workspace before trying to build Wasm targets.
if [[ -z ${toolchain_name} && -z ${disable_wasm_tests} ]]; then
# Redefine `test_args` without `--linkopt=-Wl,-v`, which breaks `wasm-ld`.
#
# https://github.com/llvm/llvm-project/issues/112836
test_args=(
"--copt=-v"
"--linkopt=-Wl,-t"
)
wasm_targets=(
"//wasm:all"
)
"${bazel}" clean --expunge
"${bazel}" ${TEST_MIGRATION:+"--strict"} --bazelrc=/dev/null test \
"${common_test_args[@]}" "${test_args[@]}" "${wasm_targets[@]}"
fi

View File

@ -30,12 +30,16 @@ set -exuo pipefail
# Common setup
export DEBIAN_FRONTEND=noninteractive
apt-get -qq update
apt-get -qq -y install apt-utils curl libtinfo5 pkg-config zlib1g-dev >/dev/null
apt-get -qq -y install apt-utils curl libtinfo5 libxml2 pkg-config zlib1g-dev >/dev/null
# The above command gives some verbose output that can not be silenced easily.
# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=288778
# The WebAssembly tests use an LLVM version that is too new for the GNU libc
# distributed in Ubuntu 20.04.
disable_wasm_tests='-W'
# Run tests
cd /src
tests/scripts/run_tests.sh
tests/scripts/run_tests.sh \${disable_wasm_tests}
"""
done

View File

@ -30,7 +30,7 @@ set -exuo pipefail
# Common setup
export DEBIAN_FRONTEND=noninteractive
apt-get -qq update
apt-get -qq -y install curl libtinfo5 zlib1g-dev >/dev/null
apt-get -qq -y install curl libtinfo5 libxml2 zlib1g-dev >/dev/null
# The above command gives some verbose output that can not be silenced easily.
# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=288778

View File

@ -97,11 +97,11 @@ dwp_file = rule(
},
)
def _transition_library_to_platform_transition_impl(_, attr):
def _transition_to_platform_transition_impl(_, attr):
return {"//command_line_option:platforms": str(attr.platform)}
_transition_library_to_platform_transition = transition(
implementation = _transition_library_to_platform_transition_impl,
_transition_to_platform_transition = transition(
implementation = _transition_to_platform_transition_impl,
inputs = [],
outputs = ["//command_line_option:platforms"],
)
@ -114,7 +114,27 @@ def _transition_library_to_platform_impl(ctx):
transition_library_to_platform = rule(
implementation = _transition_library_to_platform_impl,
attrs = {
"lib": attr.label(mandatory = True, cfg = _transition_library_to_platform_transition),
"lib": attr.label(mandatory = True, cfg = _transition_to_platform_transition),
"platform": attr.label(mandatory = True),
"_allowlist_function_transition": attr.label(
default = "@bazel_tools//tools/allowlists/function_transition_allowlist",
),
},
)
def _transition_binary_to_platform_impl(ctx):
out = ctx.actions.declare_file(ctx.attr.name)
ctx.actions.symlink(output = out, target_file = ctx.file.bin)
return DefaultInfo(files = depset([out]))
transition_binary_to_platform = rule(
implementation = _transition_binary_to_platform_impl,
attrs = {
"bin": attr.label(
mandatory = True,
allow_single_file = True,
cfg = _transition_to_platform_transition,
),
"platform": attr.label(mandatory = True),
"_allowlist_function_transition": attr.label(
default = "@bazel_tools//tools/allowlists/function_transition_allowlist",

44
tests/wasm/BUILD.bazel Normal file
View File

@ -0,0 +1,44 @@
load("@bazel_skylib//rules:build_test.bzl", "build_test")
load("@rules_cc//cc:defs.bzl", "cc_binary")
load("//:transitions.bzl", "transition_binary_to_platform")
build_test(
name = "wasm_targets_test",
targets = [
":wasm32_strlen",
":wasm32_strlen_nolibc",
":wasm64_strlen_nolibc",
],
)
cc_binary(
name = "wasm_strlen",
srcs = ["wasm_strlen.c"],
linkopts = ["-Wl,--no-entry"],
tags = ["manual"],
)
transition_binary_to_platform(
name = "wasm32_strlen",
bin = ":wasm_strlen",
platform = "@toolchains_llvm//platforms:wasm32",
)
cc_binary(
name = "wasm_strlen_nolibc",
srcs = ["wasm_strlen_nolibc.c"],
linkopts = ["-Wl,--no-entry"],
tags = ["manual"],
)
transition_binary_to_platform(
name = "wasm32_strlen_nolibc",
bin = ":wasm_strlen_nolibc",
platform = "@toolchains_llvm//platforms:wasm32",
)
transition_binary_to_platform(
name = "wasm64_strlen_nolibc",
bin = ":wasm_strlen_nolibc",
platform = "@toolchains_llvm//platforms:wasm64",
)

49
tests/wasm/wasi_sdk.bzl Normal file
View File

@ -0,0 +1,49 @@
_SYSROOT_BUILD = """
filegroup(
name = {name},
srcs = glob(["include/**/*", "lib/**/*", "share/**/*"], allow_empty=True),
visibility = ["//visibility:public"],
)
"""
_WASI_SDK_ABIS = [
"wasm32-wasi",
"wasm32-wasip1",
"wasm32-wasip1-threads",
"wasm32-wasip2",
"wasm32-wasi-threads",
]
def _wasi_sdk_sysroots(ctx):
ctx.download_and_extract(
integrity = "sha256-NRcvfSeZSFsVpGsdh/UKWF2RXsZiCA8AXZkVOlCIjwg=",
stripPrefix = "wasi-sysroot-24.0",
url = ["https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-24/wasi-sysroot-24.0.tar.gz"],
)
ctx.file("empty/BUILD.bazel", _SYSROOT_BUILD.format(
name = repr("empty"),
))
for abi in _WASI_SDK_ABIS:
ctx.file("%s/BUILD.bazel" % (abi,), _SYSROOT_BUILD.format(
name = repr(abi),
))
ctx.execute(["mv", "include/" + abi, "%s/include" % (abi,)])
ctx.execute(["mv", "lib/" + abi, "%s/lib" % (abi,)])
ctx.execute(["mv", "share/" + abi, "%s/share" % (abi,)])
wasi_sdk_sysroots = repository_rule(_wasi_sdk_sysroots)
def _libclang_rt_wasm32(ctx):
ctx.file("BUILD.bazel", """
exports_files(glob(["*.a"]))
""")
ctx.download_and_extract(
integrity = "sha256-fjPA33WLkEabHePKFY4tCn9xk01YhFJbpqNy3gs7Dsc=",
stripPrefix = "libclang_rt.builtins-wasm32-wasi-24.0",
url = ["https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-24/libclang_rt.builtins-wasm32-wasi-24.0.tar.gz"],
)
libclang_rt_wasm32 = repository_rule(_libclang_rt_wasm32)

7
tests/wasm/wasm_strlen.c Normal file
View File

@ -0,0 +1,7 @@
#include <stdint.h>
#include <string.h>
__attribute__((export_name("strlen")))
uint32_t wasm_strlen(char *s) {
return strlen(s);
}

View File

@ -0,0 +1,6 @@
__attribute__((export_name("strlen")))
unsigned long wasm_strlen(char *s) {
unsigned long len = 0;
for (; *s; len++) {}
return len;
}

View File

@ -41,6 +41,7 @@ filegroup(
srcs = [
"bin/ld.lld",
"bin/ld64.lld",
"bin/wasm-ld",
],
)

View File

@ -89,6 +89,22 @@ def cc_toolchain_config(
"clang",
"glibc_unknown",
),
"wasm32": (
"clang-wasm32",
"wasm32",
"unknown",
"clang",
"unknown",
"unknown",
),
"wasm64": (
"clang-wasm64",
"wasm64",
"unknown",
"clang",
"unknown",
"unknown",
),
}[target_os_arch_key]
# Unfiltered compiler flags; these are placed at the end of the command
@ -134,10 +150,15 @@ def cc_toolchain_config(
link_flags = [
"--target=" + target_system_name,
"-lm",
"-no-canonical-prefixes",
]
stdlib = compiler_configuration["stdlib"]
if stdlib != "none":
link_flags.extend([
"-lm",
])
# Similar to link_flags, but placed later in the command line such that
# unused symbols are not stripped.
link_libs = []
@ -166,6 +187,10 @@ def cc_toolchain_config(
archive_flags.extend([
"-static",
])
elif target_arch in ["wasm32", "wasm64"]:
# lld is invoked as wasm-ld for WebAssembly targets.
use_lld = True
use_libtool = False
else:
# Note that for xcompiling from darwin to linux, the native ld64 is
# not an option because it is not a cross-linker, so lld is the
@ -183,7 +208,6 @@ def cc_toolchain_config(
# The linker has no way of knowing if there are C++ objects; so we
# always link C++ libraries.
cxx_standard = compiler_configuration["cxx_standard"]
stdlib = compiler_configuration["stdlib"]
sysroot_path = compiler_configuration["sysroot_path"]
if stdlib == "builtin-libc++" and is_xcompile:
stdlib = "stdc++"
@ -253,11 +277,14 @@ def cc_toolchain_config(
link_flags.extend([
"-l:libstdc++.a",
])
elif stdlib == "libc":
cxx_flags = [
"-std=" + cxx_standard,
]
elif stdlib == "none":
cxx_flags = [
"-nostdlib",
]
link_flags.extend([
"-nostdlib",
])

View File

@ -12,7 +12,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
SUPPORTED_TARGETS = [("linux", "x86_64"), ("linux", "aarch64"), ("darwin", "x86_64"), ("darwin", "aarch64")]
SUPPORTED_TARGETS = [
("linux", "x86_64"),
("linux", "aarch64"),
("darwin", "x86_64"),
("darwin", "aarch64"),
("none", "wasm32"),
("none", "wasm64"),
]
# Map of tool name to its symlinked name in the tools directory.
# See tool_paths in toolchain/cc_toolchain_config.bzl.
@ -120,7 +127,7 @@ def os(rctx):
def os_bzl(os):
# Return the OS string as used in bazel platform constraints.
return {"darwin": "osx", "linux": "linux"}[os]
return {"darwin": "osx", "linux": "linux", "none": "none"}[os]
def arch(rctx):
arch = rctx.attr.exec_arch
@ -139,7 +146,12 @@ def arch(rctx):
return "x86_64"
return arch
def is_standalone_arch(os, arch):
return os == "none" and arch in ["wasm32", "wasm64"]
def os_arch_pair(os, arch):
if is_standalone_arch(os, arch):
return arch
return "{}-{}".format(os, arch)
_supported_os_arch = [os_arch_pair(os, arch) for (os, arch) in SUPPORTED_TARGETS]

View File

@ -25,6 +25,7 @@ load(
_check_os_arch_keys = "check_os_arch_keys",
_exec_os_arch_dict_value = "exec_os_arch_dict_value",
_is_absolute_path = "is_absolute_path",
_is_standalone_arch = "is_standalone_arch",
_list_to_string = "list_to_string",
_os = "os",
_os_arch_pair = "os_arch_pair",
@ -243,7 +244,10 @@ def _cc_toolchains_str(
cc_toolchains_str = ""
toolchain_names = []
for (target_os, target_arch) in _supported_targets:
suffix = "{}-{}".format(target_arch, target_os)
if _is_standalone_arch(target_os, target_arch):
suffix = target_arch
else:
suffix = "{}-{}".format(target_arch, target_os)
cc_toolchain_str = _cc_toolchain_str(
rctx,
suffix,
@ -315,6 +319,8 @@ def _cc_toolchain_str(
"darwin-aarch64": "aarch64-apple-macosx",
"linux-aarch64": "aarch64-unknown-linux-gnu",
"linux-x86_64": "x86_64-unknown-linux-gnu",
"wasm32": "wasm32-unknown-unknown",
"wasm64": "wasm64-unknown-unknown",
}[target_pair]
cxx_builtin_include_directories = [
toolchain_path_prefix + "include/c++/v1",
@ -341,6 +347,11 @@ def _cc_toolchain_str(
_join(sysroot_prefix, "/usr/include"),
_join(sysroot_prefix, "/System/Library/Frameworks"),
])
elif target_os == "none":
if sysroot_prefix:
cxx_builtin_include_directories.extend([
_join(sysroot_prefix, "/include"),
])
else:
fail("Unreachable")

View File

@ -623,6 +623,14 @@ def download_llvm(rctx):
auth = _get_auth(rctx, urls),
)
if rctx.attr.libclang_rt:
clang_versions = rctx.path("lib/clang").readdir()
for libclang_rt, lib_name in rctx.attr.libclang_rt.items():
libclang_rt_content = rctx.read(libclang_rt)
for clang_version in clang_versions:
lib_path = clang_version.get_child("lib", lib_name)
rctx.file(lib_path, libclang_rt_content, legacy_utf8 = False)
updated_attrs = _attr_dict(rctx.attr)
if update_sha256:
updated_attrs["sha256"].update([(key, res.sha256)])

View File

@ -102,6 +102,12 @@ llvm_repo_attrs.update({
"As with `llvm_mirror`, these sources will take precedence over the official LLVM " +
"release sources.",
),
"libclang_rt": attr.label_keyed_string_dict(
mandatory = False,
doc = ("Additional libclang_rt libraries to overlay into the downloaded LLVM " +
"distribution. The key is the label of a libclang_rt library, " +
"and the value is `\"{llvm_target_name}/{library_name}.a\"`."),
),
"netrc": attr.string(
mandatory = False,
doc = "Path to the netrc file for authenticated LLVM URL downloads.",