feat: add coreutils toolchain (#332)

This commit is contained in:
Sahin Yort 2023-01-16 21:02:17 +03:00 committed by GitHub
parent 3cf8ffa135
commit 4dc36a97f2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 467 additions and 1 deletions

View File

@ -123,4 +123,9 @@ stardoc_with_diff_test(
bzl_library_target = "//lib:base64",
)
stardoc_with_diff_test(
name = "repositories",
bzl_library_target = "//lib:repositories",
)
update_docs()

101
docs/repositories.md generated Normal file
View File

@ -0,0 +1,101 @@
<!-- Generated with Stardoc: http://skydoc.bazel.build -->
Macros for loading dependencies and registering toolchains
<a id="aspect_bazel_lib_dependencies"></a>
## aspect_bazel_lib_dependencies
<pre>
aspect_bazel_lib_dependencies(<a href="#aspect_bazel_lib_dependencies-override_local_config_platform">override_local_config_platform</a>)
</pre>
Load dependencies required by aspect rules
**PARAMETERS**
| Name | Description | Default Value |
| :------------- | :------------- | :------------- |
| <a id="aspect_bazel_lib_dependencies-override_local_config_platform"></a>override_local_config_platform | override the @local_config_platform repository with one that adds stardoc support for loading constraints.bzl.<br><br>Should be set in repositories that load @aspect_bazel_lib copy actions and also generate stardoc. | <code>False</code> |
<a id="register_copy_to_directory_toolchains"></a>
## register_copy_to_directory_toolchains
<pre>
register_copy_to_directory_toolchains(<a href="#register_copy_to_directory_toolchains-name">name</a>, <a href="#register_copy_to_directory_toolchains-register">register</a>)
</pre>
Registers copy_to_directory toolchain and repositories
**PARAMETERS**
| Name | Description | Default Value |
| :------------- | :------------- | :------------- |
| <a id="register_copy_to_directory_toolchains-name"></a>name | override the prefix for the generated toolchain repositories | <code>"copy_to_directory"</code> |
| <a id="register_copy_to_directory_toolchains-register"></a>register | whether to call through to native.register_toolchains. Should be True for WORKSPACE users, but false when used under bzlmod extension | <code>True</code> |
<a id="register_coreutils_toolchains"></a>
## register_coreutils_toolchains
<pre>
register_coreutils_toolchains(<a href="#register_coreutils_toolchains-name">name</a>, <a href="#register_coreutils_toolchains-version">version</a>, <a href="#register_coreutils_toolchains-register">register</a>)
</pre>
Registers coreutils toolchain and repositories
**PARAMETERS**
| Name | Description | Default Value |
| :------------- | :------------- | :------------- |
| <a id="register_coreutils_toolchains-name"></a>name | override the prefix for the generated toolchain repositories | <code>"coreutils"</code> |
| <a id="register_coreutils_toolchains-version"></a>version | the version of coreutils to execute (see https://github.com/uutils/coreutils/releases) | <code>"0.0.16"</code> |
| <a id="register_coreutils_toolchains-register"></a>register | whether to call through to native.register_toolchains. Should be True for WORKSPACE users, but false when used under bzlmod extension | <code>True</code> |
<a id="register_jq_toolchains"></a>
## register_jq_toolchains
<pre>
register_jq_toolchains(<a href="#register_jq_toolchains-name">name</a>, <a href="#register_jq_toolchains-version">version</a>, <a href="#register_jq_toolchains-register">register</a>)
</pre>
Registers jq toolchain and repositories
**PARAMETERS**
| Name | Description | Default Value |
| :------------- | :------------- | :------------- |
| <a id="register_jq_toolchains-name"></a>name | override the prefix for the generated toolchain repositories | <code>"jq"</code> |
| <a id="register_jq_toolchains-version"></a>version | the version of jq to execute (see https://github.com/stedolan/jq/releases) | <code>"1.6"</code> |
| <a id="register_jq_toolchains-register"></a>register | whether to call through to native.register_toolchains. Should be True for WORKSPACE users, but false when used under bzlmod extension | <code>True</code> |
<a id="register_yq_toolchains"></a>
## register_yq_toolchains
<pre>
register_yq_toolchains(<a href="#register_yq_toolchains-name">name</a>, <a href="#register_yq_toolchains-version">version</a>, <a href="#register_yq_toolchains-register">register</a>)
</pre>
Registers yq toolchain and repositories
**PARAMETERS**
| Name | Description | Default Value |
| :------------- | :------------- | :------------- |
| <a id="register_yq_toolchains-name"></a>name | override the prefix for the generated toolchain repositories | <code>"yq"</code> |
| <a id="register_yq_toolchains-version"></a>version | the version of yq to execute (see https://github.com/mikefarah/yq/releases) | <code>"4.25.2"</code> |
| <a id="register_yq_toolchains-register"></a>register | whether to call through to native.register_toolchains. Should be True for WORKSPACE users, but false when used under bzlmod extension | <code>True</code> |

View File

@ -4,7 +4,7 @@ Users should *not* need to install these. If users see a load()
statement from these, that's a bug in our distribution.
"""
load("//lib:repositories.bzl", "register_jq_toolchains", "register_yq_toolchains")
load("//lib:repositories.bzl", "register_jq_toolchains", "register_yq_toolchains", "register_coreutils_toolchains")
load("//lib:utils.bzl", http_archive = "maybe_http_archive")
# buildifier: disable=unnamed-macro
@ -50,3 +50,4 @@ def bazel_lib_internal_deps():
# Register toolchains for tests
register_jq_toolchains()
register_yq_toolchains()
register_coreutils_toolchains()

View File

@ -27,6 +27,10 @@ toolchain_type(
name = "copy_to_directory_toolchain_type",
)
toolchain_type(
name = "coreutils_toolchain_type",
)
bzl_library(
name = "docs",
srcs = ["docs.bzl"],
@ -189,6 +193,7 @@ bzl_library(
srcs = ["repositories.bzl"],
deps = [
":utils",
"//lib/private/docs:coreutils_toolchain",
"//lib/private/docs:copy_to_directory_toolchain",
"//lib/private/docs:jq_toolchain",
"//lib/private/docs:local_config_platform",

View File

@ -0,0 +1,202 @@
"Setup coreutils toolchain repositories and rules"
# Platform names follow the platform naming convention in @aspect_bazel_lib//:lib/private/repo_utils.bzl
COREUTILS_PLATFORMS = {
"darwin_amd64": struct(
compatible_with = [
"@platforms//os:macos",
"@platforms//cpu:x86_64",
],
),
"darwin_arm64": struct(
compatible_with = [
"@platforms//os:macos",
"@platforms//cpu:aarch64",
],
),
"linux_amd64": struct(
compatible_with = [
"@platforms//os:linux",
"@platforms//cpu:x86_64",
],
),
"linux_arm64": struct(
compatible_with = [
"@platforms//os:linux",
"@platforms//cpu:aarch64",
],
),
"windows_amd64": struct(
compatible_with = [
"@platforms//os:windows",
"@platforms//cpu:x86_64",
],
),
}
DEFAULT_COREUTILS_VERSION = "0.0.16"
# https://github.com/mikefarah/coreutils/releases
#
# The integrity hashes can be automatically fetched for the coreutils releases by running
# `tools/coreutils_mirror_release.sh`.
COREUTILS_VERSIONS = {
"0.0.16": {
"linux_arm64": {
"filename": "coreutils-0.0.16-aarch64-unknown-linux-gnu.tar.gz",
"sha256": "sha256-6tExkcMMHe7+59M9Mp9UKeq+g0y+juO7oakZfVOtOqw="
},
"darwin_amd64": {
"filename": "coreutils-0.0.16-x86_64-apple-darwin.tar.gz",
"sha256": "sha256-lQYWAuPOKS6XxwArZdiKIyczwppTVwjF4ml0iKBaB9s="
},
"windows_amd64": {
"filename": "coreutils-0.0.16-x86_64-pc-windows-msvc.zip",
"sha256": "sha256-z5E1onYAKZoaUt2U1cv1t8UHPsJinYktEd6SpE2u07o="
},
"linux_amd64": {
"filename": "coreutils-0.0.16-x86_64-unknown-linux-gnu.tar.gz",
"sha256": "sha256-Slf4qKf19sAWoK2pUVluAitmL3N2uz4eWpV4eibIEW0="
}
}
}
CoreUtilsInfo = provider(
doc = "Provide info for executing coreutils",
fields = {
"bin": "Executable coreutils binary",
},
)
def _coreutils_toolchain_impl(ctx):
binary = ctx.file.binary
# Make the $(COREUTILS_BIN) variable available in places like genrules.
# See https://docs.bazel.build/versions/main/be/make-variables.html#custom_variables
template_variables = platform_common.TemplateVariableInfo({
"COREUTILS_BIN": binary.path,
})
default_info = DefaultInfo(
files = depset([binary]),
runfiles = ctx.runfiles(files = [binary]),
)
coreutils_info = CoreUtilsInfo(
bin = binary,
)
# Export all the providers inside our ToolchainInfo
# so the resolved_toolchain rule can grab and re-export them.
toolchain_info = platform_common.ToolchainInfo(
coreutils_info = coreutils_info,
template_variables = template_variables,
default = default_info,
)
return [default_info, toolchain_info, template_variables]
coreutils_toolchain = rule(
implementation = _coreutils_toolchain_impl,
attrs = {
"binary": attr.label(
mandatory = True,
allow_single_file = True
),
},
)
def _coreutils_toolchains_repo_impl(rctx):
# Expose a concrete toolchain which is the result of Bazel resolving the toolchain
# for the execution or target platform.
# Workaround for https://github.com/bazelbuild/bazel/issues/14009
starlark_content = """# @generated by @aspect_bazel_lib//lib/private:coreutils_toolchain.bzl
# Forward all the providers
def _resolved_toolchain_impl(ctx):
toolchain_info = ctx.toolchains["@aspect_bazel_lib//lib:coreutils_toolchain_type"]
return [
toolchain_info,
toolchain_info.default,
toolchain_info.coreutils_info,
toolchain_info.template_variables,
]
# Copied from java_toolchain_alias
# https://cs.opensource.google/bazel/bazel/+/master:tools/jdk/java_toolchain_alias.bzl
resolved_toolchain = rule(
implementation = _resolved_toolchain_impl,
toolchains = ["@aspect_bazel_lib//lib:coreutils_toolchain_type"],
incompatible_use_toolchain_transition = True,
)
"""
rctx.file("defs.bzl", starlark_content)
build_content = """# @generated by @aspect_bazel_lib//lib/private:coreutils_toolchain.bzl
#
# These can be registered in the workspace file or passed to --extra_toolchains flag.
# By default all these toolchains are registered by the coreutils_register_toolchains macro
# so you don't normally need to interact with these targets.
load(":defs.bzl", "resolved_toolchain")
resolved_toolchain(name = "resolved_toolchain", visibility = ["//visibility:public"])
"""
for [platform, meta] in COREUTILS_PLATFORMS.items():
build_content += """
toolchain(
name = "{platform}_toolchain",
exec_compatible_with = {compatible_with},
toolchain = "@{user_repository_name}_{platform}//:coreutils_toolchain",
toolchain_type = "@aspect_bazel_lib//lib:coreutils_toolchain_type",
)
""".format(
platform = platform,
user_repository_name = rctx.attr.user_repository_name,
compatible_with = meta.compatible_with,
)
# Base BUILD file for this repository
rctx.file("BUILD.bazel", build_content)
coreutils_toolchains_repo = repository_rule(
_coreutils_toolchains_repo_impl,
doc = """Creates a repository with toolchain definitions for all known platforms
which can be registered or selected.""",
attrs = {
"user_repository_name": attr.string(doc = "Base name for toolchains repository"),
},
)
def _coreutils_platform_repo_impl(rctx):
is_windows = rctx.attr.platform.startswith("windows_")
platform = rctx.attr.platform
if platform == "darwin_arm64":
platform = "darwin_amd64"
filename = COREUTILS_VERSIONS[rctx.attr.version][platform]["filename"]
url = "https://github.com/uutils/coreutils/releases/download/{}/{}".format(
rctx.attr.version,
filename
)
rctx.download_and_extract(
url = url,
stripPrefix = filename.replace(".zip", "").replace(".tar.gz", ""),
integrity = COREUTILS_VERSIONS[rctx.attr.version][platform]["sha256"],
)
build_content = """# @generated by @aspect_bazel_lib//lib/private:coreutils_toolchain.bzl
load("@aspect_bazel_lib//lib/private:coreutils_toolchain.bzl", "coreutils_toolchain")
exports_files(["{0}"])
coreutils_toolchain(name = "coreutils_toolchain", binary = "{0}", visibility = ["//visibility:public"])
""".format("coreutils.exe" if is_windows else "coreutils")
# Base BUILD file for this repository
rctx.file("BUILD.bazel", build_content)
coreutils_platform_repo = repository_rule(
implementation = _coreutils_platform_repo_impl,
doc = "Fetch external tools needed for coreutils toolchain",
attrs = {
"version": attr.string(mandatory = True, values = COREUTILS_VERSIONS.keys()),
"platform": attr.string(mandatory = True, values = COREUTILS_PLATFORMS.keys()),
},
)

View File

@ -232,3 +232,8 @@ bzl_library(
name = "copy_to_directory_toolchain",
srcs = ["//lib/private:copy_to_directory_toolchain.bzl"],
)
bzl_library(
name = "coreutils_toolchain",
srcs = ["//lib/private:coreutils_toolchain.bzl"],
)

View File

@ -3,6 +3,7 @@
load("//lib/private:jq_toolchain.bzl", "JQ_PLATFORMS", "jq_host_alias_repo", "jq_platform_repo", "jq_toolchains_repo", _DEFAULT_JQ_VERSION = "DEFAULT_JQ_VERSION")
load("//lib/private:yq_toolchain.bzl", "YQ_PLATFORMS", "yq_host_alias_repo", "yq_platform_repo", "yq_toolchains_repo", _DEFAULT_YQ_VERSION = "DEFAULT_YQ_VERSION")
load("//lib/private:copy_to_directory_toolchain.bzl", "COPY_TO_DIRECTORY_PLATFORMS", "copy_to_directory_platform_repo", "copy_to_directory_toolchains_repo")
load("//lib/private:coreutils_toolchain.bzl", "COREUTILS_PLATFORMS", "coreutils_platform_repo", "coreutils_toolchains_repo", _DEFAULT_COREUTILS_VERSION = "DEFAULT_COREUTILS_VERSION")
load("//lib/private:local_config_platform.bzl", "local_config_platform")
load("//lib:utils.bzl", "is_bazel_6_or_greater", http_archive = "maybe_http_archive")
@ -39,6 +40,7 @@ def aspect_bazel_lib_dependencies(override_local_config_platform = False):
# Re-export the default versions
DEFAULT_JQ_VERSION = _DEFAULT_JQ_VERSION
DEFAULT_YQ_VERSION = _DEFAULT_YQ_VERSION
DEFAULT_COREUTILS_VERSION = _DEFAULT_COREUTILS_VERSION
def register_jq_toolchains(name = "jq", version = DEFAULT_JQ_VERSION, register = True):
"""Registers jq toolchain and repositories
@ -90,6 +92,29 @@ def register_yq_toolchains(name = "yq", version = DEFAULT_YQ_VERSION, register =
user_repository_name = name,
)
def register_coreutils_toolchains(name = "coreutils", version = DEFAULT_COREUTILS_VERSION, register = True):
"""Registers coreutils toolchain and repositories
Args:
name: override the prefix for the generated toolchain repositories
version: the version of coreutils to execute (see https://github.com/uutils/coreutils/releases)
register: whether to call through to native.register_toolchains.
Should be True for WORKSPACE users, but false when used under bzlmod extension
"""
for [platform, meta] in COREUTILS_PLATFORMS.items():
coreutils_platform_repo(
name = "%s_%s" % (name, platform),
platform = platform,
version = version
)
if register:
native.register_toolchains("@%s_toolchains//:%s_toolchain" % (name, platform))
coreutils_toolchains_repo(
name = "%s_toolchains" % name,
user_repository_name = name,
)
def register_copy_to_directory_toolchains(name = "copy_to_directory", register = True):
"""Registers copy_to_directory toolchain and repositories

View File

@ -0,0 +1,59 @@
load("//lib:testing.bzl", "assert_contains")
load("//lib/tests/jq:diff_test.bzl", "diff_test")
genrule(
name = "ls",
outs = ["ls.txt"],
cmd = "$(COREUTILS_BIN) ls > $@",
toolchains = ["@coreutils_toolchains//:resolved_toolchain"],
)
diff_test(
name = "ls_test",
file1 = "ls.txt.expected",
file2 = ":ls",
)
genrule(
name = "sha256sum",
outs = ["sha256sum.txt"],
srcs = ["test.bin"],
cmd = "$(COREUTILS_BIN) sha256sum $(location :test.bin) > $@",
toolchains = ["@coreutils_toolchains//:resolved_toolchain"],
)
assert_contains(
name = "test_sha256sum",
actual = "sha256sum.txt",
expected = """9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08 lib/tests/coreutils/test.bin""",
)
genrule(
name = "sha512sum",
outs = ["sha512sum.txt"],
srcs = ["test.bin"],
cmd = "$(COREUTILS_BIN) sha512sum $(location :test.bin) > $@",
toolchains = ["@coreutils_toolchains//:resolved_toolchain"],
)
assert_contains(
name = "test_sha512sum",
actual = "sha512sum.txt",
expected = """ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a8ff lib/tests/coreutils/test.bin""",
)
genrule(
name = "wc",
outs = ["wc.txt"],
srcs = ["test.bin"],
cmd = "$(COREUTILS_BIN) wc $(location :test.bin) > $@",
toolchains = ["@coreutils_toolchains//:resolved_toolchain"],
)
assert_contains(
name = "test_wc",
actual = "wc.txt",
expected = """0 1 4 lib/tests/coreutils/test.bin""",
)

View File

@ -0,0 +1,2 @@
bazel-out
external

View File

@ -0,0 +1 @@
test

View File

@ -0,0 +1,60 @@
#!/usr/bin/env bash
set -o errexit -o nounset -o pipefail
JQ_FILTER=\
'map({
"key": .tag_name,
"value": .assets
| map(select(
(.name | startswith("coreutils-")) and
((.name | endswith(".tar.gz")) or (.name | endswith(".zip"))) and
(.name | contains("i686") | not) and
(
( (.name | contains("windows")) and (.name | contains("gnu") | not) ) or
( (.name | contains("windows") | not) and (.name | contains("gnu") ) and (.name | contains("gnueabihf") | not) ) or
( .name | contains("darwin") )
)
))
| map({
key: .name |
ltrimstr("coreutils-") |
rtrimstr(".tar.gz") |
rtrimstr(".zip") |
sub("-pc"; "") |
sub("-apple"; "") |
sub("-unknown"; "") |
sub("x86_64"; "amd64") |
sub("aarch64"; "arm64") |
gsub("\\d+.\\d+.\\d+-"; "") |
rtrimstr("-msvc") |
rtrimstr("-gnu") |
split("-") |
reverse |
join("_"),
value: {
filename: .name,
sha256: "sha256-",
}
})
| from_entries
}) | from_entries
'
INFO="$(curl --silent -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/uutils/coreutils/releases?per_page=1 | jq "$JQ_FILTER")"
for VERSION in $(jq -r 'keys | join("\n")' <<< $INFO); do
for PLATFORM in $(jq -r ".[\"$VERSION\"] | keys | join(\"\n\")" <<< $INFO); do
FILENAME=$(jq -r ".[\"$VERSION\"][\"$PLATFORM\"].filename" <<< $INFO)
SHA256=$(curl -fLs "https://github.com/uutils/coreutils/releases/download/$VERSION/$FILENAME" | sha256sum | xxd -r -p | base64)
INFO=$(jq ".[\"$VERSION\"][\"$PLATFORM\"].sha256 = \"sha256-$SHA256\"" <<< $INFO)
done
done
echo -n "COREUTILS_VERSIONS = "
echo $INFO | jq -M
echo ""
echo "Copy the version info into lib/private/coreutils_toolchain.bzl"