bazel
/
bazel-lib
mirror of https://github.com/bazel-contrib/bazel-lib
2
0
Fork 0
Code Issues Releases Activity

refactor: rework toolchain registration for WORKSPACE and bzlmod users (#597)

This commit is contained in:
Derek Cormier 2023-10-10 12:16:13 -07:00 committed by GitHub
parent e1d9d9b58f
commit 20f1ac39e0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 280 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
.bcr/patches/go_dev_dep.patch
View File

@ -1,27 +1,27 @@
diff --git a/MODULE.bazel b/MODULE.bazel
index 07a6eff..26702ad 100644
index e63fa5b..9d78a88 100644
--- a/MODULE.bazel
+++ b/MODULE.bazel
@@ -34,19 +34,19 @@ register_toolchains(
@@ -50,19 +50,19 @@ use_repo(host, "aspect_bazel_lib_host")
bazel_dep(
name = "gazelle",
version = "0.33.0",
- # In released versions: dev_dependency = True
+ dev_dependency = True
+ dev_dependency = True,
)
bazel_dep(
name = "rules_go",
version = "0.41.0",
repo_name = "io_bazel_rules_go",
- # In released versions: dev_dependency = True
+ dev_dependency = True
+ dev_dependency = True,
)
go_deps = use_extension(
"@gazelle//:extensions.bzl",
"go_deps",
- # In released versions: dev_dependency = True
+ dev_dependency = True
+ dev_dependency = True,
)
go_deps.from_file(go_mod = "//:go.mod")
use_repo(

23
.github/workflows/release_prep.sh vendored
View File

@ -48,25 +48,16 @@ http_archive(
url = "https://github.com/aspect-build/bazel-lib/releases/download/${TAG}/${ARCHIVE}",
)
load("@aspect_bazel_lib//lib:repositories.bzl", "aspect_bazel_lib_dependencies")
load("@aspect_bazel_lib//lib:repositories.bzl", "aspect_bazel_lib_dependencies", "aspect_bazel_lib_register_toolchains")
# Required bazel-lib dependencies
aspect_bazel_lib_dependencies()
# Register bazel-lib toolchains
aspect_bazel_lib_register_toolchains()
\`\`\`
Optional toolchains:
\`\`\`starlark
# Register the following toolchain to use jq
load("@aspect_bazel_lib//lib:repositories.bzl", "register_jq_toolchains")
register_jq_toolchains()
# Register the following toolchain to use yq
load("@aspect_bazel_lib//lib:repositories.bzl", "register_yq_toolchains")
register_yq_toolchains()
\`\`\`
EOF

15
MODULE.bazel
View File

@ -14,8 +14,15 @@ bazel_dep(name = "platforms", version = "0.0.7")
# 0.5.4 is the first version with bzlmod support
bazel_dep(name = "stardoc", version = "0.5.4", repo_name = "io_bazel_stardoc")
ext = use_extension("@aspect_bazel_lib//lib:extensions.bzl", "ext")
use_repo(ext, "bsd_tar_toolchains", "copy_directory_toolchains", "copy_to_directory_toolchains", "coreutils_toolchains", "expand_template_toolchains", "jq_toolchains", "yq_toolchains")
bazel_lib_toolchains = use_extension("@aspect_bazel_lib//lib:extensions.bzl", "toolchains")
bazel_lib_toolchains.copy_directory()
bazel_lib_toolchains.copy_to_directory()
bazel_lib_toolchains.jq()
bazel_lib_toolchains.yq()
bazel_lib_toolchains.coreutils()
bazel_lib_toolchains.tar()
bazel_lib_toolchains.expand_template()
use_repo(bazel_lib_toolchains, "bsd_tar_toolchains", "copy_directory_toolchains", "copy_to_directory_toolchains", "coreutils_toolchains", "expand_template_toolchains", "jq_toolchains", "yq_toolchains")
register_toolchains(
"@copy_directory_toolchains//:all",
@ -32,6 +39,10 @@ register_toolchains(
"@bsd_tar_toolchains//:host_toolchain",
)
host = use_extension("@aspect_bazel_lib//lib:extensions.bzl", "host", dev_dependency = True)
host.host()
use_repo(host, "aspect_bazel_lib_host")
# To allow /tools to be built from source
# NOTE: when publishing to BCR, we patch this to be dev_dependency, as we publish pre-built binaries
# along with our releases.

4
WORKSPACE
View File

@ -10,10 +10,12 @@ load(":internal_deps.bzl", "bazel_lib_internal_deps")
# Fetch deps needed only locally for development
bazel_lib_internal_deps()
load("//lib:repositories.bzl", "aspect_bazel_lib_dependencies")
load("//lib:repositories.bzl", "aspect_bazel_lib_dependencies", "aspect_bazel_lib_register_toolchains")
aspect_bazel_lib_dependencies()
aspect_bazel_lib_register_toolchains()
# For running our own unit tests
load("@bazel_skylib//lib:unittest.bzl", "register_unittest_toolchains")

8
docs/jq.md generated
View File

@ -14,14 +14,6 @@ Invoke jq with a filter on a set of json input files.
For jq documentation, see https://stedolan.github.io/jq/.
To use this rule you must register the jq toolchain in your WORKSPACE:
```starlark
load("@aspect_bazel_lib//lib:repositories.bzl", "register_jq_toolchains")
register_jq_toolchains()
```
Usage examples:
```starlark

15
docs/repositories.md generated
View File

@ -14,6 +14,21 @@ Load dependencies required by aspect rules
<a id="aspect_bazel_lib_register_toolchains"></a>
## aspect_bazel_lib_register_toolchains
<pre>
aspect_bazel_lib_register_toolchains()
</pre>
Register all bazel-lib toolchains at their default versions.
To be more selective about which toolchains and versions to register,
call the individual toolchain registration macros.
<a id="register_copy_directory_toolchains"></a>
## register_copy_directory_toolchains

8
docs/yq.md generated
View File

@ -14,14 +14,6 @@ Invoke yq with an expression on a set of input files.
For yq documentation, see https://mikefarah.gitbook.io/yq.
To use this rule you must register the yq toolchain in your WORKSPACE:
```starlark
load("@aspect_bazel_lib//lib:repositories.bzl", "register_yq_toolchains")
register_yq_toolchains()
```
Usage examples:
```starlark

4
e2e/copy_to_directory/MODULE.bazel
View File

@ -10,7 +10,3 @@ local_path_override(
module_name = "aspect_bazel_lib",
path = "../..",
)
ext = use_extension("@aspect_bazel_lib//lib:extensions.bzl", "ext")
ext.host()
use_repo(ext, "aspect_bazel_lib_host")

4
e2e/copy_to_directory/WORKSPACE
View File

@ -3,10 +3,12 @@ local_repository(
path = "../..",
)
load("@aspect_bazel_lib//lib:repositories.bzl", "aspect_bazel_lib_dependencies")
load("@aspect_bazel_lib//lib:repositories.bzl", "aspect_bazel_lib_dependencies", "register_copy_to_directory_toolchains")
aspect_bazel_lib_dependencies()
register_copy_to_directory_toolchains()
############################################
# rules_go is needed to consume tools from sources

4
e2e/coreutils/MODULE.bazel
View File

@ -10,5 +10,5 @@ local_path_override(
path = "../..",
)
ext = use_extension("@aspect_bazel_lib//lib:extensions.bzl", "ext")
use_repo(ext, "coreutils_toolchains")
bazel_lib_toolchains = use_extension("@aspect_bazel_lib//lib:extensions.bzl", "toolchains")
use_repo(bazel_lib_toolchains, "coreutils_toolchains")

4
e2e/coreutils/WORKSPACE
View File

@ -3,6 +3,8 @@ local_repository(
path = "../..",
)
load("@aspect_bazel_lib//lib:repositories.bzl", "aspect_bazel_lib_dependencies")
load("@aspect_bazel_lib//lib:repositories.bzl", "aspect_bazel_lib_dependencies", "register_coreutils_toolchains")
aspect_bazel_lib_dependencies()
register_coreutils_toolchains()

6
e2e/smoke/WORKSPACE
View File

@ -3,13 +3,11 @@ local_repository(
path = "../..",
)
load("@aspect_bazel_lib//lib:repositories.bzl", "aspect_bazel_lib_dependencies", "register_jq_toolchains", "register_yq_toolchains")
load("@aspect_bazel_lib//lib:repositories.bzl", "aspect_bazel_lib_dependencies", "aspect_bazel_lib_register_toolchains")
aspect_bazel_lib_dependencies()
register_jq_toolchains()
register_yq_toolchains()
aspect_bazel_lib_register_toolchains()
############################################
# rules_go is needed to consume tools from sources

98
lib/extensions.bzl
View File

@ -2,6 +2,16 @@
load(
"@aspect_bazel_lib//lib:repositories.bzl",
"DEFAULT_COPY_DIRECTORY_REPOSITORY",
"DEFAULT_COPY_TO_DIRECTORY_REPOSITORY",
"DEFAULT_COREUTILS_REPOSITORY",
"DEFAULT_COREUTILS_VERSION",
"DEFAULT_EXPAND_TEMPLATE_REPOSITORY",
"DEFAULT_JQ_REPOSITORY",
"DEFAULT_JQ_VERSION",
"DEFAULT_TAR_REPOSITORY",
"DEFAULT_YQ_REPOSITORY",
"DEFAULT_YQ_VERSION",
"register_copy_directory_toolchains",
"register_copy_to_directory_toolchains",
"register_coreutils_toolchains",
@ -10,17 +20,10 @@ load(
"register_tar_toolchains",
"register_yq_toolchains",
)
load("//lib/private:extension_utils.bzl", "extension_utils")
load("//lib/private:host_repo.bzl", "host_repo")
def _toolchain_extension(mctx):
register_copy_directory_toolchains(register = False)
register_copy_to_directory_toolchains(register = False)
register_jq_toolchains(register = False)
register_yq_toolchains(register = False)
register_coreutils_toolchains(register = False)
register_tar_toolchains(register = False)
register_expand_template_toolchains(register = False)
def _host_extension_impl(mctx):
create_host_repo = False
for module in mctx.modules:
if len(module.tags.host) > 0:
@ -29,8 +32,77 @@ def _toolchain_extension(mctx):
if create_host_repo:
host_repo(name = "aspect_bazel_lib_host")
# TODO: some way for users to control repo name/version of the tools installed
ext = module_extension(
implementation = _toolchain_extension,
tag_classes = {"host": tag_class(attrs = {})},
host = module_extension(
implementation = _host_extension_impl,
tag_classes = {
"host": tag_class(attrs = {}),
},
)
def _toolchains_extension_impl(mctx):
extension_utils.toolchain_repos_bfs(
mctx = mctx,
get_tag_fn = lambda tags: tags.copy_directory,
toolchain_name = "copy_directory",
toolchain_repos_fn = lambda name, version: register_copy_directory_toolchains(name = name, register = False),
get_version_fn = lambda attr: None,
)
extension_utils.toolchain_repos_bfs(
mctx = mctx,
get_tag_fn = lambda tags: tags.copy_to_directory,
toolchain_name = "copy_to_directory",
toolchain_repos_fn = lambda name, version: register_copy_to_directory_toolchains(name = name, register = False),
get_version_fn = lambda attr: None,
)
extension_utils.toolchain_repos_bfs(
mctx = mctx,
get_tag_fn = lambda tags: tags.jq,
toolchain_name = "jq",
toolchain_repos_fn = lambda name, version: register_jq_toolchains(name = name, version = version, register = False),
)
extension_utils.toolchain_repos_bfs(
mctx = mctx,
get_tag_fn = lambda tags: tags.yq,
toolchain_name = "yq",
toolchain_repos_fn = lambda name, version: register_yq_toolchains(name = name, version = version, register = False),
)
extension_utils.toolchain_repos_bfs(
mctx = mctx,
get_tag_fn = lambda tags: tags.coreutils,
toolchain_name = "coreutils",
toolchain_repos_fn = lambda name, version: register_coreutils_toolchains(name = name, version = version, register = False),
)
extension_utils.toolchain_repos_bfs(
mctx = mctx,
get_tag_fn = lambda tags: tags.tar,
toolchain_name = "tar",
default_repository = DEFAULT_TAR_REPOSITORY,
toolchain_repos_fn = lambda name, version: register_tar_toolchains(name = name, register = False),
get_version_fn = lambda attr: None,
)
extension_utils.toolchain_repos_bfs(
mctx = mctx,
get_tag_fn = lambda tags: tags.expand_template,
toolchain_name = "expand_template",
toolchain_repos_fn = lambda name, version: register_expand_template_toolchains(name = name, register = False),
get_version_fn = lambda attr: None,
)
toolchains = module_extension(
implementation = _toolchains_extension_impl,
tag_classes = {
"copy_directory": tag_class(attrs = {"name": attr.string(default = DEFAULT_COPY_DIRECTORY_REPOSITORY)}),
"copy_to_directory": tag_class(attrs = {"name": attr.string(default = DEFAULT_COPY_TO_DIRECTORY_REPOSITORY)}),
"jq": tag_class(attrs = {"name": attr.string(default = DEFAULT_JQ_REPOSITORY), "version": attr.string(default = DEFAULT_JQ_VERSION)}),
"yq": tag_class(attrs = {"name": attr.string(default = DEFAULT_YQ_REPOSITORY), "version": attr.string(default = DEFAULT_YQ_VERSION)}),
"coreutils": tag_class(attrs = {"name": attr.string(default = DEFAULT_COREUTILS_REPOSITORY), "version": attr.string(default = DEFAULT_COREUTILS_VERSION)}),
"tar": tag_class(attrs = {"name": attr.string(default = DEFAULT_TAR_REPOSITORY)}),
"expand_template": tag_class(attrs = {"name": attr.string(default = DEFAULT_EXPAND_TEMPLATE_REPOSITORY)}),
},
)

8
lib/jq.bzl
View File

@ -13,14 +13,6 @@ def jq(name, srcs, filter = None, filter_file = None, args = [], out = None, **k
For jq documentation, see https://stedolan.github.io/jq/.
To use this rule you must register the jq toolchain in your WORKSPACE:
```starlark
load("@aspect_bazel_lib//lib:repositories.bzl", "register_jq_toolchains")
register_jq_toolchains()
```
Usage examples:
```starlark

104
lib/private/extension_utils.bzl Normal file
View File

@ -0,0 +1,104 @@
"""Utility functions for bzlmod extensions"""
def _toolchain_repos_bfs(mctx, get_tag_fn, toolchain_name, toolchain_repos_fn, default_repository = None, get_name_fn = None, get_version_fn = None):
"""Create toolchain repositories from bzlmod extensions using a breadth-first resolution strategy.
Toolchains are assumed to have a "default" or canonical repository name so that across
all invocations of the module extension with that name only a single toolchain repository
is created. As such, it is recommended to default the toolchain name in the extension's
tag class attributes so that diverging from the canonical name is a special case.
The resolved toolchain version will be the one invoked closest to the root module, following
Bazel's breadth-first ordering of modules in the dependency graph.
For example, given the module extension usage in a MODULE file:
```starlark
ext = use_extension("@my_lib//lib:extensions.bzl", "ext")
ext.foo_toolchain(version = "1.2.3") # Default `name = "foo"`
use_repo(ext, "foo")
register_toolchains(
"@foo//:all",
)
```
This macro would be used in the module extension implementation as follows:
```starlark
extension_utils.toolchain_repos(
mctx = mctx,
get_tag_fn = lambda tags: tags.foo_toolchain,
toolchain_name = "foo",
toolchain_repos_fn = lambda name, version: register_foo_toolchains(name = name, register = False),
get_version_fn = lambda attr: None,
)
```
Where `register_foo_toolchains` is a typical WORKSPACE macro used to register
the foo toolchain for a particular version, minus the actual registration step
which is done separately in the MODULE file.
This macro enforces that only root MODULEs may use a different name for the toolchain
in case several versions of the toolchain repository is desired.
Args:
mctx: The module context
get_tag_fn: A function that takes in `module.tags` and returns the tag used for the toolchain.
For example, `tag: lambda tags: tags.foo_toolchain`. This is required because `foo_toolchain`
cannot be accessed as a simple string key from `module.tags`.
toolchain_name: Name of the toolchain to use in error messages
toolchain_repos_fn: A function that takes (name, version) and creates a toolchain repository. This lambda
should call a typical reposotiory rule to create toolchains.
default_repository: Default name of the toolchain repository to pass to the repos_fn.
By default, it equals `toolchain_name`.
get_name_fn: A function that extracts the module name from the toolchain tag's attributes. Defaults
to grabbing the `name` attribute.
get_version_fn: A function that extracts the module version from the a tag's attributes. Defaults
to grabbing the `version` attribute. Override this to a lambda that returns `None` if
version isn't used as an attribute.
"""
if default_repository == None:
default_repository = toolchain_name
if get_name_fn == None:
get_name_fn = lambda attr: attr.name
if get_version_fn == None:
get_version_fn = lambda attr: attr.version
registrations = {}
for mod in mctx.modules:
for attr in get_tag_fn(mod.tags):
name = get_name_fn(attr)
version = get_version_fn(attr)
if name != default_repository and not mod.is_root:
fail("Only the root module may provide a name for the {} toolchain.".format(toolchain_name))
if name in registrations.keys():
if name == default_repository:
# Prioritize the root-most registration of the default toolchain version and
# ignore any further registrations (modules are processed breadth-first)
continue
if version == registrations[name]:
# No problem to register a matching toolchain twice
continue
fail("Multiple conflicting {} toolchains declared for name {} ({} and {})".format(
toolchain_name,
name,
version,
registrations[name],
))
else:
registrations[name] = version
for name, version in registrations.items():
toolchain_repos_fn(
name = name,
version = version,
)
extension_utils = struct(
toolchain_repos_bfs = _toolchain_repos_bfs,
)

58
lib/repositories.bzl
View File

@ -23,23 +23,10 @@ def aspect_bazel_lib_dependencies():
],
)
# Always register the copy_to_directory toolchain
register_copy_directory_toolchains()
register_copy_to_directory_toolchains()
# Always register the expand_template toolchain
register_expand_template_toolchains()
# Always register the coreutils toolchain and the tar toolchain
register_coreutils_toolchains()
register_tar_toolchains()
# Re-export the default versions
DEFAULT_JQ_REPOSITORY = "jq"
DEFAULT_JQ_VERSION = _DEFAULT_JQ_VERSION
DEFAULT_YQ_VERSION = _DEFAULT_YQ_VERSION
DEFAULT_COREUTILS_VERSION = _DEFAULT_COREUTILS_VERSION
def register_jq_toolchains(name = "jq", version = DEFAULT_JQ_VERSION, register = True):
def register_jq_toolchains(name = DEFAULT_JQ_REPOSITORY, version = DEFAULT_JQ_VERSION, register = True):
"""Registers jq toolchain and repositories
Args:
@ -64,7 +51,10 @@ def register_jq_toolchains(name = "jq", version = DEFAULT_JQ_VERSION, register =
user_repository_name = name,
)
def register_yq_toolchains(name = "yq", version = DEFAULT_YQ_VERSION, register = True):
DEFAULT_YQ_REPOSITORY = "yq"
DEFAULT_YQ_VERSION = _DEFAULT_YQ_VERSION
def register_yq_toolchains(name = DEFAULT_YQ_REPOSITORY, version = DEFAULT_YQ_VERSION, register = True):
"""Registers yq toolchain and repositories
Args:
@ -89,7 +79,9 @@ def register_yq_toolchains(name = "yq", version = DEFAULT_YQ_VERSION, register =
user_repository_name = name,
)
def register_tar_toolchains(name = "bsd_tar", register = True):
DEFAULT_TAR_REPOSITORY = "bsd_tar"
def register_tar_toolchains(name = DEFAULT_TAR_REPOSITORY, register = True):
"""Registers bsdtar toolchain and repositories
Args:
@ -110,7 +102,10 @@ def register_tar_toolchains(name = "bsd_tar", register = True):
user_repository_name = name,
)
def register_coreutils_toolchains(name = "coreutils", version = DEFAULT_COREUTILS_VERSION, register = True):
DEFAULT_COREUTILS_REPOSITORY = "coreutils"
DEFAULT_COREUTILS_VERSION = _DEFAULT_COREUTILS_VERSION
def register_coreutils_toolchains(name = DEFAULT_COREUTILS_REPOSITORY, version = DEFAULT_COREUTILS_VERSION, register = True):
"""Registers coreutils toolchain and repositories
Args:
@ -133,7 +128,9 @@ def register_coreutils_toolchains(name = "coreutils", version = DEFAULT_COREUTIL
user_repository_name = name,
)
def register_copy_directory_toolchains(name = "copy_directory", register = True):
DEFAULT_COPY_DIRECTORY_REPOSITORY = "copy_directory"
def register_copy_directory_toolchains(name = DEFAULT_COPY_DIRECTORY_REPOSITORY, register = True):
"""Registers copy_directory toolchain and repositories
Args:
@ -166,7 +163,9 @@ def register_copy_directory_toolchains(name = "copy_directory", register = True)
user_repository_name = name,
)
def register_copy_to_directory_toolchains(name = "copy_to_directory", register = True):
DEFAULT_COPY_TO_DIRECTORY_REPOSITORY = "copy_to_directory"
def register_copy_to_directory_toolchains(name = DEFAULT_COPY_TO_DIRECTORY_REPOSITORY, register = True):
"""Registers copy_to_directory toolchain and repositories
Args:
@ -199,7 +198,9 @@ def register_copy_to_directory_toolchains(name = "copy_to_directory", register =
user_repository_name = name,
)
def register_expand_template_toolchains(name = "expand_template", register = True):
DEFAULT_EXPAND_TEMPLATE_REPOSITORY = "expand_template"
def register_expand_template_toolchains(name = DEFAULT_EXPAND_TEMPLATE_REPOSITORY, register = True):
"""Registers expand_template toolchain and repositories
Args:
@ -231,3 +232,18 @@ def register_expand_template_toolchains(name = "expand_template", register = Tru
name = "%s_toolchains" % name,
user_repository_name = name,
)
# buildifier: disable=unnamed-macro
def aspect_bazel_lib_register_toolchains():
"""Register all bazel-lib toolchains at their default versions.
To be more selective about which toolchains and versions to register,
call the individual toolchain registration macros.
"""
register_copy_directory_toolchains()
register_copy_to_directory_toolchains()
register_expand_template_toolchains()
register_coreutils_toolchains()
register_jq_toolchains()
register_yq_toolchains()
register_tar_toolchains()

8
lib/yq.bzl
View File

@ -13,14 +13,6 @@ def yq(name, srcs, expression = ".", args = [], outs = None, **kwargs):
For yq documentation, see https://mikefarah.gitbook.io/yq.
To use this rule you must register the yq toolchain in your WORKSPACE:
```starlark
load("@aspect_bazel_lib//lib:repositories.bzl", "register_yq_toolchains")
register_yq_toolchains()
```
Usage examples:
```starlark