2
0
Fork 0
mirror of https://github.com/bazel-contrib/bazel-lib synced 2024-11-27 17:43:27 +00:00
bazel-lib/lib/BUILD.bazel

331 lines
6.4 KiB
Python
Raw Normal View History

2021-11-08 14:40:36 +00:00
load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
load("@bazel_skylib//rules:common_settings.bzl", "bool_flag")
load("//lib:utils.bzl", "is_bzlmod_enabled")
load("//lib/private:stamping.bzl", "stamp_build_setting")
2021-11-08 14:40:36 +00:00
# Ensure that users building their own rules can dep on our bzl_library targets for their stardoc
package(default_visibility = ["//visibility:public"])
exports_files(
glob(["*.bzl"]),
visibility = [
"//docs:__pkg__",
"//lib:__subpackages__",
],
)
2021-11-08 14:40:36 +00:00
2022-07-15 05:32:39 +00:00
# Macro that creates targets enabling the use of `--stamp` in Starlark rules
stamp_build_setting(name = "stamp")
# Add a config indicating that bzlmod is enabled so that we can use select()
# to skip bzlmod-incompatible targets.
config_setting(
name = "bzlmod",
flag_values = {
":flag_bzlmod": "true",
},
)
bool_flag(
name = "flag_bzlmod",
build_setting_default = True if is_bzlmod_enabled() else False,
)
perf: report unused inputs for the tar rule (#951) * perf: report unused inputs for the tar rule The `mtree` spec passed to the `tar` rule very often selects a subset of the inputs made available through the `srcs` attribute. In many cases, these subsets do not break down cleanly along dependency-tree lines and there is no simple way just pass less content to the `tar` rule. One prominent example where this occurs is when constructing the tars for OCI image layers. For instance when [building a Python-based container image](https://github.com/bazel-contrib/rules_oci/blob/main/docs/python.md), we might want to split the Python interpreter, third-party dependencies, and application code into their own layers. This is done by [filtering the `mtree_spec`](https://github.com/aspect-build/bazel-examples/blob/85cb2aaf8c6e51d5e9e086cc94b94ab896903fb0/oci_python_image/py_layer.bzl#L39). However, in the operation to construct a `tar` from a subsetted mtree, it is usually still an unsubsetted tree of `srcs` that gets passed. As a result, the subset tarball is considered dependent upon a larger set of sources than is strictly necessary. This over-scoping runs counter to a very common objective associated with breaking up an image into layers - isolating churn to a smaller slice of the application. Because of the spurious relationships established in Bazel's dependency graph, all tars get rebuilt anytime any content in the application gets changed. Tar rebuilds can even be triggered by changes to files that are completely filtered-out from all layers of the container. Redundent creation of archive content is usually not too computationally intensive, but the archives can be quite large in some cases, and avoiding a rebuild might free up gigabytes of disk and/or network bandwidth for better use. In addition, eliminating the spurious dependency edges removes erroneous constraints applied to the build action schedule; these tend to push all Tar-building operations towards the end of a build, even when some archive construction could be scheduled much earlier. ## Risk assessment and mitigation The `unused_inputs_list` mechanism used to report spurious dependency relationships is a bit difficult to use. Reporting an actually-used input as unused can create difficult to diagnose problems down the line. However, the behaviour of the `mtree`-based `tar` rule is sufficiently simple and self-contained that I am fairly confident that this rule's used/unused set can be determined accurately in a maintainable fashion. Out of an abundance of caution I have gated this feature behind a default-off flag. The `tar` rule will continue to operate as it had before - typically over-reporting dependencies - unless the `--@aspect_bazel_lib//lib:tar_compute_unused_inputs` flag is passed. ### Filter accuracy The `vis` encoding used by the `mtree` format to resiliently handle path names has a small amount of "play" to it - it is reversable but the encoded representation of a string is not unique. Two unequal encoded strings might decode to the same value; this can happen when at least one of the encoded strings contains unnecessary escapes that are nevertheless honoured by the decoder. The unused-inputs set is determined using a filter that compares `vis`-encoded strings. In the presence of non-canonically-encoded paths, false-mismatches can lead to falsely reporting that an input is unused. The only `vis`-encoded path content that is under the control of callers is the `mtree` content itself; all other `vis`-encoded strings are constructed internally to this package, not exposed publicly, and are all derived using the `lib/private/tar.bzl%_vis_encode` function; all of these paths are expected to compare exactly. Additionally, it is expected that many/most users will use this package's helpers (e.g. `mtree_spec`) when crafting their mtree content; such content is also safe. It is only when the user crafts their own mtree, or modifies an mtree spec's `content=` fields' encoding in some way, that a risk of inaccurate reporting arises. The chances for this are expected to be minor since this seems like an inconvenient and not-particularly-useful thing for a user to go out of their way to do. * Also include other bsdtar toolchain files in keep set * Add tri-state attribute to control unused-inputs behaviour This control surface provides for granular control of the feature. The interface is selected to mirror the common behaviour of `stamp` attributes. * Add bzl_library level dep * Update docs * pre-commit * Add reminder to change flag default on major-version bump * Add note about how to make unused input computation exactly correct * Add a test for unused_inputs listing * Support alternate contents= form This is accepted by bsdtar/libarchive. In fact `contents=` is the only of the pair documented in `mtree(5)`; `content=` is an undocumented alternate form supported by libarchive. * Don't try to prune the unprunable Bazel's interpretation of unused_inputs_list cannot accomodate certain things in filenames. These are also likely to mess up our own line-oriented protocol in the shellscript that produces this file. Co-authored-by: Sahin Yort <thesayyn@gmail.com> * Rerun docs update --------- Co-authored-by: Sahin Yort <thesayyn@gmail.com>
2024-10-13 16:58:56 +00:00
# TODO(3.0): change default to True.
bool_flag(
name = "tar_compute_unused_inputs",
build_setting_default = False,
)
config_setting(
name = "enable_runfiles",
values = {"enable_runfiles": "true"},
)
toolchain_type(
name = "jq_toolchain_type",
)
2022-04-20 04:45:06 +00:00
toolchain_type(
name = "yq_toolchain_type",
)
toolchain_type(
name = "copy_directory_toolchain_type",
)
toolchain_type(
name = "copy_to_directory_toolchain_type",
)
2023-01-16 18:02:17 +00:00
toolchain_type(
name = "coreutils_toolchain_type",
)
toolchain_type(
name = "expand_template_toolchain_type",
)
toolchain_type(
name = "tar_toolchain_type",
)
2024-05-03 23:12:56 +00:00
toolchain_type(
name = "zstd_toolchain_type",
)
toolchain_type(
name = "bats_toolchain_type",
)
2021-11-08 14:40:36 +00:00
bzl_library(
name = "expand_make_vars",
srcs = ["expand_make_vars.bzl"],
deps = [
"//lib/private:expand_locations",
"//lib/private:expand_variables",
],
)
bzl_library(
name = "expand_template",
srcs = ["expand_template.bzl"],
deps = [
"//lib/private:expand_template",
"@bazel_skylib//lib:types",
"@bazel_skylib//rules:write_file",
],
2021-11-08 14:40:36 +00:00
)
bzl_library(
name = "params_file",
srcs = ["params_file.bzl"],
deps = ["//lib/private:params_file"],
2021-11-08 14:40:36 +00:00
)
bzl_library(
name = "paths",
srcs = ["paths.bzl"],
deps = ["//lib/private:paths"],
)
bzl_library(
name = "utils",
srcs = ["utils.bzl"],
deps = ["//lib/private:utils"],
)
bzl_library(
name = "tar",
srcs = ["tar.bzl"],
deps = [
"//lib:expand_template",
"//lib:utils",
"//lib/private:tar",
"@bazel_skylib//lib:types",
],
)
2021-12-09 00:47:45 +00:00
bzl_library(
name = "jq",
srcs = ["jq.bzl"],
visibility = ["//visibility:public"],
deps = [
"//lib/private:jq",
],
2021-12-09 00:47:45 +00:00
)
bzl_library(
name = "directory_path",
srcs = ["directory_path.bzl"],
deps = ["//lib/private:directory_path"],
)
bzl_library(
name = "output_files",
srcs = ["output_files.bzl"],
deps = ["//lib/private:output_files"],
)
bzl_library(
name = "copy_file",
srcs = ["copy_file.bzl"],
deps = ["//lib/private:copy_file"],
)
2022-04-04 00:52:03 +00:00
bzl_library(
name = "copy_directory",
srcs = ["copy_directory.bzl"],
deps = ["//lib/private:copy_directory"],
2022-04-04 00:52:03 +00:00
)
bzl_library(
name = "copy_to_directory",
srcs = ["copy_to_directory.bzl"],
deps = ["//lib/private:copy_to_directory"],
2021-12-09 00:47:45 +00:00
)
2022-01-28 02:15:28 +00:00
2022-04-12 23:31:04 +00:00
bzl_library(
name = "copy_to_bin",
srcs = ["copy_to_bin.bzl"],
deps = ["//lib/private:copy_to_bin"],
)
#keep
bzl_library(
name = "diff_test",
srcs = ["diff_test.bzl"],
deps = ["//lib/private:diff_test"],
2022-04-12 23:31:04 +00:00
)
2022-01-28 02:15:28 +00:00
bzl_library(
name = "write_source_files",
srcs = ["write_source_files.bzl"],
deps = [
":diff_test",
2022-01-28 02:15:28 +00:00
":utils",
# "//lib/private:fail_with_message_test",
"//lib/private:write_source_file",
2022-01-28 02:15:28 +00:00
],
)
bzl_library(
name = "run_binary",
srcs = ["run_binary.bzl"],
deps = ["//lib/private:run_binary"],
)
bzl_library(
name = "repo_utils",
srcs = ["repo_utils.bzl"],
deps = [
"//lib/private:patch",
"//lib/private:repo_utils",
],
)
2022-04-20 04:45:06 +00:00
bzl_library(
name = "yq",
srcs = ["yq.bzl"],
deps = ["//lib/private:yq"],
2022-04-20 04:45:06 +00:00
)
2022-07-28 18:15:14 +00:00
bzl_library(
name = "glob_match",
srcs = ["glob_match.bzl"],
deps = ["//lib/private:glob_match"],
2022-07-28 18:15:14 +00:00
)
bzl_library(
name = "host_repo",
srcs = ["host_repo.bzl"],
deps = [
"//lib/private:host_repo",
"@bazel_skylib//lib:versions",
],
)
2022-07-15 05:32:39 +00:00
bzl_library(
name = "stamping",
srcs = ["stamping.bzl"],
deps = ["//lib/private:stamping"],
2022-07-15 05:32:39 +00:00
)
bzl_library(
name = "testing",
srcs = ["testing.bzl"],
deps = [
":diff_test",
":jq",
":params_file",
":utils",
"@bazel_skylib//lib:types",
"@bazel_skylib//rules:write_file",
],
)
bzl_library(
name = "extensions",
srcs = ["extensions.bzl"],
deps = ["@aspect_bazel_lib//lib:repositories"],
)
bzl_library(
name = "repositories",
srcs = ["repositories.bzl"],
deps = [
2022-10-27 22:55:57 +00:00
":utils",
"//lib/private:bats_toolchain",
"//lib/private:copy_directory_toolchain",
"//lib/private:copy_to_directory_toolchain",
"//lib/private:coreutils_toolchain",
"//lib/private:expand_template_toolchain",
"//lib/private:jq_toolchain",
"//lib/private:source_toolchains_repo",
"//lib/private:tar_toolchain",
"//lib/private:yq_toolchain",
2024-05-03 23:12:56 +00:00
"//lib/private:zstd_toolchain",
],
)
bzl_library(
name = "transitions",
srcs = ["transitions.bzl"],
deps = [
"@bazel_skylib//lib:paths",
],
)
bzl_library(
name = "platform_utils",
srcs = ["platform_utils.bzl"],
deps = ["//lib/private:platform_utils"],
)
bzl_library(
name = "base64",
srcs = ["base64.bzl"],
deps = ["//lib/private:base64"],
)
bzl_library(
name = "bazelrc_presets",
srcs = ["bazelrc_presets.bzl"],
deps = [":write_source_files"],
)
bzl_library(
name = "strings",
srcs = ["strings.bzl"],
deps = ["//lib/private:strings"],
2023-06-09 23:22:37 +00:00
)
bzl_library(
name = "lists",
srcs = ["lists.bzl"],
deps = ["//lib/private:lists"],
)
bzl_library(
name = "docs",
srcs = ["docs.bzl"],
deps = ["//lib/private:docs"],
)
bzl_library(
name = "windows_utils",
srcs = ["windows_utils.bzl"],
deps = ["//lib/private:paths"],
)
bzl_library(
name = "bats",
srcs = ["bats.bzl"],
deps = ["//lib/private:bats"],
)
bzl_library(
name = "resource_sets",
srcs = ["resource_sets.bzl"],
)