feat: yq (#80)
This commit is contained in:
Derek Cormier
GitHub
parent
e0af9633ed
commit
3e4024c785
|
|
@ -12,7 +12,11 @@ tasks:
|
|||
test_targets:
|
||||
- "//..."
|
||||
windows:
|
||||
build_flags:
|
||||
- "--build_tag_filters=-no-windows-ci"
|
||||
build_targets:
|
||||
- "//..."
|
||||
test_flags:
|
||||
- "--test_tag_filters=-no-windows-ci"
|
||||
test_targets:
|
||||
- "//..."
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
docs/*.md linguist-generated=true
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
docs/*.md
|
||||
lib/tests/jq/*.json
|
||||
lib/tests/yq/empty.yaml
|
||||
lib/lib/tests/write_source_files/*.js
|
||||
lib/lib/tests/write_source_files/subdir/*.js
|
||||
lib/lib/tests/write_source_files/subdir/subsubdir/*.js
|
||||
|
|
@ -83,4 +83,9 @@ stardoc_with_diff_test(
|
|||
bzl_library_target = "//lib:repo_utils",
|
||||
)
|
||||
|
||||
stardoc_with_diff_test(
|
||||
name = "yq",
|
||||
bzl_library_target = "//lib:yq",
|
||||
)
|
||||
|
||||
update_docs()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,114 @@
|
|||
<!-- Generated with Stardoc: http://skydoc.bazel.build -->
|
||||
|
||||
Public API for yq
|
||||
|
||||
<a id="#yq"></a>
|
||||
|
||||
## yq
|
||||
|
||||
<pre>
|
||||
yq(<a href="#yq-name">name</a>, <a href="#yq-srcs">srcs</a>, <a href="#yq-expression">expression</a>, <a href="#yq-args">args</a>, <a href="#yq-outs">outs</a>, <a href="#yq-kwargs">kwargs</a>)
|
||||
</pre>
|
||||
|
||||
Invoke yq with an expression on a set of input files.
|
||||
|
||||
For yq documentation, see https://mikefarah.gitbook.io/yq.
|
||||
|
||||
To use this rule you must register the yq toolchain in your WORKSPACE:
|
||||
|
||||
```starlark
|
||||
load("@aspect_bazel_lib//lib:repositories.bzl", "register_yq_toolchains")
|
||||
|
||||
register_yq_toolchains(version = "4.24.4")
|
||||
```
|
||||
|
||||
Usage examples:
|
||||
|
||||
```starlark
|
||||
load("@aspect_bazel_lib//lib:yq.bzl", "yq")
|
||||
```
|
||||
|
||||
```starlark
|
||||
# Remove fields
|
||||
yq(
|
||||
name = "safe-config",
|
||||
srcs = ["config.yaml"],
|
||||
filter = "del(.credentials)",
|
||||
)
|
||||
```
|
||||
|
||||
```starlark
|
||||
# Merge two yaml documents
|
||||
yq(
|
||||
name = "merged",
|
||||
srcs = [
|
||||
"a.yaml",
|
||||
"b.yaml",
|
||||
],
|
||||
expression = ". as $item ireduce ({}; . * $item )",
|
||||
)
|
||||
```
|
||||
|
||||
```starlark
|
||||
# Split a yaml file into several files
|
||||
yq(
|
||||
name = "split",
|
||||
srcs = ["multidoc.yaml"],
|
||||
outs = [
|
||||
"first.yml",
|
||||
"second.yml",
|
||||
],
|
||||
args = [
|
||||
"-s '.a'", # Split expression
|
||||
"--no-doc", # Exclude document separator --
|
||||
],
|
||||
)
|
||||
```
|
||||
|
||||
```starlark
|
||||
# Convert a yaml file to json
|
||||
yq(
|
||||
name = "convert-to-json",
|
||||
srcs = ["foo.yaml"],
|
||||
args = ["-o=json"],
|
||||
outs = ["foo.json"],
|
||||
)
|
||||
```
|
||||
|
||||
```starlark
|
||||
# Convert a json file to yaml
|
||||
yq(
|
||||
name = "convert",
|
||||
srcs = ["bar.json"],
|
||||
args = ["-P"],
|
||||
outs = ["bar.yaml"],
|
||||
)
|
||||
```
|
||||
|
||||
```starlark
|
||||
# Call yq in a genrule
|
||||
genrule(
|
||||
name = "generate",
|
||||
srcs = ["farm.yaml"],
|
||||
outs = ["genrule_output.yaml"],
|
||||
cmd = "$(YQ_BIN) '.moo = "cow"' $(location farm.yaml) > $@",
|
||||
toolchains = ["@yq_toolchains//:resolved_toolchain"],
|
||||
)
|
||||
```
|
||||
|
||||
yq is capable of parsing and outputting to other formats. See their [docs](https://mikefarah.gitbook.io/yq) for more examples.
|
||||
|
||||
|
||||
**PARAMETERS**
|
||||
|
||||
|
||||
| Name | Description | Default Value |
|
||||
| :------------- | :------------- | :------------- |
|
||||
| <a id="yq-name"></a>name | Name of the rule | none |
|
||||
| <a id="yq-srcs"></a>srcs | List of input file labels | none |
|
||||
| <a id="yq-expression"></a>expression | yq expression (https://mikefarah.gitbook.io/yq/commands/evaluate). Defaults to the identity expression "." | <code>"."</code> |
|
||||
| <a id="yq-args"></a>args | Additional args to pass to yq. Note that you do not need to pass _eval_ or _eval-all_ as this is handled automatically based on the number <code>srcs</code>. Passing the output format or the parse format is optional as these can be guessed based on the file extensions in <code>srcs</code> and <code>outs</code>. | <code>[]</code> |
|
||||
| <a id="yq-outs"></a>outs | Name of the output files. Defaults to a single output with the name plus a ".yaml" extension, or the extension corresponding to a passed output argment (e.g., "-o=json"). For split operations you must declare all outputs as the name of the output files depends on the expression. | <code>None</code> |
|
||||
| <a id="yq-kwargs"></a>kwargs | Other common named parameters such as <code>tags</code> or <code>visibility</code> | none |
|
||||
|
||||
|
||||
|
|
@ -6,7 +6,7 @@ statement from these, that's a bug in our distribution.
|
|||
|
||||
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
|
||||
load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
|
||||
load("//lib:repositories.bzl", "register_jq_toolchains")
|
||||
load("//lib:repositories.bzl", "register_jq_toolchains", "register_yq_toolchains")
|
||||
|
||||
# buildifier: disable=unnamed-macro
|
||||
def bazel_lib_internal_deps():
|
||||
|
|
@ -67,3 +67,4 @@ def bazel_lib_internal_deps():
|
|||
|
||||
# Register toolchains for tests
|
||||
register_jq_toolchains(version = "1.6")
|
||||
register_yq_toolchains(version = "4.24.5")
|
||||
|
|
|
|||
|
|
@ -13,6 +13,11 @@ toolchain_type(
|
|||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
toolchain_type(
|
||||
name = "yq_toolchain_type",
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
bzl_library(
|
||||
name = "docs",
|
||||
srcs = ["docs.bzl"],
|
||||
|
|
@ -144,3 +149,10 @@ bzl_library(
|
|||
"//lib/private:repo_utils",
|
||||
],
|
||||
)
|
||||
|
||||
bzl_library(
|
||||
name = "yq",
|
||||
srcs = ["yq.bzl"],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = ["//lib/private:yq"],
|
||||
)
|
||||
|
|
|
|||
|
|
@ -151,3 +151,9 @@ bzl_library(
|
|||
visibility = ["//lib:__subpackages__"],
|
||||
deps = [":repo_utils"],
|
||||
)
|
||||
|
||||
bzl_library(
|
||||
name = "yq",
|
||||
srcs = ["yq.bzl"],
|
||||
visibility = ["//lib:__subpackages__"],
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,69 @@
|
|||
"""Implementation for yq rule"""
|
||||
|
||||
_yq_attrs = {
|
||||
"srcs": attr.label_list(
|
||||
allow_files = [".yaml", ".json", ".xml"],
|
||||
mandatory = True,
|
||||
allow_empty = True,
|
||||
),
|
||||
"expression": attr.string(mandatory = False),
|
||||
"args": attr.string_list(),
|
||||
"outs": attr.output_list(mandatory = True),
|
||||
}
|
||||
|
||||
def is_split_operation(args):
|
||||
for arg in args:
|
||||
if arg.startswith("-s") or arg.startswith("--split-exp"):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _escape_path(path):
|
||||
return "/".join([".." for t in path.split("/")]) + "/"
|
||||
|
||||
def _yq_impl(ctx):
|
||||
yq_bin = ctx.toolchains["@aspect_bazel_lib//lib:yq_toolchain_type"].yqinfo.bin
|
||||
|
||||
outs = ctx.outputs.outs
|
||||
args = ctx.attr.args[:]
|
||||
inputs = ctx.files.srcs[:]
|
||||
|
||||
split_operation = is_split_operation(args)
|
||||
|
||||
if "eval" in args or "eval-all" in args:
|
||||
fail("Do not pass 'eval' or 'eval-all' into yq; this is already set based on the number of srcs")
|
||||
if not split_operation and len(outs) > 1:
|
||||
fail("Cannot specify multiple outputs when -s or --split-exp is not set")
|
||||
if "-i" in args or "--inplace" in args:
|
||||
fail("Cannot use arg -i or --inplace as it is not bazel-idiomatic to update the input file; consider using write_source_files to write back to the source tree")
|
||||
if len(ctx.attr.srcs) == 0 and "-n" not in args and "--null-input" not in args:
|
||||
args = args + ["--null-input"]
|
||||
|
||||
# For split operations, yq outputs files in the same directory so we
|
||||
# must cd to the correct output dir before executing it
|
||||
bin_dir = ctx.bin_dir.path + "/" + ctx.label.package
|
||||
escape_bin_dir = _escape_path(bin_dir)
|
||||
cmd = "cd {bin_dir} && {yq} {args} {eval_cmd} {expression} {sources} {maybe_out}".format(
|
||||
bin_dir = ctx.bin_dir.path + "/" + ctx.label.package,
|
||||
yq = escape_bin_dir + yq_bin.path,
|
||||
eval_cmd = "eval" if len(inputs) <= 1 else "eval-all",
|
||||
args = " ".join(args),
|
||||
expression = "'%s'" % ctx.attr.expression if ctx.attr.expression else "",
|
||||
sources = " ".join(["'%s%s'" % (escape_bin_dir, file.path) for file in ctx.files.srcs]),
|
||||
# In the -s/--split-exr case, the out file names are determined by the yq expression
|
||||
maybe_out = (" > %s%s" % (escape_bin_dir, outs[0].path)) if len(outs) == 1 else "",
|
||||
)
|
||||
|
||||
ctx.actions.run_shell(
|
||||
tools = [yq_bin],
|
||||
inputs = inputs,
|
||||
outputs = outs,
|
||||
command = cmd,
|
||||
mnemonic = "yq",
|
||||
)
|
||||
|
||||
return DefaultInfo(files = depset(outs), runfiles = ctx.runfiles(outs))
|
||||
|
||||
yq_lib = struct(
|
||||
attrs = _yq_attrs,
|
||||
implementation = _yq_impl,
|
||||
)
|
||||
|
|
@ -0,0 +1,207 @@
|
|||
"Setup yq toolchain repositories and rules"
|
||||
|
||||
YQ_PLATFORMS = {
|
||||
"darwin_amd64": struct(
|
||||
compatible_with = [
|
||||
"@platforms//os:macos",
|
||||
"@platforms//cpu:x86_64",
|
||||
],
|
||||
),
|
||||
"darwin_arm64": struct(
|
||||
compatible_with = [
|
||||
"@platforms//os:macos",
|
||||
"@platforms//cpu:aarch64",
|
||||
],
|
||||
),
|
||||
"linux_386": struct(
|
||||
compatible_with = [
|
||||
"@platforms//os:linux",
|
||||
"@platforms//cpu:x86_32",
|
||||
],
|
||||
),
|
||||
"linux_amd64": struct(
|
||||
compatible_with = [
|
||||
"@platforms//os:linux",
|
||||
"@platforms//cpu:x86_64",
|
||||
],
|
||||
),
|
||||
"windows_386": struct(
|
||||
compatible_with = [
|
||||
"@platforms//os:windows",
|
||||
"@platforms//cpu:x86_32",
|
||||
],
|
||||
),
|
||||
"windows_amd64": struct(
|
||||
compatible_with = [
|
||||
"@platforms//os:windows",
|
||||
"@platforms//cpu:x86_64",
|
||||
],
|
||||
),
|
||||
}
|
||||
|
||||
# https://github.com/mikefarah/yq/releases
|
||||
#
|
||||
# The integrity hashes can be automatically fetched for the latest yq release by running
|
||||
# tools/yq_mirror_release.sh. Alternatively, you can compute them manually by running
|
||||
# shasum -b -a 384 [downloaded file] | awk '{ print $1 }' | xxd -r -p | base64
|
||||
YQ_VERSIONS = {
|
||||
"v4.24.5": {
|
||||
"darwin_amd64": "sha384-Y6Utm9NAX7q69apRHLAU6oNYk5Kn5b6LUccBolbTm2CXXYye8pabeFPsaREFIHbw",
|
||||
"darwin_arm64": "sha384-d6+hFiZrsUeqnXJufnvadTi0BL/sfbd6K7LnJyLVDy31C0isjyHipVqlibKYbFSu",
|
||||
"linux_386": "sha384-skSDYmjm3uvi6xFKpzlIARzoiWaX0ml5CPAeLNxIybtRD3IBS1MSBoKkeWnS9n6h",
|
||||
"linux_amd64": "sha384-FEWzb66XTTiMfz5wA/hCs/n0N+PVj4lXzKX8ZIUXnM3JTlFlBvA9X59elqqEJUPq",
|
||||
"windows_386": "sha384-+BbsyeEO5BUN47u20qcwr0CGgVfo3Inj32BQsH6myca3C3hGqAE1nYVuy4JLBj+K",
|
||||
"windows_amd64": "sha384-6T42wIkqXZ8OCetIeMjTlTIVQDwlRpTXj8pi+SrGzU4r5waq3SwIYSrDqUxMD43j",
|
||||
},
|
||||
"v4.24.4": {
|
||||
"darwin_amd64": "sha384-H5JnUD7c0jpbOvvN1pGz12XFi3XrX+ism4iGnH9wv37i+qdkD2AdTbTe4MIFtMR+",
|
||||
"darwin_arm64": "sha384-9B85+dFTGRmMWWP2M+PVOkl8CtAb/HV4+XNGC0OBfdBvdJU85FyiTb12XGEgNjFp",
|
||||
"linux_386": "sha384-TiesqbEG9ITqnOyFNMilVnciVM65dCAlRNYp/pK19jrqs2x5MhbpJ0a7Q9XwZmz8",
|
||||
"linux_amd64": "sha384-y8vr5fWIqSvJhMoHwldoVPOJpAfLi4iHcnhfTcm/nuJAxGAJmI2MiBbk3t7lQNHC",
|
||||
"windows_386": "sha384-YJTz4Y+5rcy6Ii/J44Qb6J2JZuzfh40WHGTc6jFTHFhJ47Ht+y9s4bS6h8WX6S0m",
|
||||
"windows_amd64": "sha384-f8jkaz3oRaDcn8jiXupeDO665t6d2tTnFuU0bKwLWszXSz8r29My/USG+UoO9hOr",
|
||||
},
|
||||
}
|
||||
|
||||
YqInfo = provider(
|
||||
doc = "Provide info for executing yq",
|
||||
fields = {
|
||||
"bin": "Executable yq binary",
|
||||
},
|
||||
)
|
||||
|
||||
def _yq_toolchain_impl(ctx):
|
||||
binary = ctx.attr.bin.files.to_list()[0]
|
||||
|
||||
# Make the $(YQ_BIN) variable available in places like genrules.
|
||||
# See https://docs.bazel.build/versions/main/be/make-variables.html#custom_variables
|
||||
template_variables = platform_common.TemplateVariableInfo({
|
||||
"YQ_BIN": binary.path,
|
||||
})
|
||||
default_info = DefaultInfo(
|
||||
files = depset([binary]),
|
||||
runfiles = ctx.runfiles(files = [binary]),
|
||||
)
|
||||
yq_info = YqInfo(
|
||||
bin = binary,
|
||||
)
|
||||
|
||||
# Export all the providers inside our ToolchainInfo
|
||||
# so the resolved_toolchain rule can grab and re-export them.
|
||||
toolchain_info = platform_common.ToolchainInfo(
|
||||
yqinfo = yq_info,
|
||||
template_variables = template_variables,
|
||||
default = default_info,
|
||||
)
|
||||
|
||||
return [default_info, toolchain_info, template_variables]
|
||||
|
||||
yq_toolchain = rule(
|
||||
implementation = _yq_toolchain_impl,
|
||||
attrs = {
|
||||
"bin": attr.label(
|
||||
mandatory = True,
|
||||
allow_single_file = True,
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
def _yq_toolchains_repo_impl(repository_ctx):
|
||||
# Expose a concrete toolchain which is the result of Bazel resolving the toolchain
|
||||
# for the execution or target platform.
|
||||
# Workaround for https://github.com/bazelbuild/bazel/issues/14009
|
||||
starlark_content = """# Generated by @aspect_bazel_lib//lib/private/yq_toolchain.bzl
|
||||
|
||||
# Forward all the providers
|
||||
def _resolved_toolchain_impl(ctx):
|
||||
toolchain_info = ctx.toolchains["@aspect_bazel_lib//lib:yq_toolchain_type"]
|
||||
return [
|
||||
toolchain_info,
|
||||
toolchain_info.default,
|
||||
toolchain_info.yqinfo,
|
||||
toolchain_info.template_variables,
|
||||
]
|
||||
|
||||
# Copied from java_toolchain_alias
|
||||
# https://cs.opensource.google/bazel/bazel/+/master:tools/jdk/java_toolchain_alias.bzl
|
||||
resolved_toolchain = rule(
|
||||
implementation = _resolved_toolchain_impl,
|
||||
toolchains = ["@aspect_bazel_lib//lib:yq_toolchain_type"],
|
||||
incompatible_use_toolchain_transition = True,
|
||||
)
|
||||
"""
|
||||
repository_ctx.file("defs.bzl", starlark_content)
|
||||
|
||||
build_content = """# Generated by @aspect_bazel_lib//lib/private/yq_toolchain.bzl
|
||||
#
|
||||
# These can be registered in the workspace file or passed to --extra_toolchains flag.
|
||||
# By default all these toolchains are registered by the yq_register_toolchains macro
|
||||
# so you don't normally need to interact with these targets.
|
||||
|
||||
load(":defs.bzl", "resolved_toolchain")
|
||||
|
||||
resolved_toolchain(name = "resolved_toolchain", visibility = ["//visibility:public"])
|
||||
|
||||
"""
|
||||
|
||||
for [platform, meta] in YQ_PLATFORMS.items():
|
||||
build_content += """
|
||||
toolchain(
|
||||
name = "{platform}_toolchain",
|
||||
exec_compatible_with = {compatible_with},
|
||||
target_compatible_with = {compatible_with},
|
||||
toolchain = "@{name}_{platform}//:yq_toolchain",
|
||||
toolchain_type = "@aspect_bazel_lib//lib:yq_toolchain_type",
|
||||
)
|
||||
""".format(
|
||||
platform = platform,
|
||||
name = repository_ctx.attr.name,
|
||||
user_repository_name = repository_ctx.attr.user_repository_name,
|
||||
compatible_with = meta.compatible_with,
|
||||
)
|
||||
|
||||
# Base BUILD file for this repository
|
||||
repository_ctx.file("BUILD.bazel", build_content)
|
||||
|
||||
yq_toolchains_repo = repository_rule(
|
||||
_yq_toolchains_repo_impl,
|
||||
doc = """Creates a repository with toolchain definitions for all known platforms
|
||||
which can be registered or selected.""",
|
||||
attrs = {
|
||||
"user_repository_name": attr.string(doc = "Base name for toolchains repository"),
|
||||
},
|
||||
)
|
||||
|
||||
def _yq_platform_repo_impl(repository_ctx):
|
||||
is_windows = repository_ctx.attr.platform == "windows_386" or repository_ctx.attr.platform == "windows_amd64"
|
||||
|
||||
#https://github.com/mikefarah/yq/releases/download/v4.24.4/yq_linux_386
|
||||
url = "https://github.com/mikefarah/yq/releases/download/{0}/yq_{1}{2}".format(
|
||||
repository_ctx.attr.yq_version,
|
||||
repository_ctx.attr.platform,
|
||||
".exe" if is_windows else "",
|
||||
)
|
||||
|
||||
repository_ctx.download(
|
||||
url = url,
|
||||
output = "yq.exe" if is_windows else "yq",
|
||||
executable = True,
|
||||
integrity = YQ_VERSIONS[repository_ctx.attr.yq_version][repository_ctx.attr.platform],
|
||||
)
|
||||
build_content = """#Generated by @aspect_bazel_lib//lib/private/yq_toolchain.bzl
|
||||
load("@aspect_bazel_lib//lib/private:yq_toolchain.bzl", "yq_toolchain")
|
||||
exports_files(["{0}"])
|
||||
yq_toolchain(name = "yq_toolchain", bin = "{0}", visibility = ["//visibility:public"])
|
||||
""".format("yq.exe" if is_windows else "yq")
|
||||
|
||||
# Base BUILD file for this repository
|
||||
repository_ctx.file("BUILD.bazel", build_content)
|
||||
|
||||
yq_platform_repo = repository_rule(
|
||||
implementation = _yq_platform_repo_impl,
|
||||
doc = "Fetch external tools needed for yq toolchain",
|
||||
attrs = {
|
||||
"yq_version": attr.string(mandatory = True, values = YQ_VERSIONS.keys()),
|
||||
"platform": attr.string(mandatory = True, values = YQ_PLATFORMS.keys()),
|
||||
},
|
||||
)
|
||||
|
|
@ -3,6 +3,7 @@
|
|||
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
|
||||
load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
|
||||
load("//lib/private:jq_toolchain.bzl", "JQ_PLATFORMS", "jq_platform_repo", "jq_toolchains_repo")
|
||||
load("//lib/private:yq_toolchain.bzl", "YQ_PLATFORMS", "yq_platform_repo", "yq_toolchains_repo")
|
||||
|
||||
def aspect_bazel_lib_dependencies():
|
||||
"Load dependencies required by aspect rules"
|
||||
|
|
@ -34,3 +35,22 @@ def register_jq_toolchains(version, name = "jq"):
|
|||
jq_toolchains_repo(
|
||||
name = "%s_toolchains" % name,
|
||||
)
|
||||
|
||||
def register_yq_toolchains(version, name = "yq"):
|
||||
"""Registers yq toolchain and repositories
|
||||
|
||||
Args:
|
||||
version: the version of yq to execute (see https://github.com/mikefarah/yq/releases)
|
||||
name: override the prefix for the generated toolchain repositories
|
||||
"""
|
||||
for platform in YQ_PLATFORMS.keys():
|
||||
yq_platform_repo(
|
||||
name = "%s_toolchains_%s" % (name, platform),
|
||||
platform = platform,
|
||||
yq_version = "v%s" % version,
|
||||
)
|
||||
native.register_toolchains("@%s_toolchains//:%s_toolchain" % (name, platform))
|
||||
|
||||
yq_toolchains_repo(
|
||||
name = "%s_toolchains" % name,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,12 @@
|
|||
load("//lib/tests/jq:diff_test.bzl", "diff_test")
|
||||
load("//lib:jq.bzl", "jq")
|
||||
|
||||
exports_files([
|
||||
"a_pretty.json",
|
||||
"a.json",
|
||||
"b.json",
|
||||
])
|
||||
|
||||
# Identity filter produces identical json
|
||||
jq(
|
||||
name = "case_dot_filter",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,333 @@
|
|||
load("//lib/private:diff_test.bzl", "diff_test")
|
||||
load("//lib:yq.bzl", "yq")
|
||||
|
||||
# Identity (dot) expression produces identical yaml
|
||||
yq(
|
||||
name = "case_dot_expression",
|
||||
srcs = ["a.yaml"],
|
||||
expression = ".",
|
||||
)
|
||||
|
||||
diff_test(
|
||||
name = "case_dot_expression_test",
|
||||
file1 = "a.yaml",
|
||||
file2 = ":case_dot_expression",
|
||||
)
|
||||
|
||||
# No expression same as dot expression
|
||||
yq(
|
||||
name = "case_no_expression",
|
||||
srcs = ["a.yaml"],
|
||||
)
|
||||
|
||||
diff_test(
|
||||
name = "case_no_expression_test",
|
||||
file1 = "a.yaml",
|
||||
file2 = ":case_no_expression",
|
||||
)
|
||||
|
||||
# Output json, no out declared
|
||||
yq(
|
||||
name = "case_json_output_no_out",
|
||||
srcs = ["a.yaml"],
|
||||
args = ["-o=json"],
|
||||
expression = ".",
|
||||
)
|
||||
|
||||
diff_test(
|
||||
name = "case_json_output_no_out_test",
|
||||
file1 = "//lib/tests/jq:a_pretty.json",
|
||||
file2 = "case_json_output_no_out.json",
|
||||
)
|
||||
|
||||
# Output json, outs has ".json" extension but "-o=json" not set
|
||||
yq(
|
||||
name = "case_json_output_no_arg",
|
||||
srcs = ["a.yaml"],
|
||||
outs = ["case_json_output_no_arg.json"],
|
||||
args = [],
|
||||
expression = ".",
|
||||
)
|
||||
|
||||
diff_test(
|
||||
name = "case_json_output_no_arg_test",
|
||||
file1 = "//lib/tests/jq:a_pretty.json",
|
||||
file2 = ":case_json_output_no_arg",
|
||||
)
|
||||
|
||||
# Convert json to yaml
|
||||
yq(
|
||||
name = "case_convert_json_to_yaml",
|
||||
srcs = ["//lib/tests/jq:a_pretty.json"],
|
||||
args = ["-P"],
|
||||
expression = ".",
|
||||
)
|
||||
|
||||
diff_test(
|
||||
name = "case_convert_json_to_yaml_test",
|
||||
file1 = "a.yaml",
|
||||
file2 = ":case_convert_json_to_yaml",
|
||||
)
|
||||
|
||||
# No srcs, output is a generated expression
|
||||
yq(
|
||||
name = "case_generate_from_expression",
|
||||
srcs = [],
|
||||
expression = ".a.b.c = \"cat\"",
|
||||
# yq hangs without input srcs (https://github.com/mikefarah/yq/issues/1177)
|
||||
tags = ["no-windows-ci"],
|
||||
)
|
||||
|
||||
diff_test(
|
||||
name = "case_generate_from_expression_test",
|
||||
file1 = "generated-from-expression.yaml",
|
||||
file2 = ":case_generate_from_expression",
|
||||
# yq hangs without input srcs (https://github.com/mikefarah/yq/issues/1177)
|
||||
tags = ["no-windows-ci"],
|
||||
)
|
||||
|
||||
# No sources produces empty file (equivalent to --null-input)
|
||||
yq(
|
||||
name = "case_no_sources",
|
||||
srcs = [],
|
||||
expression = ".",
|
||||
# yq hangs without input srcs (https://github.com/mikefarah/yq/issues/1177)
|
||||
tags = ["no-windows-ci"],
|
||||
)
|
||||
|
||||
diff_test(
|
||||
name = "case_no_sources_test",
|
||||
file1 = ":case_no_sources",
|
||||
file2 = "empty.yaml",
|
||||
# yq hangs without input srcs (https://github.com/mikefarah/yq/issues/1177)
|
||||
tags = ["no-windows-ci"],
|
||||
)
|
||||
|
||||
# Merge two documents together
|
||||
yq(
|
||||
name = "case_merge_expression",
|
||||
srcs = [
|
||||
"a.yaml",
|
||||
"b.yaml",
|
||||
],
|
||||
expression = "select(fileIndex == 0) * select(fileIndex == 1)",
|
||||
)
|
||||
|
||||
diff_test(
|
||||
name = "case_merge_expression_test",
|
||||
file1 = "a_b_merged.yaml",
|
||||
file2 = ":case_merge_expression",
|
||||
)
|
||||
|
||||
# Merge two documents together (alt syntax)
|
||||
yq(
|
||||
name = "case_merge_expression_alt",
|
||||
srcs = [
|
||||
"a.yaml",
|
||||
"b.yaml",
|
||||
],
|
||||
expression = ". as $item ireduce ({}; . * $item )",
|
||||
# TODO: figure out why this doesn't work on windows (may be related to https://github.com/mikefarah/yq/issues/747)
|
||||
tags = ["no-windows-ci"],
|
||||
)
|
||||
|
||||
diff_test(
|
||||
name = "case_merge_expression_alt_test",
|
||||
file1 = "a_b_merged.yaml",
|
||||
file2 = ":case_merge_expression_alt",
|
||||
# TODO: figure out why this doesn't work on windows (may be related to https://github.com/mikefarah/yq/issues/747)
|
||||
tags = ["no-windows-ci"],
|
||||
)
|
||||
|
||||
# Split into multiple documents
|
||||
yq(
|
||||
name = "case_split_expression",
|
||||
srcs = ["multidoc.yaml"],
|
||||
outs = [
|
||||
"test_doc1.yml",
|
||||
"test_doc2.yml",
|
||||
],
|
||||
args = [
|
||||
"-s '.a'",
|
||||
"--no-doc",
|
||||
],
|
||||
expression = ".",
|
||||
)
|
||||
|
||||
diff_test(
|
||||
name = "case_split_expression_test_1",
|
||||
file1 = "split1.yaml",
|
||||
file2 = "test_doc1.yml",
|
||||
)
|
||||
|
||||
diff_test(
|
||||
name = "case_split_expression_test_2",
|
||||
file1 = "split2.yaml",
|
||||
file2 = "test_doc2.yml",
|
||||
)
|
||||
|
||||
# Outputs properties file
|
||||
yq(
|
||||
name = "case_output_properties",
|
||||
srcs = ["a.yaml"],
|
||||
outs = ["case_output_properties.properties"],
|
||||
args = ["-o=props"],
|
||||
expression = ".",
|
||||
)
|
||||
|
||||
diff_test(
|
||||
name = "case_output_properties_test",
|
||||
file1 = "a.properties",
|
||||
file2 = ":case_output_properties",
|
||||
)
|
||||
|
||||
# Outputs properties file, outs not declared
|
||||
yq(
|
||||
name = "case_output_properties_no_outs",
|
||||
srcs = ["a.yaml"],
|
||||
args = ["-o=props"],
|
||||
expression = ".",
|
||||
)
|
||||
|
||||
diff_test(
|
||||
name = "case_output_properties_no_outs_test",
|
||||
file1 = "a.properties",
|
||||
file2 = ":case_output_properties_no_outs",
|
||||
)
|
||||
|
||||
# Outputs csv file
|
||||
yq(
|
||||
name = "case_output_csv",
|
||||
srcs = ["array.yaml"],
|
||||
outs = ["case_output_csv.csv"],
|
||||
args = ["-o=c"],
|
||||
expression = ".",
|
||||
)
|
||||
|
||||
diff_test(
|
||||
name = "case_output_csv_test",
|
||||
file1 = "array.csv",
|
||||
file2 = ":case_output_csv",
|
||||
)
|
||||
|
||||
# Outputs csv file, outs not declared
|
||||
yq(
|
||||
name = "case_output_csv_no_outs",
|
||||
srcs = ["array.yaml"],
|
||||
args = ["-o=c"],
|
||||
expression = ".",
|
||||
)
|
||||
|
||||
diff_test(
|
||||
name = "case_output_csv_no_outs_test",
|
||||
file1 = "array.csv",
|
||||
file2 = ":case_output_csv_no_outs",
|
||||
)
|
||||
|
||||
# Outputs tsv file
|
||||
yq(
|
||||
name = "case_output_tsv",
|
||||
srcs = ["array.yaml"],
|
||||
outs = ["case_output_tsv.tsv"],
|
||||
args = ["-o=t"],
|
||||
expression = ".",
|
||||
)
|
||||
|
||||
diff_test(
|
||||
name = "case_output_tsv_test",
|
||||
file1 = "array.tsv",
|
||||
file2 = ":case_output_tsv",
|
||||
)
|
||||
|
||||
# Outputs tsv file, outs not declared
|
||||
yq(
|
||||
name = "case_output_tsv_no_outs",
|
||||
srcs = ["array.yaml"],
|
||||
args = ["-o=t"],
|
||||
expression = ".",
|
||||
)
|
||||
|
||||
diff_test(
|
||||
name = "case_output_tsv_no_outs_test",
|
||||
file1 = "array.tsv",
|
||||
file2 = ":case_output_tsv_no_outs",
|
||||
)
|
||||
|
||||
# Convert xml to yaml
|
||||
yq(
|
||||
name = "case_convert_xml_to_yaml",
|
||||
srcs = ["sample.xml"],
|
||||
args = ["-p=xml"],
|
||||
expression = ".",
|
||||
)
|
||||
|
||||
diff_test(
|
||||
name = "case_convert_xml_to_yaml_test",
|
||||
file1 = "sample.yaml",
|
||||
file2 = ":case_convert_xml_to_yaml",
|
||||
)
|
||||
|
||||
# Outputs xml file
|
||||
yq(
|
||||
name = "case_output_xml",
|
||||
srcs = ["a.yaml"],
|
||||
outs = ["case_output_xml.xml"],
|
||||
args = ["-o=xml"],
|
||||
expression = ".",
|
||||
)
|
||||
|
||||
diff_test(
|
||||
name = "case_output_xml_test",
|
||||
file1 = "a.xml",
|
||||
file2 = ":case_output_xml",
|
||||
)
|
||||
|
||||
# Outputs xml file, outs not declared
|
||||
yq(
|
||||
name = "case_output_xml_no_outs",
|
||||
srcs = ["a.yaml"],
|
||||
args = ["-o=xml"],
|
||||
expression = ".",
|
||||
)
|
||||
|
||||
diff_test(
|
||||
name = "case_output_xml_no_outs_test",
|
||||
file1 = "a.xml",
|
||||
file2 = ":case_output_xml_no_outs",
|
||||
)
|
||||
|
||||
# Merge two json documents together
|
||||
yq(
|
||||
name = "case_merge_expression_json",
|
||||
srcs = [
|
||||
"//lib/tests/jq:a.json",
|
||||
"//lib/tests/jq:b.json",
|
||||
],
|
||||
args = ["-P"],
|
||||
expression = ". as $item ireduce ({}; . * $item )",
|
||||
# TODO: figure out why this doesn't work on windows (may be related to https://github.com/mikefarah/yq/issues/747)
|
||||
tags = ["no-windows-ci"],
|
||||
)
|
||||
|
||||
diff_test(
|
||||
name = "case_merge_expression_json_test",
|
||||
file1 = "a_b_merged.yaml",
|
||||
file2 = ":case_merge_expression_json",
|
||||
# TODO: figure out why this doesn't work on windows (may be related to https://github.com/mikefarah/yq/issues/747)
|
||||
tags = ["no-windows-ci"],
|
||||
)
|
||||
|
||||
# Call yq within a genrule
|
||||
genrule(
|
||||
name = "case_genrule",
|
||||
srcs = ["a.yaml"],
|
||||
outs = ["genrule_output.yaml"],
|
||||
cmd = "$(YQ_BIN) '.' $(location a.yaml) > $@",
|
||||
toolchains = ["@yq_toolchains//:resolved_toolchain"],
|
||||
)
|
||||
|
||||
diff_test(
|
||||
name = "case_genrule_test",
|
||||
file1 = "genrule_output.yaml",
|
||||
file2 = "a.yaml",
|
||||
)
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
foo = bar
|
||||
value = 123
|
||||
moo.0 = 1
|
||||
moo.1 = 2
|
||||
moo.2 = 3
|
||||
a = true
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
<foo>bar</foo>
|
||||
<value>123</value>
|
||||
<moo>1</moo>
|
||||
<moo>2</moo>
|
||||
<moo>3</moo>
|
||||
<a>true</a>
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
foo: bar
|
||||
value: 123
|
||||
moo:
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
||||
a: true
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
foo: baz
|
||||
value: 456
|
||||
moo:
|
||||
- 4
|
||||
- 5
|
||||
- 6
|
||||
a: true
|
||||
b: true
|
||||
|
|
@ -0,0 +1 @@
|
|||
1,2,3,4
|
||||
|
|
|
@ -0,0 +1 @@
|
|||
1 2 3 4
|
||||
|
|
|
@ -0,0 +1 @@
|
|||
[1, 2, 3, 4]
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
foo: baz
|
||||
value: 456
|
||||
moo:
|
||||
- 4
|
||||
- 5
|
||||
- 6
|
||||
b: true
|
||||
|
|
@ -0,0 +1 @@
|
|||
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
a:
|
||||
b:
|
||||
c: cat
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
a: test_doc1
|
||||
---
|
||||
a: test_doc2
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<cat>
|
||||
<says>meow</says>
|
||||
<legs>4</legs>
|
||||
<cute>true</cute>
|
||||
</cat>
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
cat:
|
||||
says: meow
|
||||
legs: "4"
|
||||
cute: "true"
|
||||
|
|
@ -0,0 +1 @@
|
|||
a: test_doc1
|
||||
|
|
@ -0,0 +1 @@
|
|||
a: test_doc2
|
||||
|
|
@ -0,0 +1,160 @@
|
|||
"""Public API for yq"""
|
||||
|
||||
load("//lib/private:yq.bzl", _is_split_operation = "is_split_operation", _yq_lib = "yq_lib")
|
||||
|
||||
_yq_rule = rule(
|
||||
attrs = _yq_lib.attrs,
|
||||
implementation = _yq_lib.implementation,
|
||||
toolchains = ["@aspect_bazel_lib//lib:yq_toolchain_type"],
|
||||
)
|
||||
|
||||
def yq(name, srcs, expression = ".", args = [], outs = None, **kwargs):
|
||||
"""Invoke yq with an expression on a set of input files.
|
||||
|
||||
For yq documentation, see https://mikefarah.gitbook.io/yq.
|
||||
|
||||
To use this rule you must register the yq toolchain in your WORKSPACE:
|
||||
|
||||
```starlark
|
||||
load("@aspect_bazel_lib//lib:repositories.bzl", "register_yq_toolchains")
|
||||
|
||||
register_yq_toolchains(version = "4.24.4")
|
||||
```
|
||||
|
||||
Usage examples:
|
||||
|
||||
```starlark
|
||||
load("@aspect_bazel_lib//lib:yq.bzl", "yq")
|
||||
```
|
||||
|
||||
```starlark
|
||||
# Remove fields
|
||||
yq(
|
||||
name = "safe-config",
|
||||
srcs = ["config.yaml"],
|
||||
filter = "del(.credentials)",
|
||||
)
|
||||
```
|
||||
|
||||
```starlark
|
||||
# Merge two yaml documents
|
||||
yq(
|
||||
name = "merged",
|
||||
srcs = [
|
||||
"a.yaml",
|
||||
"b.yaml",
|
||||
],
|
||||
expression = ". as $item ireduce ({}; . * $item )",
|
||||
)
|
||||
```
|
||||
|
||||
```starlark
|
||||
# Split a yaml file into several files
|
||||
yq(
|
||||
name = "split",
|
||||
srcs = ["multidoc.yaml"],
|
||||
outs = [
|
||||
"first.yml",
|
||||
"second.yml",
|
||||
],
|
||||
args = [
|
||||
"-s '.a'", # Split expression
|
||||
"--no-doc", # Exclude document separator --
|
||||
],
|
||||
)
|
||||
```
|
||||
|
||||
```starlark
|
||||
# Convert a yaml file to json
|
||||
yq(
|
||||
name = "convert-to-json",
|
||||
srcs = ["foo.yaml"],
|
||||
args = ["-o=json"],
|
||||
outs = ["foo.json"],
|
||||
)
|
||||
```
|
||||
|
||||
```starlark
|
||||
# Convert a json file to yaml
|
||||
yq(
|
||||
name = "convert",
|
||||
srcs = ["bar.json"],
|
||||
args = ["-P"],
|
||||
outs = ["bar.yaml"],
|
||||
)
|
||||
```
|
||||
|
||||
```starlark
|
||||
# Call yq in a genrule
|
||||
genrule(
|
||||
name = "generate",
|
||||
srcs = ["farm.yaml"],
|
||||
outs = ["genrule_output.yaml"],
|
||||
cmd = "$(YQ_BIN) '.moo = \"cow\"' $(location farm.yaml) > $@",
|
||||
toolchains = ["@yq_toolchains//:resolved_toolchain"],
|
||||
)
|
||||
```
|
||||
|
||||
yq is capable of parsing and outputting to other formats. See their [docs](https://mikefarah.gitbook.io/yq) for more examples.
|
||||
|
||||
Args:
|
||||
name: Name of the rule
|
||||
srcs: List of input file labels
|
||||
expression: yq expression (https://mikefarah.gitbook.io/yq/commands/evaluate). Defaults to the identity
|
||||
expression "."
|
||||
args: Additional args to pass to yq. Note that you do not need to pass _eval_ or _eval-all_ as this
|
||||
is handled automatically based on the number `srcs`. Passing the output format or the parse format
|
||||
is optional as these can be guessed based on the file extensions in `srcs` and `outs`.
|
||||
outs: Name of the output files. Defaults to a single output with the name plus a ".yaml" extension, or
|
||||
the extension corresponding to a passed output argment (e.g., "-o=json"). For split operations you
|
||||
must declare all outputs as the name of the output files depends on the expression.
|
||||
**kwargs: Other common named parameters such as `tags` or `visibility`
|
||||
"""
|
||||
args = args[:]
|
||||
|
||||
if not _is_split_operation(args):
|
||||
# For split operations we can't predeclare outs because the name of the resulting files
|
||||
# depends on the expression. For non-split operations, set a default output file name
|
||||
# based on the name and the output format passed, defaulting to yaml.
|
||||
if not outs:
|
||||
outs = [name + ".yaml"]
|
||||
if "-o=json" in args or "--outputformat=json" in args:
|
||||
outs = [name + ".json"]
|
||||
if "-o=xml" in args or "--outputformat=xml" in args:
|
||||
outs = [name + ".xml"]
|
||||
elif "-o=props" in args or "--outputformat=props" in args:
|
||||
outs = [name + ".properties"]
|
||||
elif "-o=c" in args or "--outputformat=csv" in args:
|
||||
outs = [name + ".csv"]
|
||||
elif "-o=t" in args or "--outputformat=tsv" in args:
|
||||
outs = [name + ".tsv"]
|
||||
|
||||
elif outs and len(outs) == 1:
|
||||
# If an output file with an extension was provided, try to set the corresponding output
|
||||
# argument if it wasn't already passed.
|
||||
if outs[0].endswith(".json") and "-o=json" not in args and "--outputformat=json" not in args:
|
||||
args.append("-o=json")
|
||||
elif outs[0].endswith(".xml") and "-o=xml" not in args and "--outputformat=xml" not in args:
|
||||
args.append("-o=xml")
|
||||
elif outs[0].endswith(".properties") and "-o=props" not in args and "--outputformat=props" not in args:
|
||||
args.append("-o=props")
|
||||
elif outs[0].endswith(".csv") and "-o=c" not in args and "--outputformat=csv" not in args:
|
||||
args.append("-o=c")
|
||||
elif outs[0].endswith(".tsv") and "-o=t" not in args and "--outputformat=tsv" not in args:
|
||||
args.append("-o=t")
|
||||
|
||||
# If the input files are json or xml, set the parse flag if it isn't already set
|
||||
if len(srcs) > 0:
|
||||
if srcs[0].endswith(".json") and "-P" not in args:
|
||||
args.append("-P")
|
||||
elif srcs[0].endswith(".xml") and "-p=xml" not in args:
|
||||
args.append("-p=xml")
|
||||
|
||||
_yq_rule(
|
||||
name = name,
|
||||
srcs = srcs,
|
||||
expression = expression,
|
||||
args = args,
|
||||
outs = outs,
|
||||
**kwargs
|
||||
)
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
#!/bin/bash/env bash
|
||||
# Produce a dictionary for the current yq release,
|
||||
# suitable for adding to lib/private/yq_toolchain.bzl
|
||||
|
||||
set -o errexit
|
||||
|
||||
# Find the latest version
|
||||
version=$(curl --silent "https://api.github.com/repos/mikefarah/yq/releases/latest" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/')
|
||||
|
||||
# yq publishes its checksums and a script to extract them
|
||||
curl --silent --location "https://github.com/mikefarah/yq/releases/download/$version/extract-checksum.sh" -o /tmp/extract-checksum.sh
|
||||
curl --silent --location "https://github.com/mikefarah/yq/releases/download/$version/checksums_hashes_order" -o /tmp/checksums_hashes_order
|
||||
curl --silent --location "https://github.com/mikefarah/yq/releases/download/$version/checksums" -o /tmp/checksums
|
||||
|
||||
cd /tmp
|
||||
chmod u+x extract-checksum.sh
|
||||
|
||||
# Extract the checksums and output a starlark map entry
|
||||
echo "\"$version\": {"
|
||||
for release in darwin_{amd,arm}64 linux_{386,amd64} windows_{386,amd64}; do
|
||||
artifact=$release
|
||||
if [[ $release == windows* ]]; then
|
||||
artifact="$release.exe"
|
||||
fi
|
||||
echo " \"$release\": \"$(./extract-checksum.sh SHA-384 $artifact | awk '{ print $2 }' | xxd -r -p | base64 | awk '{ print "sha384-" $1 }' )\","
|
||||
done
|
||||
echo "},"
|
||||
|
||||
printf "\n"
|
||||
echo "Paste the above into VERSIONS in yq_toolchain.bzl."
|
||||
Loading…
Reference in New Issue