From 3e4024c7854ddba9bc1ffc63f003cb9ae3c0b387 Mon Sep 17 00:00:00 2001 From: Derek Cormier Date: Tue, 19 Apr 2022 21:45:06 -0700 Subject: [PATCH] feat: yq (#80) --- .bazelci/presubmit.yml | 4 + .gitattributes | 1 + .prettierignore | 1 + docs/BUILD.bazel | 5 + docs/yq.md | 114 +++++++ internal_deps.bzl | 3 +- lib/BUILD.bazel | 12 + lib/private/BUILD.bazel | 6 + lib/private/yq.bzl | 69 ++++ lib/private/yq_toolchain.bzl | 207 ++++++++++++ lib/repositories.bzl | 20 ++ lib/tests/jq/BUILD.bazel | 6 + lib/tests/yq/BUILD.bazel | 333 ++++++++++++++++++++ lib/tests/yq/a.properties | 6 + lib/tests/yq/a.xml | 6 + lib/tests/yq/a.yaml | 7 + lib/tests/yq/a_b_merged.yaml | 8 + lib/tests/yq/array.csv | 1 + lib/tests/yq/array.tsv | 1 + lib/tests/yq/array.yaml | 1 + lib/tests/yq/b.yaml | 7 + lib/tests/yq/empty.yaml | 1 + lib/tests/yq/generated-from-expression.yaml | 3 + lib/tests/yq/multidoc.yaml | 3 + lib/tests/yq/sample.xml | 6 + lib/tests/yq/sample.yaml | 4 + lib/tests/yq/split1.yaml | 1 + lib/tests/yq/split2.yaml | 1 + lib/yq.bzl | 160 ++++++++++ tools/yq_mirror_release.sh | 30 ++ 30 files changed, 1026 insertions(+), 1 deletion(-) create mode 100644 .gitattributes create mode 100755 docs/yq.md create mode 100644 lib/private/yq.bzl create mode 100644 lib/private/yq_toolchain.bzl create mode 100644 lib/tests/yq/BUILD.bazel create mode 100644 lib/tests/yq/a.properties create mode 100644 lib/tests/yq/a.xml create mode 100644 lib/tests/yq/a.yaml create mode 100644 lib/tests/yq/a_b_merged.yaml create mode 100644 lib/tests/yq/array.csv create mode 100644 lib/tests/yq/array.tsv create mode 100644 lib/tests/yq/array.yaml create mode 100644 lib/tests/yq/b.yaml create mode 100644 lib/tests/yq/empty.yaml create mode 100644 lib/tests/yq/generated-from-expression.yaml create mode 100644 lib/tests/yq/multidoc.yaml create mode 100644 lib/tests/yq/sample.xml create mode 100644 lib/tests/yq/sample.yaml create mode 100644 lib/tests/yq/split1.yaml create mode 100644 lib/tests/yq/split2.yaml create mode 100644 lib/yq.bzl create mode 100644 tools/yq_mirror_release.sh diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 576fb4b..e8953d9 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -12,7 +12,11 @@ tasks: test_targets: - "//..." windows: + build_flags: + - "--build_tag_filters=-no-windows-ci" build_targets: - "//..." + test_flags: + - "--test_tag_filters=-no-windows-ci" test_targets: - "//..." diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..fb496ed --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +docs/*.md linguist-generated=true diff --git a/.prettierignore b/.prettierignore index e0dc6af..5e4027f 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,5 +1,6 @@ docs/*.md lib/tests/jq/*.json +lib/tests/yq/empty.yaml lib/lib/tests/write_source_files/*.js lib/lib/tests/write_source_files/subdir/*.js lib/lib/tests/write_source_files/subdir/subsubdir/*.js \ No newline at end of file diff --git a/docs/BUILD.bazel b/docs/BUILD.bazel index 0610043..b92c33d 100644 --- a/docs/BUILD.bazel +++ b/docs/BUILD.bazel @@ -83,4 +83,9 @@ stardoc_with_diff_test( bzl_library_target = "//lib:repo_utils", ) +stardoc_with_diff_test( + name = "yq", + bzl_library_target = "//lib:yq", +) + update_docs() diff --git a/docs/yq.md b/docs/yq.md new file mode 100755 index 0000000..ab84a5b --- /dev/null +++ b/docs/yq.md @@ -0,0 +1,114 @@ + + +Public API for yq + + + +## yq + +
+yq(name, srcs, expression, args, outs, kwargs)
+
+ +Invoke yq with an expression on a set of input files. + +For yq documentation, see https://mikefarah.gitbook.io/yq. + +To use this rule you must register the yq toolchain in your WORKSPACE: + +```starlark +load("@aspect_bazel_lib//lib:repositories.bzl", "register_yq_toolchains") + +register_yq_toolchains(version = "4.24.4") +``` + +Usage examples: + +```starlark +load("@aspect_bazel_lib//lib:yq.bzl", "yq") +``` + +```starlark +# Remove fields +yq( + name = "safe-config", + srcs = ["config.yaml"], + filter = "del(.credentials)", +) +``` + +```starlark +# Merge two yaml documents +yq( + name = "merged", + srcs = [ + "a.yaml", + "b.yaml", + ], + expression = ". as $item ireduce ({}; . * $item )", +) +``` + +```starlark +# Split a yaml file into several files +yq( + name = "split", + srcs = ["multidoc.yaml"], + outs = [ + "first.yml", + "second.yml", + ], + args = [ + "-s '.a'", # Split expression + "--no-doc", # Exclude document separator -- + ], +) +``` + +```starlark +# Convert a yaml file to json +yq( + name = "convert-to-json", + srcs = ["foo.yaml"], + args = ["-o=json"], + outs = ["foo.json"], +) +``` + +```starlark +# Convert a json file to yaml +yq( + name = "convert", + srcs = ["bar.json"], + args = ["-P"], + outs = ["bar.yaml"], +) +``` + +```starlark +# Call yq in a genrule +genrule( + name = "generate", + srcs = ["farm.yaml"], + outs = ["genrule_output.yaml"], + cmd = "$(YQ_BIN) '.moo = "cow"' $(location farm.yaml) > $@", + toolchains = ["@yq_toolchains//:resolved_toolchain"], +) +``` + +yq is capable of parsing and outputting to other formats. See their [docs](https://mikefarah.gitbook.io/yq) for more examples. + + +**PARAMETERS** + + +| Name | Description | Default Value | +| :------------- | :------------- | :------------- | +| name | Name of the rule | none | +| srcs | List of input file labels | none | +| expression | yq expression (https://mikefarah.gitbook.io/yq/commands/evaluate). Defaults to the identity expression "." | "." | +| args | Additional args to pass to yq. Note that you do not need to pass _eval_ or _eval-all_ as this is handled automatically based on the number srcs. Passing the output format or the parse format is optional as these can be guessed based on the file extensions in srcs and outs. | [] | +| outs | Name of the output files. Defaults to a single output with the name plus a ".yaml" extension, or the extension corresponding to a passed output argment (e.g., "-o=json"). For split operations you must declare all outputs as the name of the output files depends on the expression. | None | +| kwargs | Other common named parameters such as tags or visibility | none | + + diff --git a/internal_deps.bzl b/internal_deps.bzl index 1aee104..c607667 100644 --- a/internal_deps.bzl +++ b/internal_deps.bzl @@ -6,7 +6,7 @@ statement from these, that's a bug in our distribution. load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") -load("//lib:repositories.bzl", "register_jq_toolchains") +load("//lib:repositories.bzl", "register_jq_toolchains", "register_yq_toolchains") # buildifier: disable=unnamed-macro def bazel_lib_internal_deps(): @@ -67,3 +67,4 @@ def bazel_lib_internal_deps(): # Register toolchains for tests register_jq_toolchains(version = "1.6") + register_yq_toolchains(version = "4.24.5") diff --git a/lib/BUILD.bazel b/lib/BUILD.bazel index 75cddec..d837ca6 100644 --- a/lib/BUILD.bazel +++ b/lib/BUILD.bazel @@ -13,6 +13,11 @@ toolchain_type( visibility = ["//visibility:public"], ) +toolchain_type( + name = "yq_toolchain_type", + visibility = ["//visibility:public"], +) + bzl_library( name = "docs", srcs = ["docs.bzl"], @@ -144,3 +149,10 @@ bzl_library( "//lib/private:repo_utils", ], ) + +bzl_library( + name = "yq", + srcs = ["yq.bzl"], + visibility = ["//visibility:public"], + deps = ["//lib/private:yq"], +) diff --git a/lib/private/BUILD.bazel b/lib/private/BUILD.bazel index ba040b0..329faf0 100644 --- a/lib/private/BUILD.bazel +++ b/lib/private/BUILD.bazel @@ -151,3 +151,9 @@ bzl_library( visibility = ["//lib:__subpackages__"], deps = [":repo_utils"], ) + +bzl_library( + name = "yq", + srcs = ["yq.bzl"], + visibility = ["//lib:__subpackages__"], +) diff --git a/lib/private/yq.bzl b/lib/private/yq.bzl new file mode 100644 index 0000000..790eab2 --- /dev/null +++ b/lib/private/yq.bzl @@ -0,0 +1,69 @@ +"""Implementation for yq rule""" + +_yq_attrs = { + "srcs": attr.label_list( + allow_files = [".yaml", ".json", ".xml"], + mandatory = True, + allow_empty = True, + ), + "expression": attr.string(mandatory = False), + "args": attr.string_list(), + "outs": attr.output_list(mandatory = True), +} + +def is_split_operation(args): + for arg in args: + if arg.startswith("-s") or arg.startswith("--split-exp"): + return True + return False + +def _escape_path(path): + return "/".join([".." for t in path.split("/")]) + "/" + +def _yq_impl(ctx): + yq_bin = ctx.toolchains["@aspect_bazel_lib//lib:yq_toolchain_type"].yqinfo.bin + + outs = ctx.outputs.outs + args = ctx.attr.args[:] + inputs = ctx.files.srcs[:] + + split_operation = is_split_operation(args) + + if "eval" in args or "eval-all" in args: + fail("Do not pass 'eval' or 'eval-all' into yq; this is already set based on the number of srcs") + if not split_operation and len(outs) > 1: + fail("Cannot specify multiple outputs when -s or --split-exp is not set") + if "-i" in args or "--inplace" in args: + fail("Cannot use arg -i or --inplace as it is not bazel-idiomatic to update the input file; consider using write_source_files to write back to the source tree") + if len(ctx.attr.srcs) == 0 and "-n" not in args and "--null-input" not in args: + args = args + ["--null-input"] + + # For split operations, yq outputs files in the same directory so we + # must cd to the correct output dir before executing it + bin_dir = ctx.bin_dir.path + "/" + ctx.label.package + escape_bin_dir = _escape_path(bin_dir) + cmd = "cd {bin_dir} && {yq} {args} {eval_cmd} {expression} {sources} {maybe_out}".format( + bin_dir = ctx.bin_dir.path + "/" + ctx.label.package, + yq = escape_bin_dir + yq_bin.path, + eval_cmd = "eval" if len(inputs) <= 1 else "eval-all", + args = " ".join(args), + expression = "'%s'" % ctx.attr.expression if ctx.attr.expression else "", + sources = " ".join(["'%s%s'" % (escape_bin_dir, file.path) for file in ctx.files.srcs]), + # In the -s/--split-exr case, the out file names are determined by the yq expression + maybe_out = (" > %s%s" % (escape_bin_dir, outs[0].path)) if len(outs) == 1 else "", + ) + + ctx.actions.run_shell( + tools = [yq_bin], + inputs = inputs, + outputs = outs, + command = cmd, + mnemonic = "yq", + ) + + return DefaultInfo(files = depset(outs), runfiles = ctx.runfiles(outs)) + +yq_lib = struct( + attrs = _yq_attrs, + implementation = _yq_impl, +) diff --git a/lib/private/yq_toolchain.bzl b/lib/private/yq_toolchain.bzl new file mode 100644 index 0000000..547b3b6 --- /dev/null +++ b/lib/private/yq_toolchain.bzl @@ -0,0 +1,207 @@ +"Setup yq toolchain repositories and rules" + +YQ_PLATFORMS = { + "darwin_amd64": struct( + compatible_with = [ + "@platforms//os:macos", + "@platforms//cpu:x86_64", + ], + ), + "darwin_arm64": struct( + compatible_with = [ + "@platforms//os:macos", + "@platforms//cpu:aarch64", + ], + ), + "linux_386": struct( + compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_32", + ], + ), + "linux_amd64": struct( + compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + ), + "windows_386": struct( + compatible_with = [ + "@platforms//os:windows", + "@platforms//cpu:x86_32", + ], + ), + "windows_amd64": struct( + compatible_with = [ + "@platforms//os:windows", + "@platforms//cpu:x86_64", + ], + ), +} + +# https://github.com/mikefarah/yq/releases +# +# The integrity hashes can be automatically fetched for the latest yq release by running +# tools/yq_mirror_release.sh. Alternatively, you can compute them manually by running +# shasum -b -a 384 [downloaded file] | awk '{ print $1 }' | xxd -r -p | base64 +YQ_VERSIONS = { + "v4.24.5": { + "darwin_amd64": "sha384-Y6Utm9NAX7q69apRHLAU6oNYk5Kn5b6LUccBolbTm2CXXYye8pabeFPsaREFIHbw", + "darwin_arm64": "sha384-d6+hFiZrsUeqnXJufnvadTi0BL/sfbd6K7LnJyLVDy31C0isjyHipVqlibKYbFSu", + "linux_386": "sha384-skSDYmjm3uvi6xFKpzlIARzoiWaX0ml5CPAeLNxIybtRD3IBS1MSBoKkeWnS9n6h", + "linux_amd64": "sha384-FEWzb66XTTiMfz5wA/hCs/n0N+PVj4lXzKX8ZIUXnM3JTlFlBvA9X59elqqEJUPq", + "windows_386": "sha384-+BbsyeEO5BUN47u20qcwr0CGgVfo3Inj32BQsH6myca3C3hGqAE1nYVuy4JLBj+K", + "windows_amd64": "sha384-6T42wIkqXZ8OCetIeMjTlTIVQDwlRpTXj8pi+SrGzU4r5waq3SwIYSrDqUxMD43j", + }, + "v4.24.4": { + "darwin_amd64": "sha384-H5JnUD7c0jpbOvvN1pGz12XFi3XrX+ism4iGnH9wv37i+qdkD2AdTbTe4MIFtMR+", + "darwin_arm64": "sha384-9B85+dFTGRmMWWP2M+PVOkl8CtAb/HV4+XNGC0OBfdBvdJU85FyiTb12XGEgNjFp", + "linux_386": "sha384-TiesqbEG9ITqnOyFNMilVnciVM65dCAlRNYp/pK19jrqs2x5MhbpJ0a7Q9XwZmz8", + "linux_amd64": "sha384-y8vr5fWIqSvJhMoHwldoVPOJpAfLi4iHcnhfTcm/nuJAxGAJmI2MiBbk3t7lQNHC", + "windows_386": "sha384-YJTz4Y+5rcy6Ii/J44Qb6J2JZuzfh40WHGTc6jFTHFhJ47Ht+y9s4bS6h8WX6S0m", + "windows_amd64": "sha384-f8jkaz3oRaDcn8jiXupeDO665t6d2tTnFuU0bKwLWszXSz8r29My/USG+UoO9hOr", + }, +} + +YqInfo = provider( + doc = "Provide info for executing yq", + fields = { + "bin": "Executable yq binary", + }, +) + +def _yq_toolchain_impl(ctx): + binary = ctx.attr.bin.files.to_list()[0] + + # Make the $(YQ_BIN) variable available in places like genrules. + # See https://docs.bazel.build/versions/main/be/make-variables.html#custom_variables + template_variables = platform_common.TemplateVariableInfo({ + "YQ_BIN": binary.path, + }) + default_info = DefaultInfo( + files = depset([binary]), + runfiles = ctx.runfiles(files = [binary]), + ) + yq_info = YqInfo( + bin = binary, + ) + + # Export all the providers inside our ToolchainInfo + # so the resolved_toolchain rule can grab and re-export them. + toolchain_info = platform_common.ToolchainInfo( + yqinfo = yq_info, + template_variables = template_variables, + default = default_info, + ) + + return [default_info, toolchain_info, template_variables] + +yq_toolchain = rule( + implementation = _yq_toolchain_impl, + attrs = { + "bin": attr.label( + mandatory = True, + allow_single_file = True, + ), + }, +) + +def _yq_toolchains_repo_impl(repository_ctx): + # Expose a concrete toolchain which is the result of Bazel resolving the toolchain + # for the execution or target platform. + # Workaround for https://github.com/bazelbuild/bazel/issues/14009 + starlark_content = """# Generated by @aspect_bazel_lib//lib/private/yq_toolchain.bzl + +# Forward all the providers +def _resolved_toolchain_impl(ctx): + toolchain_info = ctx.toolchains["@aspect_bazel_lib//lib:yq_toolchain_type"] + return [ + toolchain_info, + toolchain_info.default, + toolchain_info.yqinfo, + toolchain_info.template_variables, + ] + +# Copied from java_toolchain_alias +# https://cs.opensource.google/bazel/bazel/+/master:tools/jdk/java_toolchain_alias.bzl +resolved_toolchain = rule( + implementation = _resolved_toolchain_impl, + toolchains = ["@aspect_bazel_lib//lib:yq_toolchain_type"], + incompatible_use_toolchain_transition = True, +) +""" + repository_ctx.file("defs.bzl", starlark_content) + + build_content = """# Generated by @aspect_bazel_lib//lib/private/yq_toolchain.bzl +# +# These can be registered in the workspace file or passed to --extra_toolchains flag. +# By default all these toolchains are registered by the yq_register_toolchains macro +# so you don't normally need to interact with these targets. + +load(":defs.bzl", "resolved_toolchain") + +resolved_toolchain(name = "resolved_toolchain", visibility = ["//visibility:public"]) + +""" + + for [platform, meta] in YQ_PLATFORMS.items(): + build_content += """ +toolchain( + name = "{platform}_toolchain", + exec_compatible_with = {compatible_with}, + target_compatible_with = {compatible_with}, + toolchain = "@{name}_{platform}//:yq_toolchain", + toolchain_type = "@aspect_bazel_lib//lib:yq_toolchain_type", +) +""".format( + platform = platform, + name = repository_ctx.attr.name, + user_repository_name = repository_ctx.attr.user_repository_name, + compatible_with = meta.compatible_with, + ) + + # Base BUILD file for this repository + repository_ctx.file("BUILD.bazel", build_content) + +yq_toolchains_repo = repository_rule( + _yq_toolchains_repo_impl, + doc = """Creates a repository with toolchain definitions for all known platforms + which can be registered or selected.""", + attrs = { + "user_repository_name": attr.string(doc = "Base name for toolchains repository"), + }, +) + +def _yq_platform_repo_impl(repository_ctx): + is_windows = repository_ctx.attr.platform == "windows_386" or repository_ctx.attr.platform == "windows_amd64" + + #https://github.com/mikefarah/yq/releases/download/v4.24.4/yq_linux_386 + url = "https://github.com/mikefarah/yq/releases/download/{0}/yq_{1}{2}".format( + repository_ctx.attr.yq_version, + repository_ctx.attr.platform, + ".exe" if is_windows else "", + ) + + repository_ctx.download( + url = url, + output = "yq.exe" if is_windows else "yq", + executable = True, + integrity = YQ_VERSIONS[repository_ctx.attr.yq_version][repository_ctx.attr.platform], + ) + build_content = """#Generated by @aspect_bazel_lib//lib/private/yq_toolchain.bzl +load("@aspect_bazel_lib//lib/private:yq_toolchain.bzl", "yq_toolchain") +exports_files(["{0}"]) +yq_toolchain(name = "yq_toolchain", bin = "{0}", visibility = ["//visibility:public"]) +""".format("yq.exe" if is_windows else "yq") + + # Base BUILD file for this repository + repository_ctx.file("BUILD.bazel", build_content) + +yq_platform_repo = repository_rule( + implementation = _yq_platform_repo_impl, + doc = "Fetch external tools needed for yq toolchain", + attrs = { + "yq_version": attr.string(mandatory = True, values = YQ_VERSIONS.keys()), + "platform": attr.string(mandatory = True, values = YQ_PLATFORMS.keys()), + }, +) diff --git a/lib/repositories.bzl b/lib/repositories.bzl index 5e678d3..192e251 100644 --- a/lib/repositories.bzl +++ b/lib/repositories.bzl @@ -3,6 +3,7 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") load("//lib/private:jq_toolchain.bzl", "JQ_PLATFORMS", "jq_platform_repo", "jq_toolchains_repo") +load("//lib/private:yq_toolchain.bzl", "YQ_PLATFORMS", "yq_platform_repo", "yq_toolchains_repo") def aspect_bazel_lib_dependencies(): "Load dependencies required by aspect rules" @@ -34,3 +35,22 @@ def register_jq_toolchains(version, name = "jq"): jq_toolchains_repo( name = "%s_toolchains" % name, ) + +def register_yq_toolchains(version, name = "yq"): + """Registers yq toolchain and repositories + + Args: + version: the version of yq to execute (see https://github.com/mikefarah/yq/releases) + name: override the prefix for the generated toolchain repositories + """ + for platform in YQ_PLATFORMS.keys(): + yq_platform_repo( + name = "%s_toolchains_%s" % (name, platform), + platform = platform, + yq_version = "v%s" % version, + ) + native.register_toolchains("@%s_toolchains//:%s_toolchain" % (name, platform)) + + yq_toolchains_repo( + name = "%s_toolchains" % name, + ) diff --git a/lib/tests/jq/BUILD.bazel b/lib/tests/jq/BUILD.bazel index 9b5c48d..401990a 100644 --- a/lib/tests/jq/BUILD.bazel +++ b/lib/tests/jq/BUILD.bazel @@ -1,6 +1,12 @@ load("//lib/tests/jq:diff_test.bzl", "diff_test") load("//lib:jq.bzl", "jq") +exports_files([ + "a_pretty.json", + "a.json", + "b.json", +]) + # Identity filter produces identical json jq( name = "case_dot_filter", diff --git a/lib/tests/yq/BUILD.bazel b/lib/tests/yq/BUILD.bazel new file mode 100644 index 0000000..d2a9c96 --- /dev/null +++ b/lib/tests/yq/BUILD.bazel @@ -0,0 +1,333 @@ +load("//lib/private:diff_test.bzl", "diff_test") +load("//lib:yq.bzl", "yq") + +# Identity (dot) expression produces identical yaml +yq( + name = "case_dot_expression", + srcs = ["a.yaml"], + expression = ".", +) + +diff_test( + name = "case_dot_expression_test", + file1 = "a.yaml", + file2 = ":case_dot_expression", +) + +# No expression same as dot expression +yq( + name = "case_no_expression", + srcs = ["a.yaml"], +) + +diff_test( + name = "case_no_expression_test", + file1 = "a.yaml", + file2 = ":case_no_expression", +) + +# Output json, no out declared +yq( + name = "case_json_output_no_out", + srcs = ["a.yaml"], + args = ["-o=json"], + expression = ".", +) + +diff_test( + name = "case_json_output_no_out_test", + file1 = "//lib/tests/jq:a_pretty.json", + file2 = "case_json_output_no_out.json", +) + +# Output json, outs has ".json" extension but "-o=json" not set +yq( + name = "case_json_output_no_arg", + srcs = ["a.yaml"], + outs = ["case_json_output_no_arg.json"], + args = [], + expression = ".", +) + +diff_test( + name = "case_json_output_no_arg_test", + file1 = "//lib/tests/jq:a_pretty.json", + file2 = ":case_json_output_no_arg", +) + +# Convert json to yaml +yq( + name = "case_convert_json_to_yaml", + srcs = ["//lib/tests/jq:a_pretty.json"], + args = ["-P"], + expression = ".", +) + +diff_test( + name = "case_convert_json_to_yaml_test", + file1 = "a.yaml", + file2 = ":case_convert_json_to_yaml", +) + +# No srcs, output is a generated expression +yq( + name = "case_generate_from_expression", + srcs = [], + expression = ".a.b.c = \"cat\"", + # yq hangs without input srcs (https://github.com/mikefarah/yq/issues/1177) + tags = ["no-windows-ci"], +) + +diff_test( + name = "case_generate_from_expression_test", + file1 = "generated-from-expression.yaml", + file2 = ":case_generate_from_expression", + # yq hangs without input srcs (https://github.com/mikefarah/yq/issues/1177) + tags = ["no-windows-ci"], +) + +# No sources produces empty file (equivalent to --null-input) +yq( + name = "case_no_sources", + srcs = [], + expression = ".", + # yq hangs without input srcs (https://github.com/mikefarah/yq/issues/1177) + tags = ["no-windows-ci"], +) + +diff_test( + name = "case_no_sources_test", + file1 = ":case_no_sources", + file2 = "empty.yaml", + # yq hangs without input srcs (https://github.com/mikefarah/yq/issues/1177) + tags = ["no-windows-ci"], +) + +# Merge two documents together +yq( + name = "case_merge_expression", + srcs = [ + "a.yaml", + "b.yaml", + ], + expression = "select(fileIndex == 0) * select(fileIndex == 1)", +) + +diff_test( + name = "case_merge_expression_test", + file1 = "a_b_merged.yaml", + file2 = ":case_merge_expression", +) + +# Merge two documents together (alt syntax) +yq( + name = "case_merge_expression_alt", + srcs = [ + "a.yaml", + "b.yaml", + ], + expression = ". as $item ireduce ({}; . * $item )", + # TODO: figure out why this doesn't work on windows (may be related to https://github.com/mikefarah/yq/issues/747) + tags = ["no-windows-ci"], +) + +diff_test( + name = "case_merge_expression_alt_test", + file1 = "a_b_merged.yaml", + file2 = ":case_merge_expression_alt", + # TODO: figure out why this doesn't work on windows (may be related to https://github.com/mikefarah/yq/issues/747) + tags = ["no-windows-ci"], +) + +# Split into multiple documents +yq( + name = "case_split_expression", + srcs = ["multidoc.yaml"], + outs = [ + "test_doc1.yml", + "test_doc2.yml", + ], + args = [ + "-s '.a'", + "--no-doc", + ], + expression = ".", +) + +diff_test( + name = "case_split_expression_test_1", + file1 = "split1.yaml", + file2 = "test_doc1.yml", +) + +diff_test( + name = "case_split_expression_test_2", + file1 = "split2.yaml", + file2 = "test_doc2.yml", +) + +# Outputs properties file +yq( + name = "case_output_properties", + srcs = ["a.yaml"], + outs = ["case_output_properties.properties"], + args = ["-o=props"], + expression = ".", +) + +diff_test( + name = "case_output_properties_test", + file1 = "a.properties", + file2 = ":case_output_properties", +) + +# Outputs properties file, outs not declared +yq( + name = "case_output_properties_no_outs", + srcs = ["a.yaml"], + args = ["-o=props"], + expression = ".", +) + +diff_test( + name = "case_output_properties_no_outs_test", + file1 = "a.properties", + file2 = ":case_output_properties_no_outs", +) + +# Outputs csv file +yq( + name = "case_output_csv", + srcs = ["array.yaml"], + outs = ["case_output_csv.csv"], + args = ["-o=c"], + expression = ".", +) + +diff_test( + name = "case_output_csv_test", + file1 = "array.csv", + file2 = ":case_output_csv", +) + +# Outputs csv file, outs not declared +yq( + name = "case_output_csv_no_outs", + srcs = ["array.yaml"], + args = ["-o=c"], + expression = ".", +) + +diff_test( + name = "case_output_csv_no_outs_test", + file1 = "array.csv", + file2 = ":case_output_csv_no_outs", +) + +# Outputs tsv file +yq( + name = "case_output_tsv", + srcs = ["array.yaml"], + outs = ["case_output_tsv.tsv"], + args = ["-o=t"], + expression = ".", +) + +diff_test( + name = "case_output_tsv_test", + file1 = "array.tsv", + file2 = ":case_output_tsv", +) + +# Outputs tsv file, outs not declared +yq( + name = "case_output_tsv_no_outs", + srcs = ["array.yaml"], + args = ["-o=t"], + expression = ".", +) + +diff_test( + name = "case_output_tsv_no_outs_test", + file1 = "array.tsv", + file2 = ":case_output_tsv_no_outs", +) + +# Convert xml to yaml +yq( + name = "case_convert_xml_to_yaml", + srcs = ["sample.xml"], + args = ["-p=xml"], + expression = ".", +) + +diff_test( + name = "case_convert_xml_to_yaml_test", + file1 = "sample.yaml", + file2 = ":case_convert_xml_to_yaml", +) + +# Outputs xml file +yq( + name = "case_output_xml", + srcs = ["a.yaml"], + outs = ["case_output_xml.xml"], + args = ["-o=xml"], + expression = ".", +) + +diff_test( + name = "case_output_xml_test", + file1 = "a.xml", + file2 = ":case_output_xml", +) + +# Outputs xml file, outs not declared +yq( + name = "case_output_xml_no_outs", + srcs = ["a.yaml"], + args = ["-o=xml"], + expression = ".", +) + +diff_test( + name = "case_output_xml_no_outs_test", + file1 = "a.xml", + file2 = ":case_output_xml_no_outs", +) + +# Merge two json documents together +yq( + name = "case_merge_expression_json", + srcs = [ + "//lib/tests/jq:a.json", + "//lib/tests/jq:b.json", + ], + args = ["-P"], + expression = ". as $item ireduce ({}; . * $item )", + # TODO: figure out why this doesn't work on windows (may be related to https://github.com/mikefarah/yq/issues/747) + tags = ["no-windows-ci"], +) + +diff_test( + name = "case_merge_expression_json_test", + file1 = "a_b_merged.yaml", + file2 = ":case_merge_expression_json", + # TODO: figure out why this doesn't work on windows (may be related to https://github.com/mikefarah/yq/issues/747) + tags = ["no-windows-ci"], +) + +# Call yq within a genrule +genrule( + name = "case_genrule", + srcs = ["a.yaml"], + outs = ["genrule_output.yaml"], + cmd = "$(YQ_BIN) '.' $(location a.yaml) > $@", + toolchains = ["@yq_toolchains//:resolved_toolchain"], +) + +diff_test( + name = "case_genrule_test", + file1 = "genrule_output.yaml", + file2 = "a.yaml", +) diff --git a/lib/tests/yq/a.properties b/lib/tests/yq/a.properties new file mode 100644 index 0000000..fda24cb --- /dev/null +++ b/lib/tests/yq/a.properties @@ -0,0 +1,6 @@ +foo = bar +value = 123 +moo.0 = 1 +moo.1 = 2 +moo.2 = 3 +a = true diff --git a/lib/tests/yq/a.xml b/lib/tests/yq/a.xml new file mode 100644 index 0000000..98ea901 --- /dev/null +++ b/lib/tests/yq/a.xml @@ -0,0 +1,6 @@ +bar +123 +1 +2 +3 +true diff --git a/lib/tests/yq/a.yaml b/lib/tests/yq/a.yaml new file mode 100644 index 0000000..08c5e29 --- /dev/null +++ b/lib/tests/yq/a.yaml @@ -0,0 +1,7 @@ +foo: bar +value: 123 +moo: + - 1 + - 2 + - 3 +a: true diff --git a/lib/tests/yq/a_b_merged.yaml b/lib/tests/yq/a_b_merged.yaml new file mode 100644 index 0000000..3de3ed3 --- /dev/null +++ b/lib/tests/yq/a_b_merged.yaml @@ -0,0 +1,8 @@ +foo: baz +value: 456 +moo: + - 4 + - 5 + - 6 +a: true +b: true diff --git a/lib/tests/yq/array.csv b/lib/tests/yq/array.csv new file mode 100644 index 0000000..2c6de59 --- /dev/null +++ b/lib/tests/yq/array.csv @@ -0,0 +1 @@ +1,2,3,4 diff --git a/lib/tests/yq/array.tsv b/lib/tests/yq/array.tsv new file mode 100644 index 0000000..62eefe2 --- /dev/null +++ b/lib/tests/yq/array.tsv @@ -0,0 +1 @@ +1 2 3 4 diff --git a/lib/tests/yq/array.yaml b/lib/tests/yq/array.yaml new file mode 100644 index 0000000..12bae17 --- /dev/null +++ b/lib/tests/yq/array.yaml @@ -0,0 +1 @@ +[1, 2, 3, 4] diff --git a/lib/tests/yq/b.yaml b/lib/tests/yq/b.yaml new file mode 100644 index 0000000..a0bf7f8 --- /dev/null +++ b/lib/tests/yq/b.yaml @@ -0,0 +1,7 @@ +foo: baz +value: 456 +moo: + - 4 + - 5 + - 6 +b: true diff --git a/lib/tests/yq/empty.yaml b/lib/tests/yq/empty.yaml new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/lib/tests/yq/empty.yaml @@ -0,0 +1 @@ + diff --git a/lib/tests/yq/generated-from-expression.yaml b/lib/tests/yq/generated-from-expression.yaml new file mode 100644 index 0000000..a630278 --- /dev/null +++ b/lib/tests/yq/generated-from-expression.yaml @@ -0,0 +1,3 @@ +a: + b: + c: cat diff --git a/lib/tests/yq/multidoc.yaml b/lib/tests/yq/multidoc.yaml new file mode 100644 index 0000000..5a545a4 --- /dev/null +++ b/lib/tests/yq/multidoc.yaml @@ -0,0 +1,3 @@ +a: test_doc1 +--- +a: test_doc2 diff --git a/lib/tests/yq/sample.xml b/lib/tests/yq/sample.xml new file mode 100644 index 0000000..ab8708d --- /dev/null +++ b/lib/tests/yq/sample.xml @@ -0,0 +1,6 @@ + + + meow + 4 + true + \ No newline at end of file diff --git a/lib/tests/yq/sample.yaml b/lib/tests/yq/sample.yaml new file mode 100644 index 0000000..8b6718b --- /dev/null +++ b/lib/tests/yq/sample.yaml @@ -0,0 +1,4 @@ +cat: + says: meow + legs: "4" + cute: "true" diff --git a/lib/tests/yq/split1.yaml b/lib/tests/yq/split1.yaml new file mode 100644 index 0000000..2d7444f --- /dev/null +++ b/lib/tests/yq/split1.yaml @@ -0,0 +1 @@ +a: test_doc1 diff --git a/lib/tests/yq/split2.yaml b/lib/tests/yq/split2.yaml new file mode 100644 index 0000000..1fbd5fd --- /dev/null +++ b/lib/tests/yq/split2.yaml @@ -0,0 +1 @@ +a: test_doc2 diff --git a/lib/yq.bzl b/lib/yq.bzl new file mode 100644 index 0000000..34d1481 --- /dev/null +++ b/lib/yq.bzl @@ -0,0 +1,160 @@ +"""Public API for yq""" + +load("//lib/private:yq.bzl", _is_split_operation = "is_split_operation", _yq_lib = "yq_lib") + +_yq_rule = rule( + attrs = _yq_lib.attrs, + implementation = _yq_lib.implementation, + toolchains = ["@aspect_bazel_lib//lib:yq_toolchain_type"], +) + +def yq(name, srcs, expression = ".", args = [], outs = None, **kwargs): + """Invoke yq with an expression on a set of input files. + + For yq documentation, see https://mikefarah.gitbook.io/yq. + + To use this rule you must register the yq toolchain in your WORKSPACE: + + ```starlark + load("@aspect_bazel_lib//lib:repositories.bzl", "register_yq_toolchains") + + register_yq_toolchains(version = "4.24.4") + ``` + + Usage examples: + + ```starlark + load("@aspect_bazel_lib//lib:yq.bzl", "yq") + ``` + + ```starlark + # Remove fields + yq( + name = "safe-config", + srcs = ["config.yaml"], + filter = "del(.credentials)", + ) + ``` + + ```starlark + # Merge two yaml documents + yq( + name = "merged", + srcs = [ + "a.yaml", + "b.yaml", + ], + expression = ". as $item ireduce ({}; . * $item )", + ) + ``` + + ```starlark + # Split a yaml file into several files + yq( + name = "split", + srcs = ["multidoc.yaml"], + outs = [ + "first.yml", + "second.yml", + ], + args = [ + "-s '.a'", # Split expression + "--no-doc", # Exclude document separator -- + ], + ) + ``` + + ```starlark + # Convert a yaml file to json + yq( + name = "convert-to-json", + srcs = ["foo.yaml"], + args = ["-o=json"], + outs = ["foo.json"], + ) + ``` + + ```starlark + # Convert a json file to yaml + yq( + name = "convert", + srcs = ["bar.json"], + args = ["-P"], + outs = ["bar.yaml"], + ) + ``` + + ```starlark + # Call yq in a genrule + genrule( + name = "generate", + srcs = ["farm.yaml"], + outs = ["genrule_output.yaml"], + cmd = "$(YQ_BIN) '.moo = \"cow\"' $(location farm.yaml) > $@", + toolchains = ["@yq_toolchains//:resolved_toolchain"], + ) + ``` + + yq is capable of parsing and outputting to other formats. See their [docs](https://mikefarah.gitbook.io/yq) for more examples. + + Args: + name: Name of the rule + srcs: List of input file labels + expression: yq expression (https://mikefarah.gitbook.io/yq/commands/evaluate). Defaults to the identity + expression "." + args: Additional args to pass to yq. Note that you do not need to pass _eval_ or _eval-all_ as this + is handled automatically based on the number `srcs`. Passing the output format or the parse format + is optional as these can be guessed based on the file extensions in `srcs` and `outs`. + outs: Name of the output files. Defaults to a single output with the name plus a ".yaml" extension, or + the extension corresponding to a passed output argment (e.g., "-o=json"). For split operations you + must declare all outputs as the name of the output files depends on the expression. + **kwargs: Other common named parameters such as `tags` or `visibility` + """ + args = args[:] + + if not _is_split_operation(args): + # For split operations we can't predeclare outs because the name of the resulting files + # depends on the expression. For non-split operations, set a default output file name + # based on the name and the output format passed, defaulting to yaml. + if not outs: + outs = [name + ".yaml"] + if "-o=json" in args or "--outputformat=json" in args: + outs = [name + ".json"] + if "-o=xml" in args or "--outputformat=xml" in args: + outs = [name + ".xml"] + elif "-o=props" in args or "--outputformat=props" in args: + outs = [name + ".properties"] + elif "-o=c" in args or "--outputformat=csv" in args: + outs = [name + ".csv"] + elif "-o=t" in args or "--outputformat=tsv" in args: + outs = [name + ".tsv"] + + elif outs and len(outs) == 1: + # If an output file with an extension was provided, try to set the corresponding output + # argument if it wasn't already passed. + if outs[0].endswith(".json") and "-o=json" not in args and "--outputformat=json" not in args: + args.append("-o=json") + elif outs[0].endswith(".xml") and "-o=xml" not in args and "--outputformat=xml" not in args: + args.append("-o=xml") + elif outs[0].endswith(".properties") and "-o=props" not in args and "--outputformat=props" not in args: + args.append("-o=props") + elif outs[0].endswith(".csv") and "-o=c" not in args and "--outputformat=csv" not in args: + args.append("-o=c") + elif outs[0].endswith(".tsv") and "-o=t" not in args and "--outputformat=tsv" not in args: + args.append("-o=t") + + # If the input files are json or xml, set the parse flag if it isn't already set + if len(srcs) > 0: + if srcs[0].endswith(".json") and "-P" not in args: + args.append("-P") + elif srcs[0].endswith(".xml") and "-p=xml" not in args: + args.append("-p=xml") + + _yq_rule( + name = name, + srcs = srcs, + expression = expression, + args = args, + outs = outs, + **kwargs + ) diff --git a/tools/yq_mirror_release.sh b/tools/yq_mirror_release.sh new file mode 100644 index 0000000..3c501e7 --- /dev/null +++ b/tools/yq_mirror_release.sh @@ -0,0 +1,30 @@ +#!/bin/bash/env bash +# Produce a dictionary for the current yq release, +# suitable for adding to lib/private/yq_toolchain.bzl + +set -o errexit + +# Find the latest version +version=$(curl --silent "https://api.github.com/repos/mikefarah/yq/releases/latest" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/') + +# yq publishes its checksums and a script to extract them +curl --silent --location "https://github.com/mikefarah/yq/releases/download/$version/extract-checksum.sh" -o /tmp/extract-checksum.sh +curl --silent --location "https://github.com/mikefarah/yq/releases/download/$version/checksums_hashes_order" -o /tmp/checksums_hashes_order +curl --silent --location "https://github.com/mikefarah/yq/releases/download/$version/checksums" -o /tmp/checksums + +cd /tmp +chmod u+x extract-checksum.sh + +# Extract the checksums and output a starlark map entry +echo "\"$version\": {" +for release in darwin_{amd,arm}64 linux_{386,amd64} windows_{386,amd64}; do + artifact=$release + if [[ $release == windows* ]]; then + artifact="$release.exe" + fi + echo " \"$release\": \"$(./extract-checksum.sh SHA-384 $artifact | awk '{ print $2 }' | xxd -r -p | base64 | awk '{ print "sha384-" $1 }' )\"," +done +echo "}," + +printf "\n" +echo "Paste the above into VERSIONS in yq_toolchain.bzl." \ No newline at end of file