From d79f4d48675654df81bde0d9dcd80aaf68984d1f Mon Sep 17 00:00:00 2001 From: Malte Poll Date: Wed, 14 Jun 2023 16:21:55 +0200 Subject: [PATCH] feat: yq supports stamping Fixes #223 --- docs/yq.md | 15 ++++++++++++- lib/private/BUILD.bazel | 1 + lib/private/docs/BUILD.bazel | 1 + lib/private/parse_status_file.yq | 3 +++ lib/private/yq.bzl | 36 ++++++++++++++++++++++++++++++-- lib/tests/yq/BUILD.bazel | 33 +++++++++++++++++++++++++++++ lib/yq.bzl | 19 ++++++++++++++++- 7 files changed, 104 insertions(+), 4 deletions(-) create mode 100644 lib/private/parse_status_file.yq diff --git a/docs/yq.md b/docs/yq.md index 3450a71..bd1dc2a 100644 --- a/docs/yq.md +++ b/docs/yq.md @@ -96,6 +96,19 @@ genrule( ) ``` +```starlark +# With --stamp, causes properties to be replaced by version control info. +yq( + name = "stamped", + srcs = ["package.yaml"], + expression = "|".join([ + "load(strenv(STAMP)) as $stamp", + # Provide a default using the "alternative operator" in case $stamp is empty dict. + ".version = ($stamp.BUILD_EMBED_LABEL // "")", + ]), +) +``` + yq is capable of parsing and outputting to other formats. See their [docs](https://mikefarah.gitbook.io/yq) for more examples. @@ -106,7 +119,7 @@ yq is capable of parsing and outputting to other formats. See their [docs](https | :------------- | :------------- | :------------- | | name | Name of the rule | none | | srcs | List of input file labels | none | -| expression | yq expression (https://mikefarah.gitbook.io/yq/commands/evaluate). Defaults to the identity expression "." | "." | +| expression | yq expression (https://mikefarah.gitbook.io/yq/commands/evaluate). Defaults to the identity expression ".". Subject to stamp variable replacements, see [Stamping](./stamping.md). When stamping is enabled, an environment variable named "STAMP" will be available in the expression.

Be careful to write the filter so that it handles unstamped builds, as in the example above. | "." | | args | Additional args to pass to yq. Note that you do not need to pass _eval_ or _eval-all_ as this is handled automatically based on the number srcs. Passing the output format or the parse format is optional as these can be guessed based on the file extensions in srcs and outs. | [] | | outs | Name of the output files. Defaults to a single output with the name plus a ".yaml" extension, or the extension corresponding to a passed output argment (e.g., "-o=json"). For split operations you must declare all outputs as the name of the output files depends on the expression. | None | | kwargs | Other common named parameters such as tags or visibility | none | diff --git a/lib/private/BUILD.bazel b/lib/private/BUILD.bazel index 7623400..9a9278c 100644 --- a/lib/private/BUILD.bazel +++ b/lib/private/BUILD.bazel @@ -3,6 +3,7 @@ exports_files( "diff_test_tmpl.sh", "diff_test_tmpl.bat", "parse_status_file.jq", + "parse_status_file.yq", ], visibility = ["//visibility:public"], ) diff --git a/lib/private/docs/BUILD.bazel b/lib/private/docs/BUILD.bazel index 316894a..862d1d6 100644 --- a/lib/private/docs/BUILD.bazel +++ b/lib/private/docs/BUILD.bazel @@ -217,6 +217,7 @@ bzl_library( bzl_library( name = "yq", srcs = ["//lib/private:yq.bzl"], + deps = ["//lib:stamping"], ) bzl_library( diff --git a/lib/private/parse_status_file.yq b/lib/private/parse_status_file.yq new file mode 100644 index 0000000..5db6622 --- /dev/null +++ b/lib/private/parse_status_file.yq @@ -0,0 +1,3 @@ +load_str(filename) | split("\n") | .[] | select(length!=0) + | [capture("(?P[^\s]+)\s+(?P.*)")] + | from_entries diff --git a/lib/private/yq.bzl b/lib/private/yq.bzl index 14e8924..784fc23 100644 --- a/lib/private/yq.bzl +++ b/lib/private/yq.bzl @@ -1,6 +1,8 @@ """Implementation for yq rule""" -_yq_attrs = { +load("//lib:stamping.bzl", "STAMP_ATTRS", "maybe_stamp") + +_yq_attrs = dict({ "srcs": attr.label_list( allow_files = [".yaml", ".json", ".xml"], mandatory = True, @@ -9,7 +11,11 @@ _yq_attrs = { "expression": attr.string(mandatory = False), "args": attr.string_list(), "outs": attr.output_list(mandatory = True), -} + "_parse_status_file_expression": attr.label( + allow_single_file = True, + default = Label("//lib/private:parse_status_file.yq"), + ), +}, **STAMP_ATTRS) def is_split_operation(args): for arg in args: @@ -38,6 +44,31 @@ def _yq_impl(ctx): if len(ctx.attr.srcs) == 0 and "-n" not in args and "--null-input" not in args: args = args + ["--null-input"] + stamp = maybe_stamp(ctx) + stamp_yaml = ctx.actions.declare_file("_%s_stamp.yaml" % ctx.label.name) + if stamp: + # create an action that gives a YAML representation of the stamp keys + ctx.actions.run_shell( + tools = [yq_bin], + inputs = [stamp.stable_status_file, stamp.volatile_status_file, ctx.file._parse_status_file_expression], + outputs = [stamp_yaml], + command = "{yq} --from-file {expression} {stable} {volatile} > {out}".format( + yq = yq_bin.path, + expression = ctx.file._parse_status_file_expression.path, + stable = stamp.stable_status_file.path, + volatile = stamp.volatile_status_file.path, + out = stamp_yaml.path, + ), + mnemonic = "ConvertStatusToYaml", + ) + else: + # create an empty stamp file as placeholder + ctx.actions.write( + output = stamp_yaml, + content = "{}", + ) + inputs.append(stamp_yaml) + # For split operations, yq outputs files in the same directory so we # must cd to the correct output dir before executing it bin_dir = "/".join([ctx.bin_dir.path, ctx.label.package]) if ctx.label.package else ctx.bin_dir.path @@ -58,6 +89,7 @@ def _yq_impl(ctx): inputs = inputs, outputs = outs, command = cmd, + env = {"STAMP": escape_bin_dir + stamp_yaml.path}, mnemonic = "Yq", ) diff --git a/lib/tests/yq/BUILD.bazel b/lib/tests/yq/BUILD.bazel index 280b3c6..ee99c1a 100644 --- a/lib/tests/yq/BUILD.bazel +++ b/lib/tests/yq/BUILD.bazel @@ -1,5 +1,6 @@ load("//lib/private:diff_test.bzl", "diff_test") load("//lib:yq.bzl", "yq") +load("//lib:testing.bzl", "assert_contains") exports_files( [ @@ -308,6 +309,38 @@ diff_test( file2 = ":case_merge_expression_json", ) +# Expression that uses a stamp variable +[ + yq( + name = ("" if stamp else "un") + "stamped", + srcs = ["a.yaml"], + expression = "|".join([ + "load(strenv(STAMP)) as $stamp", + # Provide a default using the "alternative operator" + ".foo = ($stamp.BUILD_EMBED_LABEL // \"\")", + ".value = ($stamp.BUILD_TIMESTAMP // 1 | @yamld)", + ]), + stamp = stamp, + ) + for stamp in [ + 0, + 1, + ] +] + +assert_contains( + name = "check_stamped", + actual = "stamped.yaml", + # v1.2.3 comes from the --embed_label flag in .bazelrc + expected = """foo: v1.2.3""", +) + +assert_contains( + name = "check_unstamped", + actual = "unstamped.yaml", + expected = """foo: """, +) + # Call yq within a genrule genrule( name = "case_genrule", diff --git a/lib/yq.bzl b/lib/yq.bzl index 8951696..4958446 100644 --- a/lib/yq.bzl +++ b/lib/yq.bzl @@ -95,13 +95,30 @@ def yq(name, srcs, expression = ".", args = [], outs = None, **kwargs): ) ``` + ```starlark + # With --stamp, causes properties to be replaced by version control info. + yq( + name = "stamped", + srcs = ["package.yaml"], + expression = "|".join([ + "load(strenv(STAMP)) as $stamp", + # Provide a default using the "alternative operator" in case $stamp is empty dict. + ".version = ($stamp.BUILD_EMBED_LABEL // "")", + ]), + ) + ``` + yq is capable of parsing and outputting to other formats. See their [docs](https://mikefarah.gitbook.io/yq) for more examples. Args: name: Name of the rule srcs: List of input file labels expression: yq expression (https://mikefarah.gitbook.io/yq/commands/evaluate). Defaults to the identity - expression "." + expression ".". Subject to stamp variable replacements, see [Stamping](./stamping.md). + When stamping is enabled, an environment variable named "STAMP" will be available in the expression. + + Be careful to write the filter so that it handles unstamped builds, as in the example above. + args: Additional args to pass to yq. Note that you do not need to pass _eval_ or _eval-all_ as this is handled automatically based on the number `srcs`. Passing the output format or the parse format is optional as these can be guessed based on the file extensions in `srcs` and `outs`.