From 8e6a15ceae398bc28997b3ceb95805123e3507b0 Mon Sep 17 00:00:00 2001 From: Derek Cormier Date: Thu, 27 Jan 2022 18:15:28 -0800 Subject: [PATCH] feat: write_source_files --- .prettierignore | 2 + docs/BUILD.bazel | 5 + docs/write_source_files.md | 61 ++++++++ lib/BUILD.bazel | 12 ++ lib/private/BUILD.bazel | 15 ++ lib/private/fail_with_message_test.bzl | 12 ++ lib/private/write_source_files.bzl | 65 +++++++++ lib/tests/write_source_files/BUILD.bazel | 34 +++++ lib/tests/write_source_files/a.js | 1 + lib/tests/write_source_files/a2.js | 1 + lib/tests/write_source_files/b.js | 1 + lib/tests/write_source_files/b2.js | 1 + .../write_source_files_test.bzl | 102 ++++++++++++++ lib/write_source_files.bzl | 131 ++++++++++++++++++ 14 files changed, 443 insertions(+) create mode 100755 docs/write_source_files.md create mode 100644 lib/private/fail_with_message_test.bzl create mode 100644 lib/private/write_source_files.bzl create mode 100644 lib/tests/write_source_files/BUILD.bazel create mode 100644 lib/tests/write_source_files/a.js create mode 100644 lib/tests/write_source_files/a2.js create mode 100644 lib/tests/write_source_files/b.js create mode 100644 lib/tests/write_source_files/b2.js create mode 100644 lib/tests/write_source_files/write_source_files_test.bzl create mode 100644 lib/write_source_files.bzl diff --git a/.prettierignore b/.prettierignore index eb837ff..d6ed2fd 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,2 +1,4 @@ docs/*.md lib/tests/jq/*.json +lib/tests/write_source_files/a2.js +lib/tests/write_source_files/b2.js diff --git a/docs/BUILD.bazel b/docs/BUILD.bazel index 76333cb..ff48aae 100644 --- a/docs/BUILD.bazel +++ b/docs/BUILD.bazel @@ -38,6 +38,11 @@ stardoc_with_diff_test( out_label = "//docs:jq.md", ) +stardoc_with_diff_test( + bzl_library_target = "//lib:write_source_files", + out_label = "//docs:write_source_files.md", +) + update_docs( name = "update", docs_folder = "docs", diff --git a/docs/write_source_files.md b/docs/write_source_files.md new file mode 100755 index 0000000..280df19 --- /dev/null +++ b/docs/write_source_files.md @@ -0,0 +1,61 @@ + + +Public API for write_source_files + + + +## write_source_files + +
+write_source_files(name, files, kwargs)
+
+ +Write to one or more files in the source tree. Stamp out tests that ensure the files exists and are up to date. + +Usage: + +```starlark +load("@aspect_bazel_lib//lib:write_source_files.bzl", "write_source_files") + +write_source_files( + name = "write_foobar", + files = { + "foobar.json": "//some/generated:file", + }, +) +``` + +To update the source file, run: +```bash +bazel run //:write_foobar +``` + +A test will fail if the source file doesn't exist +```bash +bazel test //... + +//:foobar.json does not exist. To create & update this file, run: + + bazel run //:write_foobar +``` + +...or if it's out of date. +```bash +bazel test //... + +//:foobar.json is out-of-date. To update this file, run: + + bazel run //:write_foobar +``` + + +**PARAMETERS** + + +| Name | Description | Default Value | +| :------------- | :------------- | :------------- | +| name | Name of the executable target that creates or updates the source file | none | +| files | A dict where the keys are source files to write to and the values are labels pointing to the desired content. Source files must be within the same bazel package as the target. | none | +| kwargs | Other common named parameters such as tags or visibility | none | + + diff --git a/lib/BUILD.bazel b/lib/BUILD.bazel index 976ea32..c0ba3d2 100644 --- a/lib/BUILD.bazel +++ b/lib/BUILD.bazel @@ -60,3 +60,15 @@ toolchain_type( name = "jq_toolchain_type", visibility = ["//visibility:public"], ) + +bzl_library( + name = "write_source_files", + srcs = ["write_source_files.bzl"], + visibility = ["//visibility:public"], + deps = [ + ":utils", + "//lib/private:fail_with_message_test", + "//lib/private:write_source_files", + "@bazel_skylib//rules:diff_test", + ], +) diff --git a/lib/private/BUILD.bazel b/lib/private/BUILD.bazel index 37a15bd..f5377d9 100644 --- a/lib/private/BUILD.bazel +++ b/lib/private/BUILD.bazel @@ -57,3 +57,18 @@ bzl_library( srcs = ["jq.bzl"], visibility = ["//lib:__subpackages__"], ) + +bzl_library( + name = "write_source_files", + srcs = ["write_source_files.bzl"], + visibility = ["//lib:__subpackages__"], + deps = [ + "//lib:utils", + ], +) + +bzl_library( + name = "fail_with_message_test", + srcs = ["fail_with_message_test.bzl"], + visibility = ["//lib:__subpackages__"], +) diff --git a/lib/private/fail_with_message_test.bzl b/lib/private/fail_with_message_test.bzl new file mode 100644 index 0000000..4d98843 --- /dev/null +++ b/lib/private/fail_with_message_test.bzl @@ -0,0 +1,12 @@ +"Test rule that always fails and prints a message" + +def _fail_with_message_test_impl(ctx): + fail(ctx.attr.message) + +fail_with_message_test = rule( + attrs = { + "message": attr.string(mandatory = True), + }, + implementation = _fail_with_message_test_impl, + test = True, +) diff --git a/lib/private/write_source_files.bzl b/lib/private/write_source_files.bzl new file mode 100644 index 0000000..98e60b3 --- /dev/null +++ b/lib/private/write_source_files.bzl @@ -0,0 +1,65 @@ +"write_source_file implementation" + +load("//lib:utils.bzl", "is_external_label") + +_write_source_files_attrs = { + "in_files": attr.label_list(allow_files = True, allow_empty = False, mandatory = True), + "out_files": attr.label_list(allow_files = True, allow_empty = False, mandatory = True), + "is_windows": attr.bool(mandatory = True), +} + +def _write_source_files_impl(ctx): + if ctx.attr.is_windows: + fail("write_source_file is not yet implemented for windows") + + if (len(ctx.attr.in_files) != len(ctx.attr.out_files)): + fail("in_files and out_files must be the same length") + + for i in range(len(ctx.attr.in_files)): + out_file_label = ctx.attr.out_files[i].label + if is_external_label(out_file_label): + fail("out file %s must be a source file in the user workspace" % out_file_label) + + if not ctx.files.out_files[i].is_source: + fail("out file %s must be a source file, not a generated file" % out_file_label) + + if out_file_label.package != ctx.label.package: + fail("out file %s (in package '%s') must be a source file within the target's package: '%s'" % (out_file_label, out_file_label.package, ctx.label.package)) + + updater = ctx.actions.declare_file( + ctx.label.name + "_update.sh", + ) + + ctx.actions.write( + output = updater, + content = """ +#!/usr/bin/env bash +set -o errexit -o nounset -o pipefail +runfiles_dir=$PWD +# BUILD_WORKSPACE_DIRECTORY not set when running as a test, uses the sandbox instead +if [[ ! -z "${BUILD_WORKSPACE_DIRECTORY:-}" ]]; then + cd "$BUILD_WORKSPACE_DIRECTORY" +fi +""" + "\n".join([ + """ +in=$runfiles_dir/{in_file} +out={out_file} + +mkdir -p "$(dirname "$out")" +echo "Copying $in to $out in $PWD" +cp -f "$in" "$out" +chmod 644 "$out" +""".format(in_file = ctx.files.in_files[i].short_path, out_file = ctx.files.out_files[i].short_path) + for i in range(len(ctx.attr.in_files)) + ]), + ) + + return DefaultInfo( + executable = updater, + runfiles = ctx.runfiles(files = ctx.files.in_files), + ) + +write_source_files_lib = struct( + attrs = _write_source_files_attrs, + implementation = _write_source_files_impl, +) diff --git a/lib/tests/write_source_files/BUILD.bazel b/lib/tests/write_source_files/BUILD.bazel new file mode 100644 index 0000000..6b6b62c --- /dev/null +++ b/lib/tests/write_source_files/BUILD.bazel @@ -0,0 +1,34 @@ +load("//lib/tests/write_source_files:write_source_files_test.bzl", "write_source_files_test") +load("//lib:write_source_files.bzl", "write_source_files") + +genrule( + name = "a-desired", + outs = ["a-desired.js"], + cmd = "echo 'console.log(\"a*\")' > $@", +) + +genrule( + name = "b-desired", + outs = ["b-desired.js"], + cmd = "echo 'console.log(\"b*\")' > $@", +) + +write_source_files_test( + name = "write_to_source_files_test", + in_files = [ + ":a-desired", + ":b-desired", + ], + out_files = [ + "a.js", + "b.js", + ], +) + +write_source_files( + name = "macro_smoke_test", + files = { + "a2.js": ":a-desired", + "b2.js": ":b-desired", + }, +) diff --git a/lib/tests/write_source_files/a.js b/lib/tests/write_source_files/a.js new file mode 100644 index 0000000..7b2a346 --- /dev/null +++ b/lib/tests/write_source_files/a.js @@ -0,0 +1 @@ +console.log("a"); diff --git a/lib/tests/write_source_files/a2.js b/lib/tests/write_source_files/a2.js new file mode 100644 index 0000000..8c71642 --- /dev/null +++ b/lib/tests/write_source_files/a2.js @@ -0,0 +1 @@ +console.log("a*") diff --git a/lib/tests/write_source_files/b.js b/lib/tests/write_source_files/b.js new file mode 100644 index 0000000..6d012e7 --- /dev/null +++ b/lib/tests/write_source_files/b.js @@ -0,0 +1 @@ +console.log("b"); diff --git a/lib/tests/write_source_files/b2.js b/lib/tests/write_source_files/b2.js new file mode 100644 index 0000000..32f7fea --- /dev/null +++ b/lib/tests/write_source_files/b2.js @@ -0,0 +1 @@ +console.log("b*") diff --git a/lib/tests/write_source_files/write_source_files_test.bzl b/lib/tests/write_source_files/write_source_files_test.bzl new file mode 100644 index 0000000..fe6e5bf --- /dev/null +++ b/lib/tests/write_source_files/write_source_files_test.bzl @@ -0,0 +1,102 @@ +"""Tests for write_source_files""" +# Inspired by https://github.com/cgrindel/bazel-starlib/blob/main/updatesrc/private/updatesrc_update_test.bzl + +load("//lib/private:write_source_files.bzl", _lib = "write_source_files_lib") + +_write_source_files = rule( + attrs = _lib.attrs, + implementation = _lib.implementation, + executable = True, +) + +def _impl(ctx): + test = ctx.actions.declare_file( + ctx.label.name + "_test.sh", + ) + + ctx.actions.write( + output = test, + is_executable = True, + content = """ +#!/usr/bin/env bash +set -o errexit -o nounset -o pipefail + +assert_different() { + local in_file="${1}" + local out_file="${2}" + diff "${in_file}" "${out_file}" > /dev/null && (echo >&2 "Expected files to differ. in: ${in_file}, out: ${out_file}" && return -1) + return 0 +} + +assert_same() { + local in_file="${1}" + local out_file="${2}" + diff "${in_file}" "${out_file}" || (echo >&2 "Expected files to be same. in: ${in_file}, out: ${out_file}" && return -1) +} + +# Check that in and out files are different +""" + "\n".join([ + "assert_different {in_file} {out_file}".format( + in_file = ctx.files.in_files[i].short_path, + out_file = ctx.files.out_files[i].short_path, + ) + for i in range(len(ctx.files.in_files)) + ]) + """ +# Write to the source files +{write_source_files} + +# Check that in and out files are the same +""".format(write_source_files = ctx.file.write_source_files_target.short_path) + "\n".join([ + "assert_same {in_file} {out_file}".format( + in_file = ctx.files.in_files[i].short_path, + out_file = ctx.files.out_files[i].short_path, + ) + for i in range(len(ctx.files.in_files)) + ]), + ) + + return DefaultInfo( + executable = test, + runfiles = ctx.runfiles(files = [ctx.file.write_source_files_target] + ctx.files.in_files + ctx.files.out_files), + ) + +_write_source_files_test = rule( + implementation = _impl, + attrs = { + "write_source_files_target": attr.label( + allow_single_file = True, + executable = True, + cfg = "exec", + ), + "out_files": attr.label_list( + allow_files = True, + allow_empty = False, + mandatory = True, + ), + "in_files": attr.label_list( + allow_files = True, + allow_empty = False, + mandatory = True, + ), + }, + test = True, +) + +def write_source_files_test(name, in_files, out_files): + """Stamp a write_source_files executable and a test to run against it""" + + _write_source_files( + name = name + "_updater", + out_files = out_files, + in_files = in_files, + is_windows = False, + ) + + # Note that for testing we update the source files in the sandbox, + # not the actual source tree. + _write_source_files_test( + name = name, + write_source_files_target = name + "_updater", + out_files = out_files, + in_files = in_files, + ) diff --git a/lib/write_source_files.bzl b/lib/write_source_files.bzl new file mode 100644 index 0000000..cec1daa --- /dev/null +++ b/lib/write_source_files.bzl @@ -0,0 +1,131 @@ +"Public API for write_source_files" + +load("//lib/private:write_source_files.bzl", _lib = "write_source_files_lib") +load("//lib:utils.bzl", _to_label = "to_label") +load("@bazel_skylib//rules:diff_test.bzl", _diff_test = "diff_test") +load("//lib/private:fail_with_message_test.bzl", "fail_with_message_test") + +_write_source_files = rule( + attrs = _lib.attrs, + implementation = _lib.implementation, + executable = True, +) + +def write_source_files(name, files, **kwargs): + """Write to one or more files in the source tree. Stamp out tests that ensure the files exists and are up to date. + + Usage: + + ```starlark + load("@aspect_bazel_lib//lib:write_source_files.bzl", "write_source_files") + + write_source_files( + name = "write_foobar", + files = { + "foobar.json": "//some/generated:file", + }, + ) + ``` + + To update the source file, run: + ```bash + bazel run //:write_foobar + ``` + + A test will fail if the source file doesn't exist + ```bash + bazel test //... + + //:foobar.json does not exist. To create & update this file, run: + + bazel run //:write_foobar + ``` + + ...or if it's out of date. + ```bash + bazel test //... + + //:foobar.json is out-of-date. To update this file, run: + + bazel run //:write_foobar + ``` + + Args: + name: Name of the executable target that creates or updates the source file + files: A dict where the keys are source files to write to and the values are labels pointing to the desired content. + Source files must be within the same bazel package as the target. + **kwargs: Other common named parameters such as `tags` or `visibility` + """ + + out_files = files.keys() + in_files = [files[f] for f in out_files] + + # Stamp an executable rule that writes to the out file + _write_source_files( + name = name, + in_files = in_files, + out_files = out_files, + is_windows = select({ + "@bazel_tools//src/conditions:host_windows": True, + "//conditions:default": False, + }), + visibility = kwargs.get("visibility"), + tags = kwargs.get("tags"), + ) + + # Fail if user passes args that would conflict with stamped out targets below + if kwargs.pop("file1", None) != None: + fail("file1 not a valid parameter in write_source_file") + if kwargs.pop("file2", None) != None: + fail("file2 not a valid parameter in write_source_file") + if kwargs.pop("failure_message", None) != None: + fail("failure_message not a valid parameter in write_source_file") + + for i in range(len(out_files)): + out_file = _to_label(out_files[i]) + out_file_missing = _is_file_missing(out_file) + + name_test = "%s_%d_test" % (name, i) + + if out_file_missing: + # Stamp out a test that fails with a helpful message when the source file doesn't exist. + # Note that we cannot simply call fail() here since it will fail during the analysis + # phase and prevent the user from calling bazel run //update/the:file. + fail_with_message_test( + name = name_test, + message = """ + + %s does not exist. To create & update this file, run: + + bazel run //%s:%s + + """ % (out_file, native.package_name(), name), + visibility = kwargs.get("visibility"), + tags = kwargs.get("tags"), + ) + else: + # Stamp out a diff test the check that the source file is up to date + _diff_test( + name = name_test, + file1 = in_files[i], + file2 = out_file, + failure_message = """ + + %s is out-of-date. To update this file, run: + + bazel run //%s:%s + + """ % (out_file, native.package_name(), name), + **kwargs + ) + +def _is_file_missing(label): + """Check if a file is missing by passing its relative path through a glob() + + Args + label: the file's label + """ + file_abs = "%s/%s" % (label.package, label.name) + file_rel = file_abs[len(native.package_name()) + 1:] + file_glob = native.glob([file_rel]) + return len(file_glob) == 0