diff --git a/.prettierignore b/.prettierignore index 2e117bf..eb837ff 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1 +1,2 @@ docs/*.md +lib/tests/jq/*.json diff --git a/WORKSPACE b/WORKSPACE index 3ac7276..8e2c4c2 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -10,6 +10,10 @@ load(":internal_deps.bzl", "bazel_lib_internal_deps") # Fetch deps needed only locally for development bazel_lib_internal_deps() +load("//lib:repositories.bzl", "aspect_bazel_lib_dependencies") + +aspect_bazel_lib_dependencies() + # For running our own unit tests load("@bazel_skylib//lib:unittest.bzl", "register_unittest_toolchains") diff --git a/docs/BUILD.bazel b/docs/BUILD.bazel index f61f917..76333cb 100644 --- a/docs/BUILD.bazel +++ b/docs/BUILD.bazel @@ -33,6 +33,11 @@ stardoc_with_diff_test( out_label = "//docs:utils.md", ) +stardoc_with_diff_test( + bzl_library_target = "//lib:jq", + out_label = "//docs:jq.md", +) + update_docs( name = "update", docs_folder = "docs", diff --git a/docs/jq.md b/docs/jq.md new file mode 100755 index 0000000..62e3709 --- /dev/null +++ b/docs/jq.md @@ -0,0 +1,29 @@ + + +Public API for jq + + + +## jq + +
+jq(name, srcs, filter, args, out)
+
+ +Invoke jq with a filter on a set of json input files. + +For jq documentation, see https://stedolan.github.io/jq/. + + +**PARAMETERS** + + +| Name | Description | Default Value | +| :------------- | :------------- | :------------- | +| name | Name of the rule | none | +| srcs | List of input json files | none | +| filter | mandatory jq filter specification (https://stedolan.github.io/jq/manual/#Basicfilters) | none | +| args | additional args to pass to jq | [] | +| out | Name of the output json file; defaults to the rule name plus ".json" | None | + + diff --git a/internal_deps.bzl b/internal_deps.bzl index c8dd414..75df7c5 100644 --- a/internal_deps.bzl +++ b/internal_deps.bzl @@ -6,7 +6,9 @@ statement from these, that's a bug in our distribution. load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") +load("//lib:repositories.bzl", "register_jq_toolchains") +# buildifier: disable=unnamed-macro def bazel_lib_internal_deps(): "Fetch deps needed for local development" maybe( @@ -62,3 +64,6 @@ def bazel_lib_internal_deps(): "https://github.com/bazelbuild/stardoc/releases/download/0.5.0/stardoc-0.5.0.tar.gz", ], ) + + # Register toolchains for tests + register_jq_toolchains(version = "1.6") diff --git a/lib/BUILD.bazel b/lib/BUILD.bazel index 4639cab..976ea32 100644 --- a/lib/BUILD.bazel +++ b/lib/BUILD.bazel @@ -48,3 +48,15 @@ bzl_library( srcs = ["windows_utils.bzl"], visibility = ["//visibility:public"], ) + +bzl_library( + name = "jq", + srcs = ["jq.bzl"], + visibility = ["//visibility:public"], + deps = ["//lib/private:jq"], +) + +toolchain_type( + name = "jq_toolchain_type", + visibility = ["//visibility:public"], +) diff --git a/lib/jq.bzl b/lib/jq.bzl new file mode 100644 index 0000000..400f372 --- /dev/null +++ b/lib/jq.bzl @@ -0,0 +1,32 @@ +"""Public API for jq""" + +load("//lib/private:jq.bzl", _jq_lib = "jq_lib") + +_jq_rule = rule( + attrs = _jq_lib.attrs, + implementation = _jq_lib.implementation, + toolchains = ["@aspect_bazel_lib//lib:jq_toolchain_type"], +) + +def jq(name, srcs, filter, args = [], out = None): + """Invoke jq with a filter on a set of json input files. + + For jq documentation, see https://stedolan.github.io/jq/. + + Args: + name: Name of the rule + srcs: List of input json files + filter: mandatory jq filter specification (https://stedolan.github.io/jq/manual/#Basicfilters) + args: additional args to pass to jq + out: Name of the output json file; defaults to the rule name plus ".json" + """ + if not out: + out = name + ".json" + + _jq_rule( + name = name, + srcs = srcs, + filter = filter, + args = args, + out = out, + ) diff --git a/lib/private/BUILD.bazel b/lib/private/BUILD.bazel index d7e47f1..37a15bd 100644 --- a/lib/private/BUILD.bazel +++ b/lib/private/BUILD.bazel @@ -51,3 +51,9 @@ bzl_library( srcs = ["utils.bzl"], visibility = ["//lib:__subpackages__"], ) + +bzl_library( + name = "jq", + srcs = ["jq.bzl"], + visibility = ["//lib:__subpackages__"], +) diff --git a/lib/private/jq.bzl b/lib/private/jq.bzl new file mode 100644 index 0000000..d7c8734 --- /dev/null +++ b/lib/private/jq.bzl @@ -0,0 +1,46 @@ +"""Implementation for jq rule""" + +_jq_attrs = { + "srcs": attr.label_list( + allow_files = [".json"], + mandatory = True, + allow_empty = True, + ), + "filter": attr.string( + mandatory = True, + ), + "args": attr.string_list(), + "out": attr.output(mandatory = True), +} + +def _jq_impl(ctx): + jq_bin = ctx.toolchains["@aspect_bazel_lib//lib:jq_toolchain_type"].jqinfo.bin + + out = ctx.outputs.out + args = ctx.attr.args + + # jq hangs when there are no input sources unless --null-input flag is passed + if len(ctx.attr.srcs) == 0 and "-n" not in args and "--null-input" not in args: + args = args + ["--null-input"] + + cmd = "{jq} {args} '{filter}' {sources} > {out}".format( + jq = jq_bin.path, + args = " ".join(args), + filter = ctx.attr.filter, + sources = " ".join(["'%s'" % file.path for file in ctx.files.srcs]), + out = out.path, + ) + + ctx.actions.run_shell( + tools = [jq_bin], + inputs = ctx.files.srcs, + outputs = [out], + command = cmd, + ) + + return DefaultInfo(files = depset([out]), runfiles = ctx.runfiles([out])) + +jq_lib = struct( + attrs = _jq_attrs, + implementation = _jq_impl, +) diff --git a/lib/private/jq_toolchain.bzl b/lib/private/jq_toolchain.bzl new file mode 100644 index 0000000..5ae1036 --- /dev/null +++ b/lib/private/jq_toolchain.bzl @@ -0,0 +1,197 @@ +"Setup jq toolchain repositories and rules" + +JQ_PLATFORMS = { + "linux32": struct( + compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_32", + ], + ), + "linux64": struct( + compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + ), + "osx-amd64": struct( + compatible_with = [ + "@platforms//os:macos", + ], + ), + "win32": struct( + compatible_with = [ + "@platforms//os:windows", + "@platforms//cpu:x86_32", + ], + ), + "win64": struct( + compatible_with = [ + "@platforms//os:windows", + "@platforms//cpu:x86_64", + ], + ), +} + +# https://github.com/stedolan/jq/releases +# +# The integrity hashes can be computed with +# shasum -b -a 384 [downloaded file] | awk '{ print $1 }' | xxd -r -p | base64 +JQ_VERSIONS = { + "1.6": { + "linux32": "sha384-hBGwNC3R0WyEbDQnrabzvcURSSV9BGxVrUVXLCH1C+Ilo7YDlzfTJSr4gadVssVI", + "linux64": "sha384-+K6tuwxrC/P4WBYRJ7YXcpeLS7GesbbnUhq4r9w7k0lCUC1KlhyXXf0sFQgOg0dI", + "osx-amd64": "sha384-ZLZljM9OyKCJbJbv7s1SRYSeMbVxfRc6kFNUlk9U/IL10Xm2xr4cxx3SZKv93QFO", + "win32": "sha384-PO+MMFELa0agwy35NuKhrxn8C6GjNq8gnzL3NvYSWNx/pwClCl7yzCONGhLFknMc", + "win64": "sha384-O4qdyhb+0zU1XAuUKc1Mil5hlbSbCUcPQOGRtkJUqryv7X0IeKcMCIuZw97q9WGr", + }, + "1.5": { + "linux32": "sha-384MPO/DYgSPNRkrGEOCvZBZ8UvTdP4YVzXJoSYnWz9/IuywSRVqqyO6se9S72sue56", + "linux64": "sha384-/Su0ihtb867nCQTzQlTHjve+KpwfzsPws5ILj6hl7k33Qw+FwnyxAVITDh/pOOYw", + "osx-amd64": "sha384-X3VGwLkqaLafis82SySkqFPGIiJMdWdzcHPWLJ0q87XF+MGVc/e2n65a1yMBW6Nf", + "win32": "sha384-zZoz1F0nrhl5yvnGm37TxDw7dMWUQtJeDVmHfdAhLYMRGynIxefJgmB4Ty8gjNeu", + "win64": "sha384-NtaejeSFoKaXxxT1nPqxdOWRmIZAFF8wFTKjqs/4W0qYMYLohmO73AGKKR2XIg84", + }, +} + +JqInfo = provider( + doc = "Provide info for executing jq", + fields = { + "bin": "Executable jq binary", + }, +) + +def _jq_toolchain_impl(ctx): + binary = ctx.attr.bin.files.to_list()[0] + + # Make the $(JQ_BIN) variable available in places like genrules. + # See https://docs.bazel.build/versions/main/be/make-variables.html#custom_variables + template_variables = platform_common.TemplateVariableInfo({ + "JQ_BIN": binary.path, + }) + default_info = DefaultInfo( + files = depset([binary]), + runfiles = ctx.runfiles(files = [binary]), + ) + jq_info = JqInfo( + bin = binary, + ) + + # Export all the providers inside our ToolchainInfo + # so the resolved_toolchain rule can grab and re-export them. + toolchain_info = platform_common.ToolchainInfo( + jqinfo = jq_info, + template_variables = template_variables, + default = default_info, + ) + + return [default_info, toolchain_info, template_variables] + +jq_toolchain = rule( + implementation = _jq_toolchain_impl, + attrs = { + "bin": attr.label( + mandatory = True, + allow_single_file = True, + ), + }, +) + +def _jq_toolchains_repo_impl(repository_ctx): + # Expose a concrete toolchain which is the result of Bazel resolving the toolchain + # for the execution or target platform. + # Workaround for https://github.com/bazelbuild/bazel/issues/14009 + starlark_content = """# Generated by lib/private/toolchain.bzl + +# Forward all the providers +def _resolved_toolchain_impl(ctx): + toolchain_info = ctx.toolchains["@aspect_bazel_lib//lib:jq_toolchain_type"] + return [ + toolchain_info, + toolchain_info.default, + toolchain_info.jqinfo, + toolchain_info.template_variables, + ] + +# Copied from java_toolchain_alias +# https://cs.opensource.google/bazel/bazel/+/master:tools/jdk/java_toolchain_alias.bzl +resolved_toolchain = rule( + implementation = _resolved_toolchain_impl, + toolchains = ["@aspect_bazel_lib//lib:jq_toolchain_type"], + incompatible_use_toolchain_transition = True, +) +""" + repository_ctx.file("defs.bzl", starlark_content) + + build_content = """# Generated by lib/private/toolchain.bzl +# +# These can be registered in the workspace file or passed to --extra_toolchains flag. +# By default all these toolchains are registered by the jq_register_toolchains macro +# so you don't normally need to interact with these targets. + +load(":defs.bzl", "resolved_toolchain") + +resolved_toolchain(name = "resolved_toolchain", visibility = ["//visibility:public"]) + +""" + + for [platform, meta] in JQ_PLATFORMS.items(): + build_content += """ +toolchain( + name = "{platform}_toolchain", + exec_compatible_with = {compatible_with}, + target_compatible_with = {compatible_with}, + toolchain = "@{name}_{platform}//:jq_toolchain", + toolchain_type = "@aspect_bazel_lib//lib:jq_toolchain_type", +) +""".format( + platform = platform, + name = repository_ctx.attr.name, + user_repository_name = repository_ctx.attr.user_repository_name, + compatible_with = meta.compatible_with, + ) + + # Base BUILD file for this repository + repository_ctx.file("BUILD.bazel", build_content) + +jq_toolchains_repo = repository_rule( + _jq_toolchains_repo_impl, + doc = """Creates a repository with toolchain definitions for all known platforms + which can be registered or selected.""", + attrs = { + "user_repository_name": attr.string(doc = "Base name for toolchains repository"), + }, +) + +def _jq_platform_repo_impl(repository_ctx): + is_windows = repository_ctx.attr.platform == "win32" or repository_ctx.attr.platform == "win64" + url = "https://github.com/stedolan/jq/releases/download/jq-{0}/jq-{1}{2}".format( + repository_ctx.attr.jq_version, + repository_ctx.attr.platform, + ".exe" if is_windows else "", + ) + + repository_ctx.download( + url = url, + output = "jq.exe" if is_windows else "jq", + executable = True, + integrity = JQ_VERSIONS[repository_ctx.attr.jq_version][repository_ctx.attr.platform], + ) + build_content = """#Generated by lib/repositories.bzl +load("@aspect_bazel_lib//lib/private:jq_toolchain.bzl", "jq_toolchain") +jq_toolchain(name = "jq_toolchain", bin = select({ + "@bazel_tools//src/conditions:host_windows": ":jq.exe", + "//conditions:default": ":jq", +}), visibility = ["//visibility:public"]) +""" + + # Base BUILD file for this repository + repository_ctx.file("BUILD.bazel", build_content) + +jq_platform_repo = repository_rule( + implementation = _jq_platform_repo_impl, + doc = "Fetch external tools needed for jq toolchain", + attrs = { + "jq_version": attr.string(mandatory = True, values = JQ_VERSIONS.keys()), + "platform": attr.string(mandatory = True, values = JQ_PLATFORMS.keys()), + }, +) diff --git a/lib/repositories.bzl b/lib/repositories.bzl new file mode 100644 index 0000000..5e678d3 --- /dev/null +++ b/lib/repositories.bzl @@ -0,0 +1,36 @@ +"Macros for loading dependencies and registering toolchains" + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") +load("//lib/private:jq_toolchain.bzl", "JQ_PLATFORMS", "jq_platform_repo", "jq_toolchains_repo") + +def aspect_bazel_lib_dependencies(): + "Load dependencies required by aspect rules" + maybe( + http_archive, + name = "bazel_skylib", + sha256 = "c6966ec828da198c5d9adbaa94c05e3a1c7f21bd012a0b29ba8ddbccb2c93b0d", + urls = [ + "https://github.com/bazelbuild/bazel-skylib/releases/download/1.1.1/bazel-skylib-1.1.1.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.1.1/bazel-skylib-1.1.1.tar.gz", + ], + ) + +def register_jq_toolchains(version, name = "jq"): + """Registers jq toolchain and repositories + + Args: + version: the version of jq to execute (see https://github.com/stedolan/jq/releases) + name: override the prefix for the generated toolchain repositories + """ + for platform in JQ_PLATFORMS.keys(): + jq_platform_repo( + name = "%s_toolchains_%s" % (name, platform), + platform = platform, + jq_version = version, + ) + native.register_toolchains("@%s_toolchains//:%s_toolchain" % (name, platform)) + + jq_toolchains_repo( + name = "%s_toolchains" % name, + ) diff --git a/lib/tests/jq/BUILD.bazel b/lib/tests/jq/BUILD.bazel new file mode 100644 index 0000000..57a782a --- /dev/null +++ b/lib/tests/jq/BUILD.bazel @@ -0,0 +1,88 @@ +load("//lib/tests/jq:diff_test.bzl", "diff_test") +load("//lib:jq.bzl", "jq") + +# Identity filter produces identical json +jq( + name = "case_dot_filter", + srcs = ["a.json"], + filter = ".", +) + +diff_test( + name = "case_dot_filter_test", + file1 = "a_pretty.json", + file2 = ":case_dot_filter", +) + +# Merge filter with slurp merges two jsons +jq( + name = "case_merge_filter", + srcs = [ + "a.json", + "b.json", + ], + args = ["--slurp"], + filter = ".[0] * .[1]", +) + +diff_test( + name = "case_merge_filter_test", + file1 = "a_b_merged.json", + file2 = ":case_merge_filter", +) + +# Use predeclared output +jq( + name = "case_predeclared_output", + srcs = ["a.json"], + out = "foo.json", + filter = ".", +) + +diff_test( + name = "case_predeclared_output_test", + file1 = "a_pretty.json", + file2 = "foo.json", +) + +# No sources produces null (equivalent to --null-input) +jq( + name = "case_no_sources", + srcs = [], + filter = ".", +) + +diff_test( + name = "case_no_sources_test", + file1 = ":case_no_sources", + file2 = "null.json", +) + +# Sources with --null-input flag produces null +jq( + name = "case_null_input_flag", + srcs = ["a.json"], + args = ["--null-input"], + filter = ".", +) + +diff_test( + name = "case_null_input_flag_test", + file1 = ":case_null_input_flag", + file2 = "null.json", +) + +# Call jq within a genrule +genrule( + name = "case_genrule", + srcs = ["a.json"], + outs = ["genrule_output.json"], + cmd = "$(JQ_BIN) '.' $(location a.json) > $@", + toolchains = ["@jq_toolchains//:resolved_toolchain"], +) + +diff_test( + name = "case_genrule_test", + file1 = "genrule_output.json", + file2 = "a_pretty.json", +) diff --git a/lib/tests/jq/a.json b/lib/tests/jq/a.json new file mode 100644 index 0000000..cb88af3 --- /dev/null +++ b/lib/tests/jq/a.json @@ -0,0 +1,6 @@ +{ + "foo": "bar", + "value": 123, + "moo": [1, 2, 3], + "a": true +} diff --git a/lib/tests/jq/a_b_merged.json b/lib/tests/jq/a_b_merged.json new file mode 100644 index 0000000..21ecc72 --- /dev/null +++ b/lib/tests/jq/a_b_merged.json @@ -0,0 +1,11 @@ +{ + "foo": "baz", + "value": 456, + "moo": [ + 4, + 5, + 6 + ], + "a": true, + "b": true +} diff --git a/lib/tests/jq/a_pretty.json b/lib/tests/jq/a_pretty.json new file mode 100644 index 0000000..d306c3e --- /dev/null +++ b/lib/tests/jq/a_pretty.json @@ -0,0 +1,10 @@ +{ + "foo": "bar", + "value": 123, + "moo": [ + 1, + 2, + 3 + ], + "a": true +} diff --git a/lib/tests/jq/b.json b/lib/tests/jq/b.json new file mode 100644 index 0000000..f5582a7 --- /dev/null +++ b/lib/tests/jq/b.json @@ -0,0 +1,6 @@ +{ + "foo": "baz", + "value": 456, + "moo": [4, 5, 6], + "b": true +} diff --git a/lib/tests/jq/diff_test.bzl b/lib/tests/jq/diff_test.bzl new file mode 100644 index 0000000..acf4645 --- /dev/null +++ b/lib/tests/jq/diff_test.bzl @@ -0,0 +1,36 @@ +"""Override diff_test behaviour to ignore carriage returns in order to +test jq output on Windows. See https://github.com/stedolan/jq/issues/92. +""" + +load("@bazel_skylib//rules:diff_test.bzl", _diff_test = "diff_test") + +def diff_test(name, file1, file2): + """Perform a diff_test ignoring carriage returns + + Args: + name: name of the test rule + file1: first file to compare + file2: second file to compare + """ + test_files = [] + for i, file in enumerate([file1, file2], start = 1): + if file[0] == ":": + target = file[1:] + else: + target = file + + stripped_file = "%s_file%d_stripped" % (name, i) + + native.genrule( + name = "%s_file%d" % (name, i), + srcs = [file], + outs = [stripped_file], + cmd = "cat $(execpath :{target}) | sed \"s#\\r##\" > $@".format(target = target), + ) + test_files.append(stripped_file) + + _diff_test( + name = name, + file1 = test_files[0], + file2 = test_files[1], + ) diff --git a/lib/tests/jq/null.json b/lib/tests/jq/null.json new file mode 100644 index 0000000..19765bd --- /dev/null +++ b/lib/tests/jq/null.json @@ -0,0 +1 @@ +null