feat: add jq toolchain and rule

.prettierignore

@@ -1 +1,2 @@
 docs/*.md
+lib/tests/jq/*.json

WORKSPACE

@@ -10,6 +10,10 @@ load(":internal_deps.bzl", "bazel_lib_internal_deps")
 # Fetch deps needed only locally for development
 bazel_lib_internal_deps()
 
+load("//lib:repositories.bzl", "aspect_bazel_lib_dependencies")
+
+aspect_bazel_lib_dependencies()
+
 # For running our own unit tests
 load("@bazel_skylib//lib:unittest.bzl", "register_unittest_toolchains")
 

docs/BUILD.bazel

@@ -33,6 +33,11 @@ stardoc_with_diff_test(
     out_label = "//docs:utils.md",
 )
 
+stardoc_with_diff_test(
+    bzl_library_target = "//lib:jq",
+    out_label = "//docs:jq.md",
+)
+
 update_docs(
     name = "update",
     docs_folder = "docs",

docs/jq.md

@@ -0,0 +1,29 @@
+<!-- Generated with Stardoc: http://skydoc.bazel.build -->
+
+Public API for jq
+
+<a id="#jq"></a>
+
+## jq
+
+<pre>
+jq(<a href="#jq-name">name</a>, <a href="#jq-srcs">srcs</a>, <a href="#jq-filter">filter</a>, <a href="#jq-args">args</a>, <a href="#jq-out">out</a>)
+</pre>
+
+Invoke jq with a filter on a set of json input files.
+
+For jq documentation, see https://stedolan.github.io/jq/.
+
+
+**PARAMETERS**
+
+
+| Name  | Description | Default Value |
+| :------------- | :------------- | :------------- |
+| <a id="jq-name"></a>name |  Name of the rule   |  none |
+| <a id="jq-srcs"></a>srcs |  List of input json files   |  none |
+| <a id="jq-filter"></a>filter |  mandatory jq filter specification (https://stedolan.github.io/jq/manual/#Basicfilters)   |  none |
+| <a id="jq-args"></a>args |  additional args to pass to jq   |  <code>[]</code> |
+| <a id="jq-out"></a>out |  Name of the output json file; defaults to the rule name plus ".json"   |  <code>None</code> |
+
+

internal_deps.bzl

@@ -6,7 +6,9 @@ statement from these, that's a bug in our distribution.
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
+load("//lib:repositories.bzl", "register_jq_toolchains")
 
+# buildifier: disable=unnamed-macro
 def bazel_lib_internal_deps():
     "Fetch deps needed for local development"
     maybe(
@@ -62,3 +64,6 @@ def bazel_lib_internal_deps():
             "https://github.com/bazelbuild/stardoc/releases/download/0.5.0/stardoc-0.5.0.tar.gz",
         ],
     )
+
+    # Register toolchains for tests
+    register_jq_toolchains(version = "1.6")

lib/BUILD.bazel

@@ -48,3 +48,15 @@ bzl_library(
     srcs = ["windows_utils.bzl"],
     visibility = ["//visibility:public"],
 )
+
+bzl_library(
+    name = "jq",
+    srcs = ["jq.bzl"],
+    visibility = ["//visibility:public"],
+    deps = ["//lib/private:jq"],
+)
+
+toolchain_type(
+    name = "jq_toolchain_type",
+    visibility = ["//visibility:public"],
+)

lib/jq.bzl

@@ -0,0 +1,32 @@
+"""Public API for jq"""
+
+load("//lib/private:jq.bzl", _jq_lib = "jq_lib")
+
+_jq_rule = rule(
+    attrs = _jq_lib.attrs,
+    implementation = _jq_lib.implementation,
+    toolchains = ["@aspect_bazel_lib//lib:jq_toolchain_type"],
+)
+
+def jq(name, srcs, filter, args = [], out = None):
+    """Invoke jq with a filter on a set of json input files.
+
+    For jq documentation, see https://stedolan.github.io/jq/.
+
+    Args:
+        name: Name of the rule
+        srcs: List of input json files
+        filter: mandatory jq filter specification (https://stedolan.github.io/jq/manual/#Basicfilters)
+        args: additional args to pass to jq
+        out: Name of the output json file; defaults to the rule name plus ".json"
+    """
+    if not out:
+        out = name + ".json"
+
+    _jq_rule(
+        name = name,
+        srcs = srcs,
+        filter = filter,
+        args = args,
+        out = out,
+    )

lib/private/BUILD.bazel

@@ -51,3 +51,9 @@ bzl_library(
     srcs = ["utils.bzl"],
     visibility = ["//lib:__subpackages__"],
 )
+
+bzl_library(
+    name = "jq",
+    srcs = ["jq.bzl"],
+    visibility = ["//lib:__subpackages__"],
+)

lib/private/jq.bzl

@@ -0,0 +1,46 @@
+"""Implementation for jq rule"""
+
+_jq_attrs = {
+    "srcs": attr.label_list(
+        allow_files = [".json"],
+        mandatory = True,
+        allow_empty = True,
+    ),
+    "filter": attr.string(
+        mandatory = True,
+    ),
+    "args": attr.string_list(),
+    "out": attr.output(mandatory = True),
+}
+
+def _jq_impl(ctx):
+    jq_bin = ctx.toolchains["@aspect_bazel_lib//lib:jq_toolchain_type"].jqinfo.bin
+
+    out = ctx.outputs.out
+    args = ctx.attr.args
+
+    # jq hangs when there are no input sources unless --null-input flag is passed
+    if len(ctx.attr.srcs) == 0 and "-n" not in args and "--null-input" not in args:
+        args = args + ["--null-input"]
+
+    cmd = "{jq} {args} '{filter}' {sources} > {out}".format(
+        jq = jq_bin.path,
+        args = " ".join(args),
+        filter = ctx.attr.filter,
+        sources = " ".join(["'%s'" % file.path for file in ctx.files.srcs]),
+        out = out.path,
+    )
+
+    ctx.actions.run_shell(
+        tools = [jq_bin],
+        inputs = ctx.files.srcs,
+        outputs = [out],
+        command = cmd,
+    )
+
+    return DefaultInfo(files = depset([out]), runfiles = ctx.runfiles([out]))
+
+jq_lib = struct(
+    attrs = _jq_attrs,
+    implementation = _jq_impl,
+)

lib/private/jq_toolchain.bzl

@@ -0,0 +1,197 @@
+"Setup jq toolchain repositories and rules"
+
+JQ_PLATFORMS = {
+    "linux32": struct(
+        compatible_with = [
+            "@platforms//os:linux",
+            "@platforms//cpu:x86_32",
+        ],
+    ),
+    "linux64": struct(
+        compatible_with = [
+            "@platforms//os:linux",
+            "@platforms//cpu:x86_64",
+        ],
+    ),
+    "osx-amd64": struct(
+        compatible_with = [
+            "@platforms//os:macos",
+        ],
+    ),
+    "win32": struct(
+        compatible_with = [
+            "@platforms//os:windows",
+            "@platforms//cpu:x86_32",
+        ],
+    ),
+    "win64": struct(
+        compatible_with = [
+            "@platforms//os:windows",
+            "@platforms//cpu:x86_64",
+        ],
+    ),
+}
+
+# https://github.com/stedolan/jq/releases
+#
+# The integrity hashes can be computed with
+# shasum -b -a 384 [downloaded file] | awk '{ print $1 }' | xxd -r -p | base64
+JQ_VERSIONS = {
+    "1.6": {
+        "linux32": "sha384-hBGwNC3R0WyEbDQnrabzvcURSSV9BGxVrUVXLCH1C+Ilo7YDlzfTJSr4gadVssVI",
+        "linux64": "sha384-+K6tuwxrC/P4WBYRJ7YXcpeLS7GesbbnUhq4r9w7k0lCUC1KlhyXXf0sFQgOg0dI",
+        "osx-amd64": "sha384-ZLZljM9OyKCJbJbv7s1SRYSeMbVxfRc6kFNUlk9U/IL10Xm2xr4cxx3SZKv93QFO",
+        "win32": "sha384-PO+MMFELa0agwy35NuKhrxn8C6GjNq8gnzL3NvYSWNx/pwClCl7yzCONGhLFknMc",
+        "win64": "sha384-O4qdyhb+0zU1XAuUKc1Mil5hlbSbCUcPQOGRtkJUqryv7X0IeKcMCIuZw97q9WGr",
+    },
+    "1.5": {
+        "linux32": "sha-384MPO/DYgSPNRkrGEOCvZBZ8UvTdP4YVzXJoSYnWz9/IuywSRVqqyO6se9S72sue56",
+        "linux64": "sha384-/Su0ihtb867nCQTzQlTHjve+KpwfzsPws5ILj6hl7k33Qw+FwnyxAVITDh/pOOYw",
+        "osx-amd64": "sha384-X3VGwLkqaLafis82SySkqFPGIiJMdWdzcHPWLJ0q87XF+MGVc/e2n65a1yMBW6Nf",
+        "win32": "sha384-zZoz1F0nrhl5yvnGm37TxDw7dMWUQtJeDVmHfdAhLYMRGynIxefJgmB4Ty8gjNeu",
+        "win64": "sha384-NtaejeSFoKaXxxT1nPqxdOWRmIZAFF8wFTKjqs/4W0qYMYLohmO73AGKKR2XIg84",
+    },
+}
+
+JqInfo = provider(
+    doc = "Provide info for executing jq",
+    fields = {
+        "bin": "Executable jq binary",
+    },
+)
+
+def _jq_toolchain_impl(ctx):
+    binary = ctx.attr.bin.files.to_list()[0]
+
+    # Make the $(JQ_BIN) variable available in places like genrules.
+    # See https://docs.bazel.build/versions/main/be/make-variables.html#custom_variables
+    template_variables = platform_common.TemplateVariableInfo({
+        "JQ_BIN": binary.path,
+    })
+    default_info = DefaultInfo(
+        files = depset([binary]),
+        runfiles = ctx.runfiles(files = [binary]),
+    )
+    jq_info = JqInfo(
+        bin = binary,
+    )
+
+    # Export all the providers inside our ToolchainInfo
+    # so the resolved_toolchain rule can grab and re-export them.
+    toolchain_info = platform_common.ToolchainInfo(
+        jqinfo = jq_info,
+        template_variables = template_variables,
+        default = default_info,
+    )
+
+    return [default_info, toolchain_info, template_variables]
+
+jq_toolchain = rule(
+    implementation = _jq_toolchain_impl,
+    attrs = {
+        "bin": attr.label(
+            mandatory = True,
+            allow_single_file = True,
+        ),
+    },
+)
+
+def _jq_toolchains_repo_impl(repository_ctx):
+    # Expose a concrete toolchain which is the result of Bazel resolving the toolchain
+    # for the execution or target platform.
+    # Workaround for https://github.com/bazelbuild/bazel/issues/14009
+    starlark_content = """# Generated by lib/private/toolchain.bzl
+
+# Forward all the providers
+def _resolved_toolchain_impl(ctx):
+    toolchain_info = ctx.toolchains["@aspect_bazel_lib//lib:jq_toolchain_type"]
+    return [
+        toolchain_info,
+        toolchain_info.default,
+        toolchain_info.jqinfo,
+        toolchain_info.template_variables,
+    ]
+
+# Copied from java_toolchain_alias
+# https://cs.opensource.google/bazel/bazel/+/master:tools/jdk/java_toolchain_alias.bzl
+resolved_toolchain = rule(
+    implementation = _resolved_toolchain_impl,
+    toolchains = ["@aspect_bazel_lib//lib:jq_toolchain_type"],
+    incompatible_use_toolchain_transition = True,
+)
+"""
+    repository_ctx.file("defs.bzl", starlark_content)
+
+    build_content = """# Generated by lib/private/toolchain.bzl
+#
+# These can be registered in the workspace file or passed to --extra_toolchains flag.
+# By default all these toolchains are registered by the jq_register_toolchains macro
+# so you don't normally need to interact with these targets.
+
+load(":defs.bzl", "resolved_toolchain")
+
+resolved_toolchain(name = "resolved_toolchain", visibility = ["//visibility:public"])
+
+"""
+
+    for [platform, meta] in JQ_PLATFORMS.items():
+        build_content += """
+toolchain(
+    name = "{platform}_toolchain",
+    exec_compatible_with = {compatible_with},
+    target_compatible_with = {compatible_with},
+    toolchain = "@{name}_{platform}//:jq_toolchain",
+    toolchain_type = "@aspect_bazel_lib//lib:jq_toolchain_type",
+)
+""".format(
+            platform = platform,
+            name = repository_ctx.attr.name,
+            user_repository_name = repository_ctx.attr.user_repository_name,
+            compatible_with = meta.compatible_with,
+        )
+
+    # Base BUILD file for this repository
+    repository_ctx.file("BUILD.bazel", build_content)
+
+jq_toolchains_repo = repository_rule(
+    _jq_toolchains_repo_impl,
+    doc = """Creates a repository with toolchain definitions for all known platforms
+     which can be registered or selected.""",
+    attrs = {
+        "user_repository_name": attr.string(doc = "Base name for toolchains repository"),
+    },
+)
+
+def _jq_platform_repo_impl(repository_ctx):
+    is_windows = repository_ctx.attr.platform == "win32" or repository_ctx.attr.platform == "win64"
+    url = "https://github.com/stedolan/jq/releases/download/jq-{0}/jq-{1}{2}".format(
+        repository_ctx.attr.jq_version,
+        repository_ctx.attr.platform,
+        ".exe" if is_windows else "",
+    )
+
+    repository_ctx.download(
+        url = url,
+        output = "jq.exe" if is_windows else "jq",
+        executable = True,
+        integrity = JQ_VERSIONS[repository_ctx.attr.jq_version][repository_ctx.attr.platform],
+    )
+    build_content = """#Generated by lib/repositories.bzl
+load("@aspect_bazel_lib//lib/private:jq_toolchain.bzl", "jq_toolchain")
+jq_toolchain(name = "jq_toolchain", bin = select({
+    "@bazel_tools//src/conditions:host_windows": ":jq.exe",
+    "//conditions:default": ":jq",
+}), visibility = ["//visibility:public"])
+"""
+
+    # Base BUILD file for this repository
+    repository_ctx.file("BUILD.bazel", build_content)
+
+jq_platform_repo = repository_rule(
+    implementation = _jq_platform_repo_impl,
+    doc = "Fetch external tools needed for jq toolchain",
+    attrs = {
+        "jq_version": attr.string(mandatory = True, values = JQ_VERSIONS.keys()),
+        "platform": attr.string(mandatory = True, values = JQ_PLATFORMS.keys()),
+    },
+)

lib/repositories.bzl

@@ -0,0 +1,36 @@
+"Macros for loading dependencies and registering toolchains"
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
+load("//lib/private:jq_toolchain.bzl", "JQ_PLATFORMS", "jq_platform_repo", "jq_toolchains_repo")
+
+def aspect_bazel_lib_dependencies():
+    "Load dependencies required by aspect rules"
+    maybe(
+        http_archive,
+        name = "bazel_skylib",
+        sha256 = "c6966ec828da198c5d9adbaa94c05e3a1c7f21bd012a0b29ba8ddbccb2c93b0d",
+        urls = [
+            "https://github.com/bazelbuild/bazel-skylib/releases/download/1.1.1/bazel-skylib-1.1.1.tar.gz",
+            "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.1.1/bazel-skylib-1.1.1.tar.gz",
+        ],
+    )
+
+def register_jq_toolchains(version, name = "jq"):
+    """Registers jq toolchain and repositories
+
+    Args:
+        version: the version of jq to execute (see https://github.com/stedolan/jq/releases)
+        name: override the prefix for the generated toolchain repositories
+    """
+    for platform in JQ_PLATFORMS.keys():
+        jq_platform_repo(
+            name = "%s_toolchains_%s" % (name, platform),
+            platform = platform,
+            jq_version = version,
+        )
+        native.register_toolchains("@%s_toolchains//:%s_toolchain" % (name, platform))
+
+    jq_toolchains_repo(
+        name = "%s_toolchains" % name,
+    )

lib/tests/jq/BUILD.bazel

@@ -0,0 +1,88 @@
+load("//lib/tests/jq:diff_test.bzl", "diff_test")
+load("//lib:jq.bzl", "jq")
+
+# Identity filter produces identical json
+jq(
+    name = "case_dot_filter",
+    srcs = ["a.json"],
+    filter = ".",
+)
+
+diff_test(
+    name = "case_dot_filter_test",
+    file1 = "a_pretty.json",
+    file2 = ":case_dot_filter",
+)
+
+# Merge filter with slurp merges two jsons
+jq(
+    name = "case_merge_filter",
+    srcs = [
+        "a.json",
+        "b.json",
+    ],
+    args = ["--slurp"],
+    filter = ".[0] * .[1]",
+)
+
+diff_test(
+    name = "case_merge_filter_test",
+    file1 = "a_b_merged.json",
+    file2 = ":case_merge_filter",
+)
+
+# Use predeclared output
+jq(
+    name = "case_predeclared_output",
+    srcs = ["a.json"],
+    out = "foo.json",
+    filter = ".",
+)
+
+diff_test(
+    name = "case_predeclared_output_test",
+    file1 = "a_pretty.json",
+    file2 = "foo.json",
+)
+
+# No sources produces null (equivalent to --null-input)
+jq(
+    name = "case_no_sources",
+    srcs = [],
+    filter = ".",
+)
+
+diff_test(
+    name = "case_no_sources_test",
+    file1 = ":case_no_sources",
+    file2 = "null.json",
+)
+
+# Sources with --null-input flag produces null
+jq(
+    name = "case_null_input_flag",
+    srcs = ["a.json"],
+    args = ["--null-input"],
+    filter = ".",
+)
+
+diff_test(
+    name = "case_null_input_flag_test",
+    file1 = ":case_null_input_flag",
+    file2 = "null.json",
+)
+
+# Call jq within a genrule
+genrule(
+    name = "case_genrule",
+    srcs = ["a.json"],
+    outs = ["genrule_output.json"],
+    cmd = "$(JQ_BIN) '.' $(location a.json) > $@",
+    toolchains = ["@jq_toolchains//:resolved_toolchain"],
+)
+
+diff_test(
+    name = "case_genrule_test",
+    file1 = "genrule_output.json",
+    file2 = "a_pretty.json",
+)

lib/tests/jq/a.json

@@ -0,0 +1,6 @@
+{
+    "foo": "bar",
+    "value": 123,
+    "moo": [1, 2, 3],
+    "a": true
+}

lib/tests/jq/a_b_merged.json

@@ -0,0 +1,11 @@
+{
+  "foo": "baz",
+  "value": 456,
+  "moo": [
+    4,
+    5,
+    6
+  ],
+  "a": true,
+  "b": true
+}

lib/tests/jq/a_pretty.json

@@ -0,0 +1,10 @@
+{
+  "foo": "bar",
+  "value": 123,
+  "moo": [
+    1,
+    2,
+    3
+  ],
+  "a": true
+}

lib/tests/jq/b.json

@@ -0,0 +1,6 @@
+{
+    "foo": "baz",
+    "value": 456,
+    "moo": [4, 5, 6],
+    "b": true
+}

lib/tests/jq/diff_test.bzl

@@ -0,0 +1,36 @@
+"""Override diff_test behaviour to ignore carriage returns in order to
+test jq output on Windows. See https://github.com/stedolan/jq/issues/92.
+"""
+
+load("@bazel_skylib//rules:diff_test.bzl", _diff_test = "diff_test")
+
+def diff_test(name, file1, file2):
+    """Perform a diff_test ignoring carriage returns
+
+    Args:
+        name: name of the test rule
+        file1: first file to compare
+        file2: second file to compare
+    """
+    test_files = []
+    for i, file in enumerate([file1, file2], start = 1):
+        if file[0] == ":":
+            target = file[1:]
+        else:
+            target = file
+
+        stripped_file = "%s_file%d_stripped" % (name, i)
+
+        native.genrule(
+            name = "%s_file%d" % (name, i),
+            srcs = [file],
+            outs = [stripped_file],
+            cmd = "cat $(execpath :{target}) | sed \"s#\\r##\" > $@".format(target = target),
+        )
+        test_files.append(stripped_file)
+
+    _diff_test(
+        name = name,
+        file1 = test_files[0],
+        file2 = test_files[1],
+    )

lib/tests/jq/null.json

@@ -0,0 +1 @@
+null