chore: use coreutils hashsum (#333)

.bcr/presubmit.yml

@@ -1,5 +1,5 @@
 bcr_test_module:
-  module_path: "e2e/bzlmod"
+  module_path: "e2e/smoke"
   matrix:
     platform: ["debian10", "macos", "ubuntu2004", "windows"]
   tasks:

.github/workflows/ci.yaml

@@ -83,22 +83,20 @@ jobs:
         config: ${{ fromJSON(needs.matrix-prep-config.outputs.configs) }}
         bazelversion: ${{ fromJSON(needs.matrix-prep-bazelversion.outputs.bazelversions) }}
         os: ${{ fromJSON(needs.matrix-prep-os.outputs.os) }}
+        bzlmodEnabled: [true, false]
         folder:
           - "."
-          - "e2e/bzlmod"
+          - "e2e/coreutils"
           - "e2e/copy_to_directory"
-          - "e2e/workspace"
+          - "e2e/smoke"
         exclude:
-          # bzlmod broken at 5.3.2 which the RBE:
-          # ```
-          # ERROR: /home/runner/work/bazel-lib/bazel-lib/e2e/bzlmod/BUILD.bazel:37:10: While resolving
-          # toolchains for target //:test: com.google.devtools.build.lib.packages.BuildFileNotFoundException:
-          # no such package '@aspect_bazel_lib//platforms': The repository '@aspect_bazel_lib' could not be
-          # resolved: Repository '@aspect_bazel_lib' is not defined
-          # ```
-          - config: rbe
+          # this e2e meant to work under bzlmod
+          - bzlmodEnabled: false
+            folder: bzlmod_write_source_files_external
+
+          # Don't test Bazel 5.3 with bzlmod. (Unrecognized option: --enable_bzlmod)
+          - bzlmodEnabled: true
             bazelversion: 5.3.2
-            folder: e2e/bzlmod
           # Don't test MacOS with RBE to minimize MacOS minutes (billed at 10X)
           - config: rbe
             os: macos-latest
@@ -164,6 +162,14 @@ jobs:
             rm -Force .aspect/bazelrc/bazel6.bazelrc
           }
 
+          
+      - name: Set bzlmod flag
+        # Store the --enable_bzlmod flag that we add to the test command below
+        # only when we're running bzlmod in our test matrix.
+        id: set_bzlmod_flag
+        if: matrix.bzlmodEnabled
+        run: echo "bzlmod_flag=--enable_bzlmod" >> $GITHUB_OUTPUT
+        
       - name: Write engflow credentials
         if: ${{ matrix.config == 'rbe' }}
         working-directory: ${{ matrix.folder }}
@@ -200,6 +206,7 @@ jobs:
             --bazelrc=${{ github.workspace }}/.github/workflows/ci.bazelrc `
             --bazelrc=${{ github.workspace }}/.aspect/bazelrc/bazel$BAZEL_MAJOR_VERSION.bazelrc `
             --bazelrc=.bazelrc `
+            ${{ steps.set_bzlmod_flag.outputs.bzlmod_flag }} 
             test --config=${{ matrix.config }} //...
         env:
           # Bazelisk will download bazel to here

MODULE.bazel

@@ -13,11 +13,12 @@ bazel_dep(name = "stardoc", repo_name = "io_bazel_stardoc", version = "0.5.0")
 
 ext = use_extension("@aspect_bazel_lib//lib:extensions.bzl", "ext")
 
-use_repo(ext, "copy_directory_toolchains", "copy_to_directory_toolchains", "jq_toolchains", "yq_toolchains")
+use_repo(ext, "copy_directory_toolchains", "copy_to_directory_toolchains", "jq_toolchains", "yq_toolchains", "coreutils_toolchains")
 
 register_toolchains(
     "@copy_directory_toolchains//:all",
     "@copy_to_directory_toolchains//:all",
     "@jq_toolchains//:all",
     "@yq_toolchains//:all",
+    "@coreutils_toolchains//:all"
 )

e2e/bzlmod/BUILD.bazel

@@ -1,95 +0,0 @@
-"""Integration testing that aspect_bazel_lib works when used via bzlmod.
-
-NB: We don't use yq, so we can confirm that Bazel never fetches it.
-You can manually verify this after testing this repo with
-`ls $(bazel info output_base)/external | grep yq`
-You'll see a aspect_bazel_lib.ext.yq_toolchains repo, but no downloaded yq binary.
-"""
-
-load("@aspect_bazel_lib//lib:copy_directory.bzl", "copy_directory")
-load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory")
-load("@aspect_bazel_lib//lib:diff_test.bzl", "diff_test")
-load("@aspect_bazel_lib//lib:jq.bzl", "jq")
-load("@aspect_bazel_lib//lib:yq.bzl", "yq")
-load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
-load("@aspect_bazel_lib_host//:defs.bzl", "host")
-
-bzl_library(
-    name = "defs",
-    srcs = ["defs.bzl"],
-)
-
-# Validate that stardoc dependency works.
-# Note, stardoc is generally broken under bzlmod, see
-# https://github.com/bazelbuild/stardoc/issues/117
-# This happens to work because we don't reference any external repos
-# from defs.bzl.
-# TODO: re-enable this once it works
-# load("@aspect_bazel_lib//lib:docs.bzl", "stardoc_with_diff_test")
-# stardoc_with_diff_test(
-#     name = "docs",
-#     bzl_library_target = "//:defs",
-# )
-
-# Validate that JQ works and resolves its toolchain
-jq(
-    name = "jq_case_no_sources",
-    srcs = [],
-    filter = ".",
-)
-
-diff_test(
-    name = "jq_test",
-    file1 = "jq_case_no_sources",
-    file2 = "expected_jq",
-)
-
-# Validate that YQ works and resolves its toolchain
-yq(
-    name = "yq_case_no_sources",
-    srcs = [],
-    expression = ".",
-)
-
-diff_test(
-    name = "yq_test",
-    file1 = "yq_case_no_sources",
-    file2 = "expected_yq",
-)
-
-# Validate that copy_directory works and resolves its toolchain
-copy_directory(
-    name = "copy_directory_case",
-    src = "d",
-    out = "d_out",
-)
-
-diff_test(
-    name = "copy_directory_test",
-    file1 = "d",
-    file2 = "copy_directory_case",
-    # Source directories are not support on remote execution.
-    tags = ["no-remote-exec"],
-)
-
-# Validate that copy_to_directory works and resolves its toolchain
-copy_to_directory(
-    name = "copy_to_directory_case",
-    srcs = ["d"],
-    out = "d2_out",
-    replace_prefixes = {"d/": ""},
-)
-
-diff_test(
-    name = "copy_to_directory_test",
-    file1 = "d",
-    file2 = "copy_to_directory_case",
-    # Source directories are not support on remote execution.
-    tags = ["no-remote-exec"],
-)
-
-genrule(
-    name = "host_bazel_version",
-    outs = ["host_bazel_version.txt"],
-    cmd = "echo '%s' > $@" % host.bazel_version,
-)

e2e/bzlmod/WORKSPACE

@@ -1,2 +0,0 @@
-# Marker file that this folder is the root of a Bazel workspace.
-# See MODULE.bazel for dependencies and setup.

e2e/bzlmod/defs.bzl

@@ -1 +0,0 @@
-# no rules

e2e/bzlmod/docs.md

@@ -1,4 +0,0 @@
-<!-- Generated with Stardoc: http://skydoc.bazel.build -->
-
-
-

e2e/bzlmod_write_source_files_external/MODULE.bazel

@@ -1,5 +1,5 @@
 module(
-    name = "e2e_bzlmod_write_source_files_external",
+    name = "bzlmod_write_source_files_external",
     compatibility_level = 1,
     version = "0.0.0",
 )

e2e/copy_to_directory/MODULE.bazel

@@ -1,5 +1,5 @@
 module(
-    name = "e2e_bzlmod",
+    name = "copy_to_directory",
     compatibility_level = 1,
     version = "0.0.0",
 )

e2e/coreutils/BUILD.bazel

@@ -0,0 +1,40 @@
+load("@aspect_bazel_lib//lib:write_source_files.bzl", "write_source_files")
+load("@aspect_bazel_lib//tools:hashes.bzl", "hashes")
+
+genrule(
+    name = "ls",
+    outs = ["ls.txt"],
+    cmd = "$(COREUTILS_BIN) ls > $@",
+    toolchains = ["@coreutils_toolchains//:resolved_toolchain"],
+)
+
+hashes(
+    name = "hash",
+    src = "test.bin"
+)
+
+filegroup(
+    name = "md5",
+    output_group = "md5",
+    srcs = [":hash"]
+)
+
+filegroup(
+    name = "sha1",
+    output_group = "sha1",
+    srcs = [":hash"]
+)
+
+
+# This tests that the "in" file to write_source_files can be a
+# label to an external repository target when bzlmod is enabled.
+write_source_files(
+    name = "write_ls",
+    files = {
+        "ls.txt.expected": ":ls.txt",
+        "sha256.txt": ":hash",
+        "sha1.txt": ":sha1",
+        "md5.txt": ":md5"
+    },
+)
+

e2e/coreutils/MODULE.bazel

@@ -0,0 +1,15 @@
+module(
+    name = "coreutils",
+    compatibility_level = 1,
+    version = "0.0.0",
+)
+
+bazel_dep(name = "aspect_bazel_lib", version = "0.0.0")
+
+local_path_override(
+    module_name = "aspect_bazel_lib",
+    path = "../..",
+)
+
+ext = use_extension("@aspect_bazel_lib//lib:extensions.bzl", "ext")
+use_repo(ext, "coreutils_toolchains")

e2e/coreutils/WORKSPACE

@@ -0,0 +1,10 @@
+local_repository(
+    name = "aspect_bazel_lib",
+    path = "../..",
+)
+
+load("@aspect_bazel_lib//lib:repositories.bzl", "aspect_bazel_lib_dependencies", "register_coreutils_toolchains")
+
+aspect_bazel_lib_dependencies()
+
+register_coreutils_toolchains()

e2e/coreutils/ls.txt.expected

@@ -0,0 +1,2 @@
+bazel-out
+external

e2e/coreutils/md5.txt

@@ -0,0 +1 @@
+160e389707f172f2d1333084fec4e650 test.bin

e2e/coreutils/sha1.txt

@@ -0,0 +1 @@
+b3dd96b0db6a12a3ae53899e8fbc04b749c980d6 test.bin

e2e/coreutils/sha256.txt

@@ -0,0 +1 @@
+3285d399e870df47749162ab2ebfd0b727e9142925ab5f5ad221c3a42a852b08 test.bin

e2e/coreutils/test.bin

@@ -0,0 +1 @@
+act as if this is binary file

e2e/smoke/.bazelrc

@@ -4,14 +4,12 @@
 # Import Aspect bazelrc presets
 import %workspace%/.aspect/bazelrc/convenience.bazelrc
 import %workspace%/.aspect/bazelrc/correctness.bazelrc
+import %workspace%/.aspect/bazelrc/performance.bazelrc
 import %workspace%/.aspect/bazelrc/debug.bazelrc
 import %workspace%/.aspect/bazelrc/javascript.bazelrc
-import %workspace%/.aspect/bazelrc/performance.bazelrc
 
 ### PROJECT SPECIFIC OPTIONS ###
 
-common --experimental_enable_bzlmod
-
 # Load any settings & overrides specific to the current user from `.aspect/bazelrc/user.bazelrc`.
 # This file should appear in `.gitignore` so that settings are not shared with team members. This
 # should be last statement in this config so the user configuration is able to overwrite flags from

e2e/smoke/MODULE.bazel

@@ -0,0 +1,14 @@
+module(
+    name = "smoke",
+    compatibility_level = 1,
+    version = "0.0.0",
+)
+
+bazel_dep(name = "bazel_skylib", version = "1.1.1")
+
+bazel_dep(name = "aspect_bazel_lib", version = "0.0.0")
+
+local_path_override(
+    module_name = "aspect_bazel_lib",
+    path = "../..",
+)

e2e/workspace/d/1

@@ -1 +0,0 @@
-1

e2e/workspace/expected_jq

@@ -1 +0,0 @@
-null

e2e/workspace/expected_yq

@@ -1 +0,0 @@
-

lib/extensions.bzl

@@ -6,6 +6,7 @@ load(
     "register_copy_to_directory_toolchains",
     "register_jq_toolchains",
     "register_yq_toolchains",
+    "register_coreutils_toolchains"
 )
 load("//lib/private:host_repo.bzl", "host_repo")
 
@@ -14,6 +15,7 @@ def _toolchain_extension(mctx):
     register_copy_to_directory_toolchains(register = False)
     register_jq_toolchains(register = False)
     register_yq_toolchains(register = False)
+    register_coreutils_toolchains(register = False)
 
     create_host_repo = False
     for module in mctx.modules:

lib/repositories.bzl

@@ -39,6 +39,9 @@ def aspect_bazel_lib_dependencies(override_local_config_platform = False):
     register_copy_directory_toolchains()
     register_copy_to_directory_toolchains()
 
+    # Always register the coreutils toolchain
+    register_coreutils_toolchains()
+
 # Re-export the default versions
 DEFAULT_JQ_VERSION = _DEFAULT_JQ_VERSION
 DEFAULT_YQ_VERSION = _DEFAULT_YQ_VERSION

tools/hashes.bzl

@@ -6,36 +6,32 @@ via output groups.
 Based on https://github.com/bazelbuild/examples/blob/main/rules/implicit_output/hash.bzl
 """
 
+
+def _hash(ctx, algo, file):
+    coreutils = ctx.toolchains["@aspect_bazel_lib//lib:coreutils_toolchain_type"]
+    out = ctx.actions.declare_file("{}.{}".format(file.path, algo))
+    ctx.actions.run_shell(
+        outputs = [out],
+        inputs = [file],
+        tools = [coreutils.coreutils_info.bin],
+        # coreutils has --no-names option but it doesn't work in current version, so we have to use cut.
+        command = """HASH=$({coreutils} hashsum --{algo} {src} | {coreutils} cut -f1 -d " ") && {coreutils} echo -ne "$HASH {basename}" > {out}""".format(
+            coreutils = coreutils.coreutils_info.bin.path, 
+            algo = algo, 
+            src = file.path, 
+            basename = file.basename,
+            out = out.path
+        ),
+    )
+    return out
+
 def _impl(ctx):
     # Create actions to generate the three output files.
     # Actions are run only when the corresponding file is requested.
 
-    md5out = ctx.actions.declare_file("{}.md5".format(ctx.file.src.basename))
-    ctx.actions.run_shell(
-        outputs = [md5out],
-        inputs = [ctx.file.src],
-        command = "ROOT=$PWD && cd {} && md5sum {} > $ROOT/{}".format(ctx.file.src.dirname, ctx.file.src.basename, md5out.path),
-    )
-
-    sha1out = ctx.actions.declare_file("{}.sha1".format(ctx.file.src.basename))
-    ctx.actions.run_shell(
-        outputs = [sha1out],
-        inputs = [ctx.file.src],
-        command = "ROOT=$PWD && cd {} && sha1sum {} > $ROOT/{}".format(ctx.file.src.dirname, ctx.file.src.basename, sha1out.path),
-    )
-
-    sha256out = ctx.actions.declare_file("{}.sha256".format(ctx.file.src.basename))
-    ctx.actions.run_shell(
-        outputs = [sha256out],
-        inputs = [ctx.file.src],
-        command = "ROOT=$PWD && cd {dirname} && $ROOT/{sha256sum} {basename} > $ROOT/{path}".format(
-            dirname = ctx.file.src.dirname,
-            sha256sum = ctx.executable._sha256sum.path,
-            basename = ctx.file.src.basename,
-            path = sha256out.path,
-        ),
-        tools = [ctx.executable._sha256sum],
-    )
+    md5out = _hash(ctx, "md5", ctx.file.src)
+    sha1out = _hash(ctx, "sha1", ctx.file.src)
+    sha256out = _hash(ctx, "sha256", ctx.file.src)
 
     # By default (if you run `bazel build` on this target, or if you use it as a
     # source of another target), only the sha256 is computed.
@@ -52,16 +48,14 @@ def _impl(ctx):
 
 _hashes = rule(
     implementation = _impl,
+    toolchains = [
+        "@aspect_bazel_lib//lib:coreutils_toolchain_type",
+    ],
     attrs = {
         "src": attr.label(
             allow_single_file = True,
             mandatory = True,
-        ),
-        "_sha256sum": attr.label(
-            executable = True,
-            cfg = "exec",
-            default = "//tools/sha256sum",
-        ),
+        )
     },
 )
 

tools/sha256sum/BUILD.bazel

@@ -1,14 +0,0 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
-
-go_library(
-    name = "sha256sum_lib",
-    srcs = ["main.go"],
-    importpath = "github.com/aspect-build/bazel-lib/tools/sha256sum",
-    visibility = ["//visibility:private"],
-)
-
-go_binary(
-    name = "sha256sum",
-    embed = [":sha256sum_lib"],
-    visibility = ["//visibility:public"],
-)

tools/sha256sum/main.go

@@ -1,32 +0,0 @@
-package main
-
-import (
-	"crypto/sha256"
-	"fmt"
-	"io"
-	"log"
-	"os"
-)
-
-func main() {
-	var input io.Reader
-	var filename string
-	if len(os.Args) == 1 {
-		input = os.Stdin
-		filename = "-"
-	} else {
-		f, err := os.Open(os.Args[1])
-		if err != nil {
-			log.Fatal(err)
-		}
-		defer f.Close()
-		input = f
-		filename = os.Args[1]
-	}
-
-	hash := sha256.New()
-	if _, err := io.Copy(hash, input); err != nil {
-		log.Fatal(err)
-	}
-	fmt.Printf("%x  %s\n", hash.Sum(nil), filename)
-}