Add copy_directory rule (#366)

docs/BUILD

@@ -22,6 +22,11 @@ stardoc_with_diff_test(
     out_label = "//docs:common_settings_doc.md",
 )
 
+stardoc_with_diff_test(
+    bzl_library_target = "//rules:copy_directory",
+    out_label = "//docs:copy_directory_doc.md",
+)
+
 stardoc_with_diff_test(
     bzl_library_target = "//rules:copy_file",
     out_label = "//docs:copy_file_doc.md",

docs/copy_directory_doc.md

@@ -0,0 +1,63 @@
+<!-- Generated with Stardoc: http://skydoc.bazel.build -->
+
+A rule that copies a directory to another place.
+
+The rule uses a Bash command on Linux/macOS/non-Windows, and a cmd.exe command
+on Windows (no Bash is required).
+
+
+<a id="#copy_directory"></a>
+
+## copy_directory
+
+<pre>
+copy_directory(<a href="#copy_directory-name">name</a>, <a href="#copy_directory-src">src</a>, <a href="#copy_directory-out">out</a>, <a href="#copy_directory-kwargs">kwargs</a>)
+</pre>
+
+Copies a directory to another location.
+
+This rule uses a Bash command on Linux/macOS/non-Windows, and a cmd.exe command on Windows (no Bash is required).
+
+If using this rule with source directories, it is recommended that you use the
+`--host_jvm_args=-DBAZEL_TRACK_SOURCE_DIRECTORIES=1` startup option so that changes
+to files within source directories are detected. See
+https://github.com/bazelbuild/bazel/commit/c64421bc35214f0414e4f4226cc953e8c55fa0d2
+for more context.
+
+
+**PARAMETERS**
+
+
+| Name  | Description | Default Value |
+| :------------- | :------------- | :------------- |
+| <a id="copy_directory-name"></a>name |  Name of the rule.   |  none |
+| <a id="copy_directory-src"></a>src |  The directory to make a copy of. Can be a source directory or TreeArtifact.   |  none |
+| <a id="copy_directory-out"></a>out |  Path of the output directory, relative to this package.   |  none |
+| <a id="copy_directory-kwargs"></a>kwargs |  further keyword arguments, e.g. <code>visibility</code>   |  none |
+
+
+<a id="#copy_directory_action"></a>
+
+## copy_directory_action
+
+<pre>
+copy_directory_action(<a href="#copy_directory_action-ctx">ctx</a>, <a href="#copy_directory_action-src">src</a>, <a href="#copy_directory_action-dst">dst</a>, <a href="#copy_directory_action-is_windows">is_windows</a>)
+</pre>
+
+Helper function that creates an action to copy a directory from src to dst.
+
+This helper is used by copy_directory. It is exposed as a public API so it can be used within
+other rule implementations.
+
+
+**PARAMETERS**
+
+
+| Name  | Description | Default Value |
+| :------------- | :------------- | :------------- |
+| <a id="copy_directory_action-ctx"></a>ctx |  The rule context.   |  none |
+| <a id="copy_directory_action-src"></a>src |  The directory to make a copy of. Can be a source directory or TreeArtifact.   |  none |
+| <a id="copy_directory_action-dst"></a>dst |  The directory to copy to. Must be a TreeArtifact.   |  none |
+| <a id="copy_directory_action-is_windows"></a>is_windows |  If true, an cmd.exe action is created so there is no bash dependency.   |  <code>False</code> |
+
+

rules/BUILD

@@ -21,6 +21,12 @@ bzl_library(
     deps = ["//rules/private:copy_file_private"],
 )
 
+bzl_library(
+    name = "copy_directory",
+    srcs = ["copy_directory.bzl"],
+    deps = ["//rules/private:copy_directory_private"],
+)
+
 bzl_library(
     name = "write_file",
     srcs = ["write_file.bzl"],

rules/copy_directory.bzl

@@ -0,0 +1,28 @@
+# Copyright 2022 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A rule that copies a directory to another place.
+
+The rule uses a Bash command on Linux/macOS/non-Windows, and a cmd.exe command
+on Windows (no Bash is required).
+"""
+
+load(
+    "//rules/private:copy_directory_private.bzl",
+    _copy_directory = "copy_directory",
+    _copy_directory_action = "copy_directory_action",
+)
+
+copy_directory = _copy_directory
+copy_directory_action = _copy_directory_action

rules/private/BUILD

@@ -2,6 +2,12 @@ load("//:bzl_library.bzl", "bzl_library")
 
 licenses(["notice"])
 
+bzl_library(
+    name = "copy_directory_private",
+    srcs = ["copy_directory_private.bzl"],
+    visibility = ["//rules:__pkg__"],
+)
+
 bzl_library(
     name = "copy_file_private",
     srcs = ["copy_file_private.bzl"],

rules/private/copy_directory_private.bzl

@@ -0,0 +1,164 @@
+# Copyright 2022 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Implementation of copy_directory macro and underlying rules.
+
+This rule copies a directory to another location using Bash (on Linux/macOS) or
+cmd.exe (on Windows).
+"""
+
+# Hints for Bazel spawn strategy
+_execution_requirements = {
+    # Copying files is entirely IO-bound and there is no point doing this work remotely.
+    # Also, remote-execution does not allow source directory inputs, see
+    # https://github.com/bazelbuild/bazel/commit/c64421bc35214f0414e4f4226cc953e8c55fa0d2
+    # So we must not attempt to execute remotely in that case.
+    "no-remote-exec": "1",
+}
+
+def _copy_cmd(ctx, src, dst):
+    # Most Windows binaries built with MSVC use a certain argument quoting
+    # scheme. Bazel uses that scheme too to quote arguments. However,
+    # cmd.exe uses different semantics, so Bazel's quoting is wrong here.
+    # To fix that we write the command to a .bat file so no command line
+    # quoting or escaping is required.
+    # Put a hash of the file name into the name of the generated batch file to
+    # make it unique within the package, so that users can define multiple copy_file's.
+    bat = ctx.actions.declare_file("%s-%s-cmd.bat" % (ctx.label.name, hash(src.path)))
+
+    # Flags are documented at
+    # https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/robocopy
+    # NB: robocopy return non-zero exit codes on success so we must exit 0 after calling it
+    cmd_tmpl = """\
+if not exist \"{src}\\\" (
+  echo Error: \"{src}\" is not a directory
+  @exit 1
+)
+@robocopy \"{src}\" \"{dst}\" /E /MIR >NUL & @exit 0
+"""
+    mnemonic = "CopyDirectory"
+    progress_message = "Copying directory %{input}"
+
+    ctx.actions.write(
+        output = bat,
+        # Do not use lib/shell.bzl's shell.quote() method, because that uses
+        # Bash quoting syntax, which is different from cmd.exe's syntax.
+        content = cmd_tmpl.format(
+            src = src.path.replace("/", "\\"),
+            dst = dst.path.replace("/", "\\"),
+        ),
+        is_executable = True,
+    )
+    ctx.actions.run(
+        inputs = [src],
+        tools = [bat],
+        outputs = [dst],
+        executable = "cmd.exe",
+        arguments = ["/C", bat.path.replace("/", "\\")],
+        mnemonic = mnemonic,
+        progress_message = progress_message,
+        use_default_shell_env = True,
+        execution_requirements = _execution_requirements,
+    )
+
+def _copy_bash(ctx, src, dst):
+    cmd = """\
+if [ ! -d \"$1\" ]; then
+    echo \"Error: $1 is not a directory\"
+    exit 1
+fi
+
+rm -rf \"$2\" && cp -fR \"$1/\" \"$2\"
+"""
+    mnemonic = "CopyDirectory"
+    progress_message = "Copying directory %s" % src.path
+
+    ctx.actions.run_shell(
+        tools = [src],
+        outputs = [dst],
+        command = cmd,
+        arguments = [src.path, dst.path],
+        mnemonic = mnemonic,
+        progress_message = progress_message,
+        use_default_shell_env = True,
+        execution_requirements = _execution_requirements,
+    )
+
+def copy_directory_action(ctx, src, dst, is_windows = False):
+    """Helper function that creates an action to copy a directory from src to dst.
+
+    This helper is used by copy_directory. It is exposed as a public API so it can be used within
+    other rule implementations.
+
+    Args:
+        ctx: The rule context.
+        src: The directory to make a copy of. Can be a source directory or TreeArtifact.
+        dst: The directory to copy to. Must be a TreeArtifact.
+        is_windows: If true, an cmd.exe action is created so there is no bash dependency.
+    """
+    if dst.is_source or not dst.is_directory:
+        fail("dst must be a TreeArtifact")
+    if is_windows:
+        _copy_cmd(ctx, src, dst)
+    else:
+        _copy_bash(ctx, src, dst)
+
+def _copy_directory_impl(ctx):
+    dst = ctx.actions.declare_directory(ctx.attr.out)
+    copy_directory_action(ctx, ctx.file.src, dst, ctx.attr.is_windows)
+
+    files = depset(direct = [dst])
+    runfiles = ctx.runfiles(files = [dst])
+
+    return [DefaultInfo(files = files, runfiles = runfiles)]
+
+_copy_directory = rule(
+    implementation = _copy_directory_impl,
+    provides = [DefaultInfo],
+    attrs = {
+        "src": attr.label(mandatory = True, allow_single_file = True),
+        "is_windows": attr.bool(mandatory = True),
+        # Cannot declare out as an output here, because there's no API for declaring
+        # TreeArtifact outputs.
+        "out": attr.string(mandatory = True),
+    },
+)
+
+def copy_directory(name, src, out, **kwargs):
+    """Copies a directory to another location.
+
+    This rule uses a Bash command on Linux/macOS/non-Windows, and a cmd.exe command on Windows (no Bash is required).
+
+    If using this rule with source directories, it is recommended that you use the
+    `--host_jvm_args=-DBAZEL_TRACK_SOURCE_DIRECTORIES=1` startup option so that changes
+    to files within source directories are detected. See
+    https://github.com/bazelbuild/bazel/commit/c64421bc35214f0414e4f4226cc953e8c55fa0d2
+    for more context.
+
+    Args:
+      name: Name of the rule.
+      src: The directory to make a copy of. Can be a source directory or TreeArtifact.
+      out: Path of the output directory, relative to this package.
+      **kwargs: further keyword arguments, e.g. `visibility`
+    """
+    _copy_directory(
+        name = name,
+        src = src,
+        is_windows = select({
+            "@bazel_tools//src/conditions:host_windows": True,
+            "//conditions:default": False,
+        }),
+        out = out,
+        **kwargs
+    )

tests/copy_directory/BUILD.bazel

@@ -0,0 +1,44 @@
+# This package aids testing the 'copy_directory' rule.
+
+load("//rules:copy_directory.bzl", "copy_directory")
+
+licenses(["notice"])
+
+package(default_testonly = 1)
+
+# Copy of directory containing files a and b, and a subdir containing c
+copy_directory(
+    name = "copy_of_dir_with_subdir",
+    src = "dir_with_subdir",
+    out = "dir_copy",
+)
+
+genrule(
+    name = "empty_directory",
+    outs = ["empty_dir"],
+    cmd = "mkdir $@",
+)
+
+copy_directory(
+    name = "copy_of_empty_dir",
+    src = "empty_dir",
+    out = "empty_dir_copy",
+)
+
+copy_directory(
+    name = "copy_of_dir_with_symlink",
+    src = "dir_with_symlink",
+    out = "dir_with_symlink_copy",
+)
+
+sh_test(
+    name = "copy_directory_tests",
+    srcs = ["copy_directory_tests.sh"],
+    data = [
+        ":copy_of_dir_with_subdir",
+        ":copy_of_dir_with_symlink",
+        ":copy_of_empty_dir",
+        "//tests:unittest.bash",
+    ],
+    deps = ["@bazel_tools//tools/bash/runfiles"],
+)

tests/copy_directory/copy_directory_tests.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+
+# Copyright 2019 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# --- begin runfiles.bash initialization ---
+# Copy-pasted from Bazel's Bash runfiles library (tools/bash/runfiles/runfiles.bash).
+set -euo pipefail
+if [[ ! -d "${RUNFILES_DIR:-/dev/null}" && ! -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
+  if [[ -f "$0.runfiles_manifest" ]]; then
+    export RUNFILES_MANIFEST_FILE="$0.runfiles_manifest"
+  elif [[ -f "$0.runfiles/MANIFEST" ]]; then
+    export RUNFILES_MANIFEST_FILE="$0.runfiles/MANIFEST"
+  elif [[ -f "$0.runfiles/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then
+    export RUNFILES_DIR="$0.runfiles"
+  fi
+fi
+if [[ -f "${RUNFILES_DIR:-/dev/null}/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then
+  source "${RUNFILES_DIR}/bazel_tools/tools/bash/runfiles/runfiles.bash"
+elif [[ -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
+  source "$(grep -m1 "^bazel_tools/tools/bash/runfiles/runfiles.bash " \
+            "$RUNFILES_MANIFEST_FILE" | cut -d ' ' -f 2-)"
+else
+  echo >&2 "ERROR: cannot find @bazel_tools//tools/bash/runfiles:runfiles.bash"
+  exit 1
+fi
+# --- end runfiles.bash initialization ---
+
+source "$(rlocation bazel_skylib/tests/unittest.bash)" \
+  || { echo "Could not source bazel_skylib/tests/unittest.bash" >&2; exit 1; }
+
+function test_copy_dir_with_subdir__copies_a() {
+  cat "$(rlocation bazel_skylib/tests/copy_directory/dir_copy)/a" >"$TEST_log"
+  expect_log '^foo$'
+}
+
+function test_copy_dir_with_subdir__copies_b() {
+  cat "$(rlocation bazel_skylib/tests/copy_directory/dir_copy)/b" >"$TEST_log"
+  expect_log '^bar$'
+}
+
+function test_copy_dir_with_subdir__copies_c() {
+  cat "$(rlocation bazel_skylib/tests/copy_directory/dir_copy)/subdir/c" >"$TEST_log"
+  expect_log '^moocow$'
+}
+
+function test_copy_dir_with_subdir__correct_filecounts() {
+  local -r dir_filecount=$(ls "$(rlocation bazel_skylib/tests/copy_directory/dir_copy)" | wc -l)
+  assert_equals $dir_filecount 3
+  local -r subdir_filecount=$(ls "$(rlocation bazel_skylib/tests/copy_directory/dir_copy)/subdir" | wc -l)
+  assert_equals $subdir_filecount 1
+}
+
+function test_copy_empty_dir() {
+  local -r filecount=$(ls "$(rlocation bazel_skylib/tests/copy_directory/empty_dir_copy)" | wc -l)
+  assert_equals $filecount 0
+}
+
+function test_copy_dir_with_symlink__copies_file() {
+  cat "$(rlocation bazel_skylib/tests/copy_directory/dir_with_symlink_copy)/file" >"$TEST_log"
+  expect_log '^foo$'
+}
+
+function test_copy_dir_with_symlink__copies_symlink() {
+  cat "$(rlocation bazel_skylib/tests/copy_directory/dir_with_symlink_copy)/symlink" >"$TEST_log"
+  expect_log '^foo$'
+}
+
+
+run_suite "copy_directory test suite"

tests/copy_directory/dir_with_subdir/a

@@ -0,0 +1 @@
+foo

tests/copy_directory/dir_with_subdir/b

@@ -0,0 +1 @@
+bar

tests/copy_directory/dir_with_subdir/subdir/c

@@ -0,0 +1 @@
+moocow

tests/copy_directory/dir_with_symlink/file

@@ -0,0 +1 @@
+foo

tests/copy_directory/dir_with_symlink/symlink

@@ -0,0 +1 @@
+./file

tests/subpackages_tests.bzl

@@ -21,6 +21,7 @@ def _all_test(env):
     """Unit tests for subpackages.all."""
 
     all_pkgs = [
+        "copy_directory",
         "copy_file",
         "diff_test",
         "expand_template",
@@ -37,6 +38,7 @@ def _all_test(env):
 
     # These exist in all cases
     filtered_pkgs = [
+        "copy_directory",
         "copy_file",
         "expand_template",
         "select_file",