Use stricter execution requirements for copy actions (#372)

This commit is contained in:
Derek Cormier 2022-06-10 07:23:43 -07:00 committed by GitHub
parent 207acb3850
commit a501641dae
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 69 additions and 10 deletions

View File

@ -2,16 +2,24 @@ load("//:bzl_library.bzl", "bzl_library")
licenses(["notice"]) licenses(["notice"])
bzl_library(
name = "copy_common",
srcs = ["copy_common.bzl"],
visibility = ["//rules:__pkg__"],
)
bzl_library( bzl_library(
name = "copy_directory_private", name = "copy_directory_private",
srcs = ["copy_directory_private.bzl"], srcs = ["copy_directory_private.bzl"],
visibility = ["//rules:__pkg__"], visibility = ["//rules:__pkg__"],
deps = [":copy_common"],
) )
bzl_library( bzl_library(
name = "copy_file_private", name = "copy_file_private",
srcs = ["copy_file_private.bzl"], srcs = ["copy_file_private.bzl"],
visibility = ["//rules:__pkg__"], visibility = ["//rules:__pkg__"],
deps = [":copy_common"],
) )
bzl_library( bzl_library(

View File

@ -0,0 +1,54 @@
# Copyright 2022 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"Helpers for copy rules"
# Hints for Bazel spawn strategy
COPY_EXECUTION_REQUIREMENTS = {
# ----------------+-----------------------------------------------------------------------------
# no-remote | Prevents the action or test from being executed remotely or cached remotely.
# | This is equivalent to using both `no-remote-cache` and `no-remote-exec`.
# ----------------+-----------------------------------------------------------------------------
# no-cache | Results in the action or test never being cached (remotely or locally)
# ----------------+-----------------------------------------------------------------------------
# local | Precludes the action or test from being remotely cached, remotely executed,
# | or run inside the sandbox. For genrules and tests, marking the rule with the
# | local = True attribute has the same effect.
# ----------------+-----------------------------------------------------------------------------
# See https://bazel.google.cn/reference/be/common-definitions?hl=en&authuser=0#common-attributes
#
# Copying file & directories is entirely IO-bound and there is no point doing this work
# remotely.
#
# Also, remote-execution does not allow source directory inputs, see
# https://github.com/bazelbuild/bazel/commit/c64421bc35214f0414e4f4226cc953e8c55fa0d2 So we must
# not attempt to execute remotely in that case.
#
# There is also no point pulling the output file or directory from the remote cache since the
# bytes to copy are already available locally. Conversely, no point in writing to the cache if
# no one has any reason to check it for this action.
#
# Read and writing to disk cache is disabled as well primarily to reduce disk usage on the local
# machine. A disk cache hit of a directory copy could be slghtly faster than a copy since the
# disk cache stores the directory artifact as a single entry, but the slight performance bump
# comes at the cost of heavy disk cache usage, which is an unmanaged directory that grow beyond
# the bounds of the physical disk.
#
# Sandboxing for this action is wasteful as well since there is a 1:1 mapping of input
# file/directory to output file/directory and no room for non-hermetic inputs to sneak in to the
# input.
"no-remote": "1",
"no-cache": "1",
"local": "1",
}

View File

@ -18,14 +18,7 @@ This rule copies a directory to another location using Bash (on Linux/macOS) or
cmd.exe (on Windows). cmd.exe (on Windows).
""" """
# Hints for Bazel spawn strategy load(":copy_common.bzl", "COPY_EXECUTION_REQUIREMENTS")
_execution_requirements = {
# Copying files is entirely IO-bound and there is no point doing this work remotely.
# Also, remote-execution does not allow source directory inputs, see
# https://github.com/bazelbuild/bazel/commit/c64421bc35214f0414e4f4226cc953e8c55fa0d2
# So we must not attempt to execute remotely in that case.
"no-remote-exec": "1",
}
def _copy_cmd(ctx, src, dst): def _copy_cmd(ctx, src, dst):
# Most Windows binaries built with MSVC use a certain argument quoting # Most Windows binaries built with MSVC use a certain argument quoting
@ -69,7 +62,7 @@ if not exist \"{src}\\\" (
mnemonic = mnemonic, mnemonic = mnemonic,
progress_message = progress_message, progress_message = progress_message,
use_default_shell_env = True, use_default_shell_env = True,
execution_requirements = _execution_requirements, execution_requirements = COPY_EXECUTION_REQUIREMENTS,
) )
def _copy_bash(ctx, src, dst): def _copy_bash(ctx, src, dst):
@ -92,7 +85,7 @@ rm -rf \"$2\" && cp -fR \"$1/\" \"$2\"
mnemonic = mnemonic, mnemonic = mnemonic,
progress_message = progress_message, progress_message = progress_message,
use_default_shell_env = True, use_default_shell_env = True,
execution_requirements = _execution_requirements, execution_requirements = COPY_EXECUTION_REQUIREMENTS,
) )
def copy_directory_action(ctx, src, dst, is_windows = False): def copy_directory_action(ctx, src, dst, is_windows = False):

View File

@ -19,6 +19,8 @@ cmd.exe (on Windows). '_copy_xfile' marks the resulting file executable,
'_copy_file' does not. '_copy_file' does not.
""" """
load(":copy_common.bzl", "COPY_EXECUTION_REQUIREMENTS")
def copy_cmd(ctx, src, dst): def copy_cmd(ctx, src, dst):
# Most Windows binaries built with MSVC use a certain argument quoting # Most Windows binaries built with MSVC use a certain argument quoting
# scheme. Bazel uses that scheme too to quote arguments. However, # scheme. Bazel uses that scheme too to quote arguments. However,
@ -45,6 +47,7 @@ def copy_cmd(ctx, src, dst):
mnemonic = "CopyFile", mnemonic = "CopyFile",
progress_message = "Copying files", progress_message = "Copying files",
use_default_shell_env = True, use_default_shell_env = True,
execution_requirements = COPY_EXECUTION_REQUIREMENTS,
) )
def copy_bash(ctx, src, dst): def copy_bash(ctx, src, dst):
@ -56,6 +59,7 @@ def copy_bash(ctx, src, dst):
mnemonic = "CopyFile", mnemonic = "CopyFile",
progress_message = "Copying files", progress_message = "Copying files",
use_default_shell_env = True, use_default_shell_env = True,
execution_requirements = COPY_EXECUTION_REQUIREMENTS,
) )
def _copy_file_impl(ctx): def _copy_file_impl(ctx):