feat: hardlink generated files in copy_to_directory and copy_directory instead of copying (#321)
This commit is contained in:
Greg Magolan
GitHub
parent
74caa5c097
commit
63f5aff803
|
|
@ -11,7 +11,7 @@ on Windows (no Bash is required).
|
|||
## copy_directory
|
||||
|
||||
<pre>
|
||||
copy_directory(<a href="#copy_directory-name">name</a>, <a href="#copy_directory-src">src</a>, <a href="#copy_directory-out">out</a>, <a href="#copy_directory-kwargs">kwargs</a>)
|
||||
copy_directory(<a href="#copy_directory-name">name</a>, <a href="#copy_directory-src">src</a>, <a href="#copy_directory-out">out</a>, <a href="#copy_directory-hardlink">hardlink</a>, <a href="#copy_directory-kwargs">kwargs</a>)
|
||||
</pre>
|
||||
|
||||
Copies a directory to another location.
|
||||
|
|
@ -33,6 +33,7 @@ for more context.
|
|||
| <a id="copy_directory-name"></a>name | Name of the rule. | none |
|
||||
| <a id="copy_directory-src"></a>src | The directory to make a copy of. Can be a source directory or TreeArtifact. | none |
|
||||
| <a id="copy_directory-out"></a>out | Path of the output directory, relative to this package. | none |
|
||||
| <a id="copy_directory-hardlink"></a>hardlink | Controls when to use hardlinks to files instead of making copies.<br><br>Creating hardlinks is much faster than making copies of files with the caveat that hardlinks share file permissions with their source.<br><br>Since Bazel removes write permissions on files in the output tree after an action completes, hardlinks to source files within source directories is not recommended since write permissions will be inadvertently removed from sources files.<br><br>- "auto": hardlinks are used if src is a tree artifact already in the output tree - "off": files are always copied - "on": hardlinks are always used (not recommended) | <code>"auto"</code> |
|
||||
| <a id="copy_directory-kwargs"></a>kwargs | further keyword arguments, e.g. <code>visibility</code> | none |
|
||||
|
||||
|
||||
|
|
@ -44,7 +45,13 @@ for more context.
|
|||
copy_directory_action(<a href="#copy_directory_action-ctx">ctx</a>, <a href="#copy_directory_action-src">src</a>, <a href="#copy_directory_action-dst">dst</a>, <a href="#copy_directory_action-is_windows">is_windows</a>)
|
||||
</pre>
|
||||
|
||||
Factory function that creates an action to copy a directory from src to dst.
|
||||
Legacy factory function that creates an action to copy a directory from src to dst.
|
||||
|
||||
For improved analysis and runtime performance, it is recommended the switch
|
||||
to `copy_directory_bin_action` which takes a tool binary, typically the
|
||||
`@aspect_bazel_lib//tools/copy_to_directory` `go_binary` either built from
|
||||
source or provided by a toolchain and creates hard links instead of performing full
|
||||
file copies.
|
||||
|
||||
This helper is used by copy_directory. It is exposed as a public API so it can be used within
|
||||
other rule implementations.
|
||||
|
|
@ -61,3 +68,33 @@ other rule implementations.
|
|||
| <a id="copy_directory_action-is_windows"></a>is_windows | Deprecated and unused | <code>None</code> |
|
||||
|
||||
|
||||
<a id="copy_directory_bin_action"></a>
|
||||
|
||||
## copy_directory_bin_action
|
||||
|
||||
<pre>
|
||||
copy_directory_bin_action(<a href="#copy_directory_bin_action-ctx">ctx</a>, <a href="#copy_directory_bin_action-src">src</a>, <a href="#copy_directory_bin_action-dst">dst</a>, <a href="#copy_directory_bin_action-copy_directory_bin">copy_directory_bin</a>, <a href="#copy_directory_bin_action-hardlink">hardlink</a>, <a href="#copy_directory_bin_action-verbose">verbose</a>)
|
||||
</pre>
|
||||
|
||||
Factory function that creates an action to copy a directory from src to dst using a tool binary.
|
||||
|
||||
The tool binary will typically be the `@aspect_bazel_lib//tools/copy_directory` `go_binary`
|
||||
either built from source or provided by a toolchain.
|
||||
|
||||
This helper is used by the copy_directory rule. It is exposed as a public API so it can be used
|
||||
within other rule implementations.
|
||||
|
||||
|
||||
**PARAMETERS**
|
||||
|
||||
|
||||
| Name | Description | Default Value |
|
||||
| :------------- | :------------- | :------------- |
|
||||
| <a id="copy_directory_bin_action-ctx"></a>ctx | The rule context. | none |
|
||||
| <a id="copy_directory_bin_action-src"></a>src | The source directory to copy. | none |
|
||||
| <a id="copy_directory_bin_action-dst"></a>dst | The directory to copy to. Must be a TreeArtifact. | none |
|
||||
| <a id="copy_directory_bin_action-copy_directory_bin"></a>copy_directory_bin | Copy to directory tool binary. | none |
|
||||
| <a id="copy_directory_bin_action-hardlink"></a>hardlink | Controls when to use hardlinks to files instead of making copies.<br><br>See copy_directory rule documentation for more details. | <code>"auto"</code> |
|
||||
| <a id="copy_directory_bin_action-verbose"></a>verbose | If true, prints out verbose logs to stdout | <code>False</code> |
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -9,8 +9,9 @@ Copy files and directories to an output directory.
|
|||
|
||||
<pre>
|
||||
copy_to_directory(<a href="#copy_to_directory-name">name</a>, <a href="#copy_to_directory-allow_overwrites">allow_overwrites</a>, <a href="#copy_to_directory-exclude_prefixes">exclude_prefixes</a>, <a href="#copy_to_directory-exclude_srcs_packages">exclude_srcs_packages</a>,
|
||||
<a href="#copy_to_directory-exclude_srcs_patterns">exclude_srcs_patterns</a>, <a href="#copy_to_directory-include_external_repositories">include_external_repositories</a>, <a href="#copy_to_directory-include_srcs_packages">include_srcs_packages</a>,
|
||||
<a href="#copy_to_directory-include_srcs_patterns">include_srcs_patterns</a>, <a href="#copy_to_directory-out">out</a>, <a href="#copy_to_directory-replace_prefixes">replace_prefixes</a>, <a href="#copy_to_directory-root_paths">root_paths</a>, <a href="#copy_to_directory-srcs">srcs</a>, <a href="#copy_to_directory-verbose">verbose</a>)
|
||||
<a href="#copy_to_directory-exclude_srcs_patterns">exclude_srcs_patterns</a>, <a href="#copy_to_directory-hardlink">hardlink</a>, <a href="#copy_to_directory-include_external_repositories">include_external_repositories</a>,
|
||||
<a href="#copy_to_directory-include_srcs_packages">include_srcs_packages</a>, <a href="#copy_to_directory-include_srcs_patterns">include_srcs_patterns</a>, <a href="#copy_to_directory-out">out</a>, <a href="#copy_to_directory-replace_prefixes">replace_prefixes</a>, <a href="#copy_to_directory-root_paths">root_paths</a>,
|
||||
<a href="#copy_to_directory-srcs">srcs</a>, <a href="#copy_to_directory-verbose">verbose</a>)
|
||||
</pre>
|
||||
|
||||
Copies files and directories to an output directory.
|
||||
|
|
@ -51,10 +52,11 @@ for more information on supported globbing patterns.
|
|||
| Name | Description | Type | Mandatory | Default |
|
||||
| :------------- | :------------- | :------------- | :------------- | :------------- |
|
||||
| <a id="copy_to_directory-name"></a>name | A unique name for this target. | <a href="https://bazel.build/docs/build-ref.html#name">Name</a> | required | |
|
||||
| <a id="copy_to_directory-allow_overwrites"></a>allow_overwrites | If True, allow files to be overwritten if the same output file is copied to twice.<br><br>If set, then the order of srcs matters as the last copy of a particular file will win.<br><br>This setting has no effect on Windows where overwrites are always allowed. | Boolean | optional | False |
|
||||
| <a id="copy_to_directory-allow_overwrites"></a>allow_overwrites | If True, allow files to be overwritten if the same output file is copied to twice.<br><br>The order of srcs matters as the last copy of a particular file will win when overwriting. Performance of copy_to_directory will be slightly degraded when allow_overwrites is True since copies cannot be parallelized out as they are calculated. Instead all copy paths must be calculated before any copies can be started. | Boolean | optional | False |
|
||||
| <a id="copy_to_directory-exclude_prefixes"></a>exclude_prefixes | List of path prefixes (with glob support) to exclude from output directory.<br><br>DEPRECATED: use <code>exclude_srcs_patterns</code> instead<br><br>Files in srcs are not copied to the output directory if their output directory path, after applying <code>root_paths</code>, starts with or fully matches one of the patterns specified.<br><br>Forward slashes (<code>/</code>) should be used as path separators.<br><br>Files that do not have matching output directory paths are subject to subsequent filters and transformations to determine if they are copied and what their path in the output directory will be.<br><br>Globs are supported (see rule docstring above). | List of strings | optional | [] |
|
||||
| <a id="copy_to_directory-exclude_srcs_packages"></a>exclude_srcs_packages | List of Bazel packages (with glob support) to exclude from output directory.<br><br>Files in srcs are not copied to the output directory if the Bazel package of the file matches one of the patterns specified.<br><br>Forward slashes (<code>/</code>) should be used as path separators. A first character of <code>"."</code> will be replaced by the target's package path.<br><br>Files that have do not have matching Bazel packages are subject to subsequent filters and transformations to determine if they are copied and what their path in the output directory will be.<br><br>Globs are supported (see rule docstring above). | List of strings | optional | [] |
|
||||
| <a id="copy_to_directory-exclude_srcs_patterns"></a>exclude_srcs_patterns | List of paths (with glob support) to exclude from output directory.<br><br>Files in srcs are not copied to the output directory if their output directory path, after applying <code>root_paths</code>, matches one of the patterns specified.<br><br>Forward slashes (<code>/</code>) should be used as path separators.<br><br>Files that do not have matching output directory paths are subject to subsequent filters and transformations to determine if they are copied and what their path in the output directory will be.<br><br>Globs are supported (see rule docstring above). | List of strings | optional | [] |
|
||||
| <a id="copy_to_directory-hardlink"></a>hardlink | Controls when to use hardlinks to files instead of making copies.<br><br>Creating hardlinks is much faster than making copies of files with the caveat that hardlinks share file permissions with their source.<br><br>Since Bazel removes write permissions on files in the output tree after an action completes, hardlinks to source files are not recommended since write permissions will be inadvertently removed from sources files.<br><br>- <code>auto</code>: hardlinks are used for generated files already in the output tree - <code>off</code>: all files are copied - <code>on</code>: hardlinks are used for all files (not recommended) | String | optional | "auto" |
|
||||
| <a id="copy_to_directory-include_external_repositories"></a>include_external_repositories | List of external repository names (with glob support) to include in the output directory.<br><br>Files from external repositories are only copied into the output directory if the external repository they come from matches one of the external repository patterns specified.<br><br>When copied from an external repository, the file path in the output directory defaults to the file's path within the external repository. The external repository name is _not_ included in that path.<br><br>For example, the following copies <code>@external_repo//path/to:file</code> to <code>path/to/file</code> within the output directory.<br><br><pre><code> copy_to_directory( name = "dir", include_external_repositories = ["external_*"], srcs = ["@external_repo//path/to:file"], ) </code></pre><br><br>Files that come from matching external are subject to subsequent filters and transformations to determine if they are copied and what their path in the output directory will be. The external repository name of the file from an external repository is not included in the output directory path and is considered in subsequent filters and transformations.<br><br>Globs are supported (see rule docstring above). | List of strings | optional | [] |
|
||||
| <a id="copy_to_directory-include_srcs_packages"></a>include_srcs_packages | List of Bazel packages (with glob support) to include in output directory.<br><br>Files in srcs are only copied to the output directory if the Bazel package of the file matches one of the patterns specified.<br><br>Forward slashes (<code>/</code>) should be used as path separators. A first character of <code>"."</code> will be replaced by the target's package path.<br><br>Defaults to <code>["**"]</code> which includes sources from all packages.<br><br>Files that have matching Bazel packages are subject to subsequent filters and transformations to determine if they are copied and what their path in the output directory will be.<br><br>Globs are supported (see rule docstring above). | List of strings | optional | ["**"] |
|
||||
| <a id="copy_to_directory-include_srcs_patterns"></a>include_srcs_patterns | List of paths (with glob support) to include in output directory.<br><br>Files in srcs are only copied to the output directory if their output directory path, after applying <code>root_paths</code>, matches one of the patterns specified.<br><br>Forward slashes (<code>/</code>) should be used as path separators.<br><br>Defaults to <code>["**"]</code> which includes all sources.<br><br>Files that have matching output directory paths are subject to subsequent filters and transformations to determine if they are copied and what their path in the output directory will be.<br><br>Globs are supported (see rule docstring above). | List of strings | optional | ["**"] |
|
||||
|
|
@ -117,7 +119,7 @@ other rule implementations where additional_files can also be passed in.
|
|||
copy_to_directory_bin_action(<a href="#copy_to_directory_bin_action-ctx">ctx</a>, <a href="#copy_to_directory_bin_action-name">name</a>, <a href="#copy_to_directory_bin_action-dst">dst</a>, <a href="#copy_to_directory_bin_action-copy_to_directory_bin">copy_to_directory_bin</a>, <a href="#copy_to_directory_bin_action-files">files</a>, <a href="#copy_to_directory_bin_action-targets">targets</a>, <a href="#copy_to_directory_bin_action-root_paths">root_paths</a>,
|
||||
<a href="#copy_to_directory_bin_action-include_external_repositories">include_external_repositories</a>, <a href="#copy_to_directory_bin_action-include_srcs_packages">include_srcs_packages</a>,
|
||||
<a href="#copy_to_directory_bin_action-exclude_srcs_packages">exclude_srcs_packages</a>, <a href="#copy_to_directory_bin_action-include_srcs_patterns">include_srcs_patterns</a>, <a href="#copy_to_directory_bin_action-exclude_srcs_patterns">exclude_srcs_patterns</a>,
|
||||
<a href="#copy_to_directory_bin_action-exclude_prefixes">exclude_prefixes</a>, <a href="#copy_to_directory_bin_action-replace_prefixes">replace_prefixes</a>, <a href="#copy_to_directory_bin_action-allow_overwrites">allow_overwrites</a>, <a href="#copy_to_directory_bin_action-verbose">verbose</a>)
|
||||
<a href="#copy_to_directory_bin_action-exclude_prefixes">exclude_prefixes</a>, <a href="#copy_to_directory_bin_action-replace_prefixes">replace_prefixes</a>, <a href="#copy_to_directory_bin_action-allow_overwrites">allow_overwrites</a>, <a href="#copy_to_directory_bin_action-hardlink">hardlink</a>, <a href="#copy_to_directory_bin_action-verbose">verbose</a>)
|
||||
</pre>
|
||||
|
||||
Factory function to copy files to a directory using a tool binary.
|
||||
|
|
@ -149,6 +151,7 @@ other rule implementations where additional_files can also be passed in.
|
|||
| <a id="copy_to_directory_bin_action-exclude_prefixes"></a>exclude_prefixes | List of path prefixes to exclude from output directory.<br><br>See copy_to_directory rule documentation for more details. | <code>[]</code> |
|
||||
| <a id="copy_to_directory_bin_action-replace_prefixes"></a>replace_prefixes | Map of paths prefixes to replace in the output directory path when copying files.<br><br>See copy_to_directory rule documentation for more details. | <code>{}</code> |
|
||||
| <a id="copy_to_directory_bin_action-allow_overwrites"></a>allow_overwrites | If True, allow files to be overwritten if the same output file is copied to twice.<br><br>See copy_to_directory rule documentation for more details. | <code>False</code> |
|
||||
| <a id="copy_to_directory_bin_action-hardlink"></a>hardlink | Controls when to use hardlinks to files instead of making copies.<br><br>See copy_to_directory rule documentation for more details. | <code>"auto"</code> |
|
||||
| <a id="copy_to_directory_bin_action-verbose"></a>verbose | If true, prints out verbose logs to stdout | <code>False</code> |
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -20,6 +20,25 @@ Load dependencies required by aspect rules
|
|||
| <a id="aspect_bazel_lib_dependencies-override_local_config_platform"></a>override_local_config_platform | override the @local_config_platform repository with one that adds stardoc support for loading constraints.bzl.<br><br>Should be set in repositories that load @aspect_bazel_lib copy actions and also generate stardoc. | <code>False</code> |
|
||||
|
||||
|
||||
<a id="register_copy_directory_toolchains"></a>
|
||||
|
||||
## register_copy_directory_toolchains
|
||||
|
||||
<pre>
|
||||
register_copy_directory_toolchains(<a href="#register_copy_directory_toolchains-name">name</a>, <a href="#register_copy_directory_toolchains-register">register</a>)
|
||||
</pre>
|
||||
|
||||
Registers copy_directory toolchain and repositories
|
||||
|
||||
**PARAMETERS**
|
||||
|
||||
|
||||
| Name | Description | Default Value |
|
||||
| :------------- | :------------- | :------------- |
|
||||
| <a id="register_copy_directory_toolchains-name"></a>name | override the prefix for the generated toolchain repositories | <code>"copy_directory"</code> |
|
||||
| <a id="register_copy_directory_toolchains-register"></a>register | whether to call through to native.register_toolchains. Should be True for WORKSPACE users, but false when used under bzlmod extension | <code>True</code> |
|
||||
|
||||
|
||||
<a id="register_copy_to_directory_toolchains"></a>
|
||||
|
||||
## register_copy_to_directory_toolchains
|
||||
|
|
|
|||
|
|
@ -23,6 +23,10 @@ toolchain_type(
|
|||
name = "yq_toolchain_type",
|
||||
)
|
||||
|
||||
toolchain_type(
|
||||
name = "copy_directory_toolchain_type",
|
||||
)
|
||||
|
||||
toolchain_type(
|
||||
name = "copy_to_directory_toolchain_type",
|
||||
)
|
||||
|
|
@ -193,6 +197,7 @@ bzl_library(
|
|||
srcs = ["repositories.bzl"],
|
||||
deps = [
|
||||
":utils",
|
||||
"//lib/private/docs:copy_directory_toolchain",
|
||||
"//lib/private/docs:copy_to_directory_toolchain",
|
||||
"//lib/private/docs:coreutils_toolchain",
|
||||
"//lib/private/docs:jq_toolchain",
|
||||
|
|
|
|||
|
|
@ -8,7 +8,9 @@ load(
|
|||
"//lib/private:copy_directory.bzl",
|
||||
_copy_directory = "copy_directory",
|
||||
_copy_directory_action = "copy_directory_action",
|
||||
_copy_directory_bin_action = "copy_directory_bin_action",
|
||||
)
|
||||
|
||||
copy_directory = _copy_directory
|
||||
copy_directory_action = _copy_directory_action
|
||||
copy_directory_bin_action = _copy_directory_bin_action
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
load(
|
||||
"@aspect_bazel_lib//lib:repositories.bzl",
|
||||
"register_copy_directory_toolchains",
|
||||
"register_copy_to_directory_toolchains",
|
||||
"register_jq_toolchains",
|
||||
"register_yq_toolchains",
|
||||
|
|
@ -10,6 +11,7 @@ load(
|
|||
def _toolchain_extension(_):
|
||||
register_yq_toolchains(register = False)
|
||||
register_jq_toolchains(register = False)
|
||||
register_copy_directory_toolchains(register = False)
|
||||
register_copy_to_directory_toolchains(register = False)
|
||||
|
||||
# TODO: some way for users to control repo name/version of the tools installed
|
||||
|
|
|
|||
|
|
@ -62,8 +62,15 @@ def _copy_bash(ctx, src, dst):
|
|||
execution_requirements = _COPY_EXECUTION_REQUIREMENTS,
|
||||
)
|
||||
|
||||
# TODO(2.0): remove the legacy copy_directory_action helper
|
||||
def copy_directory_action(ctx, src, dst, is_windows = None):
|
||||
"""Factory function that creates an action to copy a directory from src to dst.
|
||||
"""Legacy factory function that creates an action to copy a directory from src to dst.
|
||||
|
||||
For improved analysis and runtime performance, it is recommended the switch
|
||||
to `copy_directory_bin_action` which takes a tool binary, typically the
|
||||
`@aspect_bazel_lib//tools/copy_to_directory` `go_binary` either built from
|
||||
source or provided by a toolchain and creates hard links instead of performing full
|
||||
file copies.
|
||||
|
||||
This helper is used by copy_directory. It is exposed as a public API so it can be used within
|
||||
other rule implementations.
|
||||
|
|
@ -115,6 +122,7 @@ def copy_directory_bin_action(
|
|||
copy_directory_bin: Copy to directory tool binary.
|
||||
|
||||
hardlink: Controls when to use hardlinks to files instead of making copies.
|
||||
|
||||
See copy_directory rule documentation for more details.
|
||||
|
||||
verbose: If true, prints out verbose logs to stdout
|
||||
|
|
@ -142,13 +150,26 @@ def copy_directory_bin_action(
|
|||
)
|
||||
|
||||
def _copy_directory_impl(ctx):
|
||||
copy_directory_bin = ctx.toolchains["@aspect_bazel_lib//lib:copy_directory_toolchain_type"].copy_directory_info.bin
|
||||
|
||||
dst = ctx.actions.declare_directory(ctx.attr.out)
|
||||
copy_directory_action(ctx, ctx.file.src, dst)
|
||||
|
||||
files = depset(direct = [dst])
|
||||
runfiles = ctx.runfiles(files = [dst])
|
||||
copy_directory_bin_action(
|
||||
ctx,
|
||||
src = ctx.file.src,
|
||||
dst = dst,
|
||||
# copy_directory_bin = ctx.executable._tool, # use for development
|
||||
copy_directory_bin = copy_directory_bin,
|
||||
hardlink = ctx.attr.hardlink,
|
||||
verbose = ctx.attr.verbose,
|
||||
)
|
||||
|
||||
return [DefaultInfo(files = files, runfiles = runfiles)]
|
||||
return [
|
||||
DefaultInfo(
|
||||
files = depset([dst]),
|
||||
runfiles = ctx.runfiles([dst]),
|
||||
),
|
||||
]
|
||||
|
||||
_copy_directory = rule(
|
||||
implementation = _copy_directory_impl,
|
||||
|
|
@ -158,10 +179,28 @@ _copy_directory = rule(
|
|||
# Cannot declare out as an output here, because there's no API for declaring
|
||||
# TreeArtifact outputs.
|
||||
"out": attr.string(mandatory = True),
|
||||
"hardlink": attr.string(
|
||||
values = ["auto", "off", "on"],
|
||||
default = "auto",
|
||||
),
|
||||
"verbose": attr.bool(),
|
||||
# use '_tool' attribute for development only; do not commit with this attribute active since it
|
||||
# propagates a dependency on rules_go which would be breaking for users
|
||||
# "_tool": attr.label(
|
||||
# executable = True,
|
||||
# cfg = "exec",
|
||||
# default = "//tools/copy_directory",
|
||||
# ),
|
||||
},
|
||||
toolchains = ["@aspect_bazel_lib//lib:copy_directory_toolchain_type"],
|
||||
)
|
||||
|
||||
def copy_directory(name, src, out, **kwargs):
|
||||
def copy_directory(
|
||||
name,
|
||||
src,
|
||||
out,
|
||||
hardlink = "auto",
|
||||
**kwargs):
|
||||
"""Copies a directory to another location.
|
||||
|
||||
This rule uses a Bash command on Linux/macOS/non-Windows, and a cmd.exe command on Windows (no Bash is required).
|
||||
|
|
@ -174,13 +213,30 @@ def copy_directory(name, src, out, **kwargs):
|
|||
|
||||
Args:
|
||||
name: Name of the rule.
|
||||
|
||||
src: The directory to make a copy of. Can be a source directory or TreeArtifact.
|
||||
|
||||
out: Path of the output directory, relative to this package.
|
||||
|
||||
hardlink: Controls when to use hardlinks to files instead of making copies.
|
||||
|
||||
Creating hardlinks is much faster than making copies of files with the caveat that
|
||||
hardlinks share file permissions with their source.
|
||||
|
||||
Since Bazel removes write permissions on files in the output tree after an action completes,
|
||||
hardlinks to source files within source directories is not recommended since write
|
||||
permissions will be inadvertently removed from sources files.
|
||||
|
||||
- "auto": hardlinks are used if src is a tree artifact already in the output tree
|
||||
- "off": files are always copied
|
||||
- "on": hardlinks are always used (not recommended)
|
||||
|
||||
**kwargs: further keyword arguments, e.g. `visibility`
|
||||
"""
|
||||
_copy_directory(
|
||||
name = name,
|
||||
src = src,
|
||||
out = out,
|
||||
hardlink = hardlink,
|
||||
**kwargs
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,188 @@
|
|||
"Setup copy_directory toolchain repositories and rules"
|
||||
|
||||
# https://github.com/aspect-build/bazel-lib/releases
|
||||
#
|
||||
# The integrity hashes can be automatically fetched for the latest copy_directory release by running
|
||||
# `tools/copy_directory_mirror_release.sh`. To calculate for a specific release run
|
||||
# `tools/copy_directory_mirror_release.sh <release_version>`
|
||||
|
||||
COPY_DIRECTORY_VERSION = "1.22.0"
|
||||
COPY_DIRECTORY_INTEGRITY = {
|
||||
"darwin_amd64": "sha256-zDnAx/e5J61iWmmdYuQ+pABjgh16c9Tl7gmJpPySj8Q=",
|
||||
"darwin_arm64": "sha256-wQKrD6CVyZgyi+PFG192dNkwZpPNa8u9sv9BimBY8NA=",
|
||||
"linux_amd64": "sha256-J72eI31EZ3E5WxnvH/XJ9HYJD674ZSmj0ZngSPHwfhA=",
|
||||
"linux_arm64": "sha256-iAvIQrwhzwLzHRAMs7YNAIpmb9xI8Azr/ji6QFC0QGs=",
|
||||
"windows_amd64": "sha256-e4Qslt/gGFm7zyVcoe//OcAvtrDtxkJhKWaufjayH7E=",
|
||||
}
|
||||
|
||||
# Platform names follow the platform naming convention in @aspect_bazel_lib//:lib/private/repo_utils.bzl
|
||||
COPY_DIRECTORY_PLATFORMS = {
|
||||
"darwin_amd64": struct(
|
||||
compatible_with = [
|
||||
"@platforms//os:macos",
|
||||
"@platforms//cpu:x86_64",
|
||||
],
|
||||
),
|
||||
"darwin_arm64": struct(
|
||||
compatible_with = [
|
||||
"@platforms//os:macos",
|
||||
"@platforms//cpu:aarch64",
|
||||
],
|
||||
),
|
||||
"linux_amd64": struct(
|
||||
compatible_with = [
|
||||
"@platforms//os:linux",
|
||||
"@platforms//cpu:x86_64",
|
||||
],
|
||||
),
|
||||
"linux_arm64": struct(
|
||||
compatible_with = [
|
||||
"@platforms//os:linux",
|
||||
"@platforms//cpu:aarch64",
|
||||
],
|
||||
),
|
||||
"windows_amd64": struct(
|
||||
compatible_with = [
|
||||
"@platforms//os:windows",
|
||||
"@platforms//cpu:x86_64",
|
||||
],
|
||||
),
|
||||
}
|
||||
|
||||
CopyToDirectoryInfo = provider(
|
||||
doc = "Provide info for executing copy_directory",
|
||||
fields = {
|
||||
"bin": "Executable copy_directory binary",
|
||||
},
|
||||
)
|
||||
|
||||
def _copy_directory_toolchain_impl(ctx):
|
||||
binary = ctx.attr.bin.files.to_list()[0]
|
||||
|
||||
default_info = DefaultInfo(
|
||||
files = depset([binary]),
|
||||
runfiles = ctx.runfiles(files = [binary]),
|
||||
)
|
||||
copy_directory_info = CopyToDirectoryInfo(
|
||||
bin = binary,
|
||||
)
|
||||
|
||||
# Export all the providers inside our ToolchainInfo
|
||||
# so the resolved_toolchain rule can grab and re-export them.
|
||||
toolchain_info = platform_common.ToolchainInfo(
|
||||
copy_directory_info = copy_directory_info,
|
||||
default = default_info,
|
||||
)
|
||||
|
||||
return [default_info, toolchain_info]
|
||||
|
||||
copy_directory_toolchain = rule(
|
||||
implementation = _copy_directory_toolchain_impl,
|
||||
attrs = {
|
||||
"bin": attr.label(
|
||||
mandatory = True,
|
||||
allow_single_file = True,
|
||||
executable = True,
|
||||
cfg = "exec",
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
def _copy_directory_toolchains_repo_impl(rctx):
|
||||
# Expose a concrete toolchain which is the result of Bazel resolving the toolchain
|
||||
# for the execution or target platform.
|
||||
# Workaround for https://github.com/bazelbuild/bazel/issues/14009
|
||||
starlark_content = """# @generated by @aspect_bazel_lib//lib/private:copy_directory_toolchain.bzl
|
||||
|
||||
# Forward all the providers
|
||||
def _resolved_toolchain_impl(ctx):
|
||||
toolchain_info = ctx.toolchains["@aspect_bazel_lib//lib:copy_directory_toolchain_type"]
|
||||
return [
|
||||
toolchain_info,
|
||||
toolchain_info.default,
|
||||
toolchain_info.copy_directory_info,
|
||||
toolchain_info.template_variables,
|
||||
]
|
||||
|
||||
# Copied from java_toolchain_alias
|
||||
# https://cs.opensource.google/bazel/bazel/+/master:tools/jdk/java_toolchain_alias.bzl
|
||||
resolved_toolchain = rule(
|
||||
implementation = _resolved_toolchain_impl,
|
||||
toolchains = ["@aspect_bazel_lib//lib:copy_directory_toolchain_type"],
|
||||
incompatible_use_toolchain_transition = True,
|
||||
)
|
||||
"""
|
||||
rctx.file("defs.bzl", starlark_content)
|
||||
|
||||
build_content = """# @generated by @aspect_bazel_lib//lib/private:copy_directory_toolchain.bzl
|
||||
#
|
||||
# These can be registered in the workspace file or passed to --extra_toolchains flag.
|
||||
# By default all these toolchains are registered by the copy_directory_register_toolchains macro
|
||||
# so you don't normally need to interact with these targets.
|
||||
|
||||
load(":defs.bzl", "resolved_toolchain")
|
||||
|
||||
resolved_toolchain(name = "resolved_toolchain", visibility = ["//visibility:public"])
|
||||
|
||||
"""
|
||||
|
||||
for [platform, meta] in COPY_DIRECTORY_PLATFORMS.items():
|
||||
build_content += """
|
||||
toolchain(
|
||||
name = "{platform}_toolchain",
|
||||
exec_compatible_with = {compatible_with},
|
||||
toolchain = "@{user_repository_name}_{platform}//:copy_directory_toolchain",
|
||||
toolchain_type = "@aspect_bazel_lib//lib:copy_directory_toolchain_type",
|
||||
)
|
||||
""".format(
|
||||
platform = platform,
|
||||
user_repository_name = rctx.attr.user_repository_name,
|
||||
compatible_with = meta.compatible_with,
|
||||
)
|
||||
|
||||
# Base BUILD file for this repository
|
||||
rctx.file("BUILD.bazel", build_content)
|
||||
|
||||
copy_directory_toolchains_repo = repository_rule(
|
||||
_copy_directory_toolchains_repo_impl,
|
||||
doc = """Creates a repository with toolchain definitions for all known platforms
|
||||
which can be registered or selected.""",
|
||||
attrs = {
|
||||
"user_repository_name": attr.string(doc = "Base name for toolchains repository"),
|
||||
},
|
||||
)
|
||||
|
||||
def _copy_directory_platform_repo_impl(rctx):
|
||||
is_windows = rctx.attr.platform.startswith("windows_")
|
||||
meta = COPY_DIRECTORY_PLATFORMS[rctx.attr.platform]
|
||||
release_platform = meta.release_platform if hasattr(meta, "release_platform") else rctx.attr.platform
|
||||
|
||||
# https://github.com/aspect-build/bazel-lib/releases/download/v1.19.0/copy_directory-linux_amd64
|
||||
url = "https://github.com/aspect-build/bazel-lib/releases/download/v{0}/copy_directory-{1}{2}".format(
|
||||
COPY_DIRECTORY_VERSION,
|
||||
release_platform,
|
||||
".exe" if is_windows else "",
|
||||
)
|
||||
|
||||
rctx.download(
|
||||
url = url,
|
||||
output = "copy_directory.exe" if is_windows else "copy_directory",
|
||||
executable = True,
|
||||
integrity = COPY_DIRECTORY_INTEGRITY[release_platform],
|
||||
)
|
||||
build_content = """# @generated by @aspect_bazel_lib//lib/private:copy_directory_toolchain.bzl
|
||||
load("@aspect_bazel_lib//lib/private:copy_directory_toolchain.bzl", "copy_directory_toolchain")
|
||||
exports_files(["{0}"])
|
||||
copy_directory_toolchain(name = "copy_directory_toolchain", bin = "{0}", visibility = ["//visibility:public"])
|
||||
""".format("copy_directory.exe" if is_windows else "copy_directory")
|
||||
|
||||
# Base BUILD file for this repository
|
||||
rctx.file("BUILD.bazel", build_content)
|
||||
|
||||
copy_directory_platform_repo = repository_rule(
|
||||
implementation = _copy_directory_platform_repo_impl,
|
||||
doc = "Fetch external tools needed for copy_directory toolchain",
|
||||
attrs = {
|
||||
"platform": attr.string(mandatory = True, values = COPY_DIRECTORY_PLATFORMS.keys()),
|
||||
},
|
||||
)
|
||||
|
|
@ -206,10 +206,25 @@ Globs are supported (see rule docstring above).
|
|||
# allow_overwrites
|
||||
"allow_overwrites": """If True, allow files to be overwritten if the same output file is copied to twice.
|
||||
|
||||
If set, then the order of srcs matters as the last copy of a particular file will win.
|
||||
|
||||
This setting has no effect on Windows where overwrites are always allowed.
|
||||
The order of srcs matters as the last copy of a particular file will win when overwriting.
|
||||
Performance of copy_to_directory will be slightly degraded when allow_overwrites is True
|
||||
since copies cannot be parallelized out as they are calculated. Instead all copy paths
|
||||
must be calculated before any copies can be started.
|
||||
""",
|
||||
# hardlink
|
||||
"hardlink": """Controls when to use hardlinks to files instead of making copies.
|
||||
|
||||
Creating hardlinks is much faster than making copies of files with the caveat that
|
||||
hardlinks share file permissions with their source.
|
||||
|
||||
Since Bazel removes write permissions on files in the output tree after an action completes,
|
||||
hardlinks to source files are not recommended since write permissions will be inadvertently
|
||||
removed from sources files.
|
||||
|
||||
- `auto`: hardlinks are used for generated files already in the output tree
|
||||
- `off`: all files are copied
|
||||
- `on`: hardlinks are used for all files (not recommended)
|
||||
""",
|
||||
# verbose
|
||||
"verbose": """If true, prints out verbose logs to stdout""",
|
||||
}
|
||||
|
|
@ -254,6 +269,11 @@ _copy_to_directory_attr = {
|
|||
"allow_overwrites": attr.bool(
|
||||
doc = _copy_to_directory_attr_doc["allow_overwrites"],
|
||||
),
|
||||
"hardlink": attr.string(
|
||||
values = ["auto", "off", "on"],
|
||||
default = "auto",
|
||||
doc = _copy_to_directory_attr_doc["hardlink"],
|
||||
),
|
||||
"verbose": attr.bool(
|
||||
doc = _copy_to_directory_attr_doc["verbose"],
|
||||
),
|
||||
|
|
@ -580,7 +600,8 @@ def _copy_to_directory_impl(ctx):
|
|||
ctx,
|
||||
name = ctx.attr.name,
|
||||
dst = dst,
|
||||
copy_to_directory_bin = copy_to_directory_bin, # use ctx.executable._tool for development
|
||||
# copy_to_directory_bin = ctx.executable._tool, # use for development
|
||||
copy_to_directory_bin = copy_to_directory_bin,
|
||||
files = ctx.files.srcs,
|
||||
targets = [t for t in ctx.attr.srcs if DirectoryPathInfo in t],
|
||||
root_paths = ctx.attr.root_paths,
|
||||
|
|
@ -592,6 +613,7 @@ def _copy_to_directory_impl(ctx):
|
|||
exclude_prefixes = ctx.attr.exclude_prefixes,
|
||||
replace_prefixes = ctx.attr.replace_prefixes,
|
||||
allow_overwrites = ctx.attr.allow_overwrites,
|
||||
hardlink = ctx.attr.hardlink,
|
||||
verbose = ctx.attr.verbose,
|
||||
)
|
||||
|
||||
|
|
@ -631,6 +653,7 @@ def copy_to_directory_bin_action(
|
|||
exclude_prefixes = [],
|
||||
replace_prefixes = {},
|
||||
allow_overwrites = False,
|
||||
hardlink = "auto",
|
||||
verbose = False):
|
||||
"""Factory function to copy files to a directory using a tool binary.
|
||||
|
||||
|
|
@ -689,6 +712,10 @@ def copy_to_directory_bin_action(
|
|||
|
||||
See copy_to_directory rule documentation for more details.
|
||||
|
||||
hardlink: Controls when to use hardlinks to files instead of making copies.
|
||||
|
||||
See copy_to_directory rule documentation for more details.
|
||||
|
||||
verbose: If true, prints out verbose logs to stdout
|
||||
"""
|
||||
|
||||
|
|
@ -765,6 +792,12 @@ def copy_to_directory_bin_action(
|
|||
msg = "Expected owner target label for file {} to have a workspace name but found None".format(f)
|
||||
fail(msg)
|
||||
|
||||
hardlink_file = False
|
||||
if hardlink == "on":
|
||||
hardlink_file = True
|
||||
elif hardlink == "auto":
|
||||
hardlink_file = not f.file.is_source
|
||||
|
||||
file_infos.append({
|
||||
"package": f.file.owner.package,
|
||||
"path": f.path,
|
||||
|
|
@ -772,6 +805,7 @@ def copy_to_directory_bin_action(
|
|||
"short_path": f.short_path,
|
||||
"workspace": f.file.owner.workspace_name,
|
||||
"workspace_path": f.workspace_path,
|
||||
"hardlink": hardlink_file,
|
||||
})
|
||||
file_inputs.append(f.file)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,20 @@
|
|||
"Setup copy_to_directory toolchain repositories and rules"
|
||||
|
||||
# https://github.com/aspect-build/bazel-lib/releases
|
||||
#
|
||||
# The integrity hashes can be automatically fetched for the latest copy_to_directory release by running
|
||||
# `tools/copy_to_directory_mirror_release.sh`. To calculate for a specific release run
|
||||
# `tools/copy_to_directory_mirror_release.sh <release_version>`
|
||||
|
||||
COPY_TO_DIRECTORY_VERSION = "1.22.0"
|
||||
COPY_TO_DIRECTORY_INTEGRITY = {
|
||||
"darwin_amd64": "sha256-l6P/W5CvPIhkF8ayilEu2KetQVRaK7npwsy+o7PWvDM=",
|
||||
"darwin_arm64": "sha256-xiK5HNDa4izFtfsIvKdmKqjnqXR7jJkP/+i6BrYhuNY=",
|
||||
"linux_amd64": "sha256-v6fSwJVfMoyGK7N46Io/+hxb37hVFjnGSGz8R4Bj32I=",
|
||||
"linux_arm64": "sha256-LdrJIwiQQMVPBucw7fqEg8KFECS75gVrx8TbEeXPz7E=",
|
||||
"windows_amd64": "sha256-0GNyAS19fOCWQ4ntf/E6KodUss/jm9zD634v5O29lQ8=",
|
||||
}
|
||||
|
||||
# Platform names follow the platform naming convention in @aspect_bazel_lib//:lib/private/repo_utils.bzl
|
||||
COPY_TO_DIRECTORY_PLATFORMS = {
|
||||
"darwin_amd64": struct(
|
||||
|
|
@ -34,21 +49,6 @@ COPY_TO_DIRECTORY_PLATFORMS = {
|
|||
),
|
||||
}
|
||||
|
||||
# https://github.com/aspect-build/bazel-lib/releases
|
||||
#
|
||||
# The integrity hashes can be automatically fetched for the latest copy_to_directory release by running
|
||||
# `tools/copy_to_directory_mirror_release.sh`. To calculate for a specific release run
|
||||
# `tools/copy_to_directory_mirror_release.sh <release_version>`
|
||||
|
||||
COPY_TO_DIRECTORY_VERSION = "1.19.2"
|
||||
COPY_TO_DIRECTORY_INTEGRITY = {
|
||||
"darwin_amd64": "sha256-TdU9sae6uxjkABWnIh5DhHr52KuNPy/F3Lo5MH/hbWc=",
|
||||
"darwin_arm64": "sha256-jLM7ZWI1xqp1tHcOf8MdQOurgLlAhppvhr1zSsBwOqA=",
|
||||
"linux_amd64": "sha256-38Plushs6HNUemnAGRS7nN1Bc9Fq5z209m9OOGVCrm4=",
|
||||
"linux_arm64": "sha256-TbyibQbo2QORCBpX08e4YLDPbFsCQNyk1jxePwe1cFI=",
|
||||
"windows_amd64": "sha256-0xgHiwX0qHi2TGn1v+cBK7rHTgaz2ulYxCnvF07cVh4=",
|
||||
}
|
||||
|
||||
CopyToDirectoryInfo = provider(
|
||||
doc = "Provide info for executing copy_to_directory",
|
||||
fields = {
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ bzl_library(
|
|||
srcs = ["//lib/private:copy_directory.bzl"],
|
||||
deps = [
|
||||
":copy_common",
|
||||
":paths",
|
||||
":platform_utils",
|
||||
],
|
||||
)
|
||||
|
|
@ -228,6 +229,11 @@ bzl_library(
|
|||
srcs = ["//lib/private:base64.bzl"],
|
||||
)
|
||||
|
||||
bzl_library(
|
||||
name = "copy_directory_toolchain",
|
||||
srcs = ["//lib/private:copy_directory_toolchain.bzl"],
|
||||
)
|
||||
|
||||
bzl_library(
|
||||
name = "copy_to_directory_toolchain",
|
||||
srcs = ["//lib/private:copy_to_directory_toolchain.bzl"],
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ load("//lib/private:jq_toolchain.bzl", "JQ_PLATFORMS", "jq_host_alias_repo", "jq
|
|||
load("//lib/private:yq_toolchain.bzl", "YQ_PLATFORMS", "yq_host_alias_repo", "yq_platform_repo", "yq_toolchains_repo", _DEFAULT_YQ_VERSION = "DEFAULT_YQ_VERSION")
|
||||
load("//lib/private:copy_to_directory_toolchain.bzl", "COPY_TO_DIRECTORY_PLATFORMS", "copy_to_directory_platform_repo", "copy_to_directory_toolchains_repo")
|
||||
load("//lib/private:coreutils_toolchain.bzl", "COREUTILS_PLATFORMS", "coreutils_platform_repo", "coreutils_toolchains_repo", _DEFAULT_COREUTILS_VERSION = "DEFAULT_COREUTILS_VERSION")
|
||||
load("//lib/private:copy_directory_toolchain.bzl", "COPY_DIRECTORY_PLATFORMS", "copy_directory_platform_repo", "copy_directory_toolchains_repo")
|
||||
load("//lib/private:local_config_platform.bzl", "local_config_platform")
|
||||
load("//lib:utils.bzl", "is_bazel_6_or_greater", http_archive = "maybe_http_archive")
|
||||
|
||||
|
|
@ -35,6 +36,7 @@ def aspect_bazel_lib_dependencies(override_local_config_platform = False):
|
|||
)
|
||||
|
||||
# Always register the copy_to_directory toolchain
|
||||
register_copy_directory_toolchains()
|
||||
register_copy_to_directory_toolchains()
|
||||
|
||||
# Re-export the default versions
|
||||
|
|
@ -105,7 +107,7 @@ def register_coreutils_toolchains(name = "coreutils", version = DEFAULT_COREUTIL
|
|||
coreutils_platform_repo(
|
||||
name = "%s_%s" % (name, platform),
|
||||
platform = platform,
|
||||
version = version
|
||||
version = version,
|
||||
)
|
||||
if register:
|
||||
native.register_toolchains("@%s_toolchains//:%s_toolchain" % (name, platform))
|
||||
|
|
@ -115,6 +117,27 @@ def register_coreutils_toolchains(name = "coreutils", version = DEFAULT_COREUTIL
|
|||
user_repository_name = name,
|
||||
)
|
||||
|
||||
def register_copy_directory_toolchains(name = "copy_directory", register = True):
|
||||
"""Registers copy_directory toolchain and repositories
|
||||
|
||||
Args:
|
||||
name: override the prefix for the generated toolchain repositories
|
||||
register: whether to call through to native.register_toolchains.
|
||||
Should be True for WORKSPACE users, but false when used under bzlmod extension
|
||||
"""
|
||||
for [platform, meta] in COPY_DIRECTORY_PLATFORMS.items():
|
||||
copy_directory_platform_repo(
|
||||
name = "%s_%s" % (name, platform),
|
||||
platform = platform,
|
||||
)
|
||||
if register:
|
||||
native.register_toolchains("@%s_toolchains//:%s_toolchain" % (name, platform))
|
||||
|
||||
copy_directory_toolchains_repo(
|
||||
name = "%s_toolchains" % name,
|
||||
user_repository_name = name,
|
||||
)
|
||||
|
||||
def register_copy_to_directory_toolchains(name = "copy_to_directory", register = True):
|
||||
"""Registers copy_to_directory toolchain and repositories
|
||||
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ copy_directory(
|
|||
name = "dir_copy",
|
||||
src = "dir",
|
||||
out = "copy_of_dir",
|
||||
verbose = True,
|
||||
)
|
||||
|
||||
diff_test(
|
||||
|
|
@ -35,3 +36,16 @@ diff_test(
|
|||
file1 = ":dir",
|
||||
file2 = ":dir_copy",
|
||||
)
|
||||
|
||||
copy_directory(
|
||||
name = "external_dir_copy",
|
||||
src = "@external_test_repo//:test_a",
|
||||
out = "external_dir",
|
||||
verbose = True,
|
||||
)
|
||||
|
||||
diff_test(
|
||||
name = "copy_external_directory_test",
|
||||
file1 = ":external_dir_copy",
|
||||
file2 = "@external_test_repo//:test_a",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ copy_directory(
|
|||
lib(
|
||||
name = "lib",
|
||||
srcs = [
|
||||
"1",
|
||||
"copy_1",
|
||||
],
|
||||
# intentionally dup on "1" to make sure it is gracefully handled
|
||||
others = [
|
||||
|
|
|
|||
|
|
@ -59,6 +59,7 @@ def _pkg_impl(ctx):
|
|||
files = ctx.files.srcs + symlinks + depset(transitive = additional_files_depsets).to_list(),
|
||||
dst = dst,
|
||||
copy_to_directory_bin = ctx.executable._tool,
|
||||
hardlink = "auto",
|
||||
verbose = True,
|
||||
)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue