bazel
/
bazel-lib
mirror of https://github.com/bazel-contrib/bazel-lib
2
0
Fork 0
Code Issues Releases Activity

feat: add directory_path rule and DirectoryPathInfo for specifying files within tree artifacts (#42)

This commit is contained in:
Greg Magolan 2022-03-03 16:40:32 -08:00 committed by GitHub
parent f788d286d0
commit 0fd56dc59d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 616 additions and 178 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
docs/BUILD.bazel
View File

@ -3,7 +3,7 @@
load("//lib:docs.bzl", "stardoc_with_diff_test", "update_docs")
stardoc_with_diff_test(
bzl_library_target = "//lib/private:copy_to_directory",
bzl_library_target = "//lib:copy_to_directory",
out_label = "//docs:copy_to_directory.md",
)
@ -43,6 +43,11 @@ stardoc_with_diff_test(
out_label = "//docs:write_source_files.md",
)
stardoc_with_diff_test(
bzl_library_target = "//lib:directory_path",
out_label = "//docs:directory_path.md",
)
update_docs(
name = "update",
docs_folder = "docs",

36
docs/copy_to_directory.md
View File

@ -7,8 +7,8 @@ Copy files and directories to an output directory
## copy_to_directory
<pre>
copy_to_directory(<a href="#copy_to_directory-name">name</a>, <a href="#copy_to_directory-exclude_prefixes">exclude_prefixes</a>, <a href="#copy_to_directory-include_external_repositories">include_external_repositories</a>, <a href="#copy_to_directory-is_windows">is_windows</a>,
<a href="#copy_to_directory-replace_prefixes">replace_prefixes</a>, <a href="#copy_to_directory-root_paths">root_paths</a>, <a href="#copy_to_directory-srcs">srcs</a>)
copy_to_directory(<a href="#copy_to_directory-name">name</a>, <a href="#copy_to_directory-srcs">srcs</a>, <a href="#copy_to_directory-root_paths">root_paths</a>, <a href="#copy_to_directory-include_external_repositories">include_external_repositories</a>, <a href="#copy_to_directory-exclude_prefixes">exclude_prefixes</a>,
<a href="#copy_to_directory-replace_prefixes">replace_prefixes</a>, <a href="#copy_to_directory-kwargs">kwargs</a>)
</pre>
Copies files and directories to an output directory.
@ -17,35 +17,17 @@ Files and directories can be arranged as needed in the output directory using
the `root_paths`, `exclude_prefixes` and `replace_prefixes` attributes.
**ATTRIBUTES**
| Name | Description | Type | Mandatory | Default |
| :------------- | :------------- | :------------- | :------------- | :------------- |
| <a id="copy_to_directory-name"></a>name | A unique name for this target. | <a href="https://bazel.build/docs/build-ref.html#name">Name</a> | required | |
| <a id="copy_to_directory-exclude_prefixes"></a>exclude_prefixes | List of path prefixes to exclude from output directory.<br><br>If the output directory path for a file or directory starts with or is equal to a path in the list then that file is not copied to the output directory.<br><br>Exclude prefixes are matched *before* replace_prefixes are applied. | List of strings | optional | [] |
| <a id="copy_to_directory-include_external_repositories"></a>include_external_repositories | List of external repository names to include in the output directory.<br><br>Files from external repositories are not copied into the output directory unless the external repository they come from is listed here.<br><br>When copied from an external repository, the file path in the output directory defaults to the file's path within the external repository. The external repository name is _not_ included in that path.<br><br>For example, the following copies <code>@external_repo//path/to:file</code> to <code>path/to/file</code> within the output directory.<br><br><pre><code> copy_to_directory( name = "dir", include_external_repositories = ["external_repo"], srcs = ["@external_repo//path/to:file"], ) </code></pre><br><br>Files from external repositories are subject to <code>root_paths</code>, <code>exclude_prefixes</code> and <code>replace_prefixes</code> in the same way as files form the main repository. | List of strings | optional | [] |
| <a id="copy_to_directory-is_windows"></a>is_windows | - | Boolean | required | |
| <a id="copy_to_directory-replace_prefixes"></a>replace_prefixes | Map of paths prefixes to replace in the output directory path when copying files.<br><br>If the output directory path for a file or directory starts with or is equal to a key in the dict then the matching portion of the output directory path is replaced with the dict value for that key.<br><br>Forward slashes (<code>/</code>) should be used as path separators. The final path segment of the key can be a partial match in the corresponding segment of the output directory path.<br><br>If there are multiple keys that match, the longest match wins. | <a href="https://bazel.build/docs/skylark/lib/dict.html">Dictionary: String -> String</a> | optional | {} |
| <a id="copy_to_directory-root_paths"></a>root_paths | List of paths that are roots in the output directory. If a file or directory being copied is in one of the listed paths or one of its subpaths, the output directory path is the path relative to the root path instead of the path relative to the file's workspace.<br><br>Forward slashes (<code>/</code>) should be used as path separators. Partial matches on the final path segment of a root path against the corresponding segment in the full workspace relative path of a file are not matched.<br><br>If there are multiple root paths that match, the longest match wins.<br><br>Defaults to [package_name()] so that the output directory path of files in the target's package and and sub-packages are relative to the target's package and files outside of that retain their full workspace relative paths. | List of strings | optional | [] |
| <a id="copy_to_directory-srcs"></a>srcs | Files and/or directories to copy into the output directory | <a href="https://bazel.build/docs/build-ref.html#labels">List of labels</a> | optional | [] |
<a id="#copy_to_directory_lib.impl"></a>
## copy_to_directory_lib.impl
<pre>
copy_to_directory_lib.impl(<a href="#copy_to_directory_lib.impl-ctx">ctx</a>)
</pre>
**PARAMETERS**
| Name | Description | Default Value |
| :------------- | :------------- | :------------- |
| <a id="copy_to_directory_lib.impl-ctx"></a>ctx | <p align="center"> - </p> | none |
| <a id="copy_to_directory-name"></a>name | A unique name for this target. | none |
| <a id="copy_to_directory-srcs"></a>srcs | Files and/or directories or targets that provide DirectoryPathInfo to copy into the output directory. | <code>[]</code> |
| <a id="copy_to_directory-root_paths"></a>root_paths | List of paths that are roots in the output directory.<br><br>If a file or directory being copied is in one of the listed paths or one of its subpaths, the output directory path is the path relative to the root path instead of the path relative to the file's workspace.<br><br>Forward slashes (<code>/</code>) should be used as path separators. Partial matches on the final path segment of a root path against the corresponding segment in the full workspace relative path of a file are not matched.<br><br>If there are multiple root paths that match, the longest match wins.<br><br>Defaults to [package_name()] so that the output directory path of files in the target's package and and sub-packages are relative to the target's package and files outside of that retain their full workspace relative paths. | <code>None</code> |
| <a id="copy_to_directory-include_external_repositories"></a>include_external_repositories | List of external repository names to include in the output directory.<br><br>Files from external repositories are not copied into the output directory unless the external repository they come from is listed here.<br><br>When copied from an external repository, the file path in the output directory defaults to the file's path within the external repository. The external repository name is _not_ included in that path.<br><br>For example, the following copies <code>@external_repo//path/to:file</code> to <code>path/to/file</code> within the output directory.<br><br><pre><code> copy_to_directory( name = "dir", include_external_repositories = ["external_repo"], srcs = ["@external_repo//path/to:file"], ) </code></pre><br><br>Files from external repositories are subject to <code>root_paths</code>, <code>exclude_prefixes</code> and <code>replace_prefixes</code> in the same way as files form the main repository. | <code>[]</code> |
| <a id="copy_to_directory-exclude_prefixes"></a>exclude_prefixes | List of path prefixes to exclude from output directory.<br><br>If the output directory path for a file or directory starts with or is equal to a path in the list then that file is not copied to the output directory.<br><br>Exclude prefixes are matched *before* replace_prefixes are applied. | <code>[]</code> |
| <a id="copy_to_directory-replace_prefixes"></a>replace_prefixes | Map of paths prefixes to replace in the output directory path when copying files.<br><br>If the output directory path for a file or directory starts with or is equal to a key in the dict then the matching portion of the output directory path is replaced with the dict value for that key.<br><br>Forward slashes (<code>/</code>) should be used as path separators. The final path segment of the key can be a partial match in the corresponding segment of the output directory path.<br><br>If there are multiple keys that match, the longest match wins. | <code>{}</code> |
| <a id="copy_to_directory-kwargs"></a>kwargs | Other common named parameters such as <code>tags</code> or <code>visibility</code> | none |

136
docs/directory_path.md Normal file
View File

@ -0,0 +1,136 @@
<!-- Generated with Stardoc: http://skydoc.bazel.build -->
Rule and corresponding provider that joins a label pointing to a TreeArtifact
with a path nested within that directory
<a id="#directory_path"></a>
## directory_path
<pre>
directory_path(<a href="#directory_path-name">name</a>, <a href="#directory_path-directory">directory</a>, <a href="#directory_path-path">path</a>)
</pre>
Provide DirectoryPathInfo to reference some path within a directory.
Otherwise there is no way to give a Bazel label for it.
**ATTRIBUTES**
| Name | Description | Type | Mandatory | Default |
| :------------- | :------------- | :------------- | :------------- | :------------- |
| <a id="directory_path-name"></a>name | A unique name for this target. | <a href="https://bazel.build/docs/build-ref.html#name">Name</a> | required | |
| <a id="directory_path-directory"></a>directory | a TreeArtifact (ctx.actions.declare_directory) | <a href="https://bazel.build/docs/build-ref.html#labels">Label</a> | required | |
| <a id="directory_path-path"></a>path | path relative to the directory | String | required | |
<a id="#DirectoryPathInfo"></a>
## DirectoryPathInfo
<pre>
DirectoryPathInfo(<a href="#DirectoryPathInfo-directory">directory</a>, <a href="#DirectoryPathInfo-path">path</a>)
</pre>
Joins a label pointing to a TreeArtifact with a path nested within that directory.
**FIELDS**
| Name | Description |
| :------------- | :------------- |
| <a id="DirectoryPathInfo-directory"></a>directory | a TreeArtifact (ctx.actions.declare_directory) |
| <a id="DirectoryPathInfo-path"></a>path | path relative to the directory |
<a id="#make_directory_path"></a>
## make_directory_path
<pre>
make_directory_path(<a href="#make_directory_path-name">name</a>, <a href="#make_directory_path-directory">directory</a>, <a href="#make_directory_path-path">path</a>)
</pre>
Helper function to convert generate a directory_path target and return its label.
**PARAMETERS**
| Name | Description | Default Value |
| :------------- | :------------- | :------------- |
| <a id="make_directory_path-name"></a>name | Unique name for the generated <code>directory_path</code> target. | none |
| <a id="make_directory_path-directory"></a>directory | a TreeArtifact (ctx.actions.declare_directory) | none |
| <a id="make_directory_path-path"></a>path | path relative to the directory | none |
**RETURNS**
The label `name`
<a id="#make_directory_paths"></a>
## make_directory_paths
<pre>
make_directory_paths(<a href="#make_directory_paths-name">name</a>, <a href="#make_directory_paths-dict">dict</a>)
</pre>
Helper function to convert a dict of directory to path mappings to directory_path targets and labels.
For example,
```
make_directory_paths("my_name", {
"//directory/artifact:target_1": "file/path",
"//directory/artifact:target_2": ["file/path1", "file/path2"],
})
```
generates the targets,
```
directory_path(
name = "my_name_0",
directory = "//directory/artifact:target_1",
path = "file/path"
)
directory_path(
name = "my_name_1",
directory = "//directory/artifact:target_2",
path = "file/path1"
)
directory_path(
name = "my_name_2",
directory = "//directory/artifact:target_2",
path = "file/path2"
)
```
and the list of targets is returned,
```
[
"my_name_0",
"my_name_1",
"my_name_2",
]
```
**PARAMETERS**
| Name | Description | Default Value |
| :------------- | :------------- | :------------- |
| <a id="make_directory_paths-name"></a>name | The target name to use for the generated targets & labels.<br><br>The names are generated as zero-indexed <code>name + "_" + i</code> | none |
| <a id="make_directory_paths-dict"></a>dict | The dictionary of directory keys to path or path list values. | none |
**RETURNS**
The label of the generated `directory_path` targets named `name + "_" + i`

14
lib/BUILD.bazel
View File

@ -56,6 +56,20 @@ bzl_library(
deps = ["//lib/private:jq"],
)
bzl_library(
name = "directory_path",
srcs = ["directory_path.bzl"],
visibility = ["//visibility:public"],
deps = ["//lib/private:directory_path"],
)
bzl_library(
name = "copy_to_directory",
srcs = ["copy_to_directory.bzl"],
visibility = ["//visibility:public"],
deps = ["//lib/private:copy_to_directory"],
)
toolchain_type(
name = "jq_toolchain_type",
visibility = ["//visibility:public"],

87
lib/copy_to_directory.bzl
View File

@ -1,4 +1,4 @@
"Public API for copy_to_directory"
"Copy files and directories to an output directory"
load(
"//lib/private:copy_to_directory.bzl",
@ -11,12 +11,95 @@ _copy_to_directory = rule(
attrs = lib.attrs,
)
def copy_to_directory(name, root_paths = None, **kwargs):
def copy_to_directory(
name,
srcs = [],
root_paths = None,
include_external_repositories = [],
exclude_prefixes = [],
replace_prefixes = {},
**kwargs):
"""Copies files and directories to an output directory.
Files and directories can be arranged as needed in the output directory using
the `root_paths`, `exclude_prefixes` and `replace_prefixes` attributes.
Args:
name: A unique name for this target.
srcs: Files and/or directories or targets that provide DirectoryPathInfo to copy into the output directory.
root_paths: List of paths that are roots in the output directory.
If a file or directory being copied is in one of the listed paths or one of its subpaths,
the output directory path is the path relative to the root path instead of the path
relative to the file's workspace.
Forward slashes (`/`) should be used as path separators. Partial matches
on the final path segment of a root path against the corresponding segment
in the full workspace relative path of a file are not matched.
If there are multiple root paths that match, the longest match wins.
Defaults to [package_name()] so that the output directory path of files in the
target's package and and sub-packages are relative to the target's package and
files outside of that retain their full workspace relative paths.
include_external_repositories: List of external repository names to include in the output directory.
Files from external repositories are not copied into the output directory unless
the external repository they come from is listed here.
When copied from an external repository, the file path in the output directory
defaults to the file's path within the external repository. The external repository
name is _not_ included in that path.
For example, the following copies `@external_repo//path/to:file` to
`path/to/file` within the output directory.
```
copy_to_directory(
name = "dir",
include_external_repositories = ["external_repo"],
srcs = ["@external_repo//path/to:file"],
)
```
Files from external repositories are subject to `root_paths`, `exclude_prefixes`
and `replace_prefixes` in the same way as files form the main repository.
exclude_prefixes: List of path prefixes to exclude from output directory.
If the output directory path for a file or directory starts with or is equal to
a path in the list then that file is not copied to the output directory.
Exclude prefixes are matched *before* replace_prefixes are applied.
replace_prefixes: Map of paths prefixes to replace in the output directory path when copying files.
If the output directory path for a file or directory starts with or is equal to
a key in the dict then the matching portion of the output directory path is
replaced with the dict value for that key.
Forward slashes (`/`) should be used as path separators. The final path segment
of the key can be a partial match in the corresponding segment of the output
directory path.
If there are multiple keys that match, the longest match wins.
**kwargs: Other common named parameters such as `tags` or `visibility`
"""
if root_paths == None:
root_paths = [native.package_name()]
_copy_to_directory(
name = name,
srcs = srcs,
root_paths = root_paths,
include_external_repositories = include_external_repositories,
exclude_prefixes = exclude_prefixes,
replace_prefixes = replace_prefixes,
is_windows = select({
"@bazel_tools//src/conditions:host_windows": True,
"//conditions:default": False,

30
lib/directory_path.bzl Normal file
View File

@ -0,0 +1,30 @@
# Copyright 2019 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Rule and corresponding provider that joins a label pointing to a TreeArtifact
with a path nested within that directory
"""
load(
"//lib/private:directory_path.bzl",
_DirectoryPathInfo = "DirectoryPathInfo",
_directory_path = "directory_path",
_make_directory_path = "make_directory_path",
_make_directory_paths = "make_directory_paths",
)
directory_path = _directory_path
make_directory_path = _make_directory_path
make_directory_paths = _make_directory_paths
DirectoryPathInfo = _DirectoryPathInfo

14
lib/private/BUILD.bazel
View File

@ -8,8 +8,9 @@ exports_files(
bzl_library(
name = "copy_to_directory",
srcs = ["copy_to_directory.bzl"],
visibility = ["//docs:__pkg__"],
visibility = ["//lib:__subpackages__"],
deps = [
":directory_path",
":paths",
"@bazel_skylib//lib:paths",
],
@ -62,9 +63,7 @@ bzl_library(
name = "write_source_files",
srcs = ["write_source_files.bzl"],
visibility = ["//lib:__subpackages__"],
deps = [
"//lib:utils",
],
deps = ["//lib:utils"],
)
bzl_library(
@ -72,3 +71,10 @@ bzl_library(
srcs = ["fail_with_message_test.bzl"],
visibility = ["//lib:__subpackages__"],
)
bzl_library(
name = "directory_path",
srcs = ["directory_path.bzl"],
visibility = ["//lib:__subpackages__"],
deps = ["//lib:utils"],
)

48
lib/private/copy_file.bzl
View File

@ -25,6 +25,8 @@ cmd.exe (on Windows). `_copy_xfile` marks the resulting file executable,
`_copy_file` does not.
"""
load(":directory_path.bzl", "DirectoryPathInfo")
# Hints for Bazel spawn strategy
_execution_requirements = {
# Copying files is entirely IO-bound and there is no point doing this work remotely.
@ -34,11 +36,8 @@ _execution_requirements = {
"no-remote-exec": "1",
}
def _hash_file(file):
return str(hash(file.path))
# buildifier: disable=function-docstring
def copy_cmd(ctx, src, dst):
def copy_cmd(ctx, src_file, src_path, dst):
# Most Windows binaries built with MSVC use a certain argument quoting
# scheme. Bazel uses that scheme too to quote arguments. However,
# cmd.exe uses different semantics, so Bazel's quoting is wrong here.
@ -46,7 +45,7 @@ def copy_cmd(ctx, src, dst):
# quoting or escaping is required.
# Put a hash of the file name into the name of the generated batch file to
# make it unique within the package, so that users can define multiple copy_file's.
bat = ctx.actions.declare_file("%s-%s-cmd.bat" % (ctx.label.name, _hash_file(src)))
bat = ctx.actions.declare_file("%s-%s-cmd.bat" % (ctx.label.name, hash(src_path)))
# Flags are documented at
# https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/copy
@ -55,24 +54,24 @@ def copy_cmd(ctx, src, dst):
if dst.is_directory:
cmd_tmpl = "@robocopy \"{src}\" \"{dst}\" /E >NUL & @exit 0"
mnemonic = "CopyDirectory"
progress_message = "Copying directory %s" % src.path
progress_message = "Copying directory %s" % src_path
else:
cmd_tmpl = "@copy /Y \"{src}\" \"{dst}\" >NUL"
mnemonic = "CopyFile"
progress_message = "Copying file %s" % src.path
progress_message = "Copying file %s" % src_path
ctx.actions.write(
output = bat,
# Do not use lib/shell.bzl's shell.quote() method, because that uses
# Bash quoting syntax, which is different from cmd.exe's syntax.
content = cmd_tmpl.format(
src = src.path.replace("/", "\\"),
src = src_path.replace("/", "\\"),
dst = dst.path.replace("/", "\\"),
),
is_executable = True,
)
ctx.actions.run(
inputs = [src],
inputs = [src_file],
tools = [bat],
outputs = [dst],
executable = "cmd.exe",
@ -84,21 +83,21 @@ def copy_cmd(ctx, src, dst):
)
# buildifier: disable=function-docstring
def copy_bash(ctx, src, dst):
def copy_bash(ctx, src_file, src_path, dst):
if dst.is_directory:
cmd_tmpl = "rm -rf \"$2\" && cp -rf \"$1/\" \"$2\""
mnemonic = "CopyDirectory"
progress_message = "Copying directory"
progress_message = "Copying directory %s" % src_path
else:
cmd_tmpl = "cp -f \"$1\" \"$2\""
mnemonic = "CopyFile"
progress_message = "Copying file"
progress_message = "Copying file %s" % src_path
ctx.actions.run_shell(
tools = [src],
tools = [src_file],
outputs = [dst],
command = cmd_tmpl,
arguments = [src.path, dst.path],
arguments = [src_path, dst.path],
mnemonic = mnemonic,
progress_message = progress_message,
use_default_shell_env = True,
@ -113,17 +112,28 @@ def _copy_file_impl(ctx):
else:
output = ctx.outputs.out
if ctx.attr.allow_symlink:
if len(ctx.files.src) != 1:
fail("src must be a single file when allow_symlink is True")
if output.is_directory:
fail("Cannot use both is_directory and allow_symlink")
ctx.actions.symlink(
output = output,
target_file = ctx.file.src,
target_file = ctx.files.src[0],
is_executable = ctx.attr.is_executable,
)
elif ctx.attr.is_windows:
copy_cmd(ctx, ctx.file.src, output)
else:
copy_bash(ctx, ctx.file.src, output)
if DirectoryPathInfo in ctx.attr.src:
src_file = ctx.attr.src[DirectoryPathInfo].directory
src_path = "/".join([src_file.path, ctx.attr.src[DirectoryPathInfo].path])
else:
if len(ctx.files.src) != 1:
fail("src must be a single file or a target with a DirectoryPathInfo provider")
src_file = ctx.files.src[0]
src_path = src_file.path
if ctx.attr.is_windows:
copy_cmd(ctx, src_file, src_path, output)
else:
copy_bash(ctx, src_file, src_path, output)
files = depset(direct = [output])
runfiles = ctx.runfiles(files = [output])
@ -133,7 +143,7 @@ def _copy_file_impl(ctx):
return [DefaultInfo(files = files, runfiles = runfiles)]
_ATTRS = {
"src": attr.label(mandatory = True, allow_single_file = True),
"src": attr.label(mandatory = True, allow_files = True),
"is_windows": attr.bool(mandatory = True),
"is_executable": attr.bool(mandatory = True),
"allow_symlink": attr.bool(mandatory = True),

191
lib/private/copy_to_directory.bzl
View File

@ -1,92 +1,15 @@
"Copy files and directories to an output directory"
"copy_to_directory implementation"
load("@bazel_skylib//lib:paths.bzl", skylib_paths = "paths")
load(":paths.bzl", "paths")
_DOC = """Copies files and directories to an output directory.
Files and directories can be arranged as needed in the output directory using
the `root_paths`, `exclude_prefixes` and `replace_prefixes` attributes.
"""
load(":directory_path.bzl", "DirectoryPathInfo")
_copy_to_directory_attr = {
"srcs": attr.label_list(
allow_files = True,
doc = """Files and/or directories to copy into the output directory""",
),
"root_paths": attr.string_list(
default = [],
doc = """
List of paths that are roots in the output directory. If a file or directory
being copied is in one of the listed paths or one of its subpaths, the output
directory path is the path relative to the root path instead of the path
relative to the file's workspace.
Forward slashes (`/`) should be used as path separators. Partial matches
on the final path segment of a root path against the corresponding segment
in the full workspace relative path of a file are not matched.
If there are multiple root paths that match, the longest match wins.
Defaults to [package_name()] so that the output directory path of files in the
target's package and and sub-packages are relative to the target's package and
files outside of that retain their full workspace relative paths.
""",
),
"include_external_repositories": attr.string_list(
default = [],
doc = """
List of external repository names to include in the output directory.
Files from external repositories are not copied into the output directory unless
the external repository they come from is listed here.
When copied from an external repository, the file path in the output directory
defaults to the file's path within the external repository. The external repository
name is _not_ included in that path.
For example, the following copies `@external_repo//path/to:file` to
`path/to/file` within the output directory.
```
copy_to_directory(
name = "dir",
include_external_repositories = ["external_repo"],
srcs = ["@external_repo//path/to:file"],
)
```
Files from external repositories are subject to `root_paths`, `exclude_prefixes`
and `replace_prefixes` in the same way as files form the main repository.
""",
),
"exclude_prefixes": attr.string_list(
default = [],
doc = """
List of path prefixes to exclude from output directory.
If the output directory path for a file or directory starts with or is equal to
a path in the list then that file is not copied to the output directory.
Exclude prefixes are matched *before* replace_prefixes are applied.
""",
),
"replace_prefixes": attr.string_dict(
default = {},
doc = """
Map of paths prefixes to replace in the output directory path when copying files.
If the output directory path for a file or directory starts with or is equal to
a key in the dict then the matching portion of the output directory path is
replaced with the dict value for that key.
Forward slashes (`/`) should be used as path separators. The final path segment
of the key can be a partial match in the corresponding segment of the output
directory path.
If there are multiple keys that match, the longest match wins.
""",
),
"srcs": attr.label_list(allow_files = True),
"root_paths": attr.string_list(default = []),
"include_external_repositories": attr.string_list(default = []),
"exclude_prefixes": attr.string_list(default = []),
"replace_prefixes": attr.string_dict(default = {}),
"is_windows": attr.bool(mandatory = True),
}
@ -112,43 +35,54 @@ def _longest_match(subject, tests, allow_partial = False):
high_score = score
return match
def _output_path(ctx, src):
# src can either be a File or a target with a DirectoryPathInfo
def _copy_paths(ctx, src):
if type(src) == "File":
src_file = src
src_path = src_file.path
output_path = paths.to_workspace_path(src_file)
elif DirectoryPathInfo in src:
src_file = src[DirectoryPathInfo].directory
src_path = "/".join([src_file.path, src[DirectoryPathInfo].path])
output_path = "/".join([paths.to_workspace_path(src_file), src[DirectoryPathInfo].path])
else:
fail("Unsupported type")
# if the file is from an external repository check if that repository should
# be included in the output directory
if src.owner and src.owner.workspace_name and not src.owner.workspace_name in ctx.attr.include_external_repositories:
return False
result = paths.to_workspace_path(src)
if src_file.owner and src_file.owner.workspace_name and not src_file.owner.workspace_name in ctx.attr.include_external_repositories:
return None, None, None
# strip root paths
root_path = _longest_match(result, ctx.attr.root_paths)
root_path = _longest_match(output_path, ctx.attr.root_paths)
if root_path:
strip_depth = len(root_path.split("/"))
result = "/".join(result.split("/")[strip_depth:])
output_path = "/".join(output_path.split("/")[strip_depth:])
# check if this file matches an exclude_prefix
match = _longest_match(result, ctx.attr.exclude_prefixes, True)
match = _longest_match(output_path, ctx.attr.exclude_prefixes, True)
if match:
# file is excluded due to match in exclude_prefix
return False
return None, None, None
# apply a replacement if one is found
match = _longest_match(result, ctx.attr.replace_prefixes.keys(), True)
match = _longest_match(output_path, ctx.attr.replace_prefixes.keys(), True)
if match:
result = ctx.attr.replace_prefixes[match] + result[len(match):]
return result
output_path = ctx.attr.replace_prefixes[match] + output_path[len(match):]
def _copy_to_dir_bash(ctx, srcs, dst_dir):
return src_path, output_path, src_file
def _copy_to_dir_bash(ctx, copy_paths, dst_dir):
cmds = [
"set -o errexit -o nounset -o pipefail",
"mkdir -p \"%s\"" % dst_dir.path,
]
for src in srcs:
output_path = _output_path(ctx, src)
if output_path == False:
# exclude this file/directory from the output directory
continue
dst_path = skylib_paths.normalize("/".join([dst_dir.path, output_path]))
inputs = []
for src_path, dst_path, src_file in copy_paths:
inputs.append(src_file)
cmds.append("""
if [[ ! -e "{src}" ]]; then echo "file '{src}' does not exist"; exit 1; fi
if [[ -f "{src}" ]]; then
@ -158,10 +92,10 @@ else
mkdir -p "{dst}"
cp -rf "{src}"/* "{dst}"
fi
""".format(src = src.path, dst_dir = skylib_paths.dirname(dst_path), dst = dst_path))
""".format(src = src_path, dst_dir = skylib_paths.dirname(dst_path), dst = dst_path))
ctx.actions.run_shell(
inputs = srcs,
inputs = inputs,
outputs = [dst_dir],
command = "\n".join(cmds),
mnemonic = "CopyToDirectory",
@ -170,7 +104,7 @@ fi
execution_requirements = _execution_requirements,
)
def _copy_to_dir_cmd(ctx, srcs, dst_dir):
def _copy_to_dir_cmd(ctx, copy_paths, dst_dir):
# Most Windows binaries built with MSVC use a certain argument quoting
# scheme. Bazel uses that scheme too to quote arguments. However,
# cmd.exe uses different semantics, so Bazel's quoting is wrong here.
@ -188,12 +122,11 @@ def _copy_to_dir_cmd(ctx, srcs, dst_dir):
@echo off
mkdir "%s" >NUL 2>NUL
""" % dst_dir.path.replace("/", "\\")]
for src in srcs:
output_path = _output_path(ctx, src)
if output_path == False:
# exclude this file/directory from the output directory
continue
dst_path = skylib_paths.normalize("/".join([dst_dir.path, output_path]))
inputs = []
for src_path, dst_path, src_file in copy_paths:
inputs.append(src_file)
# copy & xcopy flags are documented at
# https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/copy
@ -211,7 +144,7 @@ if exist "{src}\\*" (
copy /Y "{src}" "{dst}" >NUL
)
""".format(
src = src.path.replace("/", "\\"),
src = src_path.replace("/", "\\"),
dst_dir = skylib_paths.dirname(dst_path).replace("/", "\\"),
dst = dst_path.replace("/", "\\"),
))
@ -228,7 +161,7 @@ if exist "{src}\\*" (
)
ctx.actions.run(
inputs = srcs,
inputs = inputs,
tools = [bat],
outputs = [dst_dir],
executable = "cmd.exe",
@ -239,14 +172,30 @@ if exist "{src}\\*" (
)
def _copy_to_directory_impl(ctx):
if not ctx.files.srcs:
if not ctx.attr.srcs:
msg = "srcs must not be empty in copy_to_directory %s" % ctx.label
fail(msg)
output = ctx.actions.declare_directory(ctx.attr.name)
# Gather a list of src_path, dst_path pairs
copy_paths = []
for src in ctx.attr.srcs:
if DirectoryPathInfo in src:
src_path, output_path, src_file = _copy_paths(ctx, src)
if src_path != None:
dst_path = skylib_paths.normalize("/".join([output.path, output_path]))
copy_paths.append((src_path, dst_path, src_file))
for src_file in ctx.files.srcs:
src_path, output_path, src_file = _copy_paths(ctx, src_file)
if src_path != None:
dst_path = skylib_paths.normalize("/".join([output.path, output_path]))
copy_paths.append((src_path, dst_path, src_file))
if ctx.attr.is_windows:
_copy_to_dir_cmd(ctx, ctx.files.srcs, output)
_copy_to_dir_cmd(ctx, copy_paths, output)
else:
_copy_to_dir_bash(ctx, ctx.files.srcs, output)
_copy_to_dir_bash(ctx, copy_paths, output)
return [
DefaultInfo(
files = depset([output]),
@ -259,11 +208,3 @@ copy_to_directory_lib = struct(
impl = _copy_to_directory_impl,
provides = [DefaultInfo],
)
# For stardoc to generate documentation for the rule rather than a wrapper macro
copy_to_directory = rule(
doc = _DOC,
implementation = copy_to_directory_lib.impl,
attrs = copy_to_directory_lib.attrs,
provides = copy_to_directory_lib.provides,
)

127
lib/private/directory_path.bzl Normal file
View File

@ -0,0 +1,127 @@
"""Rule and corresponding provider that joins a label pointing to a TreeArtifact
with a path nested within that directory
"""
load("//lib:utils.bzl", _to_label = "to_label")
DirectoryPathInfo = provider(
doc = "Joins a label pointing to a TreeArtifact with a path nested within that directory.",
fields = {
"directory": "a TreeArtifact (ctx.actions.declare_directory)",
"path": "path relative to the directory",
},
)
def _directory_path(ctx):
if not ctx.file.directory.is_directory:
fail("directory attribute must be created with Bazel declare_directory (TreeArtifact)")
return [DirectoryPathInfo(path = ctx.attr.path, directory = ctx.file.directory)]
directory_path = rule(
doc = """Provide DirectoryPathInfo to reference some path within a directory.
Otherwise there is no way to give a Bazel label for it.""",
implementation = _directory_path,
attrs = {
"directory": attr.label(
doc = "a TreeArtifact (ctx.actions.declare_directory)",
mandatory = True,
allow_single_file = True,
),
"path": attr.string(
doc = "path relative to the directory",
mandatory = True,
),
},
)
def make_directory_path(name, directory, path):
"""Helper function to convert generate a directory_path target and return its label.
Args:
name: Unique name for the generated `directory_path` target.
directory: a TreeArtifact (ctx.actions.declare_directory)
path: path relative to the directory
Returns:
The label `name`
"""
directory_path(
name = name,
directory = directory,
path = path,
)
return _to_label(name)
def make_directory_paths(name, dict):
"""Helper function to convert a dict of directory to path mappings to directory_path targets and labels.
For example,
```
make_directory_paths("my_name", {
"//directory/artifact:target_1": "file/path",
"//directory/artifact:target_2": ["file/path1", "file/path2"],
})
```
generates the targets,
```
directory_path(
name = "my_name_0",
directory = "//directory/artifact:target_1",
path = "file/path"
)
directory_path(
name = "my_name_1",
directory = "//directory/artifact:target_2",
path = "file/path1"
)
directory_path(
name = "my_name_2",
directory = "//directory/artifact:target_2",
path = "file/path2"
)
```
and the list of targets is returned,
```
[
"my_name_0",
"my_name_1",
"my_name_2",
]
```
Args:
name: The target name to use for the generated targets & labels.
The names are generated as zero-indexed `name + "_" + i`
dict: The dictionary of directory keys to path or path list values.
Returns:
The label of the generated `directory_path` targets named `name + "_" + i`
"""
labels = []
pairs = []
for directory, val in dict.items():
if type(val) == "list":
for path in val:
pairs.append((directory, path))
elif type(val) == "string":
pairs.append((directory, val))
else:
fail("Value must be a list or string")
for i, pair in enumerate(pairs):
directory, path = pair
labels.append(make_directory_path(
"%s_%d" % (name, i),
directory,
path,
))
return labels

38
lib/tests/copy_file/BUILD.bazel Normal file
View File

@ -0,0 +1,38 @@
"tests for copy_files"
load("//lib:diff_test.bzl", "diff_test")
load("//lib:copy_file.bzl", "copy_file")
load("//lib:copy_to_directory.bzl", "copy_to_directory")
load("//lib:directory_path.bzl", "make_directory_path")
genrule(
name = "a_orig",
outs = ["a"],
cmd = "echo 'foo' > $@",
)
genrule(
name = "a2_orig",
outs = ["a2"],
cmd = "echo 'bar' > $@",
)
copy_to_directory(
name = "a_dir",
srcs = [
":a2_orig",
":a_orig",
],
)
copy_file(
name = "a_copy_from_dir",
src = make_directory_path("a_dir_a", ":a_dir", "a"),
out = "a_copy",
)
diff_test(
name = "copy_from_dir_test",
file1 = ":a",
file2 = ":a_copy_from_dir",
)

53
lib/tests/copy_to_directory/BUILD.bazel
View File

@ -3,6 +3,7 @@
load("//lib:diff_test.bzl", "diff_test")
load("//lib:copy_file.bzl", "copy_file")
load("//lib:copy_to_directory.bzl", "copy_to_directory")
load("//lib:directory_path.bzl", "make_directory_paths")
[
copy_file(
@ -21,6 +22,7 @@ load("//lib:copy_to_directory.bzl", "copy_to_directory")
"expected_5",
"expected_7",
"expected_8",
"expected_9",
]
]
@ -175,6 +177,57 @@ copy_to_directory(
],
)
# Case 9: directory_path srcs & replace_prefixes (with external repo)
copy_to_directory(
name = "case_9",
srcs = [
":c",
":d",
":e/e",
":e/e2",
"//lib/tests/copy_to_directory/f/f2",
"//lib/tests/copy_to_directory/f/f2:f",
"@external_test_repo//:test_c",
"@external_test_repo//:test_d",
] + make_directory_paths(
"case_9_srcs",
{
":a": "a",
":b": [
"b",
"b2",
],
"@external_test_repo//:test_a": "test_a",
"@external_test_repo//:test_b": "test_b",
},
),
include_external_repositories = ["external_test_repo"],
replace_prefixes = {
# merge a, b, c, d into a new/abcd dest folder
"a/a": "new/abcd/a",
"b": "//new///abcd////",
"c": "new/abcd/c",
"d": "////new/////abcd////d",
# put e into new/e except for e/e2 which goes into new/e2
"e": "new/e",
"e/e2": "new/e2/e2",
# f/f2/f => new/ff and f/f2/f2 => new/f2/f2
"f/f2/": "////new/////f",
"f/f2/f2": "////new/////f2///f2",
# flatten test_a & test_b to the root
"test_a": "",
"test_b": "",
# some paths that won't match
"a/a2": "wont_match_since_a2_is_not_included_in_the_directory_path_in_srcs",
},
)
diff_test(
name = "case_9_test",
file1 = "case_9",
file2 = ":expected_9",
)
diff_test(
name = "case_8_test",
file1 = "case_8",

1
lib/tests/copy_to_directory/dir_expected_9/new/abcd/a Normal file
View File

@ -0,0 +1 @@
foobar

1
lib/tests/copy_to_directory/dir_expected_9/new/abcd/b Normal file
View File

@ -0,0 +1 @@
foobar

1
lib/tests/copy_to_directory/dir_expected_9/new/abcd/b2 Normal file
View File

@ -0,0 +1 @@
foobar

1
lib/tests/copy_to_directory/dir_expected_9/new/abcd/c Normal file
View File

@ -0,0 +1 @@
foobar

1
lib/tests/copy_to_directory/dir_expected_9/new/abcd/d Normal file
View File

@ -0,0 +1 @@
foobar

1
lib/tests/copy_to_directory/dir_expected_9/new/e/e Normal file
View File

@ -0,0 +1 @@
foobar

1
lib/tests/copy_to_directory/dir_expected_9/new/e2/e2 Normal file
View File

@ -0,0 +1 @@
foobar

1
lib/tests/copy_to_directory/dir_expected_9/new/f2/f2 Normal file
View File

@ -0,0 +1 @@
foobar

1
lib/tests/copy_to_directory/dir_expected_9/new/ff Normal file
View File

@ -0,0 +1 @@
foobar

1
lib/tests/copy_to_directory/dir_expected_9/test_a Normal file
View File

@ -0,0 +1 @@
foobar

1
lib/tests/copy_to_directory/dir_expected_9/test_b Normal file
View File

@ -0,0 +1 @@
foobar

1
lib/tests/copy_to_directory/dir_expected_9/test_c Normal file
View File

@ -0,0 +1 @@
foobar

1
lib/tests/copy_to_directory/dir_expected_9/test_d Normal file
View File

@ -0,0 +1 @@
foobar