feat: improve performance of copy_to_directory by moving glob match & copying to a golang binary (#308)
This commit is contained in:
Greg Magolan
GitHub
parent
8334e127a7
commit
ec13ec9a69
11
BUILD.bazel
11
BUILD.bazel
|
|
@ -22,6 +22,17 @@ gazelle(
|
|||
gazelle = "gazelle_bin",
|
||||
)
|
||||
|
||||
gazelle(
|
||||
name = "gazelle_update_repos",
|
||||
args = [
|
||||
"-build_file_proto_mode=disable_global",
|
||||
"-from_file=go.mod",
|
||||
"-to_macro=deps.bzl%go_dependencies",
|
||||
"-prune",
|
||||
],
|
||||
command = "update-repos",
|
||||
)
|
||||
|
||||
bzl_library(
|
||||
name = "internal_deps",
|
||||
srcs = ["internal_deps.bzl"],
|
||||
|
|
|
|||
|
|
@ -29,11 +29,17 @@ load("//lib:host_repo.bzl", "host_repo")
|
|||
|
||||
host_repo(name = "aspect_bazel_lib_host")
|
||||
|
||||
load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")
|
||||
load("//:deps.bzl", "go_dependencies")
|
||||
|
||||
# gazelle:repository_macro deps.bzl%go_dependencies
|
||||
# gazelle:repository go_repository name=org_golang_x_tools importpath=golang.org/x/tools
|
||||
# https://github.com/bazelbuild/bazel-gazelle/issues/1217#issuecomment-1152236735
|
||||
go_dependencies()
|
||||
|
||||
############################################
|
||||
# Gazelle, for generating bzl_library targets
|
||||
load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies")
|
||||
load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")
|
||||
|
||||
go_rules_dependencies()
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,43 @@
|
|||
"""This module contains the project repository dependencies.
|
||||
"""
|
||||
|
||||
load("@bazel_gazelle//:deps.bzl", "go_repository")
|
||||
|
||||
def go_dependencies():
|
||||
"""The Go dependencies.
|
||||
"""
|
||||
go_repository(
|
||||
name = "com_github_gobwas_glob",
|
||||
build_file_proto_mode = "disable_global",
|
||||
importpath = "github.com/gobwas/glob",
|
||||
sum = "h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=",
|
||||
version = "v0.2.3",
|
||||
)
|
||||
go_repository(
|
||||
name = "com_github_google_go_cmp",
|
||||
build_file_proto_mode = "disable_global",
|
||||
importpath = "github.com/google/go-cmp",
|
||||
sum = "h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=",
|
||||
version = "v0.5.8",
|
||||
)
|
||||
go_repository(
|
||||
name = "org_golang_x_exp",
|
||||
build_file_proto_mode = "disable_global",
|
||||
importpath = "golang.org/x/exp",
|
||||
sum = "h1:m9O6OTJ627iFnN2JIWfdqlZCzneRO6EEBsHXI25P8ws=",
|
||||
version = "v0.0.0-20221230185412-738e83a70c30",
|
||||
)
|
||||
go_repository(
|
||||
name = "org_golang_x_mod",
|
||||
build_file_proto_mode = "disable_global",
|
||||
importpath = "golang.org/x/mod",
|
||||
sum = "h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I=",
|
||||
version = "v0.6.0",
|
||||
)
|
||||
go_repository(
|
||||
name = "org_golang_x_sys",
|
||||
build_file_proto_mode = "disable_global",
|
||||
importpath = "golang.org/x/sys",
|
||||
sum = "h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=",
|
||||
version = "v0.1.0",
|
||||
)
|
||||
|
|
@ -69,7 +69,7 @@ other rule implementations where additional_files can also be passed in.
|
|||
| <a id="copy_to_directory_action-exclude_srcs_packages"></a>exclude_srcs_packages | List of Bazel packages (with glob support) to exclude from output directory.<br><br>See copy_to_directory rule documentation for more details. | <code>[]</code> |
|
||||
| <a id="copy_to_directory_action-include_srcs_patterns"></a>include_srcs_patterns | List of paths (with glob support) to include in output directory.<br><br>See copy_to_directory rule documentation for more details. | <code>["**"]</code> |
|
||||
| <a id="copy_to_directory_action-exclude_srcs_patterns"></a>exclude_srcs_patterns | List of paths (with glob support) to exclude from output directory.<br><br>See copy_to_directory rule documentation for more details. | <code>[]</code> |
|
||||
| <a id="copy_to_directory_action-exclude_prefixes"></a>exclude_prefixes | List of path prefixes to exclude from output directory. | <code>[]</code> |
|
||||
| <a id="copy_to_directory_action-exclude_prefixes"></a>exclude_prefixes | List of path prefixes to exclude from output directory.<br><br>See copy_to_directory rule documentation for more details. | <code>[]</code> |
|
||||
| <a id="copy_to_directory_action-replace_prefixes"></a>replace_prefixes | Map of paths prefixes to replace in the output directory path when copying files.<br><br>See copy_to_directory rule documentation for more details. | <code>{}</code> |
|
||||
| <a id="copy_to_directory_action-allow_overwrites"></a>allow_overwrites | If True, allow files to be overwritten if the same output file is copied to twice.<br><br>See copy_to_directory rule documentation for more details. | <code>False</code> |
|
||||
| <a id="copy_to_directory_action-is_windows"></a>is_windows | Deprecated and unused | <code>None</code> |
|
||||
|
|
|
|||
|
|
@ -0,0 +1,8 @@
|
|||
module github.com/aspect-build/bazel-lib
|
||||
|
||||
go 1.19
|
||||
|
||||
require (
|
||||
github.com/gobwas/glob v0.2.3
|
||||
golang.org/x/exp v0.0.0-20221230185412-738e83a70c30
|
||||
)
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
|
||||
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
|
||||
golang.org/x/exp v0.0.0-20221230185412-738e83a70c30 h1:m9O6OTJ627iFnN2JIWfdqlZCzneRO6EEBsHXI25P8ws=
|
||||
golang.org/x/exp v0.0.0-20221230185412-738e83a70c30/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
|
||||
|
|
@ -7,12 +7,16 @@ the `root_paths`, `include_srcs_patters`, `exclude_srcs_patters` and `replace_pr
|
|||
load(
|
||||
"//lib/private:copy_to_directory.bzl",
|
||||
_copy_to_directory_action = "copy_to_directory_action",
|
||||
# TODO: export copy_to_directory_bin_action once ready
|
||||
# _copy_to_directory_bin_action = "copy_to_directory_bin_action",
|
||||
_copy_to_directory_lib = "copy_to_directory_lib",
|
||||
)
|
||||
|
||||
# export the starlark library as a public API
|
||||
copy_to_directory_lib = _copy_to_directory_lib
|
||||
copy_to_directory_action = _copy_to_directory_action
|
||||
# TODO: export copy_to_directory_bin_action once ready
|
||||
# copy_to_directory_bin_action = _copy_to_directory_bin_action
|
||||
|
||||
copy_to_directory = rule(
|
||||
implementation = _copy_to_directory_lib.impl,
|
||||
|
|
|
|||
|
|
@ -75,7 +75,7 @@ def copy_directory_action(ctx, src, dst, is_windows = None):
|
|||
is_windows: Deprecated and unused
|
||||
"""
|
||||
|
||||
# TODO(2.0): remove depcreated & unused is_windows parameter
|
||||
# TODO(2.0): remove deprecated & unused is_windows parameter
|
||||
if not src.is_source and not dst.is_directory:
|
||||
fail("src must be a source directory or TreeArtifact")
|
||||
if dst.is_source or not dst.is_directory:
|
||||
|
|
|
|||
|
|
@ -101,7 +101,7 @@ def copy_file_action(ctx, src, dst, dir_path = None, is_windows = None):
|
|||
is_windows: Deprecated and unused
|
||||
"""
|
||||
|
||||
# TODO(2.0): remove depcreated & unused is_windows parameter
|
||||
# TODO(2.0): remove deprecated & unused is_windows parameter
|
||||
if dst.is_directory:
|
||||
fail("dst must not be a TreeArtifact")
|
||||
if src.is_directory:
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ def copy_file_to_bin_action(ctx, file, is_windows = None):
|
|||
A File in the output tree.
|
||||
"""
|
||||
|
||||
# TODO(2.0): remove depcreated & unused is_windows parameter
|
||||
# TODO(2.0): remove deprecated & unused is_windows parameter
|
||||
if not file.is_source:
|
||||
return file
|
||||
if ctx.label.workspace_name != file.owner.workspace_name:
|
||||
|
|
@ -109,7 +109,7 @@ def copy_files_to_bin_actions(ctx, files, is_windows = None):
|
|||
List of File objects in the output tree.
|
||||
"""
|
||||
|
||||
# TODO(2.0): remove depcreated & unused is_windows parameter
|
||||
# TODO(2.0): remove deprecated & unused is_windows parameter
|
||||
return [copy_file_to_bin_action(ctx, file, is_windows = is_windows) for file in files]
|
||||
|
||||
def _copy_to_bin_impl(ctx):
|
||||
|
|
|
|||
|
|
@ -280,6 +280,15 @@ _copy_to_directory_attr = {
|
|||
|
||||
This setting has no effect on Windows where overwrites are always allowed.""",
|
||||
),
|
||||
# TODO: flip to copy_to_directory_bin_action once ready
|
||||
# "verbose": attr.bool(
|
||||
# doc = """If true, prints out verbose logs to stdout""",
|
||||
# ),
|
||||
# "_tool": attr.label(
|
||||
# executable = True,
|
||||
# cfg = "exec",
|
||||
# default = "//tools/copy_to_directory",
|
||||
# ),
|
||||
}
|
||||
|
||||
def _any_globs_match(exprs, path):
|
||||
|
|
@ -383,7 +392,7 @@ def _copy_paths(
|
|||
# file is excluded as its external repository does not match any patterns in include_external_repositories
|
||||
return None, None, None
|
||||
|
||||
# apply include_srcs_packages if "**" is not included in the list
|
||||
# apply include_srcs_packages
|
||||
if not _any_globs_match(include_srcs_packages, src_file.owner.package):
|
||||
# file is excluded as it does not match any specified include_srcs_packages
|
||||
return None, None, None
|
||||
|
|
@ -605,6 +614,26 @@ def _copy_to_directory_impl(ctx):
|
|||
allow_overwrites = ctx.attr.allow_overwrites,
|
||||
)
|
||||
|
||||
# TODO: flip to copy_to_directory_bin_action once ready
|
||||
# copy_to_directory_bin_action(
|
||||
# ctx,
|
||||
# name = ctx.attr.name,
|
||||
# dst = dst,
|
||||
# copy_to_directory_bin = ctx.executable._tool,
|
||||
# files = ctx.files.srcs,
|
||||
# targets = [t for t in ctx.attr.srcs if DirectoryPathInfo in t],
|
||||
# root_paths = ctx.attr.root_paths,
|
||||
# include_external_repositories = ctx.attr.include_external_repositories,
|
||||
# include_srcs_packages = ctx.attr.include_srcs_packages,
|
||||
# exclude_srcs_packages = ctx.attr.exclude_srcs_packages,
|
||||
# include_srcs_patterns = ctx.attr.include_srcs_patterns,
|
||||
# exclude_srcs_patterns = ctx.attr.exclude_srcs_patterns,
|
||||
# exclude_prefixes = ctx.attr.exclude_prefixes,
|
||||
# replace_prefixes = ctx.attr.replace_prefixes,
|
||||
# allow_overwrites = ctx.attr.allow_overwrites,
|
||||
# verbose = ctx.attr.verbose,
|
||||
# )
|
||||
|
||||
return [
|
||||
DefaultInfo(
|
||||
files = depset([dst]),
|
||||
|
|
@ -628,6 +657,201 @@ def _expand_src_packages_patterns(patterns, package):
|
|||
result.append(pattern)
|
||||
return result
|
||||
|
||||
def copy_to_directory_bin_action(
|
||||
ctx,
|
||||
name,
|
||||
dst,
|
||||
copy_to_directory_bin,
|
||||
files = [],
|
||||
targets = [],
|
||||
root_paths = ["."],
|
||||
include_external_repositories = [],
|
||||
include_srcs_packages = ["**"],
|
||||
exclude_srcs_packages = [],
|
||||
include_srcs_patterns = ["**"],
|
||||
exclude_srcs_patterns = [],
|
||||
exclude_prefixes = [],
|
||||
replace_prefixes = {},
|
||||
allow_overwrites = False,
|
||||
verbose = False):
|
||||
"""Helper function to copy files to a directory using a tool binary.
|
||||
|
||||
The tool binary will typically be the `@aspect_bazel_lib//tools/copy_to_directory` `go_binary`
|
||||
either built from source or provided by a toolchain.
|
||||
|
||||
This helper is used by copy_to_directory. It is exposed as a public API so it can be used within
|
||||
other rule implementations where additional_files can also be passed in.
|
||||
|
||||
Args:
|
||||
ctx: The rule context.
|
||||
|
||||
name: Name of target creating this action used for config file generation.
|
||||
|
||||
dst: The directory to copy to. Must be a TreeArtifact.
|
||||
|
||||
copy_to_directory_bin: Copy to directory tool binary.
|
||||
|
||||
files: List of files to copy into the output directory.
|
||||
|
||||
targets: List of targets that provide DirectoryPathInfo to copy into the output directory.
|
||||
|
||||
root_paths: List of paths that are roots in the output directory.
|
||||
|
||||
See copy_to_directory rule documentation for more details.
|
||||
|
||||
include_external_repositories: List of external repository names to include in the output directory.
|
||||
|
||||
See copy_to_directory rule documentation for more details.
|
||||
|
||||
include_srcs_packages: List of Bazel packages to include in output directory.
|
||||
|
||||
See copy_to_directory rule documentation for more details.
|
||||
|
||||
exclude_srcs_packages: List of Bazel packages (with glob support) to exclude from output directory.
|
||||
|
||||
See copy_to_directory rule documentation for more details.
|
||||
|
||||
include_srcs_patterns: List of paths (with glob support) to include in output directory.
|
||||
|
||||
See copy_to_directory rule documentation for more details.
|
||||
|
||||
exclude_srcs_patterns: List of paths (with glob support) to exclude from output directory.
|
||||
|
||||
See copy_to_directory rule documentation for more details.
|
||||
|
||||
exclude_prefixes: List of path prefixes to exclude from output directory.
|
||||
|
||||
See copy_to_directory rule documentation for more details.
|
||||
|
||||
replace_prefixes: Map of paths prefixes to replace in the output directory path when copying files.
|
||||
|
||||
See copy_to_directory rule documentation for more details.
|
||||
|
||||
allow_overwrites: If True, allow files to be overwritten if the same output file is copied to twice.
|
||||
|
||||
See copy_to_directory rule documentation for more details.
|
||||
|
||||
verbose: If true, prints out verbose logs to stdout
|
||||
"""
|
||||
|
||||
# Replace "." in root_paths with the package name of the target
|
||||
root_paths = [p if p != "." else ctx.label.package for p in root_paths]
|
||||
|
||||
# Replace "." and "./**" patterns in in include_srcs_packages & exclude_srcs_packages
|
||||
include_srcs_packages = _expand_src_packages_patterns(include_srcs_packages, ctx.label.package)
|
||||
exclude_srcs_packages = _expand_src_packages_patterns(exclude_srcs_packages, ctx.label.package)
|
||||
|
||||
# Convert and append exclude_prefixes to exclude_srcs_patterns
|
||||
# TODO(2.0): remove exclude_prefixes this block and in a future breaking release
|
||||
for exclude_prefix in exclude_prefixes:
|
||||
if exclude_prefix.endswith("**"):
|
||||
exclude_srcs_patterns.append(exclude_prefix)
|
||||
elif exclude_prefix.endswith("*"):
|
||||
exclude_srcs_patterns.append(exclude_prefix + "/**")
|
||||
exclude_srcs_patterns.append(exclude_prefix)
|
||||
elif exclude_prefix.endswith("/"):
|
||||
exclude_srcs_patterns.append(exclude_prefix + "**")
|
||||
else:
|
||||
exclude_srcs_patterns.append(exclude_prefix + "*/**")
|
||||
exclude_srcs_patterns.append(exclude_prefix + "*")
|
||||
|
||||
if not include_srcs_packages:
|
||||
fail("An empty 'include_srcs_packages' list will exclude all srcs and result in an empty directory")
|
||||
|
||||
if "**" in exclude_srcs_packages:
|
||||
fail("A '**' glob pattern in 'exclude_srcs_packages' will exclude all srcs and result in an empty directory")
|
||||
|
||||
if not include_srcs_patterns:
|
||||
fail("An empty 'include_srcs_patterns' list will exclude all srcs and result in an empty directory")
|
||||
|
||||
if "**" in exclude_srcs_patterns:
|
||||
fail("A '**' glob pattern in 'exclude_srcs_patterns' will exclude all srcs and result in an empty directory")
|
||||
|
||||
for replace_prefix in replace_prefixes.keys():
|
||||
if replace_prefix.endswith("**"):
|
||||
msg = "replace_prefix '{}' must not end with '**' glob expression".format(replace_prefix)
|
||||
fail(msg)
|
||||
|
||||
files_and_targets = []
|
||||
for f in files:
|
||||
files_and_targets.append(struct(
|
||||
file = f,
|
||||
path = f.path,
|
||||
root_path = f.root.path,
|
||||
short_path = f.short_path,
|
||||
workspace_path = paths.to_workspace_path(f),
|
||||
))
|
||||
for t in targets:
|
||||
if not DirectoryPathInfo in t:
|
||||
continue
|
||||
files_and_targets.append(struct(
|
||||
file = t[DirectoryPathInfo].directory,
|
||||
path = "/".join([t[DirectoryPathInfo].directory.path, t[DirectoryPathInfo].path]),
|
||||
root_path = t[DirectoryPathInfo].directory.root.path,
|
||||
short_path = "/".join([t[DirectoryPathInfo].directory.short_path, t[DirectoryPathInfo].path]),
|
||||
workspace_path = "/".join([paths.to_workspace_path(t[DirectoryPathInfo].directory), t[DirectoryPathInfo].path]),
|
||||
))
|
||||
|
||||
file_infos = []
|
||||
file_inputs = []
|
||||
for f in files_and_targets:
|
||||
if not f.file.owner:
|
||||
msg = "Expected an owner target label for file {} but found none".format(f)
|
||||
fail(msg)
|
||||
|
||||
if f.file.owner.package == None:
|
||||
msg = "Expected owner target label for file {} to have a package name but found None".format(f)
|
||||
fail(msg)
|
||||
|
||||
if f.file.owner.workspace_name == None:
|
||||
msg = "Expected owner target label for file {} to have a workspace name but found None".format(f)
|
||||
fail(msg)
|
||||
|
||||
file_infos.append({
|
||||
# the path might be a file if it came from a DirectoryPathInfo or it is a source directory
|
||||
"maybe_directory": f.file.is_directory or f.file.is_source,
|
||||
"package": f.file.owner.package,
|
||||
"path": f.path,
|
||||
"root_path": f.root_path,
|
||||
"short_path": f.short_path,
|
||||
"workspace": f.file.owner.workspace_name,
|
||||
"workspace_path": f.workspace_path,
|
||||
})
|
||||
file_inputs.append(f.file)
|
||||
|
||||
if not file_inputs:
|
||||
fail("No files to copy")
|
||||
|
||||
config = {
|
||||
"allow_overwrites": allow_overwrites,
|
||||
"dst": dst.path,
|
||||
"exclude_srcs_packages": exclude_srcs_packages,
|
||||
"exclude_srcs_patterns": exclude_srcs_patterns,
|
||||
"files": file_infos,
|
||||
"include_external_repositories": include_external_repositories,
|
||||
"include_srcs_packages": include_srcs_packages,
|
||||
"include_srcs_patterns": include_srcs_patterns,
|
||||
"replace_prefixes": replace_prefixes,
|
||||
"root_paths": root_paths,
|
||||
"verbose": verbose,
|
||||
}
|
||||
|
||||
config_file = ctx.actions.declare_file("{}_config.json".format(name))
|
||||
ctx.actions.write(
|
||||
output = config_file,
|
||||
content = json.encode_indent(config),
|
||||
)
|
||||
|
||||
ctx.actions.run(
|
||||
inputs = file_inputs + [config_file],
|
||||
outputs = [dst],
|
||||
executable = copy_to_directory_bin,
|
||||
arguments = [config_file.path],
|
||||
mnemonic = "CopyToDirectory",
|
||||
progress_message = "Copying files to directory",
|
||||
execution_requirements = _COPY_EXECUTION_REQUIREMENTS,
|
||||
)
|
||||
|
||||
def copy_to_directory_action(
|
||||
ctx,
|
||||
srcs,
|
||||
|
|
@ -683,6 +907,8 @@ def copy_to_directory_action(
|
|||
|
||||
exclude_prefixes: List of path prefixes to exclude from output directory.
|
||||
|
||||
See copy_to_directory rule documentation for more details.
|
||||
|
||||
replace_prefixes: Map of paths prefixes to replace in the output directory path when copying files.
|
||||
|
||||
See copy_to_directory rule documentation for more details.
|
||||
|
|
@ -694,7 +920,7 @@ def copy_to_directory_action(
|
|||
is_windows: Deprecated and unused
|
||||
"""
|
||||
|
||||
# TODO(2.0): remove depcreated & unused is_windows parameter
|
||||
# TODO(2.0): remove deprecated & unused is_windows parameter
|
||||
if not srcs:
|
||||
fail("srcs must not be empty")
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
1
|
||||
|
|
@ -0,0 +1 @@
|
|||
1
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
|
||||
load("//lib:diff_test.bzl", "diff_test")
|
||||
load("//lib:copy_to_directory.bzl", "copy_to_directory")
|
||||
load("//lib:copy_to_bin.bzl", "copy_to_bin")
|
||||
load(":lib.bzl", "lib")
|
||||
load(":pkg.bzl", "pkg")
|
||||
|
||||
copy_to_bin(
|
||||
name = "copy_1",
|
||||
srcs = ["1"],
|
||||
)
|
||||
|
||||
lib(
|
||||
name = "lib",
|
||||
srcs = ["1"],
|
||||
# intentionally dup on "1" to make sure it is gracefully handled
|
||||
others = [
|
||||
"1",
|
||||
# also pass in a copy_to_bin copy of "1" to spice things up;
|
||||
# this case is handled in the fix in https://github.com/aspect-build/bazel-lib/pull/205
|
||||
"copy_1",
|
||||
"2",
|
||||
"d",
|
||||
],
|
||||
)
|
||||
|
||||
# pkg should copy DefaultInfo files and OtherInfo files
|
||||
pkg(
|
||||
name = "pkg",
|
||||
srcs = [":lib"],
|
||||
out = "pkg",
|
||||
)
|
||||
|
||||
copy_to_directory(
|
||||
name = "expected_pkg",
|
||||
srcs = [
|
||||
"1",
|
||||
"2",
|
||||
"d",
|
||||
],
|
||||
)
|
||||
|
||||
diff_test(
|
||||
name = "test",
|
||||
file1 = ":pkg",
|
||||
file2 = ":expected_pkg",
|
||||
)
|
||||
|
||||
bzl_library(
|
||||
name = "other_info",
|
||||
srcs = ["other_info.bzl"],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
|
@ -0,0 +1 @@
|
|||
1
|
||||
|
|
@ -0,0 +1 @@
|
|||
1
|
||||
|
|
@ -0,0 +1 @@
|
|||
1
|
||||
|
|
@ -0,0 +1 @@
|
|||
1
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
"""
|
||||
Test rule to create a lib with a DefaultInfo and a OtherInfo
|
||||
"""
|
||||
|
||||
load(":other_info.bzl", "OtherInfo")
|
||||
|
||||
_attrs = {
|
||||
"srcs": attr.label_list(allow_files = True),
|
||||
"others": attr.label_list(allow_files = True),
|
||||
}
|
||||
|
||||
def _lib_impl(ctx):
|
||||
return [
|
||||
DefaultInfo(files = depset(ctx.files.srcs)),
|
||||
OtherInfo(files = depset(ctx.files.others)),
|
||||
]
|
||||
|
||||
lib = rule(
|
||||
implementation = _lib_impl,
|
||||
attrs = _attrs,
|
||||
provides = [DefaultInfo, OtherInfo],
|
||||
)
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
"""For testing"""
|
||||
|
||||
OtherInfo = provider(
|
||||
doc = "For testing",
|
||||
fields = {
|
||||
"files": "A depset of files",
|
||||
},
|
||||
)
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
"""
|
||||
Test rule to create a pkg with DefaultInfo and OtherInfo files
|
||||
"""
|
||||
|
||||
load("@aspect_bazel_lib//lib/private:copy_to_directory.bzl", "copy_to_directory_bin_action")
|
||||
load(":other_info.bzl", "OtherInfo")
|
||||
|
||||
_attrs = {
|
||||
"srcs": attr.label_list(allow_files = True),
|
||||
"out": attr.string(mandatory = True),
|
||||
"_tool": attr.label(
|
||||
executable = True,
|
||||
cfg = "exec",
|
||||
default = "//tools/copy_to_directory",
|
||||
),
|
||||
}
|
||||
|
||||
def _pkg_impl(ctx):
|
||||
dst = ctx.actions.declare_directory(ctx.attr.out)
|
||||
|
||||
additional_files_depsets = []
|
||||
|
||||
# include files from OtherInfo of srcs
|
||||
for src in ctx.attr.srcs:
|
||||
if OtherInfo in src:
|
||||
additional_files_depsets.append(src[OtherInfo].files)
|
||||
|
||||
copy_to_directory_bin_action(
|
||||
ctx,
|
||||
name = ctx.attr.name,
|
||||
files = ctx.files.srcs + depset(transitive = additional_files_depsets).to_list(),
|
||||
dst = dst,
|
||||
copy_to_directory_bin = ctx.executable._tool,
|
||||
verbose = True,
|
||||
)
|
||||
|
||||
return [
|
||||
DefaultInfo(files = depset([dst])),
|
||||
]
|
||||
|
||||
pkg = rule(
|
||||
implementation = _pkg_impl,
|
||||
attrs = _attrs,
|
||||
provides = [DefaultInfo],
|
||||
)
|
||||
|
|
@ -5,6 +5,10 @@ go_library(
|
|||
srcs = ["main.go"],
|
||||
importpath = "github.com/aspect-build/bazel-lib/tools/copy_to_directory",
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"@com_github_gobwas_glob//:glob",
|
||||
"@org_golang_x_exp//maps",
|
||||
],
|
||||
)
|
||||
|
||||
go_binary(
|
||||
|
|
|
|||
|
|
@ -1,7 +1,347 @@
|
|||
package main
|
||||
|
||||
import "fmt"
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/gobwas/glob"
|
||||
"golang.org/x/exp/maps"
|
||||
)
|
||||
|
||||
type fileInfo struct {
|
||||
MaybeDirectory bool `json:"maybe_directory"`
|
||||
Package string `json:"package"`
|
||||
Path string `json:"path"`
|
||||
RootPath string `json:"root_path"`
|
||||
ShortPath string `json:"short_path"`
|
||||
Workspace string `json:"workspace"`
|
||||
WorkspacePath string `json:"workspace_path"`
|
||||
}
|
||||
|
||||
type config struct {
|
||||
AllowOverwrites bool `json:"allow_overwrites"`
|
||||
Dst string `json:"dst"`
|
||||
ExcludeSrcsPackages []string `json:"exclude_srcs_packages"`
|
||||
ExcludeSrcsPatterns []string `json:"exclude_srcs_patterns"`
|
||||
Files []fileInfo `json:"files"`
|
||||
IncludeExternalRepositories []string `json:"include_external_repositories"`
|
||||
IncludeSrcsPackages []string `json:"include_srcs_packages"`
|
||||
IncludeSrcsPatterns []string `json:"include_srcs_patterns"`
|
||||
ReplacePrefixes map[string]string `json:"replace_prefixes"`
|
||||
RootPaths []string `json:"root_paths"`
|
||||
Verbose bool `json:"verbose"`
|
||||
|
||||
ExcludeSrcsPackagesGlobs []glob.Glob
|
||||
ExcludeSrcsPatternsGlobs []glob.Glob
|
||||
IncludeExternalRepositoriesGlobs []glob.Glob
|
||||
IncludeSrcsPackagesGlobs []glob.Glob
|
||||
IncludeSrcsPatternsGlobs []glob.Glob
|
||||
ReplacePrefixesGlobs []glob.Glob
|
||||
ReplacePrefixesKeys []string
|
||||
RootPathsGlobs []glob.Glob
|
||||
}
|
||||
|
||||
type copyPaths map[string]fileInfo
|
||||
|
||||
func parseConfig(configPath string) (*config, error) {
|
||||
f, err := os.Open(configPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open config file: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
byteValue, err := ioutil.ReadAll(f)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read config file: %w", err)
|
||||
}
|
||||
|
||||
var cfg config
|
||||
if err := json.Unmarshal([]byte(byteValue), &cfg); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config file: %w", err)
|
||||
}
|
||||
|
||||
// compile all globs
|
||||
cfg.ExcludeSrcsPackagesGlobs, err = compileGlobs(cfg.ExcludeSrcsPackages)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cfg.ExcludeSrcsPatternsGlobs, err = compileGlobs(cfg.ExcludeSrcsPatterns)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cfg.IncludeExternalRepositoriesGlobs, err = compileGlobs(cfg.IncludeExternalRepositories)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cfg.IncludeSrcsPackagesGlobs, err = compileGlobs(cfg.IncludeSrcsPackages)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cfg.IncludeSrcsPatternsGlobs, err = compileGlobs(cfg.IncludeSrcsPatterns)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cfg.RootPathsGlobs, err = compileGlobs(cfg.RootPaths)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cfg.ReplacePrefixesKeys = maps.Keys(cfg.ReplacePrefixes)
|
||||
cfg.ReplacePrefixesGlobs, err = compileGlobs(cfg.ReplacePrefixesKeys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
func compileGlobs(patterns []string) ([]glob.Glob, error) {
|
||||
result := make([]glob.Glob, len(patterns))
|
||||
for i, pattern := range patterns {
|
||||
g, err := glob.Compile(pattern)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to compile glob pattern '%s': %w", pattern, err)
|
||||
}
|
||||
result[i] = g
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func anyGlobsMatch(globs []glob.Glob, test string) bool {
|
||||
for _, g := range globs {
|
||||
if g.Match(test) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func longestGlobsMatch(globs []glob.Glob, test string) (string, int) {
|
||||
result := ""
|
||||
index := 0
|
||||
for i, g := range globs {
|
||||
match := longestGlobMatch(g, test)
|
||||
if len(match) > len(result) {
|
||||
result = match
|
||||
index = i
|
||||
}
|
||||
}
|
||||
return result, index
|
||||
}
|
||||
|
||||
func longestGlobMatch(g glob.Glob, test string) string {
|
||||
for i := 0; i < len(test); i++ {
|
||||
t := test[:len(test)-i]
|
||||
if g.Match(t) {
|
||||
return t
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// From https://stackoverflow.com/a/49196644
|
||||
func filePathWalkDir(root string) ([]string, error) {
|
||||
var files []string
|
||||
err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
|
||||
if !info.IsDir() {
|
||||
files = append(files, path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return files, err
|
||||
}
|
||||
|
||||
func calcCopyPath(cfg *config, copyPaths copyPaths, file fileInfo) error {
|
||||
// Apply filters and transformations in the following order:
|
||||
//
|
||||
// - `include_external_repositories`
|
||||
// - `include_srcs_packages`
|
||||
// - `exclude_srcs_packages`
|
||||
// - `root_paths`
|
||||
// - `include_srcs_patterns`
|
||||
// - `exclude_srcs_patterns`
|
||||
// - `replace_prefixes`
|
||||
//
|
||||
// If you change this order please update the docstrings in the copy_to_directory rule.
|
||||
|
||||
outputPath := file.WorkspacePath
|
||||
outputRoot := path.Dir(outputPath)
|
||||
|
||||
// apply include_external_repositories (if the file is from an external repository)
|
||||
if file.Workspace != "" {
|
||||
if !anyGlobsMatch(cfg.IncludeExternalRepositoriesGlobs, file.Workspace) {
|
||||
return nil // external workspace is not included
|
||||
}
|
||||
}
|
||||
|
||||
// apply include_srcs_packages
|
||||
if !anyGlobsMatch(cfg.IncludeSrcsPackagesGlobs, file.Package) {
|
||||
return nil // package is not included
|
||||
}
|
||||
|
||||
// apply exclude_srcs_packages
|
||||
if anyGlobsMatch(cfg.ExcludeSrcsPackagesGlobs, file.Package) {
|
||||
return nil // package is excluded
|
||||
}
|
||||
|
||||
// apply root_paths
|
||||
rootPathMatch, _ := longestGlobsMatch(cfg.RootPathsGlobs, outputRoot)
|
||||
if rootPathMatch != "" {
|
||||
outputPath = outputPath[len(rootPathMatch):]
|
||||
if strings.HasPrefix(outputPath, "/") {
|
||||
outputPath = outputPath[1:]
|
||||
}
|
||||
}
|
||||
|
||||
// apply include_srcs_patterns
|
||||
if !anyGlobsMatch(cfg.IncludeSrcsPatternsGlobs, outputPath) {
|
||||
return nil // outputPath is not included
|
||||
}
|
||||
|
||||
// apply include_srcs_patterns
|
||||
if anyGlobsMatch(cfg.ExcludeSrcsPatternsGlobs, outputPath) {
|
||||
return nil // outputPath is excluded
|
||||
}
|
||||
|
||||
// apply replace_prefixes
|
||||
replacePrefixMatch, replacePrefixIndex := longestGlobsMatch(cfg.ReplacePrefixesGlobs, outputPath)
|
||||
if replacePrefixMatch != "" {
|
||||
replaceWith := cfg.ReplacePrefixes[cfg.ReplacePrefixesKeys[replacePrefixIndex]]
|
||||
outputPath = replaceWith + outputPath[len(replacePrefixMatch):]
|
||||
}
|
||||
|
||||
outputPath = path.Join(cfg.Dst, outputPath)
|
||||
|
||||
// add this file to the copy Paths
|
||||
dup, exists := copyPaths[outputPath]
|
||||
if exists {
|
||||
if dup.ShortPath == file.ShortPath {
|
||||
// this is likely the same file listed twice: the original in the source
|
||||
// tree and the copy in the output tree
|
||||
// TODO: stat the two files to double check that they are the same
|
||||
if file.RootPath == "" {
|
||||
// when this happens prefer the output tree copy.
|
||||
return nil
|
||||
}
|
||||
} else if !cfg.AllowOverwrites {
|
||||
return fmt.Errorf("duplicate output file '%s' configured from source files '%s' and '%s'; set 'allow_overwrites' to True to allow this overwrites but keep in mind that order matters when this is set", outputPath, dup.Path, file.Path)
|
||||
}
|
||||
}
|
||||
copyPaths[outputPath] = file
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func calcCopyPaths(cfg *config) (copyPaths, error) {
|
||||
result := copyPaths{}
|
||||
|
||||
for _, file := range cfg.Files {
|
||||
if file.MaybeDirectory {
|
||||
// This entry may be a directory
|
||||
s, err := os.Stat(file.Path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to stats file %s: %w", file.Path, err)
|
||||
}
|
||||
if s.IsDir() {
|
||||
// List files in the directory recursively and copy each file individually
|
||||
files, err := filePathWalkDir(file.Path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to walk directory %s: %w", file.Path, err)
|
||||
}
|
||||
for _, f := range files {
|
||||
r, err := filepath.Rel(file.Path, f)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to walk directory %s: %w", file.Path, err)
|
||||
}
|
||||
dirFile := fileInfo{
|
||||
MaybeDirectory: false,
|
||||
Package: file.Package,
|
||||
Path: f,
|
||||
RootPath: file.RootPath,
|
||||
ShortPath: path.Join(file.ShortPath, r),
|
||||
Workspace: file.Workspace,
|
||||
WorkspacePath: path.Join(file.WorkspacePath, r),
|
||||
}
|
||||
if err := calcCopyPath(cfg, result, dirFile); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
// The entry is not a directory
|
||||
file.MaybeDirectory = false
|
||||
}
|
||||
if err := calcCopyPath(cfg, result, file); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// From https://opensource.com/article/18/6/copying-files-go
|
||||
func copy(src, dst string) (int64, error) {
|
||||
sourceFileStat, err := os.Stat(src)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if !sourceFileStat.Mode().IsRegular() {
|
||||
return 0, fmt.Errorf("%s is not a regular file", src)
|
||||
}
|
||||
|
||||
source, err := os.Open(src)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer source.Close()
|
||||
|
||||
destination, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer destination.Close()
|
||||
nBytes, err := io.Copy(destination, source)
|
||||
return nBytes, err
|
||||
}
|
||||
|
||||
func main() {
|
||||
fmt.Println("Not yet implemented!")
|
||||
cfg, err := parseConfig(os.Args[1])
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// Calculate copy paths
|
||||
copyPaths, err := calcCopyPaths(cfg)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// Perform copies
|
||||
// TODO: split out into parallel go routines?
|
||||
for to, from := range copyPaths {
|
||||
if cfg.Verbose {
|
||||
fmt.Printf("%v => %v\n", from.Path, to)
|
||||
}
|
||||
err := os.MkdirAll(path.Dir(to), os.ModePerm)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
_, err = copy(from.Path, to)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue