diff --git a/BUILD.bazel b/BUILD.bazel index a4ae68c..3620d48 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -22,6 +22,17 @@ gazelle( gazelle = "gazelle_bin", ) +gazelle( + name = "gazelle_update_repos", + args = [ + "-build_file_proto_mode=disable_global", + "-from_file=go.mod", + "-to_macro=deps.bzl%go_dependencies", + "-prune", + ], + command = "update-repos", +) + bzl_library( name = "internal_deps", srcs = ["internal_deps.bzl"], diff --git a/WORKSPACE b/WORKSPACE index f4d2578..24f8fb2 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -29,11 +29,17 @@ load("//lib:host_repo.bzl", "host_repo") host_repo(name = "aspect_bazel_lib_host") -load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies") +load("//:deps.bzl", "go_dependencies") + +# gazelle:repository_macro deps.bzl%go_dependencies +# gazelle:repository go_repository name=org_golang_x_tools importpath=golang.org/x/tools +# https://github.com/bazelbuild/bazel-gazelle/issues/1217#issuecomment-1152236735 +go_dependencies() ############################################ # Gazelle, for generating bzl_library targets load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies") +load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies") go_rules_dependencies() diff --git a/deps.bzl b/deps.bzl new file mode 100644 index 0000000..985ee9c --- /dev/null +++ b/deps.bzl @@ -0,0 +1,43 @@ +"""This module contains the project repository dependencies. +""" + +load("@bazel_gazelle//:deps.bzl", "go_repository") + +def go_dependencies(): + """The Go dependencies. + """ + go_repository( + name = "com_github_gobwas_glob", + build_file_proto_mode = "disable_global", + importpath = "github.com/gobwas/glob", + sum = "h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=", + version = "v0.2.3", + ) + go_repository( + name = "com_github_google_go_cmp", + build_file_proto_mode = "disable_global", + importpath = "github.com/google/go-cmp", + sum = "h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=", + version = "v0.5.8", + ) + go_repository( + name = "org_golang_x_exp", + build_file_proto_mode = "disable_global", + importpath = "golang.org/x/exp", + sum = "h1:m9O6OTJ627iFnN2JIWfdqlZCzneRO6EEBsHXI25P8ws=", + version = "v0.0.0-20221230185412-738e83a70c30", + ) + go_repository( + name = "org_golang_x_mod", + build_file_proto_mode = "disable_global", + importpath = "golang.org/x/mod", + sum = "h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I=", + version = "v0.6.0", + ) + go_repository( + name = "org_golang_x_sys", + build_file_proto_mode = "disable_global", + importpath = "golang.org/x/sys", + sum = "h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=", + version = "v0.1.0", + ) diff --git a/docs/copy_to_directory.md b/docs/copy_to_directory.md index acf74ae..48f880c 100644 --- a/docs/copy_to_directory.md +++ b/docs/copy_to_directory.md @@ -69,7 +69,7 @@ other rule implementations where additional_files can also be passed in. | exclude_srcs_packages | List of Bazel packages (with glob support) to exclude from output directory.

See copy_to_directory rule documentation for more details. | [] | | include_srcs_patterns | List of paths (with glob support) to include in output directory.

See copy_to_directory rule documentation for more details. | ["**"] | | exclude_srcs_patterns | List of paths (with glob support) to exclude from output directory.

See copy_to_directory rule documentation for more details. | [] | -| exclude_prefixes | List of path prefixes to exclude from output directory. | [] | +| exclude_prefixes | List of path prefixes to exclude from output directory.

See copy_to_directory rule documentation for more details. | [] | | replace_prefixes | Map of paths prefixes to replace in the output directory path when copying files.

See copy_to_directory rule documentation for more details. | {} | | allow_overwrites | If True, allow files to be overwritten if the same output file is copied to twice.

See copy_to_directory rule documentation for more details. | False | | is_windows | Deprecated and unused | None | diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..0066212 --- /dev/null +++ b/go.mod @@ -0,0 +1,8 @@ +module github.com/aspect-build/bazel-lib + +go 1.19 + +require ( + github.com/gobwas/glob v0.2.3 + golang.org/x/exp v0.0.0-20221230185412-738e83a70c30 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..da5d63f --- /dev/null +++ b/go.sum @@ -0,0 +1,4 @@ +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +golang.org/x/exp v0.0.0-20221230185412-738e83a70c30 h1:m9O6OTJ627iFnN2JIWfdqlZCzneRO6EEBsHXI25P8ws= +golang.org/x/exp v0.0.0-20221230185412-738e83a70c30/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= diff --git a/lib/copy_to_directory.bzl b/lib/copy_to_directory.bzl index 835c70f..f7a49cf 100644 --- a/lib/copy_to_directory.bzl +++ b/lib/copy_to_directory.bzl @@ -7,12 +7,16 @@ the `root_paths`, `include_srcs_patters`, `exclude_srcs_patters` and `replace_pr load( "//lib/private:copy_to_directory.bzl", _copy_to_directory_action = "copy_to_directory_action", + # TODO: export copy_to_directory_bin_action once ready + # _copy_to_directory_bin_action = "copy_to_directory_bin_action", _copy_to_directory_lib = "copy_to_directory_lib", ) # export the starlark library as a public API copy_to_directory_lib = _copy_to_directory_lib copy_to_directory_action = _copy_to_directory_action +# TODO: export copy_to_directory_bin_action once ready +# copy_to_directory_bin_action = _copy_to_directory_bin_action copy_to_directory = rule( implementation = _copy_to_directory_lib.impl, diff --git a/lib/private/copy_directory.bzl b/lib/private/copy_directory.bzl index 169ba08..20f53ac 100644 --- a/lib/private/copy_directory.bzl +++ b/lib/private/copy_directory.bzl @@ -75,7 +75,7 @@ def copy_directory_action(ctx, src, dst, is_windows = None): is_windows: Deprecated and unused """ - # TODO(2.0): remove depcreated & unused is_windows parameter + # TODO(2.0): remove deprecated & unused is_windows parameter if not src.is_source and not dst.is_directory: fail("src must be a source directory or TreeArtifact") if dst.is_source or not dst.is_directory: diff --git a/lib/private/copy_file.bzl b/lib/private/copy_file.bzl index 76cd14d..554065a 100644 --- a/lib/private/copy_file.bzl +++ b/lib/private/copy_file.bzl @@ -101,7 +101,7 @@ def copy_file_action(ctx, src, dst, dir_path = None, is_windows = None): is_windows: Deprecated and unused """ - # TODO(2.0): remove depcreated & unused is_windows parameter + # TODO(2.0): remove deprecated & unused is_windows parameter if dst.is_directory: fail("dst must not be a TreeArtifact") if src.is_directory: diff --git a/lib/private/copy_to_bin.bzl b/lib/private/copy_to_bin.bzl index 53f01af..58bfb4f 100644 --- a/lib/private/copy_to_bin.bzl +++ b/lib/private/copy_to_bin.bzl @@ -35,7 +35,7 @@ def copy_file_to_bin_action(ctx, file, is_windows = None): A File in the output tree. """ - # TODO(2.0): remove depcreated & unused is_windows parameter + # TODO(2.0): remove deprecated & unused is_windows parameter if not file.is_source: return file if ctx.label.workspace_name != file.owner.workspace_name: @@ -109,7 +109,7 @@ def copy_files_to_bin_actions(ctx, files, is_windows = None): List of File objects in the output tree. """ - # TODO(2.0): remove depcreated & unused is_windows parameter + # TODO(2.0): remove deprecated & unused is_windows parameter return [copy_file_to_bin_action(ctx, file, is_windows = is_windows) for file in files] def _copy_to_bin_impl(ctx): diff --git a/lib/private/copy_to_directory.bzl b/lib/private/copy_to_directory.bzl index 20aac54..c38d47c 100644 --- a/lib/private/copy_to_directory.bzl +++ b/lib/private/copy_to_directory.bzl @@ -280,6 +280,15 @@ _copy_to_directory_attr = { This setting has no effect on Windows where overwrites are always allowed.""", ), + # TODO: flip to copy_to_directory_bin_action once ready + # "verbose": attr.bool( + # doc = """If true, prints out verbose logs to stdout""", + # ), + # "_tool": attr.label( + # executable = True, + # cfg = "exec", + # default = "//tools/copy_to_directory", + # ), } def _any_globs_match(exprs, path): @@ -383,7 +392,7 @@ def _copy_paths( # file is excluded as its external repository does not match any patterns in include_external_repositories return None, None, None - # apply include_srcs_packages if "**" is not included in the list + # apply include_srcs_packages if not _any_globs_match(include_srcs_packages, src_file.owner.package): # file is excluded as it does not match any specified include_srcs_packages return None, None, None @@ -605,6 +614,26 @@ def _copy_to_directory_impl(ctx): allow_overwrites = ctx.attr.allow_overwrites, ) + # TODO: flip to copy_to_directory_bin_action once ready + # copy_to_directory_bin_action( + # ctx, + # name = ctx.attr.name, + # dst = dst, + # copy_to_directory_bin = ctx.executable._tool, + # files = ctx.files.srcs, + # targets = [t for t in ctx.attr.srcs if DirectoryPathInfo in t], + # root_paths = ctx.attr.root_paths, + # include_external_repositories = ctx.attr.include_external_repositories, + # include_srcs_packages = ctx.attr.include_srcs_packages, + # exclude_srcs_packages = ctx.attr.exclude_srcs_packages, + # include_srcs_patterns = ctx.attr.include_srcs_patterns, + # exclude_srcs_patterns = ctx.attr.exclude_srcs_patterns, + # exclude_prefixes = ctx.attr.exclude_prefixes, + # replace_prefixes = ctx.attr.replace_prefixes, + # allow_overwrites = ctx.attr.allow_overwrites, + # verbose = ctx.attr.verbose, + # ) + return [ DefaultInfo( files = depset([dst]), @@ -628,6 +657,201 @@ def _expand_src_packages_patterns(patterns, package): result.append(pattern) return result +def copy_to_directory_bin_action( + ctx, + name, + dst, + copy_to_directory_bin, + files = [], + targets = [], + root_paths = ["."], + include_external_repositories = [], + include_srcs_packages = ["**"], + exclude_srcs_packages = [], + include_srcs_patterns = ["**"], + exclude_srcs_patterns = [], + exclude_prefixes = [], + replace_prefixes = {}, + allow_overwrites = False, + verbose = False): + """Helper function to copy files to a directory using a tool binary. + + The tool binary will typically be the `@aspect_bazel_lib//tools/copy_to_directory` `go_binary` + either built from source or provided by a toolchain. + + This helper is used by copy_to_directory. It is exposed as a public API so it can be used within + other rule implementations where additional_files can also be passed in. + + Args: + ctx: The rule context. + + name: Name of target creating this action used for config file generation. + + dst: The directory to copy to. Must be a TreeArtifact. + + copy_to_directory_bin: Copy to directory tool binary. + + files: List of files to copy into the output directory. + + targets: List of targets that provide DirectoryPathInfo to copy into the output directory. + + root_paths: List of paths that are roots in the output directory. + + See copy_to_directory rule documentation for more details. + + include_external_repositories: List of external repository names to include in the output directory. + + See copy_to_directory rule documentation for more details. + + include_srcs_packages: List of Bazel packages to include in output directory. + + See copy_to_directory rule documentation for more details. + + exclude_srcs_packages: List of Bazel packages (with glob support) to exclude from output directory. + + See copy_to_directory rule documentation for more details. + + include_srcs_patterns: List of paths (with glob support) to include in output directory. + + See copy_to_directory rule documentation for more details. + + exclude_srcs_patterns: List of paths (with glob support) to exclude from output directory. + + See copy_to_directory rule documentation for more details. + + exclude_prefixes: List of path prefixes to exclude from output directory. + + See copy_to_directory rule documentation for more details. + + replace_prefixes: Map of paths prefixes to replace in the output directory path when copying files. + + See copy_to_directory rule documentation for more details. + + allow_overwrites: If True, allow files to be overwritten if the same output file is copied to twice. + + See copy_to_directory rule documentation for more details. + + verbose: If true, prints out verbose logs to stdout + """ + + # Replace "." in root_paths with the package name of the target + root_paths = [p if p != "." else ctx.label.package for p in root_paths] + + # Replace "." and "./**" patterns in in include_srcs_packages & exclude_srcs_packages + include_srcs_packages = _expand_src_packages_patterns(include_srcs_packages, ctx.label.package) + exclude_srcs_packages = _expand_src_packages_patterns(exclude_srcs_packages, ctx.label.package) + + # Convert and append exclude_prefixes to exclude_srcs_patterns + # TODO(2.0): remove exclude_prefixes this block and in a future breaking release + for exclude_prefix in exclude_prefixes: + if exclude_prefix.endswith("**"): + exclude_srcs_patterns.append(exclude_prefix) + elif exclude_prefix.endswith("*"): + exclude_srcs_patterns.append(exclude_prefix + "/**") + exclude_srcs_patterns.append(exclude_prefix) + elif exclude_prefix.endswith("/"): + exclude_srcs_patterns.append(exclude_prefix + "**") + else: + exclude_srcs_patterns.append(exclude_prefix + "*/**") + exclude_srcs_patterns.append(exclude_prefix + "*") + + if not include_srcs_packages: + fail("An empty 'include_srcs_packages' list will exclude all srcs and result in an empty directory") + + if "**" in exclude_srcs_packages: + fail("A '**' glob pattern in 'exclude_srcs_packages' will exclude all srcs and result in an empty directory") + + if not include_srcs_patterns: + fail("An empty 'include_srcs_patterns' list will exclude all srcs and result in an empty directory") + + if "**" in exclude_srcs_patterns: + fail("A '**' glob pattern in 'exclude_srcs_patterns' will exclude all srcs and result in an empty directory") + + for replace_prefix in replace_prefixes.keys(): + if replace_prefix.endswith("**"): + msg = "replace_prefix '{}' must not end with '**' glob expression".format(replace_prefix) + fail(msg) + + files_and_targets = [] + for f in files: + files_and_targets.append(struct( + file = f, + path = f.path, + root_path = f.root.path, + short_path = f.short_path, + workspace_path = paths.to_workspace_path(f), + )) + for t in targets: + if not DirectoryPathInfo in t: + continue + files_and_targets.append(struct( + file = t[DirectoryPathInfo].directory, + path = "/".join([t[DirectoryPathInfo].directory.path, t[DirectoryPathInfo].path]), + root_path = t[DirectoryPathInfo].directory.root.path, + short_path = "/".join([t[DirectoryPathInfo].directory.short_path, t[DirectoryPathInfo].path]), + workspace_path = "/".join([paths.to_workspace_path(t[DirectoryPathInfo].directory), t[DirectoryPathInfo].path]), + )) + + file_infos = [] + file_inputs = [] + for f in files_and_targets: + if not f.file.owner: + msg = "Expected an owner target label for file {} but found none".format(f) + fail(msg) + + if f.file.owner.package == None: + msg = "Expected owner target label for file {} to have a package name but found None".format(f) + fail(msg) + + if f.file.owner.workspace_name == None: + msg = "Expected owner target label for file {} to have a workspace name but found None".format(f) + fail(msg) + + file_infos.append({ + # the path might be a file if it came from a DirectoryPathInfo or it is a source directory + "maybe_directory": f.file.is_directory or f.file.is_source, + "package": f.file.owner.package, + "path": f.path, + "root_path": f.root_path, + "short_path": f.short_path, + "workspace": f.file.owner.workspace_name, + "workspace_path": f.workspace_path, + }) + file_inputs.append(f.file) + + if not file_inputs: + fail("No files to copy") + + config = { + "allow_overwrites": allow_overwrites, + "dst": dst.path, + "exclude_srcs_packages": exclude_srcs_packages, + "exclude_srcs_patterns": exclude_srcs_patterns, + "files": file_infos, + "include_external_repositories": include_external_repositories, + "include_srcs_packages": include_srcs_packages, + "include_srcs_patterns": include_srcs_patterns, + "replace_prefixes": replace_prefixes, + "root_paths": root_paths, + "verbose": verbose, + } + + config_file = ctx.actions.declare_file("{}_config.json".format(name)) + ctx.actions.write( + output = config_file, + content = json.encode_indent(config), + ) + + ctx.actions.run( + inputs = file_inputs + [config_file], + outputs = [dst], + executable = copy_to_directory_bin, + arguments = [config_file.path], + mnemonic = "CopyToDirectory", + progress_message = "Copying files to directory", + execution_requirements = _COPY_EXECUTION_REQUIREMENTS, + ) + def copy_to_directory_action( ctx, srcs, @@ -683,6 +907,8 @@ def copy_to_directory_action( exclude_prefixes: List of path prefixes to exclude from output directory. + See copy_to_directory rule documentation for more details. + replace_prefixes: Map of paths prefixes to replace in the output directory path when copying files. See copy_to_directory rule documentation for more details. @@ -694,7 +920,7 @@ def copy_to_directory_action( is_windows: Deprecated and unused """ - # TODO(2.0): remove depcreated & unused is_windows parameter + # TODO(2.0): remove deprecated & unused is_windows parameter if not srcs: fail("srcs must not be empty") diff --git a/lib/tests/copy_to_directory_bin_action/1 b/lib/tests/copy_to_directory_bin_action/1 new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/lib/tests/copy_to_directory_bin_action/1 @@ -0,0 +1 @@ +1 diff --git a/lib/tests/copy_to_directory_bin_action/2 b/lib/tests/copy_to_directory_bin_action/2 new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/lib/tests/copy_to_directory_bin_action/2 @@ -0,0 +1 @@ +1 diff --git a/lib/tests/copy_to_directory_bin_action/BUILD.bazel b/lib/tests/copy_to_directory_bin_action/BUILD.bazel new file mode 100644 index 0000000..b3d7778 --- /dev/null +++ b/lib/tests/copy_to_directory_bin_action/BUILD.bazel @@ -0,0 +1,53 @@ +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") +load("//lib:diff_test.bzl", "diff_test") +load("//lib:copy_to_directory.bzl", "copy_to_directory") +load("//lib:copy_to_bin.bzl", "copy_to_bin") +load(":lib.bzl", "lib") +load(":pkg.bzl", "pkg") + +copy_to_bin( + name = "copy_1", + srcs = ["1"], +) + +lib( + name = "lib", + srcs = ["1"], + # intentionally dup on "1" to make sure it is gracefully handled + others = [ + "1", + # also pass in a copy_to_bin copy of "1" to spice things up; + # this case is handled in the fix in https://github.com/aspect-build/bazel-lib/pull/205 + "copy_1", + "2", + "d", + ], +) + +# pkg should copy DefaultInfo files and OtherInfo files +pkg( + name = "pkg", + srcs = [":lib"], + out = "pkg", +) + +copy_to_directory( + name = "expected_pkg", + srcs = [ + "1", + "2", + "d", + ], +) + +diff_test( + name = "test", + file1 = ":pkg", + file2 = ":expected_pkg", +) + +bzl_library( + name = "other_info", + srcs = ["other_info.bzl"], + visibility = ["//visibility:public"], +) diff --git a/lib/tests/copy_to_directory_bin_action/d/1 b/lib/tests/copy_to_directory_bin_action/d/1 new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/lib/tests/copy_to_directory_bin_action/d/1 @@ -0,0 +1 @@ +1 diff --git a/lib/tests/copy_to_directory_bin_action/d/2 b/lib/tests/copy_to_directory_bin_action/d/2 new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/lib/tests/copy_to_directory_bin_action/d/2 @@ -0,0 +1 @@ +1 diff --git a/lib/tests/copy_to_directory_bin_action/d/d/1 b/lib/tests/copy_to_directory_bin_action/d/d/1 new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/lib/tests/copy_to_directory_bin_action/d/d/1 @@ -0,0 +1 @@ +1 diff --git a/lib/tests/copy_to_directory_bin_action/d/d/2 b/lib/tests/copy_to_directory_bin_action/d/d/2 new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/lib/tests/copy_to_directory_bin_action/d/d/2 @@ -0,0 +1 @@ +1 diff --git a/lib/tests/copy_to_directory_bin_action/lib.bzl b/lib/tests/copy_to_directory_bin_action/lib.bzl new file mode 100644 index 0000000..14544a5 --- /dev/null +++ b/lib/tests/copy_to_directory_bin_action/lib.bzl @@ -0,0 +1,22 @@ +""" +Test rule to create a lib with a DefaultInfo and a OtherInfo +""" + +load(":other_info.bzl", "OtherInfo") + +_attrs = { + "srcs": attr.label_list(allow_files = True), + "others": attr.label_list(allow_files = True), +} + +def _lib_impl(ctx): + return [ + DefaultInfo(files = depset(ctx.files.srcs)), + OtherInfo(files = depset(ctx.files.others)), + ] + +lib = rule( + implementation = _lib_impl, + attrs = _attrs, + provides = [DefaultInfo, OtherInfo], +) diff --git a/lib/tests/copy_to_directory_bin_action/other_info.bzl b/lib/tests/copy_to_directory_bin_action/other_info.bzl new file mode 100644 index 0000000..6c02679 --- /dev/null +++ b/lib/tests/copy_to_directory_bin_action/other_info.bzl @@ -0,0 +1,8 @@ +"""For testing""" + +OtherInfo = provider( + doc = "For testing", + fields = { + "files": "A depset of files", + }, +) diff --git a/lib/tests/copy_to_directory_bin_action/pkg.bzl b/lib/tests/copy_to_directory_bin_action/pkg.bzl new file mode 100644 index 0000000..b84111d --- /dev/null +++ b/lib/tests/copy_to_directory_bin_action/pkg.bzl @@ -0,0 +1,45 @@ +""" +Test rule to create a pkg with DefaultInfo and OtherInfo files +""" + +load("@aspect_bazel_lib//lib/private:copy_to_directory.bzl", "copy_to_directory_bin_action") +load(":other_info.bzl", "OtherInfo") + +_attrs = { + "srcs": attr.label_list(allow_files = True), + "out": attr.string(mandatory = True), + "_tool": attr.label( + executable = True, + cfg = "exec", + default = "//tools/copy_to_directory", + ), +} + +def _pkg_impl(ctx): + dst = ctx.actions.declare_directory(ctx.attr.out) + + additional_files_depsets = [] + + # include files from OtherInfo of srcs + for src in ctx.attr.srcs: + if OtherInfo in src: + additional_files_depsets.append(src[OtherInfo].files) + + copy_to_directory_bin_action( + ctx, + name = ctx.attr.name, + files = ctx.files.srcs + depset(transitive = additional_files_depsets).to_list(), + dst = dst, + copy_to_directory_bin = ctx.executable._tool, + verbose = True, + ) + + return [ + DefaultInfo(files = depset([dst])), + ] + +pkg = rule( + implementation = _pkg_impl, + attrs = _attrs, + provides = [DefaultInfo], +) diff --git a/tools/copy_to_directory/BUILD.bazel b/tools/copy_to_directory/BUILD.bazel index 589a201..bbf3459 100644 --- a/tools/copy_to_directory/BUILD.bazel +++ b/tools/copy_to_directory/BUILD.bazel @@ -5,6 +5,10 @@ go_library( srcs = ["main.go"], importpath = "github.com/aspect-build/bazel-lib/tools/copy_to_directory", visibility = ["//visibility:public"], + deps = [ + "@com_github_gobwas_glob//:glob", + "@org_golang_x_exp//maps", + ], ) go_binary( diff --git a/tools/copy_to_directory/main.go b/tools/copy_to_directory/main.go index 9a3b647..30c9d1a 100644 --- a/tools/copy_to_directory/main.go +++ b/tools/copy_to_directory/main.go @@ -1,7 +1,347 @@ package main -import "fmt" +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "path" + "path/filepath" + "strings" + + "github.com/gobwas/glob" + "golang.org/x/exp/maps" +) + +type fileInfo struct { + MaybeDirectory bool `json:"maybe_directory"` + Package string `json:"package"` + Path string `json:"path"` + RootPath string `json:"root_path"` + ShortPath string `json:"short_path"` + Workspace string `json:"workspace"` + WorkspacePath string `json:"workspace_path"` +} + +type config struct { + AllowOverwrites bool `json:"allow_overwrites"` + Dst string `json:"dst"` + ExcludeSrcsPackages []string `json:"exclude_srcs_packages"` + ExcludeSrcsPatterns []string `json:"exclude_srcs_patterns"` + Files []fileInfo `json:"files"` + IncludeExternalRepositories []string `json:"include_external_repositories"` + IncludeSrcsPackages []string `json:"include_srcs_packages"` + IncludeSrcsPatterns []string `json:"include_srcs_patterns"` + ReplacePrefixes map[string]string `json:"replace_prefixes"` + RootPaths []string `json:"root_paths"` + Verbose bool `json:"verbose"` + + ExcludeSrcsPackagesGlobs []glob.Glob + ExcludeSrcsPatternsGlobs []glob.Glob + IncludeExternalRepositoriesGlobs []glob.Glob + IncludeSrcsPackagesGlobs []glob.Glob + IncludeSrcsPatternsGlobs []glob.Glob + ReplacePrefixesGlobs []glob.Glob + ReplacePrefixesKeys []string + RootPathsGlobs []glob.Glob +} + +type copyPaths map[string]fileInfo + +func parseConfig(configPath string) (*config, error) { + f, err := os.Open(configPath) + if err != nil { + return nil, fmt.Errorf("failed to open config file: %w", err) + } + defer f.Close() + + byteValue, err := ioutil.ReadAll(f) + if err != nil { + return nil, fmt.Errorf("failed to read config file: %w", err) + } + + var cfg config + if err := json.Unmarshal([]byte(byteValue), &cfg); err != nil { + return nil, fmt.Errorf("failed to parse config file: %w", err) + } + + // compile all globs + cfg.ExcludeSrcsPackagesGlobs, err = compileGlobs(cfg.ExcludeSrcsPackages) + if err != nil { + return nil, err + } + + cfg.ExcludeSrcsPatternsGlobs, err = compileGlobs(cfg.ExcludeSrcsPatterns) + if err != nil { + return nil, err + } + + cfg.IncludeExternalRepositoriesGlobs, err = compileGlobs(cfg.IncludeExternalRepositories) + if err != nil { + return nil, err + } + + cfg.IncludeSrcsPackagesGlobs, err = compileGlobs(cfg.IncludeSrcsPackages) + if err != nil { + return nil, err + } + + cfg.IncludeSrcsPatternsGlobs, err = compileGlobs(cfg.IncludeSrcsPatterns) + if err != nil { + return nil, err + } + + cfg.RootPathsGlobs, err = compileGlobs(cfg.RootPaths) + if err != nil { + return nil, err + } + + cfg.ReplacePrefixesKeys = maps.Keys(cfg.ReplacePrefixes) + cfg.ReplacePrefixesGlobs, err = compileGlobs(cfg.ReplacePrefixesKeys) + if err != nil { + return nil, err + } + + return &cfg, nil +} + +func compileGlobs(patterns []string) ([]glob.Glob, error) { + result := make([]glob.Glob, len(patterns)) + for i, pattern := range patterns { + g, err := glob.Compile(pattern) + if err != nil { + return nil, fmt.Errorf("failed to compile glob pattern '%s': %w", pattern, err) + } + result[i] = g + } + return result, nil +} + +func anyGlobsMatch(globs []glob.Glob, test string) bool { + for _, g := range globs { + if g.Match(test) { + return true + } + } + return false +} + +func longestGlobsMatch(globs []glob.Glob, test string) (string, int) { + result := "" + index := 0 + for i, g := range globs { + match := longestGlobMatch(g, test) + if len(match) > len(result) { + result = match + index = i + } + } + return result, index +} + +func longestGlobMatch(g glob.Glob, test string) string { + for i := 0; i < len(test); i++ { + t := test[:len(test)-i] + if g.Match(t) { + return t + } + } + return "" +} + +// From https://stackoverflow.com/a/49196644 +func filePathWalkDir(root string) ([]string, error) { + var files []string + err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if !info.IsDir() { + files = append(files, path) + } + return nil + }) + return files, err +} + +func calcCopyPath(cfg *config, copyPaths copyPaths, file fileInfo) error { + // Apply filters and transformations in the following order: + // + // - `include_external_repositories` + // - `include_srcs_packages` + // - `exclude_srcs_packages` + // - `root_paths` + // - `include_srcs_patterns` + // - `exclude_srcs_patterns` + // - `replace_prefixes` + // + // If you change this order please update the docstrings in the copy_to_directory rule. + + outputPath := file.WorkspacePath + outputRoot := path.Dir(outputPath) + + // apply include_external_repositories (if the file is from an external repository) + if file.Workspace != "" { + if !anyGlobsMatch(cfg.IncludeExternalRepositoriesGlobs, file.Workspace) { + return nil // external workspace is not included + } + } + + // apply include_srcs_packages + if !anyGlobsMatch(cfg.IncludeSrcsPackagesGlobs, file.Package) { + return nil // package is not included + } + + // apply exclude_srcs_packages + if anyGlobsMatch(cfg.ExcludeSrcsPackagesGlobs, file.Package) { + return nil // package is excluded + } + + // apply root_paths + rootPathMatch, _ := longestGlobsMatch(cfg.RootPathsGlobs, outputRoot) + if rootPathMatch != "" { + outputPath = outputPath[len(rootPathMatch):] + if strings.HasPrefix(outputPath, "/") { + outputPath = outputPath[1:] + } + } + + // apply include_srcs_patterns + if !anyGlobsMatch(cfg.IncludeSrcsPatternsGlobs, outputPath) { + return nil // outputPath is not included + } + + // apply include_srcs_patterns + if anyGlobsMatch(cfg.ExcludeSrcsPatternsGlobs, outputPath) { + return nil // outputPath is excluded + } + + // apply replace_prefixes + replacePrefixMatch, replacePrefixIndex := longestGlobsMatch(cfg.ReplacePrefixesGlobs, outputPath) + if replacePrefixMatch != "" { + replaceWith := cfg.ReplacePrefixes[cfg.ReplacePrefixesKeys[replacePrefixIndex]] + outputPath = replaceWith + outputPath[len(replacePrefixMatch):] + } + + outputPath = path.Join(cfg.Dst, outputPath) + + // add this file to the copy Paths + dup, exists := copyPaths[outputPath] + if exists { + if dup.ShortPath == file.ShortPath { + // this is likely the same file listed twice: the original in the source + // tree and the copy in the output tree + // TODO: stat the two files to double check that they are the same + if file.RootPath == "" { + // when this happens prefer the output tree copy. + return nil + } + } else if !cfg.AllowOverwrites { + return fmt.Errorf("duplicate output file '%s' configured from source files '%s' and '%s'; set 'allow_overwrites' to True to allow this overwrites but keep in mind that order matters when this is set", outputPath, dup.Path, file.Path) + } + } + copyPaths[outputPath] = file + + return nil +} + +func calcCopyPaths(cfg *config) (copyPaths, error) { + result := copyPaths{} + + for _, file := range cfg.Files { + if file.MaybeDirectory { + // This entry may be a directory + s, err := os.Stat(file.Path) + if err != nil { + return nil, fmt.Errorf("failed to stats file %s: %w", file.Path, err) + } + if s.IsDir() { + // List files in the directory recursively and copy each file individually + files, err := filePathWalkDir(file.Path) + if err != nil { + return nil, fmt.Errorf("failed to walk directory %s: %w", file.Path, err) + } + for _, f := range files { + r, err := filepath.Rel(file.Path, f) + if err != nil { + return nil, fmt.Errorf("failed to walk directory %s: %w", file.Path, err) + } + dirFile := fileInfo{ + MaybeDirectory: false, + Package: file.Package, + Path: f, + RootPath: file.RootPath, + ShortPath: path.Join(file.ShortPath, r), + Workspace: file.Workspace, + WorkspacePath: path.Join(file.WorkspacePath, r), + } + if err := calcCopyPath(cfg, result, dirFile); err != nil { + return nil, err + } + } + continue + } + // The entry is not a directory + file.MaybeDirectory = false + } + if err := calcCopyPath(cfg, result, file); err != nil { + return nil, err + } + } + + return result, nil +} + +// From https://opensource.com/article/18/6/copying-files-go +func copy(src, dst string) (int64, error) { + sourceFileStat, err := os.Stat(src) + if err != nil { + return 0, err + } + + if !sourceFileStat.Mode().IsRegular() { + return 0, fmt.Errorf("%s is not a regular file", src) + } + + source, err := os.Open(src) + if err != nil { + return 0, err + } + defer source.Close() + + destination, err := os.Create(dst) + if err != nil { + return 0, err + } + defer destination.Close() + nBytes, err := io.Copy(destination, source) + return nBytes, err +} func main() { - fmt.Println("Not yet implemented!") + cfg, err := parseConfig(os.Args[1]) + if err != nil { + log.Fatal(err) + } + + // Calculate copy paths + copyPaths, err := calcCopyPaths(cfg) + if err != nil { + log.Fatal(err) + } + + // Perform copies + // TODO: split out into parallel go routines? + for to, from := range copyPaths { + if cfg.Verbose { + fmt.Printf("%v => %v\n", from.Path, to) + } + err := os.MkdirAll(path.Dir(to), os.ModePerm) + if err != nil { + log.Fatal(err) + } + _, err = copy(from.Path, to) + if err != nil { + log.Fatal(err) + } + } }