Copy rules_directory's globs to bazel-skylib. (#511)

Original implementation is at https://github.com/matts1/rules_directory

README.md

@@ -64,6 +64,7 @@ s = shell.quote(p)
 * [common_settings](docs/common_settings_doc.md)
 * [directories](docs/copy_directory_doc.md)
     * [directory](docs/directory_doc.md)
+    * [directory_glob](docs/directory_glob.md)
     * [subdirectory](docs/subdirectory_doc.md)
 * [copy_directory](docs/copy_directory_doc.md)
 * [copy_file](docs/copy_file_doc.md)

docs/BUILD

@@ -64,6 +64,12 @@ stardoc_with_diff_test(
     out_label = "//docs:directory_doc.md",
 )
 
+stardoc_with_diff_test(
+    name = "directory_glob",
+    bzl_library_target = "//rules/directory:glob",
+    out_label = "//docs:directory_glob_doc.md",
+)
+
 stardoc_with_diff_test(
     name = "directory_providers",
     bzl_library_target = "//rules/directory:providers",

docs/directory_glob_doc.md

@@ -0,0 +1,39 @@
+<!-- Generated with Stardoc: http://skydoc.bazel.build -->
+
+Rules to filter files from a directory.
+
+<a id="directory_glob"></a>
+
+## directory_glob
+
+<pre>
+directory_glob(<a href="#directory_glob-name">name</a>, <a href="#directory_glob-srcs">srcs</a>, <a href="#directory_glob-data">data</a>, <a href="#directory_glob-allow_empty">allow_empty</a>, <a href="#directory_glob-directory">directory</a>, <a href="#directory_glob-exclude">exclude</a>)
+</pre>
+
+globs files from a directory by relative path.
+
+Usage:
+
+```
+directory_glob(
+    name = "foo",
+    directory = ":directory",
+    srcs = ["foo/bar"],
+    data = ["foo/**"],
+    exclude = ["foo/**/*.h"]
+)
+```
+
+**ATTRIBUTES**
+
+
+| Name  | Description | Type | Mandatory | Default |
+| :------------- | :------------- | :------------- | :------------- | :------------- |
+| <a id="directory_glob-name"></a>name |  A unique name for this target.   | <a href="https://bazel.build/concepts/labels#target-names">Name</a> | required |  |
+| <a id="directory_glob-srcs"></a>srcs |  A list of globs to files within the directory to put in the files.<br><br>For example, `srcs = ["foo/**"]` would collect the file at `<directory>/foo` into the files.   | List of strings | optional |  `[]`  |
+| <a id="directory_glob-data"></a>data |  A list of globs to files within the directory to put in the runfiles.<br><br>For example, `data = ["foo/**"]` would collect all files contained within `<directory>/foo` into the runfiles.   | List of strings | optional |  `[]`  |
+| <a id="directory_glob-allow_empty"></a>allow_empty |  If true, allows globs to not match anything.   | Boolean | optional |  `False`  |
+| <a id="directory_glob-directory"></a>directory |  -   | <a href="https://bazel.build/concepts/labels">Label</a> | required |  |
+| <a id="directory_glob-exclude"></a>exclude |  A list of globs to files within the directory to exclude from the files and runfiles.   | List of strings | optional |  `[]`  |
+
+

docs/directory_providers_doc.md

@@ -7,7 +7,8 @@ Skylib module containing providers for directories.
 ## DirectoryInfo
 
 <pre>
-DirectoryInfo(<a href="#DirectoryInfo-entries">entries</a>, <a href="#DirectoryInfo-transitive_files">transitive_files</a>, <a href="#DirectoryInfo-path">path</a>, <a href="#DirectoryInfo-human_readable">human_readable</a>, <a href="#DirectoryInfo-get_path">get_path</a>, <a href="#DirectoryInfo-get_file">get_file</a>, <a href="#DirectoryInfo-get_subdirectory">get_subdirectory</a>)
+DirectoryInfo(<a href="#DirectoryInfo-entries">entries</a>, <a href="#DirectoryInfo-transitive_files">transitive_files</a>, <a href="#DirectoryInfo-path">path</a>, <a href="#DirectoryInfo-human_readable">human_readable</a>, <a href="#DirectoryInfo-get_path">get_path</a>, <a href="#DirectoryInfo-get_file">get_file</a>, <a href="#DirectoryInfo-get_subdirectory">get_subdirectory</a>,
+              <a href="#DirectoryInfo-glob">glob</a>)
 </pre>
 
 Information about a directory
@@ -24,6 +25,7 @@ Information about a directory
 | <a id="DirectoryInfo-get_path"></a>get_path |  (Function(str) -> DirectoryInfo\|File) A function to return the entry corresponding to the joined path.    |
 | <a id="DirectoryInfo-get_file"></a>get_file |  (Function(str) -> File) A function to return the entry corresponding to the joined path.    |
 | <a id="DirectoryInfo-get_subdirectory"></a>get_subdirectory |  (Function(str) -> DirectoryInfo) A function to return the entry corresponding to the joined path.    |
+| <a id="DirectoryInfo-glob"></a>glob |  (Function(include, exclude, allow_empty=False)) A function that works the same as native.glob.    |
 
 
 <a id="create_directory_info"></a>

docs/directory_utils_doc.md

@@ -2,6 +2,76 @@
 
 Skylib module containing utility functions related to directories.
 
+<a id="directory_glob"></a>
+
+## directory_glob
+
+<pre>
+directory_glob(<a href="#directory_glob-directory">directory</a>, <a href="#directory_glob-include">include</a>, <a href="#directory_glob-allow_empty">allow_empty</a>)
+</pre>
+
+native.glob, but for DirectoryInfo.
+
+**PARAMETERS**
+
+
+| Name  | Description | Default Value |
+| :------------- | :------------- | :------------- |
+| <a id="directory_glob-directory"></a>directory |  (DirectoryInfo) The directory to look relative from.   |  none |
+| <a id="directory_glob-include"></a>include |  (List[string]) A list of globs to match.   |  none |
+| <a id="directory_glob-allow_empty"></a>allow_empty |  (bool) Whether to allow a glob to not match any files.   |  `False` |
+
+**RETURNS**
+
+depset[File] A set of files that match.
+
+
+<a id="directory_glob_chunk"></a>
+
+## directory_glob_chunk
+
+<pre>
+directory_glob_chunk(<a href="#directory_glob_chunk-directory">directory</a>, <a href="#directory_glob_chunk-chunk">chunk</a>)
+</pre>
+
+Given a directory and a chunk of a glob, returns possible candidates.
+
+**PARAMETERS**
+
+
+| Name  | Description | Default Value |
+| :------------- | :------------- | :------------- |
+| <a id="directory_glob_chunk-directory"></a>directory |  (DirectoryInfo) The directory to look relative from.   |  none |
+| <a id="directory_glob_chunk-chunk"></a>chunk |  (string) A chunk of a glob to look at.   |  none |
+
+**RETURNS**
+
+List[Either[DirectoryInfo, File]]] The candidate next entries for the chunk.
+
+
+<a id="directory_single_glob"></a>
+
+## directory_single_glob
+
+<pre>
+directory_single_glob(<a href="#directory_single_glob-directory">directory</a>, <a href="#directory_single_glob-glob">glob</a>)
+</pre>
+
+Calculates all files that are matched by a glob on a directory.
+
+**PARAMETERS**
+
+
+| Name  | Description | Default Value |
+| :------------- | :------------- | :------------- |
+| <a id="directory_single_glob-directory"></a>directory |  (DirectoryInfo) The directory to look relative from.   |  none |
+| <a id="directory_single_glob-glob"></a>glob |  (string) A glob to match.   |  none |
+
+**RETURNS**
+
+List[File] A list of files that match.
+
+
 <a id="get_child"></a>
 
 ## get_child
@@ -52,3 +122,25 @@ Gets a subdirectory contained within a tree of another directory.
 (File|DirectoryInfo) The directory contained within.
 
 
+<a id="transitive_entries"></a>
+
+## transitive_entries
+
+<pre>
+transitive_entries(<a href="#transitive_entries-directory">directory</a>)
+</pre>
+
+Returns the files and directories contained within a directory transitively.
+
+**PARAMETERS**
+
+
+| Name  | Description | Default Value |
+| :------------- | :------------- | :------------- |
+| <a id="transitive_entries-directory"></a>directory |  (DirectoryInfo) The directory to look at   |  none |
+
+**RETURNS**
+
+List[Either[DirectoryInfo, File]] The entries contained within.
+
+

rules/directory/BUILD

@@ -18,11 +18,21 @@ bzl_library(
     ],
 )
 
+bzl_library(
+    name = "glob",
+    srcs = ["glob.bzl"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":providers",
+    ],
+)
+
 bzl_library(
     name = "providers",
     srcs = ["providers.bzl"],
     visibility = ["//visibility:public"],
     deps = [
+        "//rules/directory/private:glob",
         "//rules/directory/private:paths",
     ],
 )

rules/directory/glob.bzl

@@ -0,0 +1,73 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Rules to filter files from a directory."""
+
+load(":providers.bzl", "DirectoryInfo")
+
+def _directory_glob_impl(ctx):
+    directory = ctx.attr.directory[DirectoryInfo]
+    srcs = directory.glob(
+        ctx.attr.srcs,
+        exclude = ctx.attr.exclude,
+        allow_empty = ctx.attr.allow_empty,
+    )
+    data = directory.glob(
+        ctx.attr.data,
+        exclude = ctx.attr.exclude,
+        allow_empty = ctx.attr.allow_empty,
+    )
+
+    return DefaultInfo(
+        files = srcs,
+        runfiles = ctx.runfiles(transitive_files = depset(transitive = [srcs, data])),
+    )
+
+directory_glob = rule(
+    implementation = _directory_glob_impl,
+    attrs = {
+        "allow_empty": attr.bool(
+            doc = "If true, allows globs to not match anything.",
+        ),
+        "data": attr.string_list(
+            doc = """A list of globs to files within the directory to put in the runfiles.
+
+For example, `data = ["foo/**"]` would collect all files contained within `<directory>/foo` into the
+runfiles.""",
+        ),
+        "directory": attr.label(providers = [DirectoryInfo], mandatory = True),
+        "exclude": attr.string_list(
+            doc = "A list of globs to files within the directory to exclude from the files and runfiles.",
+        ),
+        "srcs": attr.string_list(
+            doc = """A list of globs to files within the directory to put in the files.
+
+For example, `srcs = ["foo/**"]` would collect the file at `<directory>/foo` into the
+files.""",
+        ),
+    },
+    doc = """globs files from a directory by relative path.
+
+Usage:
+
+```
+directory_glob(
+    name = "foo",
+    directory = ":directory",
+    srcs = ["foo/bar"],
+    data = ["foo/**"],
+    exclude = ["foo/**/*.h"]
+)
+```
+""",
+)

rules/directory/private/BUILD

@@ -8,6 +8,12 @@ exports_files(
     visibility = ["//:__subpackages__"],
 )
 
+bzl_library(
+    name = "glob",
+    srcs = ["glob.bzl"],
+    visibility = ["//visibility:public"],
+)
+
 bzl_library(
     name = "paths",
     srcs = ["paths.bzl"],

rules/directory/private/glob.bzl

@@ -0,0 +1,137 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Skylib module containing glob operations on directories."""
+
+_NO_GLOB_MATCHES = "{glob} failed to match any files in {dir}"
+
+def transitive_entries(directory):
+    """Returns the files and directories contained within a directory transitively.
+
+    Args:
+        directory: (DirectoryInfo) The directory to look at
+
+    Returns:
+        List[Either[DirectoryInfo, File]] The entries contained within.
+    """
+    entries = [directory]
+    stack = [directory]
+    for _ in range(99999999):
+        if not stack:
+            return entries
+        d = stack.pop()
+        for entry in d.entries.values():
+            entries.append(entry)
+            if type(entry) != "File":
+                stack.append(entry)
+
+    fail("Should never get to here")
+
+def directory_glob_chunk(directory, chunk):
+    """Given a directory and a chunk of a glob, returns possible candidates.
+
+    Args:
+        directory: (DirectoryInfo) The directory to look relative from.
+        chunk: (string) A chunk of a glob to look at.
+
+    Returns:
+        List[Either[DirectoryInfo, File]]] The candidate next entries for the chunk.
+    """
+    if chunk == "*":
+        return directory.entries.values()
+    elif chunk == "**":
+        return transitive_entries(directory)
+    elif "*" not in chunk:
+        if chunk in directory.entries:
+            return [directory.entries[chunk]]
+        else:
+            return []
+    elif chunk.count("*") > 2:
+        fail("glob chunks with more than two asterixes are unsupported. Got", chunk)
+
+    if chunk.count("*") == 2:
+        left, middle, right = chunk.split("*")
+    else:
+        middle = ""
+        left, right = chunk.split("*")
+    entries = []
+    for name, entry in directory.entries.items():
+        if name.startswith(left) and name.endswith(right) and len(left) + len(right) <= len(name) and middle in name[len(left):-len(right)]:
+            entries.append(entry)
+    return entries
+
+def directory_single_glob(directory, glob):
+    """Calculates all files that are matched by a glob on a directory.
+
+    Args:
+        directory: (DirectoryInfo) The directory to look relative from.
+        glob: (string) A glob to match.
+
+    Returns:
+        List[File] A list of files that match.
+    """
+
+    # Treat a glob as a nondeterministic finite state automata. We can be in
+    # multiple places at the one time.
+    candidate_dirs = [directory]
+    candidate_files = []
+    for chunk in glob.split("/"):
+        next_candidate_dirs = {}
+        candidate_files = {}
+        for candidate in candidate_dirs:
+            for e in directory_glob_chunk(candidate, chunk):
+                if type(e) == "File":
+                    candidate_files[e] = None
+                else:
+                    next_candidate_dirs[e.human_readable] = e
+        candidate_dirs = next_candidate_dirs.values()
+
+    return list(candidate_files)
+
+def glob(directory, include, exclude = [], allow_empty = False):
+    """native.glob, but for DirectoryInfo.
+
+    Args:
+        directory: (DirectoryInfo) The directory to look relative from.
+        include: (List[string]) A list of globs to match.
+        exclude: (List[string]) A list of globs to exclude.
+        allow_empty: (bool) Whether to allow a glob to not match any files.
+
+    Returns:
+        depset[File] A set of files that match.
+    """
+    include_files = []
+    for g in include:
+        matches = directory_single_glob(directory, g)
+        if not matches and not allow_empty:
+            fail(_NO_GLOB_MATCHES.format(
+                glob = repr(g),
+                dir = directory.human_readable,
+            ))
+        include_files.extend(matches)
+
+    if not exclude:
+        return depset(include_files)
+
+    include_files = {k: None for k in include_files}
+    for g in exclude:
+        matches = directory_single_glob(directory, g)
+        if not matches and not allow_empty:
+            fail(_NO_GLOB_MATCHES.format(
+                glob = repr(g),
+                dir = directory.human_readable,
+            ))
+        for f in matches:
+            include_files.pop(f, None)
+    return depset(include_files.keys())

rules/directory/providers.bzl

@@ -14,6 +14,7 @@
 
 """Skylib module containing providers for directories."""
 
+load("//rules/directory/private:glob.bzl", "glob")
 load("//rules/directory/private:paths.bzl", "DIRECTORY", "FILE", "get_path")
 
 def _init_directory_info(**kwargs):
@@ -22,6 +23,7 @@ def _init_directory_info(**kwargs):
         get_path = lambda path: get_path(self, path, require_type = None),
         get_file = lambda path: get_path(self, path, require_type = FILE),
         get_subdirectory = lambda path: get_path(self, path, require_type = DIRECTORY),
+        glob = lambda include, exclude = [], allow_empty = False: glob(self, include, exclude, allow_empty),
     )
     return kwargs
 
@@ -42,5 +44,6 @@ DirectoryInfo = provider(
         "get_path": "(Function(str) -> DirectoryInfo|File) A function to return the entry corresponding to the joined path.",
         "get_file": "(Function(str) -> File) A function to return the entry corresponding to the joined path.",
         "get_subdirectory": "(Function(str) -> DirectoryInfo) A function to return the entry corresponding to the joined path.",
+        "glob": "(Function(include, exclude, allow_empty=False)) A function that works the same as native.glob.",
     },
 )

tests/directory/BUILD

@@ -1,6 +1,7 @@
 load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
 load("@bazel_skylib//rules/directory:directory.bzl", "directory")
 load(":directory_test.bzl", "directory_test_suite")
+load(":glob_test.bzl", "glob_test_suite")
 load(":subdirectory_test.bzl", "subdirectory_test_suite")
 
 directory(
@@ -28,6 +29,10 @@ directory_test_suite(
     name = "directory_tests",
 )
 
+glob_test_suite(
+    name = "glob_tests",
+)
+
 subdirectory_test_suite(
     name = "subdirectory_tests",
 )

tests/directory/external_directory_tests.bzl

@@ -27,6 +27,7 @@ external_directory_tests = repository_rule(
         "files": attr.label_list(default = [
             "//tests/directory:BUILD",
             "//tests/directory:directory_test.bzl",
+            "//tests/directory:glob_test.bzl",
             "//tests/directory:subdirectory_test.bzl",
             "//tests/directory:testdata/f1",
             "//tests/directory:testdata/subdir/f2",

tests/directory/glob_test.bzl

@@ -0,0 +1,168 @@
+# Copyright 2024 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unit tests for the directory_glob rule."""
+
+load("@bazel_skylib//rules/directory:glob.bzl", "directory_glob")
+load("@bazel_skylib//rules/directory:providers.bzl", "DirectoryInfo")
+
+# buildifier: disable=bzl-visibility
+load("@bazel_skylib//rules/directory/private:glob.bzl", "directory_glob_chunk", "transitive_entries")
+load("@rules_testing//lib:analysis_test.bzl", "analysis_test", "test_suite")
+load("@rules_testing//lib:truth.bzl", "matching")
+load(":utils.bzl", "failure_matching", "failure_test")
+
+def _expect_glob_chunk(env, directory, chunk):
+    return env.expect.that_collection(
+        directory_glob_chunk(directory, chunk),
+        expr = "directory_glob_chunk({}, {})".format(directory.human_readable, repr(chunk)),
+    )
+
+def _expect_glob(env, directory, include, allow_empty = False):
+    return env.expect.that_collection(
+        directory.glob(include, allow_empty = allow_empty),
+        expr = "directory_glob({}, {}, allow_empty={})".format(directory.human_readable, include, allow_empty),
+    )
+
+def _with_children(children):
+    return DirectoryInfo(
+        entries = {k: k for k in children},
+        human_readable = repr(children),
+    )
+
+def _glob_test(name):
+    simple_name = "_simple_%s" % name
+    exclude_name = "_exclude_%s" % name
+    directory_glob(
+        name = simple_name,
+        srcs = ["testdata/f1"],
+        data = ["testdata/subdir/f2"],
+        directory = ":root",
+    )
+
+    directory_glob(
+        name = exclude_name,
+        srcs = [
+            "testdata/f1",
+            "nonexistent",
+        ],
+        allow_empty = True,
+        data = ["**"],
+        directory = ":root",
+        exclude = ["testdata/f1"],
+    )
+
+    analysis_test(
+        name = name,
+        impl = _glob_test_impl,
+        targets = {
+            "root": ":root",
+            "f1": ":f1_filegroup",
+            "f2": ":f2_filegroup",
+            "simple_glob": simple_name,
+            "glob_with_exclude": exclude_name,
+        },
+    )
+
+def _glob_test_impl(env, targets):
+    f1 = targets.f1.files.to_list()[0]
+    f2 = targets.f2.files.to_list()[0]
+    root = targets.root[DirectoryInfo]
+    testdata = root.entries["testdata"]
+    subdir = testdata.entries["subdir"]
+
+    env.expect.that_collection(transitive_entries(root)).contains_exactly([
+        root,
+        testdata,
+        subdir,
+        f1,
+        f2,
+    ])
+
+    _expect_glob_chunk(env, testdata, "f1").contains_exactly([f1])
+    _expect_glob_chunk(env, root, "nonexistent").contains_exactly([])
+    _expect_glob_chunk(env, testdata, "f2").contains_exactly([])
+    _expect_glob_chunk(env, root, "testdata").contains_exactly([testdata])
+    _expect_glob_chunk(env, testdata, "*").contains_exactly(
+        [f1, subdir],
+    )
+    _expect_glob_chunk(
+        env,
+        _with_children(["a", "d", "abc", "abbc", "ab.bc", ".abbc", "abbc."]),
+        "ab*bc",
+    ).contains_exactly([
+        "abbc",
+        "ab.bc",
+    ])
+    _expect_glob_chunk(
+        env,
+        _with_children(["abbc", "abbbc", "ab.b.bc"]),
+        "ab*b*bc",
+    ).contains_exactly([
+        "abbbc",
+        "ab.b.bc",
+    ])
+
+    _expect_glob(env, root, ["testdata/f1"]).contains_exactly([f1])
+    _expect_glob(env, root, ["testdata/subdir/f2"]).contains_exactly([f2])
+    _expect_glob(env, root, ["**"]).contains_exactly([f1, f2])
+    _expect_glob(env, root, ["**/f1"]).contains_exactly([f1])
+    _expect_glob(env, root, ["**/**/f1"]).contains_exactly([f1])
+    _expect_glob(env, root, ["testdata/*/f1"], allow_empty = True).contains_exactly([])
+
+    simple_glob = env.expect.that_target(targets.simple_glob)
+    with_exclude = env.expect.that_target(targets.glob_with_exclude)
+
+    env.expect.that_collection(
+        simple_glob.actual[DefaultInfo].files.to_list(),
+        expr = "simple_glob's files",
+    ).contains_exactly([f1])
+    env.expect.that_collection(
+        with_exclude.actual[DefaultInfo].files.to_list(),
+        expr = "with_exclude's files",
+    ).contains_exactly([])
+
+    # target.runfiles().contains_exactly() doesn't do what we want - it converts
+    # it to a string corresponding to the runfiles import path.
+    env.expect.that_collection(
+        simple_glob.runfiles().actual.files.to_list(),
+        expr = "simple_glob's runfiles",
+    ).contains_exactly([f1, f2])
+    env.expect.that_collection(
+        with_exclude.runfiles().actual.files.to_list(),
+        expr = "with_exclude's runfiles",
+    ).contains_exactly([f2])
+
+def _glob_with_no_match_test(name):
+    failure_test(
+        name = name,
+        impl = failure_matching(matching.contains('"nonexistent" failed to match any files in')),
+        rule = directory_glob,
+        srcs = [
+            "testdata/f1",
+            "nonexistent",
+        ],
+        data = ["testdata/f1"],
+        directory = ":root",
+    )
+
+# buildifier: disable=function-docstring
+def glob_test_suite(name):
+    test_suite(
+        name = name,
+        tests = [
+            _glob_test,
+            _glob_with_no_match_test,
+        ],
+    )