maprule: an improved version of genrule() (#86)
maprule() is an improved version of native.genrule(), with the following advantages: - Maprule can process source files in parallel, creating separate actions for each of them. - Maprule does not require declaring all output files. Instead you declare templates for the output files yielded for each source. Therefore N source files and M templates yield N*M outputs. - Maprule supports both Bash and cmd.exe syntax for its commands via the specialized rules bash_maprule and cmd_maprule. - Maprule's cmd attribute does deliberately not support $(location) expression nor Make Variables, in order to avoid issues and challenges with quoting. (In case of cmd.exe passing empty arguments is impossible). These paths can be passed as envvars instead. - Maprule's add_env attribute does support $(location) expressions (and some extra placeholders) and is the idiomatic way to pass execpaths of labels in "tools" or "srcs" (the shared sources available for all actions) to the command. See https://github.com/bazelbuild/bazel/issues/4319
This commit is contained in:
László Csomor
GitHub
parent
6dd0b9cbd6
commit
8d4f7612b2
|
|
@ -14,6 +14,8 @@ Each of the `.bzl` files in the `lib` directory defines a "module"—a
|
||||||
`struct` that contains a set of related functions and/or other symbols that can
|
`struct` that contains a set of related functions and/or other symbols that can
|
||||||
be loaded as a single unit, for convenience.
|
be loaded as a single unit, for convenience.
|
||||||
|
|
||||||
|
Skylib also provides build rules under the `rules` directory.
|
||||||
|
|
||||||
## Getting Started
|
## Getting Started
|
||||||
|
|
||||||
### `WORKSPACE` file
|
### `WORKSPACE` file
|
||||||
|
|
@ -68,6 +70,10 @@ s = shell.quote(p)
|
||||||
* [unittest](lib/unittest.bzl)
|
* [unittest](lib/unittest.bzl)
|
||||||
* [versions](lib/versions.bzl)
|
* [versions](lib/versions.bzl)
|
||||||
|
|
||||||
|
## List of rules (in rules/)
|
||||||
|
|
||||||
|
* [`cmd_maprule` and `bash_maprule`](lib/maprule.bzl)
|
||||||
|
|
||||||
## Writing a new module
|
## Writing a new module
|
||||||
|
|
||||||
Steps to add a module to Skylib:
|
Steps to add a module to Skylib:
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,20 @@
|
||||||
|
load("//:bzl_library.bzl", "bzl_library")
|
||||||
|
|
||||||
|
licenses(["notice"])
|
||||||
|
|
||||||
|
bzl_library(
|
||||||
|
name = "maprule",
|
||||||
|
srcs = ["maprule.bzl"],
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
|
deps = [":maprule_private"],
|
||||||
|
)
|
||||||
|
|
||||||
|
bzl_library(
|
||||||
|
name = "maprule_private",
|
||||||
|
srcs = ["maprule_private.bzl"],
|
||||||
|
visibility = ["//visibility:private"],
|
||||||
|
deps = [
|
||||||
|
"//lib:dicts",
|
||||||
|
"//lib:paths",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
@ -0,0 +1,28 @@
|
||||||
|
# Copyright 2019 The Bazel Authors. All rights reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
"""Maprule implementation in Starlark.
|
||||||
|
|
||||||
|
This module exports the cmd_maprule() and bash_maprule() build rules.
|
||||||
|
|
||||||
|
They are the same except for the interpreter they use (cmd.exe and Bash respectively) and for the
|
||||||
|
expected language of their `cmd` attribute.
|
||||||
|
|
||||||
|
You can read more about these rules in "maprule_private.bzl".
|
||||||
|
"""
|
||||||
|
|
||||||
|
load(":maprule_private.bzl", _bash_maprule = "bash_maprule", _cmd_maprule = "cmd_maprule")
|
||||||
|
|
||||||
|
cmd_maprule = _cmd_maprule
|
||||||
|
bash_maprule = _bash_maprule
|
||||||
|
|
@ -0,0 +1,675 @@
|
||||||
|
# Copyright 2018 The Bazel Authors. All rights reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
"""Maprule implementation in Starlark.
|
||||||
|
|
||||||
|
This module exports:
|
||||||
|
|
||||||
|
- The cmd_maprule() and bash_maprule() build rules.
|
||||||
|
They are the same except for the interpreter they use (cmd.exe and Bash respectively) and for the
|
||||||
|
expected language of their `cmd` attribute. We will refer to them collectively as `maprule`.
|
||||||
|
|
||||||
|
- The maprule_testing struct. This should only be used by maprule's own unittests.
|
||||||
|
"""
|
||||||
|
|
||||||
|
load("//lib:dicts.bzl", "dicts")
|
||||||
|
load("//lib:paths.bzl", "paths")
|
||||||
|
|
||||||
|
_cmd_maprule_intro = """
|
||||||
|
Maprule that runs a Windows Command Prompt (`cmd.exe`) command.
|
||||||
|
|
||||||
|
This rule is the same as `bash_maprule`, but uses `cmd.exe` instead of Bash, therefore the `cmd`
|
||||||
|
attribute must use Command Prompt syntax. `cmd_maprule` rules can only be built on Windows.
|
||||||
|
"""
|
||||||
|
|
||||||
|
_bash_maprule_intro = """
|
||||||
|
Maprule that runs a Bash command.
|
||||||
|
|
||||||
|
This rule is the same as `cmd_maprule` except this one uses Bash to run the command, therefore the
|
||||||
|
`cmd` attribute must use Bash syntax. `bash_maprule` rules can only be built if Bash is installed on
|
||||||
|
the build machine.
|
||||||
|
"""
|
||||||
|
|
||||||
|
_cmd_maprule_example = """
|
||||||
|
# This file is //projects/game:BUILD
|
||||||
|
|
||||||
|
load("//tools/build_rules:maprule.bzl", "cmd_maprule")
|
||||||
|
|
||||||
|
cmd_maprule(
|
||||||
|
name = "process_assets",
|
||||||
|
foreach_srcs = [
|
||||||
|
"rust.png",
|
||||||
|
"teapot.3ds",
|
||||||
|
"//assets:wood.jpg",
|
||||||
|
"//assets:models",
|
||||||
|
],
|
||||||
|
outs_templates = {
|
||||||
|
"TAGS": "{src}.tags",
|
||||||
|
"DIGEST": "digests/{src_name_noext}.md5",
|
||||||
|
},
|
||||||
|
tools = [
|
||||||
|
"//bin:asset-tagger",
|
||||||
|
"//util:md5sum",
|
||||||
|
],
|
||||||
|
add_env = {
|
||||||
|
"ASSET_TAGGER": "$(location //bin:asset-tagger)",
|
||||||
|
"MD5SUM": "$(location //util:md5sum)",
|
||||||
|
},
|
||||||
|
|
||||||
|
# Note: this command should live in a script file, we only inline it in the `cmd` attribute
|
||||||
|
# for the sake of demonstration. See Tips and Tricks section.
|
||||||
|
cmd = "%MAPRULE_ASSET_TAGGER% --input=%MAPRULE_SRC% --output=%MAPRULE_TAGS% & " +
|
||||||
|
'IF /I "%ERRORLEVEL%" EQU "0" ( %MAPRULE_MD5SUM% %MAPRULE_SRC% > %MAPRULE_DIGEST% )',
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
|
||||||
|
_bash_maprule_example = """
|
||||||
|
# This file is //projects/game:BUILD
|
||||||
|
|
||||||
|
load("//tools/build_rules:maprule.bzl", "bash_maprule")
|
||||||
|
|
||||||
|
bash_maprule(
|
||||||
|
name = "process_assets",
|
||||||
|
foreach_srcs = [
|
||||||
|
"rust.png",
|
||||||
|
"teapot.3ds",
|
||||||
|
"//assets:wood.jpg",
|
||||||
|
"//assets:models",
|
||||||
|
],
|
||||||
|
outs_templates = {
|
||||||
|
"TAGS": "{src}.tags",
|
||||||
|
"DIGEST": "digests/{src_name_noext}.md5",
|
||||||
|
},
|
||||||
|
tools = [
|
||||||
|
"//bin:asset-tagger",
|
||||||
|
"//util:md5sum",
|
||||||
|
],
|
||||||
|
add_env = {
|
||||||
|
"ASSET_TAGGER": "$(location //bin:asset-tagger)",
|
||||||
|
"MD5SUM": "$(location //util:md5sum)",
|
||||||
|
},
|
||||||
|
|
||||||
|
# Note: this command should live in a script file, we only inline it in the `cmd` attribute
|
||||||
|
# for the sake of demonstration. See Tips and Tricks section.
|
||||||
|
cmd = '"$MAPRULE_ASSET_TAGGER" --input="$MAPRULE_SRC" --output="$MAPRULE_TAGS" && ' +
|
||||||
|
'"$MAPRULE_MD5SUM" "$MAPRULE_SRC" > "$MAPRULE_DIGEST"',
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
|
||||||
|
_rule_doc_template = """
|
||||||
|
{intro}
|
||||||
|
|
||||||
|
Maprule runs a specific command for each of the "foreach" source files. This allows processing
|
||||||
|
source files in parallel, to produce some outputs for each of them.
|
||||||
|
|
||||||
|
The name "maprule" indicates that this rule can be used to map source files to output files, and is
|
||||||
|
also a reference to "genrule" that inspired this rule's design (though there are significant
|
||||||
|
differences).
|
||||||
|
|
||||||
|
Below you will find an Example, Tips and Tricks, and an FAQ.
|
||||||
|
|
||||||
|
### Example
|
||||||
|
|
||||||
|
{example}
|
||||||
|
|
||||||
|
The "process_assets" rule above will execute the command in the `cmd` to process "rust.png",
|
||||||
|
"teapot.3ds", "//assets:wood.jpg", and every file in the "//assets:models" rule, producing the
|
||||||
|
corresponding .tags and .md5 files for each of them, under the following paths:
|
||||||
|
|
||||||
|
bazel-bin/projects/game/process_assets_out/projects/game/rust.png.tags
|
||||||
|
bazel-bin/projects/game/process_assets_out/digests/rust.md5
|
||||||
|
bazel-bin/projects/game/process_assets_out/projects/game/teapot.3ds.tags
|
||||||
|
bazel-bin/projects/game/process_assets_out/digests/teapot.md5
|
||||||
|
bazel-bin/projects/game/process_assets_out/assets/wood.jpg.tags
|
||||||
|
bazel-bin/projects/game/process_assets_out/digests/wood.md5
|
||||||
|
...
|
||||||
|
|
||||||
|
You can create downstream rules, for example a filegroup or genrule (or another maprule) that put
|
||||||
|
this rule in their `srcs` attribute and get all the .tags and .md5 files.
|
||||||
|
|
||||||
|
## Tips and Tricks
|
||||||
|
|
||||||
|
*(The Tips and Tricks section is the same for `cmd_maprule` and `bash_maprule`.)*
|
||||||
|
|
||||||
|
### Use script files instead of inlining commands in the `cmd` attribute.
|
||||||
|
|
||||||
|
Unless the command is trivial, don't try to write it in `cmd`.
|
||||||
|
|
||||||
|
Properly quoting parts of the command is challenging enough, add to that escaping for the BUILD
|
||||||
|
file's syntax and the `cmd` attribute quickly gets unmaintainably complex.
|
||||||
|
|
||||||
|
It's a lot easier and maintainable to create a script file such as "foo.sh" in `bash_maprule` or
|
||||||
|
"foo.bat" in `cmd_maprule` for the commands. To do that:
|
||||||
|
|
||||||
|
1. move the commands to a script file
|
||||||
|
2. add the script file to the `tools` attribute
|
||||||
|
3. add an entry to the `add_env` attribute, e.g. "`TOOL: "$(location :tool.sh)"`"
|
||||||
|
4. replace the `cmd` with just "$MAPRULE_FOO" (in `bash_maprule`) or "%MAPRULE_FOO%" (in
|
||||||
|
`cmd_maprule`).
|
||||||
|
|
||||||
|
Doing this also avoids hitting command line length limits.
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
cmd_maprule(
|
||||||
|
...
|
||||||
|
srcs = ["//data:temperatures.txt"],
|
||||||
|
tools = [
|
||||||
|
"command.bat",
|
||||||
|
":weather-stats",
|
||||||
|
],
|
||||||
|
add_env = {{
|
||||||
|
"COMMAND": "$(location :command.bat)",
|
||||||
|
"STATS_TOOL": "$(location :weather-stats-computer)",
|
||||||
|
"TEMPERATURES": "$(location //data:temperatures.txt)",
|
||||||
|
}},
|
||||||
|
cmd = "%MAPRULE_COMMAND%",
|
||||||
|
)
|
||||||
|
|
||||||
|
### Use the `add_env` attribute to pass tool locations to the command.
|
||||||
|
|
||||||
|
Entries in the `add_env` attribute may use "$(location)" references and may also use the same
|
||||||
|
placeholders as the `outs_templates` attribute. For example you can use this mechanism to pass extra
|
||||||
|
"$(location)" references of `tools` or `srcs` to the actions.
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
cmd_maprule(
|
||||||
|
...
|
||||||
|
foreach_srcs = [...],
|
||||||
|
outs_templates = {{"STATS_OUT": "{{src}}-stats.txt"}},
|
||||||
|
srcs = ["//data:temperatures.txt"],
|
||||||
|
tools = [":weather-stats"],
|
||||||
|
add_env = {{
|
||||||
|
"STATS_TOOL": "$(location :weather-stats-computer)",
|
||||||
|
"TEMPERATURES": "$(location //data:temperatures.txt)",
|
||||||
|
}},
|
||||||
|
cmd = "%MAPRULE_STATS_TOOL% --src=%MAPRULE_SRC% --data=%MAPRULE_TEMPERATURES% > %MAPRULE_STATS_OUT%",
|
||||||
|
)
|
||||||
|
|
||||||
|
cc_binary(
|
||||||
|
name = "weather-stats",
|
||||||
|
...
|
||||||
|
)
|
||||||
|
|
||||||
|
## Environment Variables
|
||||||
|
|
||||||
|
*(The Environment Variables section is the same for `cmd_maprule` and `bash_maprule`.)*
|
||||||
|
|
||||||
|
The rule defines several environment variables available to the command may reference. All of these
|
||||||
|
envvars names start with "MAPRULE_". You can add your own envvars with the `add_env` attribute.
|
||||||
|
|
||||||
|
The command can use some envvars, all named "MAPRULE_*<something>*".
|
||||||
|
|
||||||
|
The complete list of environment variables is:
|
||||||
|
|
||||||
|
- "MAPRULE_SRC": the path of the current file from `foreach_srcs`
|
||||||
|
- "MAPRULE_SRCS": the space-separated paths of all files in the `srcs` attribute
|
||||||
|
- "MAPRULE_*<OUT>*": for each key name *<OUT>* in the `outs_templates` attribute, this
|
||||||
|
is the path of the respective output file for the current source
|
||||||
|
- "MAPRULE_*<ENV>*": for each key name *<ENV>* in the `add_env` attribute
|
||||||
|
|
||||||
|
## FAQ
|
||||||
|
|
||||||
|
*(The FAQ section is the same for `cmd_maprule` and `bash_maprule`.)*
|
||||||
|
|
||||||
|
### What's the motivation for maprule? What's wrong with genrule?
|
||||||
|
|
||||||
|
genrule creates a single action for all of its inputs. It requires specifying all output files.
|
||||||
|
Finally, it can only create Bash commands.
|
||||||
|
|
||||||
|
Maprule supports parallel processing of its inputs and doesn't require specifying all outputs, just
|
||||||
|
templates for the outputs. And not only Bash is supported (via `bash_maprule`) but so are
|
||||||
|
cmd.exe-style commands via `cmd_maprule`.
|
||||||
|
|
||||||
|
### `genrule.cmd` supports "$(location)" expressions, how come `*_maprule.cmd` doesn't?
|
||||||
|
|
||||||
|
Maprule deliberately omits support for this feature to avoid pitfalls with quoting and escaping, and
|
||||||
|
potential issues with paths containing spaces. Instead, maprule exports environment variables for
|
||||||
|
the input and outputs of the action, and allows the user to define extra envvars. These extra
|
||||||
|
envvars do support "$(location)" expressions, so you can pass paths of labels in `srcs` and `tools`.
|
||||||
|
|
||||||
|
### Why are all envvars exported with the "MAPRULE_" prefix?
|
||||||
|
|
||||||
|
To avoid conflicts with other envvars, whose names could also be attractive outs_templates names.
|
||||||
|
|
||||||
|
### Why do `outs_templates` and `add_env` keys have to be uppercase?
|
||||||
|
|
||||||
|
Because they are exported as all-uppercase envvars, so requiring that they be declared as uppercase
|
||||||
|
gives a visual cue of that. It also avoids clashes resulting from mixed lower-upper-case names like
|
||||||
|
"foo" and "Foo".
|
||||||
|
|
||||||
|
### Why don't `outs_templates` and `add_env` keys have to start with "MAPRULE_" even though they are exported as such?
|
||||||
|
|
||||||
|
For convenience. It seems to bring no benefit to have the user always type "MAPRULE_" in front of
|
||||||
|
the name when the rule itself could as well add it.
|
||||||
|
|
||||||
|
### Why are all outputs relative to "*<maprule_pkg>*/*<maprule_name>*_out/" ?
|
||||||
|
|
||||||
|
To ensure that maprules in the same package and with the same outs_templates produce distinct output
|
||||||
|
files.
|
||||||
|
|
||||||
|
### Why aren't `outs_templates` keys available as placeholders in the values of `add_env`?
|
||||||
|
|
||||||
|
Because `add_env` is meant to be used for passing extra "$(location)" information to the action, and
|
||||||
|
the output paths are already available as envvars for the action.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def _is_relative_path(p):
|
||||||
|
"""Returns True if `p` is a relative path (considering Unix and Windows semantics)."""
|
||||||
|
return p and p[0] != "/" and p[0] != "\\" and (
|
||||||
|
len(p) < 2 or not p[0].isalpha() or p[1] != ":"
|
||||||
|
)
|
||||||
|
|
||||||
|
def _validate_attributes(ctx_attr_outs_templates, ctx_attr_add_env):
|
||||||
|
"""Validates rule attributes and returns a list of error messages if there were errors."""
|
||||||
|
errors = []
|
||||||
|
|
||||||
|
envvars = {
|
||||||
|
"MAPRULE_SRC": "the source file",
|
||||||
|
"MAPRULE_SRCS": "the space-joined paths of the common sources",
|
||||||
|
}
|
||||||
|
|
||||||
|
if not ctx_attr_outs_templates:
|
||||||
|
errors.append("ERROR: \"outs_templates\" must not be empty")
|
||||||
|
|
||||||
|
names_to_paths = {}
|
||||||
|
paths_to_names = {}
|
||||||
|
|
||||||
|
# Check entries in "outs_templates".
|
||||||
|
for name, path in ctx_attr_outs_templates.items():
|
||||||
|
# Check the entry's name.
|
||||||
|
envvar_for_name = "MAPRULE_" + name.upper()
|
||||||
|
error_prefix = "ERROR: In \"outs_templates\" entry {\"%s\": \"%s\"}: " % (name, path)
|
||||||
|
if not name:
|
||||||
|
errors.append("ERROR: Bad entry in the \"outs_templates\" attribute: the name " +
|
||||||
|
"should not be empty.")
|
||||||
|
elif name.upper() != name:
|
||||||
|
errors.append(error_prefix + "the name should be all upper-case.")
|
||||||
|
elif envvar_for_name in envvars:
|
||||||
|
errors.append((error_prefix +
|
||||||
|
"please rename it, otherwise this output path would be exported " +
|
||||||
|
"as the environment variable \"%s\", conflicting with the " +
|
||||||
|
"environment variable of %s.") % (
|
||||||
|
envvar_for_name,
|
||||||
|
envvars[envvar_for_name],
|
||||||
|
))
|
||||||
|
elif not path:
|
||||||
|
errors.append(error_prefix + "output path should not be empty.")
|
||||||
|
elif not _is_relative_path(path):
|
||||||
|
errors.append(error_prefix + "output path should be relative.")
|
||||||
|
elif ".." in path:
|
||||||
|
errors.append(error_prefix + "output path should not contain uplevel references.")
|
||||||
|
elif path in paths_to_names:
|
||||||
|
errors.append(error_prefix +
|
||||||
|
"output path is already used for \"%s\"." % paths_to_names[path])
|
||||||
|
envvars[envvar_for_name] = "the \"%s\" output file declared in the \"outs_templates\" attribute" % name
|
||||||
|
names_to_paths[name] = path
|
||||||
|
paths_to_names[path] = name
|
||||||
|
|
||||||
|
# Check envvar names in "add_env".
|
||||||
|
for name, value in ctx_attr_add_env.items():
|
||||||
|
envvar_for_name = "MAPRULE_" + name.upper()
|
||||||
|
error_prefix = "ERROR: In \"add_env\" entry {\"%s\": \"%s\"}: " % (name, value)
|
||||||
|
if not name:
|
||||||
|
errors.append("ERROR: Bad entry in the \"add_env\" attribute: the name should not be " +
|
||||||
|
"empty.")
|
||||||
|
elif name.upper() != name:
|
||||||
|
errors.append(error_prefix + "the name should be all upper-case.")
|
||||||
|
elif envvar_for_name in envvars:
|
||||||
|
errors.append((error_prefix +
|
||||||
|
"please rename it, otherwise it would be exported as \"%s\", " +
|
||||||
|
"conflicting with the environment variable of %s.") % (
|
||||||
|
envvar_for_name,
|
||||||
|
envvars[envvar_for_name],
|
||||||
|
))
|
||||||
|
elif "$(location" in value:
|
||||||
|
tokens = value.split("$(location")
|
||||||
|
if len(tokens) != 2:
|
||||||
|
errors.append(error_prefix + "use only one $(location) or $(locations) function.")
|
||||||
|
elif ")" not in tokens[1]:
|
||||||
|
errors.append(error_prefix +
|
||||||
|
"incomplete $(location) or $(locations) function, missing closing " +
|
||||||
|
"parenthesis")
|
||||||
|
envvars[name] = "an additional environment declared in \"add_env\" as \"%s\"" % name
|
||||||
|
|
||||||
|
return errors
|
||||||
|
|
||||||
|
def _src_placeholders(src, strategy):
|
||||||
|
return {
|
||||||
|
"src": strategy.as_path(src.short_path),
|
||||||
|
"src_dir": strategy.as_path(paths.dirname(src.short_path) + "/"),
|
||||||
|
"src_name": src.basename,
|
||||||
|
"src_name_noext": (src.basename[:-len(src.extension) - 1] if len(src.extension) else src.basename),
|
||||||
|
}
|
||||||
|
|
||||||
|
def _create_outputs(ctx, ctx_label_name, ctx_attr_outs_templates, strategy, foreach_srcs):
|
||||||
|
errors = []
|
||||||
|
outs_dicts = {}
|
||||||
|
output_generated_by = {}
|
||||||
|
all_output_files = []
|
||||||
|
src_placeholders_dicts = {}
|
||||||
|
output_path_prefix = ctx_label_name + "_out/"
|
||||||
|
for src in foreach_srcs:
|
||||||
|
src_placeholders_dicts[src] = _src_placeholders(src, strategy)
|
||||||
|
|
||||||
|
outputs_for_src = {}
|
||||||
|
for template_name, path in ctx_attr_outs_templates.items():
|
||||||
|
output_path = path.format(**src_placeholders_dicts[src])
|
||||||
|
if output_path in output_generated_by:
|
||||||
|
existing_generator = output_generated_by[output_path]
|
||||||
|
errors.append("\n".join([
|
||||||
|
"ERROR: output file generated multiple times:",
|
||||||
|
" output file: " + output_path,
|
||||||
|
" foreach_srcs file 1: " + existing_generator[0].short_path,
|
||||||
|
" outs_templates entry 1: " + existing_generator[1],
|
||||||
|
" foreach_srcs file 2: " + src.short_path,
|
||||||
|
" outs_templates entry 2: " + template_name,
|
||||||
|
]))
|
||||||
|
output_generated_by[output_path] = (src, template_name)
|
||||||
|
output = ctx.actions.declare_file(output_path_prefix + output_path)
|
||||||
|
outputs_for_src[template_name] = output
|
||||||
|
all_output_files.append(output)
|
||||||
|
outs_dicts[src] = outputs_for_src
|
||||||
|
|
||||||
|
if errors:
|
||||||
|
# For sake of Starlark unittest we return all_output_files, so the test can create dummy
|
||||||
|
# generating actions for the files even in case of errors.
|
||||||
|
return None, all_output_files, None, errors
|
||||||
|
else:
|
||||||
|
return outs_dicts, all_output_files, src_placeholders_dicts, None
|
||||||
|
|
||||||
|
def _resolve_locations(ctx, strategy, ctx_attr_add_env, ctx_attr_tools):
|
||||||
|
# ctx.resolve_command returns a Bash command. All we need though is the inputs and runfiles
|
||||||
|
# manifests (we expand $(location) references with ctx.expand_location below), so ignore the
|
||||||
|
# tuple's middle element (the resolved command).
|
||||||
|
inputs_from_tools, _, manifests_from_tools = ctx.resolve_command(
|
||||||
|
# Pretend that the additional envvars are forming a command, so resolve_command will resolve
|
||||||
|
# $(location) references in them (which we ignore here) and add their inputs and manifests
|
||||||
|
# to the results (which we really care about).
|
||||||
|
command = " ".join(ctx_attr_add_env.values()),
|
||||||
|
tools = ctx_attr_tools,
|
||||||
|
)
|
||||||
|
|
||||||
|
errors = []
|
||||||
|
location_expressions = []
|
||||||
|
parts = {}
|
||||||
|
was_anything_to_resolve = False
|
||||||
|
for k, v in ctx_attr_add_env.items():
|
||||||
|
# Look for "$(location ...)" or "$(locations ...)", resolve if found.
|
||||||
|
# _validate_attributes already ensured that there's at most one $(location/s ...) in "v".
|
||||||
|
if "$(location" in v:
|
||||||
|
tokens = v.split("$(location")
|
||||||
|
was_anything_to_resolve = True
|
||||||
|
closing_paren = tokens[1].find(")")
|
||||||
|
location_expressions.append("$(location" + tokens[1][:closing_paren + 1])
|
||||||
|
parts[k] = (tokens[0], tokens[1][closing_paren + 1:])
|
||||||
|
else:
|
||||||
|
location_expressions.append("")
|
||||||
|
|
||||||
|
if errors:
|
||||||
|
return None, None, None, errors
|
||||||
|
|
||||||
|
resolved_add_env = {}
|
||||||
|
if was_anything_to_resolve:
|
||||||
|
# Resolve all $(location) expressions in one go. Should be faster than resolving them
|
||||||
|
# one-by-one.
|
||||||
|
all_location_expressions = "<split_here>".join(location_expressions)
|
||||||
|
all_resolved_locations = ctx.expand_location(all_location_expressions)
|
||||||
|
resolved_locations = strategy.as_path(all_resolved_locations).split("<split_here>")
|
||||||
|
|
||||||
|
i = 0
|
||||||
|
|
||||||
|
# Starlark dictionaries have a deterministic order of iteration, so the element order in
|
||||||
|
# "resolved_locations" matches the order in "location_expressions", i.e. the previous
|
||||||
|
# iteration order of "ctx_attr_add_env".
|
||||||
|
for k, v in ctx_attr_add_env.items():
|
||||||
|
if location_expressions[i]:
|
||||||
|
head, tail = parts[k]
|
||||||
|
resolved_add_env[k] = head + resolved_locations[i] + tail
|
||||||
|
else:
|
||||||
|
resolved_add_env[k] = v
|
||||||
|
i += 1
|
||||||
|
else:
|
||||||
|
resolved_add_env = ctx_attr_add_env
|
||||||
|
|
||||||
|
if errors:
|
||||||
|
return None, None, None, errors
|
||||||
|
else:
|
||||||
|
return inputs_from_tools, manifests_from_tools, resolved_add_env, None
|
||||||
|
|
||||||
|
def _custom_envmap(ctx, strategy, src_placeholders, outs_dict, add_env):
|
||||||
|
return dicts.add(
|
||||||
|
{
|
||||||
|
"MAPRULE_" + k.upper(): strategy.as_path(v)
|
||||||
|
for k, v in src_placeholders.items()
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"MAPRULE_" + k.upper(): strategy.as_path(v.path)
|
||||||
|
for k, v in outs_dict.items()
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"MAPRULE_" + k.upper(): strategy.as_path(ctx.expand_location(v)).format(**src_placeholders)
|
||||||
|
for k, v in add_env.items()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
def _fail_if_errors(errors):
|
||||||
|
if errors:
|
||||||
|
# Don't overwhelm the user; report up to ten errors.
|
||||||
|
fail("\n".join(errors[:10]))
|
||||||
|
|
||||||
|
def _maprule_main(ctx, strategy):
|
||||||
|
errors = _validate_attributes(ctx.attr.outs_templates, ctx.attr.add_env)
|
||||||
|
_fail_if_errors(errors)
|
||||||
|
|
||||||
|
# From "srcs": merge the depsets in the DefaultInfo.files of the targets.
|
||||||
|
common_srcs = depset(transitive = [t[DefaultInfo].files for t in ctx.attr.srcs])
|
||||||
|
common_srcs_list = common_srcs.to_list()
|
||||||
|
|
||||||
|
# From "foreach_srcs": by accessing the attribute's value through ctx.files (a list, not a
|
||||||
|
# depset), we flatten the depsets of DefaultInfo.files of the targets and merge them to a single
|
||||||
|
# list. This is fine, we would have to do this anyway, because we iterate over them later.
|
||||||
|
foreach_srcs = ctx.files.foreach_srcs
|
||||||
|
|
||||||
|
# Create the outputs for each file in "foreach_srcs".
|
||||||
|
foreach_src_outs_dicts, all_outputs, src_placeholders_dicts, errors = _create_outputs(
|
||||||
|
ctx,
|
||||||
|
ctx.label.name,
|
||||||
|
ctx.attr.outs_templates,
|
||||||
|
strategy,
|
||||||
|
foreach_srcs,
|
||||||
|
)
|
||||||
|
_fail_if_errors(errors)
|
||||||
|
|
||||||
|
progress_message = (ctx.attr.message or "Executing maprule") + " for %s" % ctx.label
|
||||||
|
|
||||||
|
# Create the part of the environment variables map that all actions will share.
|
||||||
|
common_envmap = dicts.add(
|
||||||
|
ctx.configuration.default_shell_env,
|
||||||
|
{"MAPRULE_SRCS": " ".join([strategy.as_path(p.path) for p in common_srcs_list])},
|
||||||
|
)
|
||||||
|
|
||||||
|
# Resolve $(location) references in "cmd" and in "add_env".
|
||||||
|
inputs_from_tools, manifests_from_tools, add_env, errors = _resolve_locations(
|
||||||
|
ctx,
|
||||||
|
strategy,
|
||||||
|
ctx.attr.add_env,
|
||||||
|
ctx.attr.tools,
|
||||||
|
)
|
||||||
|
_fail_if_errors(errors)
|
||||||
|
|
||||||
|
# Create actions for each of the "foreach" sources.
|
||||||
|
for src in foreach_srcs:
|
||||||
|
strategy.create_action(
|
||||||
|
ctx,
|
||||||
|
inputs = depset(direct = [src] + inputs_from_tools, transitive = [common_srcs]),
|
||||||
|
outputs = foreach_src_outs_dicts[src].values(),
|
||||||
|
# The custom envmap contains envvars specific to the current "src", such as MAPRULE_SRC.
|
||||||
|
env = common_envmap + _custom_envmap(
|
||||||
|
ctx,
|
||||||
|
strategy,
|
||||||
|
src_placeholders_dicts[src],
|
||||||
|
foreach_src_outs_dicts[src],
|
||||||
|
add_env,
|
||||||
|
),
|
||||||
|
command = ctx.attr.cmd,
|
||||||
|
progress_message = progress_message,
|
||||||
|
manifests_from_tools = manifests_from_tools,
|
||||||
|
)
|
||||||
|
|
||||||
|
return [DefaultInfo(files = depset(all_outputs))]
|
||||||
|
|
||||||
|
def _as_windows_path(s):
|
||||||
|
"""Returns the input path as a Windows path (replaces all of "/" with "\")."""
|
||||||
|
return s.replace("/", "\\")
|
||||||
|
|
||||||
|
def _unchanged_path(s):
|
||||||
|
"""Returns the input string (path) unchanged."""
|
||||||
|
return s
|
||||||
|
|
||||||
|
def _create_cmd_action(ctx, inputs, outputs, env, command, progress_message, manifests_from_tools):
|
||||||
|
"""Create one action using cmd.exe for one of the "foreach" sources."""
|
||||||
|
ctx.actions.run(
|
||||||
|
inputs = inputs,
|
||||||
|
outputs = outputs,
|
||||||
|
executable = "cmd.exe",
|
||||||
|
env = env,
|
||||||
|
arguments = ["/C", command],
|
||||||
|
progress_message = progress_message,
|
||||||
|
mnemonic = "Maprule",
|
||||||
|
input_manifests = manifests_from_tools,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _create_bash_action(ctx, inputs, outputs, env, command, progress_message, manifests_from_tools):
|
||||||
|
"""Create one action using Bash for one of the "foreach" sources."""
|
||||||
|
ctx.actions.run_shell(
|
||||||
|
inputs = inputs,
|
||||||
|
outputs = outputs,
|
||||||
|
env = env,
|
||||||
|
command = command,
|
||||||
|
progress_message = progress_message,
|
||||||
|
mnemonic = "Maprule",
|
||||||
|
input_manifests = manifests_from_tools,
|
||||||
|
)
|
||||||
|
|
||||||
|
_CMD_STRATEGY = struct(
|
||||||
|
as_path = _as_windows_path,
|
||||||
|
create_action = _create_cmd_action,
|
||||||
|
)
|
||||||
|
|
||||||
|
_BASH_STRATEGY = struct(
|
||||||
|
as_path = _unchanged_path,
|
||||||
|
create_action = _create_bash_action,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _cmd_maprule_impl(ctx):
|
||||||
|
return _maprule_main(ctx, _CMD_STRATEGY)
|
||||||
|
|
||||||
|
def _bash_maprule_impl(ctx):
|
||||||
|
return _maprule_main(ctx, _BASH_STRATEGY)
|
||||||
|
|
||||||
|
_ATTRS = {
|
||||||
|
"srcs": attr.label_list(
|
||||||
|
allow_files = True,
|
||||||
|
doc = "The set of source files common to all actions of this rule.",
|
||||||
|
),
|
||||||
|
"add_env": attr.string_dict(
|
||||||
|
doc = "Extra environment variables to define for the actions. Every variable's name " +
|
||||||
|
"must be uppercase. Bazel will automatically prepend \"MAPRULE_\" to the name " +
|
||||||
|
"when exporting the variable for the action. The values may use \"$(location)\" " +
|
||||||
|
"expressions for labels declared in the `srcs` and `tools` attribute, and " +
|
||||||
|
"may reference the same placeholders as the values of the `outs_templates` " +
|
||||||
|
"attribute.",
|
||||||
|
),
|
||||||
|
"cmd": attr.string(
|
||||||
|
mandatory = True,
|
||||||
|
doc = "The command to execute. It must be in the syntax corresponding to this maprule " +
|
||||||
|
"type, e.g. for `bash_maprule` this must be a Bash command, and for `cmd_maprule` " +
|
||||||
|
"a Windows Command Prompt (cmd.exe) command. Several environment variables are " +
|
||||||
|
"available for this command, storing values like the paths of the input and output " +
|
||||||
|
"files of the action. See the \"Environment Variables\" section for the complete " +
|
||||||
|
"list of environment variables available to this command.",
|
||||||
|
),
|
||||||
|
"foreach_srcs": attr.label_list(
|
||||||
|
allow_files = True,
|
||||||
|
mandatory = True,
|
||||||
|
doc = "The set of sources that will be processed one by one in parallel, to produce " +
|
||||||
|
"the templated outputs. Each of these source files will will be processed " +
|
||||||
|
"individually by its own action.",
|
||||||
|
),
|
||||||
|
"message": attr.string(
|
||||||
|
doc = "A custom progress message to display as the actions are executed.",
|
||||||
|
),
|
||||||
|
"outs_templates": attr.string_dict(
|
||||||
|
allow_empty = False,
|
||||||
|
mandatory = True,
|
||||||
|
doc = "<p>Templates for the output files. Each key defines a name for an output file " +
|
||||||
|
"and the value specifies a path template for that output. For each of the " +
|
||||||
|
"files in `foreach_srcs` this rule creates one action that produces all of " +
|
||||||
|
"these outputs. The paths of the particular output files for that input are " +
|
||||||
|
"computed from the template. The ensure the resolved templates yield unique " +
|
||||||
|
"paths, the following placeholders are supported in the path " +
|
||||||
|
"templates:</p>" +
|
||||||
|
"<ul>" +
|
||||||
|
"<li>\"{src}\": same as \"{src_dir}/{src_name}\"</li>" +
|
||||||
|
"<li>\"{src_dir}\": package path of the source file, and a trailing \"/\"</li>" +
|
||||||
|
"<li>\"{src_name}\": base name of the source file</li>" +
|
||||||
|
"<li>\"{src_name_noext}\": same as \"{src_name}\" without the file extension</li>" +
|
||||||
|
"</ul>" +
|
||||||
|
"<p>You may also add extra path components to the templates, as long as the path " +
|
||||||
|
"template is relative and does not contain uplevel references (\"..\"). " +
|
||||||
|
"Placeholders will be replaced by the values corresponding to the respective " +
|
||||||
|
"input file in `foreach_srcs`. Every output file is generated under " +
|
||||||
|
"<bazel_bin>/path/to/maprule/<maprule_name> + \"_outs/\".</p>",
|
||||||
|
),
|
||||||
|
"tools": attr.label_list(
|
||||||
|
cfg = "host",
|
||||||
|
allow_files = True,
|
||||||
|
doc = "Tools used by the command. The `cmd` attribute, and the values of the " +
|
||||||
|
"`add_env` attribute may reference these tools in \"$(location)\" expressions, " +
|
||||||
|
"similar to the genrule rule.",
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
# Maprule that uses Windows cmd.exe as the interpreter.
|
||||||
|
cmd_maprule = rule(
|
||||||
|
implementation = _cmd_maprule_impl,
|
||||||
|
doc = _rule_doc_template.format(
|
||||||
|
intro = _cmd_maprule_intro,
|
||||||
|
example = _cmd_maprule_example,
|
||||||
|
),
|
||||||
|
attrs = _ATTRS,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Maprule that uses Bash as the interpreter.
|
||||||
|
bash_maprule = rule(
|
||||||
|
implementation = _bash_maprule_impl,
|
||||||
|
doc = _rule_doc_template.format(
|
||||||
|
intro = _bash_maprule_intro,
|
||||||
|
example = _bash_maprule_example,
|
||||||
|
),
|
||||||
|
attrs = _ATTRS,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Only used in unittesting maprule.
|
||||||
|
maprule_testing = struct(
|
||||||
|
cmd_strategy = _CMD_STRATEGY,
|
||||||
|
bash_strategy = _BASH_STRATEGY,
|
||||||
|
src_placeholders = _src_placeholders,
|
||||||
|
validate_attributes = _validate_attributes,
|
||||||
|
is_relative_path = _is_relative_path,
|
||||||
|
custom_envmap = _custom_envmap,
|
||||||
|
create_outputs = _create_outputs,
|
||||||
|
)
|
||||||
|
|
@ -0,0 +1,24 @@
|
||||||
|
# Copyright 2019 The Bazel Authors. All rights reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
"""Maprule implementation in Starlark.
|
||||||
|
|
||||||
|
This module exports:
|
||||||
|
|
||||||
|
This module exports the maprule_testing struct. This should only be used by maprule's own unittests.
|
||||||
|
"""
|
||||||
|
|
||||||
|
load(":maprule_private.bzl", _maprule_testing = "maprule_testing")
|
||||||
|
|
||||||
|
maprule_testing = _maprule_testing
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
load(":collections_tests.bzl", "collections_test_suite")
|
load(":collections_tests.bzl", "collections_test_suite")
|
||||||
load(":dicts_tests.bzl", "dicts_test_suite")
|
load(":dicts_tests.bzl", "dicts_test_suite")
|
||||||
|
load(":maprule_tests.bzl", "maprule_test_suite")
|
||||||
load(":new_sets_tests.bzl", "new_sets_test_suite")
|
load(":new_sets_tests.bzl", "new_sets_test_suite")
|
||||||
load(":partial_tests.bzl", "partial_test_suite")
|
load(":partial_tests.bzl", "partial_test_suite")
|
||||||
load(":paths_tests.bzl", "paths_test_suite")
|
load(":paths_tests.bzl", "paths_test_suite")
|
||||||
|
|
@ -16,6 +17,8 @@ collections_test_suite()
|
||||||
|
|
||||||
dicts_test_suite()
|
dicts_test_suite()
|
||||||
|
|
||||||
|
maprule_test_suite()
|
||||||
|
|
||||||
partial_test_suite()
|
partial_test_suite()
|
||||||
|
|
||||||
paths_test_suite()
|
paths_test_suite()
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,498 @@
|
||||||
|
# Copyright 2018 The Bazel Authors. All rights reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
"""Unit tests for maprule.bzl."""
|
||||||
|
|
||||||
|
load("//lib:unittest.bzl", "asserts", "unittest")
|
||||||
|
load("//rules:maprule_testing.bzl", "maprule_testing")
|
||||||
|
|
||||||
|
def _dummy_generating_action(ctx, path):
|
||||||
|
ctx.actions.write(path, "hello")
|
||||||
|
|
||||||
|
def _mock_file(ctx, path):
|
||||||
|
f = ctx.actions.declare_file(path)
|
||||||
|
_dummy_generating_action(ctx, f)
|
||||||
|
return f
|
||||||
|
|
||||||
|
def _lstrip_until(s, until):
|
||||||
|
return s[s.find(until):]
|
||||||
|
|
||||||
|
def _assert_dict_keys(env, expected, actual, msg):
|
||||||
|
asserts.equals(env, {k: None for k in expected}, {k: None for k in actual}, msg)
|
||||||
|
|
||||||
|
def _assert_ends_with(env, expected_ending, s, msg):
|
||||||
|
if not s.endswith(expected_ending):
|
||||||
|
unittest.fail(env, msg + ": expected \"%s\" to end with \"%s\"" % (s, expected_ending))
|
||||||
|
|
||||||
|
def _assert_no_error(env, errors, msg):
|
||||||
|
if errors:
|
||||||
|
unittest.fail(env, msg + ": expected no errors, got: [%s]" % "\n".join(errors))
|
||||||
|
|
||||||
|
def _assert_error(env, errors, expected_fragment, msg):
|
||||||
|
for e in errors:
|
||||||
|
if expected_fragment in e:
|
||||||
|
return
|
||||||
|
unittest.fail(env, msg + ": did not find \"%s\" in: [%s]" % (expected_fragment, "\n".join(errors)))
|
||||||
|
|
||||||
|
def _contains_substrings_in_order(env, s, substrings):
|
||||||
|
index = 0
|
||||||
|
for ss in substrings:
|
||||||
|
index = s.find(ss, index)
|
||||||
|
if index < 0:
|
||||||
|
return False
|
||||||
|
index += len(ss)
|
||||||
|
return True
|
||||||
|
|
||||||
|
def _assert_error_fragments(env, errors, expected_fragments, msg):
|
||||||
|
for e in errors:
|
||||||
|
if _contains_substrings_in_order(env, e, expected_fragments):
|
||||||
|
return
|
||||||
|
unittest.fail(env, msg + ": did not find expected fragments in \"%s\" in order" % "\n".join(errors))
|
||||||
|
|
||||||
|
def _src_placeholders_test(ctx):
|
||||||
|
env = unittest.begin(ctx)
|
||||||
|
|
||||||
|
for language, strategy in [
|
||||||
|
("cmd", maprule_testing.cmd_strategy),
|
||||||
|
("bash", maprule_testing.bash_strategy),
|
||||||
|
]:
|
||||||
|
for basename, basename_noext in [("bar.txt", "bar"), ("bar.pb.h", "bar.pb")]:
|
||||||
|
actual = maprule_testing.src_placeholders(
|
||||||
|
_mock_file(ctx, language + "/foo/" + basename),
|
||||||
|
strategy,
|
||||||
|
)
|
||||||
|
_assert_dict_keys(
|
||||||
|
env,
|
||||||
|
["src", "src_dir", "src_name", "src_name_noext"],
|
||||||
|
actual,
|
||||||
|
"assertion #1 (language: %s, basename: %s)" % (language, basename),
|
||||||
|
)
|
||||||
|
_assert_ends_with(
|
||||||
|
env,
|
||||||
|
strategy.as_path(language + "/foo/" + basename),
|
||||||
|
actual["src"],
|
||||||
|
"assertion #2 (language: %s, basename: %s)" % (language, basename),
|
||||||
|
)
|
||||||
|
_assert_ends_with(
|
||||||
|
env,
|
||||||
|
strategy.as_path(language + "/foo/"),
|
||||||
|
actual["src_dir"],
|
||||||
|
"assertion #3 (language: %s, basename: %s)" % (language, basename),
|
||||||
|
)
|
||||||
|
asserts.equals(
|
||||||
|
env,
|
||||||
|
basename,
|
||||||
|
actual["src_name"],
|
||||||
|
"assertion #4 (language: %s, basename: %s)" % (language, basename),
|
||||||
|
)
|
||||||
|
asserts.equals(
|
||||||
|
env,
|
||||||
|
basename_noext,
|
||||||
|
actual["src_name_noext"],
|
||||||
|
"assertion #5 (language: %s, basename: %s)" % (language, basename),
|
||||||
|
)
|
||||||
|
|
||||||
|
return unittest.end(env)
|
||||||
|
|
||||||
|
src_placeholders_test = unittest.make(_src_placeholders_test)
|
||||||
|
|
||||||
|
def _validate_attributes_test(ctx):
|
||||||
|
"""Unit tests for maprule_testing.validate_attributes."""
|
||||||
|
env = unittest.begin(ctx)
|
||||||
|
_assert_no_error(
|
||||||
|
env,
|
||||||
|
maprule_testing.validate_attributes({"FOO": "bar"}, {"BAR": "value1"}),
|
||||||
|
"assertion #1",
|
||||||
|
)
|
||||||
|
_assert_no_error(
|
||||||
|
env,
|
||||||
|
maprule_testing.validate_attributes({"FOO": "bar"}, {}),
|
||||||
|
"assertion #2",
|
||||||
|
)
|
||||||
|
|
||||||
|
_assert_error(
|
||||||
|
env,
|
||||||
|
maprule_testing.validate_attributes({}, {}),
|
||||||
|
"\"outs_templates\" must not be empty",
|
||||||
|
"assertion #3",
|
||||||
|
)
|
||||||
|
_assert_error(
|
||||||
|
env,
|
||||||
|
maprule_testing.validate_attributes({"": "foo"}, {}),
|
||||||
|
"name should not be empty",
|
||||||
|
"assertion #4",
|
||||||
|
)
|
||||||
|
_assert_error(
|
||||||
|
env,
|
||||||
|
maprule_testing.validate_attributes({"foo": "bar"}, {}),
|
||||||
|
"name should be all upper-case",
|
||||||
|
"assertion #5",
|
||||||
|
)
|
||||||
|
_assert_error(
|
||||||
|
env,
|
||||||
|
maprule_testing.validate_attributes({"SRC": "bar"}, {}),
|
||||||
|
"conflicting with the environment variable of the source file",
|
||||||
|
"assertion #6",
|
||||||
|
)
|
||||||
|
|
||||||
|
_assert_error(
|
||||||
|
env,
|
||||||
|
maprule_testing.validate_attributes({"FOO": ""}, {}),
|
||||||
|
"output path should not be empty",
|
||||||
|
"assertion #7",
|
||||||
|
)
|
||||||
|
_assert_error(
|
||||||
|
env,
|
||||||
|
maprule_testing.validate_attributes({"FOO": "/usr/bin"}, {}),
|
||||||
|
"output path should be relative",
|
||||||
|
"assertion #8",
|
||||||
|
)
|
||||||
|
_assert_error(
|
||||||
|
env,
|
||||||
|
maprule_testing.validate_attributes({"FOO": "c:/usr/bin"}, {}),
|
||||||
|
"output path should be relative",
|
||||||
|
"assertion #9",
|
||||||
|
)
|
||||||
|
_assert_error(
|
||||||
|
env,
|
||||||
|
maprule_testing.validate_attributes({"FOO": "../foo"}, {}),
|
||||||
|
"output path should not contain uplevel references",
|
||||||
|
"assertion #10",
|
||||||
|
)
|
||||||
|
_assert_no_error(
|
||||||
|
env,
|
||||||
|
maprule_testing.validate_attributes({"FOO": "./foo"}, {}),
|
||||||
|
"assertion #11",
|
||||||
|
)
|
||||||
|
_assert_error(
|
||||||
|
env,
|
||||||
|
maprule_testing.validate_attributes({"BAR": "foo", "FOO": "foo"}, {}),
|
||||||
|
"output path is already used for \"BAR\"",
|
||||||
|
"assertion #12",
|
||||||
|
)
|
||||||
|
|
||||||
|
_assert_error(
|
||||||
|
env,
|
||||||
|
maprule_testing.validate_attributes({"FOO": "bar"}, {"": "baz"}),
|
||||||
|
"name should not be empty",
|
||||||
|
"assertion #13",
|
||||||
|
)
|
||||||
|
_assert_error(
|
||||||
|
env,
|
||||||
|
maprule_testing.validate_attributes({"FOO": "bar"}, {"Bar": "baz"}),
|
||||||
|
"name should be all upper-case",
|
||||||
|
"assertion #14",
|
||||||
|
)
|
||||||
|
_assert_error(
|
||||||
|
env,
|
||||||
|
maprule_testing.validate_attributes({"FOO": "bar"}, {"FOO": "baz"}),
|
||||||
|
"conflicting with the environment variable of the \"FOO\" output file",
|
||||||
|
"assertion #15",
|
||||||
|
)
|
||||||
|
|
||||||
|
_assert_error(
|
||||||
|
env,
|
||||||
|
maprule_testing.validate_attributes({"FOO": "bar"}, {"BAR": "$(location x) $(location y)"}),
|
||||||
|
"use only one $(location)",
|
||||||
|
"assertion #16",
|
||||||
|
)
|
||||||
|
_assert_error(
|
||||||
|
env,
|
||||||
|
maprule_testing.validate_attributes({"FOO": "bar"}, {"BAR": "a $(location b"}),
|
||||||
|
"missing closing parenthesis",
|
||||||
|
"assertion #17",
|
||||||
|
)
|
||||||
|
|
||||||
|
return unittest.end(env)
|
||||||
|
|
||||||
|
validate_attributes_test = unittest.make(_validate_attributes_test)
|
||||||
|
|
||||||
|
def _as_path_test(ctx):
|
||||||
|
"""Unit tests for maprule_testing.as_path."""
|
||||||
|
env = unittest.begin(ctx)
|
||||||
|
asserts.equals(
|
||||||
|
env,
|
||||||
|
"Foo\\Bar\\Baz\\Qux",
|
||||||
|
maprule_testing.cmd_strategy.as_path("Foo/Bar/Baz\\Qux"),
|
||||||
|
msg = "assertion #1",
|
||||||
|
)
|
||||||
|
asserts.equals(
|
||||||
|
env,
|
||||||
|
"Foo/Bar/Baz\\Qux",
|
||||||
|
maprule_testing.bash_strategy.as_path("Foo/Bar/Baz\\Qux"),
|
||||||
|
msg = "assertion #2",
|
||||||
|
)
|
||||||
|
return unittest.end(env)
|
||||||
|
|
||||||
|
as_path_test = unittest.make(_as_path_test)
|
||||||
|
|
||||||
|
def _assert_relative_path(env, path, index):
|
||||||
|
asserts.true(
|
||||||
|
env,
|
||||||
|
maprule_testing.is_relative_path(path),
|
||||||
|
msg = "assertion #%d" % index,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _assert_not_relative_path(env, path, index):
|
||||||
|
asserts.false(
|
||||||
|
env,
|
||||||
|
maprule_testing.is_relative_path(path),
|
||||||
|
msg = "assertion #%d" % index,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _is_relative_path_test(ctx):
|
||||||
|
"""Unit tests for maprule_testing.is_relative_path."""
|
||||||
|
env = unittest.begin(ctx)
|
||||||
|
_assert_relative_path(env, "Foo/Bar/Baz", 1)
|
||||||
|
_assert_relative_path(env, "Foo\\Bar\\Baz", 2)
|
||||||
|
_assert_relative_path(env, "Foo/Bar\\Baz", 3)
|
||||||
|
_assert_not_relative_path(env, "d:/Foo/Bar", 4)
|
||||||
|
_assert_not_relative_path(env, "D:/Foo/Bar", 5)
|
||||||
|
_assert_not_relative_path(env, "/Foo/Bar", 6)
|
||||||
|
_assert_not_relative_path(env, "\\Foo\\Bar", 7)
|
||||||
|
return unittest.end(env)
|
||||||
|
|
||||||
|
is_relative_path_test = unittest.make(_is_relative_path_test)
|
||||||
|
|
||||||
|
def _custom_envmap_test(ctx):
|
||||||
|
"""Unit tests for maprule_testing.custom_envmap."""
|
||||||
|
env = unittest.begin(ctx)
|
||||||
|
|
||||||
|
actual = {}
|
||||||
|
|
||||||
|
for language, strategy in [
|
||||||
|
("cmd", maprule_testing.cmd_strategy),
|
||||||
|
("bash", maprule_testing.bash_strategy),
|
||||||
|
]:
|
||||||
|
actual[language] = maprule_testing.custom_envmap(
|
||||||
|
ctx,
|
||||||
|
strategy,
|
||||||
|
src_placeholders = {"src_ph1": "Src/Ph1-value", "src_ph2": "Src/Ph2-value"},
|
||||||
|
outs_dict = {
|
||||||
|
"out1": _mock_file(ctx, language + "/Foo/Out1"),
|
||||||
|
"out2": _mock_file(ctx, language + "/Foo/Out2"),
|
||||||
|
},
|
||||||
|
add_env = {"ENV1": "Env1"},
|
||||||
|
)
|
||||||
|
_assert_dict_keys(
|
||||||
|
env,
|
||||||
|
["MAPRULE_SRC_PH1", "MAPRULE_SRC_PH2", "MAPRULE_OUT1", "MAPRULE_OUT2", "MAPRULE_ENV1"],
|
||||||
|
actual[language],
|
||||||
|
msg = "assertion #1 (language: %s)" % language,
|
||||||
|
)
|
||||||
|
actual[language]["MAPRULE_OUT1"] = _lstrip_until(actual[language]["MAPRULE_OUT1"], "Foo")
|
||||||
|
actual[language]["MAPRULE_OUT2"] = _lstrip_until(actual[language]["MAPRULE_OUT2"], "Foo")
|
||||||
|
|
||||||
|
asserts.equals(
|
||||||
|
env,
|
||||||
|
{
|
||||||
|
"MAPRULE_ENV1": "Env1",
|
||||||
|
"MAPRULE_OUT1": "Foo\\Out1",
|
||||||
|
"MAPRULE_OUT2": "Foo\\Out2",
|
||||||
|
"MAPRULE_SRC_PH1": "Src\\Ph1-value",
|
||||||
|
"MAPRULE_SRC_PH2": "Src\\Ph2-value",
|
||||||
|
},
|
||||||
|
actual["cmd"],
|
||||||
|
msg = "assertion #2",
|
||||||
|
)
|
||||||
|
|
||||||
|
asserts.equals(
|
||||||
|
env,
|
||||||
|
{
|
||||||
|
"MAPRULE_ENV1": "Env1",
|
||||||
|
"MAPRULE_OUT1": "Foo/Out1",
|
||||||
|
"MAPRULE_OUT2": "Foo/Out2",
|
||||||
|
"MAPRULE_SRC_PH1": "Src/Ph1-value",
|
||||||
|
"MAPRULE_SRC_PH2": "Src/Ph2-value",
|
||||||
|
},
|
||||||
|
actual["bash"],
|
||||||
|
msg = "assertion #3",
|
||||||
|
)
|
||||||
|
|
||||||
|
return unittest.end(env)
|
||||||
|
|
||||||
|
custom_envmap_test = unittest.make(_custom_envmap_test)
|
||||||
|
|
||||||
|
def _create_outputs_test(ctx):
|
||||||
|
"""Unit tests for maprule_testing.create_outputs."""
|
||||||
|
env = unittest.begin(ctx)
|
||||||
|
|
||||||
|
for language, strategy in [
|
||||||
|
("cmd", maprule_testing.cmd_strategy),
|
||||||
|
("bash", maprule_testing.bash_strategy),
|
||||||
|
]:
|
||||||
|
src1 = _mock_file(ctx, language + "/foo/src1.txt")
|
||||||
|
src2 = _mock_file(ctx, language + "/foo/src2.pb.h")
|
||||||
|
src3 = _mock_file(ctx, language + "/bar/src1.txt")
|
||||||
|
foreach_srcs = [src1, src2, src3]
|
||||||
|
|
||||||
|
outs_dicts, all_output_files, src_placeholders_dicts, errors = (
|
||||||
|
maprule_testing.create_outputs(
|
||||||
|
ctx,
|
||||||
|
"my_maprule",
|
||||||
|
{
|
||||||
|
"OUT1": "{src}.out1",
|
||||||
|
"OUT2": "{src_dir}/out2/{src_name_noext}.out2",
|
||||||
|
},
|
||||||
|
strategy,
|
||||||
|
foreach_srcs,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
_assert_no_error(env, errors, "assertion #1 (language: %s)" % language)
|
||||||
|
|
||||||
|
for output in all_output_files:
|
||||||
|
_dummy_generating_action(ctx, output)
|
||||||
|
|
||||||
|
_assert_dict_keys(
|
||||||
|
env,
|
||||||
|
foreach_srcs,
|
||||||
|
outs_dicts,
|
||||||
|
"assertion #2 (language: %s)" % language,
|
||||||
|
)
|
||||||
|
for src in foreach_srcs:
|
||||||
|
_assert_dict_keys(
|
||||||
|
env,
|
||||||
|
["OUT1", "OUT2"],
|
||||||
|
outs_dicts[src],
|
||||||
|
"assertion #3 (language: %s, src: %s)" % (language, src),
|
||||||
|
)
|
||||||
|
|
||||||
|
_assert_ends_with(
|
||||||
|
env,
|
||||||
|
"my_maprule_out/tests/%s/foo/src1.txt.out1" % language,
|
||||||
|
outs_dicts[src1]["OUT1"].path,
|
||||||
|
"assertion #4 (language: %s)" % language,
|
||||||
|
)
|
||||||
|
_assert_ends_with(
|
||||||
|
env,
|
||||||
|
"my_maprule_out/tests/%s/foo/out2/src1.out2" % language,
|
||||||
|
outs_dicts[src1]["OUT2"].path,
|
||||||
|
"assertion #5 (language: %s)" % language,
|
||||||
|
)
|
||||||
|
|
||||||
|
_assert_ends_with(
|
||||||
|
env,
|
||||||
|
"my_maprule_out/tests/%s/foo/src2.pb.h.out1" % language,
|
||||||
|
outs_dicts[src2]["OUT1"].path,
|
||||||
|
"assertion #6 (language: %s)" % language,
|
||||||
|
)
|
||||||
|
_assert_ends_with(
|
||||||
|
env,
|
||||||
|
"my_maprule_out/tests/%s/foo/out2/src2.pb.out2" % language,
|
||||||
|
outs_dicts[src2]["OUT2"].path,
|
||||||
|
"assertion #7 (language: %s)" % language,
|
||||||
|
)
|
||||||
|
|
||||||
|
_assert_ends_with(
|
||||||
|
env,
|
||||||
|
"my_maprule_out/tests/%s/bar/src1.txt.out1" % language,
|
||||||
|
outs_dicts[src3]["OUT1"].path,
|
||||||
|
"assertion #8 (language: %s)" % language,
|
||||||
|
)
|
||||||
|
_assert_ends_with(
|
||||||
|
env,
|
||||||
|
"my_maprule_out/tests/%s/bar/out2/src1.out2" % language,
|
||||||
|
outs_dicts[src3]["OUT2"].path,
|
||||||
|
"assertion #9 (language: %s)" % language,
|
||||||
|
)
|
||||||
|
|
||||||
|
expected = [
|
||||||
|
"my_maprule_out/tests/%s/foo/src1.txt.out1" % language,
|
||||||
|
"my_maprule_out/tests/%s/foo/out2/src1.out2" % language,
|
||||||
|
"my_maprule_out/tests/%s/foo/src2.pb.h.out1" % language,
|
||||||
|
"my_maprule_out/tests/%s/foo/out2/src2.pb.out2" % language,
|
||||||
|
"my_maprule_out/tests/%s/bar/src1.txt.out1" % language,
|
||||||
|
"my_maprule_out/tests/%s/bar/out2/src1.out2" % language,
|
||||||
|
]
|
||||||
|
for i in range(0, len(all_output_files)):
|
||||||
|
actual = _lstrip_until(all_output_files[i].path, "my_maprule_out")
|
||||||
|
asserts.equals(
|
||||||
|
env,
|
||||||
|
expected[i],
|
||||||
|
actual,
|
||||||
|
"assertion #10 (language: %s, index: %d)" % (language, i),
|
||||||
|
)
|
||||||
|
|
||||||
|
return unittest.end(env)
|
||||||
|
|
||||||
|
create_outputs_test = unittest.make(_create_outputs_test)
|
||||||
|
|
||||||
|
def _conflicting_outputs_test(ctx):
|
||||||
|
"""Unit tests for maprule_testing.create_outputs catching conflicting outputs."""
|
||||||
|
env = unittest.begin(ctx)
|
||||||
|
|
||||||
|
for language, strategy in [
|
||||||
|
("cmd", maprule_testing.cmd_strategy),
|
||||||
|
("bash", maprule_testing.bash_strategy),
|
||||||
|
]:
|
||||||
|
src1 = _mock_file(ctx, language + "/foo/src1.txt")
|
||||||
|
src2 = _mock_file(ctx, language + "/foo/src2.pb.h")
|
||||||
|
src3 = _mock_file(ctx, language + "/bar/src1.txt")
|
||||||
|
foreach_srcs = [src1, src2, src3]
|
||||||
|
|
||||||
|
_, all_output_files, _, errors = (
|
||||||
|
maprule_testing.create_outputs(
|
||||||
|
ctx,
|
||||||
|
"my_maprule",
|
||||||
|
{
|
||||||
|
"OUT1": "out1", # 3 conflicts
|
||||||
|
"OUT2": "{src_dir}/out2", # 2 conflicts
|
||||||
|
"OUT3": "out3/{src_name}", # 2 conflicts
|
||||||
|
},
|
||||||
|
strategy,
|
||||||
|
foreach_srcs,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
for output in all_output_files:
|
||||||
|
_dummy_generating_action(ctx, output)
|
||||||
|
|
||||||
|
_assert_error_fragments(
|
||||||
|
env,
|
||||||
|
errors,
|
||||||
|
["out1", language + "/foo/src1.txt", "OUT1", language + "/foo/src2.pb.h", "OUT1"],
|
||||||
|
msg = "assertion #1 (language: %s)" % language,
|
||||||
|
)
|
||||||
|
|
||||||
|
_assert_error_fragments(
|
||||||
|
env,
|
||||||
|
errors,
|
||||||
|
["out2", language + "/foo/src1.txt", "OUT2", language + "/foo/src2.pb.h", "OUT2"],
|
||||||
|
msg = "assertion #2 (language: %s)" % language,
|
||||||
|
)
|
||||||
|
|
||||||
|
_assert_error_fragments(
|
||||||
|
env,
|
||||||
|
errors,
|
||||||
|
["out3/src1.txt", language + "/foo/src1.txt", "OUT3", language + "/bar/src1.txt", "OUT3"],
|
||||||
|
msg = "assertion #5 (language: %s)" % language,
|
||||||
|
)
|
||||||
|
|
||||||
|
return unittest.end(env)
|
||||||
|
|
||||||
|
conflicting_outputs_test = unittest.make(_conflicting_outputs_test)
|
||||||
|
|
||||||
|
def maprule_test_suite():
|
||||||
|
"""Creates the test targets and test suite for maprule.bzl tests."""
|
||||||
|
|
||||||
|
unittest.suite(
|
||||||
|
"maprule_tests",
|
||||||
|
src_placeholders_test,
|
||||||
|
validate_attributes_test,
|
||||||
|
as_path_test,
|
||||||
|
is_relative_path_test,
|
||||||
|
custom_envmap_test,
|
||||||
|
conflicting_outputs_test,
|
||||||
|
)
|
||||||
Loading…
Reference in New Issue