diff --git a/README.md b/README.md index 97a9e5f..fe7c869 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,8 @@ Each of the `.bzl` files in the `lib` directory defines a "module"—a `struct` that contains a set of related functions and/or other symbols that can be loaded as a single unit, for convenience. +Skylib also provides build rules under the `rules` directory. + ## Getting Started ### `WORKSPACE` file @@ -68,6 +70,10 @@ s = shell.quote(p) * [unittest](lib/unittest.bzl) * [versions](lib/versions.bzl) +## List of rules (in rules/) + +* [`cmd_maprule` and `bash_maprule`](lib/maprule.bzl) + ## Writing a new module Steps to add a module to Skylib: diff --git a/rules/BUILD b/rules/BUILD new file mode 100644 index 0000000..4f98c5d --- /dev/null +++ b/rules/BUILD @@ -0,0 +1,20 @@ +load("//:bzl_library.bzl", "bzl_library") + +licenses(["notice"]) + +bzl_library( + name = "maprule", + srcs = ["maprule.bzl"], + visibility = ["//visibility:public"], + deps = [":maprule_private"], +) + +bzl_library( + name = "maprule_private", + srcs = ["maprule_private.bzl"], + visibility = ["//visibility:private"], + deps = [ + "//lib:dicts", + "//lib:paths", + ], +) diff --git a/rules/maprule.bzl b/rules/maprule.bzl new file mode 100644 index 0000000..3f7b524 --- /dev/null +++ b/rules/maprule.bzl @@ -0,0 +1,28 @@ +# Copyright 2019 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Maprule implementation in Starlark. + +This module exports the cmd_maprule() and bash_maprule() build rules. + +They are the same except for the interpreter they use (cmd.exe and Bash respectively) and for the +expected language of their `cmd` attribute. + +You can read more about these rules in "maprule_private.bzl". +""" + +load(":maprule_private.bzl", _bash_maprule = "bash_maprule", _cmd_maprule = "cmd_maprule") + +cmd_maprule = _cmd_maprule +bash_maprule = _bash_maprule diff --git a/rules/maprule_private.bzl b/rules/maprule_private.bzl new file mode 100644 index 0000000..007bfa5 --- /dev/null +++ b/rules/maprule_private.bzl @@ -0,0 +1,675 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Maprule implementation in Starlark. + +This module exports: + +- The cmd_maprule() and bash_maprule() build rules. + They are the same except for the interpreter they use (cmd.exe and Bash respectively) and for the + expected language of their `cmd` attribute. We will refer to them collectively as `maprule`. + +- The maprule_testing struct. This should only be used by maprule's own unittests. +""" + +load("//lib:dicts.bzl", "dicts") +load("//lib:paths.bzl", "paths") + +_cmd_maprule_intro = """ +Maprule that runs a Windows Command Prompt (`cmd.exe`) command. + +This rule is the same as `bash_maprule`, but uses `cmd.exe` instead of Bash, therefore the `cmd` +attribute must use Command Prompt syntax. `cmd_maprule` rules can only be built on Windows. +""" + +_bash_maprule_intro = """ +Maprule that runs a Bash command. + +This rule is the same as `cmd_maprule` except this one uses Bash to run the command, therefore the +`cmd` attribute must use Bash syntax. `bash_maprule` rules can only be built if Bash is installed on +the build machine. +""" + +_cmd_maprule_example = """ + # This file is //projects/game:BUILD + + load("//tools/build_rules:maprule.bzl", "cmd_maprule") + + cmd_maprule( + name = "process_assets", + foreach_srcs = [ + "rust.png", + "teapot.3ds", + "//assets:wood.jpg", + "//assets:models", + ], + outs_templates = { + "TAGS": "{src}.tags", + "DIGEST": "digests/{src_name_noext}.md5", + }, + tools = [ + "//bin:asset-tagger", + "//util:md5sum", + ], + add_env = { + "ASSET_TAGGER": "$(location //bin:asset-tagger)", + "MD5SUM": "$(location //util:md5sum)", + }, + + # Note: this command should live in a script file, we only inline it in the `cmd` attribute + # for the sake of demonstration. See Tips and Tricks section. + cmd = "%MAPRULE_ASSET_TAGGER% --input=%MAPRULE_SRC% --output=%MAPRULE_TAGS% & " + + 'IF /I "%ERRORLEVEL%" EQU "0" ( %MAPRULE_MD5SUM% %MAPRULE_SRC% > %MAPRULE_DIGEST% )', + ) +""" + +_bash_maprule_example = """ + # This file is //projects/game:BUILD + + load("//tools/build_rules:maprule.bzl", "bash_maprule") + + bash_maprule( + name = "process_assets", + foreach_srcs = [ + "rust.png", + "teapot.3ds", + "//assets:wood.jpg", + "//assets:models", + ], + outs_templates = { + "TAGS": "{src}.tags", + "DIGEST": "digests/{src_name_noext}.md5", + }, + tools = [ + "//bin:asset-tagger", + "//util:md5sum", + ], + add_env = { + "ASSET_TAGGER": "$(location //bin:asset-tagger)", + "MD5SUM": "$(location //util:md5sum)", + }, + + # Note: this command should live in a script file, we only inline it in the `cmd` attribute + # for the sake of demonstration. See Tips and Tricks section. + cmd = '"$MAPRULE_ASSET_TAGGER" --input="$MAPRULE_SRC" --output="$MAPRULE_TAGS" && ' + + '"$MAPRULE_MD5SUM" "$MAPRULE_SRC" > "$MAPRULE_DIGEST"', + ) +""" + +_rule_doc_template = """ +{intro} + +Maprule runs a specific command for each of the "foreach" source files. This allows processing +source files in parallel, to produce some outputs for each of them. + +The name "maprule" indicates that this rule can be used to map source files to output files, and is +also a reference to "genrule" that inspired this rule's design (though there are significant +differences). + +Below you will find an Example, Tips and Tricks, and an FAQ. + +### Example + +{example} + +The "process_assets" rule above will execute the command in the `cmd` to process "rust.png", +"teapot.3ds", "//assets:wood.jpg", and every file in the "//assets:models" rule, producing the +corresponding .tags and .md5 files for each of them, under the following paths: + + bazel-bin/projects/game/process_assets_out/projects/game/rust.png.tags + bazel-bin/projects/game/process_assets_out/digests/rust.md5 + bazel-bin/projects/game/process_assets_out/projects/game/teapot.3ds.tags + bazel-bin/projects/game/process_assets_out/digests/teapot.md5 + bazel-bin/projects/game/process_assets_out/assets/wood.jpg.tags + bazel-bin/projects/game/process_assets_out/digests/wood.md5 + ... + +You can create downstream rules, for example a filegroup or genrule (or another maprule) that put +this rule in their `srcs` attribute and get all the .tags and .md5 files. + +## Tips and Tricks + +*(The Tips and Tricks section is the same for `cmd_maprule` and `bash_maprule`.)* + +### Use script files instead of inlining commands in the `cmd` attribute. + +Unless the command is trivial, don't try to write it in `cmd`. + +Properly quoting parts of the command is challenging enough, add to that escaping for the BUILD +file's syntax and the `cmd` attribute quickly gets unmaintainably complex. + +It's a lot easier and maintainable to create a script file such as "foo.sh" in `bash_maprule` or +"foo.bat" in `cmd_maprule` for the commands. To do that: + +1. move the commands to a script file +2. add the script file to the `tools` attribute +3. add an entry to the `add_env` attribute, e.g. "`TOOL: "$(location :tool.sh)"`" +4. replace the `cmd` with just "$MAPRULE_FOO" (in `bash_maprule`) or "%MAPRULE_FOO%" (in + `cmd_maprule`). + +Doing this also avoids hitting command line length limits. + +Example: + + cmd_maprule( + ... + srcs = ["//data:temperatures.txt"], + tools = [ + "command.bat", + ":weather-stats", + ], + add_env = {{ + "COMMAND": "$(location :command.bat)", + "STATS_TOOL": "$(location :weather-stats-computer)", + "TEMPERATURES": "$(location //data:temperatures.txt)", + }}, + cmd = "%MAPRULE_COMMAND%", + ) + +### Use the `add_env` attribute to pass tool locations to the command. + +Entries in the `add_env` attribute may use "$(location)" references and may also use the same +placeholders as the `outs_templates` attribute. For example you can use this mechanism to pass extra +"$(location)" references of `tools` or `srcs` to the actions. + +Example: + + cmd_maprule( + ... + foreach_srcs = [...], + outs_templates = {{"STATS_OUT": "{{src}}-stats.txt"}}, + srcs = ["//data:temperatures.txt"], + tools = [":weather-stats"], + add_env = {{ + "STATS_TOOL": "$(location :weather-stats-computer)", + "TEMPERATURES": "$(location //data:temperatures.txt)", + }}, + cmd = "%MAPRULE_STATS_TOOL% --src=%MAPRULE_SRC% --data=%MAPRULE_TEMPERATURES% > %MAPRULE_STATS_OUT%", + ) + + cc_binary( + name = "weather-stats", + ... + ) + +## Environment Variables + +*(The Environment Variables section is the same for `cmd_maprule` and `bash_maprule`.)* + +The rule defines several environment variables available to the command may reference. All of these +envvars names start with "MAPRULE_". You can add your own envvars with the `add_env` attribute. + +The command can use some envvars, all named "MAPRULE_*<something>*". + +The complete list of environment variables is: + +- "MAPRULE_SRC": the path of the current file from `foreach_srcs` +- "MAPRULE_SRCS": the space-separated paths of all files in the `srcs` attribute +- "MAPRULE_*<OUT>*": for each key name *<OUT>* in the `outs_templates` attribute, this + is the path of the respective output file for the current source +- "MAPRULE_*<ENV>*": for each key name *<ENV>* in the `add_env` attribute + +## FAQ + +*(The FAQ section is the same for `cmd_maprule` and `bash_maprule`.)* + +### What's the motivation for maprule? What's wrong with genrule? + +genrule creates a single action for all of its inputs. It requires specifying all output files. +Finally, it can only create Bash commands. + +Maprule supports parallel processing of its inputs and doesn't require specifying all outputs, just +templates for the outputs. And not only Bash is supported (via `bash_maprule`) but so are +cmd.exe-style commands via `cmd_maprule`. + +### `genrule.cmd` supports "$(location)" expressions, how come `*_maprule.cmd` doesn't? + +Maprule deliberately omits support for this feature to avoid pitfalls with quoting and escaping, and +potential issues with paths containing spaces. Instead, maprule exports environment variables for +the input and outputs of the action, and allows the user to define extra envvars. These extra +envvars do support "$(location)" expressions, so you can pass paths of labels in `srcs` and `tools`. + +### Why are all envvars exported with the "MAPRULE_" prefix? + +To avoid conflicts with other envvars, whose names could also be attractive outs_templates names. + +### Why do `outs_templates` and `add_env` keys have to be uppercase? + +Because they are exported as all-uppercase envvars, so requiring that they be declared as uppercase +gives a visual cue of that. It also avoids clashes resulting from mixed lower-upper-case names like +"foo" and "Foo". + +### Why don't `outs_templates` and `add_env` keys have to start with "MAPRULE_" even though they are exported as such? + +For convenience. It seems to bring no benefit to have the user always type "MAPRULE_" in front of +the name when the rule itself could as well add it. + +### Why are all outputs relative to "*<maprule_pkg>*/*<maprule_name>*_out/" ? + +To ensure that maprules in the same package and with the same outs_templates produce distinct output +files. + +### Why aren't `outs_templates` keys available as placeholders in the values of `add_env`? + +Because `add_env` is meant to be used for passing extra "$(location)" information to the action, and +the output paths are already available as envvars for the action. +""" + +def _is_relative_path(p): + """Returns True if `p` is a relative path (considering Unix and Windows semantics).""" + return p and p[0] != "/" and p[0] != "\\" and ( + len(p) < 2 or not p[0].isalpha() or p[1] != ":" + ) + +def _validate_attributes(ctx_attr_outs_templates, ctx_attr_add_env): + """Validates rule attributes and returns a list of error messages if there were errors.""" + errors = [] + + envvars = { + "MAPRULE_SRC": "the source file", + "MAPRULE_SRCS": "the space-joined paths of the common sources", + } + + if not ctx_attr_outs_templates: + errors.append("ERROR: \"outs_templates\" must not be empty") + + names_to_paths = {} + paths_to_names = {} + + # Check entries in "outs_templates". + for name, path in ctx_attr_outs_templates.items(): + # Check the entry's name. + envvar_for_name = "MAPRULE_" + name.upper() + error_prefix = "ERROR: In \"outs_templates\" entry {\"%s\": \"%s\"}: " % (name, path) + if not name: + errors.append("ERROR: Bad entry in the \"outs_templates\" attribute: the name " + + "should not be empty.") + elif name.upper() != name: + errors.append(error_prefix + "the name should be all upper-case.") + elif envvar_for_name in envvars: + errors.append((error_prefix + + "please rename it, otherwise this output path would be exported " + + "as the environment variable \"%s\", conflicting with the " + + "environment variable of %s.") % ( + envvar_for_name, + envvars[envvar_for_name], + )) + elif not path: + errors.append(error_prefix + "output path should not be empty.") + elif not _is_relative_path(path): + errors.append(error_prefix + "output path should be relative.") + elif ".." in path: + errors.append(error_prefix + "output path should not contain uplevel references.") + elif path in paths_to_names: + errors.append(error_prefix + + "output path is already used for \"%s\"." % paths_to_names[path]) + envvars[envvar_for_name] = "the \"%s\" output file declared in the \"outs_templates\" attribute" % name + names_to_paths[name] = path + paths_to_names[path] = name + + # Check envvar names in "add_env". + for name, value in ctx_attr_add_env.items(): + envvar_for_name = "MAPRULE_" + name.upper() + error_prefix = "ERROR: In \"add_env\" entry {\"%s\": \"%s\"}: " % (name, value) + if not name: + errors.append("ERROR: Bad entry in the \"add_env\" attribute: the name should not be " + + "empty.") + elif name.upper() != name: + errors.append(error_prefix + "the name should be all upper-case.") + elif envvar_for_name in envvars: + errors.append((error_prefix + + "please rename it, otherwise it would be exported as \"%s\", " + + "conflicting with the environment variable of %s.") % ( + envvar_for_name, + envvars[envvar_for_name], + )) + elif "$(location" in value: + tokens = value.split("$(location") + if len(tokens) != 2: + errors.append(error_prefix + "use only one $(location) or $(locations) function.") + elif ")" not in tokens[1]: + errors.append(error_prefix + + "incomplete $(location) or $(locations) function, missing closing " + + "parenthesis") + envvars[name] = "an additional environment declared in \"add_env\" as \"%s\"" % name + + return errors + +def _src_placeholders(src, strategy): + return { + "src": strategy.as_path(src.short_path), + "src_dir": strategy.as_path(paths.dirname(src.short_path) + "/"), + "src_name": src.basename, + "src_name_noext": (src.basename[:-len(src.extension) - 1] if len(src.extension) else src.basename), + } + +def _create_outputs(ctx, ctx_label_name, ctx_attr_outs_templates, strategy, foreach_srcs): + errors = [] + outs_dicts = {} + output_generated_by = {} + all_output_files = [] + src_placeholders_dicts = {} + output_path_prefix = ctx_label_name + "_out/" + for src in foreach_srcs: + src_placeholders_dicts[src] = _src_placeholders(src, strategy) + + outputs_for_src = {} + for template_name, path in ctx_attr_outs_templates.items(): + output_path = path.format(**src_placeholders_dicts[src]) + if output_path in output_generated_by: + existing_generator = output_generated_by[output_path] + errors.append("\n".join([ + "ERROR: output file generated multiple times:", + " output file: " + output_path, + " foreach_srcs file 1: " + existing_generator[0].short_path, + " outs_templates entry 1: " + existing_generator[1], + " foreach_srcs file 2: " + src.short_path, + " outs_templates entry 2: " + template_name, + ])) + output_generated_by[output_path] = (src, template_name) + output = ctx.actions.declare_file(output_path_prefix + output_path) + outputs_for_src[template_name] = output + all_output_files.append(output) + outs_dicts[src] = outputs_for_src + + if errors: + # For sake of Starlark unittest we return all_output_files, so the test can create dummy + # generating actions for the files even in case of errors. + return None, all_output_files, None, errors + else: + return outs_dicts, all_output_files, src_placeholders_dicts, None + +def _resolve_locations(ctx, strategy, ctx_attr_add_env, ctx_attr_tools): + # ctx.resolve_command returns a Bash command. All we need though is the inputs and runfiles + # manifests (we expand $(location) references with ctx.expand_location below), so ignore the + # tuple's middle element (the resolved command). + inputs_from_tools, _, manifests_from_tools = ctx.resolve_command( + # Pretend that the additional envvars are forming a command, so resolve_command will resolve + # $(location) references in them (which we ignore here) and add their inputs and manifests + # to the results (which we really care about). + command = " ".join(ctx_attr_add_env.values()), + tools = ctx_attr_tools, + ) + + errors = [] + location_expressions = [] + parts = {} + was_anything_to_resolve = False + for k, v in ctx_attr_add_env.items(): + # Look for "$(location ...)" or "$(locations ...)", resolve if found. + # _validate_attributes already ensured that there's at most one $(location/s ...) in "v". + if "$(location" in v: + tokens = v.split("$(location") + was_anything_to_resolve = True + closing_paren = tokens[1].find(")") + location_expressions.append("$(location" + tokens[1][:closing_paren + 1]) + parts[k] = (tokens[0], tokens[1][closing_paren + 1:]) + else: + location_expressions.append("") + + if errors: + return None, None, None, errors + + resolved_add_env = {} + if was_anything_to_resolve: + # Resolve all $(location) expressions in one go. Should be faster than resolving them + # one-by-one. + all_location_expressions = "".join(location_expressions) + all_resolved_locations = ctx.expand_location(all_location_expressions) + resolved_locations = strategy.as_path(all_resolved_locations).split("") + + i = 0 + + # Starlark dictionaries have a deterministic order of iteration, so the element order in + # "resolved_locations" matches the order in "location_expressions", i.e. the previous + # iteration order of "ctx_attr_add_env". + for k, v in ctx_attr_add_env.items(): + if location_expressions[i]: + head, tail = parts[k] + resolved_add_env[k] = head + resolved_locations[i] + tail + else: + resolved_add_env[k] = v + i += 1 + else: + resolved_add_env = ctx_attr_add_env + + if errors: + return None, None, None, errors + else: + return inputs_from_tools, manifests_from_tools, resolved_add_env, None + +def _custom_envmap(ctx, strategy, src_placeholders, outs_dict, add_env): + return dicts.add( + { + "MAPRULE_" + k.upper(): strategy.as_path(v) + for k, v in src_placeholders.items() + }, + { + "MAPRULE_" + k.upper(): strategy.as_path(v.path) + for k, v in outs_dict.items() + }, + { + "MAPRULE_" + k.upper(): strategy.as_path(ctx.expand_location(v)).format(**src_placeholders) + for k, v in add_env.items() + }, + ) + +def _fail_if_errors(errors): + if errors: + # Don't overwhelm the user; report up to ten errors. + fail("\n".join(errors[:10])) + +def _maprule_main(ctx, strategy): + errors = _validate_attributes(ctx.attr.outs_templates, ctx.attr.add_env) + _fail_if_errors(errors) + + # From "srcs": merge the depsets in the DefaultInfo.files of the targets. + common_srcs = depset(transitive = [t[DefaultInfo].files for t in ctx.attr.srcs]) + common_srcs_list = common_srcs.to_list() + + # From "foreach_srcs": by accessing the attribute's value through ctx.files (a list, not a + # depset), we flatten the depsets of DefaultInfo.files of the targets and merge them to a single + # list. This is fine, we would have to do this anyway, because we iterate over them later. + foreach_srcs = ctx.files.foreach_srcs + + # Create the outputs for each file in "foreach_srcs". + foreach_src_outs_dicts, all_outputs, src_placeholders_dicts, errors = _create_outputs( + ctx, + ctx.label.name, + ctx.attr.outs_templates, + strategy, + foreach_srcs, + ) + _fail_if_errors(errors) + + progress_message = (ctx.attr.message or "Executing maprule") + " for %s" % ctx.label + + # Create the part of the environment variables map that all actions will share. + common_envmap = dicts.add( + ctx.configuration.default_shell_env, + {"MAPRULE_SRCS": " ".join([strategy.as_path(p.path) for p in common_srcs_list])}, + ) + + # Resolve $(location) references in "cmd" and in "add_env". + inputs_from_tools, manifests_from_tools, add_env, errors = _resolve_locations( + ctx, + strategy, + ctx.attr.add_env, + ctx.attr.tools, + ) + _fail_if_errors(errors) + + # Create actions for each of the "foreach" sources. + for src in foreach_srcs: + strategy.create_action( + ctx, + inputs = depset(direct = [src] + inputs_from_tools, transitive = [common_srcs]), + outputs = foreach_src_outs_dicts[src].values(), + # The custom envmap contains envvars specific to the current "src", such as MAPRULE_SRC. + env = common_envmap + _custom_envmap( + ctx, + strategy, + src_placeholders_dicts[src], + foreach_src_outs_dicts[src], + add_env, + ), + command = ctx.attr.cmd, + progress_message = progress_message, + manifests_from_tools = manifests_from_tools, + ) + + return [DefaultInfo(files = depset(all_outputs))] + +def _as_windows_path(s): + """Returns the input path as a Windows path (replaces all of "/" with "\").""" + return s.replace("/", "\\") + +def _unchanged_path(s): + """Returns the input string (path) unchanged.""" + return s + +def _create_cmd_action(ctx, inputs, outputs, env, command, progress_message, manifests_from_tools): + """Create one action using cmd.exe for one of the "foreach" sources.""" + ctx.actions.run( + inputs = inputs, + outputs = outputs, + executable = "cmd.exe", + env = env, + arguments = ["/C", command], + progress_message = progress_message, + mnemonic = "Maprule", + input_manifests = manifests_from_tools, + ) + +def _create_bash_action(ctx, inputs, outputs, env, command, progress_message, manifests_from_tools): + """Create one action using Bash for one of the "foreach" sources.""" + ctx.actions.run_shell( + inputs = inputs, + outputs = outputs, + env = env, + command = command, + progress_message = progress_message, + mnemonic = "Maprule", + input_manifests = manifests_from_tools, + ) + +_CMD_STRATEGY = struct( + as_path = _as_windows_path, + create_action = _create_cmd_action, +) + +_BASH_STRATEGY = struct( + as_path = _unchanged_path, + create_action = _create_bash_action, +) + +def _cmd_maprule_impl(ctx): + return _maprule_main(ctx, _CMD_STRATEGY) + +def _bash_maprule_impl(ctx): + return _maprule_main(ctx, _BASH_STRATEGY) + +_ATTRS = { + "srcs": attr.label_list( + allow_files = True, + doc = "The set of source files common to all actions of this rule.", + ), + "add_env": attr.string_dict( + doc = "Extra environment variables to define for the actions. Every variable's name " + + "must be uppercase. Bazel will automatically prepend \"MAPRULE_\" to the name " + + "when exporting the variable for the action. The values may use \"$(location)\" " + + "expressions for labels declared in the `srcs` and `tools` attribute, and " + + "may reference the same placeholders as the values of the `outs_templates` " + + "attribute.", + ), + "cmd": attr.string( + mandatory = True, + doc = "The command to execute. It must be in the syntax corresponding to this maprule " + + "type, e.g. for `bash_maprule` this must be a Bash command, and for `cmd_maprule` " + + "a Windows Command Prompt (cmd.exe) command. Several environment variables are " + + "available for this command, storing values like the paths of the input and output " + + "files of the action. See the \"Environment Variables\" section for the complete " + + "list of environment variables available to this command.", + ), + "foreach_srcs": attr.label_list( + allow_files = True, + mandatory = True, + doc = "The set of sources that will be processed one by one in parallel, to produce " + + "the templated outputs. Each of these source files will will be processed " + + "individually by its own action.", + ), + "message": attr.string( + doc = "A custom progress message to display as the actions are executed.", + ), + "outs_templates": attr.string_dict( + allow_empty = False, + mandatory = True, + doc = "

Templates for the output files. Each key defines a name for an output file " + + "and the value specifies a path template for that output. For each of the " + + "files in `foreach_srcs` this rule creates one action that produces all of " + + "these outputs. The paths of the particular output files for that input are " + + "computed from the template. The ensure the resolved templates yield unique " + + "paths, the following placeholders are supported in the path " + + "templates:

" + + "" + + "

You may also add extra path components to the templates, as long as the path " + + "template is relative and does not contain uplevel references (\"..\"). " + + "Placeholders will be replaced by the values corresponding to the respective " + + "input file in `foreach_srcs`. Every output file is generated under " + + "<bazel_bin>/path/to/maprule/<maprule_name> + \"_outs/\".

", + ), + "tools": attr.label_list( + cfg = "host", + allow_files = True, + doc = "Tools used by the command. The `cmd` attribute, and the values of the " + + "`add_env` attribute may reference these tools in \"$(location)\" expressions, " + + "similar to the genrule rule.", + ), +} + +# Maprule that uses Windows cmd.exe as the interpreter. +cmd_maprule = rule( + implementation = _cmd_maprule_impl, + doc = _rule_doc_template.format( + intro = _cmd_maprule_intro, + example = _cmd_maprule_example, + ), + attrs = _ATTRS, +) + +# Maprule that uses Bash as the interpreter. +bash_maprule = rule( + implementation = _bash_maprule_impl, + doc = _rule_doc_template.format( + intro = _bash_maprule_intro, + example = _bash_maprule_example, + ), + attrs = _ATTRS, +) + +# Only used in unittesting maprule. +maprule_testing = struct( + cmd_strategy = _CMD_STRATEGY, + bash_strategy = _BASH_STRATEGY, + src_placeholders = _src_placeholders, + validate_attributes = _validate_attributes, + is_relative_path = _is_relative_path, + custom_envmap = _custom_envmap, + create_outputs = _create_outputs, +) diff --git a/rules/maprule_testing.bzl b/rules/maprule_testing.bzl new file mode 100644 index 0000000..19f690e --- /dev/null +++ b/rules/maprule_testing.bzl @@ -0,0 +1,24 @@ +# Copyright 2019 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Maprule implementation in Starlark. + +This module exports: + +This module exports the maprule_testing struct. This should only be used by maprule's own unittests. +""" + +load(":maprule_private.bzl", _maprule_testing = "maprule_testing") + +maprule_testing = _maprule_testing diff --git a/tests/BUILD b/tests/BUILD index e845dc2..8657cae 100644 --- a/tests/BUILD +++ b/tests/BUILD @@ -1,5 +1,6 @@ load(":collections_tests.bzl", "collections_test_suite") load(":dicts_tests.bzl", "dicts_test_suite") +load(":maprule_tests.bzl", "maprule_test_suite") load(":new_sets_tests.bzl", "new_sets_test_suite") load(":partial_tests.bzl", "partial_test_suite") load(":paths_tests.bzl", "paths_test_suite") @@ -16,6 +17,8 @@ collections_test_suite() dicts_test_suite() +maprule_test_suite() + partial_test_suite() paths_test_suite() diff --git a/tests/maprule_tests.bzl b/tests/maprule_tests.bzl new file mode 100644 index 0000000..49a346d --- /dev/null +++ b/tests/maprule_tests.bzl @@ -0,0 +1,498 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for maprule.bzl.""" + +load("//lib:unittest.bzl", "asserts", "unittest") +load("//rules:maprule_testing.bzl", "maprule_testing") + +def _dummy_generating_action(ctx, path): + ctx.actions.write(path, "hello") + +def _mock_file(ctx, path): + f = ctx.actions.declare_file(path) + _dummy_generating_action(ctx, f) + return f + +def _lstrip_until(s, until): + return s[s.find(until):] + +def _assert_dict_keys(env, expected, actual, msg): + asserts.equals(env, {k: None for k in expected}, {k: None for k in actual}, msg) + +def _assert_ends_with(env, expected_ending, s, msg): + if not s.endswith(expected_ending): + unittest.fail(env, msg + ": expected \"%s\" to end with \"%s\"" % (s, expected_ending)) + +def _assert_no_error(env, errors, msg): + if errors: + unittest.fail(env, msg + ": expected no errors, got: [%s]" % "\n".join(errors)) + +def _assert_error(env, errors, expected_fragment, msg): + for e in errors: + if expected_fragment in e: + return + unittest.fail(env, msg + ": did not find \"%s\" in: [%s]" % (expected_fragment, "\n".join(errors))) + +def _contains_substrings_in_order(env, s, substrings): + index = 0 + for ss in substrings: + index = s.find(ss, index) + if index < 0: + return False + index += len(ss) + return True + +def _assert_error_fragments(env, errors, expected_fragments, msg): + for e in errors: + if _contains_substrings_in_order(env, e, expected_fragments): + return + unittest.fail(env, msg + ": did not find expected fragments in \"%s\" in order" % "\n".join(errors)) + +def _src_placeholders_test(ctx): + env = unittest.begin(ctx) + + for language, strategy in [ + ("cmd", maprule_testing.cmd_strategy), + ("bash", maprule_testing.bash_strategy), + ]: + for basename, basename_noext in [("bar.txt", "bar"), ("bar.pb.h", "bar.pb")]: + actual = maprule_testing.src_placeholders( + _mock_file(ctx, language + "/foo/" + basename), + strategy, + ) + _assert_dict_keys( + env, + ["src", "src_dir", "src_name", "src_name_noext"], + actual, + "assertion #1 (language: %s, basename: %s)" % (language, basename), + ) + _assert_ends_with( + env, + strategy.as_path(language + "/foo/" + basename), + actual["src"], + "assertion #2 (language: %s, basename: %s)" % (language, basename), + ) + _assert_ends_with( + env, + strategy.as_path(language + "/foo/"), + actual["src_dir"], + "assertion #3 (language: %s, basename: %s)" % (language, basename), + ) + asserts.equals( + env, + basename, + actual["src_name"], + "assertion #4 (language: %s, basename: %s)" % (language, basename), + ) + asserts.equals( + env, + basename_noext, + actual["src_name_noext"], + "assertion #5 (language: %s, basename: %s)" % (language, basename), + ) + + return unittest.end(env) + +src_placeholders_test = unittest.make(_src_placeholders_test) + +def _validate_attributes_test(ctx): + """Unit tests for maprule_testing.validate_attributes.""" + env = unittest.begin(ctx) + _assert_no_error( + env, + maprule_testing.validate_attributes({"FOO": "bar"}, {"BAR": "value1"}), + "assertion #1", + ) + _assert_no_error( + env, + maprule_testing.validate_attributes({"FOO": "bar"}, {}), + "assertion #2", + ) + + _assert_error( + env, + maprule_testing.validate_attributes({}, {}), + "\"outs_templates\" must not be empty", + "assertion #3", + ) + _assert_error( + env, + maprule_testing.validate_attributes({"": "foo"}, {}), + "name should not be empty", + "assertion #4", + ) + _assert_error( + env, + maprule_testing.validate_attributes({"foo": "bar"}, {}), + "name should be all upper-case", + "assertion #5", + ) + _assert_error( + env, + maprule_testing.validate_attributes({"SRC": "bar"}, {}), + "conflicting with the environment variable of the source file", + "assertion #6", + ) + + _assert_error( + env, + maprule_testing.validate_attributes({"FOO": ""}, {}), + "output path should not be empty", + "assertion #7", + ) + _assert_error( + env, + maprule_testing.validate_attributes({"FOO": "/usr/bin"}, {}), + "output path should be relative", + "assertion #8", + ) + _assert_error( + env, + maprule_testing.validate_attributes({"FOO": "c:/usr/bin"}, {}), + "output path should be relative", + "assertion #9", + ) + _assert_error( + env, + maprule_testing.validate_attributes({"FOO": "../foo"}, {}), + "output path should not contain uplevel references", + "assertion #10", + ) + _assert_no_error( + env, + maprule_testing.validate_attributes({"FOO": "./foo"}, {}), + "assertion #11", + ) + _assert_error( + env, + maprule_testing.validate_attributes({"BAR": "foo", "FOO": "foo"}, {}), + "output path is already used for \"BAR\"", + "assertion #12", + ) + + _assert_error( + env, + maprule_testing.validate_attributes({"FOO": "bar"}, {"": "baz"}), + "name should not be empty", + "assertion #13", + ) + _assert_error( + env, + maprule_testing.validate_attributes({"FOO": "bar"}, {"Bar": "baz"}), + "name should be all upper-case", + "assertion #14", + ) + _assert_error( + env, + maprule_testing.validate_attributes({"FOO": "bar"}, {"FOO": "baz"}), + "conflicting with the environment variable of the \"FOO\" output file", + "assertion #15", + ) + + _assert_error( + env, + maprule_testing.validate_attributes({"FOO": "bar"}, {"BAR": "$(location x) $(location y)"}), + "use only one $(location)", + "assertion #16", + ) + _assert_error( + env, + maprule_testing.validate_attributes({"FOO": "bar"}, {"BAR": "a $(location b"}), + "missing closing parenthesis", + "assertion #17", + ) + + return unittest.end(env) + +validate_attributes_test = unittest.make(_validate_attributes_test) + +def _as_path_test(ctx): + """Unit tests for maprule_testing.as_path.""" + env = unittest.begin(ctx) + asserts.equals( + env, + "Foo\\Bar\\Baz\\Qux", + maprule_testing.cmd_strategy.as_path("Foo/Bar/Baz\\Qux"), + msg = "assertion #1", + ) + asserts.equals( + env, + "Foo/Bar/Baz\\Qux", + maprule_testing.bash_strategy.as_path("Foo/Bar/Baz\\Qux"), + msg = "assertion #2", + ) + return unittest.end(env) + +as_path_test = unittest.make(_as_path_test) + +def _assert_relative_path(env, path, index): + asserts.true( + env, + maprule_testing.is_relative_path(path), + msg = "assertion #%d" % index, + ) + +def _assert_not_relative_path(env, path, index): + asserts.false( + env, + maprule_testing.is_relative_path(path), + msg = "assertion #%d" % index, + ) + +def _is_relative_path_test(ctx): + """Unit tests for maprule_testing.is_relative_path.""" + env = unittest.begin(ctx) + _assert_relative_path(env, "Foo/Bar/Baz", 1) + _assert_relative_path(env, "Foo\\Bar\\Baz", 2) + _assert_relative_path(env, "Foo/Bar\\Baz", 3) + _assert_not_relative_path(env, "d:/Foo/Bar", 4) + _assert_not_relative_path(env, "D:/Foo/Bar", 5) + _assert_not_relative_path(env, "/Foo/Bar", 6) + _assert_not_relative_path(env, "\\Foo\\Bar", 7) + return unittest.end(env) + +is_relative_path_test = unittest.make(_is_relative_path_test) + +def _custom_envmap_test(ctx): + """Unit tests for maprule_testing.custom_envmap.""" + env = unittest.begin(ctx) + + actual = {} + + for language, strategy in [ + ("cmd", maprule_testing.cmd_strategy), + ("bash", maprule_testing.bash_strategy), + ]: + actual[language] = maprule_testing.custom_envmap( + ctx, + strategy, + src_placeholders = {"src_ph1": "Src/Ph1-value", "src_ph2": "Src/Ph2-value"}, + outs_dict = { + "out1": _mock_file(ctx, language + "/Foo/Out1"), + "out2": _mock_file(ctx, language + "/Foo/Out2"), + }, + add_env = {"ENV1": "Env1"}, + ) + _assert_dict_keys( + env, + ["MAPRULE_SRC_PH1", "MAPRULE_SRC_PH2", "MAPRULE_OUT1", "MAPRULE_OUT2", "MAPRULE_ENV1"], + actual[language], + msg = "assertion #1 (language: %s)" % language, + ) + actual[language]["MAPRULE_OUT1"] = _lstrip_until(actual[language]["MAPRULE_OUT1"], "Foo") + actual[language]["MAPRULE_OUT2"] = _lstrip_until(actual[language]["MAPRULE_OUT2"], "Foo") + + asserts.equals( + env, + { + "MAPRULE_ENV1": "Env1", + "MAPRULE_OUT1": "Foo\\Out1", + "MAPRULE_OUT2": "Foo\\Out2", + "MAPRULE_SRC_PH1": "Src\\Ph1-value", + "MAPRULE_SRC_PH2": "Src\\Ph2-value", + }, + actual["cmd"], + msg = "assertion #2", + ) + + asserts.equals( + env, + { + "MAPRULE_ENV1": "Env1", + "MAPRULE_OUT1": "Foo/Out1", + "MAPRULE_OUT2": "Foo/Out2", + "MAPRULE_SRC_PH1": "Src/Ph1-value", + "MAPRULE_SRC_PH2": "Src/Ph2-value", + }, + actual["bash"], + msg = "assertion #3", + ) + + return unittest.end(env) + +custom_envmap_test = unittest.make(_custom_envmap_test) + +def _create_outputs_test(ctx): + """Unit tests for maprule_testing.create_outputs.""" + env = unittest.begin(ctx) + + for language, strategy in [ + ("cmd", maprule_testing.cmd_strategy), + ("bash", maprule_testing.bash_strategy), + ]: + src1 = _mock_file(ctx, language + "/foo/src1.txt") + src2 = _mock_file(ctx, language + "/foo/src2.pb.h") + src3 = _mock_file(ctx, language + "/bar/src1.txt") + foreach_srcs = [src1, src2, src3] + + outs_dicts, all_output_files, src_placeholders_dicts, errors = ( + maprule_testing.create_outputs( + ctx, + "my_maprule", + { + "OUT1": "{src}.out1", + "OUT2": "{src_dir}/out2/{src_name_noext}.out2", + }, + strategy, + foreach_srcs, + ) + ) + + _assert_no_error(env, errors, "assertion #1 (language: %s)" % language) + + for output in all_output_files: + _dummy_generating_action(ctx, output) + + _assert_dict_keys( + env, + foreach_srcs, + outs_dicts, + "assertion #2 (language: %s)" % language, + ) + for src in foreach_srcs: + _assert_dict_keys( + env, + ["OUT1", "OUT2"], + outs_dicts[src], + "assertion #3 (language: %s, src: %s)" % (language, src), + ) + + _assert_ends_with( + env, + "my_maprule_out/tests/%s/foo/src1.txt.out1" % language, + outs_dicts[src1]["OUT1"].path, + "assertion #4 (language: %s)" % language, + ) + _assert_ends_with( + env, + "my_maprule_out/tests/%s/foo/out2/src1.out2" % language, + outs_dicts[src1]["OUT2"].path, + "assertion #5 (language: %s)" % language, + ) + + _assert_ends_with( + env, + "my_maprule_out/tests/%s/foo/src2.pb.h.out1" % language, + outs_dicts[src2]["OUT1"].path, + "assertion #6 (language: %s)" % language, + ) + _assert_ends_with( + env, + "my_maprule_out/tests/%s/foo/out2/src2.pb.out2" % language, + outs_dicts[src2]["OUT2"].path, + "assertion #7 (language: %s)" % language, + ) + + _assert_ends_with( + env, + "my_maprule_out/tests/%s/bar/src1.txt.out1" % language, + outs_dicts[src3]["OUT1"].path, + "assertion #8 (language: %s)" % language, + ) + _assert_ends_with( + env, + "my_maprule_out/tests/%s/bar/out2/src1.out2" % language, + outs_dicts[src3]["OUT2"].path, + "assertion #9 (language: %s)" % language, + ) + + expected = [ + "my_maprule_out/tests/%s/foo/src1.txt.out1" % language, + "my_maprule_out/tests/%s/foo/out2/src1.out2" % language, + "my_maprule_out/tests/%s/foo/src2.pb.h.out1" % language, + "my_maprule_out/tests/%s/foo/out2/src2.pb.out2" % language, + "my_maprule_out/tests/%s/bar/src1.txt.out1" % language, + "my_maprule_out/tests/%s/bar/out2/src1.out2" % language, + ] + for i in range(0, len(all_output_files)): + actual = _lstrip_until(all_output_files[i].path, "my_maprule_out") + asserts.equals( + env, + expected[i], + actual, + "assertion #10 (language: %s, index: %d)" % (language, i), + ) + + return unittest.end(env) + +create_outputs_test = unittest.make(_create_outputs_test) + +def _conflicting_outputs_test(ctx): + """Unit tests for maprule_testing.create_outputs catching conflicting outputs.""" + env = unittest.begin(ctx) + + for language, strategy in [ + ("cmd", maprule_testing.cmd_strategy), + ("bash", maprule_testing.bash_strategy), + ]: + src1 = _mock_file(ctx, language + "/foo/src1.txt") + src2 = _mock_file(ctx, language + "/foo/src2.pb.h") + src3 = _mock_file(ctx, language + "/bar/src1.txt") + foreach_srcs = [src1, src2, src3] + + _, all_output_files, _, errors = ( + maprule_testing.create_outputs( + ctx, + "my_maprule", + { + "OUT1": "out1", # 3 conflicts + "OUT2": "{src_dir}/out2", # 2 conflicts + "OUT3": "out3/{src_name}", # 2 conflicts + }, + strategy, + foreach_srcs, + ) + ) + + for output in all_output_files: + _dummy_generating_action(ctx, output) + + _assert_error_fragments( + env, + errors, + ["out1", language + "/foo/src1.txt", "OUT1", language + "/foo/src2.pb.h", "OUT1"], + msg = "assertion #1 (language: %s)" % language, + ) + + _assert_error_fragments( + env, + errors, + ["out2", language + "/foo/src1.txt", "OUT2", language + "/foo/src2.pb.h", "OUT2"], + msg = "assertion #2 (language: %s)" % language, + ) + + _assert_error_fragments( + env, + errors, + ["out3/src1.txt", language + "/foo/src1.txt", "OUT3", language + "/bar/src1.txt", "OUT3"], + msg = "assertion #5 (language: %s)" % language, + ) + + return unittest.end(env) + +conflicting_outputs_test = unittest.make(_conflicting_outputs_test) + +def maprule_test_suite(): + """Creates the test targets and test suite for maprule.bzl tests.""" + + unittest.suite( + "maprule_tests", + src_placeholders_test, + validate_attributes_test, + as_path_test, + is_relative_path_test, + custom_envmap_test, + conflicting_outputs_test, + )