unittest.bzl: supports Windows (#84)

In this commit:

- change unittest.bzl to declare a named output
  file instead of relying on the deprecated [1]
  default output name (ctx.outputs.executable).

- define a new toolchain_type and toolchain rules
  for cmd.exe and for Bash (basically Windows and
  non-Windows)

- register the new toolchains in workspace.bzl

- let unittest.make-created test rules require the
  new toolchain_type

- write the test output script as a Windows batch
  script or as a Shell script, depending on the
  selected toolchain

This PR enables the Bazel team to break the Bash
dependency (for test execution) on Windows, and
can run Starlark unittests with the new,
Windows-native test wrapper (still under
development).

See https://github.com/bazelbuild/bazel/issues/5508

README.md

@@ -16,6 +16,8 @@ be loaded as a single unit, for convenience.
 
 ## Getting Started
 
+### `WORKSPACE` file
+
 Add the following to your `WORKSPACE` file to import the Skylib repository into
 your workspace. Replace the version number in the `tag` attribute with the
 version you wish to depend on:
@@ -28,6 +30,17 @@ git_repository(
 )
 ```
 
+If you want to use `lib/unittest.bzl` from Skylib versions released in or after
+December 2018, then you also should add to the `WORKSPACE` file:
+
+```python
+load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
+
+bazel_skylib_workspace()
+```
+
+### `BUILD` and `*.bzl` files
+
 Then, in the `BUILD` and/or `*.bzl` files in your own workspace, you can load
 the modules (listed [below](#list-of-modules)) and access the symbols by
 dotting into those structs:
@@ -85,3 +98,15 @@ Steps to add a module to Skylib:
 The `bzl_library.bzl` rule can be used to aggregate a set of
 Starlark files and its dependencies for use in test targets and
 documentation generation.
+
+## Troubleshooting
+
+If you try to use `unittest` and you get the following error:
+
+```
+ERROR: While resolving toolchains for target //foo:bar: no matching toolchains found for types @bazel_skylib//toolchains:toolchain_type
+ERROR: Analysis of target '//foo:bar' failed; build aborted: no matching toolchains found for types @bazel_skylib//toolchains:toolchain_type
+```
+
+then you probably forgot to load and call `bazel_skylib_workspace()` in your
+`WORKSPACE` file.

WORKSPACE

@@ -1 +1,5 @@
 workspace(name = "bazel_skylib")
+
+load(":workspace.bzl", "bazel_skylib_workspace")
+
+bazel_skylib_workspace()

lib/unittest.bzl

@@ -22,6 +22,43 @@ assertions used to within tests.
 load(":new_sets.bzl", new_sets = "sets")
 load(":sets.bzl", "sets")
 
+# The following function should only be called from WORKSPACE files and workspace macros.
+def register_unittest_toolchains():
+    """Registers the toolchains for unittest users."""
+    native.register_toolchains(
+        "@bazel_skylib//toolchains/unittest:cmd_toolchain",
+        "@bazel_skylib//toolchains/unittest:bash_toolchain",
+    )
+
+TOOLCHAIN_TYPE = "@bazel_skylib//toolchains/unittest:toolchain_type"
+
+_UnittestToolchain = provider(
+    doc = "Execution platform information for rules in the bazel_skylib repository.",
+    fields = ["file_ext", "success_templ", "failure_templ", "join_on"],
+)
+
+def _unittest_toolchain_impl(ctx):
+    return [
+        platform_common.ToolchainInfo(
+            unittest_toolchain_info = _UnittestToolchain(
+                file_ext = ctx.attr.file_ext,
+                success_templ = ctx.attr.success_templ,
+                failure_templ = ctx.attr.failure_templ,
+                join_on = ctx.attr.join_on,
+            ),
+        ),
+    ]
+
+unittest_toolchain = rule(
+    implementation = _unittest_toolchain_impl,
+    attrs = {
+        "file_ext": attr.string(mandatory = True),
+        "success_templ": attr.string(mandatory = True),
+        "failure_templ": attr.string(mandatory = True),
+        "join_on": attr.string(mandatory = True),
+    },
+)
+
 def _make(impl, attrs = None):
     """Creates a unit test rule from its implementation function.
 
@@ -41,7 +78,7 @@ def _make(impl, attrs = None):
 
       # Assert statements go here
 
-      unittest.end(env)
+      return unittest.end(env)
 
     your_test = unittest.make(_your_test)
     ```
@@ -75,6 +112,7 @@ def _make(impl, attrs = None):
         attrs = attrs,
         _skylark_testable = True,
         test = True,
+        toolchains = [TOOLCHAIN_TYPE],
     )
 
 def _suite(name, *test_rules):
@@ -160,17 +198,20 @@ def _end(env):
     Args:
       env: The test environment returned by `unittest.begin`.
     """
-    cmd = "\n".join([
+
-        "cat << EOF",
+    tc = env.ctx.toolchains[TOOLCHAIN_TYPE].unittest_toolchain_info
-        "\n".join(env.failures),
+    testbin = env.ctx.actions.declare_file(env.ctx.label.name + tc.file_ext)
-        "EOF",
+    if env.failures:
-        "exit %d" % len(env.failures),
+        cmd = tc.failure_templ % tc.join_on.join(env.failures)
-    ])
+    else:
+        cmd = tc.success_templ
+
     env.ctx.actions.write(
-        output = env.ctx.outputs.executable,
+        output = testbin,
         content = cmd,
         is_executable = True,
     )
+    return [DefaultInfo(executable = testbin)]
 
 def _fail(env, msg):
     """Unconditionally causes the current test to fail.

tests/collections_tests.bzl

@@ -37,7 +37,7 @@ def _after_each_test(ctx):
         collections.after_each(None, ["a", "b"]),
     )
 
-    unittest.end(env)
+    return unittest.end(env)
 
 after_each_test = unittest.make(_after_each_test)
 
@@ -61,7 +61,7 @@ def _before_each_test(ctx):
         collections.before_each(None, ["a", "b"]),
     )
 
-    unittest.end(env)
+    return unittest.end(env)
 
 before_each_test = unittest.make(_before_each_test)
 
@@ -99,7 +99,7 @@ def _uniq_test(ctx):
         ],
     )
 
-    unittest.end(env)
+    return unittest.end(env)
 
 uniq_test = unittest.make(_uniq_test)
 

tests/dicts_tests.bzl

@@ -63,7 +63,7 @@ def _add_test(ctx):
     result["a"] = 2
     asserts.equals(env, 1, original["a"])
 
-    unittest.end(env)
+    return unittest.end(env)
 
 add_test = unittest.make(_add_test)
 

tests/new_sets_tests.bzl

@@ -36,7 +36,7 @@ def _is_equal_test(ctx):
     # If passing a list, verify that duplicate elements are ignored.
     asserts.true(env, sets.is_equal(sets.make([1, 1]), sets.make([1])))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 is_equal_test = unittest.make(_is_equal_test)
 
@@ -54,7 +54,7 @@ def _is_subset_test(ctx):
     # If passing a list, verify that duplicate elements are ignored.
     asserts.true(env, sets.is_subset(sets.make([1, 1]), sets.make([1, 2])))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 is_subset_test = unittest.make(_is_subset_test)
 
@@ -72,7 +72,7 @@ def _disjoint_test(ctx):
     # If passing a list, verify that duplicate elements are ignored.
     asserts.false(env, sets.disjoint(sets.make([1, 1]), sets.make([1, 2])))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 disjoint_test = unittest.make(_disjoint_test)
 
@@ -90,7 +90,7 @@ def _intersection_test(ctx):
     # If passing a list, verify that duplicate elements are ignored.
     asserts.new_set_equals(env, sets.make([1]), sets.intersection(sets.make([1, 1]), sets.make([1, 2])))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 intersection_test = unittest.make(_intersection_test)
 
@@ -110,7 +110,7 @@ def _union_test(ctx):
     # If passing a list, verify that duplicate elements are ignored.
     asserts.new_set_equals(env, sets.make([1, 2]), sets.union(sets.make([1, 1]), sets.make([1, 2])))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 union_test = unittest.make(_union_test)
 
@@ -128,7 +128,7 @@ def _difference_test(ctx):
     # If passing a list, verify that duplicate elements are ignored.
     asserts.new_set_equals(env, sets.make([2]), sets.difference(sets.make([1, 2]), sets.make([1, 1])))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 difference_test = unittest.make(_difference_test)
 
@@ -140,7 +140,7 @@ def _to_list_test(ctx):
     asserts.equals(env, [1], sets.to_list(sets.make([1, 1, 1])))
     asserts.equals(env, [1, 2, 3], sets.to_list(sets.make([1, 2, 3])))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 to_list_test = unittest.make(_to_list_test)
 
@@ -151,7 +151,7 @@ def _make_test(ctx):
     asserts.equals(env, {}, sets.make()._values)
     asserts.equals(env, {x: None for x in [1, 2, 3]}, sets.make([1, 1, 2, 2, 3, 3])._values)
 
-    unittest.end(env)
+    return unittest.end(env)
 
 make_test = unittest.make(_make_test)
 
@@ -168,7 +168,7 @@ def _copy_test(ctx):
     copy._values[5] = None
     asserts.false(env, sets.is_equal(original, copy))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 copy_test = unittest.make(_copy_test)
 
@@ -188,7 +188,7 @@ def _insert_test(ctx):
         msg = "Insert creates a new set which is an O(n) operation, insert should be O(1).",
     )
 
-    unittest.end(env)
+    return unittest.end(env)
 
 insert_test = unittest.make(_insert_test)
 
@@ -201,7 +201,7 @@ def _contains_test(ctx):
     asserts.true(env, sets.contains(sets.make([1, 2]), 1))
     asserts.false(env, sets.contains(sets.make([2, 3]), 1))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 contains_test = unittest.make(_contains_test)
 
@@ -213,7 +213,7 @@ def _length_test(ctx):
     asserts.equals(env, 1, sets.length(sets.make([1])))
     asserts.equals(env, 2, sets.length(sets.make([1, 2])))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 length_test = unittest.make(_length_test)
 
@@ -228,7 +228,7 @@ def _remove_test(ctx):
     after_removal = sets.remove(original, 3)
     asserts.new_set_equals(env, original, after_removal)
 
-    unittest.end(env)
+    return unittest.end(env)
 
 remove_test = unittest.make(_remove_test)
 
@@ -244,7 +244,7 @@ def _repr_str_test(ctx):
     asserts.equals(env, "[1]", sets.str(sets.make([1])))
     asserts.equals(env, "[1, 2]", sets.str(sets.make([1, 2])))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 repr_str_test = unittest.make(_repr_str_test)
 

tests/partial_tests.bzl

@@ -72,7 +72,7 @@ def _make_call_test(ctx):
     foo = partial.make(_call_args_kwargs, 100, func_mult = 10)
     asserts.equals(env, 1120, partial.call(foo, 12, func_mult = 5, call_mult = 2))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 make_call_test = unittest.make(_make_call_test)
 

tests/paths_tests.bzl

@@ -37,7 +37,7 @@ def _basename_test(ctx):
     asserts.equals(env, "", paths.basename("foo/"))
     asserts.equals(env, "", paths.basename("/foo/"))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 basename_test = unittest.make(_basename_test)
 
@@ -62,7 +62,7 @@ def _dirname_test(ctx):
     asserts.equals(env, "foo", paths.dirname("foo/"))
     asserts.equals(env, "/foo", paths.dirname("/foo/"))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 dirname_test = unittest.make(_dirname_test)
 
@@ -84,7 +84,7 @@ def _is_absolute_test(ctx):
     asserts.true(env, paths.is_absolute("/foo/"))
     asserts.true(env, paths.is_absolute("/foo/bar"))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 is_absolute_test = unittest.make(_is_absolute_test)
 
@@ -128,7 +128,7 @@ def _join_test(ctx):
     asserts.equals(env, "foo", paths.join("", "", "foo"))
     asserts.equals(env, "foo/bar", paths.join("foo", "", "", "bar"))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 join_test = unittest.make(_join_test)
 
@@ -170,7 +170,7 @@ def _normalize_test(ctx):
     asserts.equals(env, "foo", paths.normalize("foo/"))
     asserts.equals(env, "foo/bar", paths.normalize("foo/bar/"))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 normalize_test = unittest.make(_normalize_test)
 
@@ -196,7 +196,7 @@ def _relativize_test(ctx):
 
     # TODO(allevato): Test failure cases, once that is possible.
 
-    unittest.end(env)
+    return unittest.end(env)
 
 relativize_test = unittest.make(_relativize_test)
 
@@ -227,7 +227,7 @@ def _replace_extension_test(ctx):
     # Verify that we don't insert a period on the extension if none is provided.
     asserts.equals(env, "foobaz", paths.replace_extension("foo.bar", "baz"))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 replace_extension_test = unittest.make(_replace_extension_test)
 
@@ -266,7 +266,7 @@ def _split_extension_test(ctx):
     asserts.equals(env, (".a/b", ".c"), paths.split_extension(".a/b.c"))
     asserts.equals(env, (".a", ".b"), paths.split_extension(".a.b"))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 split_extension_test = unittest.make(_split_extension_test)
 

tests/selects_tests.bzl

@@ -53,7 +53,7 @@ def _with_or_test(ctx):
         selects.with_or_dict(mixed_dict),
     )
 
-    unittest.end(env)
+    return unittest.end(env)
 
 with_or_test = unittest.make(_with_or_test)
 

tests/sets_tests.bzl

@@ -39,7 +39,7 @@ def _is_equal_test(ctx):
     # If passing a list, verify that duplicate elements are ignored.
     asserts.true(env, sets.is_equal([1, 1], [1]))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 is_equal_test = unittest.make(_is_equal_test)
 
@@ -58,7 +58,7 @@ def _is_subset_test(ctx):
     # If passing a list, verify that duplicate elements are ignored.
     asserts.true(env, sets.is_subset([1, 1], [1, 2]))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 is_subset_test = unittest.make(_is_subset_test)
 
@@ -77,7 +77,7 @@ def _disjoint_test(ctx):
     # If passing a list, verify that duplicate elements are ignored.
     asserts.false(env, sets.disjoint([1, 1], [1, 2]))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 disjoint_test = unittest.make(_disjoint_test)
 
@@ -96,7 +96,7 @@ def _intersection_test(ctx):
     # If passing a list, verify that duplicate elements are ignored.
     asserts.set_equals(env, [1], sets.intersection([1, 1], [1, 2]))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 intersection_test = unittest.make(_intersection_test)
 
@@ -117,7 +117,7 @@ def _union_test(ctx):
     # If passing a list, verify that duplicate elements are ignored.
     asserts.set_equals(env, [1, 2], sets.union([1, 1], [1, 2]))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 union_test = unittest.make(_union_test)
 
@@ -136,7 +136,7 @@ def _difference_test(ctx):
     # If passing a list, verify that duplicate elements are ignored.
     asserts.set_equals(env, [2], sets.difference([1, 2], [1, 1]))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 difference_test = unittest.make(_difference_test)
 

tests/shell_tests.bzl

@@ -27,7 +27,7 @@ def _shell_array_literal_test(ctx):
     asserts.equals(env, "('$foo')", shell.array_literal(["$foo"]))
     asserts.equals(env, "('qu\"o\"te')", shell.array_literal(['qu"o"te']))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 shell_array_literal_test = unittest.make(_shell_array_literal_test)
 
@@ -48,7 +48,7 @@ def _shell_quote_test(ctx):
     asserts.equals(env, "'foo\\bar'", shell.quote("foo\\bar"))
     asserts.equals(env, "'back`echo q`uote'", shell.quote("back`echo q`uote"))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 shell_quote_test = unittest.make(_shell_quote_test)
 

tests/structs_tests.bzl

@@ -42,7 +42,7 @@ def _add_test(ctx):
         structs.to_dict(struct(a = 1, b = struct(bb = 1))),
     )
 
-    unittest.end(env)
+    return unittest.end(env)
 
 add_test = unittest.make(_add_test)
 

tests/types_tests.bzl

@@ -36,7 +36,7 @@ def _is_string_test(ctx):
     asserts.false(env, types.is_string(None))
     asserts.false(env, types.is_string(_a_function))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 is_string_test = unittest.make(_is_string_test)
 
@@ -56,7 +56,7 @@ def _is_bool_test(ctx):
     asserts.false(env, types.is_bool(None))
     asserts.false(env, types.is_bool(_a_function))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 is_bool_test = unittest.make(_is_bool_test)
 
@@ -76,7 +76,7 @@ def _is_list_test(ctx):
     asserts.false(env, types.is_list(None))
     asserts.false(env, types.is_list(_a_function))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 is_list_test = unittest.make(_is_list_test)
 
@@ -96,7 +96,7 @@ def _is_none_test(ctx):
     asserts.false(env, types.is_none([1]))
     asserts.false(env, types.is_none(_a_function))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 is_none_test = unittest.make(_is_none_test)
 
@@ -117,7 +117,7 @@ def _is_int_test(ctx):
     asserts.false(env, types.is_int(None))
     asserts.false(env, types.is_int(_a_function))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 is_int_test = unittest.make(_is_int_test)
 
@@ -138,7 +138,7 @@ def _is_tuple_test(ctx):
     asserts.false(env, types.is_tuple(None))
     asserts.false(env, types.is_tuple(_a_function))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 is_tuple_test = unittest.make(_is_tuple_test)
 
@@ -159,7 +159,7 @@ def _is_dict_test(ctx):
     asserts.false(env, types.is_dict(None))
     asserts.false(env, types.is_dict(_a_function))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 is_dict_test = unittest.make(_is_dict_test)
 
@@ -179,7 +179,7 @@ def _is_function_test(ctx):
     asserts.false(env, types.is_function([1]))
     asserts.false(env, types.is_function(None))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 is_function_test = unittest.make(_is_function_test)
 

tests/versions_tests.bzl

@@ -26,7 +26,7 @@ def _parse_test(ctx):
     asserts.equals(env, (0, 4, 0), versions.parse("0.4.0"))
     asserts.equals(env, (0, 4, 0), versions.parse("0.4.0rc"))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 def _version_comparison_test(ctx):
     """Unit tests for versions.is_at_least and is_at_most"""
@@ -42,7 +42,7 @@ def _version_comparison_test(ctx):
     asserts.true(env, versions.is_at_most("0.4.0", "0.4.0rc3"))
     asserts.true(env, versions.is_at_most("1.4.0", "0.4.0rc3"))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 def _check_test(ctx):
     """Unit tests for versions.check"""
@@ -53,7 +53,7 @@ def _check_test(ctx):
     asserts.equals(env, None, versions.check("0.4.5", bazel_version = "0.10.0rc1 abcd123"))
     asserts.equals(env, None, versions.check("0.4.5", maximum_bazel_version = "1.0.0", bazel_version = "0.10.0rc1 abcd123"))
 
-    unittest.end(env)
+    return unittest.end(env)
 
 parse_test = unittest.make(_parse_test)
 version_comparison_test = unittest.make(_version_comparison_test)

toolchains/unittest/BUILD

@@ -0,0 +1,47 @@
+load("//lib:unittest.bzl", "TOOLCHAIN_TYPE", "unittest_toolchain")
+
+toolchain_type(
+    name = "toolchain_type",
+    visibility = ["//visibility:public"],
+)
+
+unittest_toolchain(
+    name = "cmd",
+    failure_templ = """@echo off
+echo %s
+exit /b 1
+""",
+    file_ext = ".bat",
+    join_on = "\necho ",
+    success_templ = "@exit /b 0",
+    visibility = ["//visibility:public"],
+)
+
+unittest_toolchain(
+    name = "bash",
+    failure_templ = """#!/bin/sh
+cat <<'EOF'
+%s
+EOF
+exit 1
+""",
+    file_ext = ".sh",
+    join_on = "\n",
+    success_templ = "#!/bin/sh\nexit 0",
+    visibility = ["//visibility:public"],
+)
+
+toolchain(
+    name = "cmd_toolchain",
+    exec_compatible_with = [
+        "@bazel_tools//platforms:windows",
+    ],
+    toolchain = ":cmd",
+    toolchain_type = TOOLCHAIN_TYPE,
+)
+
+toolchain(
+    name = "bash_toolchain",
+    toolchain = ":bash",
+    toolchain_type = TOOLCHAIN_TYPE,
+)

workspace.bzl

@@ -0,0 +1,5 @@
+load("@bazel_skylib//lib:unittest.bzl", "register_unittest_toolchains")
+
+def bazel_skylib_workspace():
+    """Registers toolchains and declares repository dependencies of the bazel_skylib repository."""
+    register_unittest_toolchains()