From 5c1be25800e0806356624c1effd7a23240b3a45e Mon Sep 17 00:00:00 2001 From: Googler Date: Mon, 9 Sep 2024 06:05:13 -0700 Subject: [PATCH] Copy a few utilities from `@_builtins` to `@rules_cc` PiperOrigin-RevId: 672519388 Change-Id: Icb077dd736af10724ffdbfb8330564516b43fb1d --- cc/common/BUILD | 18 ++++ cc/common/cc_debug_helper.bzl | 181 ++++++++++++++++++++++++++++++++++ cc/common/cc_helper.bzl | 100 ++++++++++++++++++- cc/common/visibility.bzl | 3 + 4 files changed, 298 insertions(+), 4 deletions(-) create mode 100644 cc/common/cc_debug_helper.bzl create mode 100644 cc/common/visibility.bzl diff --git a/cc/common/BUILD b/cc/common/BUILD index 2af8526..fdb3921 100644 --- a/cc/common/BUILD +++ b/cc/common/BUILD @@ -25,6 +25,24 @@ bzl_library( name = "cc_helper_bzl", srcs = ["cc_helper.bzl"], visibility = ["//visibility:public"], + deps = [":visibility_bzl"], +) + +bzl_library( + name = "cc_debug_helper_bzl", + srcs = ["cc_debug_helper.bzl"], + visibility = ["//visibility:public"], + deps = [ + ":cc_helper_bzl", + ":visibility_bzl", + "//cc:find_cc_toolchain_bzl", + ], +) + +bzl_library( + name = "visibility_bzl", + srcs = ["visibility.bzl"], + visibility = ["//visibility:private"], ) filegroup( diff --git a/cc/common/cc_debug_helper.bzl b/cc/common/cc_debug_helper.bzl new file mode 100644 index 0000000..d16e925 --- /dev/null +++ b/cc/common/cc_debug_helper.bzl @@ -0,0 +1,181 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Utilities for creating cc debug package information outputs""" + +load("//cc:find_cc_toolchain.bzl", "CC_TOOLCHAIN_TYPE") +load(":cc_helper.bzl", "linker_mode") +load(":visibility.bzl", "INTERNAL_VISIBILITY") + +visibility(INTERNAL_VISIBILITY) + +def create_debug_packager_actions( + ctx, + cc_toolchain, + dwp_output, + *, + cc_compilation_outputs, + cc_debug_context, + linking_mode, + use_pic = True, + lto_artifacts = []): + """Creates intermediate and final dwp creation action(s) + + Args: + ctx: (RuleContext) the rule context + cc_toolchain: (CcToolchainInfo) the cc toolchain + dwp_output: (File) the output of the final dwp action + cc_compilation_outputs: (CcCompilationOutputs) + cc_debug_context: (DebugContext) + linking_mode: (str) See cc_helper.bzl%linker_mode + use_pic: (bool) + lto_artifacts: ([CcLtoBackendArtifacts]) + """ + dwo_files = _collect_transitive_dwo_artifacts( + cc_compilation_outputs, + cc_debug_context, + linking_mode, + use_pic, + lto_artifacts, + ) + + # No inputs? Just generate a trivially empty .dwp. + # + # Note this condition automatically triggers for any build where fission is disabled. + # Because rules referencing .dwp targets may be invoked with or without fission, we need + # to support .dwp generation even when fission is disabled. Since no actual functionality + # is expected then, an empty file is appropriate. + dwo_files_list = dwo_files.to_list() + if len(dwo_files_list) == 0: + ctx.actions.write(dwp_output, "", False) + return + + # We apply a hierarchical action structure to limit the maximum number of inputs to any + # single action. + # + # While the dwp tool consumes .dwo files, it can also consume intermediate .dwp files, + # allowing us to split a large input set into smaller batches of arbitrary size and order. + # Aside from the parallelism performance benefits this offers, this also reduces input + # size requirements: if a.dwo, b.dwo, c.dwo, and e.dwo are each 1 KB files, we can apply + # two intermediate actions DWP(a.dwo, b.dwo) --> i1.dwp and DWP(c.dwo, e.dwo) --> i2.dwp. + # When we then apply the final action DWP(i1.dwp, i2.dwp) --> finalOutput.dwp, the inputs + # to this action will usually total far less than 4 KB. + # + # The actions form an n-ary tree with n == MAX_INPUTS_PER_DWP_ACTION. The tree is fuller + # at the leaves than the root, but that both increases parallelism and reduces the final + # action's input size. + packager = _create_intermediate_dwp_packagers(ctx, dwp_output, cc_toolchain, cc_toolchain._dwp_files, dwo_files_list, 1) + packager["outputs"].append(dwp_output) + packager["arguments"].add("-o", dwp_output) + ctx.actions.run( + mnemonic = "CcGenerateDwp", + tools = packager["tools"], + executable = packager["executable"], + toolchain = CC_TOOLCHAIN_TYPE, + arguments = [packager["arguments"]], + inputs = packager["inputs"], + outputs = packager["outputs"], + ) + +def _collect_transitive_dwo_artifacts(cc_compilation_outputs, cc_debug_context, linking_mode, use_pic, lto_backend_artifacts): + dwo_files = [] + transitive_dwo_files = depset() + if use_pic: + dwo_files.extend(cc_compilation_outputs.pic_dwo_files()) + else: + dwo_files.extend(cc_compilation_outputs.dwo_files()) + + if lto_backend_artifacts != None: + for lto_backend_artifact in lto_backend_artifacts: + if lto_backend_artifact.dwo_file() != None: + dwo_files.append(lto_backend_artifact.dwo_file()) + + if linking_mode != linker_mode.LINKING_DYNAMIC: + if use_pic: + transitive_dwo_files = cc_debug_context.pic_files + else: + transitive_dwo_files = cc_debug_context.files + return depset(dwo_files, transitive = [transitive_dwo_files]) + +def _get_intermediate_dwp_file(ctx, dwp_output, order_number): + output_path = dwp_output.short_path + + # Since it is a dwp_output we can assume that it always + # ends with .dwp suffix, because it is declared so in outputs + # attribute. + extension_stripped_output_path = output_path[0:len(output_path) - 4] + intermediate_path = extension_stripped_output_path + "-" + str(order_number) + ".dwp" + + return ctx.actions.declare_file("_dwps/" + intermediate_path) + +def _create_intermediate_dwp_packagers(ctx, dwp_output, cc_toolchain, dwp_files, dwo_files, intermediate_dwp_count): + intermediate_outputs = dwo_files + + # This long loop is a substitution for recursion, which is not currently supported in Starlark. + for _ in range(2147483647): + packagers = [] + current_packager = _new_dwp_action(ctx, cc_toolchain, dwp_files) + inputs_for_current_packager = 0 + + # Step 1: generate our batches. We currently break into arbitrary batches of fixed maximum + # input counts, but we can always apply more intelligent heuristics if the need arises. + for dwo_file in intermediate_outputs: + if inputs_for_current_packager == 100: + packagers.append(current_packager) + current_packager = _new_dwp_action(ctx, cc_toolchain, dwp_files) + inputs_for_current_packager = 0 + current_packager["inputs"].append(dwo_file) + + # add_all expands all directories to their contained files, see + # https://bazel.build/rules/lib/builtins/Args#add_all. add doesn't + # do that, so using add_all on the one-item list here allows us to + # find dwo files in directories. + current_packager["arguments"].add_all([dwo_file]) + inputs_for_current_packager += 1 + + packagers.append(current_packager) + + # Step 2: given the batches, create the actions. + if len(packagers) > 1: + # If we have multiple batches, make them all intermediate actions, then pipe their outputs + # into an additional level. + intermediate_outputs = [] + for packager in packagers: + intermediate_output = _get_intermediate_dwp_file(ctx, dwp_output, intermediate_dwp_count) + intermediate_dwp_count += 1 + packager["outputs"].append(intermediate_output) + packager["arguments"].add("-o", intermediate_output) + ctx.actions.run( + mnemonic = "CcGenerateIntermediateDwp", + tools = packager["tools"], + executable = packager["executable"], + toolchain = CC_TOOLCHAIN_TYPE, + arguments = [packager["arguments"]], + inputs = packager["inputs"], + outputs = packager["outputs"], + ) + intermediate_outputs.append(intermediate_output) + else: + return packagers[0] + + # This is to fix buildifier errors, even though we should never reach this part of the code. + return None + +def _new_dwp_action(ctx, cc_toolchain, dwp_tools): + return { + "arguments": ctx.actions.args(), + "executable": cc_toolchain._tool_paths.get("dwp", None), + "inputs": [], + "outputs": [], + "tools": dwp_tools, + } diff --git a/cc/common/cc_helper.bzl b/cc/common/cc_helper.bzl index 8b161a1..35f6812 100644 --- a/cc/common/cc_helper.bzl +++ b/cc/common/cc_helper.bzl @@ -11,10 +11,82 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """Utility functions for C++ rules.""" -# LINT.IfChange +load("//cc:find_cc_toolchain.bzl", "CC_TOOLCHAIN_TYPE") +load(":cc_common.bzl", "cc_common") +load(":visibility.bzl", "INTERNAL_VISIBILITY") + +visibility(INTERNAL_VISIBILITY) + +# LINT.IfChange(linker_mode) +linker_mode = struct( + LINKING_DYNAMIC = "dynamic_linking_mode", + LINKING_STATIC = "static_linking_mode", +) +# LINT.ThenChange(https://github.com/bazelbuild/bazel/blob/master/src/main/starlark/builtins_bzl/common/cc/cc_helper.bzl:linker_mode) + +# LINT.IfChange(forked_exports) +def _get_static_mode_params_for_dynamic_library_libraries(libs): + linker_inputs = [] + for lib in libs.to_list(): + if lib.pic_static_library: + linker_inputs.append(lib.pic_static_library) + elif lib.static_library: + linker_inputs.append(lib.static_library) + elif lib.interface_library: + linker_inputs.append(lib.interface_library) + else: + linker_inputs.append(lib.dynamic_library) + return linker_inputs + +def _create_strip_action(ctx, cc_toolchain, cpp_config, input, output, feature_configuration): + if cc_common.is_enabled(feature_configuration = feature_configuration, feature_name = "no_stripping"): + ctx.actions.symlink( + output = output, + target_file = input, + progress_message = "Symlinking original binary as stripped binary", + ) + return + + if not cc_common.action_is_enabled(feature_configuration = feature_configuration, action_name = "strip"): + fail("Expected action_config for 'strip' to be configured.") + + variables = cc_common.create_compile_variables( + cc_toolchain = cc_toolchain, + feature_configuration = feature_configuration, + output_file = output.path, + input_file = input.path, + strip_opts = cpp_config.strip_opts(), + ) + command_line = cc_common.get_memory_inefficient_command_line( + feature_configuration = feature_configuration, + action_name = "strip", + variables = variables, + ) + env = cc_common.get_environment_variables( + feature_configuration = feature_configuration, + action_name = "strip", + variables = variables, + ) + execution_info = {} + for execution_requirement in cc_common.get_tool_requirement_for_action(feature_configuration = feature_configuration, action_name = "strip"): + execution_info[execution_requirement] = "" + ctx.actions.run( + inputs = depset( + direct = [input], + transitive = [cc_toolchain._strip_files], + ), + outputs = [output], + use_default_shell_env = True, + env = env, + executable = cc_common.get_tool_for_action(feature_configuration = feature_configuration, action_name = "strip"), + toolchain = CC_TOOLCHAIN_TYPE, + execution_requirements = execution_info, + progress_message = "Stripping {} for {}".format(output.short_path, ctx.label), + mnemonic = "CcStrip", + arguments = command_line, + ) def _lookup_var(ctx, additional_vars, var): expanded_make_var_ctx = ctx.var.get(var) @@ -195,9 +267,29 @@ def _tokenize(options, options_string): if force_token or len(token) > 0: options.append("".join(token)) +def _should_use_pic(ctx, cc_toolchain, feature_configuration): + """Whether to use pic files + + Args: + ctx: (RuleContext) + cc_toolchain: (CcToolchainInfo) + feature_configuration: (FeatureConfiguration) + + Returns: + (bool) + """ + return ctx.fragments.cpp.force_pic() or ( + cc_toolchain.needs_pic_for_dynamic_libraries(feature_configuration = feature_configuration) and ( + ctx.var["COMPILATION_MODE"] != "opt" or + cc_common.is_enabled(feature_configuration = feature_configuration, feature_name = "prefer_pic_for_opt_binaries") + ) + ) + cc_helper = struct( + create_strip_action = _create_strip_action, get_expanded_env = _get_expanded_env, + get_static_mode_params_for_dynamic_library_libraries = _get_static_mode_params_for_dynamic_library_libraries, + should_use_pic = _should_use_pic, tokenize = _tokenize, ) - -# LINT.ThenChange(https://github.com/bazelbuild/bazel/blob/master/src/main/starlark/builtins_bzl/common/cc/cc_helper.bzl) +# LINT.ThenChange(https://github.com/bazelbuild/bazel/blob/master/src/main/starlark/builtins_bzl/common/cc/cc_helper.bzl:forked_exports) diff --git a/cc/common/visibility.bzl b/cc/common/visibility.bzl new file mode 100644 index 0000000..dfbc8ce --- /dev/null +++ b/cc/common/visibility.bzl @@ -0,0 +1,3 @@ +"""Bzl load visibility package specs""" + +INTERNAL_VISIBILITY = ["//cc/..."]