Migrate DexArchiveAspect from native Blaze to Starlark, and also add 2 new processors in android_binary_internal: DexProcessor and DeployJarProcessor PiperOrigin-RevId: 535526481 Change-Id: I27046c4dc89129b6b22c4ee1137d07e1c69badc8
diff --git a/rules/acls/android_binary_starlark_dex_desugar_proguard.bzl b/rules/acls/android_binary_starlark_dex_desugar_proguard.bzl index 9a212c2..1f7ec8b 100644 --- a/rules/acls/android_binary_starlark_dex_desugar_proguard.bzl +++ b/rules/acls/android_binary_starlark_dex_desugar_proguard.bzl
@@ -15,7 +15,9 @@ """Allow list for rollout of Starlark dex, desugar and proguard in android_binary_internal.""" # keep sorted -ANDROID_BINARY_STARLARK_DEX_DESUGAR_PROGUARD_ROLLOUT = [] +ANDROID_BINARY_STARLARK_DEX_DESUGAR_PROGUARD_ROLLOUT = [ + "//test/rules/android_binary_internal:__subpackages__", +] # keep sorted ANDROID_BINARY_STARLARK_DEX_DESUGAR_PROGUARD_FALLBACK = []
diff --git a/rules/android_binary_internal/attrs.bzl b/rules/android_binary_internal/attrs.bzl index 5a6f807..ed14647 100644 --- a/rules/android_binary_internal/attrs.bzl +++ b/rules/android_binary_internal/attrs.bzl
@@ -23,12 +23,14 @@ "split_config_aspect", ) load("//rules:providers.bzl", "StarlarkApkInfo") +load("//rules:dex_desugar_aspect.bzl", "dex_desugar_aspect") -def make_deps(allow_rules, providers): +def make_deps(allow_rules, providers, aspects): return attr.label_list( allow_files = True, allow_rules = allow_rules, providers = providers, + aspects = aspects, cfg = android_common.multi_cpu_configuration, ) @@ -47,6 +49,10 @@ ["AndroidResourcesInfo", "AndroidAssetsInfo"], ] +DEPS_ASPECTS = [ + dex_desugar_aspect, +] + ATTRS = _attrs.replace( _attrs.add( dict( @@ -54,7 +60,7 @@ # TODO(timpeut): Set PropertyFlag direct_compile_time_input allow_files = [".java", ".srcjar"], ), - deps = make_deps(DEPS_ALLOW_RULES, DEPS_PROVIDERS), + deps = make_deps(DEPS_ALLOW_RULES, DEPS_PROVIDERS, DEPS_ASPECTS), enable_data_binding = attr.bool(), instruments = attr.label(), manifest_values = attr.string_dict(), @@ -82,6 +88,12 @@ shrink_resources = _attrs.tristate.create( default = _attrs.tristate.auto, ), + dexopts = attr.string_list(), + main_dex_list = attr.label(allow_single_file = True), + min_sdk_version = attr.int(), + incremental_dexing = _attrs.tristate.create( + default = _attrs.tristate.auto, + ), _java_toolchain = attr.label( default = Label("//tools/jdk:toolchain_android_only"), ),
diff --git a/rules/android_binary_internal/impl.bzl b/rules/android_binary_internal/impl.bzl index 1b402c3..0bf92eb 100644 --- a/rules/android_binary_internal/impl.bzl +++ b/rules/android_binary_internal/impl.bzl
@@ -29,7 +29,9 @@ "//rules:native_deps.bzl", _process_native_deps = "process", ) -load("//rules:providers.bzl", "StarlarkApkInfo") +load("//rules:providers.bzl", "StarlarkAndroidDexInfo", "StarlarkApkInfo") +load("//rules:dex.bzl", _dex = "dex") +load("//rules:desugar.bzl", _desugar = "desugar") def _process_manifest(ctx, **unused_ctxs): manifest_ctx = _resources.bump_min_sdk( @@ -192,6 +194,111 @@ ), ) +def _process_dex(ctx, packaged_resources_ctx, jvm_ctx, deploy_ctx, **_unused_ctxs): + providers = [] + + if acls.in_android_binary_starlark_dex_desugar_proguard(str(ctx.label)): + runtime_jars = jvm_ctx.java_info.runtime_output_jars + [packaged_resources_ctx.class_jar] + forbidden_dexopts = ctx.fragments.android.get_target_dexopts_that_prevent_incremental_dexing + classes_dex_zip = None + + incremental_dexing = _dex.get_effective_incremental_dexing( + force_incremental_dexing = ctx.attr.incremental_dexing, + has_forbidden_dexopts = len([d for d in ctx.attr.dexopts if d in forbidden_dexopts]) > 0, + is_binary_optimized = len(ctx.attr.proguard_specs) > 0, + incremental_dexing_after_proguard_by_default = ctx.fragments.android.incremental_dexing_after_proguard_by_default, + incremental_dexing_shards_after_proguard = ctx.fragments.android.incremental_dexing_shards_after_proguard, + use_incremental_dexing = ctx.fragments.android.use_incremental_dexing, + ) + + # TODO(b/263473668): Implement dexing after optimization + if incremental_dexing: + classes_dex_zip = _dex.process_incremental_dexing( + ctx, + deps = ctx.attr.deps, + dexopts = ctx.attr.dexopts, + runtime_jars = runtime_jars, + main_dex_list = ctx.file.main_dex_list, + min_sdk_version = ctx.attr.min_sdk_version, + java_info = jvm_ctx.java_info, + desugar_dict = deploy_ctx.desugar_dict, + dexbuilder = get_android_toolchain(ctx).dexbuilder.files_to_run, + dexmerger = get_android_toolchain(ctx).dexmerger.files_to_run, + ) + + providers.append( + AndroidDexInfo( + deploy_jar = deploy_ctx.deploy_jar, + final_classes_dex_zip = classes_dex_zip, + java_resource_jar = deploy_ctx.deploy_jar, + ), + ) + + return ProviderInfo( + name = "dex_ctx", + value = struct(providers = providers), + ) + +def _process_deploy_jar(ctx, packaged_resources_ctx, jvm_ctx, build_info_ctx, **unused_ctx): + deploy_jar, desugar_dict = None, {} + + if acls.in_android_binary_starlark_dex_desugar_proguard(str(ctx.label)): + java_toolchain = common.get_java_toolchain(ctx) + java_info = jvm_ctx.java_info + info = _dex.merge_infos(utils.collect_providers(StarlarkAndroidDexInfo, ctx.attr.deps)) + incremental_dexopts = _dex.incremental_dexopts(ctx.attr.dexopts, ctx.fragments.android.get_dexopts_supported_in_incremental_dexing) + dex_archives = info.dex_archives_dict.get("".join(incremental_dexopts), depset()).to_list() + + if ctx.fragments.android.desugar_java8: + desugared_jars = [] + desugar_dict = {d.jar: d.desugared_jar for d in dex_archives} + + for jar in java_info.runtime_output_jars + [packaged_resources_ctx.class_jar]: + desugared_jar = ctx.actions.declare_file(ctx.label.name + "/" + jar.basename + "_desugared.jar") + _desugar.desugar( + ctx, + input = jar, + output = desugared_jar, + classpath = java_info.transitive_compile_time_jars, + bootclasspath = java_toolchain[java_common.JavaToolchainInfo].bootclasspath.to_list(), + min_sdk_version = ctx.attr.min_sdk_version, + desugar_exec = get_android_toolchain(ctx).desugar.files_to_run, + ) + desugared_jars.append(desugared_jar) + desugar_dict[jar] = desugared_jar + + for jar in java_info.transitive_runtime_jars.to_list(): + if jar in desugar_dict and desugar_dict[jar]: + desugared_jars.append(desugar_dict[jar]) + + runtime_jars = depset(desugared_jars) + else: + runtime_jars = depset(transitive = [ + java_info.runtime_output_jars, + java_info.transitive_runtime_jars, + ]) + + output = ctx.actions.declare_file(ctx.label.name + "_migrated_deploy.jar") + deploy_jar = java.create_deploy_jar( + ctx, + output = output, + runtime_jars = runtime_jars, + java_toolchain = java_toolchain, + target_name = ctx.label.name, + build_info_files = build_info_ctx.build_info_files, + deploy_manifest_lines = build_info_ctx.deploy_manifest_lines, + extra_build_info = build_info_ctx.extra_build_info, + ) + + return ProviderInfo( + name = "deploy_ctx", + value = struct( + deploy_jar = deploy_jar, + desugar_dict = desugar_dict, + providers = [], + ), + ) + def use_legacy_manifest_merger(ctx): """Whether legacy manifest merging is enabled. @@ -243,6 +350,8 @@ DataBindingProcessor = _process_data_binding, JvmProcessor = _process_jvm, BuildInfoProcessor = _process_build_info, + DeployJarProcessor = _process_deploy_jar, + DexProcessor = _process_dex, ) _PROCESSING_PIPELINE = processing_pipeline.make_processing_pipeline(
diff --git a/rules/desugar.bzl b/rules/desugar.bzl new file mode 100644 index 0000000..c498e00 --- /dev/null +++ b/rules/desugar.bzl
@@ -0,0 +1,67 @@ +# Copyright 2020 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Bazel Desugar Commands.""" + +def _desugar( + ctx, + input, + output = None, + classpath = None, + bootclasspath = [], + min_sdk_version = 0, + library_desugaring = True, + desugar_exec = None): + """Desugars a JAR. + + Args: + ctx: The context. + input: File. The jar to be desugared. + output: File. The desugared jar. + classpath: Depset of Files. The transitive classpath needed to resolve symbols in the input jar. + bootclasspath: List of Files. Bootclasspaths that was used to compile the input jar with. + min_sdk_version: Integer. The minimum targeted sdk version. + library_desugaring: Boolean. Whether to enable core library desugaring. + desugar_exec: File. The executable desugar file. + """ + + args = ctx.actions.args() + args.add("--input", input) + args.add("--output", output) + args.add_all(classpath, before_each = "--classpath_entry") + args.add_all(bootclasspath, before_each = "--bootclasspath_entry") + + if library_desugaring: + if ctx.fragments.android.check_desugar_deps: + args.add("--emit_dependency_metadata_as_needed") + + if ctx.fragments.android.desugar_java8_libs and library_desugaring: + args.add("--desugar_supported_core_libs") + + if min_sdk_version > 0: + args.add("--min_sdk_version", str(min_sdk_version)) + + ctx.actions.run( + inputs = depset([input] + bootclasspath, transitive = [classpath]), + outputs = [output], + executable = desugar_exec, + arguments = [args], + mnemonic = "Desugar", + progress_message = "Desugaring " + input.short_path + " for Android", + execution_requirements = {"supports-workers": "1"}, + ) + +desugar = struct( + desugar = _desugar, +)
diff --git a/rules/dex.bzl b/rules/dex.bzl index 5d9592d..52df692 100644 --- a/rules/dex.bzl +++ b/rules/dex.bzl
@@ -14,11 +14,72 @@ """Bazel Dex Commands.""" +load(":utils.bzl", "utils") +load(":providers.bzl", "StarlarkAndroidDexInfo") load("@bazel_skylib//lib:collections.bzl", "collections") load("//rules:attrs.bzl", _attrs = "attrs") _tristate = _attrs.tristate +def _process_incremental_dexing( + ctx, + deps = [], + runtime_jars = [], + dexopts = [], + main_dex_list = [], + min_sdk_version = 0, + java_info = None, + desugar_dict = {}, + dexbuilder = None, + dexmerger = None): + classes_dex_zip = _get_dx_artifact(ctx, "classes.dex.zip") + info = _merge_infos(utils.collect_providers(StarlarkAndroidDexInfo, deps)) + + incremental_dexopts = _incremental_dexopts(dexopts, ctx.fragments.android.get_dexopts_supported_in_incremental_dexing) + dex_archives_list = info.dex_archives_dict.get("".join(incremental_dexopts), depset()).to_list() + dex_archives = _to_dexed_classpath( + dex_archives_dict = {d.jar: d.dex for d in dex_archives_list}, + classpath = java_info.transitive_runtime_jars.to_list(), + runtime_jars = runtime_jars, + ) + + for jar in runtime_jars: + dex_archive = _get_dx_artifact(ctx, jar.basename + ".dex.zip") + _dex( + ctx, + input = desugar_dict[jar] if jar in desugar_dict else jar, + output = dex_archive, + incremental_dexopts = incremental_dexopts, + min_sdk_version = min_sdk_version, + dex_exec = dexbuilder, + ) + dex_archives.append(dex_archive) + + _dex_merge( + ctx, + output = classes_dex_zip, + inputs = dex_archives, + multidex_strategy = "minimal", + main_dex_list = main_dex_list, + dexopts = dexopts, + dexmerger = dexmerger, + ) + + return classes_dex_zip + +def _to_dexed_classpath(dex_archives_dict = {}, classpath = [], runtime_jars = []): + dexed_classpath = [] + for jar in classpath: + if jar not in dex_archives_dict: + if jar not in runtime_jars: + fail("Dependencies on .jar artifacts are not allowed in Android binaries, please use " + + "a java_import to depend on " + jar.short_path + + ". If this is an implicit dependency then the rule that " + + "introduces it will need to be fixed to account for it correctly.") + else: + dexed_classpath.append(dex_archives_dict[jar]) + return dexed_classpath + def _dex( ctx, input, @@ -101,7 +162,7 @@ args.add("--multidex", multidex_strategy) args.add_all(inputs, before_each = "--input") args.add("--output", output) - args.add_all(_merger_dexopts(ctx, dexopts)) + args.add_all(_merger_dexopts(dexopts, ctx.fragments.android.get_dexopts_supported_in_dex_merger)) if main_dex_list: inputs.append(main_dex_list) @@ -122,6 +183,19 @@ def _incremental_dexopts(tokenized_dexopts, dexopts_supported_in_incremental_dexing): return _normalize_dexopts(_filter_dexopts(tokenized_dexopts, dexopts_supported_in_incremental_dexing)) +def _merge_infos(infos): + dex_archives_dict = {} + for info in infos: + for dexopts in info.dex_archives_dict: + if dexopts not in dex_archives_dict: + dex_archives_dict[dexopts] = [info.dex_archives_dict[dexopts]] + else: + dex_archives_dict[dexopts].append(info.dex_archives_dict[dexopts]) + return StarlarkAndroidDexInfo( + dex_archives_dict = + {dexopts: depset(direct = [], transitive = dex_archives) for dexopts, dex_archives in dex_archives_dict.items()}, + ) + def _filter_dexopts(candidates, allowed): return [c for c in candidates if c in allowed] @@ -137,5 +211,7 @@ get_dx_artifact = _get_dx_artifact, get_effective_incremental_dexing = _get_effective_incremental_dexing, incremental_dexopts = _incremental_dexopts, + merge_infos = _merge_infos, normalize_dexopts = _normalize_dexopts, + process_incremental_dexing = _process_incremental_dexing, )
diff --git a/rules/dex_desugar_aspect.bzl b/rules/dex_desugar_aspect.bzl new file mode 100644 index 0000000..544000c --- /dev/null +++ b/rules/dex_desugar_aspect.bzl
@@ -0,0 +1,242 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Aspect that transitively build .dex archives and desugar jars.""" + +load(":utils.bzl", _utils = "utils") +load(":dex.bzl", _dex = "dex") +load(":desugar.bzl", _desugar = "desugar") +load(":providers.bzl", "StarlarkAndroidDexInfo") +load(":attrs.bzl", _attrs = "attrs") +load("//rules:acls.bzl", "acls") + +_tristate = _attrs.tristate + +def _aspect_attrs(): + """Attrs of the rule requiring traversal by the aspect.""" + return [ + "aidl_lib", # for the aidl runtime in the android_sdk rule + "deps", + "exports", + "runtime", + "runtime_deps", + "_android_sdk", + "_aspect_proto_toolchain_for_javalite", + "_build_stamp_mergee_manifest_lib", # To get from proto_library through proto_lang_toolchain rule to proto runtime library. + "_toolchain", # to get Kotlin toolchain component in android_library + ] + +def _aspect_impl(target, ctx): + """Adapts the rule and target data. + + Args: + target: The target. + ctx: The context. + + Returns: + A list of providers. + """ + min_sdk_version = getattr(ctx.rule.attr, "min_sdk_version", 0) + if min_sdk_version != 0 and not acls.in_android_binary_min_sdk_version_attribute(str(ctx)): + fail("Target is not allowed to set a min_sdk_version value.") + + incremental_dexing = getattr(ctx.rule.attr, "incremental_dexing", _tristate.auto) + + if incremental_dexing == _tristate.no or \ + (not ctx.fragments.android.use_incremental_dexing and + incremental_dexing == _tristate.auto): + return [] + + # TODO(b/33557068): Desugar protos if needed instead of assuming they don't need desugaring + ignore_desugar = not ctx.fragments.android.desugar_java8 or ctx.rule.kind == "proto_library" + + extra_toolchain_jars = _get_platform_based_toolchain_jars(ctx) + + if hasattr(ctx.rule.attr, "neverlink") and ctx.rule.attr.neverlink: + return [] + + dex_archives_dict = {} + runtime_jars = _get_produced_runtime_jars(target, ctx, extra_toolchain_jars) + bootclasspath = _get_boot_classpath(target, ctx) + compiletime_classpath = target[JavaInfo].transitive_compile_time_jars if JavaInfo in target else None + if runtime_jars: + basename_clash = _check_basename_clash(runtime_jars) + aspect_dexopts = _get_aspect_dexopts(ctx) + min_sdk_filename_part = "--min_sdk_version=" + min_sdk_version if min_sdk_version > 0 else "" + for jar in runtime_jars: + if not ignore_desugar: + unique_desugar_filename = (jar.path if basename_clash else jar.basename) + \ + min_sdk_filename_part + "_desugared.jar" + desugared_jar = _dex.get_dx_artifact(ctx, unique_desugar_filename) + _desugar.desugar( + ctx, + input = jar, + output = desugared_jar, + bootclasspath = bootclasspath, + classpath = compiletime_classpath, + min_sdk_version = min_sdk_version, + desugar_exec = ctx.executable._desugar_java8, + ) + else: + desugared_jar = None + + for incremental_dexopts_list in aspect_dexopts: + incremental_dexopts = "".join(incremental_dexopts_list) + + unique_dx_filename = (jar.short_path if basename_clash else jar.basename) + \ + incremental_dexopts + min_sdk_filename_part + ".dex.zip" + dex = _dex.get_dx_artifact(ctx, unique_dx_filename) + _dex.dex( + ctx, + input = desugared_jar if desugared_jar else jar, + output = dex, + incremental_dexopts = incremental_dexopts_list, + min_sdk_version = min_sdk_version, + dex_exec = ctx.executable._dexbuilder, + ) + + dex_archive = struct( + jar = jar, + desugared_jar = desugared_jar, + dex = dex, + ) + + if incremental_dexopts not in dex_archives_dict: + dex_archives_dict[incremental_dexopts] = [] + dex_archives_dict[incremental_dexopts].append(dex_archive) + + infos = _utils.collect_providers(StarlarkAndroidDexInfo, _get_aspect_deps(ctx)) + merged_info = _dex.merge_infos(infos) + + for dexopts in dex_archives_dict: + if dexopts in merged_info.dex_archives_dict: + merged_info.dex_archives_dict[dexopts] = depset(dex_archives_dict[dexopts], transitive = [merged_info.dex_archives_dict[dexopts]]) + else: + merged_info.dex_archives_dict[dexopts] = depset(dex_archives_dict[dexopts]) + + return [ + StarlarkAndroidDexInfo( + dex_archives_dict = merged_info.dex_archives_dict, + ), + ] + +def _get_aspect_deps(ctx): + deps_list = [] + for deps in [getattr(ctx.rule.attr, attr) for attr in _aspect_attrs() if hasattr(ctx.rule.attr, attr)]: + if str(type(deps)) == "list": + deps_list += deps + else: + deps_list.append(deps) + return deps_list + +def _get_produced_runtime_jars(target, ctx, extra_toolchain_jars): + if ctx.rule.kind == "proto_library": + if not getattr(ctx.rule.attr, "srcs", []): + if JavaInfo in target: + return [java_output.class_jar for java_output in target[JavaInfo].java_outputs] + return [] + else: + jars = [] + if JavaInfo in target: + jars.extend(target[JavaInfo].runtime_output_jars) + + # TODO(b/124540821): Disable R.jar desugaring (with a flag). + if AndroidIdeInfo in target and target[AndroidIdeInfo].resource_jar: + jars.append(target[AndroidIdeInfo].resource_jar.class_jar) + + if AndroidApplicationResourceInfo in target and target[AndroidApplicationResourceInfo].build_stamp_jar: + jars.append(target[AndroidApplicationResourceInfo].build_stamp_jar) + + jars.extend(extra_toolchain_jars) + return jars + +def _get_platform_based_toolchain_jars(ctx): + if not ctx.fragments.android.incompatible_use_toolchain_resolution: + return [] + + if not getattr(ctx.rule.attr, "_android_sdk", None): + return [] + + android_sdk = ctx.rule.attr._android_sdk + + if AndroidSdkInfo in android_sdk and android_sdk[AndroidSdkInfo].aidl_lib: + return android_sdk[AndroidSdkInfo].aidl_lib[JavaInfo].runtime_output_jars + + return [] + +def _get_aspect_dexopts(ctx): + return _power_set(_dex.normalize_dexopts(ctx.fragments.android.get_dexopts_supported_in_incremental_dexing)) + +def _get_boot_classpath(target, ctx): + if JavaInfo in target: + compilation_info = target[JavaInfo].compilation_info + if compilation_info and compilation_info.boot_classpath: + return compilation_info.boot_classpath + if ctx.attr._android_sdk and ctx.attr._android_sdk[AndroidSdkInfo].android_jar: + return [ctx.attr._android_sdk[AndroidSdkInfo].android_jar] + + # This shouldn't ever be reached, but if it is, we should be clear about the error. + fail("No compilation info or android jar!") + +def _check_basename_clash(artifacts): + seen = {} + for artifact in artifacts: + basename = artifact.basename + if basename not in seen: + seen[basename] = True + else: + return True + return False + +def _power_set(items): + """Calculates the power set of the given items. + """ + + def _exp(base, n): + """ Calculates base ** n.""" + res = 1 + for _ in range(n): + res *= base + return res + + power_set = [] + size = len(items) + + for i in range(_exp(2, size)): + element = [items[j] for j in range(size) if (i // _exp(2, j) % 2) != 0] + power_set.append(element) + + return power_set + +dex_desugar_aspect = aspect( + implementation = _aspect_impl, + attr_aspects = _aspect_attrs(), + attrs = _attrs.add( + { + "_desugar_java8": attr.label( + default = Label("@bazel_tools//tools/android:desugar_java8"), + cfg = "exec", + executable = True, + ), + "_dexbuilder": attr.label( + default = Label("@bazel_tools//tools/android:dexbuilder"), + allow_files = True, + cfg = "exec", + executable = True, + ), + }, + _attrs.ANDROID_SDK, + ), + fragments = ["android"], +)
diff --git a/rules/providers.bzl b/rules/providers.bzl index ef138f4..a377afe 100644 --- a/rules/providers.bzl +++ b/rules/providers.bzl
@@ -151,3 +151,12 @@ unsigned_aab = "File, the unsigned .aab", ), ) + +StarlarkAndroidDexInfo = provider( + doc = "Internal provider used to collect transitive dex info.", + fields = dict( + dex_archives_dict = ( + "A dictionary of all the transitive dex archives for all dexopts." + ), + ), +)