feat: introduce zstd toolchain (#831)
diff --git a/MODULE.bazel b/MODULE.bazel index 3bfd9b2..b4838fa 100644 --- a/MODULE.bazel +++ b/MODULE.bazel
@@ -21,9 +21,10 @@ bazel_lib_toolchains.yq() bazel_lib_toolchains.coreutils() bazel_lib_toolchains.tar() +bazel_lib_toolchains.zstd() bazel_lib_toolchains.expand_template() bazel_lib_toolchains.bats() -use_repo(bazel_lib_toolchains, "bats_toolchains", "bsd_tar_toolchains", "copy_directory_toolchains", "copy_to_directory_toolchains", "coreutils_toolchains", "expand_template_toolchains", "jq_toolchains", "yq_toolchains") +use_repo(bazel_lib_toolchains, "bats_toolchains", "bsd_tar_toolchains", "copy_directory_toolchains", "copy_to_directory_toolchains", "coreutils_toolchains", "expand_template_toolchains", "jq_toolchains", "yq_toolchains", "zstd_toolchains") register_toolchains( "@copy_directory_toolchains//:all",
diff --git a/docs/repositories.md b/docs/repositories.md index 578fc40..65ffe07 100644 --- a/docs/repositories.md +++ b/docs/repositories.md
@@ -190,3 +190,22 @@ | <a id="register_yq_toolchains-register"></a>register | whether to call through to native.register_toolchains. Should be True for WORKSPACE users, but false when used under bzlmod extension | <code>True</code> | +<a id="register_zstd_toolchains"></a> + +## register_zstd_toolchains + +<pre> +register_zstd_toolchains(<a href="#register_zstd_toolchains-name">name</a>, <a href="#register_zstd_toolchains-register">register</a>) +</pre> + +Registers zstd toolchain and repositories + +**PARAMETERS** + + +| Name | Description | Default Value | +| :------------- | :------------- | :------------- | +| <a id="register_zstd_toolchains-name"></a>name | override the prefix for the generated toolchain repositories | <code>"zstd"</code> | +| <a id="register_zstd_toolchains-register"></a>register | whether to call through to native.register_toolchains. Should be True for WORKSPACE users, but false when used under bzlmod extension | <code>True</code> | + +
diff --git a/lib/BUILD.bazel b/lib/BUILD.bazel index b03d809..c1d8087 100644 --- a/lib/BUILD.bazel +++ b/lib/BUILD.bazel
@@ -65,6 +65,10 @@ ) toolchain_type( + name = "zstd_toolchain_type", +) + +toolchain_type( name = "bats_toolchain_type", ) @@ -254,6 +258,7 @@ "//lib/private:source_toolchains_repo", "//lib/private:tar_toolchain", "//lib/private:yq_toolchain", + "//lib/private:zstd_toolchain", ], )
diff --git a/lib/extensions.bzl b/lib/extensions.bzl index 8ba94dc..635c922 100644 --- a/lib/extensions.bzl +++ b/lib/extensions.bzl
@@ -14,6 +14,7 @@ "DEFAULT_TAR_REPOSITORY", "DEFAULT_YQ_REPOSITORY", "DEFAULT_YQ_VERSION", + "DEFAULT_ZSTD_REPOSITORY", "register_bats_toolchains", "register_copy_directory_toolchains", "register_copy_to_directory_toolchains", @@ -22,6 +23,7 @@ "register_jq_toolchains", "register_tar_toolchains", "register_yq_toolchains", + "register_zstd_toolchains", ) load("//lib/private:extension_utils.bzl", "extension_utils") load("//lib/private:host_repo.bzl", "host_repo") @@ -91,6 +93,15 @@ extension_utils.toolchain_repos_bfs( mctx = mctx, + get_tag_fn = lambda tags: tags.zstd, + toolchain_name = "zstd", + default_repository = DEFAULT_ZSTD_REPOSITORY, + toolchain_repos_fn = lambda name, version: register_zstd_toolchains(name = name, register = False), + get_version_fn = lambda attr: None, + ) + + extension_utils.toolchain_repos_bfs( + mctx = mctx, get_tag_fn = lambda tags: tags.expand_template, toolchain_name = "expand_template", toolchain_repos_fn = lambda name, version: register_expand_template_toolchains(name = name, register = False), @@ -115,6 +126,7 @@ "yq": tag_class(attrs = {"name": attr.string(default = DEFAULT_YQ_REPOSITORY), "version": attr.string(default = DEFAULT_YQ_VERSION)}), "coreutils": tag_class(attrs = {"name": attr.string(default = DEFAULT_COREUTILS_REPOSITORY), "version": attr.string(default = DEFAULT_COREUTILS_VERSION)}), "tar": tag_class(attrs = {"name": attr.string(default = DEFAULT_TAR_REPOSITORY)}), + "zstd": tag_class(attrs = {"name": attr.string(default = DEFAULT_ZSTD_REPOSITORY)}), "expand_template": tag_class(attrs = {"name": attr.string(default = DEFAULT_EXPAND_TEMPLATE_REPOSITORY)}), "bats": tag_class(attrs = { "name": attr.string(default = DEFAULT_BATS_REPOSITORY),
diff --git a/lib/private/BUILD.bazel b/lib/private/BUILD.bazel index 1980f3f..3134c29 100644 --- a/lib/private/BUILD.bazel +++ b/lib/private/BUILD.bazel
@@ -356,3 +356,9 @@ srcs = ["strings.bzl"], visibility = ["//lib:__subpackages__"], ) + +bzl_library( + name = "zstd_toolchain", + srcs = ["zstd_toolchain.bzl"], + visibility = ["//lib:__subpackages__"], +)
diff --git a/lib/private/zstd_toolchain.bzl b/lib/private/zstd_toolchain.bzl new file mode 100644 index 0000000..a3ea442 --- /dev/null +++ b/lib/private/zstd_toolchain.bzl
@@ -0,0 +1,177 @@ +"Provide access to ZSTD" + +ZSTD_PLATFORMS = { + "darwin_amd64": struct( + compatible_with = [ + "@platforms//os:osx", + "@platforms//cpu:x86_64", + ], + ), + "darwin_arm64": struct( + compatible_with = [ + "@platforms//os:osx", + "@platforms//cpu:aarch64", + ], + ), + "linux_amd64": struct( + compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + ), + "linux_arm64": struct( + compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:aarch64", + ], + ), +} + +ZSTD_PREBUILT = { + "darwin_amd64": ( + "https://github.com/aspect-build/zstd-prebuilt/releases/download/v1.5.6/zstd_darwin_amd64", + "e4d517212005cf26f8b8d657455d1380318b071cb52a3ffd9dfbdf4c2ba71a13", + ), + "darwin_arm64": ( + "https://github.com/aspect-build/zstd-prebuilt/releases/download/v1.5.6/zstd_darwin_arm64", + "6e210eeae08fb6ba38c3ac2d1857075c28113aef68296f7e396f1180f7e894b9", + ), + "linux_amd64": ( + "https://github.com/aspect-build/zstd-prebuilt/releases/download/v1.5.6/zstd_linux_amd64", + "0f0bd1193509a598629d7fa745c4b0b6d5fa6719e0c94c01ef0f20e466d801a7", + ), + "linux_arm64": ( + "https://github.com/aspect-build/zstd-prebuilt/releases/download/v1.5.6/zstd_linux_arm64", + "82aacf8f1c67ff3c94e04afb0721a848bbba70fbf8249ee4bc4c9085afb84548", + ), +} + +def _zstd_binary_repo(rctx): + (url, sha256) = ZSTD_PREBUILT[rctx.attr.platform] + rctx.download( + url = url, + output = "zstd", + executable = True, + sha256 = sha256, + ) + binary = "zstd" + + rctx.file("BUILD.bazel", """\ +# @generated by @aspect_bazel_lib//lib/private:zstd_toolchain.bzl + +load("@aspect_bazel_lib//lib/private:zstd_toolchain.bzl", "zstd_toolchain") + +package(default_visibility = ["//visibility:public"]) + +zstd_toolchain(name = "zstd_toolchain", binary = "{}") +""".format(binary)) + +zstd_binary_repo = repository_rule( + implementation = _zstd_binary_repo, + attrs = { + "platform": attr.string(mandatory = True, values = ZSTD_PLATFORMS.keys()), + }, +) + +ZstdInfo = provider( + doc = "Provide info for executing zstd", + fields = { + "binary": "zstd executable", + }, +) + +def _zstd_toolchain_impl(ctx): + binary = ctx.executable.binary + + # Make the $(ZSTD_BIN) variable available in places like genrules. + # See https://docs.bazel.build/versions/main/be/make-variables.html#custom_variables + template_variables = platform_common.TemplateVariableInfo({ + "ZSTD_BIN": binary.path, + }) + + default_info = DefaultInfo( + files = depset(ctx.files.binary + ctx.files.files), + ) + zstdinfo = ZstdInfo( + binary = binary, + ) + + # Export all the providers inside our ToolchainInfo + # so the resolved_toolchain rule can grab and re-export them. + toolchain_info = platform_common.ToolchainInfo( + zstdinfo = zstdinfo, + template_variables = template_variables, + default = default_info, + ) + + return [toolchain_info, template_variables, default_info] + +zstd_toolchain = rule( + implementation = _zstd_toolchain_impl, + attrs = { + "binary": attr.label( + doc = "a command to find on the system path", + allow_files = True, + executable = True, + cfg = "exec", + ), + "files": attr.label_list(allow_files = True), + }, +) + +def _zstd_toolchains_repo_impl(rctx): + # Expose a concrete toolchain which is the result of Bazel resolving the toolchain + # for the execution or zstdget platform. + # Workaround for https://github.com/bazelbuild/bazel/issues/14009 + starlark_content = """\ +# @generated by @aspect_bazel_lib//lib/private:zstd_toolchain.bzl + +# Forward all the providers +def _resolved_toolchain_impl(ctx): + toolchain_info = ctx.toolchains["@aspect_bazel_lib//lib:zstd_toolchain_type"] + return [ + toolchain_info, + toolchain_info.default, + toolchain_info.zstdinfo, + toolchain_info.template_variables, + ] + +# Copied from java_toolchain_alias +# https://cs.opensource.google/bazel/bazel/+/master:tools/jdk/java_toolchain_alias.bzl +resolved_toolchain = rule( + implementation = _resolved_toolchain_impl, + toolchains = ["@aspect_bazel_lib//lib:zstd_toolchain_type"], + incompatible_use_toolchain_transition = True, +) +""" + rctx.file("defs.bzl", starlark_content) + + build_content = """# @generated by @aspect_bazel_lib//lib/private:zstd_toolchain.bzl +load(":defs.bzl", "resolved_toolchain") +load("@local_config_platform//:constraints.bzl", "HOST_CONSTRAINTS") + +resolved_toolchain(name = "resolved_toolchain", visibility = ["//visibility:public"])""" + + for [platform, meta] in ZSTD_PLATFORMS.items(): + build_content += """ +toolchain( + name = "{platform}_toolchain", + exec_compatible_with = {compatible_with}, + toolchain = "@{user_repository_name}_{platform}//:zstd_toolchain", + toolchain_type = "@aspect_bazel_lib//lib:zstd_toolchain_type", +) +""".format( + platform = platform, + user_repository_name = rctx.attr.user_repository_name, + compatible_with = meta.compatible_with, + ) + + rctx.file("BUILD.bazel", build_content) + +zstd_toolchains_repo = repository_rule( + _zstd_toolchains_repo_impl, + doc = """Creates a repository that exposes a zstd_toolchain_type zstdget.""", + attrs = { + "user_repository_name": attr.string(doc = "Base name for toolchains repository"), + }, +)
diff --git a/lib/repositories.bzl b/lib/repositories.bzl index 62b3abe..2cd02c9 100644 --- a/lib/repositories.bzl +++ b/lib/repositories.bzl
@@ -10,6 +10,7 @@ load("//lib/private:source_toolchains_repo.bzl", "source_toolchains_repo") load("//lib/private:tar_toolchain.bzl", "BSDTAR_PLATFORMS", "bsdtar_binary_repo", "tar_toolchains_repo") load("//lib/private:yq_toolchain.bzl", "YQ_PLATFORMS", "yq_host_alias_repo", "yq_platform_repo", "yq_toolchains_repo", _DEFAULT_YQ_VERSION = "DEFAULT_YQ_VERSION") +load("//lib/private:zstd_toolchain.bzl", "ZSTD_PLATFORMS", "zstd_binary_repo", "zstd_toolchains_repo") load("//tools:version.bzl", "IS_PRERELEASE") # buildifier: disable=unnamed-macro @@ -104,6 +105,29 @@ user_repository_name = name, ) +DEFAULT_ZSTD_REPOSITORY = "zstd" + +def register_zstd_toolchains(name = DEFAULT_ZSTD_REPOSITORY, register = True): + """Registers zstd toolchain and repositories + + Args: + name: override the prefix for the generated toolchain repositories + register: whether to call through to native.register_toolchains. + Should be True for WORKSPACE users, but false when used under bzlmod extension + """ + for [platform, _] in ZSTD_PLATFORMS.items(): + zstd_binary_repo( + name = "%s_%s" % (name, platform), + platform = platform, + ) + if register: + native.register_toolchains("@%s_toolchains//:%s_toolchain" % (name, platform)) + + zstd_toolchains_repo( + name = "%s_toolchains" % name, + user_repository_name = name, + ) + DEFAULT_BATS_REPOSITORY = "bats" DEFAULT_BATS_CORE_VERSION = "v1.10.0" @@ -324,4 +348,5 @@ register_jq_toolchains() register_yq_toolchains() register_tar_toolchains() + register_zstd_toolchains() register_bats_toolchains()
diff --git a/lib/tests/zstd/BUILD.bazel b/lib/tests/zstd/BUILD.bazel new file mode 100644 index 0000000..aede113 --- /dev/null +++ b/lib/tests/zstd/BUILD.bazel
@@ -0,0 +1,30 @@ +load("@aspect_bazel_lib//lib:testing.bzl", "assert_archive_contains") + +# Case 1: Can decompress gzip archive +genrule( + name = "tar", + srcs = [ + "srcfile", + ], + outs = ["1.tar.gz"], + cmd = "$(BSDTAR_BIN) --create --gzip --dereference --file $@ -s '#$(BINDIR)##' $(execpath srcfile)", + toolchains = ["@bsd_tar_toolchains//:resolved_toolchain"], +) + +genrule( + name = "decompress_tar", + srcs = [ + ":tar", + ], + outs = ["1.tar"], + cmd = "$(ZSTD_BIN) -f --decompress $(execpath :tar) --stdout > $@", + toolchains = ["@zstd_toolchains//:resolved_toolchain"], +) + +assert_archive_contains( + name = "test_decompressed", + archive = "1.tar", + expected = [ + "lib/tests/zstd/srcfile", + ], +)
diff --git a/lib/tests/zstd/srcfile b/lib/tests/zstd/srcfile new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/lib/tests/zstd/srcfile