feat: add coreutils toolchain (#332)
diff --git a/docs/BUILD.bazel b/docs/BUILD.bazel index 7c5c2cd..2ce1af4 100644 --- a/docs/BUILD.bazel +++ b/docs/BUILD.bazel
@@ -123,4 +123,9 @@ bzl_library_target = "//lib:base64", ) +stardoc_with_diff_test( + name = "repositories", + bzl_library_target = "//lib:repositories", +) + update_docs()
diff --git a/docs/repositories.md b/docs/repositories.md new file mode 100644 index 0000000..14cd793 --- /dev/null +++ b/docs/repositories.md
@@ -0,0 +1,101 @@ +<!-- Generated with Stardoc: http://skydoc.bazel.build --> + +Macros for loading dependencies and registering toolchains + +<a id="aspect_bazel_lib_dependencies"></a> + +## aspect_bazel_lib_dependencies + +<pre> +aspect_bazel_lib_dependencies(<a href="#aspect_bazel_lib_dependencies-override_local_config_platform">override_local_config_platform</a>) +</pre> + +Load dependencies required by aspect rules + +**PARAMETERS** + + +| Name | Description | Default Value | +| :------------- | :------------- | :------------- | +| <a id="aspect_bazel_lib_dependencies-override_local_config_platform"></a>override_local_config_platform | override the @local_config_platform repository with one that adds stardoc support for loading constraints.bzl.<br><br>Should be set in repositories that load @aspect_bazel_lib copy actions and also generate stardoc. | <code>False</code> | + + +<a id="register_copy_to_directory_toolchains"></a> + +## register_copy_to_directory_toolchains + +<pre> +register_copy_to_directory_toolchains(<a href="#register_copy_to_directory_toolchains-name">name</a>, <a href="#register_copy_to_directory_toolchains-register">register</a>) +</pre> + +Registers copy_to_directory toolchain and repositories + +**PARAMETERS** + + +| Name | Description | Default Value | +| :------------- | :------------- | :------------- | +| <a id="register_copy_to_directory_toolchains-name"></a>name | override the prefix for the generated toolchain repositories | <code>"copy_to_directory"</code> | +| <a id="register_copy_to_directory_toolchains-register"></a>register | whether to call through to native.register_toolchains. Should be True for WORKSPACE users, but false when used under bzlmod extension | <code>True</code> | + + +<a id="register_coreutils_toolchains"></a> + +## register_coreutils_toolchains + +<pre> +register_coreutils_toolchains(<a href="#register_coreutils_toolchains-name">name</a>, <a href="#register_coreutils_toolchains-version">version</a>, <a href="#register_coreutils_toolchains-register">register</a>) +</pre> + +Registers coreutils toolchain and repositories + +**PARAMETERS** + + +| Name | Description | Default Value | +| :------------- | :------------- | :------------- | +| <a id="register_coreutils_toolchains-name"></a>name | override the prefix for the generated toolchain repositories | <code>"coreutils"</code> | +| <a id="register_coreutils_toolchains-version"></a>version | the version of coreutils to execute (see https://github.com/uutils/coreutils/releases) | <code>"0.0.16"</code> | +| <a id="register_coreutils_toolchains-register"></a>register | whether to call through to native.register_toolchains. Should be True for WORKSPACE users, but false when used under bzlmod extension | <code>True</code> | + + +<a id="register_jq_toolchains"></a> + +## register_jq_toolchains + +<pre> +register_jq_toolchains(<a href="#register_jq_toolchains-name">name</a>, <a href="#register_jq_toolchains-version">version</a>, <a href="#register_jq_toolchains-register">register</a>) +</pre> + +Registers jq toolchain and repositories + +**PARAMETERS** + + +| Name | Description | Default Value | +| :------------- | :------------- | :------------- | +| <a id="register_jq_toolchains-name"></a>name | override the prefix for the generated toolchain repositories | <code>"jq"</code> | +| <a id="register_jq_toolchains-version"></a>version | the version of jq to execute (see https://github.com/stedolan/jq/releases) | <code>"1.6"</code> | +| <a id="register_jq_toolchains-register"></a>register | whether to call through to native.register_toolchains. Should be True for WORKSPACE users, but false when used under bzlmod extension | <code>True</code> | + + +<a id="register_yq_toolchains"></a> + +## register_yq_toolchains + +<pre> +register_yq_toolchains(<a href="#register_yq_toolchains-name">name</a>, <a href="#register_yq_toolchains-version">version</a>, <a href="#register_yq_toolchains-register">register</a>) +</pre> + +Registers yq toolchain and repositories + +**PARAMETERS** + + +| Name | Description | Default Value | +| :------------- | :------------- | :------------- | +| <a id="register_yq_toolchains-name"></a>name | override the prefix for the generated toolchain repositories | <code>"yq"</code> | +| <a id="register_yq_toolchains-version"></a>version | the version of yq to execute (see https://github.com/mikefarah/yq/releases) | <code>"4.25.2"</code> | +| <a id="register_yq_toolchains-register"></a>register | whether to call through to native.register_toolchains. Should be True for WORKSPACE users, but false when used under bzlmod extension | <code>True</code> | + +
diff --git a/internal_deps.bzl b/internal_deps.bzl index 54545b6..d94a9dc 100644 --- a/internal_deps.bzl +++ b/internal_deps.bzl
@@ -4,7 +4,7 @@ statement from these, that's a bug in our distribution. """ -load("//lib:repositories.bzl", "register_jq_toolchains", "register_yq_toolchains") +load("//lib:repositories.bzl", "register_jq_toolchains", "register_yq_toolchains", "register_coreutils_toolchains") load("//lib:utils.bzl", http_archive = "maybe_http_archive") # buildifier: disable=unnamed-macro @@ -50,3 +50,4 @@ # Register toolchains for tests register_jq_toolchains() register_yq_toolchains() + register_coreutils_toolchains()
diff --git a/lib/BUILD.bazel b/lib/BUILD.bazel index d574991..c178dbc 100644 --- a/lib/BUILD.bazel +++ b/lib/BUILD.bazel
@@ -27,6 +27,10 @@ name = "copy_to_directory_toolchain_type", ) +toolchain_type( + name = "coreutils_toolchain_type", +) + bzl_library( name = "docs", srcs = ["docs.bzl"], @@ -189,6 +193,7 @@ srcs = ["repositories.bzl"], deps = [ ":utils", + "//lib/private/docs:coreutils_toolchain", "//lib/private/docs:copy_to_directory_toolchain", "//lib/private/docs:jq_toolchain", "//lib/private/docs:local_config_platform",
diff --git a/lib/private/coreutils_toolchain.bzl b/lib/private/coreutils_toolchain.bzl new file mode 100644 index 0000000..f0771b5 --- /dev/null +++ b/lib/private/coreutils_toolchain.bzl
@@ -0,0 +1,202 @@ +"Setup coreutils toolchain repositories and rules" + +# Platform names follow the platform naming convention in @aspect_bazel_lib//:lib/private/repo_utils.bzl +COREUTILS_PLATFORMS = { + "darwin_amd64": struct( + compatible_with = [ + "@platforms//os:macos", + "@platforms//cpu:x86_64", + ], + ), + "darwin_arm64": struct( + compatible_with = [ + "@platforms//os:macos", + "@platforms//cpu:aarch64", + ], + ), + "linux_amd64": struct( + compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + ), + "linux_arm64": struct( + compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:aarch64", + ], + ), + "windows_amd64": struct( + compatible_with = [ + "@platforms//os:windows", + "@platforms//cpu:x86_64", + ], + ), +} + +DEFAULT_COREUTILS_VERSION = "0.0.16" + +# https://github.com/mikefarah/coreutils/releases +# +# The integrity hashes can be automatically fetched for the coreutils releases by running +# `tools/coreutils_mirror_release.sh`. +COREUTILS_VERSIONS = { + "0.0.16": { + "linux_arm64": { + "filename": "coreutils-0.0.16-aarch64-unknown-linux-gnu.tar.gz", + "sha256": "sha256-6tExkcMMHe7+59M9Mp9UKeq+g0y+juO7oakZfVOtOqw=" + }, + "darwin_amd64": { + "filename": "coreutils-0.0.16-x86_64-apple-darwin.tar.gz", + "sha256": "sha256-lQYWAuPOKS6XxwArZdiKIyczwppTVwjF4ml0iKBaB9s=" + }, + "windows_amd64": { + "filename": "coreutils-0.0.16-x86_64-pc-windows-msvc.zip", + "sha256": "sha256-z5E1onYAKZoaUt2U1cv1t8UHPsJinYktEd6SpE2u07o=" + }, + "linux_amd64": { + "filename": "coreutils-0.0.16-x86_64-unknown-linux-gnu.tar.gz", + "sha256": "sha256-Slf4qKf19sAWoK2pUVluAitmL3N2uz4eWpV4eibIEW0=" + } + } +} + +CoreUtilsInfo = provider( + doc = "Provide info for executing coreutils", + fields = { + "bin": "Executable coreutils binary", + }, +) + +def _coreutils_toolchain_impl(ctx): + binary = ctx.file.binary + + # Make the $(COREUTILS_BIN) variable available in places like genrules. + # See https://docs.bazel.build/versions/main/be/make-variables.html#custom_variables + template_variables = platform_common.TemplateVariableInfo({ + "COREUTILS_BIN": binary.path, + }) + default_info = DefaultInfo( + files = depset([binary]), + runfiles = ctx.runfiles(files = [binary]), + ) + coreutils_info = CoreUtilsInfo( + bin = binary, + ) + + # Export all the providers inside our ToolchainInfo + # so the resolved_toolchain rule can grab and re-export them. + toolchain_info = platform_common.ToolchainInfo( + coreutils_info = coreutils_info, + template_variables = template_variables, + default = default_info, + ) + + return [default_info, toolchain_info, template_variables] + +coreutils_toolchain = rule( + implementation = _coreutils_toolchain_impl, + attrs = { + "binary": attr.label( + mandatory = True, + allow_single_file = True + ), + }, +) + +def _coreutils_toolchains_repo_impl(rctx): + # Expose a concrete toolchain which is the result of Bazel resolving the toolchain + # for the execution or target platform. + # Workaround for https://github.com/bazelbuild/bazel/issues/14009 + starlark_content = """# @generated by @aspect_bazel_lib//lib/private:coreutils_toolchain.bzl + +# Forward all the providers +def _resolved_toolchain_impl(ctx): + toolchain_info = ctx.toolchains["@aspect_bazel_lib//lib:coreutils_toolchain_type"] + return [ + toolchain_info, + toolchain_info.default, + toolchain_info.coreutils_info, + toolchain_info.template_variables, + ] + +# Copied from java_toolchain_alias +# https://cs.opensource.google/bazel/bazel/+/master:tools/jdk/java_toolchain_alias.bzl +resolved_toolchain = rule( + implementation = _resolved_toolchain_impl, + toolchains = ["@aspect_bazel_lib//lib:coreutils_toolchain_type"], + incompatible_use_toolchain_transition = True, +) +""" + rctx.file("defs.bzl", starlark_content) + + build_content = """# @generated by @aspect_bazel_lib//lib/private:coreutils_toolchain.bzl +# +# These can be registered in the workspace file or passed to --extra_toolchains flag. +# By default all these toolchains are registered by the coreutils_register_toolchains macro +# so you don't normally need to interact with these targets. + +load(":defs.bzl", "resolved_toolchain") + +resolved_toolchain(name = "resolved_toolchain", visibility = ["//visibility:public"]) + +""" + + for [platform, meta] in COREUTILS_PLATFORMS.items(): + build_content += """ +toolchain( + name = "{platform}_toolchain", + exec_compatible_with = {compatible_with}, + toolchain = "@{user_repository_name}_{platform}//:coreutils_toolchain", + toolchain_type = "@aspect_bazel_lib//lib:coreutils_toolchain_type", +) +""".format( + platform = platform, + user_repository_name = rctx.attr.user_repository_name, + compatible_with = meta.compatible_with, + ) + + # Base BUILD file for this repository + rctx.file("BUILD.bazel", build_content) + +coreutils_toolchains_repo = repository_rule( + _coreutils_toolchains_repo_impl, + doc = """Creates a repository with toolchain definitions for all known platforms + which can be registered or selected.""", + attrs = { + "user_repository_name": attr.string(doc = "Base name for toolchains repository"), + }, +) + +def _coreutils_platform_repo_impl(rctx): + is_windows = rctx.attr.platform.startswith("windows_") + platform = rctx.attr.platform + if platform == "darwin_arm64": + platform = "darwin_amd64" + filename = COREUTILS_VERSIONS[rctx.attr.version][platform]["filename"] + url = "https://github.com/uutils/coreutils/releases/download/{}/{}".format( + rctx.attr.version, + filename + ) + rctx.download_and_extract( + url = url, + stripPrefix = filename.replace(".zip", "").replace(".tar.gz", ""), + integrity = COREUTILS_VERSIONS[rctx.attr.version][platform]["sha256"], + ) + build_content = """# @generated by @aspect_bazel_lib//lib/private:coreutils_toolchain.bzl +load("@aspect_bazel_lib//lib/private:coreutils_toolchain.bzl", "coreutils_toolchain") +exports_files(["{0}"]) +coreutils_toolchain(name = "coreutils_toolchain", binary = "{0}", visibility = ["//visibility:public"]) +""".format("coreutils.exe" if is_windows else "coreutils") + + # Base BUILD file for this repository + rctx.file("BUILD.bazel", build_content) + +coreutils_platform_repo = repository_rule( + implementation = _coreutils_platform_repo_impl, + doc = "Fetch external tools needed for coreutils toolchain", + attrs = { + "version": attr.string(mandatory = True, values = COREUTILS_VERSIONS.keys()), + "platform": attr.string(mandatory = True, values = COREUTILS_PLATFORMS.keys()), + }, +)
diff --git a/lib/private/docs/BUILD.bazel b/lib/private/docs/BUILD.bazel index a91d858..0c65fc3 100644 --- a/lib/private/docs/BUILD.bazel +++ b/lib/private/docs/BUILD.bazel
@@ -232,3 +232,8 @@ name = "copy_to_directory_toolchain", srcs = ["//lib/private:copy_to_directory_toolchain.bzl"], ) + +bzl_library( + name = "coreutils_toolchain", + srcs = ["//lib/private:coreutils_toolchain.bzl"], +)
diff --git a/lib/repositories.bzl b/lib/repositories.bzl index ed7ff59..9bfe6ab 100644 --- a/lib/repositories.bzl +++ b/lib/repositories.bzl
@@ -3,6 +3,7 @@ load("//lib/private:jq_toolchain.bzl", "JQ_PLATFORMS", "jq_host_alias_repo", "jq_platform_repo", "jq_toolchains_repo", _DEFAULT_JQ_VERSION = "DEFAULT_JQ_VERSION") load("//lib/private:yq_toolchain.bzl", "YQ_PLATFORMS", "yq_host_alias_repo", "yq_platform_repo", "yq_toolchains_repo", _DEFAULT_YQ_VERSION = "DEFAULT_YQ_VERSION") load("//lib/private:copy_to_directory_toolchain.bzl", "COPY_TO_DIRECTORY_PLATFORMS", "copy_to_directory_platform_repo", "copy_to_directory_toolchains_repo") +load("//lib/private:coreutils_toolchain.bzl", "COREUTILS_PLATFORMS", "coreutils_platform_repo", "coreutils_toolchains_repo", _DEFAULT_COREUTILS_VERSION = "DEFAULT_COREUTILS_VERSION") load("//lib/private:local_config_platform.bzl", "local_config_platform") load("//lib:utils.bzl", "is_bazel_6_or_greater", http_archive = "maybe_http_archive") @@ -39,6 +40,7 @@ # Re-export the default versions DEFAULT_JQ_VERSION = _DEFAULT_JQ_VERSION DEFAULT_YQ_VERSION = _DEFAULT_YQ_VERSION +DEFAULT_COREUTILS_VERSION = _DEFAULT_COREUTILS_VERSION def register_jq_toolchains(name = "jq", version = DEFAULT_JQ_VERSION, register = True): """Registers jq toolchain and repositories @@ -90,6 +92,29 @@ user_repository_name = name, ) +def register_coreutils_toolchains(name = "coreutils", version = DEFAULT_COREUTILS_VERSION, register = True): + """Registers coreutils toolchain and repositories + + Args: + name: override the prefix for the generated toolchain repositories + version: the version of coreutils to execute (see https://github.com/uutils/coreutils/releases) + register: whether to call through to native.register_toolchains. + Should be True for WORKSPACE users, but false when used under bzlmod extension + """ + for [platform, meta] in COREUTILS_PLATFORMS.items(): + coreutils_platform_repo( + name = "%s_%s" % (name, platform), + platform = platform, + version = version + ) + if register: + native.register_toolchains("@%s_toolchains//:%s_toolchain" % (name, platform)) + + coreutils_toolchains_repo( + name = "%s_toolchains" % name, + user_repository_name = name, + ) + def register_copy_to_directory_toolchains(name = "copy_to_directory", register = True): """Registers copy_to_directory toolchain and repositories
diff --git a/lib/tests/coreutils/BUILD.bazel b/lib/tests/coreutils/BUILD.bazel new file mode 100644 index 0000000..1fa2437 --- /dev/null +++ b/lib/tests/coreutils/BUILD.bazel
@@ -0,0 +1,59 @@ +load("//lib:testing.bzl", "assert_contains") +load("//lib/tests/jq:diff_test.bzl", "diff_test") + +genrule( + name = "ls", + outs = ["ls.txt"], + cmd = "$(COREUTILS_BIN) ls > $@", + toolchains = ["@coreutils_toolchains//:resolved_toolchain"], +) + +diff_test( + name = "ls_test", + file1 = "ls.txt.expected", + file2 = ":ls", +) + + +genrule( + name = "sha256sum", + outs = ["sha256sum.txt"], + srcs = ["test.bin"], + cmd = "$(COREUTILS_BIN) sha256sum $(location :test.bin) > $@", + toolchains = ["@coreutils_toolchains//:resolved_toolchain"], +) + +assert_contains( + name = "test_sha256sum", + actual = "sha256sum.txt", + expected = """9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08 lib/tests/coreutils/test.bin""", +) + + +genrule( + name = "sha512sum", + outs = ["sha512sum.txt"], + srcs = ["test.bin"], + cmd = "$(COREUTILS_BIN) sha512sum $(location :test.bin) > $@", + toolchains = ["@coreutils_toolchains//:resolved_toolchain"], +) + +assert_contains( + name = "test_sha512sum", + actual = "sha512sum.txt", + expected = """ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a8ff lib/tests/coreutils/test.bin""", +) + +genrule( + name = "wc", + outs = ["wc.txt"], + srcs = ["test.bin"], + cmd = "$(COREUTILS_BIN) wc $(location :test.bin) > $@", + toolchains = ["@coreutils_toolchains//:resolved_toolchain"], +) + +assert_contains( + name = "test_wc", + actual = "wc.txt", + expected = """0 1 4 lib/tests/coreutils/test.bin""", +) \ No newline at end of file
diff --git a/lib/tests/coreutils/ls.txt.expected b/lib/tests/coreutils/ls.txt.expected new file mode 100644 index 0000000..8c28eab --- /dev/null +++ b/lib/tests/coreutils/ls.txt.expected
@@ -0,0 +1,2 @@ +bazel-out +external
diff --git a/lib/tests/coreutils/test.bin b/lib/tests/coreutils/test.bin new file mode 100644 index 0000000..30d74d2 --- /dev/null +++ b/lib/tests/coreutils/test.bin
@@ -0,0 +1 @@ +test \ No newline at end of file
diff --git a/tools/coreutils_mirror_release.sh b/tools/coreutils_mirror_release.sh new file mode 100755 index 0000000..135a68c --- /dev/null +++ b/tools/coreutils_mirror_release.sh
@@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +set -o errexit -o nounset -o pipefail + +JQ_FILTER=\ +'map({ + "key": .tag_name, + "value": .assets + | map(select( + (.name | startswith("coreutils-")) and + ((.name | endswith(".tar.gz")) or (.name | endswith(".zip"))) and + (.name | contains("i686") | not) and + ( + ( (.name | contains("windows")) and (.name | contains("gnu") | not) ) or + ( (.name | contains("windows") | not) and (.name | contains("gnu") ) and (.name | contains("gnueabihf") | not) ) or + ( .name | contains("darwin") ) + ) + )) + | map({ + key: .name | + ltrimstr("coreutils-") | + rtrimstr(".tar.gz") | + rtrimstr(".zip") | + sub("-pc"; "") | + sub("-apple"; "") | + sub("-unknown"; "") | + sub("x86_64"; "amd64") | + sub("aarch64"; "arm64") | + gsub("\\d+.\\d+.\\d+-"; "") | + rtrimstr("-msvc") | + rtrimstr("-gnu") | + split("-") | + reverse | + join("_"), + value: { + filename: .name, + sha256: "sha256-", + } + }) + | from_entries +}) | from_entries +' + + +INFO="$(curl --silent -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/uutils/coreutils/releases?per_page=1 | jq "$JQ_FILTER")" + + +for VERSION in $(jq -r 'keys | join("\n")' <<< $INFO); do + for PLATFORM in $(jq -r ".[\"$VERSION\"] | keys | join(\"\n\")" <<< $INFO); do + FILENAME=$(jq -r ".[\"$VERSION\"][\"$PLATFORM\"].filename" <<< $INFO) + SHA256=$(curl -fLs "https://github.com/uutils/coreutils/releases/download/$VERSION/$FILENAME" | sha256sum | xxd -r -p | base64) + INFO=$(jq ".[\"$VERSION\"][\"$PLATFORM\"].sha256 = \"sha256-$SHA256\"" <<< $INFO) + done +done + +echo -n "COREUTILS_VERSIONS = " +echo $INFO | jq -M + +echo "" +echo "Copy the version info into lib/private/coreutils_toolchain.bzl" \ No newline at end of file