feat: nixos toolchains for bsdtar
diff --git a/lib/private/tar_toolchain.bzl b/lib/private/tar_toolchain.bzl index 61a1cce..d2eeac4 100644 --- a/lib/private/tar_toolchain.bzl +++ b/lib/private/tar_toolchain.bzl
@@ -15,6 +15,12 @@ "@platforms//cpu:aarch64", ], ), + "darwin_arm64": struct( + compatible_with = [ + "@platforms//os:macos", + "@platforms//cpu:aarch64", + ], + ), "windows_amd64": struct( release_platform = "win64", compatible_with = [ @@ -29,71 +35,6 @@ ), } -WINDOWS_DEPS = ( - "e06f10043b1b148eb38ad06cff678af05beade0bdd2edd8735a198c521fa3993", - "https://github.com/libarchive/libarchive/releases/download/v3.7.2/libarchive-v3.7.2-amd64.zip", -) - -# note, using Ubuntu Focal packages as they link with older glibc versions. -# Ubuntu Jammy packages will fail on ubuntu 20.02 with -# bsdtar: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.33' not found -# bsdtar: /lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.34' not found -# -# TODO: this is only a partial listing of the transitive deps of libarchive-tools -# so we expect a bunch of compress modes are broken, for example. - -LINUX_LIB_DEPS = { - "linux_arm64": [ - ( - "6d18525e248e84b8a4ee39a226fd1195ca9b9d0d5a1c7909ae4f997d46378848", - "http://ports.ubuntu.com/pool/main/n/nettle/libnettle7_3.5.1+really3.5.1-2ubuntu0.2_arm64.deb", - ), - ( - "aa5e31d05a9d6bde8093137bd1c82b5a20a5f470bd5109642014f895c20f323a", - "http://ports.ubuntu.com/pool/main/liba/libarchive/libarchive13_3.4.0-2ubuntu1_arm64.deb", - ), - ( - "6d089f878507b536d8ca51b1ad80a80706a1dd7dbbcce7600800d3f9f98be2ab", - "http://ports.ubuntu.com/pool/main/liba/libarchive/libarchive-tools_3.2.1-2~ubuntu16.04.1_arm64.deb", - ), - ( - "6242892cb032859044ddfcfbe61bac5678a95c585d8fff4525acaf45512e3d39", - "http://ports.ubuntu.com/pool/main/libx/libxml2/libxml2_2.9.10+dfsg-5_arm64.deb", - ), - ( - "6302e309ab002af30ddfa0d68de26c68f7c034ed2f45b1d97a712bff1a03999a", - "http://ports.ubuntu.com/pool/main/i/icu/libicu66_66.1-2ubuntu2_arm64.deb", - ), - ], - "linux_amd64": [ - # https://packages.ubuntu.com/focal/amd64/libarchive-tools/download - ( - "12a19878d34b407e6f4893d3b26b7758a26c5534a066d76184c8b764b2df1652", - "http://security.ubuntu.com/ubuntu/pool/universe/liba/libarchive/libarchive-tools_3.4.0-2ubuntu1.2_amd64.deb", - ), - # https://packages.ubuntu.com/focal/amd64/libarchive13/download - ( - "8ba7507f61bb3ea8da488702ec0badcbfb726d36ea6886e3421ac59082aaf2d1", - "http://security.ubuntu.com/ubuntu/pool/main/liba/libarchive/libarchive13_3.4.0-2ubuntu1.2_amd64.deb", - ), - # https://packages.ubuntu.com/focal/amd64/libnettle7/download - ( - "3496aed83407fde71e0dc5988b28e8fd7f07a2f27fcf3e0f214c7cd86667eecd", - "http://security.ubuntu.com/ubuntu/pool/main/n/nettle/libnettle7_3.5.1+really3.5.1-2ubuntu0.2_amd64.deb", - ), - # https://packages.ubuntu.com/focal/amd64/libxml2/download - ( - "a8cbd10a0d74ff8ec43a7e6c09ad07629f20efea9972799d9ff7f63c4e82bfcf", - "http://security.ubuntu.com/ubuntu/pool/main/libx/libxml2/libxml2_2.9.10+dfsg-5ubuntu0.20.04.6_amd64.deb", - ), - # https://packages.ubuntu.com/focal/amd64/libicu66/download - ( - "00d0de456134668f41bd9ea308a076bc0a6a805180445af8a37209d433f41efe", - "http://security.ubuntu.com/ubuntu/pool/main/i/icu/libicu66_66.1-2ubuntu2.1_amd64.deb", - ), - ], -} - def _find_usable_system_tar(rctx, tar_name): tar = rctx.which(tar_name) if not tar: @@ -109,6 +50,25 @@ fail("tar isn't a BSD tar") +EXPERIMENTAL_NIXOS_BSDTAR_TOOLCHAINS = { + "linux_arm64": ("https://cache.nixos.org/nar/06sac7pp4zx71jxvrngxqgqgjjbzppwvwkiyrqyhzhl3clmkz1sq.nar.xz", "58873f2b6583c20f3dce3e4ebef9bd7f49f9f0c3fdd9bcbb0ca77f72ef614a1b", (7301328, 6923120)), + "linux_amd64": ("https://cache.nixos.org/nar/1zi03mz2l6aidzy8kzc9iri2hvm74dn0mdhv0sk42zpxdl91y2v0.nar.xz", "600b1f126dfd7e41a6061bb60a6c23a76e28628e89fd89fc6f51192a7e1d20fe", (7903336, 7533384)), + "darwin_arm64": ("https://cache.nixos.org/nar/0q8ysphisaikwv2fx0pj2fs76rx3xj12j8p4xxggdyyj81p37xxy.nar.xz", "bef7336e40d2fbf65eefe4222982eca36773b413f282eec4e6332a1de1d51e61", (5217904, 3866160)), +} + +WINDOWS_DEPS = ( + "e06f10043b1b148eb38ad06cff678af05beade0bdd2edd8735a198c521fa3993", + "https://github.com/libarchive/libarchive/releases/download/v3.7.2/libarchive-v3.7.2-amd64.zip", +) + +NAR_EXTRACT = """\ +#!/bin/bash +gzcat $1 | tail -c $2 > ./tmp && +head -c $3 ./tmp > $4 && +chmod +x $4 && +rm tmp extract.sh +""" + def _bsdtar_binary_repo(rctx): tar_name = "tar.exe" if repo_utils.is_windows(rctx) else "tar" build_header = """\ @@ -125,42 +85,26 @@ tar = _find_usable_system_tar(rctx, tar_name) output = rctx.path(tar_name) rctx.symlink(tar, output) - rctx.file("BUILD.bazel", build_header + """tar_toolchain(name = "bsdtar_toolchain", binary = "tar")""") - return - if repo_utils.is_windows(rctx): + elif repo_utils.is_windows(rctx): rctx.download_and_extract( url = WINDOWS_DEPS[1], type = "zip", sha256 = WINDOWS_DEPS[0], ) - rctx.file("BUILD.bazel", build_header + """tar_toolchain(name = "bsdtar_toolchain", binary = "libarchive/bin/bsdtar.exe")""") - return + tar_name = "libarchive/bin/bsdtar.exe" - # Other platforms, we have more work to do. - libs_dir = "usr/lib/x86_64-linux-gnu" if rctx.attr.platform.endswith("amd64") else "usr/lib/aarch64-linux-gnu" - - for lib in LINUX_LIB_DEPS[rctx.attr.platform]: - rctx.download_and_extract( - url = lib[1], - type = "deb", - sha256 = lib[0], + else: + remote = EXPERIMENTAL_NIXOS_BSDTAR_TOOLCHAINS[rctx.attr.platform] + rctx.download( + url = remote[0], + output = "bsdtar.nar.xz", + sha256 = remote[1], ) - rctx.extract("data.tar.xz") + rctx.file("extract.sh", executable = True, content = NAR_EXTRACT) + rctx.execute([rctx.path("extract.sh"), "bsdtar.nar.xz", "+%s" % (remote[2][0] + 1), "%s" % remote[2][1], tar_name]) - rctx.file("bsdtar.sh", """#!/usr/bin/env bash -readonly wksp="$(dirname "${{BASH_SOURCE[0]}}")" -LD_LIBRARY_PATH=$wksp/{libs_dir} exec $wksp/usr/bin/bsdtar $@ -""".format(name = rctx.name, libs_dir = libs_dir)) - - rctx.file("BUILD.bazel", build_header + """\ -tar_toolchain( - name = "bsdtar_toolchain", - files = glob(["{libs}/*.so.*"]) + ["usr/bin/bsdtar"], - binary = "bsdtar.sh", - visibility = ["//visibility:public"], -) -""".format(libs = libs_dir, name = rctx.name)) + rctx.file("BUILD.bazel", build_header + """tar_toolchain(name = "bsdtar_toolchain", binary = "{}")""".format(tar_name)) bsdtar_binary_repo = repository_rule( implementation = _bsdtar_binary_repo,
diff --git a/scripts/bsdtar_mirror_release.sh b/scripts/bsdtar_mirror_release.sh new file mode 100755 index 0000000..d4417bd --- /dev/null +++ b/scripts/bsdtar_mirror_release.sh
@@ -0,0 +1,31 @@ +#!/bin/bash + +set -o errexit -o nounset -o pipefail + +function get_nar () { + local platform="$1" + nix-env --uninstall libarchive.\* 2>&2 + nix-env -iA --prebuilt-only --attr nixpkgs.pkgsStatic.libarchive --argstr system "$platform" 2>&2 + hash=$(nix-env -q --out-path libarchive.\* --no-name | cut -c12-43) + + nar=$(curl -fsSL "https://cache.nixos.org/$hash.narinfo" | grep 'URL: ' | tr -d "URL: ") + nar_url="https://cache.nixos.org/$nar" + + archive="$(mktemp)" + curl -fsSL $nar_url | gzcat > $archive + sha256=$(curl -fsSL $nar_url | shasum -a 256 | awk '{print $1}') + range=$(./scripts/read_nar.sh "$archive") + echo " \"$2\": (\"$nar_url\", \"$sha256\", $range)" +} + + +nl=$'\n' +output="EXPERIMENTAL_NIXOS_BSDTAR_TOOLCHAINS = {$nl" +output+="$(get_nar aarch64-linux linux_arm64),$nl" +output+="$(get_nar x86_64-linux linux_amd64),$nl" +output+="$(get_nar aarch64-darwin darwin_arm64)$nl" +output+="}" + +clear + +echo "$output"
diff --git a/scripts/read_nar.sh b/scripts/read_nar.sh new file mode 100755 index 0000000..4141c13 --- /dev/null +++ b/scripts/read_nar.sh
@@ -0,0 +1,114 @@ +#!/bin/bash + +set -o errexit -o nounset -o pipefail + +archive="$1" + +pad_len=8 +pos_file=$(mktemp) +echo 0 > $pos_file + +function get_pos() { cat $pos_file; } +function move_pos () { local pos=$(get_pos); echo $((pos + $1)) > $pos_file; } +function u64 () { od -t u8 -An -v | awk '{print $1}'; } +function assert_eq() { + if [[ "$1" != "$2" ]]; then + echo "assertion failed($BASH_LINENO): '$1' != '$2'"; + exit 1 + fi +} + +function read_bytes () { + local len=$1 + if [[ $len -eq 0 ]]; then + return 0 + fi + local pos=$(get_pos) + move_pos $len + # echo "reading from $pos, $len bytes" >&2 + tail -c +"$((pos+1))" "$archive" | head -c "$len" || true +} + + +function read_bytes_padded() { + local content_len=$(read_bytes $pad_len | u64) + # echo "content_len $content_len" >&2 + read_bytes $content_len + local remainder=$((content_len % pad_len)) + if [[ $remainder -gt 0 ]]; then + # echo "remainder $header_len $remainder" >&2 + read_bytes $((pad_len - remainder)) > /dev/null + fi +} + +trap 'echo $LINENO' ERR + +magic="$(read_bytes_padded)" + +if [[ $magic != "nix-archive-1" ]]; then + echo "wrong nar $magic"; + exit 1 +fi + +function read_entry () { + assert_eq "$(read_bytes_padded)" "(" + assert_eq $(read_bytes_padded) "type" + local type=$(read_bytes_padded) + case "$type" in + "regular") + local executable=0 + local tag=$(read_bytes_padded) + if [[ "$tag" == "executable" ]]; then + executable=1; + read_bytes_padded + tag=$(read_bytes_padded) + fi + assert_eq "$tag" "contents" + local startpos=$(get_pos) + startpos=$((startpos+8)) + + local data=$(mktemp) + read_bytes_padded > $data + + local len=$(cat $data | wc -c | tr -d "") + if [[ "$name" == "bsdtar" ]]; then + echo "($startpos,$len)" + exit 0 + fi + + assert_eq "$(read_bytes_padded)" ")" + ;; + "symlink") + echo "symlink" + exit + ;; + "directory") + while [ true ]; do + case "$(read_bytes_padded)" in + "entry") + assert_eq "$(read_bytes_padded)" "(" + assert_eq "$(read_bytes_padded)" "name" + local name=$(read_bytes_padded) + assert_eq "$(read_bytes_padded)" "node" + read_entry + local endpos=$(get_pos) + assert_eq "$(read_bytes_padded)" ")" + ;; + ")") + break + ;; + *) + ;; + esac + done + ;; + *) + echo "illegal value" + exit 1 + ;; + esac +} + +read_entry + +exit 1