util/fetch_shas/fetch_shas.py

#!/usr/bin/env python3.11
"""A script for generating the `//rust:known_shas.bzl` file."""

import json
import logging
import os
import shutil
import subprocess
import sys
import tempfile
import tomllib
from pathlib import Path
from typing import Any, Dict, Sequence

KNOWN_SHAS_TEMPLATE = """\
\"\"\"A module containing a mapping of Rust tools to checksums

This is a generated file -- see //util/fetch_shas
\"\"\"

FILE_KEY_TO_SHA = {}
"""


def download_manifest_data(
    stable_versions: Sequence[str], nightly_versions: Sequence[str], output_dir: Path
) -> Dict[str, Sequence[Dict[str, Any]]]:
    """Download and deserialize `channel-rust-*.toml` files for the requested versions

    Args:
        stable_versions: A list of Rust versions. E.g. `1.81.0`
        nightly_versions: A list of nightly iso dates. E.g. `2024-09-05`
        output_dir: The location where the intermediate files should be written.

    Returns:
        A mapping of channel name (`stable`, `nightly`) to deserialized data.
    """

    output_dir.mkdir(exist_ok=True, parents=True)
    curl_config = output_dir / "curl_config.txt"
    curl_config_lines = [
        "--fail",
        "--parallel",
        "--silent",
        "--create-dirs",
    ]

    stable_files = {}
    for version in stable_versions:
        output = output_dir / f"channel-rust-{version}.toml"
        curl_config_lines.extend(
            [
                f"--output {output}",
                f"--url https://static.rust-lang.org/dist/channel-rust-{version}.toml",
            ]
        )
        stable_files[version] = output
    nightly_files = {}
    for version in nightly_versions:
        output = output_dir / version / "channel-rust-nightly.toml"
        curl_config_lines.extend(
            [
                f"--output {output}",
                f"--url https://static.rust-lang.org/dist/{version}/channel-rust-nightly.toml",
            ]
        )
        nightly_files[version] = output

    curl_config.write_text("\n".join(curl_config_lines), encoding="utf-8")

    logging.info("Downloading data...")
    subprocess.run(
        [
            "curl",
            "--config",
            curl_config,
        ],
        check=True,
    )
    logging.info("Done.")

    logging.info(
        "Deserializing %s tomls...",
        len(list(stable_files.keys()) + list(nightly_files.keys())),
    )
    for collection in (stable_files, nightly_files):
        for version, file in collection.items():
            data = file.read_text(encoding="utf-8")
            assert data
            try:
                collection[version] = tomllib.loads(data)
            except:
                logging.error("Failed to parse toml: %s\n%s", file, data)
                raise

    logging.info("Done.")
    return {
        "stable": stable_files,
        "nightly": nightly_files,
    }


def download_direct_sha256s(
    artifacts: Sequence[str], output_dir: Path
) -> Dict[str, str]:
    """_summary_

    This function is mostly here for backward compatibility. There are artifacts
    referenced by the `channel-rust-*.toml` files which are marked as `available: false`
    and probably intented to not be downloaded. But for now this is ignored and instead
    a collection of artifacts whose hash data could not be found is explicitly checked
    by trying to download the `.sha256` files directly. A 404 indicates the artifact
    genuinely does not exist and anything else we find is extra data we can retain
    in `known_shas.bzl`.

    Args:
        artifacts: A list of paths within `https://static.rust-lang.org/dist` to download.
        output_dir: The location where the intermediate files should be written.

    Returns:
        A mapping of `artifacts` entries to their sha256 value.
    """
    output_dir.mkdir(exist_ok=True, parents=True)
    status_config = output_dir / "status" / "curl_config.txt"
    sha256_config = output_dir / "sha256" / "curl_config.txt"
    common_config_lines = [
        "--parallel",
        "--create-dirs",
    ]
    status_config_lines = common_config_lines + [
        "--silent",
        "--head",
    ]

    statuses = {}
    for url_path in artifacts:
        output = status_config.parent / f"{url_path}.sha256.status"
        status_config_lines.extend(
            [
                f"--output {output}",
                f"--url https://static.rust-lang.org/dist/{url_path}.sha256",
            ]
        )
        statuses[url_path] = output

    status_config.parent.mkdir(exist_ok=True, parents=True)
    status_config.write_text("\n".join(status_config_lines), encoding="utf-8")

    logging.info("Checking for %s missing artifacts...", len(statuses))
    result = subprocess.run(
        [
            "curl",
            "--config",
            status_config,
        ],
        check=True,
    )
    logging.info("Done.")

    checksums = {}
    missing = []
    for url_path, status_file in statuses.items():
        if not status_file.exists():
            missing.append(f"https://static.rust-lang.org/dist/{url_path}.sha256")
            continue

        if status_file.read_text(encoding="utf-8").startswith("HTTP/2 404"):
            continue

        checksums[url_path] = sha256_config.parent / f"{url_path}.sha256"

    if missing:
        logging.warning(
            "Status not found for %s artifacts:\n%s",
            len(missing),
            json.dumps(sorted(missing), indent=2),
        )

    logging.info("Downloading %s missing artifacts...", len(checksums))
    sha256_config_lines = common_config_lines + [
        "--verbose",
    ]
    for url_path, output in checksums.items():
        sha256_config_lines.extend(
            [
                f"--output {output}",
                f"--url https://static.rust-lang.org/dist/{url_path}.sha256",
            ]
        )

    sha256_config.parent.mkdir(exist_ok=True, parents=True)
    sha256_config.write_text("\n".join(sha256_config_lines), encoding="utf-8")
    result = subprocess.run(
        [
            "curl",
            "--config",
            sha256_config,
        ],
        encoding="utf-8",
        check=False,
        stderr=subprocess.STDOUT,
        stdout=subprocess.PIPE,
    )

    if result.returncode:
        print(result.stdout, file=sys.stderr)
        sys.exit(result.returncode)

    logging.info("Done.")

    return {
        name: file.read_text(encoding="utf-8").split(" ")[0].strip()
        for name, file in checksums.items()
    }


def load_data(file: Path) -> Sequence[str]:
    """Load a `fetch_shas_*.txt` file

    Args:
        file: The file to load

    Returns:
        A list of the file's contents.
    """
    data = []
    for line in file.read_text(encoding="utf-8").splitlines():
        text = line.strip()
        if not text:
            continue
        data.append(text)

    return sorted(set(data))


def main() -> None:
    """The  main entrypoint."""
    if "BUILD_WORKSPACE_DIRECTORY" in os.environ:
        workspace_dir = Path(os.environ["BUILD_WORKSPACE_DIRECTORY"])
    else:
        workspace_dir = Path(__file__).parent.parent.parent

    logging.basicConfig(
        level=(
            logging.INFO
            if "RULES_RUST_FETCH_SHAS_DEBUG" not in os.environ
            else logging.DEBUG
        ),
        format="%(asctime)s - %(levelname)s - %(message)s",
        datefmt="%Y-%m-%d %H:%M:%S",
    )
    logging.info("Fetching known sha256 data...")

    tools = load_data(workspace_dir / "util/fetch_shas/fetch_shas_TOOLS.txt")
    host_tools = load_data(workspace_dir / "util/fetch_shas/fetch_shas_HOST_TOOLS.txt")
    targets = load_data(workspace_dir / "util/fetch_shas/fetch_shas_TARGETS.txt")
    stable_versions = load_data(
        workspace_dir / "util/fetch_shas/fetch_shas_VERSIONS.txt"
    )
    nightly_iso_dates = load_data(
        workspace_dir / "util/fetch_shas/fetch_shas_NIGHTLY_ISO_DATES.txt"
    )

    # Allow this directory to be optionally cleaned up.
    tmp_dir = Path(tempfile.mkdtemp(prefix="rules-rust-fetch-shas-"))
    logging.debug("Temp dir: %s", tmp_dir)
    try:
        manifest_data = download_manifest_data(
            stable_versions=stable_versions,
            nightly_versions=nightly_iso_dates,
            output_dir=tmp_dir,
        )

        file_key_to_sha = {}

        artifacts = []

        logging.info("Parsing artifacts...")
        for channel, versioned_info in manifest_data.items():
            for version, info in versioned_info.items():
                if channel == "stable":
                    tool_tpl = f"{{pkg}}-{version}-{{target}}.{{ext}}"
                    host_tool_tpl = f"{{pkg}}-{version}.{{ext}}"
                else:
                    tool_tpl = f"{version}/{{pkg}}-{channel}-{{target}}.{{ext}}"
                    host_tool_tpl = f"{version}/{{pkg}}-{channel}.{{ext}}"

                # Artifacts are commonly referred to with an internal (to the Rust org) name.
                # In order to correctly evaluate whether or not a particular `channel-rust-*.toml`
                # has an artifact, a mapping is made from what the internal Rust names are to
                # what they're advertised as externally (e.g. what users would use when trying
                # to download files directly).
                renames = {
                    "clippy-preview": "clippy",
                    "llvm-tools-preview": "llvm-tools",
                    "rustfmt-preview": "rustfmt",
                }
                if "renames" in info:
                    for pkg, rename in info["renames"].items():
                        if "to" in rename:
                            renames[rename["to"]] = pkg

                logging.debug("Renames (%s %s): %s", channel, version, renames)
                for pkg, pkg_data in info["pkg"].items():
                    pkg_name = renames.get(pkg, pkg)
                    if pkg_name in tools:
                        tool_template = tool_tpl
                    elif pkg_name in host_tools:
                        tool_template = host_tool_tpl
                    else:
                        logging.debug("Skipping %s %s %s", channel, version, pkg_name)
                        continue

                    drain_targets = list(targets)
                    for target, target_data in pkg_data["target"].items():
                        # One variant of the template has an extra format string field.
                        # Replace is used to account for that.
                        template = tool_template.replace("{target}", target)

                        if target not in drain_targets:
                            continue

                        drain_targets.remove(target)

                        if "hash" in target_data:
                            file_key_to_sha[
                                template.format(pkg=pkg_name, ext="tar.gz")
                            ] = target_data["hash"]
                        if "xz_hash" in target_data:
                            file_key_to_sha[
                                template.format(pkg=pkg_name, ext="tar.xz")
                            ] = target_data["xz_hash"]

                    # If an artifact is not advertised to be available for a particular
                    # target then we track this and see if the sha256 data can be
                    # downloaded directly.
                    for target in drain_targets:
                        # One variant of the template has an extra format string field.
                        # Replace is used to account for that.
                        template = tool_template.replace("{target}", target)

                        # See if we can download the file directly.
                        artifacts.extend(
                            [
                                template.format(pkg=pkg_name, ext="tar.gz"),
                                template.format(pkg=pkg_name, ext="tar.xz"),
                            ]
                        )

        logging.info("Done. Identified %s artifacts.", len(file_key_to_sha))

        # Do a brute force check to find additional sha256 values.
        file_key_to_sha.update(
            download_direct_sha256s(
                artifacts=sorted(set(artifacts)),
                output_dir=tmp_dir / "retries",
            )
        )

    finally:
        if not "RULES_RUST_FETCH_SHAS_DEBUG" in os.environ:
            shutil.rmtree(tmp_dir)

    known_shas_file = workspace_dir / "rust/known_shas.bzl"
    known_shas_file.write_text(
        KNOWN_SHAS_TEMPLATE.format(
            json.dumps(file_key_to_sha, sort_keys=True, indent=4).replace(
                '"\n}', '",\n}'
            )
        )
    )
    logging.info("Done. Wrote %s", known_shas_file.relative_to(workspace_dir))


if __name__ == "__main__":
    main()