fix: don't materialize rules_python when reading the artifacts manifest (#4370)

The `config` extension reads `artifacts_manifest.json` with
`mctx.read(Label(...))`, so with
`--experimental_remote_repo_contents_cache` every consumer still
materializes the whole `@rules_python` repo. This is the remaining gap
after #4294.

Following #4294, the manifest becomes
`python/private/artifacts_manifest.bzl`, whose `ARTIFACTS_MANIFEST_JSON`
string the extension loads. It stays empty in git, so source checkouts
build the tools from source. The release generator now writes this
`.bzl` file, and the archive script bundles it in place of the empty
one. `configure_tool_toolchains` no longer needs `module_ctx`, and tests
pass their manifest JSON directly.

---------

Co-authored-by: Spencer Putt <sputt@sputt-macbook-pro.local>
diff --git a/.github/workflows/create_archive_and_notes.sh b/.github/workflows/create_archive_and_notes.sh
index 2dc9b22..df9c04e 100755
--- a/.github/workflows/create_archive_and_notes.sh
+++ b/.github/workflows/create_archive_and_notes.sh
@@ -44,12 +44,13 @@
   "--prefix=${PREFIX}/" \
   "--add-file=MODULE.bazel" \
   "--prefix=${PREFIX}/python/private/" \
-  "--add-file=artifacts_manifest.json" \
+  "--add-file=artifacts_manifest.bzl" \
   "--prefix=${PREFIX}/" \
   "$TAG" \
   . \
   ":!MODULE.bazel" \
-  ":!python/private/artifacts_manifest.json" | gzip > "$ARCHIVE"
+  ":!python/private/artifacts_manifest.bzl" |
+  gzip > "$ARCHIVE"
 SHA=$(shasum -a 256 "$ARCHIVE" | awk '{print $1}')
 
 cat > release_notes.txt << EOF
diff --git a/.github/workflows/release_build_artifacts.yaml b/.github/workflows/release_build_artifacts.yaml
index bbab6c6..aa73b07 100644
--- a/.github/workflows/release_build_artifacts.yaml
+++ b/.github/workflows/release_build_artifacts.yaml
@@ -4,7 +4,7 @@
 #   see dev/release_artifacts/build.sh. Linux x86_64 runners cross-compile the
 #   files for other Linux CPUs. macOS and Windows files need those OSes' SDKs,
 #   so they're built on those OSes.
-# - `SHA256SUMS` and `artifacts_manifest.json` for the built platform binaries.
+# - `SHA256SUMS` and `artifacts_manifest.bzl` for the built platform binaries.
 # - The release source archive (`rules_python-<ref>.tar.gz`) and
 #   `release_notes.txt`.
 # - The `bazel_runfiles` wheel (`//python/runfiles:wheel.dist`).
@@ -187,14 +187,14 @@
           hashFiles('release-artifacts/*') != ''
         run: >-
           dev/release_artifacts/generate_artifacts_manifest.py
-          "$REF" release-artifacts artifacts_manifest.json
+          "$REF" release-artifacts artifacts_manifest.bzl
 
       - name: Upload artifacts manifest
-        if: hashFiles('artifacts_manifest.json') != ''
+        if: hashFiles('artifacts_manifest.bzl') != ''
         uses: actions/upload-artifact@v7
         with:
           name: release-manifest
-          path: artifacts_manifest.json
+          path: artifacts_manifest.bzl
           if-no-files-found: error
 
       - name: Create release archive and notes
diff --git a/dev/release_artifacts/generate_artifacts_manifest.py b/dev/release_artifacts/generate_artifacts_manifest.py
index 99aab4d..30b5b4b 100755
--- a/dev/release_artifacts/generate_artifacts_manifest.py
+++ b/dev/release_artifacts/generate_artifacts_manifest.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Generates an artifacts_manifest.json file for built release artifacts."""
+"""Generates an artifacts_manifest.bzl file for built release artifacts."""
 
 from __future__ import annotations
 
@@ -86,6 +86,19 @@
     return {"artifacts": dict(sorted(artifacts.items()))}
 
 
+def render_bzl(manifest: dict[str, dict[str, list[ArtifactEntry]]]) -> str:
+    manifest_json = json.dumps(manifest, indent=1, sort_keys=True)
+    # Embedded in a Starlark string literal, so it must not end the literal
+    # early or rely on escape sequences.
+    if '"""' in manifest_json or "\\" in manifest_json:
+        raise ValueError("Manifest JSON cannot be embedded in a .bzl string")
+    return (
+        '"""Manifest of prebuilt tool binaries for this release."""\n'
+        "\n"
+        f'ARTIFACTS_MANIFEST_JSON = """\n{manifest_json}\n"""\n'
+    )
+
+
 def _main(argv: list[str] | None = None) -> int:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("tag", help="Release tag name (e.g. 2.4.0).")
@@ -97,7 +110,7 @@
     parser.add_argument(
         "output_file",
         type=Path,
-        help="Path to the artifacts_manifest.json file to write.",
+        help="Path to the artifacts_manifest.bzl file to write.",
     )
     parser.add_argument(
         "repo",
@@ -108,14 +121,12 @@
     args = parser.parse_args(argv)
     try:
         manifest = generate_manifest(args.tag, args.artifacts_dir, args.repo)
+        manifest_bzl = render_bzl(manifest)
     except ValueError as err:
         print(f"::error::{err}")
         return 1
     args.output_file.parent.mkdir(parents=True, exist_ok=True)
-    args.output_file.write_text(
-        json.dumps(manifest, indent=2, sort_keys=True) + "\n",
-        encoding="utf-8",
-    )
+    args.output_file.write_text(manifest_bzl, encoding="utf-8")
     return 0
 
 
diff --git a/python/extensions/config.bzl b/python/extensions/config.bzl
index 63d0a6e..f836574 100644
--- a/python/extensions/config.bzl
+++ b/python/extensions/config.bzl
@@ -88,7 +88,7 @@
             if mod.is_root:
                 explicit_init_py_modules[""] = str(tag.default)
 
-    configure_tool_toolchains(module_ctx)
+    configure_tool_toolchains()
     internal_config_repo(
         name = "rules_python_internal",
         transition_setting_generators = transition_setting_generators,
diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel
index 075b77b..74013a5 100644
--- a/python/private/BUILD.bazel
+++ b/python/private/BUILD.bazel
@@ -252,6 +252,11 @@
 )
 
 bzl_library(
+    name = "artifacts_manifest",
+    srcs = ["artifacts_manifest.bzl"],
+)
+
+bzl_library(
     name = "attr_builders",
     srcs = ["attr_builders.bzl"],
     deps = [
diff --git a/python/private/artifacts_manifest.bzl b/python/private/artifacts_manifest.bzl
new file mode 100644
index 0000000..13958b0
--- /dev/null
+++ b/python/private/artifacts_manifest.bzl
@@ -0,0 +1,16 @@
+"""Manifest of prebuilt tool binaries.
+
+Empty in git, so source checkouts build the tools from source. The release
+process replaces this file in the release archive with one whose
+`ARTIFACTS_MANIFEST_JSON` holds the JSON manifest of the release's binaries.
+
+It's a .bzl file so the `config` extension can load it rather than read it by
+Label, which would make Bazel's remote repository contents cache materialize
+the entire repository. Once https://github.com/bazelbuild/bazel/pull/31493
+lands, reading by Label no longer materializes, and this can go back to being
+a plain JSON file.
+"""
+
+# A JSON string rather than Starlark values, so the builtin manifest is parsed
+# by the same code path as a custom prebuilt tools manifest.
+ARTIFACTS_MANIFEST_JSON = ""
diff --git a/python/private/artifacts_manifest.json b/python/private/artifacts_manifest.json
deleted file mode 100644
index 0416886..0000000
--- a/python/private/artifacts_manifest.json
+++ /dev/null
@@ -1,3 +0,0 @@
-# Intentionally empty in git so source checkouts build tools from source.
-# In releases, this contains a JSON manifest of prebuilt tool binaries.
-# Generated during the release process and bundled into the release archive.
diff --git a/python/private/toolchains/BUILD.bazel b/python/private/toolchains/BUILD.bazel
index 3587baa..7ca7aaf 100644
--- a/python/private/toolchains/BUILD.bazel
+++ b/python/private/toolchains/BUILD.bazel
@@ -58,6 +58,7 @@
     name = "repo",
     srcs = ["repo.bzl"],
     deps = [
+        "//python/private:artifacts_manifest",
         "//python/private:bazel_tools",
         "//python/private:text_util",
         "//python/private:toolchain_types",
diff --git a/python/private/toolchains/repo.bzl b/python/private/toolchains/repo.bzl
index 05bc5c1..af2fac9 100644
--- a/python/private/toolchains/repo.bzl
+++ b/python/private/toolchains/repo.bzl
@@ -3,10 +3,13 @@
 """
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file")
+
+# Loaded rather than read by Label: reading a file of @rules_python by Label
+# makes --experimental_remote_repo_contents_cache materialize the whole repo.
+load("//python/private:artifacts_manifest.bzl", "ARTIFACTS_MANIFEST_JSON")
 load("//python/private:text_util.bzl", "render")
 load("//python/private:toolchain_types.bzl", "EXE_ZIP_MAKER_TOOLCHAIN_TYPE")
 
-_ARTIFACTS_MANIFEST_LABEL = Label("//python/private:artifacts_manifest.json")
 _IS_RUST_EXE_ZIP_MAKER_ENABLED = (
     "@rules_python//python/private/toolchains:is_rust_exe_zip_maker_enabled"
 )
@@ -110,38 +113,34 @@
 
 # Called by _config_impl in //python/extensions:config.bzl.
 def configure_tool_toolchains(
-        mctx,
         http_file = http_file,
         prebuilt_exe_zip_maker_toolchain_impl_repo = (
             prebuilt_exe_zip_maker_toolchain_impl_repo
         ),
-        tools_toolchains_repo = tools_toolchains_repo):
+        tools_toolchains_repo = tools_toolchains_repo,
+        artifacts_manifest_json = ARTIFACTS_MANIFEST_JSON):
     """Configures prebuilt or from-source toolchains in `@rules_python_tools`.
 
     Called by the `config` module extension (`//python/extensions:config.bzl`).
-    If `//python/private:artifacts_manifest.json` contains non-comment JSON
-    content (as in release archives), creates an `http_file` repo and a
+    If `ARTIFACTS_MANIFEST_JSON` from `//python/private:artifacts_manifest.bzl`
+    is non-empty (as in release archives), creates an `http_file` repo and a
     `prebuilt_exe_zip_maker_toolchain_impl_repo` repo for each entry in the
     manifest, and registers `toolchain()` declarations in
     `@rules_python_tools//toolchains`.
-    Otherwise (when the manifest file contains only `#`-comments, as in git
-    checkouts and source builds), registers the from-source `toolchain()`
-    declaration in `@rules_python_tools//toolchains`.
+    Otherwise (as in git checkouts and source builds), registers the
+    from-source `toolchain()` declaration in `@rules_python_tools//toolchains`.
 
     Args:
-        mctx: The `module_ctx` object.
         http_file: Repository rule callable for downloading prebuilt tool
             binaries (injectable for testing).
         prebuilt_exe_zip_maker_toolchain_impl_repo: Repository rule callable for
             creating `py_exe_zip_maker_toolchain` repos (injectable for testing).
         tools_toolchains_repo: Repository rule callable for creating
             `@rules_python_tools` (injectable for testing).
+        artifacts_manifest_json: The JSON manifest of prebuilt tool binaries,
+            or empty to build from source (injectable for testing).
     """
-    manifest_content = "\n".join([
-        line
-        for line in mctx.read(_ARTIFACTS_MANIFEST_LABEL).splitlines()
-        if not line.strip().startswith("#")
-    ]).strip()
+    manifest_content = artifacts_manifest_json.strip()
     toolchain_names = []
     toolchain_impls = []
     exec_compatible_with = {}
diff --git a/tests/exe_zip_maker_toolchain/exe_zip_maker_toolchain_tests.bzl b/tests/exe_zip_maker_toolchain/exe_zip_maker_toolchain_tests.bzl
index ed235fb..1e7fd19 100644
--- a/tests/exe_zip_maker_toolchain/exe_zip_maker_toolchain_tests.bzl
+++ b/tests/exe_zip_maker_toolchain/exe_zip_maker_toolchain_tests.bzl
@@ -125,20 +125,14 @@
     )
 
 def _test_configure_tool_toolchains_from_source_impl(env):
-    manifest_label = str(Label("//python/private:artifacts_manifest.json"))
     http_file = mocks.repo()
     toolchain_repo = mocks.repo()
     tools_toolchains_repo = mocks.repo()
-    mctx = mocks.mctx(
-        mock_files = {
-            manifest_label: "# Stub manifest with only comments.\n# Another comment.\n\n",
-        },
-    )
     configure_tool_toolchains(
-        mctx,
         http_file = http_file.rule,
         prebuilt_exe_zip_maker_toolchain_impl_repo = toolchain_repo.rule,
         tools_toolchains_repo = tools_toolchains_repo.rule,
+        artifacts_manifest_json = "\n",
     )
     env.expect.that_collection(http_file.calls).contains_exactly([])
     env.expect.that_collection(toolchain_repo.calls).contains_exactly([])
@@ -164,7 +158,6 @@
     )
 
 def _test_configure_tool_toolchains_prebuilt_impl(env):
-    manifest_label = str(Label("//python/private:artifacts_manifest.json"))
     manifest_json = json.encode({
         "artifacts": {
             "exe_zip_maker": [
@@ -192,12 +185,11 @@
     http_file = mocks.repo()
     toolchain_repo = mocks.repo()
     tools_toolchains_repo = mocks.repo()
-    mctx = mocks.mctx(mock_files = {manifest_label: manifest_json})
     configure_tool_toolchains(
-        mctx,
         http_file = http_file.rule,
         prebuilt_exe_zip_maker_toolchain_impl_repo = toolchain_repo.rule,
         tools_toolchains_repo = tools_toolchains_repo.rule,
+        artifacts_manifest_json = manifest_json,
     )
     env.expect.that_collection(http_file.calls).contains_exactly([
         {
diff --git a/tests/integration/prebuilt_toolchain_test.py b/tests/integration/prebuilt_toolchain_test.py
index ab12a9c..28f31db 100644
--- a/tests/integration/prebuilt_toolchain_test.py
+++ b/tests/integration/prebuilt_toolchain_test.py
@@ -34,7 +34,7 @@
 
         # Simulate a release archive where tool dependencies (such as
         # rules_rust) are dev-only and prebuilt toolchains are configured via
-        # artifacts_manifest.json.
+        # artifacts_manifest.bzl.
         module_bazel = self.rules_python_dir / "MODULE.bazel"
         module_content = module_bazel.read_text(encoding="utf-8")
         self.assertIn("\nTOOL_DEPS_DEV_ONLY = False\n", module_content)
@@ -66,9 +66,9 @@
                 ],
             },
         }
-        manifest_path = self.rules_python_dir / "python/private/artifacts_manifest.json"
+        manifest_path = self.rules_python_dir / "python/private/artifacts_manifest.bzl"
         manifest_path.write_text(
-            json.dumps(manifest, indent=2) + "\n",
+            f'ARTIFACTS_MANIFEST_JSON = """{json.dumps(manifest, indent=1)}"""\n',
             encoding="utf-8",
         )
 
diff --git a/tests/support/mocks/mocks.bzl b/tests/support/mocks/mocks.bzl
index b7c721e..addf05a 100644
--- a/tests/support/mocks/mocks.bzl
+++ b/tests/support/mocks/mocks.bzl
@@ -102,8 +102,19 @@
         is_root = is_root,
     )
 
+_RULES_PYTHON_REPO_NAME = Label("//:MODULE.bazel").repo_name
+
+def _fail_on_rules_python_file(x):
+    # Reading or taking the path of a file of @rules_python by Label makes
+    # --experimental_remote_repo_contents_cache materialize the whole repo.
+    # https://github.com/bazelbuild/bazel/pull/31493 avoids this for reads.
+    if (type(x) == "Label" and x.repo_name == _RULES_PYTHON_REPO_NAME and
+        (x.package + "/").startswith("python/")):
+        fail("module_ctx must not read files of @rules_python: " + str(x))
+
 def _mctx_read(self, x, watch = None):
     _ = watch  # @unused
+    _fail_on_rules_python_file(x)
     path_str = x._path if hasattr(x, "_path") else str(x)
     if path_str in self.mock_files:
         return self.mock_files[path_str]
@@ -113,6 +124,7 @@
     fail("File not found in mock_files: " + path_str)
 
 def _mctx_path(self, x):
+    _fail_on_rules_python_file(x)
     return _path_new(str(x), self.mock_files)
 
 def _get_download_file_name(url, output = ""):
diff --git a/tests/workflows/release_artifacts_test.py b/tests/workflows/release_artifacts_test.py
index f811645..c503d7a 100644
--- a/tests/workflows/release_artifacts_test.py
+++ b/tests/workflows/release_artifacts_test.py
@@ -20,7 +20,7 @@
 
     artifacts_dir = tmp_path / "release-artifacts"
     artifacts_dir.mkdir(parents=True)
-    manifest_file = tmp_path / "artifacts_manifest.json"
+    manifest_file = tmp_path / "artifacts_manifest.bzl"
 
     linux_bin = artifacts_dir / "exe_zip_maker-x86_64-unknown-linux-gnu"
     linux_bin.write_bytes(b"linux-binary-contents")
@@ -45,7 +45,10 @@
         check=True,
     )
 
-    manifest = json.loads(manifest_file.read_text(encoding="utf-8"))
+    manifest_bzl = manifest_file.read_text(encoding="utf-8")
+    _, marker, manifest_json = manifest_bzl.partition('ARTIFACTS_MANIFEST_JSON = """')
+    assert marker
+    manifest = json.loads(manifest_json.removesuffix('"""\n'))
     base_url = "https://github.com/bazel-contrib/rules_python/releases/download/2.4.0"
     assert manifest == {
         "artifacts": {
@@ -93,8 +96,8 @@
     repo_dir.mkdir()
     shutil.copy2(module_bazel, repo_dir / "MODULE.bazel")
     (repo_dir / "python/private").mkdir(parents=True)
-    (repo_dir / "python/private/artifacts_manifest.json").write_text(
-        "# stub\n", encoding="utf-8"
+    (repo_dir / "python/private/artifacts_manifest.bzl").write_text(
+        'ARTIFACTS_MANIFEST_JSON = ""\n', encoding="utf-8"
     )
     (repo_dir / "README.md").write_text("# rules_python\n", encoding="utf-8")
 
@@ -114,7 +117,7 @@
             "git",
             "add",
             "MODULE.bazel",
-            "python/private/artifacts_manifest.json",
+            "python/private/artifacts_manifest.bzl",
             "README.md",
         ],
         cwd=repo_dir,
@@ -128,10 +131,8 @@
         check=True,
     )
 
-    manifest_content = '{"artifacts": {"exe_zip_maker": []}}\n'
-    (repo_dir / "artifacts_manifest.json").write_text(
-        manifest_content, encoding="utf-8"
-    )
+    manifest_content = 'ARTIFACTS_MANIFEST_JSON = """{"artifacts": {}}"""\n'
+    (repo_dir / "artifacts_manifest.bzl").write_text(manifest_content, encoding="utf-8")
 
     subprocess.run(
         [str(archive_script), "2.4.0"],
@@ -146,12 +147,11 @@
         names = tar.getnames()
         assert names.count("rules_python-2.4.0/MODULE.bazel") == 1
         assert (
-            names.count("rules_python-2.4.0/python/private/artifacts_manifest.json")
-            == 1
+            names.count("rules_python-2.4.0/python/private/artifacts_manifest.bzl") == 1
         )
 
         manifest_member = tar.extractfile(
-            "rules_python-2.4.0/python/private/artifacts_manifest.json"
+            "rules_python-2.4.0/python/private/artifacts_manifest.bzl"
         )
         assert manifest_member is not None
         archived_manifest = manifest_member.read().decode("utf-8")
diff --git a/tests/workflows/release_build_artifacts_test.py b/tests/workflows/release_build_artifacts_test.py
index 11232dc..9262d49 100644
--- a/tests/workflows/release_build_artifacts_test.py
+++ b/tests/workflows/release_build_artifacts_test.py
@@ -120,8 +120,8 @@
         "TOOL_DEPS_DEV_ONLY = False\n", encoding="utf-8"
     )
     (origin / "python/private").mkdir(parents=True)
-    (origin / "python/private/artifacts_manifest.json").write_text(
-        "# stub\n", encoding="utf-8"
+    (origin / "python/private/artifacts_manifest.bzl").write_text(
+        'ARTIFACTS_MANIFEST_JSON = ""\n', encoding="utf-8"
     )
     (origin / "python/features.bzl").write_text(
         '_VERSION_PRIVATE = "$Format:%(describe:tags=true)$"\n'
@@ -133,7 +133,7 @@
         "add",
         ".gitattributes",
         "MODULE.bazel",
-        "python/private/artifacts_manifest.json",
+        "python/private/artifacts_manifest.bzl",
         "python/features.bzl",
     )
     git(origin, "commit", "-m", "Release commit")
@@ -149,7 +149,9 @@
     # Clone without tags at the later commit (matches actions/checkout default)
     clone = tmp_path / "clone"
     git(tmp_path, "clone", "--no-tags", str(origin), str(clone))
-    (clone / "artifacts_manifest.json").write_text("{}\n", encoding="utf-8")
+    (clone / "artifacts_manifest.bzl").write_text(
+        'ARTIFACTS_MANIFEST_JSON = "{}"\n', encoding="utf-8"
+    )
 
     # Git < 2.35 (e.g. Debian 11 / Ubuntu 22.04 on Buildkite) does not support
     # %(describe:tags=true) and leaves the literal placeholder unexpanded.