feat: auto-fetch node versions in module extension (#3919)
More aggressive alternative to #3916: We even do the fetching if facts
are not available.
## PR Checklist
Please check if your PR fulfills the following requirements:
- [x] Tests for the changes have been added (for bug fixes / features)
- [x] Docs have been added / updated (for bug fixes / features)
## PR Type
What kind of change does this PR introduce?
<!-- Please check the one that applies to this PR using "x". -->
- [ ] Bugfix
- [x] Feature (please, look at the "Scope of the project" section in the
README.md file)
- [ ] Code style update (formatting, local variables)
- [ ] Refactoring (no functional changes, no api changes)
- [ ] Build related changes
- [ ] CI related changes
- [ ] Documentation content changes
- [ ] Other... Please describe:
## What is the current behavior?
<!-- Please describe the current behavior that you are modifying, or
link to a relevant issue. -->
Node versions / digests are hardcoded and shipped with rules_nodejs.
## What is the new behavior?
rules_nodejs retrieves required Node digests as needed and stores them
in facts or in generatedRepoSpecs in the bzlmod lockfile.
## Does this PR introduce a breaking change?
- [ ] Yes
- [x] No
<!-- If this PR contains a breaking change, please describe the impact
and migration path for existing applications below. -->
## Other information
- #3916 (fact only alternative)
- [Example lockfile with bazel
8.1.0](https://gist.github.com/gzm0/777a23ce5201d2c70a3b18bde4b83d4a)
- [Example lockfile with bazel
9.1.1](https://gist.github.com/gzm0/eeb1e54658e78259a33f763376a79004)
diff --git a/e2e/nodejs_host/BUILD.bazel b/e2e/nodejs_host/BUILD.bazel
index 1871163..9a54ae2 100644
--- a/e2e/nodejs_host/BUILD.bazel
+++ b/e2e/nodejs_host/BUILD.bazel
@@ -23,6 +23,7 @@
"node16",
"node16_nvmrc",
"node17_custom",
+ "node26_auto_fetch",
]
]
@@ -66,5 +67,7 @@
("node16_nvmrc", "npx"),
("node17_custom", "npm"),
("node17_custom", "npx"),
+ ("node26_auto_fetch", "npm"),
+ ("node26_auto_fetch", "npx"),
]
]
diff --git a/e2e/nodejs_host/MODULE.bazel b/e2e/nodejs_host/MODULE.bazel
index 8c2b4ff..ff12b2f 100644
--- a/e2e/nodejs_host/MODULE.bazel
+++ b/e2e/nodejs_host/MODULE.bazel
@@ -35,6 +35,10 @@
node_urls = ["https://nodejs.org/dist/v17.0.1/{filename}"],
node_version = "17.0.1.custom",
)
+node.toolchain(
+ name = "node26_auto_fetch",
+ node_version = "26.3.0",
+)
# FIXME(6.0): a repo rule with name=foo should create a repo named @foo, not @foo_toolchains
use_repo(
@@ -60,6 +64,13 @@
"node17_custom_linux_arm64",
"node17_custom_toolchains",
"node17_custom_windows_amd64",
+ "node26_auto_fetch",
+ "node26_auto_fetch_darwin_amd64",
+ "node26_auto_fetch_darwin_arm64",
+ "node26_auto_fetch_linux_amd64",
+ "node26_auto_fetch_linux_arm64",
+ "node26_auto_fetch_toolchains",
+ "node26_auto_fetch_windows_amd64",
"nodejs",
"nodejs_darwin_amd64",
"nodejs_darwin_arm64",
diff --git a/nodejs/BUILD.bazel b/nodejs/BUILD.bazel
index 622b1f7..202be5a 100644
--- a/nodejs/BUILD.bazel
+++ b/nodejs/BUILD.bazel
@@ -9,7 +9,13 @@
bzl_library(
name = "extensions",
srcs = ["extensions.bzl"],
- deps = [":repositories"],
+ deps = [
+ ":repositories",
+ "//nodejs/private:fetch_node_repositories",
+ "//nodejs/private:node_versions",
+ "//nodejs/private:version_from_attr",
+ "@bazel_features//:features",
+ ],
)
bzl_library(
@@ -29,6 +35,7 @@
"//nodejs/private:nodejs_repo_host_os_alias",
"//nodejs/private:nodejs_toolchains_repo",
"//nodejs/private:os_name",
+ "//nodejs/private:version_from_attr",
],
)
diff --git a/nodejs/extensions.bzl b/nodejs/extensions.bzl
index 3b4b53f..720b056 100644
--- a/nodejs/extensions.bzl
+++ b/nodejs/extensions.bzl
@@ -9,6 +9,10 @@
```
"""
+load("@bazel_features//:features.bzl", "bazel_features")
+load("//nodejs/private:fetch_node_repositories.bzl", "fetch_node_repositories")
+load("//nodejs/private:node_versions.bzl", "NODE_VERSIONS")
+load("//nodejs/private:version_from_attr.bzl", "version_from_attr")
load(
":repositories.bzl",
"DEFAULT_NODE_REPOSITORY",
@@ -52,17 +56,50 @@
else:
registrations[toolchain.name] = toolchain
+ supports_facts = hasattr(module_ctx, "facts")
+ new_repository_facts = {}
+
for k, v in registrations.items():
+ node_version = version_from_attr(module_ctx, v)
+ node_repositories = v.node_repositories or NODE_VERSIONS.get(node_version, {})
+
+ if not node_repositories:
+ node_repositories = (
+ new_repository_facts.get(node_version) or
+ (module_ctx.facts.get(node_version) if supports_facts else None) or
+ # TODO: Add support for node_urls?
+ fetch_node_repositories(module_ctx, node_version)
+ )
+
+ new_repository_facts[node_version] = node_repositories
+
nodejs_register_toolchains(
name = k,
- node_version = v.node_version,
- node_version_from_nvmrc = v.node_version_from_nvmrc,
+ node_version = node_version,
node_urls = v.node_urls,
- node_repositories = v.node_repositories,
+ node_repositories = node_repositories,
include_headers = v.include_headers,
register = False,
)
+ if not hasattr(module_ctx, "extension_metadata"):
+ return # buildifier: disable=return-value (allow no value)
+
+ if not bazel_features.external_deps.extension_metadata_has_reproducible:
+ return module_ctx.extension_metadata()
+
+ if not supports_facts:
+ return module_ctx.extension_metadata(
+ # if we have new_repository_facts is non-empty, we called
+ # fetch_node_repositories making this invocation non-reproducible.
+ reproducible = not new_repository_facts,
+ )
+
+ return module_ctx.extension_metadata(
+ reproducible = True,
+ facts = new_repository_facts,
+ )
+
_ATTRS = {
"name": attr.string(
doc = "Base name for generated repositories",
diff --git a/nodejs/private/BUILD.bazel b/nodejs/private/BUILD.bazel
index 7388dc4..9c92ac7 100644
--- a/nodejs/private/BUILD.bazel
+++ b/nodejs/private/BUILD.bazel
@@ -41,3 +41,13 @@
srcs = ["user_build_settings.bzl"],
deps = ["//nodejs/private/providers:user_build_settings"],
)
+
+bzl_library(
+ name = "version_from_attr",
+ srcs = ["version_from_attr.bzl"],
+)
+
+bzl_library(
+ name = "fetch_node_repositories",
+ srcs = ["fetch_node_repositories.bzl"],
+)
diff --git a/nodejs/private/fetch_node_repositories.bzl b/nodejs/private/fetch_node_repositories.bzl
new file mode 100644
index 0000000..ab0627a
--- /dev/null
+++ b/nodejs/private/fetch_node_repositories.bzl
@@ -0,0 +1,58 @@
+"""Implementation of node SHASUM fetching for facts."""
+
+_REPOSITORY_TYPES = {
+ "darwin-arm64.tar.gz": "darwin_arm64",
+ "darwin-x64.tar.gz": "darwin_amd64",
+ "linux-x64.tar.xz": "linux_amd64",
+ "linux-arm64.tar.xz": "linux_arm64",
+ "linux-s390x.tar.xz": "linux_s390x",
+ "win-x64.zip": "windows_amd64",
+ "win-arm64.zip": "windows_arm64",
+ "linux-ppc64le.tar.xz": "linux_ppc64le",
+}
+
+def fetch_node_repositories(module_ctx, version):
+ """Fetches node repositories for the given node version.
+
+ Port of scripts/update-nodejs-versions.js
+
+ Args:
+ module_ctx: Module context
+ version: The node version to fetch repositories for.
+
+ Returns:
+ A dictionary in the shape of node_repositories.
+ """
+
+ shasums_filename = "{version}-SHASUMS256.txt".format(version = version)
+ url = "https://nodejs.org/dist/v{version}/SHASUMS256.txt".format(version = version)
+
+ result = module_ctx.download(url = url, output = shasums_filename)
+ if not result.success:
+ fail("Failed to fetch node shasums:", url, result, sep = "\n")
+
+ shasums = module_ctx.read(shasums_filename)
+
+ result = {}
+
+ for line in shasums.splitlines():
+ line = line.strip()
+ if not line:
+ continue
+
+ parts = line.split(" ")
+ if len(parts) != 2:
+ fail("{url} contains unexpected line:\n{line}".format(
+ url = url,
+ line = line,
+ ))
+
+ sha, filename = parts
+ type = _REPOSITORY_TYPES.get(filename.removeprefix("node-v%s-" % version))
+ if not type:
+ continue
+
+ strip_prefix = filename.removesuffix(".tar.gz").removesuffix(".tar.xz").removesuffix(".zip")
+ result[version + "-" + type] = (filename, strip_prefix, sha)
+
+ return result
diff --git a/nodejs/private/version_from_attr.bzl b/nodejs/private/version_from_attr.bzl
new file mode 100644
index 0000000..6d7fa20
--- /dev/null
+++ b/nodejs/private/version_from_attr.bzl
@@ -0,0 +1,28 @@
+"""Helper to get node version from user config."""
+
+def _verify_version_is_valid(version):
+ major, minor, patch = (version.split(".") + [None, None, None])[:3]
+ if not major.isdigit() or not minor.isdigit() or not patch.isdigit():
+ fail("Invalid node version: %s" % version)
+
+def version_from_attr(ctx, attr):
+ """Extract the node version from attr.
+
+ Verifies if the extracted version is valid.
+
+ Args:
+ ctx: repository or module context
+ attr: A struct with fields node_version and node_version_from_nvmrc
+
+ Returns:
+ The node version.
+ """
+
+ node_version = attr.node_version
+
+ if attr.node_version_from_nvmrc:
+ node_version = str(ctx.read(attr.node_version_from_nvmrc)).strip()
+
+ _verify_version_is_valid(node_version)
+
+ return node_version
diff --git a/nodejs/repositories.bzl b/nodejs/repositories.bzl
index 3a278cf..d5f7ab2 100644
--- a/nodejs/repositories.bzl
+++ b/nodejs/repositories.bzl
@@ -3,6 +3,7 @@
load("//nodejs/private:node_versions.bzl", "NODE_VERSIONS")
load("//nodejs/private:nodejs_repo_host_os_alias.bzl", "nodejs_repo_host_os_alias")
load("//nodejs/private:nodejs_toolchains_repo.bzl", "PLATFORMS", "nodejs_toolchains_repo")
+load("//nodejs/private:version_from_attr.bzl", "version_from_attr")
# Default base name for node toolchain repositories
# created by the module extension
@@ -66,12 +67,7 @@
# @nodejs_PLATFORM where PLATFORM is one of BUILT_IN_NODE_PLATFORMS
host_os = repository_ctx.attr.platform or repository_ctx.name.split("nodejs_", 1)[1]
- node_version = repository_ctx.attr.node_version
-
- if repository_ctx.attr.node_version_from_nvmrc:
- node_version = str(repository_ctx.read(repository_ctx.attr.node_version_from_nvmrc)).strip()
-
- _verify_version_is_valid(node_version)
+ node_version = version_from_attr(repository_ctx, repository_ctx.attr)
node_repositories = repository_ctx.attr.node_repositories
@@ -302,11 +298,6 @@
return path[len("bin/"):]
-def _verify_version_is_valid(version):
- major, minor, patch = (version.split(".") + [None, None, None])[:3]
- if not major.isdigit() or not minor.isdigit() or not patch.isdigit():
- fail("Invalid node version: %s" % version)
-
def _nodejs_repositories_impl(repository_ctx):
reproducible = _download_node(repository_ctx)
_prepare_node(repository_ctx)
