Only download necessary toolchains (#16)
Fixes #13.
We now create a repo per 'environment' (os/cpu combo) in addition to the hub repo, and then refer to the downloads of those repos from the hub.
diff --git a/multitool/private/env_repo_template/BUILD.bazel.template b/multitool/private/env_repo_template/BUILD.bazel.template
new file mode 100644
index 0000000..14f9e5b
--- /dev/null
+++ b/multitool/private/env_repo_template/BUILD.bazel.template
@@ -0,0 +1 @@
+# generated by multitool
diff --git a/multitool/private/env_repo_template/tools/BUILD.bazel.template b/multitool/private/env_repo_template/tools/BUILD.bazel.template
new file mode 100644
index 0000000..14f9e5b
--- /dev/null
+++ b/multitool/private/env_repo_template/tools/BUILD.bazel.template
@@ -0,0 +1 @@
+# generated by multitool
diff --git a/multitool/private/env_repo_tool_template/BUILD.bazel.template b/multitool/private/env_repo_tool_template/BUILD.bazel.template
new file mode 100644
index 0000000..d2f7a23
--- /dev/null
+++ b/multitool/private/env_repo_tool_template/BUILD.bazel.template
@@ -0,0 +1,3 @@
+# generated by multitool
+
+exports_files(glob(include=["*_executable"]))
diff --git a/multitool/private/tool_template/BUILD.bazel.template b/multitool/private/hub_repo_tool_template/BUILD.bazel.template
similarity index 81%
rename from multitool/private/tool_template/BUILD.bazel.template
rename to multitool/private/hub_repo_tool_template/BUILD.bazel.template
index 0165c6d..112b679 100644
--- a/multitool/private/tool_template/BUILD.bazel.template
+++ b/multitool/private/hub_repo_tool_template/BUILD.bazel.template
@@ -2,8 +2,6 @@
load(":tool.bzl", "tool")
-exports_files(glob(include=["*_executable"]))
-
toolchain_type(
name = "toolchain_type",
visibility = ["//:__subpackages__"],
diff --git a/multitool/private/tool_template/tool.bzl.template b/multitool/private/hub_repo_tool_template/tool.bzl.template
similarity index 91%
rename from multitool/private/tool_template/tool.bzl.template
rename to multitool/private/hub_repo_tool_template/tool.bzl.template
index 5ab6394..c89e8fb 100644
--- a/multitool/private/tool_template/tool.bzl.template
+++ b/multitool/private/hub_repo_tool_template/tool.bzl.template
@@ -15,7 +15,7 @@
def _declare_toolchain(name, os, cpu):
toolchain_info(
name = "{name}_{os}_{cpu}_toolchain_info".format(name=name, os=os, cpu=cpu),
- executable = "//tools/{name}:{os}_{cpu}_executable".format(name=name, os=os, cpu=cpu),
+ executable = "@@{hub_name}.{os}_{cpu}//tools/{name}:{os}_{cpu}_executable".format(name=name, os=os, cpu=cpu),
os = os,
cpu = cpu,
)
diff --git a/multitool/private/multitool.bzl b/multitool/private/multitool.bzl
index 10b08f9..aeb258b 100644
--- a/multitool/private/multitool.bzl
+++ b/multitool/private/multitool.bzl
@@ -1,30 +1,64 @@
-"multitool hub implementation"
+"""
+multitool
-_HUB_TEMPLATE = "//multitool/private:hub_repo_template/{filename}.template"
-_TOOL_TEMPLATE = "//multitool/private:tool_template/{filename}.template"
+Multitool takes as input a JSON lockfile and emits the following repos:
-def _render_hub(rctx, filename, substitutions = None):
- rctx.template(
- filename,
- Label(_HUB_TEMPLATE.format(filename = filename)),
- substitutions = substitutions or {},
- )
+ - [hub].[os]_[cpu], for each [os]/[cpu] combo in _SUPPORTED_ENVS:
+ This repository holds os/cpu specific binaries for all tools in the provided
+ lockfile(s) and constructs clean symlinks to their content for inclusion in
+ toolchains defined in the [hub] repo.
-def _render_tool(rctx, tool_name, filename, substitutions = None):
- rctx.template(
- "tools/{tool_name}/{filename}".format(tool_name = tool_name, filename = filename),
- Label(_TOOL_TEMPLATE.format(filename = filename)),
- substitutions = {
- "{name}": tool_name,
- } | (substitutions or {}),
- )
+ The structure of this repo is, very simply:
+ tools/
+ [tool-name]/
+ BUILD.bazel (export all *_executable files)
+ [os]_[cpu]_executable (a downloaded file or a symlink to a file in a
+ downloaded and extracted archive)
+
+ - [hub]:
+ This repository holds toolchain definitions for all tools in the provided
+ lockfile(s), as well as an executable tool target that will pick the
+ appropriate toolchain.
+
+ The structure of this repo is:
+ toolchains/
+ BUILD.bazel (a single file containing all declared toolchains for easy registration)
+ tools/
+ [tool-name]/
+ BUILD.bazel (declares the toolchain_type and the executable tool target)
+ tool.bzl (scaffolding for the tool target and toolchain declarations in toolchains/BUILD.bazel)
+ toolchain_info.bzl (common scaffolding for toolchain declarations)
+
+ (additional BUILD.bazel and a WORKSPACE file are included as required by Bazel)
+
+To keep things orderly, we keep all the toolchain Bazel goo in the [hub] repo and only stash
+the binaries in the [hub].[os]_[cpu] repos. It's a conscious decision not to place some fragments
+of the toolchain definitions in the latter repos to make the dependencies run exactly one way:
+[hub] -> [hub].[os]_[cpu].
+
+This implementation depends on rendering a number of templates, which are defined in sibling
+folders and managed by the templates starlark file.
+
+To maintain support both bzlmod and non-bzlmod setups, we provide two entrypoints to the rule:
+ - (bzlmod) hub : invoked by the hub tag in extension.bzl
+ - (non-bzlmod) multitool : invoked in WORKSPACE or related macros, and additionally registers toolchains
+"""
+
+load(":templates.bzl", "templates")
+
+_SUPPORTED_ENVS = [
+ ("linux", "arm64"),
+ ("linux", "x86_64"),
+ ("macos", "arm64"),
+ ("macos", "x86_64"),
+]
def _check(condition, message):
"fails iff condition is False and emits message"
if not condition:
fail(message)
-def _multitool_hub_impl(rctx):
+def _load_tools(rctx):
tools = {}
for lockfile in rctx.attr.lockfiles:
# TODO: validate no conflicts from multiple hub declarations and/or
@@ -33,15 +67,33 @@
# (this is also a very naive merge at the tool level)
tools = tools | json.decode(rctx.read(lockfile))
- loads = []
- defines = []
+ # validation
+ for tool_name, tool in tools.items():
+ for binary in tool["binaries"]:
+ _check(
+ binary["os"] in ["linux", "macos"],
+ "{tool_name}: Unknown os '{os}'".format(
+ tool_name = tool_name,
+ os = binary["os"],
+ ),
+ )
+ _check(
+ binary["cpu"] in ["x86_64", "arm64"],
+ "{tool_name}: Unknown cpu '{cpu}'".format(
+ tool_name = tool_name,
+ cpu = binary["cpu"],
+ ),
+ )
+
+ return tools
+
+def _env_specific_tools_impl(rctx):
+ tools = _load_tools(rctx)
for tool_name, tool in tools.items():
- toolchains = []
-
for binary in tool["binaries"]:
- _check(binary["os"] in ["linux", "macos"], "Unknown os '{os}'".format(os = binary["os"]))
- _check(binary["cpu"] in ["x86_64", "arm64"], "Unknown cpu '{cpu}'".format(cpu = binary["cpu"]))
+ if binary["os"] != rctx.attr.os or binary["cpu"] != rctx.attr.cpu:
+ continue
target_executable = "tools/{tool_name}/{os}_{cpu}_executable".format(
tool_name = tool_name,
@@ -104,14 +156,42 @@
else:
fail("Unknown 'kind' {kind}".format(kind = binary["kind"]))
+ templates.env_tool(rctx, tool_name, "BUILD.bazel")
+
+ templates.env(rctx, "tools/BUILD.bazel")
+ templates.env(rctx, "BUILD.bazel")
+
+_env_specific_tools = repository_rule(
+ attrs = {
+ "lockfiles": attr.label_list(mandatory = True, allow_files = True),
+ "os": attr.string(),
+ "cpu": attr.string(),
+ },
+ implementation = _env_specific_tools_impl,
+)
+
+def _sort_fn(tup):
+ return tup[0]
+
+def _multitool_hub_impl(rctx):
+ tools = _load_tools(rctx)
+
+ loads = []
+ defines = []
+
+ for tool_name, tool in sorted(tools.items(), key = _sort_fn):
+ toolchains = []
+
+ for binary in tool["binaries"]:
toolchains.append('\n _declare_toolchain(name="{name}", os="{os}", cpu="{cpu}")'.format(
name = tool_name,
cpu = binary["cpu"],
os = binary["os"],
))
- _render_tool(rctx, tool_name, "BUILD.bazel")
- _render_tool(rctx, tool_name, "tool.bzl", {
+ templates.hub_tool(rctx, tool_name, "BUILD.bazel")
+ templates.hub_tool(rctx, tool_name, "tool.bzl", {
+ "{hub_name}": rctx.attr.name,
"{toolchains}": "\n".join(toolchains),
})
@@ -122,10 +202,10 @@
))
defines.append("declare_{clean_name}_toolchains()".format(clean_name = clean_name))
- _render_hub(rctx, "BUILD.bazel")
- _render_hub(rctx, "toolchain_info.bzl")
- _render_hub(rctx, "tools/BUILD.bazel")
- _render_hub(rctx, "toolchains/BUILD.bazel", {
+ templates.hub(rctx, "BUILD.bazel")
+ templates.hub(rctx, "toolchain_info.bzl")
+ templates.hub(rctx, "tools/BUILD.bazel")
+ templates.hub(rctx, "toolchains/BUILD.bazel", {
"{loads}": "\n".join(loads),
"{defines}": "\n".join(defines),
})
@@ -139,11 +219,16 @@
def hub(name, lockfiles):
"Create a multitool hub."
+ for env in _SUPPORTED_ENVS:
+ _env_specific_tools(
+ name = "{name}.{os}_{cpu}".format(name = name, os = env[0], cpu = env[1]),
+ lockfiles = lockfiles,
+ os = env[0],
+ cpu = env[1],
+ )
_multitool_hub(name = name, lockfiles = lockfiles)
def multitool(name, lockfile):
"(non-bzlmod) Create a multitool hub and register its toolchains."
-
- _multitool_hub(name = name, lockfiles = [lockfile])
-
- native.register_toolchains("@multitool//toolchains:all")
+ hub(name, [lockfile])
+ native.register_toolchains("@{name}//toolchains:all".format(name = name))
diff --git a/multitool/private/templates.bzl b/multitool/private/templates.bzl
new file mode 100644
index 0000000..91a5db8
--- /dev/null
+++ b/multitool/private/templates.bzl
@@ -0,0 +1,46 @@
+"multitool templating"
+
+_ENV_TEMPLATE = "//multitool/private:env_repo_template/{filename}.template"
+_ENV_TOOL_TEMPLATE = "//multitool/private:env_repo_tool_template/{filename}.template"
+
+_HUB_TEMPLATE = "//multitool/private:hub_repo_template/{filename}.template"
+_HUB_TOOL_TEMPLATE = "//multitool/private:hub_repo_tool_template/{filename}.template"
+
+def _render_env(rctx, filename, substitutions = None):
+ rctx.template(
+ filename,
+ Label(_ENV_TEMPLATE.format(filename = filename)),
+ substitutions = substitutions or {},
+ )
+
+def _render_env_tool(rctx, tool_name, filename, substitutions = None):
+ rctx.template(
+ "tools/{tool_name}/{filename}".format(tool_name = tool_name, filename = filename),
+ Label(_ENV_TOOL_TEMPLATE.format(filename = filename)),
+ substitutions = {
+ "{name}": tool_name,
+ } | (substitutions or {}),
+ )
+
+def _render_hub(rctx, filename, substitutions = None):
+ rctx.template(
+ filename,
+ Label(_HUB_TEMPLATE.format(filename = filename)),
+ substitutions = substitutions or {},
+ )
+
+def _render_hub_tool(rctx, tool_name, filename, substitutions = None):
+ rctx.template(
+ "tools/{tool_name}/{filename}".format(tool_name = tool_name, filename = filename),
+ Label(_HUB_TOOL_TEMPLATE.format(filename = filename)),
+ substitutions = {
+ "{name}": tool_name,
+ } | (substitutions or {}),
+ )
+
+templates = struct(
+ env = _render_env,
+ env_tool = _render_env_tool,
+ hub = _render_hub,
+ hub_tool = _render_hub_tool,
+)
