ci: verify a pinned sha256 for every toolchain download (#4098)
Fixes #4079
diff --git a/.circleci/config2.yml b/.circleci/config2.yml
index 7cc8848..a539c78 100644
--- a/.circleci/config2.yml
+++ b/.circleci/config2.yml
@@ -12,12 +12,6 @@
type: string
steps:
- - run:
- name: Set toolchain url
- command: |
- toolchain_url=$(jq -r '."<< parameters.toolchain >>"' .github/actions/setup_toolchain/toolchain.json)
- echo "export toolchain_url=$toolchain_url" >> $BASH_ENV
-
- restore_cache:
name: Restore Toolchain Cache
key: deps-<< parameters.toolchain >>-{{ checksum ".github/actions/setup_toolchain/toolchain.json" }}
@@ -30,17 +24,18 @@
# download if folder does not exist (not cached)
if [ ! -d ~/cache/<< parameters.toolchain >> ]; then
mkdir -p ~/cache/<< parameters.toolchain >>
+ fetch=.github/actions/setup_toolchain/download/fetch.sh
if [[ << parameters.toolchain >> == rx-gcc ]]; then
- wget --progress=dot:giga $toolchain_url -O toolchain.run
+ $fetch << parameters.toolchain >> toolchain.run
chmod +x toolchain.run
./toolchain.run -p ~/cache/<< parameters.toolchain >>/gnurx -y
elif [[ << parameters.toolchain >> == ft9xx-gcc ]]; then
- wget --progress=dot:giga $toolchain_url -O ~/cache/<< parameters.toolchain >>/ft9xxtoolchain.deb
+ $fetch << parameters.toolchain >> ~/cache/<< parameters.toolchain >>/ft9xxtoolchain.deb
elif [[ << parameters.toolchain >> == arm-iar ]]; then
- wget --progress=dot:giga https://netstorage.iar.com/FileStore/STANDARD/001/003/926/iar-lmsc-tools_1.8_amd64.deb -O ~/cache/<< parameters.toolchain >>/iar-lmsc-tools.deb
- wget --progress=dot:giga $toolchain_url -O ~/cache/<< parameters.toolchain >>/toolchain.deb
+ $fetch iar-lmsc-tools ~/cache/<< parameters.toolchain >>/iar-lmsc-tools.deb
+ $fetch << parameters.toolchain >> ~/cache/<< parameters.toolchain >>/toolchain.deb
else
- wget --progress=dot:giga $toolchain_url -O toolchain.tar.gz
+ $fetch << parameters.toolchain >> toolchain.tar.gz
tar -C ~/cache/<< parameters.toolchain >> -xaf toolchain.tar.gz
fi
fi
diff --git a/.github/actions/setup_toolchain/action.yml b/.github/actions/setup_toolchain/action.yml
index f78fe4d..fba3bf1 100644
--- a/.github/actions/setup_toolchain/action.yml
+++ b/.github/actions/setup_toolchain/action.yml
@@ -25,7 +25,7 @@
env:
TOOLCHAIN: ${{ inputs.toolchain }}
run: |
- TOOLCHAIN_URL=$(jq -r --arg tc "$TOOLCHAIN" '.[$tc]' .github/actions/setup_toolchain/toolchain.json)
+ TOOLCHAIN_URL=$(jq -er --arg tc "$TOOLCHAIN" '.[$tc].url' .github/actions/setup_toolchain/toolchain.json)
echo "toolchain_url=$TOOLCHAIN_URL"
echo "toolchain_url=$TOOLCHAIN_URL" >> $GITHUB_OUTPUT
shell: bash
diff --git a/.github/actions/setup_toolchain/download/action.yml b/.github/actions/setup_toolchain/download/action.yml
index 28a8b9f..c85bf57 100644
--- a/.github/actions/setup_toolchain/download/action.yml
+++ b/.github/actions/setup_toolchain/download/action.yml
@@ -17,33 +17,35 @@
id: cache-toolchain-download
with:
path: ~/cache/${{ inputs.toolchain }}
- key: ${{ runner.os }}-${{ inputs.toolchain }}-${{ inputs.toolchain_url }}
+ # whole manifest: arm-iar also installs the iar-lmsc-tools entry
+ key: ${{ runner.os }}-${{ inputs.toolchain }}-${{ hashFiles('.github/actions/setup_toolchain/toolchain.json') }}
- name: Install Toolchain
if: steps.cache-toolchain-download.outputs.cache-hit != 'true'
env:
TOOLCHAIN: ${{ inputs.toolchain }}
TOOLCHAIN_URL: ${{ inputs.toolchain_url }}
+ FETCH: .github/actions/setup_toolchain/download/fetch.sh
run: |
mkdir -p ~/cache/${TOOLCHAIN}
FILE_EXT="${TOOLCHAIN_URL##*.}"
if [[ ${TOOLCHAIN} == rx-gcc ]]; then
- wget --progress=dot:giga ${TOOLCHAIN_URL} -O toolchain.run
+ "$FETCH" "$TOOLCHAIN" toolchain.run
chmod +x toolchain.run
./toolchain.run -p ~/cache/${TOOLCHAIN}/gnurx -y
elif [[ ${TOOLCHAIN} == ft9xx-gcc ]]; then
- wget --progress=dot:giga ${TOOLCHAIN_URL} -O ~/cache/${TOOLCHAIN}/ft9xxtoolchain.deb
+ "$FETCH" "$TOOLCHAIN" ~/cache/${TOOLCHAIN}/ft9xxtoolchain.deb
elif [[ ${TOOLCHAIN} == arm-iar ]]; then
- wget --progress=dot:giga https://netstorage.iar.com/FileStore/STANDARD/001/003/926/iar-lmsc-tools_1.8_amd64.deb -O ~/cache/${TOOLCHAIN}/iar-lmsc-tools.deb
- wget --progress=dot:giga ${TOOLCHAIN_URL} -O ~/cache/${TOOLCHAIN}/cxarm.deb
+ "$FETCH" iar-lmsc-tools ~/cache/${TOOLCHAIN}/iar-lmsc-tools.deb
+ "$FETCH" "$TOOLCHAIN" ~/cache/${TOOLCHAIN}/cxarm.deb
elif [[ ${FILE_EXT} == zip ]]; then
- curl -L "$TOOLCHAIN_URL" -o toolchain.zip
+ "$FETCH" "$TOOLCHAIN" toolchain.zip
# Arm's Windows zip has no top-level directory; Setup Toolchain expects one
unzip -q toolchain.zip -d ~/cache/${TOOLCHAIN}/toolchain
~/cache/${TOOLCHAIN}/toolchain/bin/arm-none-eabi-gcc.exe --version
elif [[ ${FILE_EXT} =~ ^(gz|bz2|xz)$ ]]; then
- curl -fsSL --proto '=https' --proto-redir '=https' --retry 5 --connect-timeout 30 --speed-limit 1024 --speed-time 60 "$TOOLCHAIN_URL" -o toolchain.tar.${FILE_EXT}
+ "$FETCH" "$TOOLCHAIN" toolchain.tar.${FILE_EXT}
tar -C ~/cache/${TOOLCHAIN} -xaf toolchain.tar.${FILE_EXT}
else
echo "Unsupported toolchain file extension: ${FILE_EXT}"
diff --git a/.github/actions/setup_toolchain/download/fetch.sh b/.github/actions/setup_toolchain/download/fetch.sh
new file mode 100755
index 0000000..b5fbfc2
--- /dev/null
+++ b/.github/actions/setup_toolchain/download/fetch.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+# Usage: fetch.sh NAME OUT - download toolchain.json's NAME entry over https only and verify its sha256
+set -euo pipefail
+
+name=$1 out=$2
+manifest=$(dirname "$0")/../toolchain.json
+url=$(jq -ej --arg n "$name" '.[$n].url' "$manifest")
+sha256=$(jq -ej --arg n "$name" '.[$n].sha256' "$manifest")
+
+curl -fsSL --proto '=https' --proto-redir '=https' --retry 5 --connect-timeout 30 --speed-limit 1024 --speed-time 60 "$url" -o "$out"
+
+# macOS runners have shasum but no sha256sum
+if command -v sha256sum > /dev/null; then
+ echo "$sha256 $out" | sha256sum -c -
+else
+ echo "$sha256 $out" | shasum -a 256 -c -
+fi
diff --git a/.github/actions/setup_toolchain/toolchain.json b/.github/actions/setup_toolchain/toolchain.json
index ec09a01..fa269d3 100644
--- a/.github/actions/setup_toolchain/toolchain.json
+++ b/.github/actions/setup_toolchain/toolchain.json
@@ -1,12 +1,46 @@
{
- "aarch64-gcc": "https://developer.arm.com/-/media/Files/downloads/gnu-a/10.3-2021.07/binrel/gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz",
- "arm-clang": "https://github.com/ARM-software/LLVM-embedded-toolchain-for-Arm/releases/download/release-19.1.1/LLVM-ET-Arm-19.1.1-Linux-x86_64.tar.xz",
- "arm-gcc": "https://gitlab.arm.com/api/v4/projects/tooling%2Fgnu-toolchains-for-arm/packages/generic/gnu-toolchain/14.2.rel1/arm-gnu-toolchain-14.2.rel1-x86_64-arm-none-eabi.tar.xz",
- "ft9xx-gcc": "https://github.com/Bridgetek/ft32-toolchain-linux/releases/download/v2.7.6/ft9xxtoolchain_2.7.6_amd64.deb",
- "arm-gcc-macos-latest": "https://gitlab.arm.com/api/v4/projects/tooling%2Fgnu-toolchains-for-arm/packages/generic/gnu-toolchain/14.2.rel1/arm-gnu-toolchain-14.2.rel1-darwin-arm64-arm-none-eabi.tar.xz",
- "arm-gcc-windows-latest": "https://gitlab.arm.com/api/v4/projects/tooling%2Fgnu-toolchains-for-arm/packages/generic/gnu-toolchain/14.2.rel1/arm-gnu-toolchain-14.2.rel1-mingw-w64-x86_64-arm-none-eabi.zip",
- "msp430-gcc": "https://software-dl.ti.com/msp430/msp430_public_sw/mcu/msp430/MSPGCC/9_2_0_0/export/msp430-gcc-9.2.0.50_linux64.tar.bz2",
- "riscv-gcc": "https://github.com/xpack-dev-tools/riscv-none-elf-gcc-xpack/releases/download/v13.2.0-2/xpack-riscv-none-elf-gcc-13.2.0-2-linux-x64.tar.gz",
- "rx-gcc": "https://github.com/hathach/rx_device/releases/download/0.0.1/gcc-8.3.0.202411-GNURX-ELF.run",
- "arm-iar": "https://netstorage.iar.com/FileStore/STANDARD/001/003/723/cxarm-9.70.1.deb"
+ "aarch64-gcc": {
+ "url": "https://developer.arm.com/-/media/Files/downloads/gnu-a/10.3-2021.07/binrel/gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz",
+ "sha256": "6f74b1ee370caeb716688d2e467e5b44727fdc0ed56023fe5c72c0620019ecef"
+ },
+ "arm-clang": {
+ "url": "https://github.com/ARM-software/LLVM-embedded-toolchain-for-Arm/releases/download/release-19.1.1/LLVM-ET-Arm-19.1.1-Linux-x86_64.tar.xz",
+ "sha256": "f659c625302f6d3fb50f040f748206f6fd6bb1fc7e398057dd2deaf1c1f5e8d1"
+ },
+ "arm-gcc": {
+ "url": "https://gitlab.arm.com/api/v4/projects/tooling%2Fgnu-toolchains-for-arm/packages/generic/gnu-toolchain/14.2.rel1/arm-gnu-toolchain-14.2.rel1-x86_64-arm-none-eabi.tar.xz",
+ "sha256": "62a63b981fe391a9cbad7ef51b17e49aeaa3e7b0d029b36ca1e9c3b2a9b78823"
+ },
+ "ft9xx-gcc": {
+ "url": "https://github.com/Bridgetek/ft32-toolchain-linux/releases/download/v2.7.6/ft9xxtoolchain_2.7.6_amd64.deb",
+ "sha256": "cedc2ed01a3eb068b2341f1c9348c07de7d408fe15fe0144e38dd575356e038b"
+ },
+ "arm-gcc-macos-latest": {
+ "url": "https://gitlab.arm.com/api/v4/projects/tooling%2Fgnu-toolchains-for-arm/packages/generic/gnu-toolchain/14.2.rel1/arm-gnu-toolchain-14.2.rel1-darwin-arm64-arm-none-eabi.tar.xz",
+ "sha256": "c7c78ffab9bebfce91d99d3c24da6bf4b81c01e16cf551eb2ff9f25b9e0a3818"
+ },
+ "arm-gcc-windows-latest": {
+ "url": "https://gitlab.arm.com/api/v4/projects/tooling%2Fgnu-toolchains-for-arm/packages/generic/gnu-toolchain/14.2.rel1/arm-gnu-toolchain-14.2.rel1-mingw-w64-x86_64-arm-none-eabi.zip",
+ "sha256": "f074615953f76036e9a51b87f6577fdb4ed8e77d3322a6f68214e92e7859888f"
+ },
+ "msp430-gcc": {
+ "url": "https://software-dl.ti.com/msp430/msp430_public_sw/mcu/msp430/MSPGCC/9_2_0_0/export/msp430-gcc-9.2.0.50_linux64.tar.bz2",
+ "sha256": "a439316f586e5c8e4b18843e7f7648dfce01f3ea0ebd9ed6837bdcdf931c45c3"
+ },
+ "riscv-gcc": {
+ "url": "https://github.com/xpack-dev-tools/riscv-none-elf-gcc-xpack/releases/download/v13.2.0-2/xpack-riscv-none-elf-gcc-13.2.0-2-linux-x64.tar.gz",
+ "sha256": "52545afb900200fbf65fe05f7ce7090b8a42c64091f4f5d43cae6bb68ea2434a"
+ },
+ "rx-gcc": {
+ "url": "https://github.com/hathach/rx_device/releases/download/0.0.1/gcc-8.3.0.202411-GNURX-ELF.run",
+ "sha256": "498ba69b8f2dfe8b731e12709a1cf4c7d591ef2223fa1140e901fa9277f650f8"
+ },
+ "arm-iar": {
+ "url": "https://netstorage.iar.com/FileStore/STANDARD/001/003/723/cxarm-9.70.1.deb",
+ "sha256": "54c97d4379d1cc3bee3ee0e10423167aaf406aa037261e6f72894fd740753ece"
+ },
+ "iar-lmsc-tools": {
+ "url": "https://netstorage.iar.com/FileStore/STANDARD/001/003/926/iar-lmsc-tools_1.8_amd64.deb",
+ "sha256": "c54bdbaee9b4c73723ee4de4285bb6915bf05c9ec4643eaedf0b56cd8bf3e538"
+ }
}
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 059ec1f..a77ac95 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -92,6 +92,12 @@
entry: sh -c "python3 test/hil/test/test_ci_select.py && python3 test/hil/test/test_ci_metrics.py && cd test/hil/test && python3 -m unittest -q test_hil_util.BottomLayer"
pass_filenames: false
language: system
+ - id: setup-toolchain-test
+ name: setup-toolchain-test
+ files: ^(\.github/actions/setup_toolchain/|test/hil/test/test_setup_toolchain\.py$)
+ entry: python3 test/hil/test/test_setup_toolchain.py
+ pass_filenames: false
+ language: system
# language: python, not system, for the next two hooks: their suites are
# stdlib-only except where they parse YAML (skill front matter,
# .pre-commit-config.yaml), so pre-commit's venv supplies PyYAML rather than
diff --git a/test/hil/test/test_ci_select.py b/test/hil/test/test_ci_select.py
index 74c0cde..ca239f8 100644
--- a/test/hil/test/test_ci_select.py
+++ b/test/hil/test/test_ci_select.py
@@ -1291,6 +1291,7 @@
'test/hil/test/test_hil_usbtest_verdict.py',
'test/hil/test/test_hil_util.py',
'test/hil/test/test_membrowse_cli.py',
+ 'test/hil/test/test_setup_toolchain.py',
'test/hil/test/usbtest_harness.py',
], 'test/hil/test/ gained or lost a file; it is carved out of rule 2, so confirm '
'the rig still does not read anything in there before updating this list')
diff --git a/test/hil/test/test_setup_toolchain.py b/test/hil/test/test_setup_toolchain.py
new file mode 100644
index 0000000..2a1c51b
--- /dev/null
+++ b/test/hil/test/test_setup_toolchain.py
@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+"""Tests for .github/actions/setup_toolchain: the toolchain manifest and fetch.sh, with curl stubbed."""
+import hashlib
+import json
+import os
+import shutil
+import subprocess
+import tempfile
+import unittest
+
+REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..'))
+ACTION = os.path.join(REPO, '.github', 'actions', 'setup_toolchain')
+FETCH = os.path.join(ACTION, 'download', 'fetch.sh')
+PAYLOAD = 'toolchain bytes\n'
+DIGEST = hashlib.sha256(PAYLOAD.encode()).hexdigest()
+
+# writes CURL_PAYLOAD to the -o file and logs the call, or fails like curl -f on a 404 when CURL_FAIL is set
+CURL_STUB = '''#!/bin/sh
+echo "$@" >> "$CURL_LOG"
+[ -n "$CURL_FAIL" ] && exit 22
+while [ $# -gt 0 ]; do
+ [ "$1" = -o ] && { printf '%s' "$CURL_PAYLOAD" > "$2"; exit 0; }
+ shift
+done
+exit 2
+'''
+
+class Manifest(unittest.TestCase):
+ def test_every_entry_pins_an_https_url_and_sha256(self):
+ with open(os.path.join(ACTION, 'toolchain.json')) as f:
+ manifest = json.load(f)
+ for name, entry in manifest.items():
+ with self.subTest(name):
+ self.assertEqual(set(entry), {'url', 'sha256'})
+ self.assertTrue(entry['url'].startswith('https://'))
+ self.assertRegex(entry['sha256'], r'^[0-9a-f]{64}$')
+
+
+class Fetch(unittest.TestCase):
+ def setUp(self):
+ self.tmp = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, self.tmp)
+ # fetch.sh reads ../toolchain.json beside itself, so run a copy next to a fixture manifest
+ os.mkdir(os.path.join(self.tmp, 'download'))
+ self.fetch_sh = shutil.copy(FETCH, os.path.join(self.tmp, 'download', 'fetch.sh'))
+ self.bin = os.path.join(self.tmp, 'bin')
+ os.mkdir(self.bin)
+ self._write_exe('curl', CURL_STUB)
+ self.curl_log = os.path.join(self.tmp, 'curl.log')
+ self.out = os.path.join(self.tmp, 'toolchain.tar.xz')
+
+ def _write_exe(self, name, text):
+ path = os.path.join(self.bin, name)
+ with open(path, 'w') as f:
+ f.write(text)
+ os.chmod(path, 0o755)
+
+ def _manifest(self, **entries):
+ with open(os.path.join(self.tmp, 'toolchain.json'), 'w') as f:
+ json.dump(entries, f)
+
+ def fetch(self, name='tc', path=None, **env):
+ env = {'PATH': path or f'{self.bin}:{os.environ["PATH"]}', 'CURL_LOG': self.curl_log, 'CURL_PAYLOAD': PAYLOAD, **env}
+ return subprocess.run([self.fetch_sh, name, self.out], env=env, capture_output=True, text=True)
+
+ def curl_called(self):
+ return os.path.exists(self.curl_log)
+
+ def test_matching_digest_succeeds(self):
+ self._manifest(tc={'url': 'https://example.invalid/tc.tar.xz', 'sha256': DIGEST})
+ r = self.fetch()
+ self.assertEqual(r.returncode, 0, r.stderr)
+ with open(self.out) as f:
+ self.assertEqual(f.read(), PAYLOAD)
+ with open(self.curl_log) as f:
+ self.assertIn('https://example.invalid/tc.tar.xz', f.read())
+
+ def test_mismatched_digest_fails(self):
+ self._manifest(tc={'url': 'https://example.invalid/tc.tar.xz', 'sha256': '0' * 64})
+ self.assertNotEqual(self.fetch().returncode, 0)
+
+ def test_unknown_name_fails_before_download(self):
+ self._manifest(tc={'url': 'https://example.invalid/tc.tar.xz', 'sha256': DIGEST})
+ self.assertNotEqual(self.fetch('other').returncode, 0)
+ self.assertFalse(self.curl_called())
+
+ def test_missing_digest_fails_before_download(self):
+ self._manifest(tc={'url': 'https://example.invalid/tc.tar.xz'})
+ self.assertNotEqual(self.fetch().returncode, 0)
+ self.assertFalse(self.curl_called())
+
+ def test_download_failure_stops_before_verification(self):
+ self._manifest(tc={'url': 'https://example.invalid/tc.tar.xz', 'sha256': DIGEST})
+ self.assertEqual(self.fetch(CURL_FAIL='1').returncode, 22)
+
+ @unittest.skipUnless(shutil.which('shasum') and shutil.which('perl'), 'shasum not installed')
+ def test_falls_back_to_shasum_without_sha256sum(self):
+ # PATH holds only what fetch.sh and shasum need, so sha256sum is absent as on macOS
+ for tool in ('bash', 'dirname', 'jq', 'perl', 'shasum'):
+ os.symlink(shutil.which(tool), os.path.join(self.bin, tool))
+ self._manifest(good={'url': 'https://example.invalid/a', 'sha256': DIGEST},
+ bad={'url': 'https://example.invalid/b', 'sha256': '0' * 64})
+ good = self.fetch('good', path=self.bin)
+ self.assertEqual(good.returncode, 0, good.stderr)
+ self.assertNotEqual(self.fetch('bad', path=self.bin).returncode, 0)
+
+ def test_native_windows_jq_line_endings(self):
+ # native jq.exe turns every LF it prints into CRLF; command substitution keeps the CR
+ self._write_exe('jq', f'#!/bin/bash\nset -o pipefail\n{shutil.which("jq")} "$@" | perl -pe "s/\\n/\\r\\n/"\n')
+ self._manifest(tc={'url': 'https://example.invalid/tc.tar.xz', 'sha256': DIGEST})
+ r = self.fetch()
+ self.assertEqual(r.returncode, 0, r.stderr)
+ with open(self.curl_log) as f:
+ self.assertIn('https://example.invalid/tc.tar.xz -o', f.read())
+
+ def test_curl_restricts_to_https_including_redirects(self):
+ # a behavioural check needs a TLS server; pin the flags that enforce it instead
+ with open(FETCH) as f:
+ script = f.read()
+ self.assertIn("--proto '=https'", script)
+ self.assertIn("--proto-redir '=https'", script)
+
+
+if __name__ == '__main__':
+ unittest.main()
