[veer] add FPGA board bootstrap and remote test runner
diff --git a/.bazelrc b/.bazelrc index fd02d63..4980dab 100644 --- a/.bazelrc +++ b/.bazelrc
@@ -120,10 +120,20 @@ build:k_veer --build_tag_filters=-do_not_build,-kernel_doc_test # Caliptra VCK190 FPGA board via SSH. -# Host-side test binaries compile for the VCK190's ARM cores (//toolchain/aarch64_linux) +# Host-side test binaries compile for the VCK190's ARM cores (@pigweed//targets/host:linux_aarch64) # while MCU firmware images transition to //target/veer:veer with target_type=fpga. +# +# Usage: bazel test --config=fpga --test_env=VCK190_FPGA_HOST=<host> //target/veer/... +# (or set `test:fpga --test_env=VCK190_FPGA_HOST=<host>` in .bazelrc-site). common:fpga --platforms=@pigweed//targets/host:linux_aarch64 build:fpga --build_tag_filters=-do_not_build,-kernel_doc_test +test:fpga --local_test_jobs=1 +test:fpga --run_under=//target/veer/tooling/fpga:run_remote +test:fpga --strategy=TestRunner=local +test:fpga --test_env=SSH_AUTH_SOCK +test:fpga --test_env=VCK190_FPGA_HOST +test:fpga --test_tag_filters=fpga,-disabled,-do_not_build,-do_not_run_test,-kernel_doc_test +test:fpga --test_timeout=600 # ── Release builds (SLSA provenance) ── # Used by .github/workflows/release.yml. Fails the build if MODULE.bazel.lock
diff --git a/target/veer/tooling/caliptra_runner.bzl b/target/veer/tooling/caliptra_runner.bzl index 94a8409..2860cf5 100644 --- a/target/veer/tooling/caliptra_runner.bzl +++ b/target/veer/tooling/caliptra_runner.bzl
@@ -292,10 +292,16 @@ } test_env.update(env) + test_data = list(data) + [image_label] + if iface == "fpga": + # Board-side script and bitstream used by + # //target/veer/tooling/fpga:run_remote. + test_data.append("//target/veer/tooling/fpga:board_files") + rust_test( name = target_name, srcs = srcs, - data = list(data) + [image_label], + data = test_data, deps = full_deps, edition = edition, env = test_env,
diff --git a/target/veer/tooling/fpga/BUILD.bazel b/target/veer/tooling/fpga/BUILD.bazel new file mode 100644 index 0000000..8564f54 --- /dev/null +++ b/target/veer/tooling/fpga/BUILD.bazel
@@ -0,0 +1,44 @@ +# Licensed under the Apache-2.0 license +# SPDX-License-Identifier: Apache-2.0 + +load("@bazel_skylib//rules:copy_file.bzl", "copy_file") +load("@bazel_skylib//rules:native_binary.bzl", "native_binary") + +# Symlink the external @caliptra_fpga_bitstream PDI into this package so it +# lands at _main/target/veer/tooling/fpga/ in FPGA tests' runfiles, next to +# run_on_board.sh. +copy_file( + name = "bitstream_pdi", + src = "@caliptra_fpga_bitstream//file", + out = "caliptra-bitstream.pdi", + allow_symlink = True, + tags = ["fpga"], + target_compatible_with = [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + ], +) + +# Board-side files that run_remote expects in every FPGA test's runfiles +# (added as `data` by `caliptra_test`). +filegroup( + name = "board_files", + srcs = [ + "run_on_board.sh", + ":bitstream_pdi", + ], + tags = ["fpga"], + target_compatible_with = [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + ], + visibility = ["//target/veer:__subpackages__"], +) + +# `--run_under` wrapper for `bazel test --config=fpga`. +native_binary( + name = "run_remote", + src = "run_remote.sh", + out = "run_remote", + visibility = ["//visibility:public"], +)
diff --git a/target/veer/tooling/fpga/README.md b/target/veer/tooling/fpga/README.md deleted file mode 100644 index 48a74d0..0000000 --- a/target/veer/tooling/fpga/README.md +++ /dev/null
@@ -1,2 +0,0 @@ -Tools for running OpenPRoT images on a VCK190 FPGA running the Caliptra -Subsystem RTL
diff --git a/target/veer/tooling/fpga/run_on_board.sh b/target/veer/tooling/fpga/run_on_board.sh new file mode 100755 index 0000000..4cd9f87 --- /dev/null +++ b/target/veer/tooling/fpga/run_on_board.sh
@@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# Licensed under the Apache-2.0 license +# SPDX-License-Identifier: Apache-2.0 +# +# Board-side half of run_remote.sh: runs on the VCK190 (as root, via sudo) to +# bootstrap the FPGA and run one test binary. +# +# Usage: run_on_board.sh <runfiles-dir> <test-binary> [args...] +# +# runfiles-dir The test's runfiles tree, as synced by run_remote.sh. +# test-binary Path to the test binary, relative to <runfiles-dir>/_main. +# +# steps: +# 1. take the board lock, to serialize tests using this board +# 2. Bootstrap the FPGA with the appropriate bitstream +# 3. Run the test with RUNFILES_DIR set. + +set -euo pipefail + +readonly LOCK_FILE=/run/lock/openprot-fpga.lock +readonly MAX_LOCK_WAIT_SECONDS=120 +# Our bitstream, synced with the test (path relative to <runfiles-dir>/_main). +readonly BITSTREAM=target/veer/tooling/fpga/caliptra-bitstream.pdi +# Where the kernel FPGA manager loads bitstreams from (by name). Upstream xtask +# loads its bitstream from the same path, so comparing it against BITSTREAM +# tells whether the last-loaded bitstream is ours. +readonly LOADED_BITSTREAM=/lib/firmware/caliptra-bitstream.pdi +# Upstream xtask's record of the loaded FPGA configuration. Lives in /dev/shm, +# so it is cleared on power cycle. +readonly CONFIG_MARKER=/dev/shm/fpga-config +readonly FPGA_MGR=/sys/class/fpga_manager/fpga0 + +log() { echo "[run_on_board] $*" >&2; } +die() { + log "error: $*" + exit 2 +} + +take_lock() { + exec 9>"$LOCK_FILE" + flock -w "$MAX_LOCK_WAIT_SECONDS" 9 || + die "board is busy; $LOCK_FILE held for over ${MAX_LOCK_WAIT_SECONDS}s" +} + +sha256() { { sha256sum "$1" 2>/dev/null || true; } | awk '{print $1}'; } + +bootstrap_fpga() { + # The Caliptra image ships the kernel modules and supports loading segmented + # bitstreams at runtime; stock Ubuntu images do neither. + [[ "$(hostname)" == "caliptra-fpga" ]] || + die "unsupported board image (hostname '$(hostname)'); use the Caliptra" \ + "FPGA image from caliptra-sw" + [[ -r "$BITSTREAM" ]] || die "bitstream not found: $BITSTREAM" + [[ -d "$FPGA_MGR" ]] || die "FPGA manager sysfs node missing: $FPGA_MGR" + + # Reload unless the subsystem bitstream is loaded and it's ours. + local sha + sha="$(sha256 "$BITSTREAM")" + if [[ "$(cat "$CONFIG_MARKER" 2>/dev/null || true)" != "subsystem" || + "$(sha256 "$LOADED_BITSTREAM")" != "$sha" ]]; then + log "Loading bitstream (${sha:0:12}...) into fpga0" + rm -f "$CONFIG_MARKER" # So a failed load is retried next time. + mkdir -p "$(dirname "$LOADED_BITSTREAM")" + cp -f "$BITSTREAM" "$LOADED_BITSTREAM" + basename "$LOADED_BITSTREAM" >"$FPGA_MGR/firmware" || + die "failed to write bitstream to $FPGA_MGR/firmware" + echo subsystem >"$CONFIG_MARKER" + chmod 666 "$CONFIG_MARKER" # Upstream xtask runs as a regular user. + fi + + local state + state="$(cat "$FPGA_MGR/state" 2>/dev/null || true)" + [[ "$state" == "operating" ]] || die "fpga0 state is '$state' (expected 'operating')" + for uio in /dev/uio0 /dev/uio1; do + [[ -e "$uio" ]] || die "required UIO device $uio is missing" + done +} + +[[ $# -ge 2 ]] || die "usage: run_on_board.sh <runfiles-dir> <test-binary> [args...]" +((EUID == 0)) || die "must run as root" +runfiles="$(realpath "$1")" +test_bin="$2" +shift 2 +cd "$runfiles/_main" + +take_lock +bootstrap_fpga +RUNFILES_DIR="$runfiles" exec "$test_bin" "$@"
diff --git a/target/veer/tooling/fpga/run_remote.sh b/target/veer/tooling/fpga/run_remote.sh new file mode 100755 index 0000000..e6dc02e --- /dev/null +++ b/target/veer/tooling/fpga/run_remote.sh
@@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# Licensed under the Apache-2.0 license +# SPDX-License-Identifier: Apache-2.0 +# +# Bazel `--run_under` wrapper (`bazel test --config=fpga`) that runs a +# cross-compiled aarch64 FPGA test binary on a remote VCK190 board. +# +# Invoked by Bazel as `run_remote <test-binary> [args...]`. It: +# 1. Copies the test's runfiles tree ($TEST_SRCDIR) to the board. The tree +# includes run_on_board.sh and the bitstream (added as data by +# `caliptra_test`). +# 2. Runs run_on_board.sh there as root, which bootstraps the FPGA and runs +# the test. See that script for details. +# +# Environment: +# VCK190_FPGA_HOST - Anything `ssh` accepts: an alias from ~/.ssh/config or +# user@host. + +set -euo pipefail + +readonly REMOTE_RUNFILES=openprot-fpga/runfiles # Relative to the SSH user's home. +# Test environment to forward to the board, if set. +readonly FORWARDED_ENV=(CALIPTRA_BOOT_ENV CPTRA_UIO_NUM RUST_BACKTRACE RUST_LOG RUST_TEST_THREADS) +# Share one SSH connection between rsync, ssh, and consecutive tests (kept open +# for 60s after the last use) instead of paying ~2s of connection setup for +# each. BatchMode makes ssh fail rather than prompt. +readonly SSH_OPTS=( + -o BatchMode=yes + -o ControlMaster=auto + -o "ControlPath=/tmp/openprot-fpga-ssh-%u-%C" + -o ControlPersist=60 +) + +: "${VCK190_FPGA_HOST:?is not set (pass --test_env=VCK190_FPGA_HOST=<host>)}" +: "${TEST_SRCDIR:?is not set; run_remote must be used via bazel test --run_under}" +: "${TEST_BINARY:?is not set; run_remote must be used via bazel test --run_under}" +[[ $# -ge 1 ]] || { + echo "usage: run_remote <test-binary> [args...]" >&2 + exit 2 +} +shift # The test binary; $TEST_BINARY is the same path, relative to _main. + +# 1. Copy the runfiles tree, resolving Bazel's symlinks. Bazel's MANIFEST maps +# to absolute host paths and would put the `runfiles` crate into manifest mode +# on the board, so skip it. +rsync -a --copy-links --exclude=/MANIFEST \ + -e "ssh ${SSH_OPTS[*]}" \ + --rsync-path="mkdir -p $REMOTE_RUNFILES && rsync" \ + "$TEST_SRCDIR/" "$VCK190_FPGA_HOST:$REMOTE_RUNFILES/" || { + echo "error: failed to copy runfiles to $VCK190_FPGA_HOST" >&2 + exit 2 +} + +# 2. Run on the board. `sudo` drops the environment, so forward it explicitly. +env_vars=() +for var in "${FORWARDED_ENV[@]}"; do + if [[ -n "${!var:-}" ]]; then + env_vars+=("$var=${!var}") + fi +done +remote_cmd=( + sudo env "${env_vars[@]}" + "$REMOTE_RUNFILES/_main/target/veer/tooling/fpga/run_on_board.sh" + "$REMOTE_RUNFILES" "./$TEST_BINARY" "$@" +) + +rc=0 +ssh -tt "${SSH_OPTS[@]}" -o LogLevel=ERROR "$VCK190_FPGA_HOST" \ + "$(printf '%q ' "${remote_cmd[@]}")" | tr -d '\r' || rc=$? + +if ((rc == 255)); then + echo "error: SSH connection to $VCK190_FPGA_HOST failed" >&2 + exit 2 +fi +exit "$rc"
diff --git a/third_party/caliptra/uprev.py b/third_party/caliptra/uprev.py index 0030bd3..4674de8 100644 --- a/third_party/caliptra/uprev.py +++ b/third_party/caliptra/uprev.py
@@ -672,6 +672,10 @@ print("") print(f" caliptra_dpe : {dpe_sha} (Cargo.toml, unchanged)") print(" ureg : check git_override in MODULE.bazel (unchanged)") + print( + " bitstream : check @caliptra_fpga_bitstream in MODULE.bazel" + " against hw/fpga/bitstream_manifests/subsystem.toml" + ) print("") print("------------------------------------------------------------") print(" Next steps:")