orchestrator: Replace BootMonitor with checkpoint-embedded evidence checks A BootCheckpoint is timing policy plus its own evidence check: a capture-less fn handed the board's device context, so the channel underneath never leaks past the check and an unobservable checkpoint is unrepresentable. config.rs defines the schema (BootSignal is gone); the board table declares the checkpoints against its own context and error types. BootStatus stays as the shared vocabulary and absorbs the latch-cleared-by-reset contract; GpioBootMonitor keeps its behavior as a plain reader. BootWatch/WalkVerdict is the erased seam the orchestrator polls — timeout and retry-budget judgment lands with the walker that implements it. Assisted-by: Claude:claude-fable-5 Signed-off-by: Christina Quast <christina.quast@9elements.com>
diff --git a/services/orchestrator/capabilities/BUILD.bazel b/services/orchestrator/capabilities/BUILD.bazel index 7b873d2..3ff2734 100644 --- a/services/orchestrator/capabilities/BUILD.bazel +++ b/services/orchestrator/capabilities/BUILD.bazel
@@ -7,7 +7,8 @@ name = "orchestrator_capabilities", srcs = [ "src/boot_control.rs", - "src/boot_monitor.rs", + "src/boot_status.rs", + "src/boot_watch.rs", "src/lib.rs", ], edition = "2024",
diff --git a/services/orchestrator/capabilities/src/boot_control.rs b/services/orchestrator/capabilities/src/boot_control.rs index ae13cd5..83f46ae 100644 --- a/services/orchestrator/capabilities/src/boot_control.rs +++ b/services/orchestrator/capabilities/src/boot_control.rs
@@ -33,7 +33,7 @@ /// dev.hold_in_reset()?; /// store.set_trial(new_slot)?; // tentative boot selection — not yet committed /// dev.release()?; // boot the trial image -/// match monitor.await_boot(window)? { +/// match supervise_boot(window)? { /// Booted => store.commit(new_slot)?, // observed good => make it active /// Failed | Timeout => { /* nothing committed; previous slot still active */ } /// }
diff --git a/services/orchestrator/capabilities/src/boot_monitor.rs b/services/orchestrator/capabilities/src/boot_monitor.rs deleted file mode 100644 index 9695688..0000000 --- a/services/orchestrator/capabilities/src/boot_monitor.rs +++ /dev/null
@@ -1,180 +0,0 @@ -// Licensed under the Apache-2.0 license -// SPDX-License-Identifier: Apache-2.0 - -//! Observation capability: read a managed device's boot liveness. - -/// Liveness of a managed device's boot: Boot Confirmation only. -/// -/// Reports only that a device came up, never what booted; confirming the -/// running image is the one the RoT staged is attestation, a separate step. -/// `Failed` is optional device-reported evidence and never the only failure -/// path, since a hung device reports nothing — a stuck boot is caught by the -/// orchestrator's timeout, not by this enum. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum BootStatus { - /// Released, but boot completion not yet observed. - Booting, - /// Boot completion observed. - Booted, - /// Device reported a boot failure. - Failed, -} - -/// Observation capability: read a managed device's boot liveness. -/// -/// Pull-shaped: where the underlying signal is an edge or pulse, the interrupt -/// latches a flag beneath this seam and `boot_status` only reads it. No -/// callback registration, which would require allocation and invert control -/// into device implementations. -/// -/// The reported status must describe the **current** boot cycle. An -/// implementation backed by a latched signal must guarantee the latch is -/// cleared whenever the device re-enters reset, so evidence left over from a -/// previous boot never reads as [`BootStatus::Booted`]. This trait -/// deliberately has no re-arm operation: clearing is the reset path's job -/// (hardware tying the latch to the device's reset line, or the same platform -/// code that drives `BootControl`), not the observer's — a monitor that could -/// clear its own evidence would let a read race a reset. -pub trait BootMonitor { - /// The error type reported by this device's boot monitor. - /// - /// Requires [`core::error::Error`] (in `core` since Rust 1.81) so the - /// orchestrator gets `Display` and a `source()` cause chain, not just a - /// `Debug` dump. Error categories stay implementation-defined — this - /// crate names no error vocabulary of its own; a consumer that knows the - /// concrete adapter can recover its details by downcasting the - /// `&dyn core::error::Error`. - type Error: core::error::Error; - - /// Returns the current liveness of the device. - /// - /// Any given monitor may only ever produce a *subset* of [`BootStatus`], - /// depending on the signals it can access: a single ready pin yields only - /// `Booting`/`Booted`, while a fault-channel backend can also report - /// `Failed`. This is a capability difference between backends, not an - /// incomplete implementation. Consumers must still handle the full set — - /// they cannot know statically which backend they hold. - /// - /// # Errors - /// - /// Returns an error if the underlying liveness signal cannot be read. - fn boot_status(&self) -> Result<BootStatus, Self::Error>; -} - -#[cfg(test)] -#[allow(clippy::bool_assert_comparison)] -mod tests { - use super::*; - use core::cell::Cell; - - // ── Trait contract ────────────────────────────────────────────────── - // MockMonitor implements the trait without any HAL dependency. If a - // HAL-specific bound sneaks back onto `Error`, this module stops - // compiling. - - struct MockMonitor { - ready_after: usize, - polls: Cell<usize>, - fail: bool, - } - - #[derive(Debug, PartialEq, Eq)] - struct MockFault; - - impl core::fmt::Display for MockFault { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - f.write_str("mock monitor fault") - } - } - - impl core::error::Error for MockFault {} - - impl BootMonitor for MockMonitor { - type Error = MockFault; - - fn boot_status(&self) -> Result<BootStatus, MockFault> { - if self.fail { - return Err(MockFault); - } - let polls = self.polls.get(); - self.polls.set(polls + 1); - Ok(if polls >= self.ready_after { - BootStatus::Booted - } else { - BootStatus::Booting - }) - } - } - - // A device that is still coming up reads Booting, then Booted once it - // is up. - #[test] - fn status_progresses_from_booting_to_booted() { - let mon = MockMonitor { - ready_after: 1, - polls: Cell::new(0), - fail: false, - }; - - assert_eq!( - mon.boot_status().expect("boot_status failed"), - BootStatus::Booting - ); - assert_eq!( - mon.boot_status().expect("boot_status failed"), - BootStatus::Booted - ); - } - - #[test] - fn errors_surface_through_the_generic_seam() { - let mon = MockMonitor { - ready_after: 0, - polls: Cell::new(0), - fail: true, - }; - - let err = comes_up_within(&mon, 1).expect_err("expected the monitor fault"); - - // Display comes from the core::error::Error bound, not a Debug dump. - assert_eq!(err.to_string(), "mock monitor fault"); - } - - // ── The orchestrator's future shape ───────────────────────────────── - // Usage examples for the future orchestrator, not API guarantees; move - // these to the orchestrator crate once it exists. - - /// Poll a monitor up to `poll_budget` times. `Booting` is not a failure; - /// `Ok(false)` means the budget ran out before the device came up. - fn comes_up_within<M: BootMonitor>(mon: &M, poll_budget: usize) -> Result<bool, M::Error> { - for _ in 0..poll_budget { - if mon.boot_status()? == BootStatus::Booted { - return Ok(true); - } - } - Ok(false) - } - - // A device that comes up within the poll budget reads Booted. - #[test] - fn a_device_that_comes_up_within_budget_is_booted() { - let mon = MockMonitor { - ready_after: 2, - polls: Cell::new(0), - fail: false, - }; - - assert_eq!(comes_up_within(&mon, 5).expect("boot_status failed"), true); - } - - #[test] - fn a_device_that_never_comes_up_is_a_timeout_not_an_error() { - let mon = MockMonitor { - ready_after: usize::MAX, - polls: Cell::new(0), - fail: false, - }; - - assert_eq!(comes_up_within(&mon, 3).expect("boot_status failed"), false); - } -}
diff --git a/services/orchestrator/capabilities/src/boot_status.rs b/services/orchestrator/capabilities/src/boot_status.rs new file mode 100644 index 0000000..a6f84f9 --- /dev/null +++ b/services/orchestrator/capabilities/src/boot_status.rs
@@ -0,0 +1,36 @@ +// Licensed under the Apache-2.0 license +// SPDX-License-Identifier: Apache-2.0 + +//! Shared vocabulary for boot-liveness evidence. + +/// Liveness of a managed device's boot: Boot Confirmation only. +/// +/// Reports only that a device came up, never what booted; confirming the +/// running image is the one the RoT staged is attestation, a separate step. +/// `Failed` is optional device-reported evidence and never the only failure +/// path, since a hung device reports nothing — a stuck boot is caught by the +/// orchestrator's timeout, not by this enum. +/// +/// Any given evidence source may only ever produce a *subset* of these +/// statuses: a single ready pin yields only `Booting`/`Booted`, while a +/// fault-channel backend can also report `Failed`. That is a capability +/// difference between sources, not an incomplete implementation — consumers +/// must handle the full set. +/// +/// A status must describe the **current** boot cycle. Where the underlying +/// signal is an edge or pulse, it is latched beneath the read, and the latch +/// must be cleared whenever the device re-enters reset — by hardware tying +/// the latch to the device's reset line, or by the platform code that drives +/// `BootControl` — so evidence left over from a previous boot never reads as +/// [`Booted`](BootStatus::Booted). Clearing is deliberately the reset path's +/// job, not the reader's: a reader that could clear its own evidence would +/// let a read race a reset. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BootStatus { + /// Released, but boot completion not yet observed. + Booting, + /// Boot completion observed. + Booted, + /// Device reported a boot failure. + Failed, +}
diff --git a/services/orchestrator/capabilities/src/boot_watch.rs b/services/orchestrator/capabilities/src/boot_watch.rs new file mode 100644 index 0000000..74b3213 --- /dev/null +++ b/services/orchestrator/capabilities/src/boot_watch.rs
@@ -0,0 +1,127 @@ +// Licensed under the Apache-2.0 license +// SPDX-License-Identifier: Apache-2.0 + +//! The orchestrator-facing seam of boot supervision. + +/// One device's boot walk, pollable without knowing the device type. +/// +/// Everything device-specific — the driver type, its error type, the +/// checkpoint list — stays inside the concrete walk; the orchestrator's +/// fleet view is uniform. Object-safe so a heterogeneous fleet can sit +/// behind `&mut dyn BootWatch`; a board preferring static dispatch wraps +/// its walks in an enum and matches, without touching anything below the +/// seam. +pub trait BootWatch { + /// Judges the walk at `now_millis` (monotonic). Never sleeps — time is + /// injected, so every decision is host-testable. + fn poll(&mut self, now_millis: u64) -> WalkVerdict; +} + +/// Everything the orchestrator needs to know about a boot walk. +/// +/// Deliberately free of device and error types: the orchestrator acts the +/// same whatever the cause, so the concrete detail is logged by the walk +/// while it is still in scope, not carried across the seam. +/// +/// Intentionally exhaustive (not `#[non_exhaustive]`): adding a verdict is +/// a breaking change, so the compiler forces every consumer — in particular +/// the orchestrator's event mapping — to handle it explicitly. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WalkVerdict { + /// Nothing to decide yet; poll again by `deadline_millis`. + Waiting { + /// When the awaited checkpoint's window expires. + deadline_millis: u64, + }, + /// Every checkpoint passed — the device is up. + Complete, + /// A window expired or the device reported failure, with retry budget + /// left; the window is re-armed. The caller re-resets the device and + /// keeps polling — what a retry re-runs is the caller's policy. + Retry { + /// The checkpoint that failed. + checkpoint: &'static str, + /// Attempts left after this one. + retries_left: u8, + }, + /// Retry budget exhausted — this boot is dead. Recovery is the + /// caller's move. + Dead { + /// The checkpoint the boot died at. + checkpoint: &'static str, + }, +} + +#[cfg(test)] +mod tests { + use super::*; + + // A BootWatch implemented against no walker at all — the seam must be + // satisfiable by anything that can produce verdicts, and must stay + // object-safe (the fleet array below fails to compile otherwise). + + struct ScriptedWalk { + verdicts: &'static [WalkVerdict], + next: usize, + } + + impl BootWatch for ScriptedWalk { + fn poll(&mut self, _now_millis: u64) -> WalkVerdict { + let v = self.verdicts[self.next]; + self.next += 1; + v + } + } + + #[test] + fn a_heterogeneous_fleet_pumps_through_the_erased_seam() { + let mut bmc = ScriptedWalk { + verdicts: &[ + WalkVerdict::Waiting { + deadline_millis: 90_000, + }, + WalkVerdict::Complete, + ], + next: 0, + }; + let mut nic = ScriptedWalk { + verdicts: &[ + WalkVerdict::Retry { + checkpoint: "heartbeat", + retries_left: 1, + }, + WalkVerdict::Dead { + checkpoint: "heartbeat", + }, + ], + next: 0, + }; + + let fleet: &mut [&mut dyn BootWatch] = &mut [&mut bmc, &mut nic]; + + let first: [WalkVerdict; 2] = [fleet[0].poll(0), fleet[1].poll(0)]; + let second: [WalkVerdict; 2] = [fleet[0].poll(1), fleet[1].poll(1)]; + + assert_eq!( + first, + [ + WalkVerdict::Waiting { + deadline_millis: 90_000 + }, + WalkVerdict::Retry { + checkpoint: "heartbeat", + retries_left: 1 + }, + ] + ); + assert_eq!( + second, + [ + WalkVerdict::Complete, + WalkVerdict::Dead { + checkpoint: "heartbeat" + }, + ] + ); + } +}
diff --git a/services/orchestrator/capabilities/src/lib.rs b/services/orchestrator/capabilities/src/lib.rs index f5e7b3e..8b974e8 100644 --- a/services/orchestrator/capabilities/src/lib.rs +++ b/services/orchestrator/capabilities/src/lib.rs
@@ -7,23 +7,29 @@ //! single managed device's reset without knowing which controller line it //! maps to. //! -//! `BootMonitor` is the observation capability: the orchestrator reads a -//! device's boot liveness. +//! `BootStatus` is the shared vocabulary for boot-liveness evidence. There +//! is deliberately no observation *trait*: each `BootCheckpoint` a board +//! table declares (`DeviceConfig::checkpoints` in `orchestrator-config`) +//! carries its own evidence check, so how a signal is read stays inside the +//! check. +//! +//! `BootWatch` is the seam the orchestrator polls: one device's boot walk, +//! erased of every device-specific type, answering with a `WalkVerdict`. //! //! This crate is a dependency-free leaf: it holds the capability contracts, -//! and everything depends downward on it. Concrete adapters bind a trait to a -//! signal source and live in their own crates, so naming a capability never -//! drags in the stack behind it — the HAL-backed `HalBootControl` and -//! `GpioBootMonitor` are in `orchestrator-hal-adapters`; other backends (for -//! example an MCTP-ready `BootMonitor`) implement the same traits from their -//! own transport crate. The per-board device table schema lives in the -//! separate `orchestrator-config` crate; board tables -//! (`target/<board>/devices.rs`) declare the values. +//! and everything depends downward on it. Concrete adapters bind a capability +//! to a signal source and live in their own crates, so naming a capability +//! never drags in the stack behind it — the HAL-backed `HalBootControl` and +//! the `GpioBootMonitor` read helper are in `orchestrator-hal-adapters`. The +//! per-board device table schema lives in the separate `orchestrator-config` +//! crate; board tables (`target/<board>/devices.rs`) declare the values. #![cfg_attr(not(test), no_std)] mod boot_control; -mod boot_monitor; +mod boot_status; +mod boot_watch; pub use boot_control::BootControl; -pub use boot_monitor::{BootMonitor, BootStatus}; +pub use boot_status::BootStatus; +pub use boot_watch::{BootWatch, WalkVerdict};
diff --git a/services/orchestrator/config/BUILD.bazel b/services/orchestrator/config/BUILD.bazel index 55ad6b8..5093408 100644 --- a/services/orchestrator/config/BUILD.bazel +++ b/services/orchestrator/config/BUILD.bazel
@@ -8,6 +8,7 @@ srcs = ["src/lib.rs"], edition = "2024", visibility = ["//visibility:public"], + deps = ["//services/orchestrator/capabilities:orchestrator_capabilities"], ) # Host tests: build on the host platform, no kernel/QEMU.
diff --git a/services/orchestrator/config/src/lib.rs b/services/orchestrator/config/src/lib.rs index 34aae93..ca6d320 100644 --- a/services/orchestrator/config/src/lib.rs +++ b/services/orchestrator/config/src/lib.rs
@@ -7,6 +7,8 @@ #![cfg_attr(not(test), no_std)] +use orchestrator_capabilities::BootStatus; + /// What the orchestrator requires before it commits a staged image. /// /// Intentionally exhaustive (not `#[non_exhaustive]`): adding a variant is @@ -21,65 +23,103 @@ LivenessAndAttestation, } -/// How the orchestrator observes a device's boot-progress signal. +/// One boot checkpoint: timing policy plus the evidence check itself. /// -/// Generic over the id type `G` the board's boot monitor uses to read a -/// boot-complete line, for the same reason `DeviceConfig` is generic over -/// its reset signal: signal ids are board-specific. +/// The check is handed the board's device context `D`, so the channel +/// underneath it (a GPIO line, a progress register, a message path) stays +/// inside the check and a checkpoint nothing can observe is +/// unrepresentable. /// -/// Intentionally exhaustive (not `#[non_exhaustive]`): adding a signal -/// kind is a breaking change, so every consumer that dispatches on it is -/// forced to handle the new kind explicitly. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum BootSignal<G> { - /// The device raises a boot-complete GPIO line. - GpioBootComplete(G), - /// The device sends a heartbeat message. - Heartbeat, - /// The device's MCTP endpoint answers as ready. - MctpReady, - /// The device answers a firmware version query. - VersionQuery, +/// `passed` is a capture-less `fn` pointer rather than a closure: a table +/// of closures each capturing `&mut D` cannot exist, while the walker +/// holding the one `&mut D` and passing it in can — and capture-less +/// closures coerce to `fn` in const tables. The division of state: +/// per-checkpoint parameters belong in the `fn` body, per-device and +/// per-board state belongs in `D`. +pub struct BootCheckpoint<D: ?Sized, E> { + /// Names the checkpoint in failure reports ("bl1", "kernel", …). + pub name: &'static str, + /// Window for one attempt at this checkpoint. Expiry is the + /// orchestrator's own judgment; hung devices report nothing. + pub timeout: core::time::Duration, + /// Attempts allowed beyond the first before the failure is final. + pub max_retries: u8, + /// The evidence check. The status must describe the current boot + /// cycle — see [`BootStatus`] for the latching contract. + pub passed: fn(&mut D) -> Result<BootStatus, E>, } -/// One boot-progress checkpoint: a signal the orchestrator waits for, and -/// how long it waits. -#[derive(Debug, Clone, Copy)] -pub struct BootCheckpoint<G> { - /// Names the checkpoint in timeout reports. - pub name: &'static str, - pub signal: BootSignal<G>, - /// How long the orchestrator waits for `signal` before it declares the - /// checkpoint — and the device's boot — failed. Expiry is the - /// orchestrator's own judgment; hung devices report nothing. - pub window: core::time::Duration, +// Manual impls: deriving would demand `D: Clone`/`D: Debug` bounds the +// fields never need (`D` only appears behind the `fn` pointer). +impl<D: ?Sized, E> Clone for BootCheckpoint<D, E> { + fn clone(&self) -> Self { + *self + } +} + +impl<D: ?Sized, E> Copy for BootCheckpoint<D, E> {} + +impl<D: ?Sized, E> core::fmt::Debug for BootCheckpoint<D, E> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("BootCheckpoint") + .field("name", &self.name) + .field("timeout", &self.timeout) + .field("max_retries", &self.max_retries) + .finish_non_exhaustive() + } } /// One managed downstream device, as declared by the board config. /// -/// Generic over the board's reset signal type `R`, which must match the +/// Generic over the board's reset signal type `R` (which must match the /// `ResetId` of the reset controller behind the board's `BootControl` -/// implementation — the compiler rejects a table whose ids the controller -/// cannot accept. +/// implementation), the board's device context `D` every evidence check +/// receives, and the board-wide check error `E` — one context and one +/// error type per table, both board-defined. /// /// Intentionally exhaustive (not `#[non_exhaustive]`): board tables /// construct this struct by literal, which the attribute would forbid. /// Adding a field is a breaking change that updates every board table. -#[derive(Debug, Clone, Copy)] -pub struct DeviceConfig<R, G: 'static> { +pub struct DeviceConfig<R, D: ?Sized + 'static, E: 'static> { pub name: &'static str, /// Reset signal id, passed to HalBootControl::new. pub reset_signal: R, - /// Boot-progress checkpoints, in the order the device passes them. - /// The device counts as booted when the last one is reached; a - /// checkpoint whose window expires fails the boot. - pub checkpoints: &'static [BootCheckpoint<G>], + /// Boot checkpoints, in the order the device passes them. The device + /// counts as booted when the last one is reached; a checkpoint whose + /// window and retry budget are exhausted fails the boot. + pub checkpoints: &'static [BootCheckpoint<D, E>], pub commit_policy: CommitPolicy, } +// Manual impls for the same reason as BootCheckpoint's: only `R` is held +// by value, so only `R` gets a bound. +impl<R: Clone, D: ?Sized, E> Clone for DeviceConfig<R, D, E> { + fn clone(&self) -> Self { + Self { + name: self.name, + reset_signal: self.reset_signal.clone(), + checkpoints: self.checkpoints, + commit_policy: self.commit_policy, + } + } +} + +impl<R: Copy, D: ?Sized, E> Copy for DeviceConfig<R, D, E> {} + +impl<R: core::fmt::Debug, D: ?Sized, E> core::fmt::Debug for DeviceConfig<R, D, E> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("DeviceConfig") + .field("name", &self.name) + .field("reset_signal", &self.reset_signal) + .field("checkpoints", &self.checkpoints) + .field("commit_policy", &self.commit_policy) + .finish() + } +} + /// Checks a device table. Board configs call this in a const context so a /// bad table fails the build. -pub const fn validate<R, G>(devices: &[DeviceConfig<R, G>]) { +pub const fn validate<R, D: ?Sized, E>(devices: &[DeviceConfig<R, D, E>]) { let mut i = 0; while i < devices.len() { assert!(!devices[i].name.is_empty(), "device name must not be empty"); @@ -94,8 +134,8 @@ "checkpoint name must not be empty" ); assert!( - !devices[i].checkpoints[c].window.is_zero(), - "checkpoint window must not be zero" + !devices[i].checkpoints[c].timeout.is_zero(), + "checkpoint timeout must not be zero" ); c += 1; } @@ -112,17 +152,77 @@ // build error nobody can assert on. These tests call it at runtime to // prove the reject paths actually fire — a vacuous loop would pass // every `const _` check silently. + // + // The fixture is a staged-boot device: one monotonic progress register + // serves four checkpoints through one reader, and a poison value fails + // every one — the pattern a real SoC table is expected to use. - const CHECKPOINT: BootCheckpoint<u8> = BootCheckpoint { - name: "boot-complete", - signal: BootSignal::GpioBootComplete(0), - window: Duration::from_secs(1), + const POISON: u8 = 0xFF; + + struct SocBoard { + level: u8, + fail: bool, + } + + #[derive(Debug, PartialEq, Eq)] + struct RegFault; + + impl core::fmt::Display for RegFault { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str("progress register unreadable") + } + } + + impl core::error::Error for RegFault {} + + impl SocBoard { + fn progress_at_least(&mut self, level: u8) -> Result<BootStatus, RegFault> { + if self.fail { + return Err(RegFault); + } + Ok(match self.level { + POISON => BootStatus::Failed, + l if l >= level => BootStatus::Booted, + _ => BootStatus::Booting, + }) + } + } + + // Named so the reject-path fixtures below can `..BL1` — an indexed + // `CHECKPOINTS[0]` would not promote to 'static. + const BL1: BootCheckpoint<SocBoard, RegFault> = BootCheckpoint { + name: "bl1", + timeout: Duration::from_millis(200), + max_retries: 0, + passed: |soc| soc.progress_at_least(1), }; - const DEVICE: DeviceConfig<u8, u8> = DeviceConfig { - name: "dev", + const CHECKPOINTS: &[BootCheckpoint<SocBoard, RegFault>] = &[ + BL1, + BootCheckpoint { + name: "bl2", + timeout: Duration::from_secs(1), + max_retries: 0, + passed: |soc| soc.progress_at_least(2), + }, + BootCheckpoint { + name: "kernel", + timeout: Duration::from_secs(10), + max_retries: 2, + passed: |soc| soc.progress_at_least(3), + }, + BootCheckpoint { + name: "service", + timeout: Duration::from_secs(30), + max_retries: 2, + passed: |soc| soc.progress_at_least(4), + }, + ]; + + const DEVICE: DeviceConfig<u8, SocBoard, RegFault> = DeviceConfig { + name: "soc", reset_signal: 0, - checkpoints: &[CHECKPOINT], + checkpoints: CHECKPOINTS, commit_policy: CommitPolicy::Liveness, }; @@ -150,26 +250,68 @@ #[should_panic(expected = "checkpoint name must not be empty")] fn rejects_an_empty_checkpoint_name() { validate(&[DeviceConfig { - checkpoints: &[BootCheckpoint { - name: "", - ..CHECKPOINT - }], + checkpoints: &[BootCheckpoint { name: "", ..BL1 }], ..DEVICE }]); } #[test] - #[should_panic(expected = "checkpoint window must not be zero")] - fn rejects_a_zero_checkpoint_window() { + #[should_panic(expected = "checkpoint timeout must not be zero")] + fn rejects_a_zero_checkpoint_timeout() { validate(&[DeviceConfig { - checkpoints: &[ - CHECKPOINT, - BootCheckpoint { - window: Duration::ZERO, - ..CHECKPOINT - }, - ], + checkpoints: &[BootCheckpoint { + timeout: Duration::ZERO, + ..BL1 + }], ..DEVICE }]); } + + // One register, four checkpoints: each check sees exactly its own + // threshold, so a device mid-boot passes the early ones and not the + // late ones. + #[test] + fn checks_resolve_through_the_board_context() { + let mut soc = SocBoard { + level: 2, + fail: false, + }; + let read = + |soc: &mut SocBoard, i: usize| (CHECKPOINTS[i].passed)(soc).expect("check failed"); + + assert_eq!(read(&mut soc, 0), BootStatus::Booted); // bl1 + assert_eq!(read(&mut soc, 1), BootStatus::Booted); // bl2 + assert_eq!(read(&mut soc, 2), BootStatus::Booting); // kernel + assert_eq!(read(&mut soc, 3), BootStatus::Booting); // service + } + + // A poisoned register must read Failed from every checkpoint, whichever + // one the walk happens to be awaiting. + #[test] + fn a_poisoned_register_fails_every_checkpoint() { + let mut soc = SocBoard { + level: POISON, + fail: false, + }; + + for cp in CHECKPOINTS { + assert_eq!( + (cp.passed)(&mut soc).expect("check failed"), + BootStatus::Failed + ); + } + } + + #[test] + fn errors_surface_through_the_check() { + let mut soc = SocBoard { + level: 0, + fail: true, + }; + + let err = (CHECKPOINTS[0].passed)(&mut soc).expect_err("expected the register fault"); + + // Display comes from the core::error::Error bound, not a Debug dump. + assert_eq!(err.to_string(), "progress register unreadable"); + } }
diff --git a/services/orchestrator/hal-adapters/src/gpio_boot_monitor.rs b/services/orchestrator/hal-adapters/src/gpio_boot_monitor.rs index a90cdcb..3591d2b 100644 --- a/services/orchestrator/hal-adapters/src/gpio_boot_monitor.rs +++ b/services/orchestrator/hal-adapters/src/gpio_boot_monitor.rs
@@ -1,19 +1,20 @@ // Licensed under the Apache-2.0 license // SPDX-License-Identifier: Apache-2.0 -//! HAL-backed [`BootMonitor`]: read a device's boot-complete signal off a GPIO -//! input line. +//! HAL-backed boot-status reader: read a device's boot-complete signal off a +//! GPIO input line into a [`BootStatus`]. use openprot_hal_blocking::gpio_port::{ ActivePolarity, GpioError, GpioErrorKind, GpioPort, PinMask, }; -use orchestrator_capabilities::{BootMonitor, BootStatus}; +use orchestrator_capabilities::BootStatus; /// Adapts any HAL GPIO error into a [`core::error::Error`]. /// /// GPIO ports keep implementing the HAL `GpioError`/`kind()` pattern /// unchanged; this wrapper supplies the `Display` and `core::error::Error` -/// machinery [`BootMonitor::Error`] requires, so no per-implementation work is +/// machinery the orchestrator expects of boot-evidence errors, so no +/// per-implementation work is /// needed. The underlying category stays reachable via [`MonitorError::kind`], /// and the concrete HAL error through the /// [`source()`](core::error::Error::source) chain, downcast to @@ -79,15 +80,15 @@ /// ready signals routinely share one). Platform configuration keeps the bank /// alive for as long as its monitors. /// -/// A single ready line can only ever answer "up yet?", so this backend +/// A single ready line can only ever answer "up yet?", so this reader /// reports the [`BootStatus::Booting`]/[`BootStatus::Booted`] subset — see -/// [`BootMonitor::boot_status`] on why that is a capability difference, not -/// an incomplete implementation. +/// [`BootStatus`] on why that is a capability difference, not an incomplete +/// implementation. /// /// Where a hardware latch is used, the platform must clear it whenever the /// device re-enters reset (typically by wiring the latch's clear to the -/// device's reset line) — [`BootMonitor`] requires that evidence from a -/// previous boot never reads as [`BootStatus::Booted`], and this adapter only +/// device's reset line) — [`BootStatus`] requires that evidence from a +/// previous boot never reads as [`BootStatus::Booted`], and this reader only /// reads the line, it cannot re-arm it. /// /// [`HalBootControl`]: crate::HalBootControl @@ -128,16 +129,16 @@ // `P::Error: 'static` because `source()` hands out `&(dyn Error + 'static)` // referencing the wrapped HAL error. Error types are plain data; this costs // no real implementation anything. -impl<P: GpioPort> BootMonitor for GpioBootMonitor<'_, P> +impl<P: GpioPort> GpioBootMonitor<'_, P> where P::Error: 'static, { - type Error = MonitorError<P::Error>; - + /// Returns the current liveness of the device. + /// /// # Errors /// /// Propagates any error returned by the port's `read_input`. - fn boot_status(&self) -> Result<BootStatus, Self::Error> { + pub fn boot_status(&self) -> Result<BootStatus, MonitorError<P::Error>> { let high = self.port.read_input()?.contains(self.ready_pin); let booted = match self.active { ActivePolarity::ActiveHigh => high, @@ -156,7 +157,8 @@ use super::*; use openprot_hal_blocking::gpio_port::GpioErrorType; - // BMC boot-complete on line 4. Normally set in config.rs. + // BMC boot-complete on line 4. Everything that is config is normally + // declared in the board device table (`target/<board>/devices.rs`). const BMC_READY: Mask = Mask(1 << 4); /// Bitmask over a single mock GPIO bank. @@ -238,15 +240,15 @@ } fn configure(&mut self, _: Mask, _: ()) -> Result<(), MockError> { - panic!("BootMonitor must never configure pins"); + panic!("the boot-status reader must never configure pins"); } fn set_reset(&mut self, _: Mask, _: Mask) -> Result<(), MockError> { - panic!("BootMonitor must never drive outputs"); + panic!("the boot-status reader must never drive outputs"); } fn toggle(&mut self, _: Mask) -> Result<(), MockError> { - panic!("BootMonitor must never drive outputs"); + panic!("the boot-status reader must never drive outputs"); } } @@ -297,9 +299,9 @@ GpioBootMonitor::new(&port, Mask::empty(), ActivePolarity::ActiveHigh); } - // A controller error surfaces through BootMonitor unchanged. + // A controller error surfaces through the reader unchanged. #[test] - fn port_error_propagates_through_boot_monitor() { + fn port_error_propagates_through_the_reader() { let port = MockGpioPort::failing(GpioErrorKind::HardwareFailure); let mon = GpioBootMonitor::new(&port, BMC_READY, ActivePolarity::ActiveHigh);
diff --git a/services/orchestrator/hal-adapters/src/hal_boot_control.rs b/services/orchestrator/hal-adapters/src/hal_boot_control.rs index f7eb93d..2f351a5 100644 --- a/services/orchestrator/hal-adapters/src/hal_boot_control.rs +++ b/services/orchestrator/hal-adapters/src/hal_boot_control.rs
@@ -74,7 +74,8 @@ use core::time::Duration; use openprot_hal_blocking::system_control::{Error as HalError, ErrorKind, ErrorType}; - // Normally set in config.rs + // Everything that is config is normally declared in the board device + // table (`target/<board>/devices.rs`). const BMC_LINE: u8 = 7; #[derive(Debug, PartialEq, Eq, Clone, Copy)]
diff --git a/services/orchestrator/hal-adapters/src/lib.rs b/services/orchestrator/hal-adapters/src/lib.rs index 4f71ebd..3c9d0c5 100644 --- a/services/orchestrator/hal-adapters/src/lib.rs +++ b/services/orchestrator/hal-adapters/src/lib.rs
@@ -3,10 +3,10 @@ //! HAL-backed adapters for the Boot Orchestrator capability traits. //! -//! Each type here implements a capability trait from `orchestrator-capabilities` -//! against a HAL-blocking trait: [`HalBootControl`] drives `BootControl` over a -//! `ResetControl` line, and [`GpioBootMonitor`] reads `BootMonitor` off a -//! `GpioPort` input line. Adapters live in this crate — not in the leaf +//! Each type here binds an orchestrator-facing seam to a HAL-blocking trait: +//! [`HalBootControl`] drives `BootControl` over a `ResetControl` line, and +//! [`GpioBootMonitor`] reads a `GpioPort` input line into a `BootStatus`. +//! Adapters live in this crate — not in the leaf //! `orchestrator-capabilities` — so that depending on a capability contract //! never pulls in the HAL. A transport-backed adapter belongs in its own crate //! depending on its own stack, by the same rule.
diff --git a/target/mock/BUILD.bazel b/target/mock/BUILD.bazel index a4dbf97..751ebde 100644 --- a/target/mock/BUILD.bazel +++ b/target/mock/BUILD.bazel
@@ -10,5 +10,8 @@ srcs = ["devices.rs"], crate_name = "board_devices", edition = "2024", - deps = ["//services/orchestrator/config:orchestrator_config"], + deps = [ + "//services/orchestrator/capabilities:orchestrator_capabilities", + "//services/orchestrator/config:orchestrator_config", + ], )
diff --git a/target/mock/devices.rs b/target/mock/devices.rs index e5c3af8..aa484c5 100644 --- a/target/mock/devices.rs +++ b/target/mock/devices.rs
@@ -7,16 +7,40 @@ #![no_std] +use core::convert::Infallible; use core::time::Duration; -use orchestrator_config::{BootCheckpoint, BootSignal, CommitPolicy, DeviceConfig}; +use orchestrator_capabilities::BootStatus; +use orchestrator_config::{BootCheckpoint, CommitPolicy, DeviceConfig}; + +/// The mock board's device context: the signal state every checkpoint +/// check reads. Stands in for real drivers until the mock platform grows +/// them; the reset path is responsible for clearing latched fields (see +/// `BootStatus`). +#[derive(Debug, Default)] +pub struct MockBoard { + /// bmc boot-complete line. + pub bmc_ready: bool, + /// nic MCTP endpoint answers as ready. + pub nic_mctp_ready: bool, + /// nic heartbeat observed (latched). + pub nic_heartbeat: bool, +} + +const fn up(ready: bool) -> BootStatus { + if ready { + BootStatus::Booted + } else { + BootStatus::Booting + } +} /// Declaration order is the boot order: the orchestrator releases devices /// top to bottom, one at a time. /// -/// The mock board's reset controller and boot monitor both address -/// signals by plain index, so both id types are `u8`. -pub const MANAGED_DEVICES: &[DeviceConfig<u8, u8>] = &[ +/// The mock board's reset controller addresses reset lines by plain index, +/// so the reset id type is `u8`. +pub const MANAGED_DEVICES: &[DeviceConfig<u8, MockBoard, Infallible>] = &[ // Direct-flash SPI device (BMC archetype): the eRoT fronts its flash. // Single checkpoint: it raises a boot-complete GPIO. DeviceConfig { @@ -24,26 +48,30 @@ reset_signal: 7, checkpoints: &[BootCheckpoint { name: "boot-complete", - signal: BootSignal::GpioBootComplete(12), - window: Duration::from_secs(90), + timeout: Duration::from_secs(90), + max_retries: 1, + passed: |b| Ok(up(b.bmc_ready)), }], commit_policy: CommitPolicy::Liveness, }, // PLDM device (NIC archetype): self-updating, SPDM-capable. Two - // checkpoints, exercising the multi-checkpoint path. + // checkpoints, exercising the multi-checkpoint path: transport up + // first, then proof the workload is alive. DeviceConfig { name: "nic", reset_signal: 3, checkpoints: &[ BootCheckpoint { name: "mctp-ready", - signal: BootSignal::MctpReady, - window: Duration::from_secs(20), + timeout: Duration::from_secs(20), + max_retries: 2, + passed: |b| Ok(up(b.nic_mctp_ready)), }, BootCheckpoint { name: "heartbeat", - signal: BootSignal::Heartbeat, - window: Duration::from_secs(10), + timeout: Duration::from_secs(10), + max_retries: 0, + passed: |b| Ok(up(b.nic_heartbeat)), }, ], commit_policy: CommitPolicy::LivenessAndAttestation,