orchestrator/sm: device-agnostic boot-progress watchdog Track boot-progress per component instead of watching only Active devices. Passive components released from reset are now watched for liveness (new Booted event) under the same watchdog as Active ComponentReady; a released component that misses its window is recovered from any state. Timeout is handled uniformly in the supervisor and PreSupervision; the AwaitingReady-only Timeout arm is removed.
diff --git a/services/orchestrator/sm/src/lib.rs b/services/orchestrator/sm/src/lib.rs index d2265cf..877e1c9 100644 --- a/services/orchestrator/sm/src/lib.rs +++ b/services/orchestrator/sm/src/lib.rs
@@ -148,6 +148,14 @@ lifecycle: ComponentLifecycle, /// Consecutive failed-restore count (INV7: consecutive only). retry: u8, + /// Set while this component has been released from reset but has not yet + /// reported its boot-progress signal ([`Event::ComponentReady`] for an + /// `Active` component, [`Event::Booted`] for a `Passive` one). The shell + /// arms a per-component watchdog on release; this bit is what a later + /// [`Event::Timeout`] consults to tell a real boot failure from a stale or + /// spurious timeout. Orthogonal to `lifecycle`: a gated component owes no + /// boot-progress signal, so gating clears it. + awaiting_boot: bool, } impl Default for ComponentStatus { @@ -155,6 +163,7 @@ Self { lifecycle: ComponentLifecycle::Nominal, retry: 0, + awaiting_boot: false, } } } @@ -236,6 +245,10 @@ ctx.emit(Effect::ReportIsolated(id)); if let Some(i) = self.status_index(id) { self.statuses[i].lifecycle = ComponentLifecycle::Isolated; + // A gated component is held in reset and no longer owes a + // boot-progress signal; drop any pending watchdog so a late + // `Timeout` can't drag an already-isolated component into recovery. + self.statuses[i].awaiting_boot = false; } true } @@ -262,6 +275,32 @@ } } + /// Record that `id` has been released and now owes a boot-progress signal. + /// Paired with the `ReleaseReset` emitted at each release site: the shell + /// arms its per-component boot watchdog there, and this arms ours. + fn mark_awaiting_boot(&mut self, id: ComponentId) { + if let Some(i) = self.status_index(id) { + self.statuses[i].awaiting_boot = true; + } + } + + /// Clear `id`'s boot-progress watchdog because it reported in + /// ([`Event::ComponentReady`] or [`Event::Booted`]). Idempotent: a report + /// for a component not awaiting boot simply changes nothing. + fn clear_awaiting_boot(&mut self, id: ComponentId) { + if let Some(i) = self.status_index(id) { + self.statuses[i].awaiting_boot = false; + } + } + + /// Whether `id` has been released and still owes a boot-progress signal. + /// A [`Event::Timeout`] is a real boot failure only for such a component; + /// otherwise it is stale/spurious. + fn is_awaiting_boot(&self, id: ComponentId) -> bool { + self.status_index(id) + .is_some_and(|i| self.statuses[i].awaiting_boot) + } + /// Advance `cursor` from `start_idx` to the first component not in /// `gated`, emitting its `ReadFirmware`/`VerifyFirmware`. Returns `true` if /// found. If the rest of the chain is exhausted or entirely gated, sets @@ -398,6 +437,8 @@ // consecutive-failure streak ends (INV7: consecutive only). self.clear_retry(*id); ctx.emit(Effect::ReleaseReset(*id)); + // Released: arm its boot-progress watchdog (both tiers). + self.mark_awaiting_boot(*id); let current_kind = self.chain.get(self.cursor as usize).map(|(_, a)| a.kind); let next_idx = (self.cursor as usize).saturating_add(1); if self.advance_to_next_ungated(ctx, next_idx) { @@ -426,6 +467,30 @@ // unhandled here (falls through to `Outcome::Super` and is // discarded) — that's a separate question. Event::CorruptionDetected(id) => self.handle_corruption(*id, ctx), + // Boot-progress liveness for a passive component released + // speculatively earlier in this same walk. Clear its watchdog + // even though `PreSupervision` is unsupervised — acting on a + // report we already have is never worse than dropping it (same + // rationale as `CorruptionDetected` above). An `Active` + // component's `ComponentReady` cannot arrive here: releasing an + // active moves the machine straight to `AwaitingReady`. + Event::Booted(id) => { + self.clear_awaiting_boot(*id); + Outcome::Handled + } + // Device-agnostic boot-progress watchdog. A passive component + // released speculatively can miss its window while the walk is + // still in `PreSupervision`; treat that as a boot failure and + // recover it, exactly as the supervised states do. A timeout for + // a component not awaiting boot (e.g. still under verification) + // is spurious and dropped. + Event::Timeout(id) => { + if self.is_awaiting_boot(*id) { + Outcome::Transition(State::Recovering(*id)) + } else { + Outcome::Handled + } + } Event::EffectFailed => Outcome::Transition(State::Locked), _ => Outcome::Super, }, @@ -436,8 +501,13 @@ // entry action — do not add one. State::AwaitingReady(awaiting) => match event { Event::ComponentReady(id) => { + // Clear this component's boot-progress watchdog whether or + // not it is the one this state's slot is tracking: a later + // `Active` released speculatively reports its readiness here + // too, and its watchdog must be cleared just the same. + self.clear_awaiting_boot(*id); if awaiting != Some(*id) { - return Outcome::Handled; // spurious / stale (INV9) + return Outcome::Handled; // not the tracked slot (INV9) } // If cursor is past the end, the eRoT side of the walk has // already finished (chain done, or the remainder is held) — @@ -455,6 +525,8 @@ // consecutive-failure streak ends (INV7: consecutive only). self.clear_retry(*id); ctx.emit(Effect::ReleaseReset(*id)); + // Released: arm its boot-progress watchdog (both tiers). + self.mark_awaiting_boot(*id); let next_idx = (self.cursor as usize).saturating_add(1); if self.advance_to_next_ungated(ctx, next_idx) { // `Handled` preserves the current payload: `awaiting` is @@ -469,18 +541,10 @@ // of the component's recovery-failure policy. Outcome::Transition(State::Recovering(*id)) } - Event::Timeout(id) => { - // The boot watchdog fired. Only the component we are - // actually waiting on matters; a timeout for any other id - // is stale or spurious and is dropped (same treatment as a - // stale `ComponentReady`, INV9). The awaited component is - // treated as a verification failure and enters recovery. - if awaiting != Some(*id) { - Outcome::Handled - } else { - Outcome::Transition(State::Recovering(*id)) - } - } + // `Timeout` is intentionally not handled here: it falls through + // to `handle_supervising`, which runs the device-agnostic + // boot-progress watchdog uniformly across every supervised state + // (an `AwaitingReady` timeout is no longer special-cased). _ => Outcome::Super, }, @@ -601,6 +665,26 @@ Outcome::Handled } Event::CorruptionDetected(id) => self.handle_corruption(*id, ctx), + // Boot-progress signals arriving after the walk left `PreSupervision` + // / `AwaitingReady` (e.g. once the machine is already `Ready`): clear + // the component's watchdog. `ComponentReady` is the active tier, + // `Booted` the passive tier; both just retire the outstanding wait. + Event::ComponentReady(id) | Event::Booted(id) => { + self.clear_awaiting_boot(*id); + Outcome::Handled + } + // Device-agnostic boot-progress watchdog across every supervised + // state: a released component that never reported in before its + // window closed is recovered like any other boot failure. A timeout + // for a component not awaiting boot (already reported, gated, or + // never released) is stale/spurious and dropped. + Event::Timeout(id) => { + if self.is_awaiting_boot(*id) { + Outcome::Transition(State::Recovering(*id)) + } else { + Outcome::Handled + } + } Event::EffectFailed => Outcome::Transition(State::Locked), _ => Outcome::Super, } @@ -620,6 +704,10 @@ ctx.emit(Effect::StageUpdate); } State::Recovering(failed) => { + // The component under recovery is being restored, not booting; + // drop any pending boot-progress watchdog so a late `Timeout` + // can't re-enter recovery for it. + self.clear_awaiting_boot(failed); ctx.emit(Effect::RestoreGoldenImage(failed)); } State::Locked => {
diff --git a/services/orchestrator/sm/src/model.rs b/services/orchestrator/sm/src/model.rs index d13891c..a937c84 100644 --- a/services/orchestrator/sm/src/model.rs +++ b/services/orchestrator/sm/src/model.rs
@@ -32,8 +32,15 @@ /// and iRoT-side (local self-verification) checks apply. The machine waits in /// [`State::AwaitingReady`] for [`Event::ComponentReady`] before advancing. Active, - /// No integrated iRoT. The eRoT's signature + SVN check is the only gate. - /// The chain walk advances immediately after `ReleaseReset`. + /// No integrated iRoT. The eRoT's signature + SVN check is the only *trust* + /// gate, so the chain walk advances speculatively after `ReleaseReset` + /// without blocking in [`State::AwaitingReady`]. The released component is + /// still watched for boot-progress liveness ([`Event::Booted`]) under the + /// same per-component watchdog as an `Active` component's + /// [`Event::ComponentReady`]: a passive device that never reports in before + /// its [`Event::Timeout`] is recovered like any other boot failure. CSA + /// boot-progress checkpointing is device-agnostic — every released device + /// owes a boot-progress signal, iRoT or not. Passive, } @@ -187,6 +194,12 @@ VerificationFailed(ComponentId), /// An `Active` component's iRoT has finished local verification and is ready. ComponentReady(ComponentId), + /// A `Passive` component reported boot-progress liveness — its firmware came + /// up. The passive-tier counterpart to [`Event::ComponentReady`]: a passive + /// component has no iRoT to self-verify, so "it booted" is the only + /// post-release signal it can produce. Clears that component's boot-progress + /// watchdog. Mirrors fwmanager's `BootProgress::Booted`. + Booted(ComponentId), /// A challenger has requested a signed attestation. AttestationChallenge, /// A firmware update has been requested. @@ -206,10 +219,14 @@ Restored(ComponentId), /// A required component's recovery was exhausted. RecoveryFailed, - /// The shell's boot-progress watchdog fired: `id` did not report readiness - /// within its configured boot timeout. Treated as a verification failure — - /// the awaited component enters recovery; a timeout for any other `id` is - /// stale/spurious and dropped. + /// The shell's boot-progress watchdog fired: `id` did not report its + /// boot-progress signal ([`Event::ComponentReady`] for an `Active` + /// component, [`Event::Booted`] for a `Passive` one) within its configured + /// boot timeout. Treated as a verification failure — a component still + /// awaiting boot-progress enters recovery. A timeout for a component that is + /// not awaiting boot (never released, already reported in, or already gated) + /// is stale/spurious and dropped. The watchdog is per component and + /// device-agnostic, matching CSA boot-progress checkpointing. Timeout(ComponentId), /// The shell could not carry out an emitted [`Effect`]; fail-closed, it /// latches to [`State::Locked`] from any state. Injected by the driver when @@ -273,7 +290,9 @@ PowerOnReset, PreSupervision, /// eRoT has released an `Active` component; the payload is the component - /// whose iRoT readiness is outstanding (INV9). + /// whose iRoT readiness is outstanding (INV9). Only the active-tier readiness + /// checkpoint lives on this payload; per-component boot-progress liveness + /// (for both tiers) is tracked in each component's status, not here. /// /// - `AwaitingReady(Some(id))` — waiting for `id`'s /// [`Event::ComponentReady`].
diff --git a/services/orchestrator/sm/src/tests.rs b/services/orchestrator/sm/src/tests.rs index f41979a..99896e0 100644 --- a/services/orchestrator/sm/src/tests.rs +++ b/services/orchestrator/sm/src/tests.rs
@@ -440,8 +440,10 @@ assert!(effects.contains(&Effect::RestoreGoldenImage(C0))); } -/// D2 (INV9): a timeout for a component we are not awaiting is stale/spurious -/// and is dropped — the machine keeps waiting on the real component. +/// D2: a timeout for a component that is not awaiting boot-progress is +/// stale/spurious and is dropped. Here `C1` has only been *verified* +/// speculatively, not released, so no boot watchdog is armed for it; the +/// machine keeps waiting on the component it actually released (`C0`). #[test] fn timeout_stale_id_ignored() { let (effects, state) = drive( @@ -452,13 +454,66 @@ &[ BOOT, Event::VerificationPassed(C0), - Event::Timeout(C1), // not the awaited id + Event::Timeout(C1), // verified but never released → not awaiting boot ], ); assert_eq!(state, State::AwaitingReady(Some(C0))); assert!(!effects.contains(&Effect::RestoreGoldenImage(C1))); } +/// Device-agnostic boot-progress: a *passive* component that is released but +/// never reports [`Event::Booted`] before its watchdog fires is recovered like +/// any other boot failure — even while the walk is still in `PreSupervision`. +/// This closes the release-and-forget gap (CSA boot-progress checkpointing is +/// device-agnostic; fwmanager arms a `boot_timeout` for every device). +#[test] +fn passive_boot_timeout_enters_recovering() { + let (effects, state) = drive( + passive_required(&[C0, C1]), + // C0 released (watchdog armed), walk speculatively verifies C1, then + // C0's boot window closes with no `Booted`. + &[BOOT, Event::VerificationPassed(C0), Event::Timeout(C0)], + ); + assert_eq!(state, State::Recovering(C0)); + assert!(effects.contains(&Effect::ReleaseReset(C0))); + assert!(effects.contains(&Effect::RestoreGoldenImage(C0))); +} + +/// A passive boot timeout is caught even after the chain walk has completed and +/// the machine has reached `Ready`: speculative release means a component can +/// still owe a boot-progress signal in `Ready`, and the supervisor runs the +/// same device-agnostic watchdog there. +#[test] +fn passive_boot_timeout_in_ready_enters_recovering() { + let (effects, state) = drive( + passive_required(&[C0]), + // Single passive: reaches `Ready` on VerificationPassed while still + // awaiting C0's boot-progress; the timeout then fires in `Ready`. + &[BOOT, Event::VerificationPassed(C0), Event::Timeout(C0)], + ); + assert_eq!(state, State::Recovering(C0)); + assert!(effects.contains(&Effect::RestoreGoldenImage(C0))); +} + +/// A passive component that reports [`Event::Booted`] retires its watchdog, so a +/// later timeout for it is stale and dropped — the machine stays `Ready`. +#[test] +fn passive_booted_clears_watchdog_then_timeout_is_stale() { + let (effects, state) = drive( + passive_required(&[C0, C1]), + &[ + BOOT, + Event::VerificationPassed(C0), + Event::Booted(C0), // C0 reports in → watchdog cleared + Event::VerificationPassed(C1), + Event::Booted(C1), + Event::Timeout(C0), // stale: C0 already booted + ], + ); + assert_eq!(state, State::Ready); + assert!(!effects.contains(&Effect::RestoreGoldenImage(C0))); +} + /// D2: full path — timeout drives recovery, restore rewalks from the top, and /// the chain then completes normally. #[test]