orchestrator-sm: add Timeout event and AwaitingReady recovery arm Boot-progress watchdog timeouts now enter recovery for the awaited component; stale/spurious timeouts are dropped (INV9). Adds three tests.
diff --git a/services/orchestrator/sm/src/lib.rs b/services/orchestrator/sm/src/lib.rs index edf80f7..7226787 100644 --- a/services/orchestrator/sm/src/lib.rs +++ b/services/orchestrator/sm/src/lib.rs
@@ -419,6 +419,18 @@ // of the component's recovery-failure policy. Outcome::Transition(State::Recovering(*id)) } + Event::Timeout(id) => { + // The boot watchdog fired. Only the component we are + // actually waiting on matters; a timeout for any other id + // is stale or spurious and is dropped (same treatment as a + // stale `ComponentReady`, INV9). The awaited component is + // treated as a verification failure and enters recovery. + if awaiting != Some(*id) { + Outcome::Handled + } else { + Outcome::Transition(State::Recovering(*id)) + } + } _ => Outcome::Super, },
diff --git a/services/orchestrator/sm/src/model.rs b/services/orchestrator/sm/src/model.rs index e71842c..0bf3d86 100644 --- a/services/orchestrator/sm/src/model.rs +++ b/services/orchestrator/sm/src/model.rs
@@ -201,6 +201,11 @@ Restored(ComponentId), /// A required component's recovery was exhausted. RecoveryFailed, + /// The shell's boot-progress watchdog fired: `id` did not report readiness + /// within its configured boot timeout. Treated as a verification failure — + /// the awaited component enters recovery; a timeout for any other `id` is + /// stale/spurious and dropped. + Timeout(ComponentId), /// The shell could not carry out an emitted [`Effect`]; fail-closed, it /// latches to [`State::Locked`] from any state. Injected by the driver when /// a [`Platform::execute`](crate::Platform::execute) call fails; never
diff --git a/services/orchestrator/sm/src/tests.rs b/services/orchestrator/sm/src/tests.rs index 61392c0..af0153f 100644 --- a/services/orchestrator/sm/src/tests.rs +++ b/services/orchestrator/sm/src/tests.rs
@@ -424,6 +424,64 @@ assert_eq!(effects.last(), Some(&Effect::SignAttestation)); } +/// D2: a boot-progress timeout for the awaited component is treated as a +/// verification failure and enters recovery. +#[test] +fn timeout_awaited_enters_recovering() { + let (effects, state) = drive( + chain(&[ + (C0, ComponentAttrs::active_required()), + (C1, ComponentAttrs::passive_required()), + ]), + &[BOOT, Event::VerificationPassed(C0), Event::Timeout(C0)], + ); + assert_eq!(state, State::Recovering(C0)); + assert!(effects.contains(&Effect::RestoreGoldenImage(C0))); +} + +/// D2 (INV9): a timeout for a component we are not awaiting is stale/spurious +/// and is dropped — the machine keeps waiting on the real component. +#[test] +fn timeout_stale_id_ignored() { + let (effects, state) = drive( + chain(&[ + (C0, ComponentAttrs::active_required()), + (C1, ComponentAttrs::passive_required()), + ]), + &[ + BOOT, + Event::VerificationPassed(C0), + Event::Timeout(C1), // not the awaited id + ], + ); + assert_eq!(state, State::AwaitingReady(Some(C0))); + assert!(!effects.contains(&Effect::RestoreGoldenImage(C1))); +} + +/// D2: full path — timeout drives recovery, restore rewalks from the top, and +/// the chain then completes normally. +#[test] +fn timeout_recovers_then_rewalks_to_ready() { + let (effects, state) = drive( + chain(&[ + (C0, ComponentAttrs::active_required()), + (C1, ComponentAttrs::passive_required()), + ]), + &[ + BOOT, + Event::VerificationPassed(C0), + Event::Timeout(C0), // → Recovering(C0) + Event::Restored(C0), // → PreSupervision, rewalk from top + Event::VerificationPassed(C0), + Event::ComponentReady(C0), + Event::VerificationPassed(C1), + ], + ); + assert_eq!(state, State::Ready); + assert!(effects.contains(&Effect::RestoreGoldenImage(C0))); + assert!(effects.contains(&Effect::ReleaseReset(C1))); +} + /// Isolable component: every `VerificationFailed` is retried through a full /// recovery episode first; only once retries are exhausted does the /// component get held in reset and the walk continue to `Ready`.