orchestrator: gate Isolable/Cascading immediately on DeviceFatal boot failure A DeviceFatal BootFailed means the device itself reported its image as bad. For Isolable/Cascading components, going through recovery first is wasted work: gate them immediately via the same policy path as CorruptionDetected, so isolation and cascade happen without a recovery attempt. Required components still enter Recovering. The split is about what gating costs, not about whether a retry could work: recovery can try a different source each attempt, so a golden image may fix any component, but gating an Isolable component only holds it in reset while gating a Required one locks the platform down. The halt-on-exhaustion path applies normally. Also adds Effect::ReportBootFailed (and Report::BootFailed in the driver) so the checkpoint name and failure kind reach management software. The report is emitted before the transition for every non-stale BootFailed, regardless of kind or policy. Closes #477. Assisted-by: Claude (Anthropic)
diff --git a/services/orchestrator/driver/src/board.rs b/services/orchestrator/driver/src/board.rs index dfed222..dc86591 100644 --- a/services/orchestrator/driver/src/board.rs +++ b/services/orchestrator/driver/src/board.rs
@@ -4,7 +4,7 @@ //! What the board supplies to the driver: traits and wiring data only. //! Boards (or test mocks) implement these. -use openprot_orchestrator_sm::{ComponentId, ComponentKind}; +use openprot_orchestrator_sm::{BootFailureKind, ComponentId, ComponentKind}; pub use orchestrator_capabilities::{BootControl, BootWatch}; use orchestrator_capabilities::{Svn, SvnFloor}; @@ -112,6 +112,14 @@ /// Recovery attempts exhausted and the platform halts. Reported before /// the lockdown latch, while there is still a platform to report from. RecoveryFailed(ComponentId), + /// A boot walk failed at a named checkpoint. The `kind` tells whether + /// this was silence (timed out), a retriable device fault, or a terminal + /// device fault. + BootFailed { + id: ComponentId, + checkpoint: &'static str, + kind: BootFailureKind, + }, /// An update request declined because the platform was busy. Nothing /// staged, and the requester may ask again. Platform-wide, not /// per-component: the machine supervises one update at a time and
diff --git a/services/orchestrator/driver/src/driver.rs b/services/orchestrator/driver/src/driver.rs index c012d4e..901aea5 100644 --- a/services/orchestrator/driver/src/driver.rs +++ b/services/orchestrator/driver/src/driver.rs
@@ -297,6 +297,18 @@ self.report(Report::UpdateAborted); Ok(None) } + Effect::ReportBootFailed { + id, + checkpoint, + kind, + } => { + self.report(Report::BootFailed { + id, + checkpoint, + kind, + }); + Ok(None) + } // No board capability is composed for these seams yet, so they // fail closed here instead of behind stub methods. Each group // gains an executor when its capability joins
diff --git a/services/orchestrator/driver/src/tests.rs b/services/orchestrator/driver/src/tests.rs index 9f3051a..2765931 100644 --- a/services/orchestrator/driver/src/tests.rs +++ b/services/orchestrator/driver/src/tests.rs
@@ -888,10 +888,15 @@ } /// One of each report, so a test covers the whole enum. -fn every_report() -> [Report; 4] { +fn every_report() -> [Report; 5] { [ Report::Isolated(C0), Report::RecoveryFailed(C0), + Report::BootFailed { + id: C0, + checkpoint: "self-test", + kind: BootFailureKind::DeviceFatal, + }, Report::UpdateDeferred, Report::UpdateAborted, ] @@ -901,7 +906,7 @@ // over; the unit sink is a wiring choice and satisfies the same caller. #[test] fn every_report_reaches_a_sink() { - fn tell<S: ReportSink>(sink: &mut S, reports: [Report; 4]) { + fn tell<S: ReportSink>(sink: &mut S, reports: [Report; 5]) { for report in reports { sink.report(report); } @@ -1027,6 +1032,11 @@ for effect in [ Effect::ReportIsolated(C0), Effect::ReportRecoveryFailed(C0), + Effect::ReportBootFailed { + id: C0, + checkpoint: "self-test", + kind: BootFailureKind::DeviceFatal, + }, Effect::ReportUpdateDeferred, Effect::ReportUpdateAborted, ] {
diff --git a/services/orchestrator/sm/src/lib.rs b/services/orchestrator/sm/src/lib.rs index d9adbec..4a95c0b 100644 --- a/services/orchestrator/sm/src/lib.rs +++ b/services/orchestrator/sm/src/lib.rs
@@ -634,17 +634,41 @@ self.clear_awaiting_boot(*id); Outcome::Handled } - // Boot failure: either the walk judged a checkpoint failure - // (BootFailed) or the fleet-level watchdog fired (Timeout). - // Both recover the component if it is still awaiting boot; - // stale/spurious events are dropped. - Event::BootFailed { id, .. } | Event::Timeout(id) => { + // Timeout always recovers: silence says nothing about the + // image, so a fresh attempt is never futile. Stale ids + // (component not awaiting boot) are dropped. + Event::Timeout(id) => { if self.is_awaiting_boot(*id) { Outcome::Transition(State::Recovering(*id)) } else { Outcome::Handled } } + // A device-reported boot failure. DeviceFatal means the + // device itself declared the image unrecoverable by retry, + // so Isolable/Cascading components are gated immediately + // (no recovery budget burned); Required still recovers + // because a different recovery source could help. + // Non-fatal kinds recover unconditionally, like Timeout. + Event::BootFailed { + id, + checkpoint, + kind, + } => { + if !self.is_awaiting_boot(*id) { + return Outcome::Handled; + } + ctx.emit(Effect::ReportBootFailed { + id: *id, + checkpoint, + kind: *kind, + }); + if *kind == BootFailureKind::DeviceFatal { + self.handle_corruption_advancing(*id, ctx) + } else { + Outcome::Transition(State::Recovering(*id)) + } + } Event::EffectFailed => Outcome::Transition(State::Locked), _ => Outcome::Super, }, @@ -711,10 +735,26 @@ // Releases off `chain[cursor]` too, so the cursor must move // off a component the cascade gated. `Handled` keeps `awaiting`. Event::CorruptionDetected(id) => self.handle_corruption_advancing(*id, ctx), - // `Timeout` is intentionally not handled here: it falls through - // to `handle_supervising`, which runs the device-agnostic - // boot-progress watchdog uniformly across every supervised state - // (an `AwaitingReady` timeout is no longer special-cased). + // DeviceFatal: the device declared the image unrecoverable. + // Gate Isolable/Cascading immediately (cursor-advancing), + // recover Required. Non-fatal kinds and Timeout fall through + // to handle_supervising below. + Event::BootFailed { + id, + kind: BootFailureKind::DeviceFatal, + checkpoint, + } if self.is_awaiting_boot(*id) => { + ctx.emit(Effect::ReportBootFailed { + id: *id, + checkpoint, + kind: BootFailureKind::DeviceFatal, + }); + self.handle_corruption_advancing(*id, ctx) + } + // `Timeout` and non-fatal `BootFailed` fall through to + // `handle_supervising`, which runs the device-agnostic + // boot-progress watchdog uniformly across every supervised + // state. _ => Outcome::Super, }, @@ -863,16 +903,37 @@ self.clear_awaiting_boot(*id); Outcome::Handled } - // Boot failure across every supervised state: a walk checkpoint - // failure (BootFailed) or fleet-level watchdog (Timeout) recovers - // the component if it is still awaiting boot. Stale events dropped. - Event::BootFailed { id, .. } | Event::Timeout(id) => { + // Timeout always recovers: silence is ambiguous, so retrying + // is never futile. Stale ids (already booted) are dropped. + Event::Timeout(id) => { if self.is_awaiting_boot(*id) { Outcome::Transition(State::Recovering(*id)) } else { Outcome::Handled } } + // Same DeviceFatal split as PreSupervision/AwaitingReady, but + // non-advancing: Ready/Recovering/Updating are not walking the + // chain, so the cursor is not the walk position. + Event::BootFailed { + id, + checkpoint, + kind, + } => { + if !self.is_awaiting_boot(*id) { + return Outcome::Handled; + } + ctx.emit(Effect::ReportBootFailed { + id: *id, + checkpoint, + kind: *kind, + }); + if *kind == BootFailureKind::DeviceFatal { + self.handle_corruption(*id, ctx) + } else { + Outcome::Transition(State::Recovering(*id)) + } + } // A new update cannot start from any supervised state except // `Ready` (the machine is mid-walk, mid-update, or mid-recovery). // `Ready` accepts `UpdateRequest` upstream in `handle` (→ `Updating`)
diff --git a/services/orchestrator/sm/src/model.rs b/services/orchestrator/sm/src/model.rs index 8d25d01..a7774bd 100644 --- a/services/orchestrator/sm/src/model.rs +++ b/services/orchestrator/sm/src/model.rs
@@ -242,9 +242,19 @@ RecoveryFailed, /// A boot walk failed at a specific checkpoint with a classified cause. /// Produced by the platform driver when `CheckpointWalk::poll` returns - /// `WalkVerdict::Failed`. A component awaiting boot enters recovery. - /// The `checkpoint` and `kind` fields are preserved so the state - /// machine can differentiate (e.g. skip retries on `DeviceFatal`). + /// `WalkVerdict::Failed`. `TimedOut` and `DeviceRetriable` enter recovery + /// (the retry cap limits attempts). `DeviceFatal` gates + /// `Isolable`/`Cascading` components immediately (no recovery attempt) + /// and enters recovery for `Required` (halt-on-exhaustion still applies). + /// + /// The split is about what gating costs, not about whether a retry could + /// work. Recovery does not re-run the bad image: `attempt` rides on + /// [`Effect::RecoverComponent`] so the driver can try a different source + /// each time, and an untried golden image may fix any component. Gating an + /// `Isolable`/`Cascading` component holds it in reset and leaves the + /// platform running, so skipping straight there costs little. Gating a + /// `Required` one locks the platform down, so every recovery source is + /// tried first. BootFailed { id: ComponentId, checkpoint: &'static str, @@ -369,6 +379,14 @@ /// immediately before the machine latches to [`Locked`](crate::State::Locked). /// Names the component that forced the halt. ReportRecoveryFailed(ComponentId), + /// Report that a boot walk failed at a named checkpoint. Emitted before + /// the transition (recovery or gate) so management software learns where + /// in the boot sequence the failure occurred. + ReportBootFailed { + id: ComponentId, + checkpoint: &'static str, + kind: BootFailureKind, + }, /// Report that an [`Event::UpdateRequest`] was declined because the machine /// is busy in a supervised state other than [`Ready`](crate::State::Ready) /// — a chain walk is finishing ([`AwaitingReady`](crate::State::AwaitingReady)),
diff --git a/services/orchestrator/sm/src/tests.rs b/services/orchestrator/sm/src/tests.rs index 080a1e7..1847fdb 100644 --- a/services/orchestrator/sm/src/tests.rs +++ b/services/orchestrator/sm/src/tests.rs
@@ -567,11 +567,10 @@ assert!(!effects.contains(&Effect::RecoverComponent { id: C1, attempt: 0 })); } -/// A `BootFailed` for the awaited component enters recovery, same as -/// `Timeout`. The checkpoint and kind are preserved in the event but do not -/// affect the transition. +/// A retriable `BootFailed` for the awaited component enters recovery, +/// same as `Timeout`. #[test] -fn boot_failed_awaited_enters_recovering() { +fn boot_failed_retriable_enters_recovering() { let (effects, state) = drive( chain(&[ (C0, ComponentAttrs::active_required()), @@ -589,6 +588,65 @@ ); assert_eq!(state, State::Recovering(C0)); assert!(effects.contains(&Effect::RecoverComponent { id: C0, attempt: 0 })); + assert!(effects.contains(&Effect::ReportBootFailed { + id: C0, + checkpoint: "heartbeat", + kind: BootFailureKind::DeviceRetriable, + })); +} + +/// A fatal `BootFailed` for an `Isolable` component gates it immediately +/// with no recovery attempt. +#[test] +fn boot_failed_fatal_isolable_gates_without_recovery() { + let (effects, state) = drive( + chain(&[ + (C0, ComponentAttrs::active_isolable()), + (C1, ComponentAttrs::passive_required()), + ]), + &[ + BOOT, + Event::VerificationPassed(C0), + Event::BootFailed { + id: C0, + checkpoint: "bl1", + kind: BootFailureKind::DeviceFatal, + }, + ], + ); + assert_eq!(state, State::AwaitingReady(Some(C0))); + assert!(!effects.contains(&Effect::RecoverComponent { id: C0, attempt: 0 })); + assert!(effects.contains(&Effect::AssertReset(C0))); + assert!(effects.contains(&Effect::ReportIsolated(C0))); + assert!(effects.contains(&Effect::ReportBootFailed { + id: C0, + checkpoint: "bl1", + kind: BootFailureKind::DeviceFatal, + })); +} + +/// A fatal `BootFailed` for a `Required` component still enters recovery +/// (the device said "this image is bad" but a different recovery source +/// could help). +#[test] +fn boot_failed_fatal_required_still_recovers() { + let (effects, state) = drive( + chain(&[ + (C0, ComponentAttrs::active_required()), + (C1, ComponentAttrs::passive_required()), + ]), + &[ + BOOT, + Event::VerificationPassed(C0), + Event::BootFailed { + id: C0, + checkpoint: "bl1", + kind: BootFailureKind::DeviceFatal, + }, + ], + ); + assert_eq!(state, State::Recovering(C0)); + assert!(effects.contains(&Effect::RecoverComponent { id: C0, attempt: 0 })); } /// A `BootFailed` for a component not awaiting boot is stale and dropped. @@ -613,6 +671,71 @@ assert!(!effects.contains(&Effect::RecoverComponent { id: C1, attempt: 0 })); } +/// A fatal `BootFailed` in `PreSupervision` for a speculatively released +/// passive isolable component gates it and advances the cursor past it, +/// same as `CorruptionDetected`. +#[test] +fn boot_failed_fatal_in_pre_supervision_gates_passive() { + let (effects, state) = drive( + chain(&[ + (C0, ComponentAttrs::passive_isolable()), + (C1, ComponentAttrs::passive_required()), + ]), + &[ + BOOT, + Event::VerificationPassed(C0), + Event::BootFailed { + id: C0, + checkpoint: "self-test", + kind: BootFailureKind::DeviceFatal, + }, + Event::VerificationPassed(C1), + ], + ); + assert_eq!(state, State::Ready); + assert!(effects.contains(&Effect::AssertReset(C0))); + assert!(effects.contains(&Effect::ReportIsolated(C0))); + assert!(effects.contains(&Effect::ReportBootFailed { + id: C0, + checkpoint: "self-test", + kind: BootFailureKind::DeviceFatal, + })); + assert!(!effects.contains(&Effect::RecoverComponent { id: C0, attempt: 0 })); +} + +/// A fatal `BootFailed` in `Ready` for a passive isolable component that +/// was released speculatively and still owes a boot signal: gates it +/// without recovery and stays `Ready`. +#[test] +fn boot_failed_fatal_in_ready_gates_passive() { + let (effects, state) = drive( + chain(&[ + (C0, ComponentAttrs::passive_isolable()), + (C1, ComponentAttrs::passive_required()), + ]), + &[ + BOOT, + Event::VerificationPassed(C0), + Event::VerificationPassed(C1), + // Both released; machine is Ready with C0/C1 still awaiting boot. + Event::BootFailed { + id: C0, + checkpoint: "pcie-link", + kind: BootFailureKind::DeviceFatal, + }, + ], + ); + assert_eq!(state, State::Ready); + assert!(effects.contains(&Effect::AssertReset(C0))); + assert!(effects.contains(&Effect::ReportIsolated(C0))); + assert!(effects.contains(&Effect::ReportBootFailed { + id: C0, + checkpoint: "pcie-link", + kind: BootFailureKind::DeviceFatal, + })); + assert!(!effects.contains(&Effect::RecoverComponent { id: C0, attempt: 0 })); +} + /// An out-of-chain id in a `VerificationFailed` report is dropped: the core /// supervises only chain components, so a verdict for an id the chain does not /// contain neither enters `Recovering` nor emits `RecoverComponent`.