orchestrator-sm: fail-closed effect-failure channel, panic-free buffers, non_exhaustive vocabulary
diff --git a/services/orchestrator/sm/src/lib.rs b/services/orchestrator/sm/src/lib.rs index 292d726..601e95c 100644 --- a/services/orchestrator/sm/src/lib.rs +++ b/services/orchestrator/sm/src/lib.rs
@@ -36,9 +36,23 @@ // deployment. The board owns `N` (chain length), `E` (effect-buffer size) and // max_retry. -/// Max pending events while settling one outside event (original + Emit follow-ups). +/// Upper bound on how many events one settle can queue: the triggering outside +/// event, at most one `Emit` follow-up (`RecoveryFailed`, emitted at most once +/// before latching), and at most one injected `EffectFailed` (de-duplicated in +/// `dispatch_with` — it is idempotent and terminal, so a second is never +/// queued). Three total; `PENDING_CAP` keeps headroom above that so the pushes +/// in `dispatch_with` can never overflow. const PENDING_CAP: usize = 8; +/// Compile-time floor tying the queue capacity to that worst case, mirroring +/// `Rot::EFFECT_CAP_OK` for the effect buffer. Evaluated at build time (an +/// anonymous `const`), so an under-sized `PENDING_CAP` fails to compile rather +/// than risking a runtime overflow. +const _: () = assert!( + PENDING_CAP >= 3, + "PENDING_CAP must hold one outside event + one Emit follow-up + one EffectFailed", +); + /// Superstate entered once the eRoT exits [`State::PreSupervision`] — i.e. on /// release of the first `Active` component, or once the whole chain has /// finished if it is all-`Passive`. Provides two platform-wide guarantees @@ -93,13 +107,22 @@ } /// Append one effect. `E` is sized so overflow is impossible for a machine - /// that compiles (the `E >= N + 2` floor in `Rot::new`); the panic is a - /// loud, fail-closed backstop for a future handler that emits beyond the - /// proven worst case, never a silent drop of a security-critical effect. + /// that compiles: `Rot::EFFECT_CAP_OK` proves `E >= N + 2` and no handler + /// emits more than `N + 2` effects into one `Sink`, so the push below can + /// never fail. The `Err` arm is therefore dead — dropped rather than + /// panicked, since a runtime panic here would be unreachable code shipped in + /// the binary. + /// + /// Effects buffered in one handler are actuated by the driver in emission + /// order and are **not** atomic: if effect *k* fails, effects `0..k` have + /// already hit hardware and `k+1..` still run before the injected + /// `EffectFailed` latches lockdown. Emit the most irreversible effect of a + /// batch last, so a mid-batch failure latches before it rather than after. pub fn emit(&mut self, effect: Effect) { - if self.effects.push(effect).is_err() { - panic!("effect buffer overflow: E must be >= chain length N + 2"); - } + // Dead Err arm: overflow is proved impossible by `Rot::EFFECT_CAP_OK` + // (`E >= N + 2`) plus the reducer never emitting more than `N + 2` + // effects into one Sink. + let _ = self.effects.push(effect); } pub fn effects(&self) -> &[Effect] { @@ -337,6 +360,7 @@ Event::PowerGood(PowerOnResult::SelfVerificationFailed) => { Outcome::Transition(State::Locked) } + Event::EffectFailed => Outcome::Transition(State::Locked), _ => Outcome::Super, }, @@ -377,6 +401,7 @@ // unhandled here (falls through to `Outcome::Super` and is // discarded) — that's a separate question. Event::CorruptionDetected(id) => rot.handle_corruption(*id, ctx), + Event::EffectFailed => Outcome::Transition(State::Locked), _ => Outcome::Super, }, @@ -532,16 +557,37 @@ Outcome::Handled } Event::CorruptionDetected(id) => rot.handle_corruption(*id, ctx), + Event::EffectFailed => Outcome::Transition(State::Locked), _ => Outcome::Super, }, } } } -/// Outward connection to the platform. Carry out one effect. Never called with +/// Signals that the shell could not carry out an [`Effect`]. The machine does +/// not need the shell's error detail — **every** actuation failure is treated +/// the same, fail-closed: the driver injects [`Event::EffectFailed`] and the +/// machine latches to [`State::Locked`]. This blanket policy is deliberate and +/// is what lets the failure signal stay a payload-less marker; a future design +/// that needs per-effect recovery must add a *new*, descriptive event rather +/// than widen this type. The shell logs the specifics on its side. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct EffectError; + +/// Outward connection to the platform. Carry out one effect, reporting +/// [`EffectError`] if it could not be performed. Never called with /// [`Effect::Emit`] — the orchestrator consumes those internally. +/// +/// Contract the reducer relies on: +/// - **Honest, complete feedback.** The reducer's correctness rests entirely on +/// the event stream the shell feeds back; dropping, reordering, or +/// synthesizing events silently breaks the state machine's invariants. +/// - **A failed [`Effect::LatchLockdown`] is a hard fault.** Lockdown is the top +/// of the escalation ladder — the reducer has nothing stronger to emit and +/// will *believe* it is `Locked`. The shell must treat that failure as +/// terminal (halt/reset), not a recoverable error. pub trait Platform { - fn execute(&mut self, effect: Effect); + fn execute(&mut self, effect: Effect) -> Result<(), EffectError>; } /// A handle for a caller's own event loop. Wraps the statig machine so callers @@ -563,8 +609,19 @@ /// Handle one event all the way through — including any [`Effect::Emit`] /// follow-ups — calling `on_effect` for each external effect in order. - pub fn dispatch_with(&mut self, event: Event, mut on_effect: impl FnMut(Effect)) { + /// + /// If `on_effect` reports an [`EffectError`], the driver injects an + /// [`Event::EffectFailed`] into the same run, so a failed actuation is + /// handled fail-closed (the machine latches to [`State::Locked`]) rather + /// than silently ignored. + pub fn dispatch_with( + &mut self, + event: Event, + mut on_effect: impl FnMut(Effect) -> Result<(), EffectError>, + ) { let mut pending: heapless::Vec<Event, PENDING_CAP> = heapless::Vec::new(); + // Dead Err arm: `pending` is empty and `PENDING_CAP >= 3` (asserted at + // build time), so the first push always fits. let _ = pending.push(event); let mut i = 0; @@ -578,9 +635,23 @@ for &effect in buf.effects() { match effect { Effect::Emit(internal) => { + // Dead Err arm: the reducer emits at most one `Emit` + // (`RecoveryFailed`) per settle, well within PENDING_CAP. let _ = pending.push(internal); } - external => on_effect(external), + external => { + if on_effect(external).is_err() && !pending.contains(&Event::EffectFailed) { + // Fail-closed, but only once: `EffectFailed` is + // idempotent and terminal (drives to `Locked`, which + // discards everything after), so a second injection + // would be a no-op. De-duping against this + // append-only queue — which still holds the first + // `EffectFailed` as its own marker — caps the queue at + // the worst case `PENDING_CAP` is sized for. Dead Err + // arm: that bound is below PENDING_CAP. + let _ = pending.push(Event::EffectFailed); + } + } } } }
diff --git a/services/orchestrator/sm/src/model.rs b/services/orchestrator/sm/src/model.rs index 0a0ac4c..abb1a9f 100644 --- a/services/orchestrator/sm/src/model.rs +++ b/services/orchestrator/sm/src/model.rs
@@ -26,6 +26,7 @@ /// Corresponds directly to the two-tier model in the CSA architecture document: /// `Active` = eRoT gate + iRoT gate; `Passive` = eRoT gate only. #[derive(Clone, Copy, PartialEq, Eq, Debug)] +#[non_exhaustive] pub enum ComponentKind { /// Has an integrated iRoT (e.g. Caliptra). Both eRoT-side (signature + SVN) /// and iRoT-side (local self-verification) checks apply. The machine waits in @@ -46,6 +47,7 @@ /// (The narrative design docs sometimes call the `Required` outcome "platform /// halt" — same behavior, this is the type-level name.) #[derive(Clone, Copy, PartialEq, Eq, Debug)] +#[non_exhaustive] pub enum FailurePolicy { /// Stop the boot sequence entirely: self-emits [`Event::RecoveryFailed`], /// which drives the machine to [`State::Locked`]. @@ -163,6 +165,7 @@ /// The result of the board's power-on checks, delivered inside [`Event::PowerGood`]. #[derive(Clone, Copy, PartialEq, Eq, Debug)] +#[non_exhaustive] pub enum PowerOnResult { /// Self-verified and provisioned. Provisioned, @@ -174,21 +177,35 @@ /// Everything the outside world can tell the state machine. #[derive(Clone, Copy, PartialEq, Eq, Debug)] +#[non_exhaustive] pub enum Event { /// Power-on, carrying the shell's self-verification and provisioning result. PowerGood(PowerOnResult), + /// The eRoT's signature + SVN check on this component passed. VerificationPassed(ComponentId), + /// The eRoT's signature + SVN check on this component failed. VerificationFailed(ComponentId), - /// An `Active` component's iRoT has finished local verification and is ready - /// (e.g. MCTP channel established). + /// An `Active` component's iRoT has finished local verification and is ready. ComponentReady(ComponentId), + /// A challenger has requested a signed attestation. AttestationChallenge, + /// A firmware update has been requested. UpdateRequest, + /// The staged update authenticated successfully. UpdateVerified, + /// The staged update failed authentication. UpdateRejected, + /// This component was found corrupt at runtime. CorruptionDetected(ComponentId), + /// This component's golden image has been restored. Restored(ComponentId), + /// A required component's recovery was exhausted. RecoveryFailed, + /// The shell could not carry out an emitted [`Effect`]; fail-closed, it + /// latches to [`State::Locked`] from any state. Injected by the driver when + /// a [`Platform::execute`](crate::Platform::execute) call fails; never + /// produced by a handler. + EffectFailed, } /// Everything the state machine can ask the outside world to do. @@ -197,6 +214,7 @@ /// queues the carried event for immediate handling, making follow-up events /// visible in the effect trace instead of hidden state changes. #[derive(Clone, Copy, PartialEq, Eq, Debug)] +#[non_exhaustive] pub enum Effect { ReadFirmware(ComponentId), VerifyFirmware(ComponentId), @@ -221,6 +239,7 @@ /// The states the machine can be in. None carry data; all mutable state lives /// in [`Rot`](crate::Rot) shared storage. #[derive(Clone, Copy, PartialEq, Eq, Debug)] +#[non_exhaustive] pub enum State { PowerOnReset, PreSupervision, @@ -260,6 +279,7 @@ /// Why a `heapless::Vec` of components is not a valid [`Chain`]. #[derive(Clone, Copy, PartialEq, Eq, Debug)] +#[non_exhaustive] pub enum ChainError { /// The chain has no components. Empty,
diff --git a/services/orchestrator/sm/src/tests.rs b/services/orchestrator/sm/src/tests.rs index f17c118..1c0dd70 100644 --- a/services/orchestrator/sm/src/tests.rs +++ b/services/orchestrator/sm/src/tests.rs
@@ -47,8 +47,9 @@ } impl Platform for Recorder { - fn execute(&mut self, effect: Effect) { + fn execute(&mut self, effect: Effect) -> Result<(), EffectError> { self.recorded.push(effect); + Ok(()) } } @@ -249,7 +250,10 @@ Event::Restored(C0), Event::VerificationPassed(C0), ] { - orch.dispatch_with(ev, |e| effects.push(e)); + orch.dispatch_with(ev, |e| { + effects.push(e); + Ok(()) + }); } assert_eq!(orch.state(), State::Ready); @@ -259,7 +263,10 @@ Event::Restored(C0), Event::VerificationPassed(C0), ] { - orch.dispatch_with(ev, |e| effects.push(e)); + orch.dispatch_with(ev, |e| { + effects.push(e); + Ok(()) + }); } assert_eq!(orch.state(), State::Ready); assert!(!effects[start..].contains(&Effect::LatchLockdown)); @@ -291,7 +298,10 @@ Event::VerificationPassed(C0), // re-walk restarts at the top Event::VerificationPassed(C1), // chain done → Ready ] { - orch.dispatch_with(ev, |e| effects.push(e)); + orch.dispatch_with(ev, |e| { + effects.push(e); + Ok(()) + }); } assert_eq!(orch.state(), State::Ready); @@ -313,7 +323,10 @@ Event::CorruptionDetected(C0), Event::Restored(C0), ] { - orch.dispatch_with(ev, |e| effects.push(e)); + orch.dispatch_with(ev, |e| { + effects.push(e); + Ok(()) + }); } assert_eq!(orch.state(), State::Locked); assert_eq!(effects.last(), Some(&Effect::LatchLockdown)); @@ -335,7 +348,10 @@ Event::VerificationPassed(C1), Event::VerificationPassed(C2), ] { - orch.dispatch_with(ev, |e| effects.push(e)); + orch.dispatch_with(ev, |e| { + effects.push(e); + Ok(()) + }); } assert_eq!(orch.state(), State::Ready); assert_eq!(effects.last(), Some(&Effect::ReleaseReset(C2))); @@ -731,7 +747,10 @@ let mut effects: Vec<Effect> = Vec::new(); for ev in [BOOT, Event::VerificationFailed(C0), Event::Restored(C0)] { - orch.dispatch_with(ev, |e| effects.push(e)); + orch.dispatch_with(ev, |e| { + effects.push(e); + Ok(()) + }); } assert_eq!(orch.state(), State::Locked); @@ -743,7 +762,10 @@ Event::UpdateRequest, Event::CorruptionDetected(C0), ] { - orch.dispatch_with(ev, |e| effects.push(e)); + orch.dispatch_with(ev, |e| { + effects.push(e); + Ok(()) + }); } assert_eq!( effects.len(), @@ -794,14 +816,20 @@ ); let mut effects: Vec<Effect> = Vec::new(); - orch.dispatch_with(BOOT, |e| effects.push(e)); + orch.dispatch_with(BOOT, |e| { + effects.push(e); + Ok(()) + }); assert_eq!( effects, std::vec![Effect::ReadFirmware(C0), Effect::VerifyFirmware(C0)], ); effects.clear(); - orch.dispatch_with(Event::VerificationPassed(C0), |e| effects.push(e)); + orch.dispatch_with(Event::VerificationPassed(C0), |e| { + effects.push(e); + Ok(()) + }); // All three effects emitted in the same handler, before ComponentReady. assert_eq!( effects, @@ -905,3 +933,118 @@ ]); assert!(Chain::try_from(v).is_ok()); } + +/// A [`Platform`] that records every effect and fails a chosen one, to exercise +/// the effect failure channel. +struct FailOn { + trigger: Effect, + recorded: Vec<Effect>, + failed: bool, +} + +impl FailOn { + fn new(trigger: Effect) -> Self { + Self { + trigger, + recorded: Vec::new(), + failed: false, + } + } +} + +impl Platform for FailOn { + fn execute(&mut self, effect: Effect) -> Result<(), EffectError> { + self.recorded.push(effect); + if effect == self.trigger { + self.failed = true; + Err(EffectError) + } else { + Ok(()) + } + } +} + +/// A failed reset actuation is fail-closed: the driver injects `EffectFailed` +/// and the machine latches to `Locked`, emitting `LatchLockdown`. +#[test] +fn effect_failure_latches_lockdown() { + let mut c = heapless::Vec::<(ComponentId, ComponentAttrs), CAPACITY>::new(); + c.push((C0, ComponentAttrs::passive_required())).unwrap(); + let mut orch = + Orchestrator::<CAPACITY, ECAP>::new(c.try_into().expect("valid chain"), MAX_RETRY); + let mut plat = FailOn::new(Effect::ReleaseReset(C0)); + + orch.dispatch(&mut plat, BOOT); // ReadFirmware/VerifyFirmware C0 — both succeed + orch.dispatch(&mut plat, Event::VerificationPassed(C0)); // ReleaseReset(C0) fails + + assert!(plat.failed, "the trigger effect should have been attempted"); + assert_eq!(orch.state(), State::Locked); + assert!(plat.recorded.contains(&Effect::LatchLockdown)); +} + +/// A failed isolation actuation (`AssertReset`) is equally fail-closed: even a +/// non-required component's containment failing latches the platform. +#[test] +fn failed_isolation_actuation_latches_lockdown() { + let mut c = heapless::Vec::<(ComponentId, ComponentAttrs), CAPACITY>::new(); + c.push((C0, ComponentAttrs::passive_required())).unwrap(); + c.push((C1, ComponentAttrs::passive_isolable())).unwrap(); + let mut orch = + Orchestrator::<CAPACITY, ECAP>::new(c.try_into().expect("valid chain"), MAX_RETRY); + let mut plat = FailOn::new(Effect::AssertReset(C1)); + + orch.dispatch(&mut plat, BOOT); + orch.dispatch(&mut plat, Event::VerificationPassed(C0)); + orch.dispatch(&mut plat, Event::VerificationPassed(C1)); // C1 released → Ready + orch.dispatch(&mut plat, Event::CorruptionDetected(C1)); // isolable → AssertReset(C1) fails + + assert!(plat.failed); + assert_eq!(orch.state(), State::Locked); + assert!(plat.recorded.contains(&Effect::LatchLockdown)); +} + +/// A failed recovery actuation is fail-closed too: if the shell cannot even +/// restore a required component's golden image, the platform latches rather +/// than continuing with an unrecovered component. +#[test] +fn failed_restore_actuation_latches_lockdown() { + let mut c = heapless::Vec::<(ComponentId, ComponentAttrs), CAPACITY>::new(); + c.push((C0, ComponentAttrs::passive_required())).unwrap(); + let mut orch = + Orchestrator::<CAPACITY, ECAP>::new(c.try_into().expect("valid chain"), MAX_RETRY); + let mut plat = FailOn::new(Effect::RestoreGoldenImage(C0)); + + orch.dispatch(&mut plat, BOOT); + orch.dispatch(&mut plat, Event::VerificationFailed(C0)); // → Recovering → RestoreGoldenImage(C0) fails + + assert!(plat.failed); + assert_eq!(orch.state(), State::Locked); + assert!(plat.recorded.contains(&Effect::LatchLockdown)); +} + +/// The lockdown latch is the last line of defense: even if *it* fails to +/// actuate, the machine must not spin. The re-injected `EffectFailed` is +/// ignored while `Locked`, so dispatch terminates and the latch is attempted +/// exactly once. +#[test] +fn failed_lockdown_actuation_does_not_loop() { + let mut c = heapless::Vec::<(ComponentId, ComponentAttrs), CAPACITY>::new(); + c.push((C0, ComponentAttrs::passive_required())).unwrap(); + let mut orch = + Orchestrator::<CAPACITY, ECAP>::new(c.try_into().expect("valid chain"), MAX_RETRY); + let mut plat = FailOn::new(Effect::LatchLockdown); + + // An unprovisioned power-on latches immediately; the latch actuation fails. + orch.dispatch(&mut plat, Event::PowerGood(PowerOnResult::Unprovisioned)); + + assert!(plat.failed, "the lockdown latch should have been attempted"); + assert_eq!(orch.state(), State::Locked); + assert_eq!( + plat.recorded + .iter() + .filter(|&&e| e == Effect::LatchLockdown) + .count(), + 1, + "a failing latch must not re-latch forever", + ); +}