orchestrator-sm: abort effect batch on first actuation failure
diff --git a/services/orchestrator/sm/src/lib.rs b/services/orchestrator/sm/src/lib.rs index b6b19ce..2b644d9 100644 --- a/services/orchestrator/sm/src/lib.rs +++ b/services/orchestrator/sm/src/lib.rs
@@ -87,9 +87,11 @@ /// /// Effects buffered in one handler are actuated by the driver in emission /// order and are **not** atomic: if effect *k* fails, effects `0..k` have - /// already hit hardware and `k+1..` still run before the injected - /// `EffectFailed` latches lockdown. Emit the most irreversible effect of a - /// batch last, so a mid-batch failure latches before it rather than after. + /// already hit hardware. Actuation is **fail-fast**, though — the driver + /// abandons `k+1..` and injects `EffectFailed` to latch lockdown, so no + /// effect ordered *after* a failure ever runs. A partially-applied prefix is + /// still possible, so emit the effect whose partial application is most + /// dangerous last, where it is least likely to be reached before a latch. pub fn emit(&mut self, effect: Effect) { // Dead Err arm: overflow is proved impossible by `Rot::EFFECT_CAP_OK` // (`E >= N + 2`) plus the reducer never emitting more than `N + 2` @@ -655,16 +657,26 @@ let _ = pending.push(internal); } external => { - if on_effect(external).is_err() && !pending.contains(&Event::EffectFailed) { - // Fail-closed, but only once: `EffectFailed` is - // idempotent and terminal (drives to `Locked`, which - // discards everything after), so a second injection - // would be a no-op. De-duping against this - // append-only queue — which still holds the first - // `EffectFailed` as its own marker — caps the queue at - // the worst case `PENDING_CAP` is sized for. Dead Err - // arm: that bound is below PENDING_CAP. - let _ = pending.push(Event::EffectFailed); + if on_effect(external).is_err() { + // Fail-closed AND fail-fast: abandon the rest of this + // batch so no effect ordered after the failed one hits + // hardware. `step` has already advanced the state as if + // the whole batch applied, so actuating `k+1..` would + // carry out effects for a transition we are about to + // override by latching to `Locked`. + // + // Inject `EffectFailed` once: it is idempotent and + // terminal (drives to `Locked`, which discards + // everything after), so a second injection would be a + // no-op. De-duping against this append-only queue — + // which still holds the first `EffectFailed` as its own + // marker — caps the queue at the worst case + // `PENDING_CAP` is sized for. Dead Err arm: that bound + // is below PENDING_CAP. + if !pending.contains(&Event::EffectFailed) { + let _ = pending.push(Event::EffectFailed); + } + break; } } }
diff --git a/services/orchestrator/sm/src/tests.rs b/services/orchestrator/sm/src/tests.rs index b4c1d46..af5c5e6 100644 --- a/services/orchestrator/sm/src/tests.rs +++ b/services/orchestrator/sm/src/tests.rs
@@ -1048,3 +1048,36 @@ "a failing latch must not re-latch forever", ); } + +/// Actuation is fail-fast: once an effect in a batch fails, no effect ordered +/// *after* it is attempted. Here `VerificationPassed(C0)` emits the batch +/// `[ReleaseReset(C0), ReadFirmware(C1), VerifyFirmware(C1)]`; failing the first +/// effect must abandon the two speculative reads of `C1` and latch, rather than +/// actuate them for a transition that is immediately overridden by `Locked`. +#[test] +fn batch_actuation_is_fail_fast() { + let mut orch = Orchestrator::<CAPACITY, ECAP>::new( + passive_required(&[C0, C1]).try_into().expect("valid chain"), + MAX_RETRY, + ); + let mut plat = FailOn::new(Effect::ReleaseReset(C0)); + + orch.dispatch(&mut plat, BOOT); // ReadFirmware/VerifyFirmware C0 — both succeed + orch.dispatch(&mut plat, Event::VerificationPassed(C0)); // ReleaseReset(C0) fails first + + assert!(plat.failed, "the failing effect should have been attempted"); + assert!( + plat.recorded.contains(&Effect::ReleaseReset(C0)), + "the failing effect itself is attempted", + ); + assert!( + !plat.recorded.contains(&Effect::ReadFirmware(C1)), + "an effect ordered after the failure must not be actuated", + ); + assert!( + !plat.recorded.contains(&Effect::VerifyFirmware(C1)), + "an effect ordered after the failure must not be actuated", + ); + assert_eq!(orch.state(), State::Locked); + assert!(plat.recorded.contains(&Effect::LatchLockdown)); +}