Aligning to the latest CSA
diff --git a/docs/src/design/orchestrator/orchestrator-machine.md b/docs/src/design/orchestrator/orchestrator-machine.md index 0671828..68c7f0c 100644 --- a/docs/src/design/orchestrator/orchestrator-machine.md +++ b/docs/src/design/orchestrator/orchestrator-machine.md
@@ -12,31 +12,32 @@ PowerOnReset --> Locked : PowerGood(Unprovisioned) PowerOnReset --> Locked : PowerGood(SelfVerificationFailed) - VerifyingPlatform --> VerifyingPlatform : VerificationPassed [more, Passive]\n/ ReleaseReset · ReadFirmware · VerifyFirmware - VerifyingPlatform --> AwaitingReady : VerificationPassed [more, Active]\n/ ReleaseReset · ReadFirmware · VerifyFirmware - VerifyingPlatform --> Ready : VerificationPassed [chain done]\n/ ReleaseReset - VerifyingPlatform --> VerifyingPlatform : VerificationFailed [optional]\n(skip — held in reset) - VerifyingPlatform --> Recovering : VerificationFailed [required]\n/ RestoreGoldenImage + VerifyingPlatform --> VerifyingPlatform : VerificationPassed [more, Passive]<br/>/ ReleaseReset · ReadFirmware · VerifyFirmware + VerifyingPlatform --> AwaitingReady : VerificationPassed [more, Active]<br/>/ ReleaseReset · ReadFirmware · VerifyFirmware + VerifyingPlatform --> Ready : VerificationPassed [chain done]<br/>/ ReleaseReset + VerifyingPlatform --> VerifyingPlatform : VerificationFailed [Isolable or Cascading]<br/>(skip — held in reset) + VerifyingPlatform --> Recovering : VerificationFailed [Required]<br/>/ RestoreGoldenImage - AwaitingReady --> AwaitingReady : VerificationPassed [more]\n/ ReleaseReset · ReadFirmware · VerifyFirmware + AwaitingReady --> AwaitingReady : VerificationPassed [more]<br/>/ ReleaseReset · ReadFirmware · VerifyFirmware AwaitingReady --> Ready : ComponentReady [chain done or cursor past end] AwaitingReady --> AwaitingReady : ComponentReady [more] - AwaitingReady --> AwaitingReady : VerificationFailed [optional, iRoT pending] - AwaitingReady --> Ready : VerificationFailed [optional, no iRoT pending, chain done] - AwaitingReady --> Recovering : VerificationFailed [required]\n/ RestoreGoldenImage + AwaitingReady --> AwaitingReady : VerificationFailed [Isolable or Cascading, iRoT pending] + AwaitingReady --> Ready : VerificationFailed [Isolable or Cascading, no iRoT pending, chain done] + AwaitingReady --> Recovering : VerificationFailed [Required]<br/>/ RestoreGoldenImage + AwaitingReady --> Recovering : Timeout(id) [id == awaiting]<br/>/ RestoreGoldenImage state Operational { [*] --> Ready - Ready --> Updating : UpdateRequest\n/ AuthenticateUpdate · StageUpdate + Ready --> Updating : UpdateRequest<br/>/ AuthenticateUpdate · StageUpdate Updating --> Ready : UpdateVerified / ActivateUpdate Updating --> Ready : UpdateRejected / DiscardStaged - Ready --> Recovering : CorruptionDetected\n/ RestoreGoldenImage - Updating --> Recovering : CorruptionDetected\n/ RestoreGoldenImage - AwaitingReady --> Recovering : CorruptionDetected\n/ RestoreGoldenImage + Ready --> Recovering : CorruptionDetected<br/>/ RestoreGoldenImage + Updating --> Recovering : CorruptionDetected<br/>/ RestoreGoldenImage + AwaitingReady --> Recovering : CorruptionDetected<br/>/ RestoreGoldenImage } Recovering --> VerifyingPlatform : Restored [retry < max_retry] - Recovering --> Locked : Restored [retry ≥ max_retry]\n(self-emits RecoveryFailed)\n/ LatchLockdown + Recovering --> Locked : Restored [retry ≥ max_retry]<br/>(self-emits RecoveryFailed)<br/>/ LatchLockdown Locked --> Locked : (terminal — all events ignored) ``` @@ -52,8 +53,9 @@ | Field | Type | Purpose | |---|---|---| | `chain` | `Vec<(ComponentId, ComponentAttrs), N>` | Ordered trust chain, supplied by the shell at construction time. Never mutated after build. | -| `cursor` | `u8` | Index of the component currently under verification. Reset to 0 on every `VerifyingPlatform` entry. Advances on each `VerificationPassed` (and on optional `VerificationFailed`) via `Outcome::Handled`. | -| `failed` | `Option<ComponentId>` | The component that triggered the current recovery episode; `None` while healthy. Set on required `VerificationFailed` or `CorruptionDetected`. | +| `cursor` | `u8` | Index of the component currently under verification. Reset to 0 on every `VerifyingPlatform` entry. Advances on each `VerificationPassed` (and on `Isolable`/`Cascading` `VerificationFailed`, or cascade-skip) via `Outcome::Handled`. | +| `held` | `Vec<ComponentId, N>` | Set of component IDs held in reset due to `Cascading` failure or cascade-skip (`Isolable` failures do not populate this set). Checked before emitting `ReadFirmware` for each new component to evaluate `depends_on`. Cleared on `VerifyingPlatform` entry alongside `cursor`. | +| `failed` | `Option<ComponentId>` | The component that triggered the current recovery episode; `None` while healthy. Set on `Required` `VerificationFailed`, `Timeout`, or `CorruptionDetected`. | | `retry_count` | `u8` | Number of consecutive failed restore attempts. Cleared to 0 in `Ready`'s entry action — consecutive only (INV7). | | `max_retry` | `u8` | Shell-chosen ceiling for `retry_count`. When `retry_count >= max_retry` the machine self-emits `RecoveryFailed` instead of re-walking the chain. | | `awaiting` | `Option<ComponentId>` | The `Active` component whose iRoT readiness is currently outstanding. `Some` only while in `AwaitingReady`; `None` everywhere else (INV9). | @@ -105,8 +107,9 @@ | `VerificationPassed(id)` | more, current `Passive` | `ReleaseReset` · `ReadFirmware(next)` · `VerifyFirmware(next)` | `Handled` (cursor ++) | | `VerificationPassed(id)` | more, current `Active` | `ReleaseReset` · `ReadFirmware(next)` · `VerifyFirmware(next)` | `AwaitingReady` (awaiting = Some(id)) | | `VerificationPassed(id)` | chain done | `ReleaseReset(id)` | `Ready` | -| `VerificationFailed(id)` | `attrs.required` | — | `Recovering` (failed = Some(id)) | -| `VerificationFailed(id)` | `!attrs.required` | — | `Handled` (skip; cursor ++; if chain done → `Ready`) | +| `VerificationFailed(id)` | `attrs.failure_policy == Required` | — | `Recovering` (failed = Some(id)) | +| `VerificationFailed(id)` | `Isolable` | — | `Handled` (skip; cursor ++; if chain done → `Ready`) | +| `VerificationFailed(id)` | `Cascading` | — | `Handled` (add to `held`; cascade-skip dependents; cursor ++; if chain done → `Ready`) | | anything else | — | — | `Outcome::Super` → `Operational` | --- @@ -128,9 +131,13 @@ | `ComponentReady(id)` | `id == awaiting`, cursor past end | — | `Ready` | | `VerificationPassed(id)` | more | `ReleaseReset` · `ReadFirmware(next)` · `VerifyFirmware(next)` | `Handled` (cursor ++) | | `VerificationPassed(id)` | chain done | `ReleaseReset(id)` | `Ready` | -| `VerificationFailed(id)` | `attrs.required` | — | `Recovering` (failed = Some(id), awaiting = None) | -| `VerificationFailed(id)` | `!attrs.required`, iRoT pending | — | `Handled` (skip; cursor ++) | -| `VerificationFailed(id)` | `!attrs.required`, no iRoT pending, chain done | — | `Ready` | +| `Timeout(id)` | `id == awaiting` | — | `Recovering` (failed = Some(id), awaiting = None) | +| `Timeout(id)` | `id != awaiting` | — | `Handled` (stale — ignore) | +| `VerificationFailed(id)` | `attrs.failure_policy == Required` | — | `Recovering` (failed = Some(id), awaiting = None) | +| `VerificationFailed(id)` | `Isolable`, iRoT pending | — | `Handled` (skip; cursor ++) | +| `VerificationFailed(id)` | `Isolable`, no iRoT pending, chain done | — | `Ready` | +| `VerificationFailed(id)` | `Cascading`, iRoT pending | — | `Handled` (add to `held`; cascade-skip dependents; cursor ++) | +| `VerificationFailed(id)` | `Cascading`, no iRoT pending, chain done | — | `Ready` | | anything else | — | — | `Outcome::Super` → `Operational` | `ComponentReady` and `VerificationPassed` are independent and may arrive in @@ -174,8 +181,11 @@ The machine is attempting to restore a corrupted or rejected component. -**Entry action**: emit `RestoreGoldenImage(rot.failed)` — exactly the named -component, not the whole chain (INV5). +**Entry action**: emit `RestoreGoldenImage(rot.failed)` — targets the failed +component's *recovery region*: all components sharing the same `RegionId` are +restored together. The core supplies the failed component ID; the shell resolves +region membership from the chain at startup. Only the named component triggers +the restore, but the entire region is affected (not the whole chain — INV5). | Event | Guard | Effects | Next state | |---|---|---|---| @@ -184,9 +194,26 @@ | `RecoveryFailed` | — | — | `Locked` | | anything else | — | — | `Outcome::Super` → `Operational` | -`Effect::Emit(RecoveryFailed)` is the *feedback-as-data* mechanism: the core -produces the event internally, the orchestrator intercepts and re-dispatches it -before returning, and the decision is visible in the effect trace. +`Effect::Emit(RecoveryFailed)` is the *feedback-as-data* mechanism. It is easiest +to understand by asking why the machine doesn't just jump straight to `Locked` +when the retry cap is hit. + +When the last restore attempt fails, the machine has a decision to make: give up +and lock down. It could act on that decision silently, transitioning directly +from `Recovering` to `Locked` inside the handler. Instead it does something that +looks indirect at first: it emits `RecoveryFailed` as an *effect* — a piece of +data saying "a follow-up event named `RecoveryFailed` should happen next" — and +returns. The orchestrator sees that effect, puts `RecoveryFailed` at the front of +the queue, and dispatches it right away. That second event is what actually moves +the machine to `Locked`. + +The payoff is that the give-up decision becomes a visible event rather than a +hidden jump. Anyone reading the effect trace sees `RecoveryFailed` appear at the +exact moment the cap was reached, and `Locked` is only ever entered one way — by +handling that event — no matter where the lockdown was triggered from. The core +never reaches out and changes its own state behind the scenes; every transition, +including its own internally-generated ones, travels through the same event path +and shows up in the same trace. **Why re-walk from `cursor = 0`?** After restoring a component the machine re-enters `VerifyingPlatform` and re-verifies the entire chain from scratch @@ -196,8 +223,8 @@ (NIST SP 800-193) — requires that trust be re-established end-to-end before the platform is considered healthy again. The CSA document does not prescribe the exact recovery sequencing, but the re-walk implements the spirit of that -principle. Optional components that fail during the re-walk are skipped (held in -reset) as during initial boot; they are re-released only if they pass +principle. `Isolable` and `Cascading` components that fail during the re-walk are skipped +(held in reset) as during initial boot; they are re-released only if they pass `VerificationPassed` in the new walk. --- @@ -225,6 +252,124 @@ --- +## Centralizing the Operational Contract + +Four states run once the platform is up: `Ready`, `Updating`, `Recovering`, and +`AwaitingReady`. Two things must be true in all four, no matter which one the +machine is in — an attestation challenge always gets answered, and a corruption +report always starts recovery. Call those two shared rules the **operational +contract**. + +The question is where to write the contract down. One option is to copy it into +each of the four states. The problem with copies is that they drift: someone +edits one and forgets the others, and now corruption is handled in three states +but quietly ignored in the fourth. The other option is to write it once, in a +single place the four states share. That is what "centralizing" means here, and +it is the safer choice for a rule this important. + +A **hierarchical state machine (HSM)** gives us that single place. In a plain +state machine every state handles its own events and nothing more. An HSM adds a +*superstate* — a parent that several states sit under. When a state does not +handle an event itself, the event falls through to the parent. So each state +handles what is unique to it, and the parent handles what they all share. Our +four operational states sit under one superstate, `Operational`, and that is +where the operational contract lives — written exactly once. + +To be fair to the alternative: a careful flat state machine could get the same +result by giving every state a default branch that calls one shared function. +That works, and it behaves identically. The difference is that the shared +function is a convention people have to remember to wire up in each state, while +the superstate is a single declared link. The rest of this section is about why +that difference is worth having. + +The first reason is that there is only one copy to get right. In the flat +version each state's default branch is written by hand, so they can quietly +diverge — one calls the shared function, another does something slightly +different. With the superstate, each state points at the one `Operational` +handler and nothing else. What that single copy buys us for auditing and +verification is covered in [Invariant Verification](#invariant-verification) +below. + +The second reason is that the superstate also covers the *in-between* states, +which are the easy ones to forget. `Operational` includes not just the settled +`Ready` state but also `AwaitingReady` (still booting) and `Recovering` (still +restoring). Corruption has to be handled even during those brief windows — and +those are exactly the states a developer is tempted to skip as "temporary." +Putting them under the same parent means the corruption rule applies to them +automatically, without anyone having to remember. + +Extensibility helps too, though less dramatically. An event that applies +everywhere — attestation today, telemetry later — has one obvious home and is +added once. An event that needs to behave differently per state is no harder +than before: the state handles it itself, which simply overrides the parent. + +One honest caveat: this is a strong default, not an ironclad guarantee. It works +as long as each state actually falls through to the parent and is correctly +linked to it. Someone can still break it by handling an event by mistake, or by +forgetting to link a new state. The real win is that the *safe* thing is the +easy thing — do nothing and the event falls through to the shared rule — whereas +the flat version makes the safe thing the line you have to remember to add. And +the payoff today is small in raw terms: the superstate shares just two events +across four states. The case rests on *which* two events they are — the +platform's corruption response — not on saving lines. + +The cost is that reading one state no longer tells the whole story. To know what +`Ready` does with an event, you also have to know it falls through to +`Operational` and go read that. This is a trade, not a free win: the design takes +on more indirection \u2014 and a bit more machinery, since a flat state machine is just\na match on state and event while this adds superstates and the fall-through rule \u2014\nin exchange for removing duplication of the one rule where duplication is most\ndangerous. We accept that because the `statig` library is +used here without macros, so the fall-through is plain, visible code rather than +hidden generation, and the small amount of library machinery involved is +covered by the machine's tests. The alternative is not "no library" — it is a +hand-written dispatch loop we would have to maintain and test ourselves. + +On balance the hierarchy is the right call. The rule it centralizes is the +platform's corruption response, so the single copy sits exactly where a silent +mismatch would do the most harm. And the shared set is expected to grow, not +stay put — transit tamper detection and telemetry queries are both +platform-wide operational events on the OpenPRoT roadmap, and both fit this +pattern directly. Two events is a floor, not a ceiling. Only if it stayed at +two forever would a flat state machine with a shared default become the simpler +choice worth reconsidering. + +--- + +## Invariant Verification + +The invariants for the operational regime describe the whole regime, not one +state at a time. Because the hierarchy stores each rule at that same whole-regime +level, checking that the code matches the spec stays simple instead of turning +into a state-by-state comparison. + +Take INV6: *"an attestation challenge is answered in any operational state +without changing state."* In this design the rule itself is one line of code — +the `AttestationChallenge` row in the `Operational` handler. Reading that row +tells you *what* happens; to know it happens *everywhere it should*, you also +check that each of the four operational states is linked to the superstate (its +`superstate()` points at `Operational`) and does not handle the event itself. +That is one row plus four link checks: + +| Invariant | Where it lives | To verify | +|---|---|---| +| INV6 — attestation answered in any operational state, no transition | `AttestationChallenge → SignAttestation`, `Handled` (one row in `Operational`) | Read one row + confirm four states link to `Operational` | +| INV5 — corruption triggers recovery from any operational state | `CorruptionDetected → Recovering` (one row in `Operational`) | Read one row + confirm four states link to `Operational` | + +This is not free — the four links matter, because a state that forgets to link, +or handles the event itself, silently drops out of the rule. But it is far less +work than the flat alternative, which spreads each rule across all four state +tables. Checking INV6 there means finding all four copies, confirming they match, +and doing it again every time a new state is added. If two copies ever disagree, +the invariant silently holds in some states and fails in others, with nothing to +flag it. Both designs require you to look at all four states; the difference is +that the superstate leaves one authoritative copy of the rule to compare against, +while the flat version leaves four copies that must be proven equal to each +other. + +That is what it means for the hierarchy to line up with the invariants: the rule +is written and implemented in one place, and verifying it is checking that one +place plus the links into it — not reconciling four independent copies. + +--- + ## `statig` integration The machine uses `statig` 0.4.1 with hand-written trait impls — no proc-macros.
diff --git a/docs/src/design/orchestrator/orchestrator-model.md b/docs/src/design/orchestrator/orchestrator-model.md index 03b3be5..c56f278 100644 --- a/docs/src/design/orchestrator/orchestrator-model.md +++ b/docs/src/design/orchestrator/orchestrator-model.md
@@ -52,29 +52,68 @@ Passive — no integrated iRoT; only the eRoT check applies ``` +In CSA terminology: `Active` corresponds to a *SoC with an integrated iRoT* +(e.g. a BMC or CPU with Caliptra). `Passive` corresponds to a *symbiont device* +— per NIST SP 800-193 §3.4, a device that lacks the capability to perform its +own firmware verification and relies on an external RoT (the eRoT, or an +intermediate RoT in a tiered model) to do so on its behalf. + +### `FailurePolicy` + +Determines what the orchestrator does when a component's firmware fails +verification. + +``` +Required — attempt recovery; re-walk the chain after `Restored`; latch + `Locked` if the retry cap is reached +Isolable — hold the component in reset; advance past it; continue the walk +Cascading — same as Isolable, and additionally hold in reset any component + whose `depends_on` names this one +``` + +### `RegionId` + +An opaque `u8` that groups components into a *recovery region*. Components +assigned the same `RegionId` must be restored together; when any region member +triggers `Recovering`, the shell issues a joint restore operation for the +entire region. The core treats the id as an equality key only and never +inspects the membership. + ### `ComponentAttrs` -Per-component attributes that combine two orthogonal axes: +Per-component attributes that combine three orthogonal axes: ```rust pub struct ComponentAttrs { - pub kind: ComponentKind, // iRoT gate: Active | Passive - pub required: bool, // failure policy: true = recover, false = skip + pub kind: ComponentKind, // iRoT gate: Active | Passive + pub failure_policy: FailurePolicy, // verification-failure handling + pub recovery_region: RegionId, // which components are recovered together + pub depends_on: Option<ComponentId>, // cascade trigger: held if named component was skipped } ``` -| `kind` | `required` | `VerificationFailed` behaviour | -|---|---|---| -| Active / Passive | `true` | → `Recovering`; component held in reset; chain walk halts | -| Active / Passive | `false` | component held in reset; cursor advances; chain walk continues | +| `failure_policy` | `VerificationFailed` behaviour | +|---|---| +| `Required` | → `Recovering`; component held in reset; chain walk halts until restored | +| `Isolable` | component held in reset; cursor advances; chain walk continues; no cascade to dependents | +| `Cascading` | component held in reset; added to `Rot.held`; any component whose `depends_on` matches this id is also held in reset and skipped; cursor advances; chain walk continues | -A `required: false` component that fails verification is **never** released from -reset — releasing a component whose firmware failed verification would mean -running untrusted code, which breaks the trust invariant regardless of the -recovery policy. +A component that fails verification is **never** released from reset regardless +of its failure policy — releasing a component whose firmware failed verification +would mean running untrusted code, which breaks the trust invariant. + +`depends_on` is only meaningful when the named component has `FailurePolicy::Cascading`. +Only `Cascading`-failed components (and their transitively cascade-skipped dependents) +are added to `Rot.held`. Before emitting `ReadFirmware` for any component, the core +checks whether its `depends_on` names a component in `Rot.held`; if so, the depending +component is also held in reset and added to `held` without emitting `ReadFirmware` or +`VerifyFirmware`. This check repeats for each newly-held component's successors until +no further cascade is triggered. An `Isolable` failure skips only the failed component; +it does not populate `held` and causes no downstream cascade. Convenience constructors: `ComponentAttrs::active_required()`, -`passive_required()`, `active_optional()`, `passive_optional()`. +`passive_required()`, `active_isolable()`, `passive_isolable()`, +`active_cascading()`, `passive_cascading()`. ### `ComponentId` @@ -88,6 +127,7 @@ | `VerificationPassed(ComponentId)` | shell → core | The eRoT-side check passed: signature and SVN valid. | | `VerificationFailed(ComponentId)` | shell → core | The eRoT-side check failed: image rejected. | | `ComponentReady(ComponentId)` | shell → core | An `Active` component's integrated iRoT has finished its local verification and the component is operational (e.g. MCTP channel established). | +| `Timeout(ComponentId)` | shell → core | The shell's watchdog fired: the named `Active` component did not deliver `ComponentReady` within the platform-policy window. The shell arms the watchdog after emitting `ReleaseReset` and cancels it on `ComponentReady`. Treated as a verification failure for recovery purposes. | ### Effects the core emits for verification work @@ -107,7 +147,7 @@ ### Active → Passive (happy path) ``` -chain: [(C0, {Active, required}), (C1, {Passive, required})] +chain: [(C0, {Active, Required}), (C1, {Passive, Required})] VerifyingPlatform (entry): emit ReadFirmware(C0) @@ -129,10 +169,10 @@ chain done → Ready ``` -### Optional component failure (skip, continue) +### Isolable component failure (skip, continue) ``` -chain: [(BMC, {Active, required}), (NIC, {Passive, optional})] +chain: [(BMC, {Active, Required}), (NIC, {Passive, Isolable})] VerificationPassed(BMC): emit ReleaseReset(BMC) @@ -140,7 +180,7 @@ emit VerifyFirmware(NIC) awaiting = Some(BMC) → AwaitingReady -VerificationFailed(NIC): ← NIC firmware rejected; optional → skip +VerificationFailed(NIC): ← NIC firmware rejected; Isolable → skip NIC stays held in reset cursor advances past end awaiting is still Some(BMC) → stay in AwaitingReady @@ -154,7 +194,7 @@ This matches the CSA single-node boot sequence. ``` -chain: [(BMC, {Active, required}), (HOST, {Active, required}), (NIC, {Passive, optional})] +chain: [(BMC, {Active, Required}), (HOST, {Active, Required}), (NIC, {Passive, Isolable})] VerifyingPlatform (entry): emit ReadFirmware(BMC) @@ -189,7 +229,7 @@ If NIC fails verification instead: ``` -VerificationFailed(NIC): ← NIC optional → skip; NIC stays held in reset +VerificationFailed(NIC): ← NIC Isolable → skip; NIC stays held in reset cursor = 3 (past end) awaiting = None (already cleared by ComponentReady(HOST)) → Ready @@ -215,6 +255,16 @@ eRoT firmware read of the next. The two checks are independent (different hardware paths), so the overlap is safe. +**Deliberate divergence from the CSA boot sequence diagram.** The CSA diagram +shows strictly sequential ordering — for example, the BMC MCTP channel is +established before the eRoT begins reading the CPU firmware. The +speculative-read pattern departs from this: the eRoT starts the next +component's firmware read as soon as the current one is released from reset, +without waiting for `ComponentReady`. The trust guarantee is fully preserved: +`ReleaseReset` for the next component is never emitted until that component's +own `VerificationPassed` arrives. The overlap reduces boot time on real +hardware where iRoT initialization can take several seconds. + --- ## 5. The Platform Boundary @@ -229,7 +279,9 @@ | Verify signature / SVN | emits `VerifyFirmware(id)` | executes: eRoT checks against RIM/PFM; responds with `VerificationPassed` or `VerificationFailed` | | Release from reset | emits `ReleaseReset(id)` | executes: eRoT drives reset GPIO or equivalent | | Detect iRoT readiness | waits for `ComponentReady(id)` event | observes: integrated iRoT signals readiness (MCTP channel-up, GPIO, etc.); calls `dispatch` | -| Required vs optional failure policy | checks `attrs.required` in handler | none — policy is encoded in the chain at startup | +| Per-component failure policy | checks `attrs.failure_policy` in handler | none — policy is encoded in the chain at startup | +| Cascade-skip evaluation | checks `attrs.depends_on` against `Rot.held` before emitting `ReadFirmware` | encodes the dependency graph at chain-build time | +| Recovery region membership | reads `attrs.recovery_region` when entering `Recovering` | assigns each component to a region at chain-build time | --- @@ -242,3 +294,10 @@ `Operational` superstate, not part of the boot-time verification chain. - **Firmware update verification** (`AuthenticateUpdate`): handled in the `Updating` state, distinct from boot-time chain verification. +- **Multiple intermediate boot-progress checkpoints per component**: the CSA + architecture allows platform policy to require multiple intermediate + readiness signals before a component is considered fully booted. This model + simplifies that to a single `ComponentReady` event per `Active` component. + The shell is responsible for aggregating any intermediate signals and + delivering `ComponentReady` only once all platform-policy checkpoints have + been satisfied.
diff --git a/docs/src/design/orchestrator/orchestrator-overview.md b/docs/src/design/orchestrator/orchestrator-overview.md index c782f50..a85643f 100644 --- a/docs/src/design/orchestrator/orchestrator-overview.md +++ b/docs/src/design/orchestrator/orchestrator-overview.md
@@ -31,7 +31,8 @@ **Feedback as data.** Internal follow-up signals (e.g. the retry-cap lockdown `RecoveryFailed`) are emitted as `Effect::Emit(event)`. The orchestrator queues and handles them immediately, making them visible in the effect trace rather than -hiding them as implicit state changes. +hiding them as implicit state changes. See the +[`Recovering` state](./orchestrator-machine.md#recovering) for a worked example. **Board-supplied policy.** The core hard-codes no deployment-specific values. The shell supplies the trust chain (component ids, kinds, and required/optional @@ -47,4 +48,45 @@ | eRoT holds component in reset until firmware verified | `VerifyingPlatform` emits `ReleaseReset` only on `VerificationPassed` | | Component with Caliptra iRoT requires two independent checks | `ComponentKind::Active` → `AwaitingReady` until `ComponentReady` | | Passive component (no iRoT): eRoT check only | `ComponentKind::Passive` → advance immediately after `ReleaseReset` | -| Optional component: failure skips, not blocks | `ComponentAttrs::required = false` → advance without `Recovering` | +| Isolable component: failure skips, not blocks | `FailurePolicy::Isolable` → skip (held in reset); advance without `Recovering`; no cascade | +| Cascading skip: failure also holds dependents | `FailurePolicy::Cascading` + `ComponentAttrs::depends_on` → cascade-skip in `Rot.held` | +| Boot-progress watchdog: component must signal readiness in time | `Timeout(ComponentId)` event → `AwaitingReady` → `Recovering` | +| Recovery scope groups components that restore together | `ComponentAttrs::recovery_region` (`RegionId`) → shell restores full region on `RestoreGoldenImage` | + +## Applicability Across Admissible Architectures + +The orchestrator is **eRoT-scoped**: one instance runs per discrete eRoT chip +running OpenPRoT firmware. The same crate and binary are used at every such +tier — the only difference between deployments is the chain configuration +supplied at startup. eRoT chips not running OpenPRoT (e.g. a third-party AMC +or a legacy DC-SCM implementation) appear as opaque `ComponentId` entries in +the chain of the nearest OpenPRoT eRoT above them. + +**Model 1 — Module** (single PCIe add-in card): a discrete eRoT is optional. +When present, it runs one orchestrator instance with a short chain containing +the card's SoC. When absent, the module relies on the parent node's eRoT +instance to verify and release it as a `ComponentId` in the node's chain. + +**Model 2 — Single Node Compute**: the DC-SCM discrete eRoT runs one +orchestrator instance whose chain covers all critical node devices (CPU, BMC, +NIC, storage). This is the canonical single-instance deployment. + +**Model 3 — Complex Heterogeneous Compute**: the three-tier hierarchy maps to +independent orchestrator instances at each tier: + +- Each **AMC / EAM** (subsystem eRoT) runs one instance whose chain covers the + GPU or AI accelerator devices it manages. +- The **DC-SCM node eRoT** runs one instance whose chain covers CPUs, BMC, + SmartNICs, storage, and the AMC/EAM chips themselves. + +The node eRoT treats each AMC/EAM as just another `ComponentId` — `Active` if +the AMC has its own integrated iRoT, `Passive` if not. It has no visibility +into the AMC's internal chain walk; it only observes the AMC's +`VerificationPassed` / `ComponentReady` signals, like any other component. + +| Admissible Architecture | Orchestrator instances | Chain scope per instance | +|---|---|---| +| Module (eRoT present) | 1 | Card SoC | +| Module (no eRoT) | 0 | — (verified by parent node eRoT) | +| Single Node Compute | 1 | All node critical devices | +| Complex Heterogeneous Compute | 1 per subsystem eRoT + 1 node eRoT | Subsystem devices / all node devices including subsystem eRoTs |
diff --git a/services/orchestrator/sm/src/lib.rs b/services/orchestrator/sm/src/lib.rs index 59388f9..b5277bb 100644 --- a/services/orchestrator/sm/src/lib.rs +++ b/services/orchestrator/sm/src/lib.rs
@@ -970,4 +970,223 @@ assert_eq!(state, State::Recovering); assert!(effects.contains(&Effect::RestoreGoldenImage(C0))); } + + /// Boot-time VerificationFailed on a required component → Recovering. + /// (Distinct from CorruptionDetected: this is a failed eRoT-side check + /// before the component is ever released from reset.) + #[test] + fn boot_failure_required_enters_recovering() { + let (effects, state) = drive( + passive_required(&[C0, C1]), + &[BOOT, Event::VerificationFailed(C0)], + ); + assert_eq!(state, State::Recovering); + assert!(effects.contains(&Effect::RestoreGoldenImage(C0))); + // Component must never be released when its eRoT check failed. + assert!(!effects.contains(&Effect::ReleaseReset(C0))); + } + + /// Full boot-failure recovery cycle: VerificationFailed → Recovering → + /// Restored → re-walk from top → VerificationPassed → Ready. + #[test] + fn boot_failure_recovery_cycle_completes() { + let (effects, state) = drive( + passive_required(&[C0]), + &[ + BOOT, + Event::VerificationFailed(C0), + Event::Restored(C0), + Event::VerificationPassed(C0), + ], + ); + assert_eq!(state, State::Ready); + assert!(effects.contains(&Effect::RestoreGoldenImage(C0))); + // ReleaseReset only after the recovery re-walk passes. + assert!(effects.contains(&Effect::ReleaseReset(C0))); + } + + /// VerificationFailed (required) on a speculative check while the machine + /// is in AwaitingReady → enters Recovering without releasing the component. + #[test] + fn required_failure_in_awaiting_ready_enters_recovering() { + let (effects, state) = drive( + chain(&[ + (C0, ComponentAttrs::active_required()), + (C1, ComponentAttrs::passive_required()), + ]), + &[ + BOOT, + Event::VerificationPassed(C0), // → AwaitingReady; spec check of C1 starts + Event::VerificationFailed(C1), // required → Recovering + ], + ); + assert_eq!(state, State::Recovering); + assert!(effects.contains(&Effect::RestoreGoldenImage(C1))); + assert!(!effects.contains(&Effect::ReleaseReset(C1))); + } + + /// CorruptionDetected while in AwaitingReady (required component) → + /// Recovering via the Operational superstate handler. + #[test] + fn corruption_in_awaiting_ready_triggers_recovery() { + let (effects, state) = drive( + chain(&[ + (C0, ComponentAttrs::active_required()), + (C1, ComponentAttrs::passive_required()), + ]), + &[ + BOOT, + Event::VerificationPassed(C0), // → AwaitingReady + Event::CorruptionDetected(C0), + ], + ); + assert_eq!(state, State::Recovering); + assert!(effects.contains(&Effect::RestoreGoldenImage(C0))); + } + + /// CorruptionDetected while in Updating (required component) → Recovering + /// via the Operational superstate handler. + #[test] + fn corruption_in_updating_triggers_recovery() { + let (effects, state) = drive( + passive_required(&[C0]), + &[ + BOOT, + Event::VerificationPassed(C0), + Event::UpdateRequest, + Event::CorruptionDetected(C0), + ], + ); + assert_eq!(state, State::Recovering); + assert!(effects.contains(&Effect::RestoreGoldenImage(C0))); + } + + /// UpdateVerified activates the staged image and returns to Ready. + /// (Complements update_rollback_is_not_recovery which tests UpdateRejected.) + #[test] + fn update_verified_activates_update() { + let (effects, state) = drive( + passive_required(&[C0]), + &[ + BOOT, + Event::VerificationPassed(C0), + Event::UpdateRequest, + Event::UpdateVerified, + ], + ); + assert_eq!(state, State::Ready); + assert!(effects.contains(&Effect::ActivateUpdate)); + assert!(!effects.contains(&Effect::DiscardStaged)); + assert!(!effects.contains(&Effect::RestoreGoldenImage(C0))); + } + + /// Locked is a terminal state: no effects are produced in response to any + /// event after the machine latches. + #[test] + fn locked_is_terminal() { + let mut c: heapless::Vec<(ComponentId, ComponentAttrs), CAPACITY> = + heapless::Vec::new(); + c.push((C0, ComponentAttrs::passive_required())).unwrap(); + // max_retry = 1 so the first failed restore latches immediately. + let mut orch = Orchestrator::new(c, 1); + let mut effects: Vec<Effect> = Vec::new(); + + for ev in [BOOT, Event::VerificationFailed(C0), Event::Restored(C0)] { + orch.dispatch_with(ev, |e| effects.push(e)); + } + assert_eq!(orch.state(), State::Locked); + + let count_before = effects.len(); + for ev in [ + BOOT, + Event::VerificationPassed(C0), + Event::AttestationChallenge, + Event::UpdateRequest, + Event::CorruptionDetected(C0), + ] { + orch.dispatch_with(ev, |e| effects.push(e)); + } + assert_eq!( + effects.len(), + count_before, + "Locked state must produce no effects" + ); + } + + /// An optional component at the head of the chain can fail and the walk + /// continues to the remaining required components. + #[test] + fn optional_first_component_skipped_walk_continues() { + let (effects, state) = drive( + chain(&[ + (C0, ComponentAttrs::passive_optional()), + (C1, ComponentAttrs::passive_required()), + ]), + &[ + BOOT, + Event::VerificationFailed(C0), // optional → skip C0 + Event::VerificationPassed(C1), + ], + ); + assert_eq!(state, State::Ready); + assert!(!effects.contains(&Effect::ReleaseReset(C0))); + assert!(effects.contains(&Effect::ReleaseReset(C1))); + assert!(!effects.contains(&Effect::RestoreGoldenImage(C0))); + } + + /// The speculative read emits ReleaseReset · ReadFirmware · VerifyFirmware + /// all in the same handler as VerificationPassed for an Active component, + /// before ComponentReady has arrived. Verifies both presence and order. + #[test] + fn speculative_read_effects_are_emitted_together() { + let mut orch = Orchestrator::<CAPACITY>::new( + chain(&[ + (C0, ComponentAttrs::active_required()), + (C1, ComponentAttrs::passive_required()), + ]), + MAX_RETRY, + ); + let mut effects: Vec<Effect> = Vec::new(); + + orch.dispatch_with(BOOT, |e| effects.push(e)); + assert_eq!( + effects, + std::vec![Effect::ReadFirmware(C0), Effect::VerifyFirmware(C0)], + ); + + effects.clear(); + orch.dispatch_with(Event::VerificationPassed(C0), |e| effects.push(e)); + // All three effects emitted in the same handler, before ComponentReady. + assert_eq!( + effects, + std::vec![ + Effect::ReleaseReset(C0), + Effect::ReadFirmware(C1), + Effect::VerifyFirmware(C1), + ], + ); + assert_eq!(orch.state(), State::AwaitingReady); + } + + /// A chain with a single Active component goes directly to Ready on + /// VerificationPassed — no AwaitingReady, no ComponentReady required. + /// This exercises the `chain done` branch of VerifyingPlatform for an + /// Active component (distinct from the multi-component Active path which + /// transitions to AwaitingReady). + #[test] + fn single_active_chain_goes_directly_to_ready() { + let mut c: heapless::Vec<(ComponentId, ComponentAttrs), CAPACITY> = + heapless::Vec::new(); + c.push((C0, ComponentAttrs::active_required())).unwrap(); + let (effects, state) = drive(c, &[BOOT, Event::VerificationPassed(C0)]); + assert_eq!(state, State::Ready); + assert_eq!( + effects, + std::vec![ + Effect::ReadFirmware(C0), + Effect::VerifyFirmware(C0), + Effect::ReleaseReset(C0), + ], + ); + } }