cpustress+orchestrator: serial CPU/RAM passes + silent-skip guard

Orion's run (log 20:49 → 20:54) shipped GREEN while silently skipping
CPUStress. Two compounding bugs:

1. CPUStress ran --cpu N AND --vm N --vm-bytes 90% concurrently.
   On a 4-core 8 GiB N95, that's 360% RAM overcommit; the OOM-killer
   fired, usually on the agent itself. Replaced with two sequential
   passes — CPU (all methods, --verify) for 3 min, then RAM (--vm 1,
   --vm-bytes capped to MemAvailable − 1.5 GiB, floor 256 MiB, --verify)
   for 3 min. Each pass now also asserts elapsed ≥ target − 2s so a
   premature clean exit counts as failure instead of a silent pass.

2. On systemd-restart after the OOM, the agent hardcoded nextStage :=
   "Inventory" and re-ran it. The orchestrator's /result handler
   advances run state via TriggerStageCompleted against the *current*
   RunState, not against body.Stage — so an Inventory result posted
   while the run was in StateCPUStress silently advanced CPUStress →
   Storage and marked CPUStress passed without it ever running.

Two-layer defense for #2:
- agent-side: /claim response now carries current_state; agent resumes
  at the matching stage on a re-claim (happy path).
- server-side: new TriggerStageMismatch + StageNameForState helper
  backstop. If body.Stage doesn't match the run's current stage, /result
  parks the run in FailedHolding with failed_stage labeled
  "<got> (expected <expected>)" and returns 409.

Other stages audited for similar unbounded concurrency — none found;
only CPUStress was unsafe.

Tests:
- cpustress_test.go — parseMemAvailable parses real meminfo, errors on
  missing/malformed; cap calc hits floor on tiny boxes, uses 1.5 GiB
  headroom on normal/huge boxes.
- statemachine_test.go — TriggerStageMismatch lands at FailedHolding
  from every stage state and is rejected from pre-stage/terminal
  states; StageNameForState round-trips the stageStates map.
- agent_handlers_test.go — TestResult_RejectsMismatchedStage proves
  the Orion scenario now 409s + FailedHolding; TestResult_AcceptsMatchingStage
  proves the guard doesn't break the happy path.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

internal/orchestrator/statemachine.go

@@ -16,6 +16,7 @@ const (
 	TriggerPXEObserved      Trigger = "PXEObserved"      // iPXE fetched cmdline for MAC
 	TriggerAgentClaimed     Trigger = "AgentClaimed"     // agent POSTed /claim with valid token
 	TriggerStageFailed      Trigger = "StageFailed"      // a stage reported failure
+	TriggerStageMismatch    Trigger = "StageMismatch"    // agent reported a stage that doesn't match current run state (silent-skip guard)
 	TriggerStageCompleted   Trigger = "StageCompleted"   // a stage reported success → advance
 	TriggerAllStagesPassed  Trigger = "AllStagesPassed"  // final stage passed
 	TriggerOperatorReleased  Trigger = "OperatorReleased"  // user clicked Release on a held run
@@ -65,6 +66,7 @@ var table = map[Trigger]transition{
 	TriggerPXEObserved:      {from: []model.RunState{model.StateWaitingReboot, model.StateWaitingWoL, model.StateBooting}, to: model.StateBooting},
 	TriggerAgentClaimed:     {from: []model.RunState{model.StateBooting, model.StateWaitingReboot, model.StateWaitingWoL}, to: model.StateInventoryCheck},
 	TriggerStageFailed:       {from: allActiveStates(), to: model.StateFailedHolding},
+	TriggerStageMismatch:     {from: stageExecutionStates(), to: model.StateFailedHolding},
 	TriggerAllStagesPassed:   {from: []model.RunState{model.StateReporting}, to: model.StateCompleted},
 	TriggerOperatorReleased:  {from: []model.RunState{model.StateFailedHolding}, to: model.StateReleased},
 	TriggerOperatorCancelled: {from: allActiveStates(), to: model.StateCancelled},
@@ -111,6 +113,21 @@ func StateForStage(name string) (model.RunState, bool) {
 	return s, ok
 }
 
+// StageNameForState is the inverse of StateForStage: given a run state
+// that maps to a stage, returns the stage name (e.g. StateCPUStress →
+// "CPUStress"). Empty string when the state isn't a stage-execution
+// state (Queued, Booting, FailedHolding, etc.). Used by /result to
+// detect when an agent submitted a stage name that doesn't match where
+// the orchestrator thinks the run is — the silent-skip guard.
+func StageNameForState(s model.RunState) string {
+	for name, state := range stageStates {
+		if state == s {
+			return name
+		}
+	}
+	return ""
+}
+
 func nextStageState(current model.RunState) (model.RunState, error) {
 	for i, s := range stageOrder {
 		if s == current {
@@ -131,3 +148,11 @@ func allActiveStates() []model.RunState {
 		model.StateGPU, model.StatePSU, model.StateReporting,
 	}
 }
+
+// stageExecutionStates returns only the stage-execution states — no
+// pre-stages, no terminals. Used as the valid "from" set for
+// TriggerStageMismatch: it's nonsensical to fire a stage-mismatch from
+// Queued or Booting because no stage result should arrive then.
+func stageExecutionStates() []model.RunState {
+	return append([]model.RunState(nil), stageOrder...)
+}

internal/orchestrator/statemachine_test.go

@@ -74,6 +74,70 @@ func TestTriggerAgentClaimedFromWaitingReboot(t *testing.T) {
 	}
 }
 
+// TestTriggerStageMismatch asserts the silent-skip guard: from every
+// stage-execution state, a mismatch lands the run in FailedHolding, and
+// from non-stage states (pre-stages, terminals) the trigger is rejected.
+func TestTriggerStageMismatch(t *testing.T) {
+	stageStates := []model.RunState{
+		model.StateInventoryCheck,
+		model.StateSpecValidate,
+		model.StateSMART,
+		model.StateCPUStress,
+		model.StateStorage,
+		model.StateNetwork,
+		model.StateGPU,
+		model.StatePSU,
+		model.StateReporting,
+	}
+	for _, from := range stageStates {
+		got, err := orchestrator.Next(from, orchestrator.TriggerStageMismatch)
+		if err != nil {
+			t.Fatalf("StageMismatch from %q: %v", from, err)
+		}
+		if got != model.StateFailedHolding {
+			t.Fatalf("StageMismatch from %q = %q, want FailedHolding", from, got)
+		}
+	}
+	for _, bad := range []model.RunState{
+		model.StateRegistered, model.StateQueued, model.StateBooting,
+		model.StateWaitingReboot, model.StateCompleted, model.StateFailedHolding,
+	} {
+		if _, err := orchestrator.Next(bad, orchestrator.TriggerStageMismatch); err == nil {
+			t.Fatalf("StageMismatch from %q: expected error", bad)
+		}
+	}
+}
+
+// TestStageNameForState round-trips the stageStates map: every name in
+// StateForStage must come back from StageNameForState, and non-stage
+// run states return empty.
+func TestStageNameForState(t *testing.T) {
+	pairs := map[string]model.RunState{
+		"Inventory":    model.StateInventoryCheck,
+		"SpecValidate": model.StateSpecValidate,
+		"SMART":        model.StateSMART,
+		"CPUStress":    model.StateCPUStress,
+		"Storage":      model.StateStorage,
+		"Network":      model.StateNetwork,
+		"GPU":          model.StateGPU,
+		"PSU":          model.StatePSU,
+		"Reporting":    model.StateReporting,
+	}
+	for name, state := range pairs {
+		if got := orchestrator.StageNameForState(state); got != name {
+			t.Errorf("StageNameForState(%q) = %q, want %q", state, got, name)
+		}
+	}
+	for _, s := range []model.RunState{
+		model.StateRegistered, model.StateQueued, model.StateBooting,
+		model.StateWaitingReboot, model.StateCompleted, model.StateFailedHolding,
+	} {
+		if got := orchestrator.StageNameForState(s); got != "" {
+			t.Errorf("StageNameForState(%q) = %q, want empty", s, got)
+		}
+	}
+}
+
 func TestNextStageWalk(t *testing.T) {
 	// Walking StageCompleted from each stage should land on the next
 	// one in the canonical order, and from Reporting onto Completed.