cpustress+orchestrator: serial CPU/RAM passes + silent-skip guard
Orion's run (log 20:49 → 20:54) shipped GREEN while silently skipping CPUStress. Two compounding bugs: 1. CPUStress ran --cpu N AND --vm N --vm-bytes 90% concurrently. On a 4-core 8 GiB N95, that's 360% RAM overcommit; the OOM-killer fired, usually on the agent itself. Replaced with two sequential passes — CPU (all methods, --verify) for 3 min, then RAM (--vm 1, --vm-bytes capped to MemAvailable − 1.5 GiB, floor 256 MiB, --verify) for 3 min. Each pass now also asserts elapsed ≥ target − 2s so a premature clean exit counts as failure instead of a silent pass. 2. On systemd-restart after the OOM, the agent hardcoded nextStage := "Inventory" and re-ran it. The orchestrator's /result handler advances run state via TriggerStageCompleted against the *current* RunState, not against body.Stage — so an Inventory result posted while the run was in StateCPUStress silently advanced CPUStress → Storage and marked CPUStress passed without it ever running. Two-layer defense for #2: - agent-side: /claim response now carries current_state; agent resumes at the matching stage on a re-claim (happy path). - server-side: new TriggerStageMismatch + StageNameForState helper backstop. If body.Stage doesn't match the run's current stage, /result parks the run in FailedHolding with failed_stage labeled "<got> (expected <expected>)" and returns 409. Other stages audited for similar unbounded concurrency — none found; only CPUStress was unsafe. Tests: - cpustress_test.go — parseMemAvailable parses real meminfo, errors on missing/malformed; cap calc hits floor on tiny boxes, uses 1.5 GiB headroom on normal/huge boxes. - statemachine_test.go — TriggerStageMismatch lands at FailedHolding from every stage state and is rejected from pre-stage/terminal states; StageNameForState round-trips the stageStates map. - agent_handlers_test.go — TestResult_RejectsMismatchedStage proves the Orion scenario now 409s + FailedHolding; TestResult_AcceptsMatchingStage proves the guard doesn't break the happy path. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -74,6 +74,70 @@ func TestTriggerAgentClaimedFromWaitingReboot(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestTriggerStageMismatch asserts the silent-skip guard: from every
|
||||
// stage-execution state, a mismatch lands the run in FailedHolding, and
|
||||
// from non-stage states (pre-stages, terminals) the trigger is rejected.
|
||||
func TestTriggerStageMismatch(t *testing.T) {
|
||||
stageStates := []model.RunState{
|
||||
model.StateInventoryCheck,
|
||||
model.StateSpecValidate,
|
||||
model.StateSMART,
|
||||
model.StateCPUStress,
|
||||
model.StateStorage,
|
||||
model.StateNetwork,
|
||||
model.StateGPU,
|
||||
model.StatePSU,
|
||||
model.StateReporting,
|
||||
}
|
||||
for _, from := range stageStates {
|
||||
got, err := orchestrator.Next(from, orchestrator.TriggerStageMismatch)
|
||||
if err != nil {
|
||||
t.Fatalf("StageMismatch from %q: %v", from, err)
|
||||
}
|
||||
if got != model.StateFailedHolding {
|
||||
t.Fatalf("StageMismatch from %q = %q, want FailedHolding", from, got)
|
||||
}
|
||||
}
|
||||
for _, bad := range []model.RunState{
|
||||
model.StateRegistered, model.StateQueued, model.StateBooting,
|
||||
model.StateWaitingReboot, model.StateCompleted, model.StateFailedHolding,
|
||||
} {
|
||||
if _, err := orchestrator.Next(bad, orchestrator.TriggerStageMismatch); err == nil {
|
||||
t.Fatalf("StageMismatch from %q: expected error", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestStageNameForState round-trips the stageStates map: every name in
|
||||
// StateForStage must come back from StageNameForState, and non-stage
|
||||
// run states return empty.
|
||||
func TestStageNameForState(t *testing.T) {
|
||||
pairs := map[string]model.RunState{
|
||||
"Inventory": model.StateInventoryCheck,
|
||||
"SpecValidate": model.StateSpecValidate,
|
||||
"SMART": model.StateSMART,
|
||||
"CPUStress": model.StateCPUStress,
|
||||
"Storage": model.StateStorage,
|
||||
"Network": model.StateNetwork,
|
||||
"GPU": model.StateGPU,
|
||||
"PSU": model.StatePSU,
|
||||
"Reporting": model.StateReporting,
|
||||
}
|
||||
for name, state := range pairs {
|
||||
if got := orchestrator.StageNameForState(state); got != name {
|
||||
t.Errorf("StageNameForState(%q) = %q, want %q", state, got, name)
|
||||
}
|
||||
}
|
||||
for _, s := range []model.RunState{
|
||||
model.StateRegistered, model.StateQueued, model.StateBooting,
|
||||
model.StateWaitingReboot, model.StateCompleted, model.StateFailedHolding,
|
||||
} {
|
||||
if got := orchestrator.StageNameForState(s); got != "" {
|
||||
t.Errorf("StageNameForState(%q) = %q, want empty", s, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNextStageWalk(t *testing.T) {
|
||||
// Walking StageCompleted from each stage should land on the next
|
||||
// one in the canonical order, and from Reporting onto Completed.
|
||||
|
||||
Reference in New Issue
Block a user