cpustress+orchestrator: serial CPU/RAM passes + silent-skip guard
Orion's run (log 20:49 → 20:54) shipped GREEN while silently skipping CPUStress. Two compounding bugs: 1. CPUStress ran --cpu N AND --vm N --vm-bytes 90% concurrently. On a 4-core 8 GiB N95, that's 360% RAM overcommit; the OOM-killer fired, usually on the agent itself. Replaced with two sequential passes — CPU (all methods, --verify) for 3 min, then RAM (--vm 1, --vm-bytes capped to MemAvailable − 1.5 GiB, floor 256 MiB, --verify) for 3 min. Each pass now also asserts elapsed ≥ target − 2s so a premature clean exit counts as failure instead of a silent pass. 2. On systemd-restart after the OOM, the agent hardcoded nextStage := "Inventory" and re-ran it. The orchestrator's /result handler advances run state via TriggerStageCompleted against the *current* RunState, not against body.Stage — so an Inventory result posted while the run was in StateCPUStress silently advanced CPUStress → Storage and marked CPUStress passed without it ever running. Two-layer defense for #2: - agent-side: /claim response now carries current_state; agent resumes at the matching stage on a re-claim (happy path). - server-side: new TriggerStageMismatch + StageNameForState helper backstop. If body.Stage doesn't match the run's current stage, /result parks the run in FailedHolding with failed_stage labeled "<got> (expected <expected>)" and returns 409. Other stages audited for similar unbounded concurrency — none found; only CPUStress was unsafe. Tests: - cpustress_test.go — parseMemAvailable parses real meminfo, errors on missing/malformed; cap calc hits floor on tiny boxes, uses 1.5 GiB headroom on normal/huge boxes. - statemachine_test.go — TriggerStageMismatch lands at FailedHolding from every stage state and is rejected from pre-stage/terminal states; StageNameForState round-trips the stageStates map. - agent_handlers_test.go — TestResult_RejectsMismatchedStage proves the Orion scenario now 409s + FailedHolding; TestResult_AcceptsMatchingStage proves the guard doesn't break the happy path. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -180,6 +180,15 @@ func (a *Agent) Claim(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
// Re-fetch run state: the Transition above may have advanced us from
|
||||
// Booting → InventoryCheck, and we want to hand that fresh state to
|
||||
// the agent so a re-claim after a crash resumes at the stored state
|
||||
// instead of silently replaying Inventory.
|
||||
currentState := run.State
|
||||
if fresh, err := a.Runs.Get(r.Context(), runID); err == nil && fresh != nil {
|
||||
currentState = fresh.State
|
||||
}
|
||||
|
||||
log.Printf("agent claimed: run=%d agent_ip=%s", runID, agentIP)
|
||||
if a.Logs != nil {
|
||||
if w, err := a.Logs.WriterFor(runID); err == nil {
|
||||
@@ -213,6 +222,7 @@ func (a *Agent) Claim(w http.ResponseWriter, r *http.Request) {
|
||||
"expected_disks": expectedDisks,
|
||||
"iperf_port": iperfPort,
|
||||
"non_destructive": run.NonDestructive,
|
||||
"current_state": string(currentState),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -411,6 +421,42 @@ func (a *Agent) Result(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// Silent-skip guard. Orchestrator advances the run state via
|
||||
// TriggerStageCompleted against the *current* state, not against
|
||||
// body.Stage — so an Inventory result posted while the run is in
|
||||
// StateCPUStress would silently advance CPUStress → Storage and mark
|
||||
// CPUStress as passed without it ever running. That's exactly what
|
||||
// happened on Orion when the agent OOM-crashed mid-CPUStress,
|
||||
// systemd restarted it, and the restarted agent (which hardcoded
|
||||
// "Inventory" as its first stage) re-ran Inventory and reported it.
|
||||
// Guard: if body.Stage doesn't match the stage the run is currently
|
||||
// in, park the run in FailedHolding so the operator can investigate
|
||||
// rather than trusting the claim and cascading silent passes.
|
||||
expectedStage := orchestrator.StageNameForState(run.State)
|
||||
if expectedStage != "" && body.Stage != expectedStage {
|
||||
failedLabel := fmt.Sprintf("%s (expected %s)", body.Stage, expectedStage)
|
||||
if err := a.Runs.SetFailedStage(r.Context(), runID, failedLabel); err != nil {
|
||||
log.Printf("result: set failed stage on mismatch run %d: %v", runID, err)
|
||||
}
|
||||
if _, err := a.Runner.Transition(r.Context(), runID, orchestrator.TriggerStageMismatch); err != nil {
|
||||
log.Printf("result: stage-mismatch transition run %d: %v", runID, err)
|
||||
}
|
||||
hostName := a.hostNameFor(r.Context(), run.HostID)
|
||||
a.dispatchEvent(notify.Event{
|
||||
Kind: notify.KindStageFailed,
|
||||
Severity: notify.SeverityCritical,
|
||||
RunID: runID,
|
||||
HostName: hostName,
|
||||
Title: fmt.Sprintf("[vetting] %s stage mismatch: %s", hostName, body.Stage),
|
||||
Body: fmt.Sprintf("Run %d reported stage %s while orchestrator expected %s — parked in FailedHolding to prevent silent skip.",
|
||||
runID, body.Stage, expectedStage),
|
||||
URL: a.runLinkURL(runID),
|
||||
})
|
||||
log.Printf("result: stage mismatch run=%d got=%s expected=%s — parked", runID, body.Stage, expectedStage)
|
||||
http.Error(w, "stage mismatch: got "+body.Stage+", expected "+expectedStage, http.StatusConflict)
|
||||
return
|
||||
}
|
||||
|
||||
stageState := model.StagePassed
|
||||
if !body.Passed {
|
||||
stageState = model.StageFailed
|
||||
|
||||
Reference in New Issue
Block a user