Files
Vetting/internal/orchestrator/statemachine_test.go
T
josh 23c689aa5b
CI / Lint + build + test (push) Failing after 1m57s
Release / release (push) Has been cancelled
deep profile + threshold gating + firmware stage + Burn super-stage
Ships all five phases of the deep-profile overhaul together. Runs now
carry a profile (quick/deep/soak); every profile walks the same
11-stage order — Inventory → Firmware → SpecValidate → SMART →
CPUStress → Storage → Network → Burn → GPU → PSU → Reporting —
with only per-stage durations and concurrency scaled.

Phase 1: profiles.ProfileRegistry loaded from vetting.yaml; runs.profile
column + CreateWithProfile; threshold table + evaluator seeded per-run
from the shared vetting.thresholds block; breach flips result at
/sensor + /result.

Phase 2: upgraded CPUStress (stress-ng --cpu-method=all --verify +
EDAC/MCE poll), Storage (fio --verify=md5 + SMART start/end delta),
Network (sustained iperf + /proc/net/dev deltas) with per-profile
knobs from Deps.

Phase 3: Burn super-stage with goroutine fan-out for CPU + memory +
fio + iperf, PSU rails sampled across the Burn window, SensorMux
(2 s flush, 500-sample cap) to absorb backpressure.

Phase 4: Firmware stage + firmware_snapshots table; probes dmidecode
(BIOS), ipmitool (BMC), ethtool -i (NIC), nvme (sysfs + id-ctrl),
lspci (HBA), /proc/cpuinfo (microcode). spec.DiffFirmware folds into
SpecValidate with pin-by-identifier and fan-out-across-component
matching; mismatches park the run in FailedHolding.

Phase 5: profile radio on the host start form, profile chip on the
run header, Firmware section in the HTML report, coverage artifact
uploaded from CI, agent/tests/fakes/ scaffold with Deps.LookPath
seam + stress_ng and dmidecode example fakes.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2026-04-18 22:50:57 -04:00

172 lines
5.5 KiB
Go

package orchestrator_test
import (
"testing"
"vetting/internal/model"
"vetting/internal/orchestrator"
)
func TestNextForOverride(t *testing.T) {
tests := []struct {
name string
from model.RunState
failedStage string
want model.RunState
wantErr bool
}{
{"storage override", model.StateFailedHolding, "Storage", model.StateStorage, false},
{"smart override", model.StateFailedHolding, "SMART", model.StateSMART, false},
{"inventory override", model.StateFailedHolding, "Inventory", model.StateInventoryCheck, false},
{"unknown stage", model.StateFailedHolding, "NotAStage", "", true},
{"not holding", model.StateStorage, "Storage", "", true},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got, err := orchestrator.NextForOverride(tc.from, tc.failedStage)
if tc.wantErr {
if err == nil {
t.Fatalf("expected error, got %q", got)
}
return
}
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got != tc.want {
t.Fatalf("got %q, want %q", got, tc.want)
}
})
}
}
// TestTriggerRebootCommanded exercises the new heartbeat-first trigger:
// Queued → WaitingReboot, and any other current state is an error.
func TestTriggerRebootCommanded(t *testing.T) {
got, err := orchestrator.Next(model.StateQueued, orchestrator.TriggerRebootCommanded)
if err != nil {
t.Fatalf("Queued + RebootCommanded: %v", err)
}
if got != model.StateWaitingReboot {
t.Fatalf("got %q, want %q", got, model.StateWaitingReboot)
}
for _, bad := range []model.RunState{
model.StateRegistered, model.StateBooting, model.StateInventoryCheck, model.StateCompleted,
} {
if _, err := orchestrator.Next(bad, orchestrator.TriggerRebootCommanded); err == nil {
t.Fatalf("RebootCommanded from %q: expected error", bad)
}
}
}
// TestTriggerAgentClaimedFromWaitingReboot: the agent's /claim must
// advance the run out of WaitingReboot (new happy path) AND out of
// legacy WaitingWoL, otherwise live boots wouldn't be recognised.
func TestTriggerAgentClaimedFromWaitingReboot(t *testing.T) {
for _, from := range []model.RunState{model.StateWaitingReboot, model.StateWaitingWoL, model.StateBooting} {
got, err := orchestrator.Next(from, orchestrator.TriggerAgentClaimed)
if err != nil {
t.Fatalf("AgentClaimed from %q: %v", from, err)
}
if got != model.StateInventoryCheck {
t.Fatalf("AgentClaimed from %q = %q, want InventoryCheck", from, got)
}
}
}
// TestTriggerStageMismatch asserts the silent-skip guard: from every
// stage-execution state, a mismatch lands the run in FailedHolding, and
// from non-stage states (pre-stages, terminals) the trigger is rejected.
func TestTriggerStageMismatch(t *testing.T) {
stageStates := []model.RunState{
model.StateInventoryCheck,
model.StateFirmware,
model.StateSpecValidate,
model.StateSMART,
model.StateCPUStress,
model.StateStorage,
model.StateNetwork,
model.StateBurn,
model.StateGPU,
model.StatePSU,
model.StateReporting,
}
for _, from := range stageStates {
got, err := orchestrator.Next(from, orchestrator.TriggerStageMismatch)
if err != nil {
t.Fatalf("StageMismatch from %q: %v", from, err)
}
if got != model.StateFailedHolding {
t.Fatalf("StageMismatch from %q = %q, want FailedHolding", from, got)
}
}
for _, bad := range []model.RunState{
model.StateRegistered, model.StateQueued, model.StateBooting,
model.StateWaitingReboot, model.StateCompleted, model.StateFailedHolding,
} {
if _, err := orchestrator.Next(bad, orchestrator.TriggerStageMismatch); err == nil {
t.Fatalf("StageMismatch from %q: expected error", bad)
}
}
}
// TestStageNameForState round-trips the stageStates map: every name in
// StateForStage must come back from StageNameForState, and non-stage
// run states return empty.
func TestStageNameForState(t *testing.T) {
pairs := map[string]model.RunState{
"Inventory": model.StateInventoryCheck,
"Firmware": model.StateFirmware,
"SpecValidate": model.StateSpecValidate,
"SMART": model.StateSMART,
"CPUStress": model.StateCPUStress,
"Storage": model.StateStorage,
"Network": model.StateNetwork,
"Burn": model.StateBurn,
"GPU": model.StateGPU,
"PSU": model.StatePSU,
"Reporting": model.StateReporting,
}
for name, state := range pairs {
if got := orchestrator.StageNameForState(state); got != name {
t.Errorf("StageNameForState(%q) = %q, want %q", state, got, name)
}
}
for _, s := range []model.RunState{
model.StateRegistered, model.StateQueued, model.StateBooting,
model.StateWaitingReboot, model.StateCompleted, model.StateFailedHolding,
} {
if got := orchestrator.StageNameForState(s); got != "" {
t.Errorf("StageNameForState(%q) = %q, want empty", s, got)
}
}
}
func TestNextStageWalk(t *testing.T) {
// Walking StageCompleted from each stage should land on the next
// one in the canonical order, and from Reporting onto Completed.
chain := []model.RunState{
model.StateInventoryCheck,
model.StateFirmware,
model.StateSpecValidate,
model.StateSMART,
model.StateCPUStress,
model.StateStorage,
model.StateNetwork,
model.StateBurn,
model.StateGPU,
model.StatePSU,
model.StateReporting,
model.StateCompleted,
}
for i := 0; i < len(chain)-1; i++ {
got, err := orchestrator.Next(chain[i], orchestrator.TriggerStageCompleted)
if err != nil {
t.Fatalf("Next(%q): %v", chain[i], err)
}
if got != chain[i+1] {
t.Fatalf("Next(%q) = %q, want %q", chain[i], got, chain[i+1])
}
}
}