62bddac110
A held run sits indefinitely at an SSH prompt waiting for operator
investigation. Previously the only exits were Override (re-enter the
failed stage) or leaving the host on forever — Cancel rejected any
terminal state, including FailedHolding, and there was no button in
the UI anyway.
Add a dedicated exit path:
- statemachine: TriggerOperatorCancelled now accepts FailedHolding
as a valid source, transitioning to Cancelled like any other
live state.
- CancelRun handler: treats FailedHolding as cancellable even
though IsTerminal reports true.
- heartbeat: Cancelled runs fork on FailedStage. Set means the
agent is parked in waitForOverride with no subprocess in
flight, so cmd=reboot tells it to systemctl reboot; the host
falls through iPXE's no-active-run script to the local disk.
Empty FailedStage keeps the pre-existing cmd=cancel_stage path
for mid-stage cancels (kill stage ctx, then power off).
- UI: canCancel now returns true for FailedHolding, and the
run-detail page renders a distinct "Cancel & reboot" button
with a hold-specific confirm message so the action doesn't
look identical to a mid-run cancel.
Tests cover the new statemachine transition, the heartbeat fork
(reboot vs cancel_stage), and keep the pre-existing mid-run cancel
behaviour locked in.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
196 lines
6.5 KiB
Go
196 lines
6.5 KiB
Go
package orchestrator_test
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"vetting/internal/model"
|
|
"vetting/internal/orchestrator"
|
|
)
|
|
|
|
func TestNextForOverride(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
from model.RunState
|
|
failedStage string
|
|
want model.RunState
|
|
wantErr bool
|
|
}{
|
|
{"storage override", model.StateFailedHolding, "Storage", model.StateStorage, false},
|
|
{"smart override", model.StateFailedHolding, "SMART", model.StateSMART, false},
|
|
{"inventory override", model.StateFailedHolding, "Inventory", model.StateInventoryCheck, false},
|
|
{"unknown stage", model.StateFailedHolding, "NotAStage", "", true},
|
|
{"not holding", model.StateStorage, "Storage", "", true},
|
|
}
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
got, err := orchestrator.NextForOverride(tc.from, tc.failedStage)
|
|
if tc.wantErr {
|
|
if err == nil {
|
|
t.Fatalf("expected error, got %q", got)
|
|
}
|
|
return
|
|
}
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if got != tc.want {
|
|
t.Fatalf("got %q, want %q", got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestTriggerRebootCommanded exercises the new heartbeat-first trigger:
|
|
// Queued → WaitingReboot, and any other current state is an error.
|
|
func TestTriggerRebootCommanded(t *testing.T) {
|
|
got, err := orchestrator.Next(model.StateQueued, orchestrator.TriggerRebootCommanded)
|
|
if err != nil {
|
|
t.Fatalf("Queued + RebootCommanded: %v", err)
|
|
}
|
|
if got != model.StateWaitingReboot {
|
|
t.Fatalf("got %q, want %q", got, model.StateWaitingReboot)
|
|
}
|
|
for _, bad := range []model.RunState{
|
|
model.StateRegistered, model.StateBooting, model.StateInventoryCheck, model.StateCompleted,
|
|
} {
|
|
if _, err := orchestrator.Next(bad, orchestrator.TriggerRebootCommanded); err == nil {
|
|
t.Fatalf("RebootCommanded from %q: expected error", bad)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestTriggerAgentClaimedFromWaitingReboot: the agent's /claim must
|
|
// advance the run out of WaitingReboot (new happy path) AND out of
|
|
// legacy WaitingWoL, otherwise live boots wouldn't be recognised.
|
|
func TestTriggerAgentClaimedFromWaitingReboot(t *testing.T) {
|
|
for _, from := range []model.RunState{model.StateWaitingReboot, model.StateWaitingWoL, model.StateBooting} {
|
|
got, err := orchestrator.Next(from, orchestrator.TriggerAgentClaimed)
|
|
if err != nil {
|
|
t.Fatalf("AgentClaimed from %q: %v", from, err)
|
|
}
|
|
if got != model.StateInventoryCheck {
|
|
t.Fatalf("AgentClaimed from %q = %q, want InventoryCheck", from, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestTriggerStageMismatch asserts the silent-skip guard: from every
|
|
// stage-execution state, a mismatch lands the run in FailedHolding, and
|
|
// from non-stage states (pre-stages, terminals) the trigger is rejected.
|
|
func TestTriggerStageMismatch(t *testing.T) {
|
|
stageStates := []model.RunState{
|
|
model.StateInventoryCheck,
|
|
model.StateFirmware,
|
|
model.StateSpecValidate,
|
|
model.StateSMART,
|
|
model.StateCPUStress,
|
|
model.StateStorage,
|
|
model.StateNetwork,
|
|
model.StateBurn,
|
|
model.StateGPU,
|
|
model.StatePSU,
|
|
model.StateReporting,
|
|
}
|
|
for _, from := range stageStates {
|
|
got, err := orchestrator.Next(from, orchestrator.TriggerStageMismatch)
|
|
if err != nil {
|
|
t.Fatalf("StageMismatch from %q: %v", from, err)
|
|
}
|
|
if got != model.StateFailedHolding {
|
|
t.Fatalf("StageMismatch from %q = %q, want FailedHolding", from, got)
|
|
}
|
|
}
|
|
for _, bad := range []model.RunState{
|
|
model.StateRegistered, model.StateQueued, model.StateBooting,
|
|
model.StateWaitingReboot, model.StateCompleted, model.StateFailedHolding,
|
|
} {
|
|
if _, err := orchestrator.Next(bad, orchestrator.TriggerStageMismatch); err == nil {
|
|
t.Fatalf("StageMismatch from %q: expected error", bad)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestStageNameForState round-trips the stageStates map: every name in
|
|
// StateForStage must come back from StageNameForState, and non-stage
|
|
// run states return empty.
|
|
func TestStageNameForState(t *testing.T) {
|
|
pairs := map[string]model.RunState{
|
|
"Inventory": model.StateInventoryCheck,
|
|
"Firmware": model.StateFirmware,
|
|
"SpecValidate": model.StateSpecValidate,
|
|
"SMART": model.StateSMART,
|
|
"CPUStress": model.StateCPUStress,
|
|
"Storage": model.StateStorage,
|
|
"Network": model.StateNetwork,
|
|
"Burn": model.StateBurn,
|
|
"GPU": model.StateGPU,
|
|
"PSU": model.StatePSU,
|
|
"Reporting": model.StateReporting,
|
|
}
|
|
for name, state := range pairs {
|
|
if got := orchestrator.StageNameForState(state); got != name {
|
|
t.Errorf("StageNameForState(%q) = %q, want %q", state, got, name)
|
|
}
|
|
}
|
|
for _, s := range []model.RunState{
|
|
model.StateRegistered, model.StateQueued, model.StateBooting,
|
|
model.StateWaitingReboot, model.StateCompleted, model.StateFailedHolding,
|
|
} {
|
|
if got := orchestrator.StageNameForState(s); got != "" {
|
|
t.Errorf("StageNameForState(%q) = %q, want empty", s, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestTriggerOperatorCancelledFromHold: cancelling a held run must be
|
|
// allowed so the operator can walk away from a FailedHolding SSH prompt
|
|
// and have the host reboot to local disk. Before this, FailedHolding
|
|
// was considered terminal and Cancel errored out with "trigger not
|
|
// allowed from FailedHolding".
|
|
func TestTriggerOperatorCancelledFromHold(t *testing.T) {
|
|
got, err := orchestrator.Next(model.StateFailedHolding, orchestrator.TriggerOperatorCancelled)
|
|
if err != nil {
|
|
t.Fatalf("FailedHolding + OperatorCancelled: %v", err)
|
|
}
|
|
if got != model.StateCancelled {
|
|
t.Fatalf("got %q, want %q", got, model.StateCancelled)
|
|
}
|
|
// Sanity: other terminal states still reject the trigger so we don't
|
|
// accidentally allow Cancel after Completed/Cancelled/Released.
|
|
for _, bad := range []model.RunState{
|
|
model.StateCompleted, model.StateCancelled, model.StateReleased, model.StateFailed,
|
|
} {
|
|
if _, err := orchestrator.Next(bad, orchestrator.TriggerOperatorCancelled); err == nil {
|
|
t.Fatalf("OperatorCancelled from %q: expected error, got none", bad)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestNextStageWalk(t *testing.T) {
|
|
// Walking StageCompleted from each stage should land on the next
|
|
// one in the canonical order, and from Reporting onto Completed.
|
|
chain := []model.RunState{
|
|
model.StateInventoryCheck,
|
|
model.StateFirmware,
|
|
model.StateSpecValidate,
|
|
model.StateSMART,
|
|
model.StateCPUStress,
|
|
model.StateStorage,
|
|
model.StateNetwork,
|
|
model.StateBurn,
|
|
model.StateGPU,
|
|
model.StatePSU,
|
|
model.StateReporting,
|
|
model.StateCompleted,
|
|
}
|
|
for i := 0; i < len(chain)-1; i++ {
|
|
got, err := orchestrator.Next(chain[i], orchestrator.TriggerStageCompleted)
|
|
if err != nil {
|
|
t.Fatalf("Next(%q): %v", chain[i], err)
|
|
}
|
|
if got != chain[i+1] {
|
|
t.Fatalf("Next(%q) = %q, want %q", chain[i], got, chain[i+1])
|
|
}
|
|
}
|
|
}
|