Files
Vetting/internal/orchestrator/statemachine.go
T
josh 62bddac110
CI / Lint + build + test (push) Successful in 1m38s
Release / release (push) Successful in 6m10s
feat(cancel): allow cancel from FailedHolding, reboot to local disk
A held run sits indefinitely at an SSH prompt waiting for operator
investigation. Previously the only exits were Override (re-enter the
failed stage) or leaving the host on forever — Cancel rejected any
terminal state, including FailedHolding, and there was no button in
the UI anyway.

Add a dedicated exit path:
  - statemachine: TriggerOperatorCancelled now accepts FailedHolding
    as a valid source, transitioning to Cancelled like any other
    live state.
  - CancelRun handler: treats FailedHolding as cancellable even
    though IsTerminal reports true.
  - heartbeat: Cancelled runs fork on FailedStage. Set means the
    agent is parked in waitForOverride with no subprocess in
    flight, so cmd=reboot tells it to systemctl reboot; the host
    falls through iPXE's no-active-run script to the local disk.
    Empty FailedStage keeps the pre-existing cmd=cancel_stage path
    for mid-stage cancels (kill stage ctx, then power off).
  - UI: canCancel now returns true for FailedHolding, and the
    run-detail page renders a distinct "Cancel & reboot" button
    with a hold-specific confirm message so the action doesn't
    look identical to a mid-run cancel.

Tests cover the new statemachine transition, the heartbeat fork
(reboot vs cancel_stage), and keep the pre-existing mid-run cancel
behaviour locked in.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2026-04-19 22:59:34 -04:00

163 lines
6.6 KiB
Go

package orchestrator
import (
"fmt"
"vetting/internal/model"
)
// Trigger is an event that drives a state transition.
type Trigger string
const (
TriggerStartRequested Trigger = "StartRequested" // user clicks Start Vetting
TriggerDispatched Trigger = "Dispatched" // dispatcher picked this run (manual-WoL override path; dormant in happy path)
TriggerRebootCommanded Trigger = "RebootCommanded" // dispatcher (or heartbeat race) told the reporter to reboot
TriggerPXEObserved Trigger = "PXEObserved" // iPXE fetched cmdline for MAC
TriggerAgentClaimed Trigger = "AgentClaimed" // agent POSTed /claim with valid token
TriggerStageFailed Trigger = "StageFailed" // a stage reported failure
TriggerStageMismatch Trigger = "StageMismatch" // agent reported a stage that doesn't match current run state (silent-skip guard)
TriggerStageCompleted Trigger = "StageCompleted" // a stage reported success → advance
TriggerAllStagesPassed Trigger = "AllStagesPassed" // final stage passed
TriggerOperatorReleased Trigger = "OperatorReleased" // user clicked Release on a held run
TriggerOperatorOverride Trigger = "OperatorOverride" // user overrode a held stage; re-enter it
TriggerOperatorCancelled Trigger = "OperatorCancelled" // user clicked Cancel on an active run
)
// stageStates maps the canonical stage name (from DefaultStageOrder)
// to the matching RunState. Named differently for historical reasons:
// the first stage is "Inventory" (stage row name) but the run state is
// "InventoryCheck". Later stages share a name with their state.
var stageStates = map[string]model.RunState{
"Inventory": model.StateInventoryCheck,
"Firmware": model.StateFirmware,
"SpecValidate": model.StateSpecValidate,
"SMART": model.StateSMART,
"CPUStress": model.StateCPUStress,
"Storage": model.StateStorage,
"Network": model.StateNetwork,
"Burn": model.StateBurn,
"GPU": model.StateGPU,
"PSU": model.StatePSU,
"Reporting": model.StateReporting,
}
// stageOrder is the sequence of RunStates the run walks through from
// first stage to Completed. Kept in sync with store.DefaultStageOrder.
var stageOrder = []model.RunState{
model.StateInventoryCheck,
model.StateFirmware,
model.StateSpecValidate,
model.StateSMART,
model.StateCPUStress,
model.StateStorage,
model.StateNetwork,
model.StateBurn,
model.StateGPU,
model.StatePSU,
model.StateReporting,
}
type transition struct {
from []model.RunState
to model.RunState
}
var table = map[Trigger]transition{
TriggerStartRequested: {from: []model.RunState{model.StateRegistered}, to: model.StateQueued},
TriggerDispatched: {from: []model.RunState{model.StateQueued}, to: model.StateWaitingWoL},
TriggerRebootCommanded: {from: []model.RunState{model.StateQueued}, to: model.StateWaitingReboot},
TriggerPXEObserved: {from: []model.RunState{model.StateWaitingReboot, model.StateWaitingWoL, model.StateBooting}, to: model.StateBooting},
TriggerAgentClaimed: {from: []model.RunState{model.StateBooting, model.StateWaitingReboot, model.StateWaitingWoL}, to: model.StateInventoryCheck},
TriggerStageFailed: {from: allActiveStates(), to: model.StateFailedHolding},
TriggerStageMismatch: {from: stageExecutionStates(), to: model.StateFailedHolding},
TriggerAllStagesPassed: {from: []model.RunState{model.StateReporting}, to: model.StateCompleted},
TriggerOperatorReleased: {from: []model.RunState{model.StateFailedHolding}, to: model.StateReleased},
TriggerOperatorCancelled: {from: append(allActiveStates(), model.StateFailedHolding), to: model.StateCancelled},
}
// Next computes the target state for a trigger against the current state.
// StageCompleted is handled specially: it advances through stageOrder.
func Next(current model.RunState, t Trigger) (model.RunState, error) {
if t == TriggerStageCompleted {
return nextStageState(current)
}
tr, ok := table[t]
if !ok {
return "", fmt.Errorf("unknown trigger %q", t)
}
for _, s := range tr.from {
if s == current {
return tr.to, nil
}
}
return "", fmt.Errorf("trigger %q not allowed from %q", t, current)
}
// NextForOverride returns the state we should jump to when the operator
// overrides a held stage. It's separate from the generic table because
// the target depends on the failed_stage, not on the current state
// (which is always FailedHolding).
func NextForOverride(current model.RunState, failedStage string) (model.RunState, error) {
if current != model.StateFailedHolding {
return "", fmt.Errorf("override not allowed from %q", current)
}
s, ok := stageStates[failedStage]
if !ok {
return "", fmt.Errorf("override: unknown failed stage %q", failedStage)
}
return s, nil
}
// StateForStage returns the RunState that corresponds to a stage name.
// Used by handlers that receive a stage name and want to guard against
// stale/out-of-order agent reports.
func StateForStage(name string) (model.RunState, bool) {
s, ok := stageStates[name]
return s, ok
}
// StageNameForState is the inverse of StateForStage: given a run state
// that maps to a stage, returns the stage name (e.g. StateCPUStress →
// "CPUStress"). Empty string when the state isn't a stage-execution
// state (Queued, Booting, FailedHolding, etc.). Used by /result to
// detect when an agent submitted a stage name that doesn't match where
// the orchestrator thinks the run is — the silent-skip guard.
func StageNameForState(s model.RunState) string {
for name, state := range stageStates {
if state == s {
return name
}
}
return ""
}
func nextStageState(current model.RunState) (model.RunState, error) {
for i, s := range stageOrder {
if s == current {
if i+1 >= len(stageOrder) {
return model.StateCompleted, nil
}
return stageOrder[i+1], nil
}
}
return "", fmt.Errorf("StageCompleted not valid from %q", current)
}
func allActiveStates() []model.RunState {
return []model.RunState{
model.StateQueued, model.StateWaitingWoL, model.StateWaitingReboot, model.StateBooting,
model.StateInventoryCheck, model.StateFirmware, model.StateSpecValidate, model.StateSMART,
model.StateCPUStress, model.StateStorage, model.StateNetwork,
model.StateBurn, model.StateGPU, model.StatePSU, model.StateReporting,
}
}
// stageExecutionStates returns only the stage-execution states — no
// pre-stages, no terminals. Used as the valid "from" set for
// TriggerStageMismatch: it's nonsensical to fire a stage-mismatch from
// Queued or Booting because no stage result should arrive then.
func stageExecutionStates() []model.RunState {
return append([]model.RunState(nil), stageOrder...)
}