feat(cancel): allow cancel from FailedHolding, reboot to local disk
A held run sits indefinitely at an SSH prompt waiting for operator
investigation. Previously the only exits were Override (re-enter the
failed stage) or leaving the host on forever — Cancel rejected any
terminal state, including FailedHolding, and there was no button in
the UI anyway.
Add a dedicated exit path:
- statemachine: TriggerOperatorCancelled now accepts FailedHolding
as a valid source, transitioning to Cancelled like any other
live state.
- CancelRun handler: treats FailedHolding as cancellable even
though IsTerminal reports true.
- heartbeat: Cancelled runs fork on FailedStage. Set means the
agent is parked in waitForOverride with no subprocess in
flight, so cmd=reboot tells it to systemctl reboot; the host
falls through iPXE's no-active-run script to the local disk.
Empty FailedStage keeps the pre-existing cmd=cancel_stage path
for mid-stage cancels (kill stage ctx, then power off).
- UI: canCancel now returns true for FailedHolding, and the
run-detail page renders a distinct "Cancel & reboot" button
with a hold-specific confirm message so the action doesn't
look identical to a mid-run cancel.
Tests cover the new statemachine transition, the heartbeat fork
(reboot vs cancel_stage), and keep the pre-existing mid-run cancel
behaviour locked in.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -131,6 +131,63 @@ func TestHeartbeatRebootWhenCompleted(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestHeartbeatRebootWhenCancelledFromHold: operator hit Cancel on a
|
||||
// FailedHolding run. Because there's no in-flight stage subprocess (the
|
||||
// agent is parked in waitForOverride), the heartbeat must answer with
|
||||
// cmd=reboot — not cmd=cancel_stage which only makes sense mid-stage.
|
||||
// The FailedStage marker is the discriminator: set means we came
|
||||
// through hold; empty means a mid-stage cancel.
|
||||
func TestHeartbeatRebootWhenCancelledFromHold(t *testing.T) {
|
||||
a, runID, token := setupAgent(t)
|
||||
a.Runner = &orchestrator.Runner{Runs: a.Runs, Hosts: a.Hosts, Stages: &store.Stages{DB: a.Runs.DB}, EventHub: events.NewHub()}
|
||||
if err := a.Runs.SetFailedStage(context.Background(), runID, "Storage"); err != nil {
|
||||
t.Fatalf("set failed stage: %v", err)
|
||||
}
|
||||
if err := a.Runs.SetState(context.Background(), runID, model.StateCancelled); err != nil {
|
||||
t.Fatalf("set state: %v", err)
|
||||
}
|
||||
req := routedRequest(runID, http.MethodPost, "/api/v1/runs/"+strconv.FormatInt(runID, 10)+"/heartbeat", nil)
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
rr := httptest.NewRecorder()
|
||||
a.Heartbeat(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var resp map[string]any
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if resp["cmd"] != "reboot" {
|
||||
t.Fatalf("cmd = %v, want reboot", resp["cmd"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestHeartbeatCancelStageWhenCancelledMidRun: the mid-stage cancel
|
||||
// path (no FailedStage marker) still answers cmd=cancel_stage so the
|
||||
// agent kills its in-flight subprocess before powering off. This is
|
||||
// the pre-existing behaviour; the hold-cancel branch is additive.
|
||||
func TestHeartbeatCancelStageWhenCancelledMidRun(t *testing.T) {
|
||||
a, runID, token := setupAgent(t)
|
||||
a.Runner = &orchestrator.Runner{Runs: a.Runs, Hosts: a.Hosts, Stages: &store.Stages{DB: a.Runs.DB}, EventHub: events.NewHub()}
|
||||
if err := a.Runs.SetState(context.Background(), runID, model.StateCancelled); err != nil {
|
||||
t.Fatalf("set state: %v", err)
|
||||
}
|
||||
req := routedRequest(runID, http.MethodPost, "/api/v1/runs/"+strconv.FormatInt(runID, 10)+"/heartbeat", nil)
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
rr := httptest.NewRecorder()
|
||||
a.Heartbeat(rr, req)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var resp map[string]any
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if resp["cmd"] != "cancel_stage" {
|
||||
t.Fatalf("cmd = %v, want cancel_stage", resp["cmd"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestResult_RejectsMismatchedStage is the silent-skip guard's unit
|
||||
// test. The Orion failure mode: agent crashes mid-CPUStress, systemd
|
||||
// restarts it, restarted agent replays Inventory and /results it.
|
||||
|
||||
Reference in New Issue
Block a user