feat(cancel): allow cancel from FailedHolding, reboot to local disk

A held run sits indefinitely at an SSH prompt waiting for operator
investigation. Previously the only exits were Override (re-enter the
failed stage) or leaving the host on forever — Cancel rejected any
terminal state, including FailedHolding, and there was no button in
the UI anyway.

Add a dedicated exit path:
  - statemachine: TriggerOperatorCancelled now accepts FailedHolding
    as a valid source, transitioning to Cancelled like any other
    live state.
  - CancelRun handler: treats FailedHolding as cancellable even
    though IsTerminal reports true.
  - heartbeat: Cancelled runs fork on FailedStage. Set means the
    agent is parked in waitForOverride with no subprocess in
    flight, so cmd=reboot tells it to systemctl reboot; the host
    falls through iPXE's no-active-run script to the local disk.
    Empty FailedStage keeps the pre-existing cmd=cancel_stage path
    for mid-stage cancels (kill stage ctx, then power off).
  - UI: canCancel now returns true for FailedHolding, and the
    run-detail page renders a distinct "Cancel & reboot" button
    with a hold-specific confirm message so the action doesn't
    look identical to a mid-run cancel.

Tests cover the new statemachine transition, the heartbeat fork
(reboot vs cancel_stage), and keep the pre-existing mid-run cancel
behaviour locked in.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

internal/api/agent_handlers.go

@@ -272,9 +272,20 @@ func (a *Agent) Heartbeat(w http.ResponseWriter, r *http.Request) {
 		// intervention.
 		cmd = "reboot"
 	case run.State == model.StateCancelled:
-		// Operator clicked Cancel — agent cancels the active stage ctx,
-		// posts a cancelled outcome, and powers off.
-		cmd = "cancel_stage"
+		// Operator clicked Cancel. Two sub-cases:
+		//   - FailedStage set → run was sitting in FailedHolding with no
+		//     in-flight stage subprocess; the agent is parked in
+		//     waitForOverride. Send cmd=reboot so the heartbeat loop
+		//     reboots the host, falls through iPXE's no-active-run
+		//     script and boots local disk.
+		//   - FailedStage empty → cancel mid-stage; kill the stage ctx
+		//     first so the running subprocess exits cleanly, then the
+		//     agent powers off via its existing cancel path.
+		if run.FailedStage != "" {
+			cmd = "reboot"
+		} else {
+			cmd = "cancel_stage"
+		}
 	case run.State == model.StateFailedHolding || run.State == model.StateReleased:
 		cmd = "abort"
 	case run.FailedStage == "Storage" && overrideWipeSet(run.OverrideFlagsJSON):

internal/api/agent_handlers_test.go

@@ -131,6 +131,63 @@ func TestHeartbeatRebootWhenCompleted(t *testing.T) {
 	}
 }
 
+// TestHeartbeatRebootWhenCancelledFromHold: operator hit Cancel on a
+// FailedHolding run. Because there's no in-flight stage subprocess (the
+// agent is parked in waitForOverride), the heartbeat must answer with
+// cmd=reboot — not cmd=cancel_stage which only makes sense mid-stage.
+// The FailedStage marker is the discriminator: set means we came
+// through hold; empty means a mid-stage cancel.
+func TestHeartbeatRebootWhenCancelledFromHold(t *testing.T) {
+	a, runID, token := setupAgent(t)
+	a.Runner = &orchestrator.Runner{Runs: a.Runs, Hosts: a.Hosts, Stages: &store.Stages{DB: a.Runs.DB}, EventHub: events.NewHub()}
+	if err := a.Runs.SetFailedStage(context.Background(), runID, "Storage"); err != nil {
+		t.Fatalf("set failed stage: %v", err)
+	}
+	if err := a.Runs.SetState(context.Background(), runID, model.StateCancelled); err != nil {
+		t.Fatalf("set state: %v", err)
+	}
+	req := routedRequest(runID, http.MethodPost, "/api/v1/runs/"+strconv.FormatInt(runID, 10)+"/heartbeat", nil)
+	req.Header.Set("Authorization", "Bearer "+token)
+	rr := httptest.NewRecorder()
+	a.Heartbeat(rr, req)
+	if rr.Code != http.StatusOK {
+		t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
+	}
+	var resp map[string]any
+	if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	if resp["cmd"] != "reboot" {
+		t.Fatalf("cmd = %v, want reboot", resp["cmd"])
+	}
+}
+
+// TestHeartbeatCancelStageWhenCancelledMidRun: the mid-stage cancel
+// path (no FailedStage marker) still answers cmd=cancel_stage so the
+// agent kills its in-flight subprocess before powering off. This is
+// the pre-existing behaviour; the hold-cancel branch is additive.
+func TestHeartbeatCancelStageWhenCancelledMidRun(t *testing.T) {
+	a, runID, token := setupAgent(t)
+	a.Runner = &orchestrator.Runner{Runs: a.Runs, Hosts: a.Hosts, Stages: &store.Stages{DB: a.Runs.DB}, EventHub: events.NewHub()}
+	if err := a.Runs.SetState(context.Background(), runID, model.StateCancelled); err != nil {
+		t.Fatalf("set state: %v", err)
+	}
+	req := routedRequest(runID, http.MethodPost, "/api/v1/runs/"+strconv.FormatInt(runID, 10)+"/heartbeat", nil)
+	req.Header.Set("Authorization", "Bearer "+token)
+	rr := httptest.NewRecorder()
+	a.Heartbeat(rr, req)
+	if rr.Code != http.StatusOK {
+		t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
+	}
+	var resp map[string]any
+	if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	if resp["cmd"] != "cancel_stage" {
+		t.Fatalf("cmd = %v, want cancel_stage", resp["cmd"])
+	}
+}
+
 // TestResult_RejectsMismatchedStage is the silent-skip guard's unit
 // test. The Orion failure mode: agent crashes mid-CPUStress, systemd
 // restarts it, restarted agent replays Inventory and /results it.

internal/api/ui_handlers.go

@@ -660,7 +660,11 @@ func (u *UI) CancelRun(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
-	if latest == nil || latest.State.IsTerminal() {
+	// FailedHolding is "terminal" for the pipeline but the host is still
+	// alive and holding at an SSH prompt; the operator can walk away from
+	// the hold via Cancel (which reboots to local disk). Every other
+	// terminal state is truly done — nothing to cancel.
+	if latest == nil || (latest.State.IsTerminal() && latest.State != model.StateFailedHolding) {
 		http.Error(w, "no active run to cancel", http.StatusConflict)
 		return
 	}