feat(cancel): allow cancel from FailedHolding, reboot to local disk

A held run sits indefinitely at an SSH prompt waiting for operator
investigation. Previously the only exits were Override (re-enter the
failed stage) or leaving the host on forever — Cancel rejected any
terminal state, including FailedHolding, and there was no button in
the UI anyway.

Add a dedicated exit path:
  - statemachine: TriggerOperatorCancelled now accepts FailedHolding
    as a valid source, transitioning to Cancelled like any other
    live state.
  - CancelRun handler: treats FailedHolding as cancellable even
    though IsTerminal reports true.
  - heartbeat: Cancelled runs fork on FailedStage. Set means the
    agent is parked in waitForOverride with no subprocess in
    flight, so cmd=reboot tells it to systemctl reboot; the host
    falls through iPXE's no-active-run script to the local disk.
    Empty FailedStage keeps the pre-existing cmd=cancel_stage path
    for mid-stage cancels (kill stage ctx, then power off).
  - UI: canCancel now returns true for FailedHolding, and the
    run-detail page renders a distinct "Cancel & reboot" button
    with a hold-specific confirm message so the action doesn't
    look identical to a mid-run cancel.

Tests cover the new statemachine transition, the heartbeat fork
(reboot vs cancel_stage), and keep the pre-existing mid-run cancel
behaviour locked in.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

internal/api/agent_handlers.go

@@ -272,9 +272,20 @@ func (a *Agent) Heartbeat(w http.ResponseWriter, r *http.Request) {
 		// intervention.
 		cmd = "reboot"
 	case run.State == model.StateCancelled:
-		// Operator clicked Cancel — agent cancels the active stage ctx,
+		// Operator clicked Cancel. Two sub-cases:
-		// posts a cancelled outcome, and powers off.
+		//   - FailedStage set → run was sitting in FailedHolding with no
-		cmd = "cancel_stage"
+		//     in-flight stage subprocess; the agent is parked in
+		//     waitForOverride. Send cmd=reboot so the heartbeat loop
+		//     reboots the host, falls through iPXE's no-active-run
+		//     script and boots local disk.
+		//   - FailedStage empty → cancel mid-stage; kill the stage ctx
+		//     first so the running subprocess exits cleanly, then the
+		//     agent powers off via its existing cancel path.
+		if run.FailedStage != "" {
+			cmd = "reboot"
+		} else {
+			cmd = "cancel_stage"
+		}
 	case run.State == model.StateFailedHolding || run.State == model.StateReleased:
 		cmd = "abort"
 	case run.FailedStage == "Storage" && overrideWipeSet(run.OverrideFlagsJSON):

internal/api/agent_handlers_test.go

@@ -131,6 +131,63 @@ func TestHeartbeatRebootWhenCompleted(t *testing.T) {
 	}
 }
 
+// TestHeartbeatRebootWhenCancelledFromHold: operator hit Cancel on a
+// FailedHolding run. Because there's no in-flight stage subprocess (the
+// agent is parked in waitForOverride), the heartbeat must answer with
+// cmd=reboot — not cmd=cancel_stage which only makes sense mid-stage.
+// The FailedStage marker is the discriminator: set means we came
+// through hold; empty means a mid-stage cancel.
+func TestHeartbeatRebootWhenCancelledFromHold(t *testing.T) {
+	a, runID, token := setupAgent(t)
+	a.Runner = &orchestrator.Runner{Runs: a.Runs, Hosts: a.Hosts, Stages: &store.Stages{DB: a.Runs.DB}, EventHub: events.NewHub()}
+	if err := a.Runs.SetFailedStage(context.Background(), runID, "Storage"); err != nil {
+		t.Fatalf("set failed stage: %v", err)
+	}
+	if err := a.Runs.SetState(context.Background(), runID, model.StateCancelled); err != nil {
+		t.Fatalf("set state: %v", err)
+	}
+	req := routedRequest(runID, http.MethodPost, "/api/v1/runs/"+strconv.FormatInt(runID, 10)+"/heartbeat", nil)
+	req.Header.Set("Authorization", "Bearer "+token)
+	rr := httptest.NewRecorder()
+	a.Heartbeat(rr, req)
+	if rr.Code != http.StatusOK {
+		t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
+	}
+	var resp map[string]any
+	if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	if resp["cmd"] != "reboot" {
+		t.Fatalf("cmd = %v, want reboot", resp["cmd"])
+	}
+}
+
+// TestHeartbeatCancelStageWhenCancelledMidRun: the mid-stage cancel
+// path (no FailedStage marker) still answers cmd=cancel_stage so the
+// agent kills its in-flight subprocess before powering off. This is
+// the pre-existing behaviour; the hold-cancel branch is additive.
+func TestHeartbeatCancelStageWhenCancelledMidRun(t *testing.T) {
+	a, runID, token := setupAgent(t)
+	a.Runner = &orchestrator.Runner{Runs: a.Runs, Hosts: a.Hosts, Stages: &store.Stages{DB: a.Runs.DB}, EventHub: events.NewHub()}
+	if err := a.Runs.SetState(context.Background(), runID, model.StateCancelled); err != nil {
+		t.Fatalf("set state: %v", err)
+	}
+	req := routedRequest(runID, http.MethodPost, "/api/v1/runs/"+strconv.FormatInt(runID, 10)+"/heartbeat", nil)
+	req.Header.Set("Authorization", "Bearer "+token)
+	rr := httptest.NewRecorder()
+	a.Heartbeat(rr, req)
+	if rr.Code != http.StatusOK {
+		t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
+	}
+	var resp map[string]any
+	if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	if resp["cmd"] != "cancel_stage" {
+		t.Fatalf("cmd = %v, want cancel_stage", resp["cmd"])
+	}
+}
+
 // TestResult_RejectsMismatchedStage is the silent-skip guard's unit
 // test. The Orion failure mode: agent crashes mid-CPUStress, systemd
 // restarts it, restarted agent replays Inventory and /results it.

internal/api/ui_handlers.go

@@ -660,7 +660,11 @@ func (u *UI) CancelRun(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
-	if latest == nil || latest.State.IsTerminal() {
+	// FailedHolding is "terminal" for the pipeline but the host is still
+	// alive and holding at an SSH prompt; the operator can walk away from
+	// the hold via Cancel (which reboots to local disk). Every other
+	// terminal state is truly done — nothing to cancel.
+	if latest == nil || (latest.State.IsTerminal() && latest.State != model.StateFailedHolding) {
 		http.Error(w, "no active run to cancel", http.StatusConflict)
 		return
 	}

internal/orchestrator/statemachine.go

@@ -73,7 +73,7 @@ var table = map[Trigger]transition{
 	TriggerStageMismatch:     {from: stageExecutionStates(), to: model.StateFailedHolding},
 	TriggerAllStagesPassed:   {from: []model.RunState{model.StateReporting}, to: model.StateCompleted},
 	TriggerOperatorReleased:  {from: []model.RunState{model.StateFailedHolding}, to: model.StateReleased},
-	TriggerOperatorCancelled: {from: allActiveStates(), to: model.StateCancelled},
+	TriggerOperatorCancelled: {from: append(allActiveStates(), model.StateFailedHolding), to: model.StateCancelled},
 }
 
 // Next computes the target state for a trigger against the current state.

internal/orchestrator/statemachine_test.go

@@ -142,6 +142,30 @@ func TestStageNameForState(t *testing.T) {
 	}
 }
 
+// TestTriggerOperatorCancelledFromHold: cancelling a held run must be
+// allowed so the operator can walk away from a FailedHolding SSH prompt
+// and have the host reboot to local disk. Before this, FailedHolding
+// was considered terminal and Cancel errored out with "trigger not
+// allowed from FailedHolding".
+func TestTriggerOperatorCancelledFromHold(t *testing.T) {
+	got, err := orchestrator.Next(model.StateFailedHolding, orchestrator.TriggerOperatorCancelled)
+	if err != nil {
+		t.Fatalf("FailedHolding + OperatorCancelled: %v", err)
+	}
+	if got != model.StateCancelled {
+		t.Fatalf("got %q, want %q", got, model.StateCancelled)
+	}
+	// Sanity: other terminal states still reject the trigger so we don't
+	// accidentally allow Cancel after Completed/Cancelled/Released.
+	for _, bad := range []model.RunState{
+		model.StateCompleted, model.StateCancelled, model.StateReleased, model.StateFailed,
+	} {
+		if _, err := orchestrator.Next(bad, orchestrator.TriggerOperatorCancelled); err == nil {
+			t.Fatalf("OperatorCancelled from %q: expected error, got none", bad)
+		}
+	}
+}
+
 func TestNextStageWalk(t *testing.T) {
 	// Walking StageCompleted from each stage should land on the next
 	// one in the canonical order, and from Reporting onto Completed.

internal/web/templates/host_tile.templ

@@ -89,12 +89,20 @@ func canStartIfOnline(r *model.Run) bool {
 	return r.State.IsTerminal()
 }
 
-// canCancel is true for any non-terminal run — the Cancel button shows
+// canCancel is true for any non-terminal run, plus FailedHolding —
-// whenever the pipeline is live (Queued through the stage states). The
+// a held run technically classifies as terminal for the pipeline but
-// handler refuses the action once the run enters a terminal state, so
+// the host is still live on the SSH hold prompt, and the operator
-// the render decision just has to mirror that.
+// can walk away from it via Cancel (which reboots to local disk).
+// Every other terminal state is truly done, so no Cancel button.
+// The server-side CancelRun handler mirrors this predicate.
 func canCancel(r *model.Run) bool {
-	return r != nil && !r.State.IsTerminal()
+	if r == nil {
+		return false
+	}
+	if !r.State.IsTerminal() {
+		return true
+	}
+	return r.State == model.StateFailedHolding
 }
 
 func tileStatus(r *model.Run) string {

internal/web/templates/host_tile_templ.go

@@ -284,12 +284,20 @@ func canStartIfOnline(r *model.Run) bool {
 	return r.State.IsTerminal()
 }
 
-// canCancel is true for any non-terminal run — the Cancel button shows
+// canCancel is true for any non-terminal run, plus FailedHolding —
-// whenever the pipeline is live (Queued through the stage states). The
+// a held run technically classifies as terminal for the pipeline but
-// handler refuses the action once the run enters a terminal state, so
+// the host is still live on the SSH hold prompt, and the operator
-// the render decision just has to mirror that.
+// can walk away from it via Cancel (which reboots to local disk).
+// Every other terminal state is truly done, so no Cancel button.
+// The server-side CancelRun handler mirrors this predicate.
 func canCancel(r *model.Run) bool {
-	return r != nil && !r.State.IsTerminal()
+	if r == nil {
+		return false
+	}
+	if !r.State.IsTerminal() {
+		return true
+	}
+	return r.State == model.StateFailedHolding
 }
 
 func tileStatus(r *model.Run) string {

internal/web/templates/run_detail.templ

@@ -100,9 +100,15 @@ templ RunHeader(d RunPageData) {
 		</div>
 		<div class="run-header-right">
 			if canCancel(&d.Run) {
-				<form method="post" action={ templ.SafeURL(fmt.Sprintf("/hosts/%d/cancel", d.Host.ID)) } class="inline" onsubmit="return confirm('Cancel run? Destructive stages may leave the host in an intermediate state requiring manual cleanup.');">
+				if d.Run.State == model.StateFailedHolding {
-					<button type="submit" class="btn-danger">Cancel run</button>
+					<form method="post" action={ templ.SafeURL(fmt.Sprintf("/hosts/%d/cancel", d.Host.ID)) } class="inline" onsubmit="return confirm('Cancel held run? The host will reboot to local disk.');">
-				</form>
+						<button type="submit" class="btn-danger">Cancel &amp; reboot</button>
+					</form>
+				} else {
+					<form method="post" action={ templ.SafeURL(fmt.Sprintf("/hosts/%d/cancel", d.Host.ID)) } class="inline" onsubmit="return confirm('Cancel run? Destructive stages may leave the host in an intermediate state requiring manual cleanup.');">
+						<button type="submit" class="btn-danger">Cancel run</button>
+					</form>
+				}
 			}
 			if canOverrideWipe(&d.Run) {
 				<form method="post" action={ templ.SafeURL(fmt.Sprintf("/hosts/%d/override-wipe", d.Host.ID)) } class="inline">

internal/web/templates/run_detail_templ.go

@@ -405,82 +405,102 @@ func RunHeader(d RunPageData) templ.Component {
 			return templ_7745c5c3_Err
 		}
 		if canCancel(&d.Run) {
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 28, "<form method=\"post\" action=\"")
+			if d.Run.State == model.StateFailedHolding {
-			if templ_7745c5c3_Err != nil {
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 28, "<form method=\"post\" action=\"")
-				return templ_7745c5c3_Err
+				if templ_7745c5c3_Err != nil {
-			}
+					return templ_7745c5c3_Err
-			var templ_7745c5c3_Var22 templ.SafeURL
+				}
-			templ_7745c5c3_Var22, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/hosts/%d/cancel", d.Host.ID)))
+				var templ_7745c5c3_Var22 templ.SafeURL
-			if templ_7745c5c3_Err != nil {
+				templ_7745c5c3_Var22, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/hosts/%d/cancel", d.Host.ID)))
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/templates/run_detail.templ`, Line: 103, Col: 90}
+				if templ_7745c5c3_Err != nil {
-			}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/templates/run_detail.templ`, Line: 104, Col: 91}
-			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var22))
+				}
-			if templ_7745c5c3_Err != nil {
+				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var22))
-				return templ_7745c5c3_Err
+				if templ_7745c5c3_Err != nil {
-			}
+					return templ_7745c5c3_Err
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 29, "\" class=\"inline\" onsubmit=\"return confirm('Cancel run? Destructive stages may leave the host in an intermediate state requiring manual cleanup.');\"><button type=\"submit\" class=\"btn-danger\">Cancel run</button></form>")
+				}
-			if templ_7745c5c3_Err != nil {
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 29, "\" class=\"inline\" onsubmit=\"return confirm('Cancel held run? The host will reboot to local disk.');\"><button type=\"submit\" class=\"btn-danger\">Cancel &amp; reboot</button></form>")
-				return templ_7745c5c3_Err
+				if templ_7745c5c3_Err != nil {
+					return templ_7745c5c3_Err
+				}
+			} else {
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 30, "<form method=\"post\" action=\"")
+				if templ_7745c5c3_Err != nil {
+					return templ_7745c5c3_Err
+				}
+				var templ_7745c5c3_Var23 templ.SafeURL
+				templ_7745c5c3_Var23, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/hosts/%d/cancel", d.Host.ID)))
+				if templ_7745c5c3_Err != nil {
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/templates/run_detail.templ`, Line: 108, Col: 91}
+				}
+				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var23))
+				if templ_7745c5c3_Err != nil {
+					return templ_7745c5c3_Err
+				}
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 31, "\" class=\"inline\" onsubmit=\"return confirm('Cancel run? Destructive stages may leave the host in an intermediate state requiring manual cleanup.');\"><button type=\"submit\" class=\"btn-danger\">Cancel run</button></form>")
+				if templ_7745c5c3_Err != nil {
+					return templ_7745c5c3_Err
+				}
 			}
 		}
 		if canOverrideWipe(&d.Run) {
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 30, "<form method=\"post\" action=\"")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 32, "<form method=\"post\" action=\"")
-			if templ_7745c5c3_Err != nil {
-				return templ_7745c5c3_Err
-			}
-			var templ_7745c5c3_Var23 templ.SafeURL
-			templ_7745c5c3_Var23, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/hosts/%d/override-wipe", d.Host.ID)))
-			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/templates/run_detail.templ`, Line: 108, Col: 97}
-			}
-			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var23))
-			if templ_7745c5c3_Err != nil {
-				return templ_7745c5c3_Err
-			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 31, "\" class=\"inline\"><button type=\"submit\" class=\"btn-danger\">Override wipe-probe</button></form>")
-			if templ_7745c5c3_Err != nil {
-				return templ_7745c5c3_Err
-			}
-		}
-		if hasReport(&d.Run) {
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 32, "<a class=\"button-like\" href=\"")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var24 templ.SafeURL
-			templ_7745c5c3_Var24, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/reports/%d", d.Run.ID)))
+			templ_7745c5c3_Var24, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/hosts/%d/override-wipe", d.Host.ID)))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/templates/run_detail.templ`, Line: 113, Col: 85}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/templates/run_detail.templ`, Line: 114, Col: 97}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var24))
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 33, "\" target=\"_blank\" rel=\"noopener\">View report</a> ")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 33, "\" class=\"inline\"><button type=\"submit\" class=\"btn-danger\">Override wipe-probe</button></form>")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 		}
-		if d.Run.State.IsTerminal() {
+		if hasReport(&d.Run) {
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 34, "<form method=\"post\" action=\"")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 34, "<a class=\"button-like\" href=\"")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var25 templ.SafeURL
-			templ_7745c5c3_Var25, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/hosts/%d/start", d.Host.ID)))
+			templ_7745c5c3_Var25, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/reports/%d", d.Run.ID)))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/templates/run_detail.templ`, Line: 116, Col: 89}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/templates/run_detail.templ`, Line: 119, Col: 85}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var25))
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 35, "\" class=\"inline\"><button type=\"submit\" class=\"btn-primary\">Start new run</button></form>")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 35, "\" target=\"_blank\" rel=\"noopener\">View report</a> ")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 36, "</div></header>")
+		if d.Run.State.IsTerminal() {
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 36, "<form method=\"post\" action=\"")
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
+			var templ_7745c5c3_Var26 templ.SafeURL
+			templ_7745c5c3_Var26, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/hosts/%d/start", d.Host.ID)))
+			if templ_7745c5c3_Err != nil {
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/templates/run_detail.templ`, Line: 122, Col: 89}
+			}
+			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var26))
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 37, "\" class=\"inline\"><button type=\"submit\" class=\"btn-primary\">Start new run</button></form>")
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
+		}
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 38, "</div></header>")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
@@ -508,83 +528,83 @@ func HoldBanner(d RunPageData) templ.Component {
 			}()
 		}
 		ctx = templ.InitializeContext(ctx)
-		templ_7745c5c3_Var26 := templ.GetChildren(ctx)
+		templ_7745c5c3_Var27 := templ.GetChildren(ctx)
-		if templ_7745c5c3_Var26 == nil {
+		if templ_7745c5c3_Var27 == nil {
-			templ_7745c5c3_Var26 = templ.NopComponent
+			templ_7745c5c3_Var27 = templ.NopComponent
 		}
 		ctx = templ.ClearChildren(ctx)
 		if d.Run.State == model.StateFailedHolding && d.Run.HoldIP != "" {
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 37, "<section id=\"")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 39, "<section id=\"")
-			if templ_7745c5c3_Err != nil {
-				return templ_7745c5c3_Err
-			}
-			var templ_7745c5c3_Var27 string
-			templ_7745c5c3_Var27, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("detail-hold-%d", d.Run.ID))
-			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/templates/run_detail.templ`, Line: 131, Col: 47}
-			}
-			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var27))
-			if templ_7745c5c3_Err != nil {
-				return templ_7745c5c3_Err
-			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 38, "\" class=\"hold-banner\" sse-swap=\"")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var28 string
 			templ_7745c5c3_Var28, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("detail-hold-%d", d.Run.ID))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/templates/run_detail.templ`, Line: 133, Col: 53}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/templates/run_detail.templ`, Line: 137, Col: 47}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var28))
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 39, "\" hx-swap=\"outerHTML\"><span class=\"hold-banner-label\">Host is holding — SSH available:</span> <code class=\"hold-ssh\">")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 40, "\" class=\"hold-banner\" sse-swap=\"")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var29 string
-			templ_7745c5c3_Var29, templ_7745c5c3_Err = templ.JoinStringErrs(sshInvocation(d.HoldKeyPath, d.Run.HoldIP))
+			templ_7745c5c3_Var29, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("detail-hold-%d", d.Run.ID))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/templates/run_detail.templ`, Line: 137, Col: 70}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/templates/run_detail.templ`, Line: 139, Col: 53}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var29))
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 40, "</code></section>")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 41, "\" hx-swap=\"outerHTML\"><span class=\"hold-banner-label\">Host is holding — SSH available:</span> <code class=\"hold-ssh\">")
-			if templ_7745c5c3_Err != nil {
-				return templ_7745c5c3_Err
-			}
-		} else {
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 41, "<section id=\"")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var30 string
-			templ_7745c5c3_Var30, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("detail-hold-%d", d.Run.ID))
+			templ_7745c5c3_Var30, templ_7745c5c3_Err = templ.JoinStringErrs(sshInvocation(d.HoldKeyPath, d.Run.HoldIP))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/templates/run_detail.templ`, Line: 141, Col: 47}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/templates/run_detail.templ`, Line: 143, Col: 70}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var30))
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 42, "\" class=\"detail-hold-placeholder\" sse-swap=\"")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 42, "</code></section>")
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
+		} else {
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 43, "<section id=\"")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 			var templ_7745c5c3_Var31 string
 			templ_7745c5c3_Var31, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("detail-hold-%d", d.Run.ID))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/templates/run_detail.templ`, Line: 143, Col: 53}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/templates/run_detail.templ`, Line: 147, Col: 47}
 			}
 			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var31))
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 43, "\" hx-swap=\"outerHTML\"></section>")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 44, "\" class=\"detail-hold-placeholder\" sse-swap=\"")
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
+			var templ_7745c5c3_Var32 string
+			templ_7745c5c3_Var32, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("detail-hold-%d", d.Run.ID))
+			if templ_7745c5c3_Err != nil {
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/templates/run_detail.templ`, Line: 149, Col: 53}
+			}
+			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var32))
+			if templ_7745c5c3_Err != nil {
+				return templ_7745c5c3_Err
+			}
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 45, "\" hx-swap=\"outerHTML\"></section>")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
@@ -612,138 +632,138 @@ func RunSpecDiffs(d RunPageData) templ.Component {
 			}()
 		}
 		ctx = templ.InitializeContext(ctx)
-		templ_7745c5c3_Var32 := templ.GetChildren(ctx)
+		templ_7745c5c3_Var33 := templ.GetChildren(ctx)
-		if templ_7745c5c3_Var32 == nil {
+		if templ_7745c5c3_Var33 == nil {
-			templ_7745c5c3_Var32 = templ.NopComponent
+			templ_7745c5c3_Var33 = templ.NopComponent
 		}
 		ctx = templ.ClearChildren(ctx)
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 44, "<section id=\"")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 46, "<section id=\"")
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		var templ_7745c5c3_Var33 string
-		templ_7745c5c3_Var33, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("detail-specdiffs-%d", d.Run.ID))
-		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/templates/run_detail.templ`, Line: 154, Col: 51}
-		}
-		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var33))
-		if templ_7745c5c3_Err != nil {
-			return templ_7745c5c3_Err
-		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 45, "\" class=\"detail-section detail-diffs\" sse-swap=\"")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
 		var templ_7745c5c3_Var34 string
 		templ_7745c5c3_Var34, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("detail-specdiffs-%d", d.Run.ID))
 		if templ_7745c5c3_Err != nil {
-			return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/templates/run_detail.templ`, Line: 156, Col: 57}
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/templates/run_detail.templ`, Line: 160, Col: 51}
 		}
 		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var34))
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 46, "\" hx-swap=\"outerHTML\">")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 47, "\" class=\"detail-section detail-diffs\" sse-swap=\"")
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		var templ_7745c5c3_Var35 string
+		templ_7745c5c3_Var35, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("detail-specdiffs-%d", d.Run.ID))
+		if templ_7745c5c3_Err != nil {
+			return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/templates/run_detail.templ`, Line: 162, Col: 57}
+		}
+		_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var35))
+		if templ_7745c5c3_Err != nil {
+			return templ_7745c5c3_Err
+		}
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 48, "\" hx-swap=\"outerHTML\">")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}
 		if len(d.SpecDiffs) > 0 {
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 47, "<details")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 49, "<details")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 			if hasCriticalDiff(d.SpecDiffs) {
-				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 48, " open")
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 50, " open")
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 49, "><summary><h2>Spec diffs (")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 51, "><summary><h2>Spec diffs (")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			var templ_7745c5c3_Var35 string
+			var templ_7745c5c3_Var36 string
-			templ_7745c5c3_Var35, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", len(d.SpecDiffs)))
+			templ_7745c5c3_Var36, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", len(d.SpecDiffs)))
 			if templ_7745c5c3_Err != nil {
-				return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/templates/run_detail.templ`, Line: 161, Col: 66}
+				return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/templates/run_detail.templ`, Line: 167, Col: 66}
 			}
-			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var35))
+			_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var36))
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 50, ")</h2></summary><ul class=\"diff-list\">")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 52, ")</h2></summary><ul class=\"diff-list\">")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 			for _, diff := range d.SpecDiffs {
-				var templ_7745c5c3_Var36 = []any{"diff-row", "diff-" + diff.Severity}
+				var templ_7745c5c3_Var37 = []any{"diff-row", "diff-" + diff.Severity}
-				templ_7745c5c3_Err = templ.RenderCSSItems(ctx, templ_7745c5c3_Buffer, templ_7745c5c3_Var36...)
+				templ_7745c5c3_Err = templ.RenderCSSItems(ctx, templ_7745c5c3_Buffer, templ_7745c5c3_Var37...)
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
-				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 51, "<li class=\"")
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 53, "<li class=\"")
-				if templ_7745c5c3_Err != nil {
-					return templ_7745c5c3_Err
-				}
-				var templ_7745c5c3_Var37 string
-				templ_7745c5c3_Var37, templ_7745c5c3_Err = templ.JoinStringErrs(templ.CSSClasses(templ_7745c5c3_Var36).String())
-				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/templates/run_detail.templ`, Line: 1, Col: 0}
-				}
-				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var37))
-				if templ_7745c5c3_Err != nil {
-					return templ_7745c5c3_Err
-				}
-				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 52, "\"><div class=\"diff-field\">")
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
 				var templ_7745c5c3_Var38 string
-				templ_7745c5c3_Var38, templ_7745c5c3_Err = templ.JoinStringErrs(diff.Field)
+				templ_7745c5c3_Var38, templ_7745c5c3_Err = templ.JoinStringErrs(templ.CSSClasses(templ_7745c5c3_Var37).String())
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/templates/run_detail.templ`, Line: 165, Col: 43}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/templates/run_detail.templ`, Line: 1, Col: 0}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var38))
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
-				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 53, "</div><div class=\"diff-expected\">expected: <code>")
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 54, "\"><div class=\"diff-field\">")
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
 				var templ_7745c5c3_Var39 string
-				templ_7745c5c3_Var39, templ_7745c5c3_Err = templ.JoinStringErrs(diff.Expected)
+				templ_7745c5c3_Var39, templ_7745c5c3_Err = templ.JoinStringErrs(diff.Field)
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/templates/run_detail.templ`, Line: 166, Col: 65}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/templates/run_detail.templ`, Line: 171, Col: 43}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var39))
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
-				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 54, "</code></div><div class=\"diff-actual\">actual: <code>")
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 55, "</div><div class=\"diff-expected\">expected: <code>")
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
 				var templ_7745c5c3_Var40 string
-				templ_7745c5c3_Var40, templ_7745c5c3_Err = templ.JoinStringErrs(diff.Actual)
+				templ_7745c5c3_Var40, templ_7745c5c3_Err = templ.JoinStringErrs(diff.Expected)
 				if templ_7745c5c3_Err != nil {
-					return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/templates/run_detail.templ`, Line: 167, Col: 59}
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/templates/run_detail.templ`, Line: 172, Col: 65}
 				}
 				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var40))
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
-				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 55, "</code></div></li>")
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 56, "</code></div><div class=\"diff-actual\">actual: <code>")
+				if templ_7745c5c3_Err != nil {
+					return templ_7745c5c3_Err
+				}
+				var templ_7745c5c3_Var41 string
+				templ_7745c5c3_Var41, templ_7745c5c3_Err = templ.JoinStringErrs(diff.Actual)
+				if templ_7745c5c3_Err != nil {
+					return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/web/templates/run_detail.templ`, Line: 173, Col: 59}
+				}
+				_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var41))
+				if templ_7745c5c3_Err != nil {
+					return templ_7745c5c3_Err
+				}
+				templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 57, "</code></div></li>")
 				if templ_7745c5c3_Err != nil {
 					return templ_7745c5c3_Err
 				}
 			}
-			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 56, "</ul></details>")
+			templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "</ul></details>")
 			if templ_7745c5c3_Err != nil {
 				return templ_7745c5c3_Err
 			}
 		}
-		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 57, "</section>")
+		templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 59, "</section>")
 		if templ_7745c5c3_Err != nil {
 			return templ_7745c5c3_Err
 		}