Reconcile recovered campaign targets (Refs #166)

src/lib/fleet-overview.ts

@@ -51,6 +51,7 @@ export interface FleetOperation {
     href: string;
     state: 'active' | 'attention' | 'done';
     startedAt: string;
+    evidenceAt?: string | null;
     activityLabel?: string;
     resultState?: string;
 }

src/lib/server/fleet/overview.ts

@@ -20,16 +20,23 @@ function readOperations(now: number, server?: string): FleetOperation[] {
     const operations: FleetOperation[] = [];
     const slots = handle
         .prepare(
-            `SELECT server, requested_at, error FROM restart_rollout WHERE healthy_at IS NULL ${scope('server')}`
+            `SELECT server, requested_at, checked_at, error FROM restart_rollout WHERE healthy_at IS NULL ${scope('server')}`
         )
-        .all(bindings) as { server: string; requested_at: string; error: string | null }[];
+        .all(bindings) as {
+        server: string;
+        requested_at: string;
+        checked_at: string | null;
+        error: string | null;
+    }[];
     const recovering = new Map(slots.map((slot) => [slot.server, slot]));
     const representedRecovery = new Set<string>();
     const campaigns = handle
         .prepare(
             `SELECT d.id, d.action, d.started_at, ds.server,
-        ds.restart_state, ds.error, ds.restarted_at, ds.players
+        ds.restart_state, ds.error, ds.restarted_at, ds.players,
+        COALESCE(op.finished_at,op.observed_at,op.checked_at,ds.checked_at,d.started_at) AS evidence_at
         FROM deployments d JOIN deployment_servers ds ON ds.deployment_id = d.id
+        LEFT JOIN server_deployments op ON op.id=ds.provisioning_id
         WHERE ds.server_deleted_at IS NULL ${scope('ds.server')} AND (ds.restart_state != 'done'
             ${server === undefined ? '' : 'OR ds.checked_at >= @since OR d.finished_at >= @since OR d.started_at >= @since'})`
         )
@@ -37,6 +44,7 @@ function readOperations(now: number, server?: string): FleetOperation[] {
         id: string;
         action: string;
         started_at: string;
+        evidence_at: string;
         server: string;
         restart_state: string;
         error: string | null;
@@ -72,6 +80,7 @@ function readOperations(now: number, server?: string): FleetOperation[] {
             label,
             state,
             startedAt: row.started_at,
+            evidenceAt: row.evidence_at,
             detail:
                 state === 'done'
                     ? 'Update applied and restart checks completed'
@@ -89,18 +98,21 @@ function readOperations(now: number, server?: string): FleetOperation[] {
     // index for detail history. Completed work is never loaded for the full fleet.
     const lifecycle = handle
         .prepare(
-            `SELECT id, server, action, state, stage, error, started_at FROM server_deployments
+            `SELECT id, server, action, state, stage, error, started_at,
+                COALESCE(finished_at,observed_at,checked_at,started_at) AS evidence_at FROM server_deployments
         WHERE server_deleted_at IS NULL ${scope('server')} AND state IN ('queued', 'running', 'waiting', 'awaiting-health')
         ${
             server === undefined
                 ? ''
                 : `UNION
-        SELECT id, server, action, state, stage, error, started_at FROM server_deployments
+        SELECT id, server, action, state, stage, error, started_at,
+            COALESCE(finished_at,observed_at,checked_at,started_at) AS evidence_at FROM server_deployments
         WHERE server_deleted_at IS NULL AND server = @server
             AND +state IN ('done', 'warning', 'cancelled', 'failed', 'interrupted') AND finished_at >= @since`
         }
         UNION
-        SELECT id, server, action, state, stage, error, started_at FROM server_deployments
+        SELECT id, server, action, state, stage, error, started_at,
+            COALESCE(finished_at,observed_at,checked_at,started_at) AS evidence_at FROM server_deployments
         WHERE server_deleted_at IS NULL ${scope('server')} AND +state IN ('failed', 'interrupted')
             AND rowid IN (SELECT MAX(rowid) FROM server_deployments WHERE 1=1 ${scope('server')} GROUP BY server)`
         )
@@ -112,6 +124,7 @@ function readOperations(now: number, server?: string): FleetOperation[] {
         stage: string;
         error: string | null;
         started_at: string;
+        evidence_at: string;
     }[];
     for (const row of lifecycle) {
         if (row.state === 'awaiting-health') representedRecovery.add(row.server);
@@ -127,6 +140,7 @@ function readOperations(now: number, server?: string): FleetOperation[] {
             server: row.server,
             state,
             startedAt: row.started_at,
+            evidenceAt: row.evidence_at,
             activityLabel: `${actionLabels[row.action as ProvisioningAction] ?? row.action} instance`,
             resultState: row.state,
             label:
@@ -145,12 +159,13 @@ function readOperations(now: number, server?: string): FleetOperation[] {
     }
     const betas = handle
         .prepare(
-            `SELECT b.id, b.opened_at, bs.server, bs.stage, bs.error
+            `SELECT b.id, b.opened_at, bs.server, bs.stage, bs.error, bs.updated_at
         FROM betas b JOIN beta_servers bs ON bs.beta_id = b.id WHERE bs.server_deleted_at IS NULL ${scope('bs.server')} AND b.state IN ('running', 'ready-promote')`
         )
         .all(bindings) as {
         id: number;
         opened_at: string;
+        updated_at: string;
         server: string;
         stage: string;
         error: string | null;
@@ -174,6 +189,7 @@ function readOperations(now: number, server?: string): FleetOperation[] {
             kind: 'beta',
             server: row.server,
             startedAt: row.opened_at,
+            evidenceAt: row.updated_at,
             state: row.stage === 'failed' ? 'attention' : 'active',
             label: stages[row.stage] ?? row.stage,
             detail: row.error ?? recovering.get(row.server)?.error ?? `Beta ${row.id}`,
@@ -191,6 +207,7 @@ function readOperations(now: number, server?: string): FleetOperation[] {
         kind: string;
         detail: string | null;
         requested_at: string;
+        finished_at: string | null;
     }[];
     for (const job of maintenance) {
         operations.push({
@@ -200,6 +217,7 @@ function readOperations(now: number, server?: string): FleetOperation[] {
             server: job.server,
             state: 'attention',
             startedAt: job.requested_at,
+            evidenceAt: job.finished_at ?? job.requested_at,
             label:
                 job.kind === 'update'
                     ? 'Update verification needs attention'
@@ -216,6 +234,7 @@ function readOperations(now: number, server?: string): FleetOperation[] {
             kind: 'recovery',
             server: slot.server,
             startedAt: slot.requested_at,
+            evidenceAt: slot.checked_at ?? slot.requested_at,
             state: slot.error ? 'attention' : 'active',
             label: 'Recovering from restart',
             detail: slot.error ?? 'Verifying server health',

src/lib/server/pipeline/campaign-recovery.ts

@@ -0,0 +1,125 @@
+import { posix } from 'node:path';
+import { db } from '../db/index.js';
+import { relDestForKind, type ArtifactKind } from '../config.js';
+import { recipeSummaryForDeployment } from '../fleet/launch-recipes.js';
+import { getServerScript } from '../fleet/inventory.js';
+import { recordProgress } from '../provisioning/progress.js';
+import type { LifecycleOperation } from '../provisioning/requests.js';
+
+/** Adopt proven replacement files, never health alone or an unrelated restart. */
+export function reconcileCampaignRecovery(
+    campaign: string,
+    failed: LifecycleOperation,
+    filesChanged: number
+): LifecycleOperation | null {
+    if (!['failed', 'interrupted', 'cancelled'].includes(failed.state)) return null;
+    return db().transaction(() => {
+        const files = db()
+            .prepare(
+                `SELECT kind, rel_path, content_hash, state FROM staged_file_changes
+                WHERE source_kind='campaign' AND source_id=? AND server=?`
+            )
+            .all(campaign, failed.server) as {
+            kind: ArtifactKind;
+            rel_path: string;
+            content_hash: string | null;
+            state: string;
+        }[];
+        // Partial staging and legacy campaigns without exact desired hashes need review.
+        if (
+            !files.length ||
+            files.length !== filesChanged ||
+            files.some((f) => f.state === 'cancelled')
+        )
+            return null;
+        const candidates = db()
+            .prepare(
+                `SELECT d.*, n.verified_at, n.files_json, i.generation AS current_generation
+                FROM server_deployments d
+                JOIN native_provisioning n ON n.deployment_id=d.id
+                JOIN native_instances i ON i.id=n.instance_id AND i.server=d.server AND i.retired_at IS NULL
+                JOIN ec_host_operations o ON o.id=n.operation_id AND o.instance_id=i.id AND o.host_id=i.host_id
+                WHERE d.server=? AND d.server_deleted_at IS NULL AND d.action='rebuild' AND d.state='done'
+                AND d.rowid > (SELECT rowid FROM server_deployments WHERE id=?)
+                AND d.started_at >= ? AND d.finished_at IS NOT NULL
+                AND n.applied_at IS NOT NULL AND n.verified_at IS NOT NULL
+                AND o.kind='instance.rebuild' AND o.state='completed' AND o.error IS NULL
+                AND n.generation=o.generation AND i.generation=o.generation+1
+                ORDER BY d.rowid DESC`
+            )
+            .all(
+                failed.server,
+                failed.id,
+                failed.finished_at ?? failed.started_at
+            ) as (LifecycleOperation & {
+            verified_at: string;
+            files_json: string | null;
+            current_generation: number;
+        })[];
+        for (const candidate of candidates) {
+            const recipe = recipeSummaryForDeployment(candidate.id)?.recipe;
+            if (!recipe) continue;
+            // A campaign rebuild can override the promoted recipe with staged files.
+            const overrides = JSON.parse(candidate.files_json ?? '[]') as {
+                path: string;
+                sha256: string | null;
+            }[];
+            const matches = files.every((file) => {
+                const override = overrides.find(
+                    (entry) =>
+                        entry.path ===
+                        posix.join(relDestForKind(file.kind, recipe.game), file.rel_path)
+                );
+                const installed = recipe.files.find(
+                    (entry) => entry.kind === file.kind && entry.path === file.rel_path
+                );
+                const hash = override ? override.sha256 : (installed?.sha256 ?? null);
+                return (
+                    hash === file.content_hash &&
+                    (getServerScript(failed.server, file.rel_path)?.sha256 ?? null) ===
+                        file.content_hash
+                );
+            });
+            if (!matches) continue;
+            const changed = db()
+                .prepare(
+                    `UPDATE deployment_servers SET provisioning_id=?, restarted_at=?
+                    WHERE deployment_id=? AND server=? AND server_deleted_at IS NULL
+                    AND provisioning_id=? AND restart_state IN ('pending','failed')`
+                )
+                .run(candidate.id, candidate.submitted_at, campaign, failed.server, failed.id);
+            if (!changed.changes) return null;
+            for (const id of [failed.id, candidate.id])
+                db()
+                    .prepare(
+                        `INSERT OR IGNORE INTO provisioning_sources(operation_id,kind,source_id,server)
+                        VALUES (?,'campaign',?,?)`
+                    )
+                    .run(id, campaign, failed.server);
+            // Recovery proves the final bytes, but cannot invent an old rollback snapshot.
+            db()
+                .prepare(
+                    `UPDATE staged_file_changes SET state='applied',applied_at=COALESCE(applied_at,?)
+                    WHERE source_kind='campaign' AND source_id=? AND server=? AND state IN ('pending','applying')`
+                )
+                .run(candidate.verified_at, campaign, failed.server);
+            const message = `${failed.server}: recovered by rebuild ${candidate.id} at ${candidate.verified_at} (generation ${candidate.current_generation}); original attempt ${failed.id} retained.`;
+            db()
+                .prepare('INSERT INTO campaign_events(deployment_id,message) VALUES (?,?)')
+                .run(campaign, message);
+            recordProgress(candidate.id, {
+                event: {
+                    kind: 'campaign-recovered',
+                    message,
+                    details: {
+                        campaignId: campaign,
+                        failedOperationId: failed.id,
+                        generation: candidate.current_generation
+                    }
+                }
+            });
+            return { ...candidate, checked_at: candidate.verified_at };
+        }
+        return null;
+    })();
+}

src/lib/server/pipeline/sweep.ts

@@ -1,6 +1,7 @@
 import { db } from '../db/index.js';
 import { emit } from './bus.js';
 import { requestRestart } from '../provisioning/index.js';
+import { reconcileCampaignRecovery } from './campaign-recovery.js';
 
 interface PendingRow {
     deployment_id: string;
@@ -40,7 +41,9 @@ async function runSweep(): Promise<SweepResult> {
     const result: SweepResult = { checked: 0, restarted: 0, healthy: 0, held: 0, failed: 0 };
     for (const row of pending) {
         if (!row.files_changed) continue; // A failed file write is not a restart request.
-        const operation = requestRestart(row.server, { kind: 'campaign', id: row.deployment_id });
+        const requested = requestRestart(row.server, { kind: 'campaign', id: row.deployment_id });
+        const operation =
+            reconcileCampaignRecovery(row.deployment_id, requested, row.files_changed) ?? requested;
         result.checked++;
         const state =
             operation.state === 'done'

src/routes/+page.svelte

@@ -391,6 +391,9 @@
                         (item) => item.server === server.name
                     )}
                     {@const operationIssue = operation && status.state === 'online'}
+                    {@const evidenceAt = operationIssue
+                        ? (operation.evidenceAt ?? operation.startedAt)
+                        : status.since}
                     {@const actionLabel = operationIssue ? 'Review operation' : status.actionLabel}
                     <div class="attention-row">
                         <div class="attention-identity">
@@ -412,9 +415,9 @@
                                 >{operationIssue ? operation.label : status.reason}</span
                             ><span class="concise-reason"
                                 >{operationIssue ? operation.label : status.shortReason}</span
-                            >{#if status.since}<span class="evidence-time" title={status.since}
+                            >{#if evidenceAt}<span class="evidence-time" title={evidenceAt}
                                     ><span class="evidence-separator">{' · '}</span>Evidence {elapsedSince(
-                                        status.since,
+                                        evidenceAt,
                                         $uiNow
                                     )} ago</span
                                 >{/if}

src/routes/api/campaigns/[id]/+server.ts

@@ -10,6 +10,13 @@ export async function GET({ params }: { params: { id: string } }) {
     const servers = db()
         .prepare('SELECT * FROM deployment_servers WHERE deployment_id = ? ORDER BY server')
         .all(params.id);
+    const operations = db()
+        .prepare(
+            `SELECT d.id,d.server,d.action,d.state,d.finished_at FROM provisioning_sources s
+            JOIN server_deployments d ON d.id=s.operation_id
+            WHERE s.kind='campaign' AND s.source_id=? ORDER BY d.started_at,d.rowid`
+        )
+        .all(params.id);
 
     // Don't ship blobs in the summary — the client asks for a single file when
     // it opens the diff viewer.
@@ -56,6 +63,7 @@ export async function GET({ params }: { params: { id: string } }) {
         fileContentsExpired,
         deployment: dep,
         servers,
+        operations,
         files,
         scripts,
         events,

src/routes/api/campaigns/[id]/resume/+server.ts

@@ -9,18 +9,42 @@ export function POST({ params }: { params: { id: string } }) {
             .prepare(
                 `SELECT ds.server, d.state FROM deployment_servers ds
             LEFT JOIN server_deployments d ON d.id = ds.provisioning_id
-            WHERE ds.deployment_id = ? AND ds.server_deleted_at IS NULL AND ds.restart_state IN ('held','manual')`
+            WHERE ds.deployment_id = ? AND ds.server_deleted_at IS NULL
+            AND (ds.restart_state IN ('held','manual') OR (ds.restart_state='failed' AND ds.files_changed>0))`
             )
             .all(params.id) as { server: string; state: string | null }[];
         for (const row of rows) {
             if (row.state && ['failed', 'interrupted', 'cancelled'].includes(row.state)) {
                 releaseRestartRequest(row.server, { kind: 'campaign', id: params.id });
+                // A failed health check may follow successful file application. A newer
+                // generation can have different files, so retry the retained desired bytes.
+                // Keep the original rollback image and refresh an existing compare guard.
+                db()
+                    .prepare(
+                        `UPDATE staged_file_changes SET state='pending',applied_at=NULL,
+                        preserve_before_image=1,expected_before_sha=CASE WHEN check_before=1 THEN content_hash ELSE expected_before_sha END
+                        WHERE source_kind='campaign' AND source_id=? AND server=? AND state='applied'`
+                    )
+                    .run(params.id, row.server);
+                db()
+                    .prepare(
+                        `UPDATE deployment_servers SET restarted_at=NULL,checked_at=NULL,players=NULL
+                        WHERE deployment_id=? AND server=?`
+                    )
+                    .run(params.id, row.server);
+                db()
+                    .prepare('INSERT INTO campaign_events(deployment_id,message) VALUES (?,?)')
+                    .run(
+                        params.id,
+                        `${row.server}: retry requested; previous operation retained in history.`
+                    );
             }
         }
         const result = db()
             .prepare(
                 `UPDATE deployment_servers SET restart_state = 'pending', error = NULL
-            WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state IN ('held','manual')`
+            WHERE deployment_id = ? AND server_deleted_at IS NULL
+            AND (restart_state IN ('held','manual') OR (restart_state='failed' AND files_changed>0))`
             )
             .run(params.id);
         if (result.changes)
@@ -28,7 +52,10 @@ export function POST({ params }: { params: { id: string } }) {
         return result.changes;
     })();
     if (!changed)
-        return json({ error: 'This campaign has no held changes to resume.' }, { status: 409 });
+        return json(
+            { error: 'This campaign has no held or failed activations to retry.' },
+            { status: 409 }
+        );
     void kick().catch((err) => console.error('[campaign] activation kick failed:', err));
     return json({ ok: true }, { status: 202 });
 }

src/routes/campaigns/[id]/+page.svelte

@@ -63,6 +63,13 @@
             rolled_back_by: string | null;
         };
         servers: ServerRow[];
+        operations: {
+            id: string;
+            server: string;
+            action: string;
+            state: string;
+            finished_at: string | null;
+        }[];
         files: FileRow[];
         events: ActivityEvent[];
         watchdogIncidentId?: string | null;
@@ -386,9 +393,16 @@
                             >All servers activated</span
                         >{/if}
                 </div>
-                {#if writable && detail.servers.some((server) => !server.server_deleted_at && ['held', 'manual'].includes(server.restart_state))}
+                {#if writable && detail.servers.some((server) => !server.server_deleted_at && (['held', 'manual'].includes(server.restart_state) || (server.restart_state === 'failed' && server.files_changed > 0)))}
                     <Button size="sm" onclick={resume} loading={resuming} disabled={resuming}
-                        >Resume when empty</Button
+                        >{detail.servers.some(
+                            (server) =>
+                                !server.server_deleted_at &&
+                                server.restart_state === 'failed' &&
+                                server.files_changed > 0
+                        )
+                            ? 'Retry when empty'
+                            : 'Resume when empty'}</Button
                     >
                 {/if}
             </div>
@@ -525,6 +539,20 @@
                                                         $page.url
                                                     )}>View provisioning</a
                                                 >{/if}
+                                            {#each detail.operations.filter((operation) => operation.server === s.server && operation.id !== s.provisioning_id) as operation (operation.id)}
+                                                <a
+                                                    href={contextHref(
+                                                        `/provisioning/${encodeURIComponent(operation.id)}`,
+                                                        $page.url
+                                                    )}
+                                                >
+                                                    Previous {operation.action}: {operation.state}
+                                                    {#if operation.finished_at}
+                                                        · <RelativeTime
+                                                            value={operation.finished_at}
+                                                        />{/if}
+                                                </a>
+                                            {/each}
                                         </div>
                                     </div>
                                 </td>