Offer rebuild retry for failed creation commands

docs/guides/collections.md

@@ -287,12 +287,22 @@ server launch until the health checks pass.
 
 For failed or interrupted creations, **Resume health verification** appears only
 when the original server is still registered on its assigned port and no later
-lifecycle operation has targeted that ID. It resumes observation only; it never
+lifecycle operation has targeted that ID. Native commands must have completed
-recreates or restarts a container. Missing servers and later destruction attempts
+before readiness verification can resume. If the container is unhealthy, the
-require inspection rather than replaying provisioning automatically.
+operation may continue its remaining automatic restart and rebuild recovery steps.
-After an application restart, the worker automatically resumes observation of
+
-eligible interrupted startups so their occupied slots are retained. It never
+When the host conclusively reports that the initial creation command failed,
-repeats container creation or deployment as part of this recovery.
+**Retry with rebuild** starts a new provisioning operation for the same lobby and
+port. It deletes the container and instance files and uses the latest promoted
+setup. Normal player safety, capacity, and protected work checks still apply.
+The original failure stays in history and links to the recovery operation.
+This retry is unavailable for unknown command results, replacement instances,
+cancelled creation cleanup, or a later operation on the same server.
+
+After an application restart, the worker reconciles the original host command
+before continuing eligible interrupted startups. Unknown command results retain
+their original command ID and occupied slots until the host confirms the outcome.
+Missing server registrations and later destruction attempts require inspection.
 
 ## API
 
@@ -339,7 +349,9 @@ there while preserving query parameters.
 - `POST /api/servers`: existing request plus optional `previewId`; the game and
   map must match that preview. `expectedName` remains supported.
 - `GET /api/deployments/:id`: includes `recipe`, `blueprintName`, and
-  `canResumeVerification` for lifecycle operations.
+  `canResumeVerification` and `canRetryWithRebuild` for lifecycle operations.
 - `POST /api/deployments/:id/resume`: resume eligible creation health checks.
+- `POST /api/provisioning/:id/retry-rebuild`: request a new rebuild for an eligible
+  failed creation and link it from the original operation.
 
 Collection launches begin processing immediately after all requests are queued.

src/lib/provisioning-detail.ts

@@ -101,6 +101,7 @@ export interface ProvisioningDetail {
     safetyOverride?: { actorId: string; actorName: string; confirmedAt: string } | null;
     canCancel?: boolean;
     canCancelCreation?: boolean;
+    canRetryWithRebuild?: boolean;
     cancellation?: { cleanupId: string; state: string; stage: string; error: string | null } | null;
     backend?: 'native';
     workerActivity?: WorkerActivity[];

src/lib/server/provisioning/history.ts

@@ -14,6 +14,7 @@ import { canCancelCreation } from './cancel-creation.js';
 import { canCancelProvisioningJob, provisioningCancelled, rebalanceCancellationGuard } from './cancellation.js';
 import { committedRebalanceStartup } from './concurrency.js';
 import { nativeDeployment } from '../backends/native.js';
+import { ensureHostSchema } from '../hosts/schema.js';
 import { safetyOverride, safetyOverrideAvailability } from './safety-override.js';
 import { serverTitle } from '../../ui/server-labels.js';
 import { provisioningQueueStatus } from '../../ui/provisioning-queue.js';
@@ -149,7 +150,8 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
             activity: { ...activitySummary(id), failureExcerpt: latestFailureEvent(id) }, sources,
             recipe: launch?.recipe ?? null, blueprintName: launch?.blueprintName ?? null,
             canResumeVerification: canResumeVerification(id), canResumeCleanup: canResumeCleanup(id),
-            canResumeRebuild: canResumeRebuild(id), canCancelRemoval: canCancelRemoval(id), collection: collection ?? null,
+            canResumeRebuild: canResumeRebuild(id), canRetryWithRebuild: canRetryWithRebuild(id),
+            canCancelRemoval: canCancelRemoval(id), collection: collection ?? null,
             progress: readProgress(row), capacity: capacityFor(deployment, generatedAt),
             warningMessages: warningMessages(deployment.warnings), generatedAt };
     })();
@@ -197,17 +199,43 @@ export function resumeCleanup(id: string): boolean {
 }
 
 export function canResumeVerification(id: string): boolean {
+    ensureHostSchema(db());
     return !!db().prepare(`SELECT d.id FROM server_deployments d
         JOIN server_creation_jobs j ON j.deployment_id = d.id
-        JOIN servers s ON s.name = j.server AND s.game_port = j.port
+        JOIN servers s ON s.name = j.server AND s.game_port = j.port AND s.destroyed_at IS NULL
-        WHERE d.id = ? AND d.server_deleted_at IS NULL AND (d.action = 'create' OR (d.action = 'rebuild' AND (EXISTS (
+        LEFT JOIN native_provisioning n ON n.deployment_id=d.id
-            SELECT 1 FROM server_rebuild_jobs r WHERE r.deployment_id = d.id AND r.removed_at IS NOT NULL) OR EXISTS (
+        LEFT JOIN native_instances i ON i.id=n.instance_id AND i.server=d.server AND i.retired_at IS NULL
-            SELECT 1 FROM ec_host_migrations m JOIN native_provisioning n ON n.deployment_id=m.deployment_id
+        LEFT JOIN ec_host_operations o ON o.id=n.operation_id
-                WHERE m.deployment_id=d.id AND m.removed_at IS NOT NULL AND n.operation_id IS NOT NULL)))) AND d.state IN ('failed', 'interrupted')
+        WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.action IN ('create','rebuild')
+        AND d.state IN ('failed', 'interrupted')
+        AND ((n.deployment_id IS NOT NULL AND i.port=j.port AND o.state='completed'
+            AND o.instance_id=i.id AND o.host_id=i.host_id AND o.generation=n.generation
+            AND ((o.kind IN ('instance.create','instance.restart') AND i.generation=n.generation)
+                OR (o.kind='instance.rebuild' AND i.generation IN (n.generation,n.generation+1))))
+            OR (n.deployment_id IS NULL AND (d.action='create' OR EXISTS (
+                SELECT 1 FROM server_rebuild_jobs r WHERE r.deployment_id=d.id AND r.removed_at IS NOT NULL))))
         AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
         .get(id);
 }
 
+/** A conclusively failed initial create can enter the normal rebuild flow. */
+export function canRetryWithRebuild(id: string): boolean {
+    ensureHostSchema(db());
+    return !!db().prepare(`SELECT d.id FROM server_deployments d
+        JOIN server_creation_jobs j ON j.deployment_id=d.id
+        JOIN servers s ON s.name=d.server AND s.game_port=j.port AND s.destroyed_at IS NULL
+        JOIN native_provisioning n ON n.deployment_id=d.id AND n.recovery_step IS NULL
+        JOIN native_instances i ON i.id=n.instance_id AND i.server=d.server AND i.retired_at IS NULL
+            AND i.generation=n.generation AND i.port=j.port
+        JOIN ec_host_operations o ON o.id=n.operation_id AND o.instance_id=i.id
+            AND o.host_id=i.host_id AND o.generation=n.generation
+        WHERE d.id=? AND d.server_deleted_at IS NULL AND d.action='create'
+        AND d.state IN ('failed','interrupted') AND o.kind='instance.create' AND o.state='failed'
+        AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations c WHERE c.creation_id=d.id)
+        AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server=d.server AND newer.rowid>d.rowid)`)
+        .get(id);
+}
+
 export function canResumeRebuild(id: string): boolean {
     return !!db().prepare(`SELECT d.id FROM server_deployments d JOIN server_rebuild_jobs r ON r.deployment_id = d.id
         WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.state IN ('failed','interrupted') AND d.stage = ? AND r.removed_at IS NULL
@@ -226,7 +254,7 @@ export function resumeRebuild(id: string): boolean {
     })();
 }
 
-/** Resume observation only. Never repeat a container creation or restart. */
+/** Resume readiness after a completed command; unhealthy recovery still applies. */
 export function resumeVerification(id: string): boolean {
     return db().transaction(() => {
         if (!canResumeVerification(id)) return false;

src/lib/server/provisioning/index.ts

@@ -1,6 +1,6 @@
 // Request/observation boundary used by APIs, campaigns, beta tests, and schedules.
 // Execution is driven exclusively by the provisioning worker.
-export { requestLifecycle, requestRestart, requestWatchdogLifecycle, escalateWatchdogRestart, provisioningOperation, cancelProvisioning, releaseRestartRequest } from './requests.js';
+export { requestLifecycle, requestRetryWithRebuild, requestRestart, requestWatchdogLifecycle, escalateWatchdogRestart, provisioningOperation, cancelProvisioning, releaseRestartRequest } from './requests.js';
 export type { LifecycleOperation, RestartSource } from './requests.js';
 export { queueCreateServer, destroyServer, previewCreation, creationQueue, nextLobbyNumber } from './instances.js';
 export { getDeployment, listDeployments, resumeVerification, resumeCleanup, resumeRebuild, cancelRemoval } from './history.js';

src/lib/server/provisioning/requests.ts

@@ -3,7 +3,7 @@ import { currentRequestUsername } from '../auth/request-context.js';
 import { maintenanceAllowed } from '../maintenance/policies.js';
 import { db } from '../db/index.js';
 import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
-import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent } from './history.js';
+import { beginDeployment, canRetryWithRebuild, deploymentStage, finishDeployment, recordDeploymentEvent } from './history.js';
 import { queueRebuildServer } from './instances.js';
 import { runtimeActions, type RuntimeAction } from '../../provisioning.js';
 import { recordProgress } from './progress.js';
@@ -166,6 +166,23 @@ export function requestLifecycle(server: string, action: RuntimeAction) {
 
 type LifecycleRequestResult = { ok: true; deploymentId: string } | { ok: false; status: number; error: string };
 
+/** Keep the failed launch immutable and let normal rebuild admission own recovery. */
+export function requestRetryWithRebuild(id: string): LifecycleRequestResult {
+    return db().transaction(() => {
+        if (!canRetryWithRebuild(id)) return { ok: false as const, status: 409,
+            error: 'This failed creation can no longer be retried with a rebuild. Refresh and inspect its current operation.' };
+        const original = provisioningOperation(id)!;
+        const result = requestLifecycle(original.server, 'rebuild');
+        if (!result.ok) return result;
+        recordProgress(id, { recovery: { state: 'transferred', operationId: result.deploymentId,
+            message: 'A new rebuild was requested for this server. The failed creation and its error are preserved.' },
+            event: { kind: 'retry-requested', message: 'Retry with rebuild requested', details: { operationId: result.deploymentId } } });
+        recordProgress(result.deploymentId, { event: { kind: 'retry-requested',
+            message: 'Rebuild requested after failed creation', details: { previousOperationId: id } } });
+        return result;
+    })();
+}
+
 function watchdogRequestGuard(server: string, incidentId: string): Extract<LifecycleRequestResult, { ok: false }> | null {
     const incident = db().prepare("SELECT kind FROM watchdog_incidents WHERE id = ? AND server = ? AND server_deleted_at IS NULL AND state IN ('open','recovering')").get(incidentId, server) as { kind: 'health' | 'drift' } | undefined;
     if (!incident) return { ok: false, status: 409, error: 'The Watchdog incident is no longer active.' };

src/lib/ui/provisioning-presentation.ts

@@ -76,11 +76,13 @@ export function provisioningOutcome(detail: ProvisioningDetail) {
     let description: string;
     if (detail.canResumeCleanup) description = 'The remote deployment was deleted. Resume the remaining local cleanup without repeating container removal.';
     else if (detail.canResumeRebuild) description = 'The remote deployment was deleted. Resume local cleanup, then create a fresh instance in the reserved server slot.';
-    else if (detail.canResumeVerification) description = 'Resume observation of the existing instance. This does not create, restart, or rebuild a container.';
+    else if (detail.canResumeVerification) description = 'Resume readiness checks for the existing instance. An unhealthy container may continue through this operation’s remaining restart and rebuild recovery steps.';
+    else if (detail.canRetryWithRebuild) description = 'The host reported that creation failed. Retry with rebuild starts a new attempt for the same lobby and port using the latest promoted setup.';
     else if (detail.cancellation) description = detail.cancellation.state === 'done'
         ? 'Creation was cancelled. Its instance files, container, and port reservations have been removed.'
         : 'Creation was cancelled. Cleanup is tracked separately below; ports remain reserved until the host confirms removal.';
     else if (operation.state === 'cancelled') description = 'This request is cancelled. Recorded prior work is preserved below; remaining work will not run.';
+    else if (failure && progress?.recovery?.state === 'transferred' && progress.recovery.operationId && progress.recovery.operationId !== operation.id) description = 'A recovery operation has been requested for this server. Follow the linked operation below; this record keeps the original failure.';
     else if (failure) description = progress?.command?.state === 'uncertain'
         ? 'The remote response was uncertain. Inspect the recorded result and server before requesting another operation.'
         : 'Review the recorded failure and execution evidence before submitting another operation.';
@@ -162,6 +164,7 @@ export function isProvisioningDetail(value: unknown, id: string): value is Provi
     if (!record(value) || !record(value.deployment)) return false;
     if (value.canCancel !== undefined && typeof value.canCancel !== 'boolean') return false;
     if (value.canCancelCreation !== undefined && typeof value.canCancelCreation !== 'boolean') return false;
+    if (value.canRetryWithRebuild !== undefined && typeof value.canRetryWithRebuild !== 'boolean') return false;
     if (value.cancellation !== undefined && value.cancellation !== null && !(record(value.cancellation)
         && text(value.cancellation.cleanupId) && text(value.cancellation.state) && text(value.cancellation.stage)
         && nullableText(value.cancellation.error))) return false;

src/lib/ui/provisioning-refresh.ts

@@ -1,4 +1,4 @@
-export type ProvisioningMutation = 'resume' | 'cancel' | 'cancel-creation' | 'override';
+export type ProvisioningMutation = 'resume' | 'retry-rebuild' | 'cancel' | 'cancel-creation' | 'override';
 
 export interface ProvisioningRefreshState<T> {
     id: string;

src/routes/api/provisioning/[id]/retry-rebuild/+server.ts

@@ -0,0 +1,10 @@
+import { json } from '@sveltejs/kit';
+import { requestRetryWithRebuild } from '$lib/server/provisioning/index.js';
+import { kickProvisioning } from '$lib/server/pipeline/worker.js';
+
+export function POST({ params }: { params: { id: string } }) {
+    const result = requestRetryWithRebuild(params.id);
+    if (!result.ok) return json({ error: result.error }, { status: result.status });
+    void kickProvisioning().catch(err => console.error('[provisioning] rebuild retry kick failed:', err));
+    return json(result, { status: 202 });
+}

src/routes/provisioning/[id]/+page.svelte

@@ -59,6 +59,17 @@
     });
     onMount(() => pollVisible(() => { now = Date.now(); return controller.refresh(); }, 5000, false));
     onDestroy(() => controller.dispose());
+    async function retryWithRebuild() {
+        if (!deployment || !detail?.canRetryWithRebuild || view.busy) return;
+        const id = deployment.id;
+        const confirmed = await confirmAction({
+            title: 'Retry creation with a rebuild?',
+            message: `Rebuild ${serverTitle(deployment.server)} (${deployment.server}) using the latest promoted setup. This keeps the same lobby and port, deletes its container and all instance files, and starts a new provisioning operation.\n\nPlayer safety, capacity limits, and protected work checks still apply. The original failure remains in this operation’s history.`,
+            confirmLabel: 'Retry with rebuild', danger: true
+        });
+        if (confirmed && view.id === id && detail?.canRetryWithRebuild) await controller.act('retry-rebuild',
+            'Rebuild retry requested. Follow the linked recovery operation below for progress.');
+    }
     async function overrideSafety() {
         if (!deployment || !canOverride || view.busy || detail?.safetyOverrideUnavailableReason) return;
         const id = deployment.id;
@@ -107,10 +118,11 @@
             {#if detail.warningMessages.length}<div class="warnings"><h3>Recorded warnings</h3><ul>{#each detail.warningMessages as warning}<li>{warning}</li>{/each}</ul></div>{/if}
 
             {#if detail.safetyOverride}<div class="warnings"><h3>Admin override enabled</h3><p>Player safety checks bypassed for this operation. Confirmed by {detail.safetyOverride.actorName} · <RelativeTime value={detail.safetyOverride.confirmedAt} />.</p></div>{/if}
-            {#if detail.canResumeVerification || detail.canResumeCleanup || detail.canResumeRebuild || detail.canCancel || detail.canCancelCreation || canOverride}
+            {#if detail.canResumeVerification || detail.canResumeCleanup || detail.canResumeRebuild || detail.canRetryWithRebuild || detail.canCancel || detail.canCancelCreation || canOverride}
                 <div class="recovery-actions">
                     {#if canOverride}<div><Button variant="danger" disabled={!!view.busy || !!detail.safetyOverrideUnavailableReason} loading={view.busy === 'override'} onclick={overrideSafety}>Admin override…</Button><p class="action-help">{detail.safetyOverrideUnavailableReason ?? 'Confirm to bypass player safety checks for this operation.'}</p></div>{/if}
-                    {#if detail.canResumeVerification}<Button variant="primary" loading={view.busy === 'resume'} disabled={!!view.busy} onclick={() => controller.act('resume', 'Health verification requested. The existing instance will be observed without another creation or restart.')}>Resume health verification</Button>{/if}
+                    {#if detail.canResumeVerification}<Button variant="primary" loading={view.busy === 'resume'} disabled={!!view.busy} onclick={() => controller.act('resume', 'Health verification requested. An unhealthy container may continue through this operation’s remaining recovery steps.')}>Resume health verification</Button>{/if}
+                    {#if detail.canRetryWithRebuild}<div><Button variant="primary" loading={view.busy === 'retry-rebuild'} disabled={!!view.busy} onclick={retryWithRebuild}>Retry with rebuild</Button><p class="action-help">Starts a new rebuild for the same lobby and port. Deletes instance files and uses the latest promoted setup.</p></div>{/if}
                     {#if detail.canResumeRebuild}<Button variant="primary" loading={view.busy === 'resume'} disabled={!!view.busy} onclick={() => controller.act('resume', 'Rebuild cleanup requested. Fresh creation waits for local cleanup to finish.')}>Resume rebuild cleanup</Button>{/if}
                     {#if detail.canResumeCleanup}<Button variant="primary" loading={view.busy === 'resume'} disabled={!!view.busy} onclick={() => controller.act('resume', 'Local cleanup requested. Remote container deletion will not be repeated.')}>Resume cleanup</Button>{/if}
                     {#if detail.canCancel}<div><Button disabled={!!view.busy} loading={view.busy === 'cancel'} onclick={() => controller.act('cancel', 'Operation cancelled. Already-submitted commands may finish; no further steps will be started.')}>Cancel operation</Button><p class="action-help">Stops remaining work and health checks. Changes already made are not undone.</p></div>{/if}

tests/unit/native-provisioning.test.ts

@@ -1503,6 +1503,169 @@ describe('automatic capacity admission and placement', () => {
     });
 });
 
+describe('native verification resume and explicit failed-creation retry', () => {
+    async function failedCreation(state: 'failed' | 'interrupted' = 'failed') {
+        const job = await queue(); await advance();
+        const original = (await operationFor(job.deploymentId))!;
+        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        reportHostOperation(hostId, original.id, { state: 'failed', error: 'Host runtime preflight failed' });
+        await advance();
+        if (state === 'interrupted') {
+            const { db } = await import('../../src/lib/server/db/index.js');
+            db().prepare("UPDATE server_deployments SET state='interrupted' WHERE id=?").run(job.deploymentId);
+        }
+        return { job, original };
+    }
+
+    it.each(['failed', 'interrupted'] as const)('retries a %s creation through a new rebuild while retaining the original failure', async state => {
+        const { job, original } = await failedCreation(state);
+        const history = await import('../../src/lib/server/provisioning/history.js');
+        expect(history.getDeployment(job.deploymentId)).toMatchObject({ canResumeVerification: false, canRetryWithRebuild: true });
+        const { POST: resume } = await import('../../src/routes/api/provisioning/[id]/resume/+server.js');
+        expect(resume({ params: { id: job.deploymentId } }).status).toBe(409);
+        const { POST: retry } = await import('../../src/routes/api/provisioning/[id]/retry-rebuild/+server.js');
+        const response = await retry({ params: { id: job.deploymentId } });
+        expect(response.status).toBe(202);
+        const next = await response.json() as { ok: boolean; deploymentId: string };
+        expect(next.ok).toBe(true);
+        expect(next.deploymentId).not.toBe(job.deploymentId);
+        expect(await detail(next.deploymentId)).toMatchObject({ action: 'rebuild', state: 'waiting', server: job.name });
+        expect(history.getDeployment(job.deploymentId)).toMatchObject({
+            canRetryWithRebuild: false,
+            deployment: { state, error: 'Host runtime preflight failed' },
+            progress: { recovery: { state: 'transferred', operationId: next.deploymentId } }
+        });
+        expect((await retry({ params: { id: job.deploymentId } })).status).toBe(409);
+        const { db } = await import('../../src/lib/server/db/index.js');
+        expect(db().prepare('SELECT COUNT(*) AS count FROM server_deployments WHERE server=?').get(job.name)).toEqual({ count: 2 });
+        const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
+        expect(nativeInstance(job.name)).toMatchObject({ id: original.instanceId, generation: 1, port: original.payload.port });
+        remote.state = 'missing'; await advance();
+        const rebuilding = (await operationFor(next.deploymentId))!;
+        expect(rebuilding).toMatchObject({ kind: 'instance.rebuild', generation: 1, instanceId: original.instanceId,
+            payload: { name: job.name, port: original.payload.port, nextGeneration: 2, env: { PLUTO_SERVER_KEY: 'saved-key' } } });
+        expect(rebuilding.id).not.toBe(original.id);
+        expect(nativeInstance(job.name)?.generation).toBe(1);
+        remote.state = 'running'; await acknowledge(next.deploymentId);
+        expect((await detail(next.deploymentId)).state).toBe('done');
+        expect(nativeInstance(job.name)?.generation).toBe(2);
+        // An interrupted original is reconciled to its already-failed receipt
+        // on the next worker pass; the failure itself is never cleared.
+        expect(await detail(job.deploymentId)).toMatchObject({ state: 'failed', error: 'Host runtime preflight failed' });
+        expect(await operationFor(job.deploymentId)).toMatchObject({ id: original.id, state: 'failed' });
+        expect(db().prepare("SELECT kind FROM ec_host_operations WHERE kind IN ('instance.create','instance.rebuild','instance.restart') ORDER BY rowid").all())
+            .toEqual([{ kind: 'instance.create' }, { kind: 'instance.rebuild' }]);
+    });
+
+    it.each([
+        ['create', false], ['create', true], ['rebuild', false], ['rebuild', true],
+        ['recovery-restart', false], ['recovery-restart', true], ['recovery-rebuild', false], ['recovery-rebuild', true]
+    ] as const)('resumes completed %s verification with the same command (receipt applied=%s)', async (action, applied) => {
+        let job = await queue(); await advance();
+        if (action === 'rebuild') {
+            await acknowledge(job.deploymentId);
+            const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
+            const rebuild = queueNativeRebuild(job.name, 'test');
+            if (!rebuild.ok) throw new Error(rebuild.message);
+            job = rebuild; await advance();
+        } else if (action.startsWith('recovery')) {
+            remote.state = 'unhealthy'; await acknowledge(job.deploymentId);
+            if (action === 'recovery-rebuild') await acknowledge(job.deploymentId);
+        }
+        const operation = (await operationFor(job.deploymentId))!;
+        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        remote.state = 'starting'; reportHostOperation(hostId, operation.id, { state: 'completed' });
+        if (applied) await advance();
+        const history = await import('../../src/lib/server/provisioning/history.js');
+        history.finishDeployment(job.deploymentId, { ok: false, message: 'Verification interrupted for inspection' });
+        expect(history.getDeployment(job.deploymentId)).toMatchObject({ canResumeVerification: true, canRetryWithRebuild: false });
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const commands = db().prepare("SELECT id FROM ec_host_operations WHERE kind LIKE 'instance.%' ORDER BY rowid").all();
+        const { POST } = await import('../../src/routes/api/provisioning/[id]/resume/+server.js');
+        expect(POST({ params: { id: job.deploymentId } }).status).toBe(202);
+        remote.state = 'running'; await advance();
+        expect((await detail(job.deploymentId)).state).toBe('done');
+        expect((await operationFor(job.deploymentId))!.id).toBe(operation.id);
+        expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind LIKE 'instance.%' ORDER BY rowid").all()).toEqual(commands);
+        const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
+        expect(nativeInstance(job.name)?.generation).toBe(operation.generation! + (operation.kind === 'instance.rebuild' ? 1 : 0));
+        expect(history.getDeployment(job.deploymentId)!.events).toContainEqual(expect.objectContaining({
+            kind: 'outcome', message: 'Failed: Verification interrupted for inspection'
+        }));
+    });
+
+    it.each(['pending', 'accepted', 'running', 'unknown', 'failed'] as const)('does not offer verification of a %s command or replay an ambiguous result', async state => {
+        const job = await queue(); await advance();
+        const operation = (await operationFor(job.deploymentId))!;
+        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        if (state !== 'pending') reportHostOperation(hostId, operation.id, { state });
+        const history = await import('../../src/lib/server/provisioning/history.js');
+        history.finishDeployment(job.deploymentId, { ok: false, message: 'Controller observation interrupted' });
+        expect(history.getDeployment(job.deploymentId)).toMatchObject({ canResumeVerification: false, canRetryWithRebuild: state === 'failed' });
+        expect(history.resumeVerification(job.deploymentId)).toBe(false);
+        if (state !== 'failed') {
+            const { requestRetryWithRebuild } = await import('../../src/lib/server/provisioning/requests.js');
+            expect(requestRetryWithRebuild(job.deploymentId)).toMatchObject({ ok: false, status: 409 });
+        }
+        const { db } = await import('../../src/lib/server/db/index.js');
+        if (state === 'unknown') db().prepare("UPDATE server_deployments SET state='interrupted' WHERE id=?").run(job.deploymentId);
+        await advance();
+        expect((await operationFor(job.deploymentId))!.id).toBe(operation.id);
+        expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind IN ('instance.create','instance.restart','instance.rebuild')").all())
+            .toEqual([{ id: operation.id }]);
+    });
+
+    it.each(['instance generation', 'command generation', 'command instance', 'command host', 'retired instance', 'server port', 'newer work', 'creation cleanup', 'recovery attempt'])
+        ('refuses to retry a failed creation after its %s changes', async change => {
+            const { job, original } = await failedCreation();
+            const { db } = await import('../../src/lib/server/db/index.js');
+            if (change === 'instance generation') db().prepare('UPDATE native_instances SET generation=generation+1 WHERE server=?').run(job.name);
+            if (change === 'command generation') db().prepare('UPDATE ec_host_operations SET generation=generation+1 WHERE id=?').run(original.id);
+            if (change === 'command instance') db().prepare("UPDATE ec_host_operations SET instance_id='different-instance' WHERE id=?").run(original.id);
+            if (change === 'command host') {
+                const { createHost } = await import('../../src/lib/server/hosts/index.js');
+                db().prepare('UPDATE ec_host_operations SET host_id=? WHERE id=?').run(createHost({ name: 'Different host' }).id, original.id);
+            }
+            if (change === 'retired instance') db().prepare('UPDATE native_instances SET retired_at=? WHERE server=?').run(new Date().toISOString(), job.name);
+            if (change === 'server port') db().prepare('UPDATE servers SET game_port=game_port+1 WHERE name=?').run(job.name);
+            if (change === 'newer work') {
+                const { beginDeployment } = await import('../../src/lib/server/provisioning/history.js');
+                beginDeployment(job.name, 'restart');
+            }
+            if (change === 'creation cleanup') {
+                const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
+                expect(cancelCreation(job.deploymentId)).toEqual(expect.any(String));
+            }
+            if (change === 'recovery attempt') db().prepare("UPDATE native_provisioning SET recovery_step='restart' WHERE deployment_id=?").run(job.deploymentId);
+            const history = await import('../../src/lib/server/provisioning/history.js');
+            const { requestRetryWithRebuild } = await import('../../src/lib/server/provisioning/requests.js');
+            expect(history.getDeployment(job.deploymentId)?.canRetryWithRebuild).toBe(false);
+            expect(requestRetryWithRebuild(job.deploymentId)).toMatchObject({ ok: false, status: 409 });
+            expect(db().prepare("SELECT id FROM server_deployments WHERE action='rebuild'").all()).toEqual([]);
+        });
+
+    it.each(['instance generation', 'command generation', 'command instance', 'command host', 'retired instance', 'server port', 'newer work'])
+        ('refuses to resume a completed creation after its %s changes', async change => {
+            const job = await queue(); await advance();
+            const original = (await operationFor(job.deploymentId))!;
+            const { reportHostOperation, createHost } = await import('../../src/lib/server/hosts/index.js');
+            reportHostOperation(hostId, original.id, { state: 'completed' });
+            const history = await import('../../src/lib/server/provisioning/history.js');
+            history.finishDeployment(job.deploymentId, { ok: false, message: 'Verification interrupted' });
+            const { db } = await import('../../src/lib/server/db/index.js');
+            if (change === 'instance generation') db().prepare('UPDATE native_instances SET generation=generation+1 WHERE server=?').run(job.name);
+            if (change === 'command generation') db().prepare('UPDATE ec_host_operations SET generation=generation+1 WHERE id=?').run(original.id);
+            if (change === 'command instance') db().prepare("UPDATE ec_host_operations SET instance_id='different-instance' WHERE id=?").run(original.id);
+            if (change === 'command host') db().prepare('UPDATE ec_host_operations SET host_id=? WHERE id=?').run(createHost({ name: 'Different host' }).id, original.id);
+            if (change === 'retired instance') db().prepare('UPDATE native_instances SET retired_at=? WHERE server=?').run(new Date().toISOString(), job.name);
+            if (change === 'server port') db().prepare('UPDATE servers SET game_port=game_port+1 WHERE name=?').run(job.name);
+            if (change === 'newer work') history.beginDeployment(job.name, 'restart');
+            expect(history.getDeployment(job.deploymentId)?.canResumeVerification).toBe(false);
+            expect(history.resumeVerification(job.deploymentId)).toBe(false);
+            expect((await operationFor(job.deploymentId))!.id).toBe(original.id);
+        });
+});
+
 describe('unhealthy recovery within native create and rebuild operations', () => {
     async function launch(action: 'create' | 'rebuild') {
         const job = await queue(); await advance();

tests/unit/provisioning-presentation.test.ts

@@ -19,6 +19,20 @@ function fixture(): ProvisioningDetail {
 }
 
 describe('provisioning outcome and evidence presentation', () => {
+    it('distinguishes a failed creation retry from readiness verification and its linked recovery', () => {
+        const detail = fixture();
+        detail.deployment.state = 'failed';
+        detail.canRetryWithRebuild = true;
+        expect(provisioningOutcome(detail)).toMatchObject({ active: false, headline: 'Create failed' });
+        expect(provisioningOutcome(detail).description).toContain('new attempt for the same lobby and port');
+        detail.canRetryWithRebuild = false;
+        detail.progress = progress({ recovery: { state: 'transferred', operationId: 'recovery-a', at, message: 'Rebuild requested' } });
+        expect(provisioningOutcome(detail).headline).toBe('Create failed');
+        expect(provisioningOutcome(detail).description).toContain('linked operation');
+        detail.progress = progress();
+        detail.canResumeVerification = true;
+        expect(provisioningOutcome(detail).description).toContain('remaining restart and rebuild recovery steps');
+    });
     it('distinguishes cancellation from confirmed cleanup and validates its link data', () => {
         const detail = fixture();
         detail.deployment.state = 'cancelled';
@@ -142,6 +156,15 @@ describe('provisioning outcome and evidence presentation', () => {
 });
 
 describe('provisioning response and return-context boundaries', () => {
+    it('accepts optional retry eligibility only as an authoritative boolean', () => {
+        const detail = fixture();
+        for (const canRetryWithRebuild of [undefined, false, true]) {
+            expect(isProvisioningDetail({ ...detail, canRetryWithRebuild }, detail.deployment.id)).toBe(true);
+        }
+        for (const canRetryWithRebuild of ['true', 1, null]) {
+            expect(isProvisioningDetail({ ...detail, canRetryWithRebuild }, detail.deployment.id)).toBe(false);
+        }
+    });
     it('accepts persisted usernames and legacy details but rejects malformed requesters', () => {
         const detail = fixture();
         for (const requested_by of [undefined, null, 'alice']) {

tests/unit/provisioning-refresh.test.ts

@@ -23,6 +23,15 @@ function deferred() {
 }
 
 describe('provisioning detail refresh ownership', () => {
+    it('requests a rebuild retry and refreshes the original failed operation', async () => {
+        const fetcher = vi.fn().mockResolvedValueOnce(Response.json({ ok: true, deploymentId: 'recovery-a' })).mockResolvedValueOnce(response('a', 2));
+        const page = setup(fetcher);
+        page.setOperation('a', detail('a'));
+        await page.act('retry-rebuild', 'Follow the linked recovery operation.');
+        expect(fetcher).toHaveBeenNthCalledWith(1, '/api/provisioning/a/retry-rebuild', expect.objectContaining({ method: 'POST' }));
+        expect(fetcher).toHaveBeenNthCalledWith(2, '/api/provisioning/a?events=recent', expect.anything());
+        expect(page.state()).toMatchObject({ id: 'a', detail: detail('a', 2), notice: 'Follow the linked recovery operation.', busy: null });
+    });
     it('sends explicit override confirmation and refreshes the operation after acceptance', async () => {
         const fetcher = vi.fn().mockResolvedValueOnce(Response.json({ ok: true })).mockResolvedValueOnce(response('a', 2));
         const page = setup(fetcher);