Fix rebuild generation reconciliation (Refs #155)

docs/guides/collections.md

@@ -304,6 +304,27 @@ before continuing eligible interrupted startups. Unknown command results retain
 their original command ID and occupied slots until the host confirms the outcome.
 Missing server registrations and later destruction attempts require inspection.
 
+### Recovering a completed rebuild generation
+
+A completed rebuild records the replacement generation before EC verifies health.
+If fresh host inventory disagrees, verification pauses with the controller and
+host generations in the operation's recovery message.
+
+For an existing mismatch, open the **completed rebuild's Provisioning detail**
+(even if that operation was later cancelled), then choose **Reconcile replacement
+generation**. EC requires a completed command for the original host and immutable
+instance ID, the command's requested replacement generation, and matching host
+inventory observed within the last 30 seconds. Wait for that host to reconnect
+and report fresh inventory if the action is unavailable. A server name alone is
+not sufficient evidence.
+
+Reconciliation preserves cancellation and submitted commands. Use **Resume health
+verification** when offered, or request a new lifecycle operation after resolving
+any remaining active operation. Fresh health checks, log reads, and new lifecycle
+commands use the replacement generation. EC requires fresh container and game
+evidence before reporting the server online. The agent continues to reject stale
+commands; an admin player-safety override does not bypass generation checks.
+
 ## API
 
 Collections retain the existing `/api/blueprints` endpoints and `blueprintId` /
@@ -351,6 +372,9 @@ there while preserving query parameters.
 - `GET /api/deployments/:id`: includes `recipe`, `blueprintName`, and
   `canResumeVerification` and `canRetryWithRebuild` for lifecycle operations.
 - `POST /api/deployments/:id/resume`: resume eligible creation health checks.
+- `POST /api/provisioning/:id/reconcile-generation`: reconcile a completed rebuild
+  using fresh inventory from its original host. Returns 409 when identity or
+  completion evidence cannot be validated; repeated application is safe.
 - `POST /api/provisioning/:id/retry-rebuild`: request a new rebuild for an eligible
   failed creation and link it from the original operation.
 

src/lib/provisioning-detail.ts

@@ -117,6 +117,13 @@ export interface ProvisioningCapacity {
     }[];
 }
 export interface ProvisioningDetail {
+    generationRecovery?: {
+        expectedGeneration: number;
+        replacementGeneration: number;
+        observedGeneration: number | null;
+        available: boolean;
+        reason: string | null;
+    } | null;
     canOverrideSafety?: boolean;
     safetyOverrideUnavailableReason?: string | null;
     safetyOverride?: { actorId: string; actorName: string; confirmedAt: string } | null;

src/lib/server/db/migrations.ts

@@ -120,7 +120,21 @@ export const migrations: readonly Migration[] = [
     { version: 20, name: 'persistent player bank and transfer ledger', run: migrateBank },
     { version: 21, name: 'instance-owned bank credentials', run: migrateBankCredentials },
     { version: 22, name: 'remembered browser sessions', run: migrateRememberedSessions },
-    { version: 23, name: 'host temperature history', run: migrateTemperatures }
+    { version: 23, name: 'host temperature history', run: migrateTemperatures },
+    {
+        version: 24,
+        name: 'rebuild generation evidence boundary',
+        run: (handle) => {
+            if (
+                !(handle.pragma('table_info(native_instances)') as { name: string }[]).some(
+                    (column) => column.name === 'generation_reconciled_at'
+                )
+            )
+                handle.exec(
+                    'ALTER TABLE native_instances ADD COLUMN generation_reconciled_at TEXT'
+                );
+        }
+    }
 ];
 
 export const databaseVersion = migrations.length;

src/lib/server/fleet/lifecycle-evidence.ts

@@ -7,9 +7,10 @@ export function latestGameSubmission(server: string): string | null {
             .prepare(
                 `SELECT MAX(at) AS at FROM (
         SELECT MAX(submitted_at) AS at FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild')
-        UNION ALL SELECT requested_at AS at FROM restart_rollout WHERE server=?)`
+        UNION ALL SELECT requested_at AS at FROM restart_rollout WHERE server=?
+        UNION ALL SELECT generation_reconciled_at AS at FROM native_instances WHERE server=? AND retired_at IS NULL)`
             )
-            .get(server, server) as { at: string | null }
+            .get(server, server, server) as { at: string | null }
     ).at;
 }
 
@@ -21,7 +22,8 @@ export function latestGameSubmissions(): Map<string, string | null> {
         SELECT s.name AS server, (SELECT MAX(submitted_at) FROM server_deployments
             WHERE server=s.name AND server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild')) AS at
         FROM servers s
-        UNION ALL SELECT server, requested_at AS at FROM restart_rollout) GROUP BY server`
+        UNION ALL SELECT server, requested_at AS at FROM restart_rollout
+        UNION ALL SELECT server, generation_reconciled_at AS at FROM native_instances WHERE retired_at IS NULL) GROUP BY server`
         )
         .all() as { server: string; at: string | null }[];
     return new Map(rows.map((row) => [row.server, row.at]));

src/lib/server/provisioning/cancellation.ts

@@ -128,6 +128,9 @@ export function unsettledCancelledCommand(
         WHERE d.server=? AND d.server_deleted_at IS NULL AND d.state='cancelled'
         AND (o.state NOT IN ('completed','failed')
             OR (o.state='completed' AND o.id=n.operation_id AND n.files_json IS NOT NULL AND n.applied_at IS NULL)
+            OR (o.state='completed' AND o.id=n.operation_id AND o.kind='instance.rebuild'
+                AND EXISTS (SELECT 1 FROM native_instances i WHERE i.id=n.instance_id
+                    AND i.retired_at IS NULL AND i.generation<=o.generation))
             OR (o.state='failed' AND o.id=n.operation_id AND o.kind='instance.rebuild'
                 AND json_type(o.result,'$.rebuildFailure') IS NOT NULL AND n.applied_at IS NULL))
         AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations c WHERE c.creation_id=d.id AND c.cleanup_id=?)

src/lib/server/provisioning/history.ts

@@ -14,6 +14,7 @@ import { fleetSettings } from '../settings.js';
 import { activitySummary, deploymentEvents, latestFailureEvent } from './activity.js';
 import { readProgress, recordProgress, type ProgressUpdate } from './progress.js';
 import { bindNativeOperation } from './native.js';
+import { rebuildGenerationRecovery } from './rebuild-receipt.js';
 import { workerActivity } from '../pipeline/activity.js';
 import { canCancelCreation } from './cancel-creation.js';
 import {
@@ -337,6 +338,7 @@ export function getDeployment(
             canResumeCleanup: canResumeCleanup(id),
             canResumeRebuild: canResumeRebuild(id),
             canRetryWithRebuild: canRetryWithRebuild(id),
+            generationRecovery: rebuildGenerationRecovery(id),
             canCancelRemoval: canCancelRemoval(id),
             collection: collection ?? null,
             progress: readProgress(row),

src/lib/server/provisioning/native.ts

@@ -70,7 +70,11 @@ import { recordProgress, type CheckInput } from './progress.js';
 import { trackActivity } from '../pipeline/activity.js';
 import { fleetSettings } from '../settings.js';
 import { resolveRecoveredHealthOperation } from '../maintenance/recovery.js';
-import { probeServerStatus, probePlayerEvidence } from '../fleet/status-evidence.js';
+import {
+    probeServerStatus,
+    probePlayerEvidence,
+    latestGameSubmission
+} from '../fleet/status-evidence.js';
 import { monitoringIdentity } from '../fleet/health-observations.js';
 import { handoffVerifiedHealth } from './health-handoff.js';
 import { checkRelease, freshRelease, probeVersion } from '../integrations/plutonium.js';
@@ -85,6 +89,12 @@ import { reconcileHostDrains } from './host-drain.js';
 import { hostMigration, isRebalanceMigration, processMigrationRemoval } from './migrations.js';
 import { safetyOverride, safetyOverrideCapability } from './safety-override.js';
 import { runtimeLaunchSql } from '../fleet/runtime-config.js';
+import {
+    reconcileCompletedRebuild,
+    assertVerificationGeneration,
+    GenerationRecoveryRequired,
+    observedGeneration
+} from './rebuild-receipt.js';
 
 export type NativeSelection = import('../../hosts.js').NativePlacement;
 type NativeRecipe = LaunchRecipePlan & { native?: NativeSelection };
@@ -813,27 +823,11 @@ export function reconcileNativeProvisioningReceipt(id: string): void {
         finishCreationCancellation(job.id);
         return;
     }
-    if (operation.kind === 'instance.rebuild' && instance.generation === job.generation) {
+    if (operation.kind === 'instance.rebuild') {
-        db()
+        // An older applied receipt must never undo a later successful replacement.
-            .prepare('UPDATE native_instances SET generation=generation+1 WHERE id=?')
+        if (job.applied_at && instance.generation > operation.generation! + 1) return;
-            .run(instance.id);
+        reconcileCompletedRebuild(id);
-        // Replacements queued while the delivered rebuild was settling bound the
+    } else if (instance.generation !== job.generation) return;
-        // previous generation. Only untouched, newer jobs may follow this receipt.
-        if (job.state === 'cancelled')
-            db()
-                .prepare(
-                    `UPDATE native_provisioning SET generation=?
-            WHERE instance_id=? AND generation=? AND operation_id IS NULL AND stage_operation_id IS NULL
-            AND deployment_id IN (SELECT newer.id FROM server_deployments newer
-                WHERE newer.rowid>(SELECT rowid FROM server_deployments WHERE id=?)
-                AND newer.server_deleted_at IS NULL AND newer.state IN ('queued','waiting'))`
-                )
-                .run(job.generation + 1, instance.id, job.generation, id);
-    } else if (
-        instance.generation !== job.generation &&
-        !(operation.kind === 'instance.rebuild' && instance.generation === job.generation + 1)
-    )
-        return;
     applyReceipt(job, (operation.result ?? {}) as Record<string, unknown>);
     db()
         .prepare(
@@ -1032,10 +1026,12 @@ export async function processNativeProvisioning(
             `SELECT d.id FROM server_deployments d
         JOIN native_provisioning n ON n.deployment_id=d.id
         LEFT JOIN ec_host_operations o ON o.id=n.operation_id
+        LEFT JOIN native_instances i ON i.id=n.instance_id AND i.retired_at IS NULL
         LEFT JOIN ec_host_migrations m ON m.deployment_id=d.id
         LEFT JOIN ec_host_operations removal ON removal.id=m.remove_operation_id
         WHERE d.state='cancelled' AND d.server_deleted_at IS NULL AND (
-            (o.state='completed' AND (n.applied_at IS NULL OR d.action='destroy'))
+            (o.state='completed' AND (n.applied_at IS NULL OR d.action='destroy'
+                OR (o.kind='instance.rebuild' AND i.generation=o.generation)))
             OR (o.state='failed' AND o.kind='instance.rebuild' AND json_type(o.result,'$.rebuildFailure') IS NOT NULL AND n.applied_at IS NULL)
             OR (o.state='failed' AND EXISTS (SELECT 1 FROM staged_file_changes f JOIN provisioning_sources s
                 ON s.kind=f.source_kind AND s.source_id=f.source_id AND s.server=f.server
@@ -1061,7 +1057,8 @@ export async function processNativeProvisioning(
             `SELECT d.*,n.instance_id,n.generation,n.stage_operation_id,n.operation_id,n.applied_at,n.recovery_step,q.selection
         FROM server_deployments d LEFT JOIN native_provisioning n ON n.deployment_id=d.id
         LEFT JOIN native_creation_intents q ON q.deployment_id=d.id
-        WHERE (n.deployment_id IS NOT NULL OR q.deployment_id IS NOT NULL) AND d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') ORDER BY d.started_at,d.rowid`
+        WHERE (n.deployment_id IS NOT NULL OR q.deployment_id IS NOT NULL) AND d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted')
+        AND COALESCE(CASE WHEN json_valid(d.observation_json) THEN json_extract(d.observation_json,'$.blocker.code') END,'')!='generation-recovery' ORDER BY d.started_at,d.rowid`
         )
         .all() as Job[];
     // One host's observation must not prevent another job being admitted.
@@ -1075,7 +1072,27 @@ export async function processNativeProvisioning(
             () => processJob(job, eligibility)
         )
             .catch((error) => {
-                if (stillActive(job)) hold(job, (error as Error).message);
+                if (!stillActive(job)) return;
+                if (
+                    error instanceof GenerationRecoveryRequired ||
+                    /generation has been superseded or removed/i.test((error as Error).message)
+                ) {
+                    const instance = nativeInstance(job.server);
+                    const message =
+                        error instanceof GenerationRecoveryRequired
+                            ? error.message
+                            : `Generation recovery required: controller expects ${instance?.generation ?? job.generation}, host reports ${instance ? (observedGeneration(instance) ?? 'unknown') : 'unknown'}. Reconcile the completed rebuild before resuming verification.`;
+                    db()
+                        .prepare(
+                            "UPDATE server_deployments SET state='interrupted',error=? WHERE id=?"
+                        )
+                        .run(message, job.id);
+                    deploymentStage(job.id, message, {
+                        blocker: { code: 'generation-recovery', message, automatic: false },
+                        checks: [],
+                        recovery: { state: 'held', operationId: job.id, message }
+                    });
+                } else hold(job, (error as Error).message);
             })
             .finally(() => {
                 activeJobs.delete(job.id);
@@ -1328,6 +1345,15 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
         job.instance_id = instance.id;
         job.generation = instance.generation;
     }
+    // Reconcile before comparing the job snapshot or issuing any observations.
+    // Historical jobs can have a stale binding even though their command is correct.
+    if (job.operation_id) {
+        const receipt = getHostOperationStatus(job.operation_id);
+        if (receipt?.kind === 'instance.rebuild' && receipt.state === 'completed') {
+            instance = reconcileCompletedRebuild(job.id);
+            job.generation = receipt.generation!;
+        }
+    }
     const migration = hostMigration(job.id);
     const rebuilding =
         job.recovery_step === 'rebuild' ||
@@ -1336,7 +1362,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
         !instance ||
         instance.id !== job.instance_id ||
         (instance.generation !== job.generation &&
-            !(rebuilding && instance.generation === job.generation + 1))
+            !(rebuilding && job.operation_id && instance.generation === job.generation + 1))
     ) {
         finishDeployment(job.id, {
             ok: false,
@@ -1493,12 +1519,6 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             })();
             return;
         }
-        if (operation.kind === 'instance.rebuild' && instance.generation === job.generation) {
-            db()
-                .prepare('UPDATE native_instances SET generation=? WHERE id=? AND generation=?')
-                .run(job.generation + 1, instance.id, job.generation);
-            instance = nativeInstance(job.server)!;
-        }
         applyReceipt(job, (operation.result ?? {}) as Record<string, unknown>);
         db()
             .prepare(
@@ -1542,6 +1562,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             blocker: null,
             checks
         });
+        assertVerificationGeneration(instance, operation.updatedAt);
         const verificationIdentity = monitoringIdentity(job.server);
         const health = await nativeRequest<{ healthy: boolean; state: string }>(
             instance,
@@ -1601,7 +1622,11 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             return;
         }
         deploymentStage(job.id, 'Checking game status', { phase: 'verify-game', blocker: null });
-        const status = await probeServerStatus(job.server, operation.updatedAt);
+        const submission = latestGameSubmission(job.server);
+        const status = await probeServerStatus(
+            job.server,
+            submission && submission > operation.updatedAt ? submission : operation.updatedAt
+        );
         if (!stillActive(job)) return;
         check(
             {
@@ -2255,8 +2280,10 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
                   )
                 : queueHostOperation(instance!.host_id, kind, payload, operationOptions);
             db()
-                .prepare('UPDATE native_provisioning SET operation_id=? WHERE deployment_id=?')
+                .prepare(
-                .run(operation.id, job.id);
+                    'UPDATE native_provisioning SET operation_id=?,generation=? WHERE deployment_id=?'
+                )
+                .run(operation.id, instance!.generation, job.id);
             for (const file of files)
                 db()
                     .prepare(

src/lib/server/provisioning/rebuild-receipt.ts

@@ -0,0 +1,173 @@
+import { db } from '../db/index.js';
+import { nativeInstance, type NativeInstance } from '../backends/native.js';
+import { getHost, getHostOperationStatus } from '../hosts/index.js';
+import { recordProgress } from './progress.js';
+
+export class GenerationRecoveryRequired extends Error {}
+
+/** Inventory is evidence only when it belongs to this host and immutable instance. */
+export function observedGeneration(instance: NativeInstance, notBefore?: string): number | null {
+    const host = getHost(instance.host_id);
+    if (!host?.connected) return null;
+    const states = host.inventory.instanceStates?.filter((state) => {
+        const age = Date.now() - Date.parse(state.observedAt);
+        return (
+            state.instanceId === instance.id &&
+            age >= 0 &&
+            age <= 30_000 &&
+            (!notBefore || Date.parse(state.observedAt) >= Date.parse(notBefore))
+        );
+    });
+    return states?.length === 1 ? states[0].generation : null;
+}
+
+function receipt(id: string) {
+    const binding = db()
+        .prepare(
+            `SELECT d.server,n.instance_id,n.generation,n.operation_id
+        FROM server_deployments d JOIN native_provisioning n ON n.deployment_id=d.id
+        WHERE d.id=? AND d.server_deleted_at IS NULL
+        AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations c WHERE c.creation_id=d.id)`
+        )
+        .get(id) as
+        | { server: string; instance_id: string; generation: number; operation_id: string }
+        | undefined;
+    const instance = binding && nativeInstance(binding.server);
+    const operation = binding?.operation_id && getHostOperationStatus(binding.operation_id);
+    if (
+        !binding ||
+        !instance ||
+        !operation ||
+        operation.kind !== 'instance.rebuild' ||
+        operation.state !== 'completed' ||
+        operation.error ||
+        instance.id !== binding.instance_id ||
+        operation.instanceId !== instance.id ||
+        operation.hostId !== instance.host_id ||
+        !Number.isSafeInteger(operation.generation) ||
+        operation.generation! < 1
+    ) {
+        throw new GenerationRecoveryRequired(
+            'Recovery requires a completed rebuild for this host and immutable instance ID. Inspect the original host command.'
+        );
+    }
+    const requested = db()
+        .prepare(
+            `SELECT COALESCE(
+        json_extract(COALESCE(p.payload,o.payload),'$.nextGeneration'),
+        json_extract(COALESCE(p.payload,o.payload),'$.newGeneration')) AS generation
+        FROM ec_host_operations o LEFT JOIN ec_host_operation_payloads p ON p.operation_id=o.id WHERE o.id=?`
+        )
+        .get(operation.id) as { generation: number | null };
+    const generation = requested.generation;
+    const result = operation.result as { instanceId?: unknown; generation?: unknown } | null;
+    if (
+        !Number.isSafeInteger(generation) ||
+        generation !== operation.generation! + 1 ||
+        (result?.instanceId !== undefined && result.instanceId !== instance.id) ||
+        (result?.generation !== undefined && result.generation !== generation) ||
+        ![operation.generation, generation].includes(instance.generation)
+    ) {
+        throw new GenerationRecoveryRequired(
+            'The completed rebuild does not match the registered instance and requested replacement generation. Inspect its host command; no generation was adopted.'
+        );
+    }
+    return { binding, instance, operation, generation: generation! };
+}
+
+/** Read-only availability for the operator recovery action, including old cancelled jobs. */
+export function rebuildGenerationRecovery(id: string) {
+    try {
+        const { binding, instance, operation, generation } = receipt(id);
+        const observed = observedGeneration(instance, operation.updatedAt);
+        if (instance.generation === generation && binding.generation === operation.generation)
+            return null;
+        return {
+            expectedGeneration: instance.generation,
+            replacementGeneration: generation,
+            observedGeneration: observed,
+            available: observed === generation,
+            reason:
+                observed === generation
+                    ? null
+                    : `Waiting for fresh inventory from the original host confirming generation ${generation} for this instance.`
+        };
+    } catch {
+        return null;
+    }
+}
+
+/** The durable command, not a potentially stale job snapshot, owns the transition. */
+export function reconcileCompletedRebuild(id: string, requireInventory = false): NativeInstance {
+    return db().transaction(() => {
+        const { binding, instance, operation, generation } = receipt(id);
+        const observed = observedGeneration(instance, operation.updatedAt);
+        if (
+            (requireInventory && observed !== generation) ||
+            (observed !== null && observed !== generation)
+        ) {
+            throw new GenerationRecoveryRequired(
+                `Generation recovery required: controller expects ${instance.generation}, completed rebuild requests ${generation}, host reports ${observed ?? 'unknown'}. Refresh the host inventory and reconcile the completed rebuild.`
+            );
+        }
+        const changed = instance.generation !== generation;
+        const repaired = binding.generation !== operation.generation;
+        if (changed) {
+            const at = new Date().toISOString();
+            db()
+                .prepare(
+                    `UPDATE native_instances SET generation=?,generation_reconciled_at=?
+                WHERE id=? AND host_id=? AND generation=? AND retired_at IS NULL`
+                )
+                .run(generation, at, instance.id, instance.host_id, instance.generation);
+            db()
+                .prepare(
+                    `UPDATE servers SET container_state='unknown',container_checked_at=?,
+                last_checked_at=NULL,last_players=NULL,last_roster=NULL WHERE name=?`
+                )
+                .run(at, instance.server);
+        }
+        db()
+            .prepare(
+                'UPDATE native_provisioning SET generation=? WHERE deployment_id=? AND operation_id=?'
+            )
+            .run(operation.generation, id, operation.id);
+        // Untouched successors may follow a cancelled, delivered rebuild. Already
+        // submitted commands retain their immutable target and must never be retargeted.
+        db()
+            .prepare(
+                `UPDATE native_provisioning SET generation=? WHERE instance_id=? AND generation=?
+            AND operation_id IS NULL AND stage_operation_id IS NULL
+            AND deployment_id IN (SELECT newer.id FROM server_deployments newer
+                WHERE newer.rowid>(SELECT rowid FROM server_deployments WHERE id=?)
+                AND newer.submitted_at IS NULL AND newer.server_deleted_at IS NULL
+                AND newer.state IN ('queued','waiting'))`
+            )
+            .run(generation, instance.id, instance.generation, id);
+        if (changed || repaired)
+            recordProgress(id, {
+                checks: [],
+                event: {
+                    kind: 'generation-reconciled',
+                    message: `Recorded replacement generation ${generation}; fresh health verification is required.`,
+                    details: {
+                        operationId: operation.id,
+                        hostId: instance.host_id,
+                        instanceId: instance.id,
+                        previousGeneration: instance.generation,
+                        generation
+                    }
+                }
+            });
+        return nativeInstance(instance.server)!;
+    })();
+}
+
+export function assertVerificationGeneration(instance: NativeInstance, notBefore: string): void {
+    const observed = observedGeneration(instance, notBefore);
+    if (observed !== null && observed !== instance.generation) {
+        throw new GenerationRecoveryRequired(
+            `Generation recovery required: controller expects ${instance.generation}, host reports ${observed}. Reconcile the completed rebuild before resuming verification.`
+        );
+    }
+}

src/lib/server/retention.ts

@@ -31,7 +31,7 @@ const unsettledDeployment = `d.state NOT IN ${terminal}
             OR (a.id=n.operation_id AND a.state='completed' AND (
             (n.applied_at IS NULL AND COALESCE(json_array_length(n.files_json),0)>0)
             OR (d.action='destroy' AND i.retired_at IS NULL)
-            OR (a.kind='instance.rebuild' AND i.retired_at IS NULL AND i.generation=n.generation)))))
+            OR (a.kind='instance.rebuild' AND i.retired_at IS NULL AND i.generation<=a.generation)))))
     OR EXISTS (SELECT 1 FROM ec_host_migrations m JOIN ec_host_operations a ON a.id=m.remove_operation_id
         WHERE m.deployment_id=d.id AND (a.state NOT IN ('completed','failed') OR (a.state='completed' AND m.removed_at IS NULL)))`;
 // Recovery replaces operation_id with its latest command. The original launch

src/lib/ui/provisioning-refresh.ts

@@ -1,5 +1,5 @@
 export type ProvisioningMutation =
-    'resume' | 'retry-rebuild' | 'cancel' | 'cancel-creation' | 'override';
+    'resume' | 'retry-rebuild' | 'cancel' | 'cancel-creation' | 'override' | 'reconcile-generation';
 
 export interface ProvisioningRefreshState<T> {
     id: string;

src/routes/api/provisioning/[id]/reconcile-generation/+server.ts

@@ -0,0 +1,26 @@
+import { json } from '@sveltejs/kit';
+import { reconcileCompletedRebuild } from '$lib/server/provisioning/rebuild-receipt.js';
+import { reconcileNativeProvisioningReceipt } from '$lib/server/provisioning/native.js';
+import { db } from '$lib/server/db/index.js';
+import { recordProgress } from '$lib/server/provisioning/progress.js';
+
+export function POST({ params }: { params: { id: string } }) {
+    try {
+        const instance = db().transaction(() => {
+            const instance = reconcileCompletedRebuild(params.id, true);
+            reconcileNativeProvisioningReceipt(params.id);
+            recordProgress(params.id, {
+                blocker: null,
+                checks: [],
+                event: {
+                    kind: 'generation-recovery',
+                    message: `Replacement generation ${instance.generation} reconciled with fresh host inventory. Resume health verification or request a new lifecycle operation.`
+                }
+            });
+            return instance;
+        })();
+        return json({ ok: true, generation: instance.generation });
+    } catch (error) {
+        return json({ error: (error as Error).message }, { status: 409 });
+    }
+}

src/routes/provisioning/[id]/+page.svelte

@@ -324,6 +324,27 @@
                         )}.
                     </p>
                 </div>{/if}
+            {#if detail.generationRecovery}<div class="warnings">
+                    <h3>Replacement generation recovery</h3>
+                    <p>
+                        Controller generation {detail.generationRecovery.expectedGeneration};
+                        completed rebuild generation {detail.generationRecovery
+                            .replacementGeneration}; host generation {detail.generationRecovery
+                            .observedGeneration ?? 'unknown'}.
+                    </p>
+                    {#if detail.generationRecovery.reason}<p>
+                            {detail.generationRecovery.reason}
+                        </p>{/if}
+                    {#if writable}<Button
+                            disabled={!!view.busy || !detail.generationRecovery.available}
+                            loading={view.busy === 'reconcile-generation'}
+                            onclick={() =>
+                                controller.act(
+                                    'reconcile-generation',
+                                    'Replacement generation reconciled. Fresh health verification is required; cancelled operations remain cancelled.'
+                                )}>Reconcile replacement generation</Button
+                        >{/if}
+                </div>{/if}
             {#if writable && (detail.canResumeVerification || detail.canResumeCleanup || detail.canResumeRebuild || detail.canRetryWithRebuild || detail.canCancel || detail.canCancelCreation || canOverride)}
                 <div class="recovery-actions">
                     {#if canOverride}<div>

tests/unit/auth-access.test.ts

@@ -90,6 +90,7 @@ describe('management route permissions', () => {
         ['/api/rebalancing', 'POST'],
         ['/api/provisioning/[id]/cancel', 'POST'],
         ['/api/provisioning/[id]/resume', 'POST'],
+        ['/api/provisioning/[id]/reconcile-generation', 'POST'],
         ['/api/deployment-collections', 'POST'],
         ['/api/deployment-collections/preview', 'POST'],
         ['/api/deployment-collections/[id]/cancel', 'POST'],

tests/unit/database-migrations.test.ts

@@ -291,3 +291,18 @@ it('adds provisioning report invalidation at version 14 without invalidating for
     upgraded.exec('DELETE FROM server_deployments');
     expect(revision()).toBeGreaterThan(completed);
 });
+
+it('adds the generation evidence boundary to a version 23 database without changing instance identity', async () => {
+    const { db, closeDb } = await import('../../src/lib/server/db/index.js');
+    const handle = db();
+    handle.exec(
+        "INSERT INTO native_instances(id,server,host_id,generation,library_id,image,port,created_at) VALUES('instance','s','host',2,'lib','image',1,'2026-10-14T00:00:00Z')"
+    );
+    handle.exec('ALTER TABLE native_instances DROP COLUMN generation_reconciled_at');
+    handle.pragma('user_version = 23');
+    closeDb();
+    expect(
+        db().prepare('SELECT id,generation,generation_reconciled_at FROM native_instances').get()
+    ).toEqual({ id: 'instance', generation: 2, generation_reconciled_at: null });
+    expect(db().pragma('foreign_key_check')).toEqual([]);
+});

tests/unit/native-provisioning.test.ts

@@ -465,7 +465,16 @@ describe('provisioning health handoff', () => {
                 .run(old, old, old, job.name);
             vi.useFakeTimers({ toFake: ['Date'] });
             vi.setSystemTime(Date.now() + 1000);
-            const observedAt = new Date().toISOString();
+            if (action === 'rebuild') {
+                // A new generation now establishes its own durable evidence boundary.
+                // Observe game health strictly after that transition.
+                const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
+                vi.mocked(nativeRequest).mockImplementationOnce(async () => {
+                    vi.setSystemTime(Date.now() + 1);
+                    return { healthy: true, state: 'running', exists: true } as never;
+                });
+            }
+            const observedAt = new Date(Date.now() + (action === 'rebuild' ? 1 : 0)).toISOString();
             const roster = [
                 { num: 0, name: 'Player', guid: '12345678901234567', score: 500, ping: 20 }
             ];
@@ -4288,3 +4297,276 @@ describe('game key pool launch lifecycle', () => {
         expect(serverKeyPool('t5')).toEqual({ total: 1, available: 1 });
     });
 });
+
+describe('completed rebuild generation receipts (#155)', () => {
+    async function rebuild() {
+        const job = await queue();
+        await advance();
+        await acknowledge(job.deploymentId);
+        const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
+        const requested = requestLifecycle(job.name, 'rebuild');
+        if (!requested.ok) throw new Error(requested.error);
+        await advance();
+        return {
+            job,
+            id: requested.deploymentId,
+            operation: (await operationFor(requested.deploymentId))!
+        };
+    }
+
+    async function inventoryGeneration(
+        instanceId: string,
+        generation: number,
+        observedAt = new Date().toISOString(),
+        targetHost = hostId
+    ) {
+        const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
+        heartbeat(targetHost, {
+            ...inventory(),
+            instanceStates: [
+                { instanceId, generation, state: 'running', healthy: true, observedAt }
+            ]
+        });
+    }
+
+    it('repairs an older job binding from the actual command before verification, including duplicate receipts and restart', async () => {
+        const { job, id } = await rebuild();
+        await acknowledge(id);
+        const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
+        const next = requestLifecycle(job.name, 'rebuild');
+        if (!next.ok) throw new Error(next.error);
+        await advance();
+        const operation = (await operationFor(next.deploymentId))!;
+        expect(operation.generation).toBe(2);
+        const { db, closeDb } = await import('../../src/lib/server/db/index.js');
+        // Historical job snapshots could stay on generation 1 while the submitted
+        // command correctly targeted 2 -> 3. The old completion path kept 2.
+        db()
+            .prepare('UPDATE native_provisioning SET generation=1 WHERE deployment_id=?')
+            .run(next.deploymentId);
+        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        reportHostOperation(hostId, operation.id, {
+            state: 'completed',
+            result: { instanceId: operation.instanceId, generation: 3, healthy: true }
+        });
+        await inventoryGeneration(operation.instanceId!, 3);
+        closeDb();
+        const { nativeRequest, nativeInstance } =
+            await import('../../src/lib/server/backends/native.js');
+        vi.mocked(nativeRequest).mockClear();
+        remote.state = 'starting';
+        await advance();
+        expect(nativeInstance(job.name)?.generation).toBe(3);
+        expect(nativeRequest).toHaveBeenCalledWith(
+            expect.objectContaining({ id: operation.instanceId, generation: 3 }),
+            'instance.inspect'
+        );
+        expect(
+            db()
+                .prepare(
+                    'SELECT generation,applied_at FROM native_provisioning WHERE deployment_id=?'
+                )
+                .get(next.deploymentId)
+        ).toEqual({ generation: 2, applied_at: expect.any(String) });
+        const { reconcileNativeProvisioningReceipt } =
+            await import('../../src/lib/server/provisioning/native.js');
+        const before = db()
+            .prepare('SELECT generation_reconciled_at FROM native_instances WHERE id=?')
+            .get(operation.instanceId);
+        reconcileNativeProvisioningReceipt(next.deploymentId);
+        closeDb();
+        reconcileNativeProvisioningReceipt(next.deploymentId);
+        expect(nativeInstance(job.name)?.generation).toBe(3);
+        expect(
+            db()
+                .prepare('SELECT generation_reconciled_at FROM native_instances WHERE id=?')
+                .get(operation.instanceId)
+        ).toEqual(before);
+        expect((await detail(next.deploymentId)).state).toBe('awaiting-health');
+        for (const observation of await observations(job.name))
+            expect(observation.health).not.toBe('online');
+        const { latestGameSubmission, latestGameSubmissions } =
+            await import('../../src/lib/server/fleet/lifecycle-evidence.js');
+        expect(latestGameSubmission(job.name)).toBe(
+            (before as { generation_reconciled_at: string }).generation_reconciled_at
+        );
+        expect(latestGameSubmissions().get(job.name)).toBe(latestGameSubmission(job.name));
+        remote.state = 'running';
+        await advance();
+        expect((await detail(next.deploymentId)).state).toBe('done');
+        const restart = requestLifecycle(job.name, 'restart');
+        if (!restart.ok) throw new Error(restart.error);
+        await advance();
+        expect(await operationFor(restart.deploymentId)).toMatchObject({ generation: 3 });
+    });
+
+    it.each(['before', 'after'] as const)(
+        'reconciles delayed completion reported %s cancellation across restart',
+        async (when) => {
+            const { job, id, operation } = await rebuild();
+            const { reportHostOperation, pendingHostOperations } =
+                await import('../../src/lib/server/hosts/index.js');
+            const { cancelProvisioningJob } =
+                await import('../../src/lib/server/provisioning/cancellation.js');
+            const { closeDb } = await import('../../src/lib/server/db/index.js');
+            pendingHostOperations(hostId);
+            if (when === 'before')
+                reportHostOperation(hostId, operation.id, { state: 'completed' });
+            expect(cancelProvisioningJob(id)).toBe(true);
+            closeDb();
+            if (when === 'after') reportHostOperation(hostId, operation.id, { state: 'completed' });
+            await advance();
+            await advance();
+            const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
+            expect(nativeInstance(job.name)?.generation).toBe(2);
+            expect((await detail(id)).state).toBe('cancelled');
+        }
+    );
+
+    it('recovers an already-applied cancelled mismatch through the API and targets replacement logs, health and removal', async () => {
+        const { job, id, operation } = await rebuild();
+        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        reportHostOperation(hostId, operation.id, {
+            state: 'completed',
+            result: { instanceId: operation.instanceId, generation: 2 }
+        });
+        const { cancelProvisioningJob } =
+            await import('../../src/lib/server/provisioning/cancellation.js');
+        expect(cancelProvisioningJob(id)).toBe(true);
+        const { db } = await import('../../src/lib/server/db/index.js');
+        db()
+            .prepare('UPDATE native_provisioning SET applied_at=? WHERE deployment_id=?')
+            .run(operation.updatedAt, id);
+        await inventoryGeneration(operation.instanceId!, 2);
+        const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
+        expect(getDeployment(id)?.generationRecovery).toMatchObject({
+            expectedGeneration: 1,
+            observedGeneration: 2,
+            available: true
+        });
+        const { POST } =
+            await import('../../src/routes/api/provisioning/[id]/reconcile-generation/+server.js');
+        expect(await POST({ params: { id } }).json()).toEqual({ ok: true, generation: 2 });
+        expect(POST({ params: { id } }).status).toBe(200);
+        expect((await detail(id)).state).toBe('cancelled');
+        expect(getDeployment(id)?.generationRecovery).toBeNull();
+        const backend = await import('../../src/lib/server/backends/native.js');
+        vi.mocked(backend.nativeRequest).mockRestore();
+        const hosts = await import('../../src/lib/server/hosts/index.js');
+        vi.spyOn(hosts, 'waitHostOperationStatus').mockImplementation(async (id) =>
+            hosts.reportHostOperationStatus(hostId, id, {
+                state: 'completed',
+                result: { healthy: true, state: 'running' }
+            })
+        );
+        await backend.nativeRequest(backend.nativeInstance(job.name)!, 'logs.read', {
+            file: 'console.log'
+        });
+        expect(await backend.nativeHealth(job.name)).toMatchObject({ ok: true });
+        expect(
+            db()
+                .prepare(
+                    "SELECT generation FROM ec_host_operations WHERE kind IN ('logs.read','instance.inspect')"
+                )
+                .all()
+                .every((row) => (row as { generation: number }).generation === 2)
+        ).toBe(true);
+        const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
+        const removal = destroyServer(job.name);
+        if (!removal.ok || !removal.deploymentId) throw new Error('Removal not queued');
+        await advance();
+        expect(await operationFor(removal.deploymentId)).toMatchObject({
+            generation: 2,
+            kind: 'instance.remove'
+        });
+    });
+
+    it.each([
+        'host',
+        'instance',
+        'result-instance',
+        'result-generation',
+        'payload',
+        'stale-inventory',
+        'missing-inventory',
+        'inventory-instance'
+    ] as const)('rejects recovery evidence with %s mismatch', async (mismatch) => {
+        const { job, id, operation } = await rebuild();
+        const { reportHostOperation, createHost } =
+            await import('../../src/lib/server/hosts/index.js');
+        reportHostOperation(hostId, operation.id, {
+            state: 'completed',
+            result: {
+                instanceId:
+                    mismatch === 'result-instance' ? 'another-instance' : operation.instanceId,
+                generation: mismatch === 'result-generation' ? 3 : 2
+            }
+        });
+        const { db } = await import('../../src/lib/server/db/index.js');
+        if (mismatch === 'host')
+            db()
+                .prepare('UPDATE ec_host_operations SET host_id=? WHERE id=?')
+                .run(createHost({ name: 'Other host' }).id, operation.id);
+        if (mismatch === 'instance')
+            db()
+                .prepare('UPDATE ec_host_operations SET instance_id=? WHERE id=?')
+                .run('another-instance', operation.id);
+        if (mismatch === 'payload')
+            db()
+                .prepare(
+                    "UPDATE ec_host_operation_payloads SET payload=json_set(payload,'$.nextGeneration',3) WHERE operation_id=?"
+                )
+                .run(operation.id);
+        if (mismatch !== 'missing-inventory')
+            await inventoryGeneration(
+                mismatch === 'inventory-instance' ? 'another-instance' : operation.instanceId!,
+                2,
+                mismatch === 'stale-inventory' ? '2000-01-01T00:00:00Z' : new Date().toISOString()
+            );
+        const { POST } =
+            await import('../../src/routes/api/provisioning/[id]/reconcile-generation/+server.js');
+        expect(POST({ params: { id } }).status).toBe(409);
+        const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
+        expect(nativeInstance(job.name)?.generation).toBe(1);
+    });
+
+    it('pauses after an agent stale-generation rejection even when inventory is unavailable', async () => {
+        const { id, operation } = await rebuild();
+        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        reportHostOperation(hostId, operation.id, { state: 'completed' });
+        const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
+        vi.mocked(nativeRequest).mockClear();
+        vi.mocked(nativeRequest).mockRejectedValueOnce(
+            new Error('Instance generation has been superseded or removed')
+        );
+        await advance();
+        expect((await detail(id)).state).toBe('interrupted');
+        expect((await detail(id)).error).toContain('controller expects 2, host reports unknown');
+        await advance();
+        expect(nativeRequest).toHaveBeenCalledTimes(1);
+    });
+
+    it('stops verification retries on a confirmed generation mismatch and keeps expected/observed evidence', async () => {
+        const { id, operation } = await rebuild();
+        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        reportHostOperation(hostId, operation.id, { state: 'completed' });
+        await inventoryGeneration(operation.instanceId!, 3);
+        const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
+        vi.mocked(nativeRequest).mockClear();
+        await advance();
+        const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
+        const before = getDeployment(id)!;
+        expect(before.deployment.state).toBe('interrupted');
+        expect(before.progress?.blocker).toMatchObject({
+            code: 'generation-recovery',
+            automatic: false,
+            message: expect.stringContaining(
+                'controller expects 1, completed rebuild requests 2, host reports 3'
+            )
+        });
+        await advance();
+        await advance();
+        expect(nativeRequest).not.toHaveBeenCalled();
+        expect(getDeployment(id)?.events).toEqual(before.events);
+    });
+});

tests/unit/provisioning-history-performance.test.ts

@@ -157,7 +157,7 @@ describe('provisioning reads with retained lifecycle history', () => {
         };
         expectIndexed(
             () => expect(latestGameSubmission('current')).toBe('2026-01-01'),
-            ['current', 'current']
+            ['current', 'current', 'current']
         );
         expectIndexed(() => expect(hasActiveProvisioning('current')).toBe(true), ['current', '']);
         expectIndexed(() => expect(housePartyTargets()).toEqual(['current']));

tests/unit/retention.test.ts

@@ -710,3 +710,35 @@ it.each(['cancelled', 'failed'])(
         ).toEqual({ history_expired_at: expect.any(String) });
     }
 );
+
+it('retains completed rebuild evidence when a stale job binding hid an unreconciled generation', async () => {
+    const a = await api();
+    const hosts = await import('../../src/lib/server/hosts/index.js');
+    const host = hosts.createHost({ name: 'Host' });
+    a.db
+        .prepare(
+            "INSERT INTO native_instances(id,server,host_id,generation,library_id,image,port,created_at) VALUES('instance','s',?,2,'lib','image',1,?)"
+        )
+        .run(host.id, old);
+    a.db
+        .prepare(
+            "INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by) VALUES('cancelled','s','rebuild','cancelled','cancelled',?,?,'test')"
+        )
+        .run(old, old);
+    const command = hosts.queueHostOperation(
+        host.id,
+        'instance.rebuild',
+        { nextGeneration: 3 },
+        { instanceId: 'instance', generation: 2 }
+    );
+    hosts.reportHostOperation(host.id, command.id, { state: 'completed' });
+    a.db
+        .prepare(
+            "INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id,applied_at) VALUES('cancelled','instance',1,?,?)"
+        )
+        .run(command.id, old);
+    a.saveFleetSettings({ ...a.fleetSettings(), provisioningKeepDays: 1, historyKeepCount: 0 });
+    expect(a.pruneHistory('provisioning')).toBe(0);
+    a.db.prepare("UPDATE native_instances SET generation=3 WHERE id='instance'").run();
+    expect(a.pruneHistory('provisioning')).toBe(1);
+});