Fix rebuild generation reconciliation (Refs #155)

tests/unit/auth-access.test.ts

@@ -90,6 +90,7 @@ describe('management route permissions', () => {
         ['/api/rebalancing', 'POST'],
         ['/api/provisioning/[id]/cancel', 'POST'],
         ['/api/provisioning/[id]/resume', 'POST'],
+        ['/api/provisioning/[id]/reconcile-generation', 'POST'],
         ['/api/deployment-collections', 'POST'],
         ['/api/deployment-collections/preview', 'POST'],
         ['/api/deployment-collections/[id]/cancel', 'POST'],

tests/unit/database-migrations.test.ts

@@ -291,3 +291,18 @@ it('adds provisioning report invalidation at version 14 without invalidating for
     upgraded.exec('DELETE FROM server_deployments');
     expect(revision()).toBeGreaterThan(completed);
 });
+
+it('adds the generation evidence boundary to a version 23 database without changing instance identity', async () => {
+    const { db, closeDb } = await import('../../src/lib/server/db/index.js');
+    const handle = db();
+    handle.exec(
+        "INSERT INTO native_instances(id,server,host_id,generation,library_id,image,port,created_at) VALUES('instance','s','host',2,'lib','image',1,'2026-10-14T00:00:00Z')"
+    );
+    handle.exec('ALTER TABLE native_instances DROP COLUMN generation_reconciled_at');
+    handle.pragma('user_version = 23');
+    closeDb();
+    expect(
+        db().prepare('SELECT id,generation,generation_reconciled_at FROM native_instances').get()
+    ).toEqual({ id: 'instance', generation: 2, generation_reconciled_at: null });
+    expect(db().pragma('foreign_key_check')).toEqual([]);
+});

tests/unit/native-provisioning.test.ts

@@ -465,7 +465,16 @@ describe('provisioning health handoff', () => {
                 .run(old, old, old, job.name);
             vi.useFakeTimers({ toFake: ['Date'] });
             vi.setSystemTime(Date.now() + 1000);
-            const observedAt = new Date().toISOString();
+            if (action === 'rebuild') {
+                // A new generation now establishes its own durable evidence boundary.
+                // Observe game health strictly after that transition.
+                const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
+                vi.mocked(nativeRequest).mockImplementationOnce(async () => {
+                    vi.setSystemTime(Date.now() + 1);
+                    return { healthy: true, state: 'running', exists: true } as never;
+                });
+            }
+            const observedAt = new Date(Date.now() + (action === 'rebuild' ? 1 : 0)).toISOString();
             const roster = [
                 { num: 0, name: 'Player', guid: '12345678901234567', score: 500, ping: 20 }
             ];
@@ -4288,3 +4297,276 @@ describe('game key pool launch lifecycle', () => {
         expect(serverKeyPool('t5')).toEqual({ total: 1, available: 1 });
     });
 });
+
+describe('completed rebuild generation receipts (#155)', () => {
+    async function rebuild() {
+        const job = await queue();
+        await advance();
+        await acknowledge(job.deploymentId);
+        const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
+        const requested = requestLifecycle(job.name, 'rebuild');
+        if (!requested.ok) throw new Error(requested.error);
+        await advance();
+        return {
+            job,
+            id: requested.deploymentId,
+            operation: (await operationFor(requested.deploymentId))!
+        };
+    }
+
+    async function inventoryGeneration(
+        instanceId: string,
+        generation: number,
+        observedAt = new Date().toISOString(),
+        targetHost = hostId
+    ) {
+        const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
+        heartbeat(targetHost, {
+            ...inventory(),
+            instanceStates: [
+                { instanceId, generation, state: 'running', healthy: true, observedAt }
+            ]
+        });
+    }
+
+    it('repairs an older job binding from the actual command before verification, including duplicate receipts and restart', async () => {
+        const { job, id } = await rebuild();
+        await acknowledge(id);
+        const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
+        const next = requestLifecycle(job.name, 'rebuild');
+        if (!next.ok) throw new Error(next.error);
+        await advance();
+        const operation = (await operationFor(next.deploymentId))!;
+        expect(operation.generation).toBe(2);
+        const { db, closeDb } = await import('../../src/lib/server/db/index.js');
+        // Historical job snapshots could stay on generation 1 while the submitted
+        // command correctly targeted 2 -> 3. The old completion path kept 2.
+        db()
+            .prepare('UPDATE native_provisioning SET generation=1 WHERE deployment_id=?')
+            .run(next.deploymentId);
+        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        reportHostOperation(hostId, operation.id, {
+            state: 'completed',
+            result: { instanceId: operation.instanceId, generation: 3, healthy: true }
+        });
+        await inventoryGeneration(operation.instanceId!, 3);
+        closeDb();
+        const { nativeRequest, nativeInstance } =
+            await import('../../src/lib/server/backends/native.js');
+        vi.mocked(nativeRequest).mockClear();
+        remote.state = 'starting';
+        await advance();
+        expect(nativeInstance(job.name)?.generation).toBe(3);
+        expect(nativeRequest).toHaveBeenCalledWith(
+            expect.objectContaining({ id: operation.instanceId, generation: 3 }),
+            'instance.inspect'
+        );
+        expect(
+            db()
+                .prepare(
+                    'SELECT generation,applied_at FROM native_provisioning WHERE deployment_id=?'
+                )
+                .get(next.deploymentId)
+        ).toEqual({ generation: 2, applied_at: expect.any(String) });
+        const { reconcileNativeProvisioningReceipt } =
+            await import('../../src/lib/server/provisioning/native.js');
+        const before = db()
+            .prepare('SELECT generation_reconciled_at FROM native_instances WHERE id=?')
+            .get(operation.instanceId);
+        reconcileNativeProvisioningReceipt(next.deploymentId);
+        closeDb();
+        reconcileNativeProvisioningReceipt(next.deploymentId);
+        expect(nativeInstance(job.name)?.generation).toBe(3);
+        expect(
+            db()
+                .prepare('SELECT generation_reconciled_at FROM native_instances WHERE id=?')
+                .get(operation.instanceId)
+        ).toEqual(before);
+        expect((await detail(next.deploymentId)).state).toBe('awaiting-health');
+        for (const observation of await observations(job.name))
+            expect(observation.health).not.toBe('online');
+        const { latestGameSubmission, latestGameSubmissions } =
+            await import('../../src/lib/server/fleet/lifecycle-evidence.js');
+        expect(latestGameSubmission(job.name)).toBe(
+            (before as { generation_reconciled_at: string }).generation_reconciled_at
+        );
+        expect(latestGameSubmissions().get(job.name)).toBe(latestGameSubmission(job.name));
+        remote.state = 'running';
+        await advance();
+        expect((await detail(next.deploymentId)).state).toBe('done');
+        const restart = requestLifecycle(job.name, 'restart');
+        if (!restart.ok) throw new Error(restart.error);
+        await advance();
+        expect(await operationFor(restart.deploymentId)).toMatchObject({ generation: 3 });
+    });
+
+    it.each(['before', 'after'] as const)(
+        'reconciles delayed completion reported %s cancellation across restart',
+        async (when) => {
+            const { job, id, operation } = await rebuild();
+            const { reportHostOperation, pendingHostOperations } =
+                await import('../../src/lib/server/hosts/index.js');
+            const { cancelProvisioningJob } =
+                await import('../../src/lib/server/provisioning/cancellation.js');
+            const { closeDb } = await import('../../src/lib/server/db/index.js');
+            pendingHostOperations(hostId);
+            if (when === 'before')
+                reportHostOperation(hostId, operation.id, { state: 'completed' });
+            expect(cancelProvisioningJob(id)).toBe(true);
+            closeDb();
+            if (when === 'after') reportHostOperation(hostId, operation.id, { state: 'completed' });
+            await advance();
+            await advance();
+            const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
+            expect(nativeInstance(job.name)?.generation).toBe(2);
+            expect((await detail(id)).state).toBe('cancelled');
+        }
+    );
+
+    it('recovers an already-applied cancelled mismatch through the API and targets replacement logs, health and removal', async () => {
+        const { job, id, operation } = await rebuild();
+        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        reportHostOperation(hostId, operation.id, {
+            state: 'completed',
+            result: { instanceId: operation.instanceId, generation: 2 }
+        });
+        const { cancelProvisioningJob } =
+            await import('../../src/lib/server/provisioning/cancellation.js');
+        expect(cancelProvisioningJob(id)).toBe(true);
+        const { db } = await import('../../src/lib/server/db/index.js');
+        db()
+            .prepare('UPDATE native_provisioning SET applied_at=? WHERE deployment_id=?')
+            .run(operation.updatedAt, id);
+        await inventoryGeneration(operation.instanceId!, 2);
+        const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
+        expect(getDeployment(id)?.generationRecovery).toMatchObject({
+            expectedGeneration: 1,
+            observedGeneration: 2,
+            available: true
+        });
+        const { POST } =
+            await import('../../src/routes/api/provisioning/[id]/reconcile-generation/+server.js');
+        expect(await POST({ params: { id } }).json()).toEqual({ ok: true, generation: 2 });
+        expect(POST({ params: { id } }).status).toBe(200);
+        expect((await detail(id)).state).toBe('cancelled');
+        expect(getDeployment(id)?.generationRecovery).toBeNull();
+        const backend = await import('../../src/lib/server/backends/native.js');
+        vi.mocked(backend.nativeRequest).mockRestore();
+        const hosts = await import('../../src/lib/server/hosts/index.js');
+        vi.spyOn(hosts, 'waitHostOperationStatus').mockImplementation(async (id) =>
+            hosts.reportHostOperationStatus(hostId, id, {
+                state: 'completed',
+                result: { healthy: true, state: 'running' }
+            })
+        );
+        await backend.nativeRequest(backend.nativeInstance(job.name)!, 'logs.read', {
+            file: 'console.log'
+        });
+        expect(await backend.nativeHealth(job.name)).toMatchObject({ ok: true });
+        expect(
+            db()
+                .prepare(
+                    "SELECT generation FROM ec_host_operations WHERE kind IN ('logs.read','instance.inspect')"
+                )
+                .all()
+                .every((row) => (row as { generation: number }).generation === 2)
+        ).toBe(true);
+        const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
+        const removal = destroyServer(job.name);
+        if (!removal.ok || !removal.deploymentId) throw new Error('Removal not queued');
+        await advance();
+        expect(await operationFor(removal.deploymentId)).toMatchObject({
+            generation: 2,
+            kind: 'instance.remove'
+        });
+    });
+
+    it.each([
+        'host',
+        'instance',
+        'result-instance',
+        'result-generation',
+        'payload',
+        'stale-inventory',
+        'missing-inventory',
+        'inventory-instance'
+    ] as const)('rejects recovery evidence with %s mismatch', async (mismatch) => {
+        const { job, id, operation } = await rebuild();
+        const { reportHostOperation, createHost } =
+            await import('../../src/lib/server/hosts/index.js');
+        reportHostOperation(hostId, operation.id, {
+            state: 'completed',
+            result: {
+                instanceId:
+                    mismatch === 'result-instance' ? 'another-instance' : operation.instanceId,
+                generation: mismatch === 'result-generation' ? 3 : 2
+            }
+        });
+        const { db } = await import('../../src/lib/server/db/index.js');
+        if (mismatch === 'host')
+            db()
+                .prepare('UPDATE ec_host_operations SET host_id=? WHERE id=?')
+                .run(createHost({ name: 'Other host' }).id, operation.id);
+        if (mismatch === 'instance')
+            db()
+                .prepare('UPDATE ec_host_operations SET instance_id=? WHERE id=?')
+                .run('another-instance', operation.id);
+        if (mismatch === 'payload')
+            db()
+                .prepare(
+                    "UPDATE ec_host_operation_payloads SET payload=json_set(payload,'$.nextGeneration',3) WHERE operation_id=?"
+                )
+                .run(operation.id);
+        if (mismatch !== 'missing-inventory')
+            await inventoryGeneration(
+                mismatch === 'inventory-instance' ? 'another-instance' : operation.instanceId!,
+                2,
+                mismatch === 'stale-inventory' ? '2000-01-01T00:00:00Z' : new Date().toISOString()
+            );
+        const { POST } =
+            await import('../../src/routes/api/provisioning/[id]/reconcile-generation/+server.js');
+        expect(POST({ params: { id } }).status).toBe(409);
+        const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
+        expect(nativeInstance(job.name)?.generation).toBe(1);
+    });
+
+    it('pauses after an agent stale-generation rejection even when inventory is unavailable', async () => {
+        const { id, operation } = await rebuild();
+        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        reportHostOperation(hostId, operation.id, { state: 'completed' });
+        const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
+        vi.mocked(nativeRequest).mockClear();
+        vi.mocked(nativeRequest).mockRejectedValueOnce(
+            new Error('Instance generation has been superseded or removed')
+        );
+        await advance();
+        expect((await detail(id)).state).toBe('interrupted');
+        expect((await detail(id)).error).toContain('controller expects 2, host reports unknown');
+        await advance();
+        expect(nativeRequest).toHaveBeenCalledTimes(1);
+    });
+
+    it('stops verification retries on a confirmed generation mismatch and keeps expected/observed evidence', async () => {
+        const { id, operation } = await rebuild();
+        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        reportHostOperation(hostId, operation.id, { state: 'completed' });
+        await inventoryGeneration(operation.instanceId!, 3);
+        const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
+        vi.mocked(nativeRequest).mockClear();
+        await advance();
+        const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
+        const before = getDeployment(id)!;
+        expect(before.deployment.state).toBe('interrupted');
+        expect(before.progress?.blocker).toMatchObject({
+            code: 'generation-recovery',
+            automatic: false,
+            message: expect.stringContaining(
+                'controller expects 1, completed rebuild requests 2, host reports 3'
+            )
+        });
+        await advance();
+        await advance();
+        expect(nativeRequest).not.toHaveBeenCalled();
+        expect(getDeployment(id)?.events).toEqual(before.events);
+    });
+});

tests/unit/provisioning-history-performance.test.ts

@@ -157,7 +157,7 @@ describe('provisioning reads with retained lifecycle history', () => {
         };
         expectIndexed(
             () => expect(latestGameSubmission('current')).toBe('2026-01-01'),
-            ['current', 'current']
+            ['current', 'current', 'current']
         );
         expectIndexed(() => expect(hasActiveProvisioning('current')).toBe(true), ['current', '']);
         expectIndexed(() => expect(housePartyTargets()).toEqual(['current']));

tests/unit/retention.test.ts

@@ -710,3 +710,35 @@ it.each(['cancelled', 'failed'])(
         ).toEqual({ history_expired_at: expect.any(String) });
     }
 );
+
+it('retains completed rebuild evidence when a stale job binding hid an unreconciled generation', async () => {
+    const a = await api();
+    const hosts = await import('../../src/lib/server/hosts/index.js');
+    const host = hosts.createHost({ name: 'Host' });
+    a.db
+        .prepare(
+            "INSERT INTO native_instances(id,server,host_id,generation,library_id,image,port,created_at) VALUES('instance','s',?,2,'lib','image',1,?)"
+        )
+        .run(host.id, old);
+    a.db
+        .prepare(
+            "INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by) VALUES('cancelled','s','rebuild','cancelled','cancelled',?,?,'test')"
+        )
+        .run(old, old);
+    const command = hosts.queueHostOperation(
+        host.id,
+        'instance.rebuild',
+        { nextGeneration: 3 },
+        { instanceId: 'instance', generation: 2 }
+    );
+    hosts.reportHostOperation(host.id, command.id, { state: 'completed' });
+    a.db
+        .prepare(
+            "INSERT INTO native_provisioning(deployment_id,instance_id,generation,operation_id,applied_at) VALUES('cancelled','instance',1,?,?)"
+        )
+        .run(command.id, old);
+    a.saveFleetSettings({ ...a.fleetSettings(), provisioningKeepDays: 1, historyKeepCount: 0 });
+    expect(a.pruneHistory('provisioning')).toBe(0);
+    a.db.prepare("UPDATE native_instances SET generation=3 WHERE id='instance'").run();
+    expect(a.pruneHistory('provisioning')).toBe(1);
+});