Reconcile rebuilds interrupted before replacement creation

2026-10-08 14:24:52 -04:00
parent e01b143a09
commit c4e2ff4384
11 changed files with 571 additions and 14 deletions
@@ -1946,6 +1946,141 @@ describe('native provisioning cancellation and rollback', () => {
    });
 });

+describe('partial rebuild failure reconciliation', () => {
+    const originalError = 'Replacement game library could not be prepared';
+    const failure = (generation = 1) => ({ reconciled: true, originalError, rebuildFailure: {
+        phase: 'disrupting', generation, nextGeneration: generation + 1,
+        oldContainerMissing: true, replacementContainerMissing: true, replacementFilesRemoved: true
+    } });
+
+    async function cancelledRebuild() {
+        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
+        const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
+        const first = requestLifecycle(job.name, 'rebuild');
+        if (!first.ok) throw new Error(first.error);
+        await advance();
+        const command = (await operationFor(first.deploymentId))!;
+        const { pendingHostOperations, reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        pendingHostOperations(hostId);
+        reportHostOperation(hostId, command.id, { state: 'unknown', error: originalError });
+        const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
+        expect(cancelProvisioningJob(first.deploymentId)).toBe(true);
+        const replacement = requestLifecycle(job.name, 'rebuild');
+        if (!replacement.ok) throw new Error(replacement.error);
+        return { job, first, command, replacement };
+    }
+
+    it('records missing containers before releasing a queued rebuild without replaying the cancelled command', async () => {
+        const { job, first, command, replacement } = await cancelledRebuild();
+        const { db, closeDb } = await import('../../src/lib/server/db/index.js');
+        const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
+        const { getHostOperation, reportHostOperation, pendingHostOperations } = await import('../../src/lib/server/hosts/index.js');
+        const { nativeInstance, nativeRequest } = await import('../../src/lib/server/backends/native.js');
+        const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
+        const { unsettledCancelledCommand } = await import('../../src/lib/server/provisioning/cancellation.js');
+        const ports = db().prepare('SELECT host_id,instance_id,port,protocol FROM ec_host_ports ORDER BY protocol').all();
+        await advance();
+        expect(await operationFor(replacement.deploymentId)).toBeNull();
+        expect(getDeployment(replacement.deploymentId)?.progress?.blocker).toMatchObject({ code: 'cancelled-command',
+            href: `/provisioning/${first.deploymentId}`, label: 'View cancelled operation',
+            message: expect.stringContaining(originalError) });
+        expect(getDeployment(replacement.deploymentId)?.progress?.blocker?.message).toContain('unknown');
+        reportHostOperation(hostId, command.id, { state: 'failed',
+            error: `Old container removed; replacement creation failed. ${originalError}`, result: failure() });
+        expect(unsettledCancelledCommand(job.name, replacement.deploymentId)).not.toBeNull();
+        expect(db().prepare('SELECT applied_at FROM native_provisioning WHERE deployment_id=?').get(first.deploymentId))
+            .toEqual({ applied_at: null });
+        remote.state = 'missing';
+        vi.mocked(nativeRequest).mockImplementationOnce(async () => {
+            expect(db().prepare('SELECT container_state,last_players FROM servers WHERE name=?').get(job.name))
+                .toEqual({ container_state: 'missing', last_players: null });
+            expect(db().prepare('SELECT applied_at FROM native_provisioning WHERE deployment_id=?').get(first.deploymentId))
+                .toEqual({ applied_at: expect.any(String) });
+            expect(unsettledCancelledCommand(job.name, replacement.deploymentId)).toBeNull();
+            return { exists: false, healthy: false, state: 'missing' } as never;
+        });
+        closeDb(); await advance();
+        const next = (await operationFor(replacement.deploymentId))!;
+        expect(next).toMatchObject({ kind: 'instance.rebuild', generation: 1, instanceId: command.instanceId,
+            payload: { nextGeneration: 2, env: { PLUTO_SERVER_KEY: 'saved-key' } } });
+        expect(next.id).not.toBe(command.id);
+        expect(nativeInstance(job.name)).toMatchObject({ id: command.instanceId, generation: 1, retired_at: null });
+        expect(getServerKey(job.name)).toBe('saved-key');
+        expect(db().prepare('SELECT host_id,instance_id,port,protocol FROM ec_host_ports ORDER BY protocol').all()).toEqual(ports);
+        expect((await detail(first.deploymentId)).state).toBe('cancelled');
+        expect(getHostOperation(command.id)).toMatchObject({ state: 'failed', result: failure() });
+        expect(pendingHostOperations(hostId).some(operation => operation.id === command.id)).toBe(false);
+        await advance();
+        expect((await operationFor(replacement.deploymentId))!.id).toBe(next.id);
+        expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'").get()).toEqual({ count: 2 });
+        remote.state = 'running'; await acknowledge(replacement.deploymentId);
+        expect((await detail(replacement.deploymentId)).state).toBe('done');
+        expect(nativeInstance(job.name)?.generation).toBe(2);
+        expect((await detail(first.deploymentId)).state).toBe('cancelled');
+    });
+
+    it.each(['disrupting', 'creating'])('keeps an ambiguous %s rebuild blocked even when inspections currently see no containers', async phase => {
+        const { job, first, command, replacement } = await cancelledRebuild();
+        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        reportHostOperation(hostId, command.id, { state: 'unknown', error: originalError,
+            result: { checkpoint: { phase }, state: 'missing', oldContainerMissing: true, replacementContainerMissing: true } });
+        remote.state = 'missing';
+        const { nativeRequest, nativeInstance } = await import('../../src/lib/server/backends/native.js');
+        vi.mocked(nativeRequest).mockClear();
+        const { closeDb, db } = await import('../../src/lib/server/db/index.js'); closeDb();
+        await advance(); await advance();
+        expect(await operationFor(replacement.deploymentId)).toBeNull();
+        expect(vi.mocked(nativeRequest)).not.toHaveBeenCalled();
+        expect((await operationFor(first.deploymentId))!.id).toBe(command.id);
+        expect(nativeInstance(job.name)?.generation).toBe(1);
+        expect(db().prepare('SELECT applied_at FROM native_provisioning WHERE deployment_id=?').get(first.deploymentId))
+            .toEqual({ applied_at: null });
+        expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'").get()).toEqual({ count: 1 });
+    });
+
+    it.each(['creation started', 'replacement files remain', 'wrong target generation', 'command target mismatch'])('does not release a replacement from invalid partial evidence: %s', async evidence => {
+        const { first, command, replacement } = await cancelledRebuild();
+        const result = failure();
+        if (evidence === 'creation started') result.rebuildFailure.phase = 'creating';
+        if (evidence === 'replacement files remain') result.rebuildFailure.replacementFilesRemoved = false;
+        if (evidence === 'wrong target generation') result.rebuildFailure.nextGeneration = 3;
+        const { db } = await import('../../src/lib/server/db/index.js');
+        if (evidence === 'command target mismatch') db().prepare("UPDATE ec_host_operation_payloads SET payload=json_set(payload,'$.nextGeneration',3) WHERE operation_id=?")
+            .run(command.id);
+        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        reportHostOperation(hostId, command.id, { state: 'failed', error: originalError, result });
+        remote.state = 'missing'; await advance();
+        expect(await operationFor(replacement.deploymentId)).toBeNull();
+        expect(db().prepare('SELECT applied_at FROM native_provisioning WHERE deployment_id=?').get(first.deploymentId))
+            .toEqual({ applied_at: null });
+        expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'").get()).toEqual({ count: 1 });
+    });
+
+    it('records an uncancelled partial rebuild failure without claiming replacement generation or successful health', async () => {
+        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
+        const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
+        const first = requestLifecycle(job.name, 'rebuild');
+        if (!first.ok) throw new Error(first.error);
+        await advance();
+        const command = (await operationFor(first.deploymentId))!;
+        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        reportHostOperation(hostId, command.id, { state: 'failed', result: failure(),
+            error: `Old container removed; replacement creation failed. ${originalError}` });
+        await advance();
+        const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
+        const { db } = await import('../../src/lib/server/db/index.js');
+        expect((await detail(first.deploymentId)).state).toBe('failed');
+        expect(nativeInstance(job.name)).toMatchObject({ generation: 1, retired_at: null });
+        expect(db().prepare('SELECT container_state,last_players FROM servers WHERE name=?').get(job.name))
+            .toEqual({ container_state: 'missing', last_players: null });
+        expect(db().prepare('SELECT applied_at,verified_at FROM native_provisioning WHERE deployment_id=?').get(first.deploymentId))
+            .toEqual({ applied_at: expect.any(String), verified_at: null });
+        expect(db().prepare('SELECT COUNT(*) AS count FROM ec_host_ports WHERE instance_id=?').get(command.instanceId)).toEqual({ count: 2 });
+        const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
+        expect(getServerKey(job.name)).toBe('saved-key');
+    });
+});
+
 describe('cancelled native mutation bookkeeping', () => {
    it.each([null, 'before beta'])('keeps the original host receipt across beta revisions (original=%s)', async original => {
        const job = await queue(); await advance(); await acknowledge(job.deploymentId);