Fix native rebuild runtime verification deadlock

2026-10-04 14:15:05 -04:00
parent f8c60fd101
commit 1bbee98b62
4 changed files with 111 additions and 5 deletions
@@ -159,6 +159,59 @@ describe('native instance provisioning', () => {
        await advance();
        expect(nativeInstance(job.name)?.generation).toBe(2);
    });
+    it('finishes native rebuild verification without waiting for its own recovery slot to be released', async () => {
+        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
+        const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
+        const rebuild = queueNativeRebuild(job.name, 'test'); if (!rebuild.ok) throw new Error(rebuild.message);
+        await advance();
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
+        const actual = await vi.importActual<typeof import('../../src/lib/server/integrations/plutonium.js')>('../../src/lib/server/integrations/plutonium.js');
+        const komodo = await import('../../src/lib/server/integrations/komodo.js');
+        vi.spyOn(komodo, 'komodoInspectHealthy').mockResolvedValue({ ok: true, state: 'running' });
+        vi.mocked(probeVersion).mockImplementationOnce(actual.probeVersion);
+        // Version evidence must follow the submitted command, even on a fast test host.
+        db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(new Date(Date.now() - 1000).toISOString(), rebuild.deploymentId);
+        db().prepare('UPDATE restart_rollout SET requested_at=? WHERE operation_id=?').run(new Date(Date.now() - 1000).toISOString(), rebuild.deploymentId);
+        await acknowledge(rebuild.deploymentId);
+        expect((await detail(job.deploymentId)).state).toBe('done');
+        expect((await detail(rebuild.deploymentId)).state).toBe('done');
+        expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE operation_id=?').get(rebuild.deploymentId))
+            .toEqual({ healthy_at: expect.any(String) });
+        expect(actual.runningVersions().find(version => version.server === job.name)?.revision).toBe(5354);
+        const { getServer } = await import('../../src/lib/server/fleet/servers.js');
+        const { serverObservation } = await import('../../src/lib/server/fleet/status.js');
+        const { fleetOperations } = await import('../../src/lib/server/fleet/overview.js');
+        expect(serverObservation(getServer(job.name)!).health).not.toBe('provisioning');
+        expect(fleetOperations().filter(operation => operation.server === job.name && operation.state !== 'done')).toEqual([]);
+    });
+    it('resumes a rebuild waiting on version evidence after controller restart without rebuilding again', async () => {
+        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
+        const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
+        const rebuild = queueNativeRebuild(job.name, 'test'); if (!rebuild.ok) throw new Error(rebuild.message);
+        await advance();
+        const operation = (await operationFor(rebuild.deploymentId))!;
+        const { db, closeDb } = await import('../../src/lib/server/db/index.js');
+        const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
+        const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
+        const actual = await vi.importActual<typeof import('../../src/lib/server/integrations/plutonium.js')>('../../src/lib/server/integrations/plutonium.js');
+        const komodo = await import('../../src/lib/server/integrations/komodo.js');
+        vi.spyOn(komodo, 'komodoInspectHealthy').mockResolvedValue({ ok: true, state: 'running' });
+        const submittedAt = new Date(Date.now() - 1000).toISOString();
+        db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(submittedAt, rebuild.deploymentId);
+        db().prepare('UPDATE restart_rollout SET requested_at=? WHERE operation_id=?').run(submittedAt, rebuild.deploymentId);
+        vi.mocked(rconPlutoniumVersion).mockResolvedValueOnce({ revision: null, error: 'Version response unavailable' });
+        vi.mocked(probeVersion).mockImplementationOnce(actual.probeVersion);
+        await acknowledge(rebuild.deploymentId);
+        expect(await detail(rebuild.deploymentId)).toMatchObject({ state: 'awaiting-health', stage: 'Waiting to verify the Plutonium runtime revision.' });
+        expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE operation_id=?').get(rebuild.deploymentId)).toEqual({ healthy_at: null });
+        closeDb();
+        vi.mocked(probeVersion).mockImplementationOnce(actual.probeVersion);
+        await advance();
+        expect((await detail(rebuild.deploymentId)).state).toBe('done');
+        expect((await operationFor(rebuild.deploymentId))!.id).toBe(operation.id);
+        expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'").get()).toEqual({ count: 1 });
+    });
    it('blocks new admission on draining hosts and retires existing instances only when empty', async () => {
        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
        const { updateHost } = await import('../../src/lib/server/hosts/index.js'); updateHost(hostId, { mode: 'draining' });