Fix native rebuild runtime verification deadlock

src/lib/server/integrations/plutonium.ts

@@ -35,15 +35,21 @@ export function runningVersions(): PlutoniumVersion[] {
                 : revision < target ? 'outdated' : revision > target ? 'ahead' : 'current' };
     });
 }
-export async function probeVersion(server: string, options: { maintenance?: boolean } = {}): Promise<number | null> {
+export async function probeVersion(server: string, options: { maintenance?: boolean; operationId?: string } = {}): Promise<number | null> {
     // Startup verification calls this directly; only maintenance observers yield
     // ownership, without replacing provisioning's version evidence with an error.
-    const suppressed = () => options.maintenance && hasActiveProvisioning(server);
+    const suppressed = () => (options.maintenance && hasActiveProvisioning(server)) || (options.operationId !== undefined
+        && !db().prepare(`SELECT 1 FROM server_deployments WHERE id=? AND server=? AND server_deleted_at IS NULL
+            AND action IN ('create','rebuild') AND state='awaiting-health' AND submitted_at IS NOT NULL`).get(options.operationId, server));
+    // The verifying operation keeps its recovery slot until every readiness
+    // check passes. Waiting on its own slot would deadlock native rebuilds.
+    const recoveryBlocked = () => restartRecoveryPending(server) && !(options.operationId
+        && db().prepare('SELECT 1 FROM restart_rollout WHERE server=? AND operation_id=? AND healthy_at IS NULL').get(server, options.operationId));
     if (suppressed()) return null;
     const original = getServer(server);
     if (!original) return null;
     const epoch = latestGameSubmission(server);
-    const recovering = restartRecoveryPending(server);
+    const recovering = recoveryBlocked();
     const health = !recovering && original.desired_state !== 'stopped'
         ? await komodoInspectHealthy(server).catch(() => ({ ok: false })) : { ok: false };
     if (suppressed()) return null;
@@ -55,7 +61,7 @@ export async function probeVersion(server: string, options: { maintenance?: bool
     // or replacement registration, including replacements with no submission yet.
     if (!current || current.created_at !== original.created_at || epoch !== latestGameSubmission(server)) return null;
     const checkedAt = nowIso();
-    if (restartRecoveryPending(server)) {
+    if (recoveryBlocked()) {
         result = { revision: null, error: 'Waiting for restart recovery.' };
     } else if (current.desired_state === 'stopped') {
         result = { revision: null, error: 'Server is intentionally stopped.' };

src/lib/server/provisioning/native.ts

@@ -319,7 +319,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             const recipe = recipeForDeployment(job.id)!.recipe;
             if (status.map !== recipe.expectedMap) { hold(job, `Waiting for selected map ${recipe.expectedMap}; observed ${status.map ?? 'unknown'}.`); return; }
             await checkRelease();
-            const revision = await probeVersion(job.server); const target = freshRelease();
+            const revision = await probeVersion(job.server, { operationId: job.id }); const target = freshRelease();
             if (revision === null || target === null) { hold(job, 'Waiting to verify the Plutonium runtime revision.'); return; }
             if (revision < target) { finishDeployment(job.id, { ok: false, message: `Running Plutonium r${revision} is older than production r${target}.` }); return; }
         }

tests/unit/native-provisioning.test.ts

@@ -159,6 +159,59 @@ describe('native instance provisioning', () => {
         await advance();
         expect(nativeInstance(job.name)?.generation).toBe(2);
     });
+    it('finishes native rebuild verification without waiting for its own recovery slot to be released', async () => {
+        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
+        const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
+        const rebuild = queueNativeRebuild(job.name, 'test'); if (!rebuild.ok) throw new Error(rebuild.message);
+        await advance();
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
+        const actual = await vi.importActual<typeof import('../../src/lib/server/integrations/plutonium.js')>('../../src/lib/server/integrations/plutonium.js');
+        const komodo = await import('../../src/lib/server/integrations/komodo.js');
+        vi.spyOn(komodo, 'komodoInspectHealthy').mockResolvedValue({ ok: true, state: 'running' });
+        vi.mocked(probeVersion).mockImplementationOnce(actual.probeVersion);
+        // Version evidence must follow the submitted command, even on a fast test host.
+        db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(new Date(Date.now() - 1000).toISOString(), rebuild.deploymentId);
+        db().prepare('UPDATE restart_rollout SET requested_at=? WHERE operation_id=?').run(new Date(Date.now() - 1000).toISOString(), rebuild.deploymentId);
+        await acknowledge(rebuild.deploymentId);
+        expect((await detail(job.deploymentId)).state).toBe('done');
+        expect((await detail(rebuild.deploymentId)).state).toBe('done');
+        expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE operation_id=?').get(rebuild.deploymentId))
+            .toEqual({ healthy_at: expect.any(String) });
+        expect(actual.runningVersions().find(version => version.server === job.name)?.revision).toBe(5354);
+        const { getServer } = await import('../../src/lib/server/fleet/servers.js');
+        const { serverObservation } = await import('../../src/lib/server/fleet/status.js');
+        const { fleetOperations } = await import('../../src/lib/server/fleet/overview.js');
+        expect(serverObservation(getServer(job.name)!).health).not.toBe('provisioning');
+        expect(fleetOperations().filter(operation => operation.server === job.name && operation.state !== 'done')).toEqual([]);
+    });
+    it('resumes a rebuild waiting on version evidence after controller restart without rebuilding again', async () => {
+        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
+        const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
+        const rebuild = queueNativeRebuild(job.name, 'test'); if (!rebuild.ok) throw new Error(rebuild.message);
+        await advance();
+        const operation = (await operationFor(rebuild.deploymentId))!;
+        const { db, closeDb } = await import('../../src/lib/server/db/index.js');
+        const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
+        const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
+        const actual = await vi.importActual<typeof import('../../src/lib/server/integrations/plutonium.js')>('../../src/lib/server/integrations/plutonium.js');
+        const komodo = await import('../../src/lib/server/integrations/komodo.js');
+        vi.spyOn(komodo, 'komodoInspectHealthy').mockResolvedValue({ ok: true, state: 'running' });
+        const submittedAt = new Date(Date.now() - 1000).toISOString();
+        db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(submittedAt, rebuild.deploymentId);
+        db().prepare('UPDATE restart_rollout SET requested_at=? WHERE operation_id=?').run(submittedAt, rebuild.deploymentId);
+        vi.mocked(rconPlutoniumVersion).mockResolvedValueOnce({ revision: null, error: 'Version response unavailable' });
+        vi.mocked(probeVersion).mockImplementationOnce(actual.probeVersion);
+        await acknowledge(rebuild.deploymentId);
+        expect(await detail(rebuild.deploymentId)).toMatchObject({ state: 'awaiting-health', stage: 'Waiting to verify the Plutonium runtime revision.' });
+        expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE operation_id=?').get(rebuild.deploymentId)).toEqual({ healthy_at: null });
+        closeDb();
+        vi.mocked(probeVersion).mockImplementationOnce(actual.probeVersion);
+        await advance();
+        expect((await detail(rebuild.deploymentId)).state).toBe('done');
+        expect((await operationFor(rebuild.deploymentId))!.id).toBe(operation.id);
+        expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'").get()).toEqual({ count: 1 });
+    });
     it('blocks new admission on draining hosts and retires existing instances only when empty', async () => {
         const job = await queue(); await advance(); await acknowledge(job.deploymentId);
         const { updateHost } = await import('../../src/lib/server/hosts/index.js'); updateHost(hostId, { mode: 'draining' });

tests/unit/plutonium.test.ts

@@ -118,6 +118,53 @@ describe('Plutonium versions', () => {
         await checkRelease();
         expect(fetch).toHaveBeenCalledTimes(2);
     });
+    it('lets the verifying rebuild probe its own recovery while maintenance still yields', async () => {
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { probeVersion, runningVersions } = await import('../../src/lib/server/integrations/plutonium.js');
+        const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
+        const submittedAt = new Date(Date.now() - 1000).toISOString();
+        db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by)
+            VALUES ('rebuild', ?, 'rebuild', 'awaiting-health', 'Verifying', ?, ?, 'test')`).run(servers[0], submittedAt, submittedAt);
+        db().prepare('INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)').run(servers[0], submittedAt, 'rebuild');
+        expect(await probeVersion(servers[0], { operationId: 'rebuild' })).toBe(5354);
+        expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE server=?').get(servers[0])).toEqual({ healthy_at: null });
+        // A successful revision check alone cannot advertise completed recovery.
+        expect(runningVersions().find(version => version.server === servers[0])?.revision).toBeNull();
+        const saved = db().prepare('SELECT * FROM plutonium_versions WHERE server=?').get(servers[0]);
+        vi.mocked(rconPlutoniumVersion).mockClear();
+        expect(await probeVersion(servers[0], { maintenance: true })).toBeNull();
+        expect(rconPlutoniumVersion).not.toHaveBeenCalled();
+        expect(db().prepare('SELECT * FROM plutonium_versions WHERE server=?').get(servers[0])).toEqual(saved);
+    });
+    it.each(['another-owner', 'another-server', 'unsubmitted', 'finished'] as const)('does not bypass recovery for %s', async scenario => {
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
+        const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
+        const submittedAt = new Date(Date.now() - 1000).toISOString();
+        db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by)
+            VALUES ('rebuild', ?, 'rebuild', ?, 'Verifying', ?, ?, 'test')`).run(scenario === 'another-server' ? servers[1] : servers[0],
+                scenario === 'finished' ? 'done' : 'awaiting-health', submittedAt, scenario === 'unsubmitted' ? null : submittedAt);
+        db().prepare('INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)')
+            .run(servers[0], submittedAt, scenario === 'another-owner' ? 'other-rebuild' : 'rebuild');
+        expect(await probeVersion(servers[0], { operationId: 'rebuild' })).toBeNull();
+        expect(rconPlutoniumVersion).not.toHaveBeenCalled();
+        expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE server=?').get(servers[0])).toEqual({ healthy_at: null });
+    });
+    it('discards version evidence if its verifying operation finishes during the probe', async () => {
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
+        const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
+        const submittedAt = new Date(Date.now() - 1000).toISOString();
+        db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by)
+            VALUES ('rebuild', ?, 'rebuild', 'awaiting-health', 'Verifying', ?, ?, 'test')`).run(servers[0], submittedAt, submittedAt);
+        db().prepare('INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)').run(servers[0], submittedAt, 'rebuild');
+        vi.mocked(rconPlutoniumVersion).mockImplementationOnce(async () => {
+            db().prepare("UPDATE server_deployments SET state='failed' WHERE id='rebuild'").run();
+            return { revision: 5354, error: null };
+        });
+        expect(await probeVersion(servers[0], { operationId: 'rebuild' })).toBeNull();
+        expect(db().prepare('SELECT * FROM plutonium_versions WHERE server=?').get(servers[0])).toBeUndefined();
+    });
     it('refreshes hourly fleet versions even when provisioning recently refreshed the release manifest', async () => {
         const m = await api();
         const { checkRelease, HOUR } = await import('../../src/lib/server/integrations/plutonium.js');