From 1bbee98b623dc574238d9fa5f51800dc541fa3bb Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Sun, 4 Oct 2026 14:15:05 -0400 Subject: [PATCH] Fix native rebuild runtime verification deadlock --- src/lib/server/integrations/plutonium.ts | 14 +++++-- src/lib/server/provisioning/native.ts | 2 +- tests/unit/native-provisioning.test.ts | 53 ++++++++++++++++++++++++ tests/unit/plutonium.test.ts | 47 +++++++++++++++++++++ 4 files changed, 111 insertions(+), 5 deletions(-) diff --git a/src/lib/server/integrations/plutonium.ts b/src/lib/server/integrations/plutonium.ts index 5b658ad..66227f6 100644 --- a/src/lib/server/integrations/plutonium.ts +++ b/src/lib/server/integrations/plutonium.ts @@ -35,15 +35,21 @@ export function runningVersions(): PlutoniumVersion[] { : revision < target ? 'outdated' : revision > target ? 'ahead' : 'current' }; }); } -export async function probeVersion(server: string, options: { maintenance?: boolean } = {}): Promise { +export async function probeVersion(server: string, options: { maintenance?: boolean; operationId?: string } = {}): Promise { // Startup verification calls this directly; only maintenance observers yield // ownership, without replacing provisioning's version evidence with an error. - const suppressed = () => options.maintenance && hasActiveProvisioning(server); + const suppressed = () => (options.maintenance && hasActiveProvisioning(server)) || (options.operationId !== undefined + && !db().prepare(`SELECT 1 FROM server_deployments WHERE id=? AND server=? AND server_deleted_at IS NULL + AND action IN ('create','rebuild') AND state='awaiting-health' AND submitted_at IS NOT NULL`).get(options.operationId, server)); + // The verifying operation keeps its recovery slot until every readiness + // check passes. Waiting on its own slot would deadlock native rebuilds. + const recoveryBlocked = () => restartRecoveryPending(server) && !(options.operationId + && db().prepare('SELECT 1 FROM restart_rollout WHERE server=? AND operation_id=? AND healthy_at IS NULL').get(server, options.operationId)); if (suppressed()) return null; const original = getServer(server); if (!original) return null; const epoch = latestGameSubmission(server); - const recovering = restartRecoveryPending(server); + const recovering = recoveryBlocked(); const health = !recovering && original.desired_state !== 'stopped' ? await komodoInspectHealthy(server).catch(() => ({ ok: false })) : { ok: false }; if (suppressed()) return null; @@ -55,7 +61,7 @@ export async function probeVersion(server: string, options: { maintenance?: bool // or replacement registration, including replacements with no submission yet. if (!current || current.created_at !== original.created_at || epoch !== latestGameSubmission(server)) return null; const checkedAt = nowIso(); - if (restartRecoveryPending(server)) { + if (recoveryBlocked()) { result = { revision: null, error: 'Waiting for restart recovery.' }; } else if (current.desired_state === 'stopped') { result = { revision: null, error: 'Server is intentionally stopped.' }; diff --git a/src/lib/server/provisioning/native.ts b/src/lib/server/provisioning/native.ts index 7de207c..ef25d9e 100644 --- a/src/lib/server/provisioning/native.ts +++ b/src/lib/server/provisioning/native.ts @@ -319,7 +319,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) { const recipe = recipeForDeployment(job.id)!.recipe; if (status.map !== recipe.expectedMap) { hold(job, `Waiting for selected map ${recipe.expectedMap}; observed ${status.map ?? 'unknown'}.`); return; } await checkRelease(); - const revision = await probeVersion(job.server); const target = freshRelease(); + const revision = await probeVersion(job.server, { operationId: job.id }); const target = freshRelease(); if (revision === null || target === null) { hold(job, 'Waiting to verify the Plutonium runtime revision.'); return; } if (revision < target) { finishDeployment(job.id, { ok: false, message: `Running Plutonium r${revision} is older than production r${target}.` }); return; } } diff --git a/tests/unit/native-provisioning.test.ts b/tests/unit/native-provisioning.test.ts index e28ec2b..7699b57 100644 --- a/tests/unit/native-provisioning.test.ts +++ b/tests/unit/native-provisioning.test.ts @@ -159,6 +159,59 @@ describe('native instance provisioning', () => { await advance(); expect(nativeInstance(job.name)?.generation).toBe(2); }); + it('finishes native rebuild verification without waiting for its own recovery slot to be released', async () => { + const job = await queue(); await advance(); await acknowledge(job.deploymentId); + const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js'); + const rebuild = queueNativeRebuild(job.name, 'test'); if (!rebuild.ok) throw new Error(rebuild.message); + await advance(); + const { db } = await import('../../src/lib/server/db/index.js'); + const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js'); + const actual = await vi.importActual('../../src/lib/server/integrations/plutonium.js'); + const komodo = await import('../../src/lib/server/integrations/komodo.js'); + vi.spyOn(komodo, 'komodoInspectHealthy').mockResolvedValue({ ok: true, state: 'running' }); + vi.mocked(probeVersion).mockImplementationOnce(actual.probeVersion); + // Version evidence must follow the submitted command, even on a fast test host. + db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(new Date(Date.now() - 1000).toISOString(), rebuild.deploymentId); + db().prepare('UPDATE restart_rollout SET requested_at=? WHERE operation_id=?').run(new Date(Date.now() - 1000).toISOString(), rebuild.deploymentId); + await acknowledge(rebuild.deploymentId); + expect((await detail(job.deploymentId)).state).toBe('done'); + expect((await detail(rebuild.deploymentId)).state).toBe('done'); + expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE operation_id=?').get(rebuild.deploymentId)) + .toEqual({ healthy_at: expect.any(String) }); + expect(actual.runningVersions().find(version => version.server === job.name)?.revision).toBe(5354); + const { getServer } = await import('../../src/lib/server/fleet/servers.js'); + const { serverObservation } = await import('../../src/lib/server/fleet/status.js'); + const { fleetOperations } = await import('../../src/lib/server/fleet/overview.js'); + expect(serverObservation(getServer(job.name)!).health).not.toBe('provisioning'); + expect(fleetOperations().filter(operation => operation.server === job.name && operation.state !== 'done')).toEqual([]); + }); + it('resumes a rebuild waiting on version evidence after controller restart without rebuilding again', async () => { + const job = await queue(); await advance(); await acknowledge(job.deploymentId); + const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js'); + const rebuild = queueNativeRebuild(job.name, 'test'); if (!rebuild.ok) throw new Error(rebuild.message); + await advance(); + const operation = (await operationFor(rebuild.deploymentId))!; + const { db, closeDb } = await import('../../src/lib/server/db/index.js'); + const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js'); + const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); + const actual = await vi.importActual('../../src/lib/server/integrations/plutonium.js'); + const komodo = await import('../../src/lib/server/integrations/komodo.js'); + vi.spyOn(komodo, 'komodoInspectHealthy').mockResolvedValue({ ok: true, state: 'running' }); + const submittedAt = new Date(Date.now() - 1000).toISOString(); + db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(submittedAt, rebuild.deploymentId); + db().prepare('UPDATE restart_rollout SET requested_at=? WHERE operation_id=?').run(submittedAt, rebuild.deploymentId); + vi.mocked(rconPlutoniumVersion).mockResolvedValueOnce({ revision: null, error: 'Version response unavailable' }); + vi.mocked(probeVersion).mockImplementationOnce(actual.probeVersion); + await acknowledge(rebuild.deploymentId); + expect(await detail(rebuild.deploymentId)).toMatchObject({ state: 'awaiting-health', stage: 'Waiting to verify the Plutonium runtime revision.' }); + expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE operation_id=?').get(rebuild.deploymentId)).toEqual({ healthy_at: null }); + closeDb(); + vi.mocked(probeVersion).mockImplementationOnce(actual.probeVersion); + await advance(); + expect((await detail(rebuild.deploymentId)).state).toBe('done'); + expect((await operationFor(rebuild.deploymentId))!.id).toBe(operation.id); + expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'").get()).toEqual({ count: 1 }); + }); it('blocks new admission on draining hosts and retires existing instances only when empty', async () => { const job = await queue(); await advance(); await acknowledge(job.deploymentId); const { updateHost } = await import('../../src/lib/server/hosts/index.js'); updateHost(hostId, { mode: 'draining' }); diff --git a/tests/unit/plutonium.test.ts b/tests/unit/plutonium.test.ts index 4d50b3d..0b17e78 100644 --- a/tests/unit/plutonium.test.ts +++ b/tests/unit/plutonium.test.ts @@ -118,6 +118,53 @@ describe('Plutonium versions', () => { await checkRelease(); expect(fetch).toHaveBeenCalledTimes(2); }); + it('lets the verifying rebuild probe its own recovery while maintenance still yields', async () => { + const { db } = await import('../../src/lib/server/db/index.js'); + const { probeVersion, runningVersions } = await import('../../src/lib/server/integrations/plutonium.js'); + const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); + const submittedAt = new Date(Date.now() - 1000).toISOString(); + db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by) + VALUES ('rebuild', ?, 'rebuild', 'awaiting-health', 'Verifying', ?, ?, 'test')`).run(servers[0], submittedAt, submittedAt); + db().prepare('INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)').run(servers[0], submittedAt, 'rebuild'); + expect(await probeVersion(servers[0], { operationId: 'rebuild' })).toBe(5354); + expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE server=?').get(servers[0])).toEqual({ healthy_at: null }); + // A successful revision check alone cannot advertise completed recovery. + expect(runningVersions().find(version => version.server === servers[0])?.revision).toBeNull(); + const saved = db().prepare('SELECT * FROM plutonium_versions WHERE server=?').get(servers[0]); + vi.mocked(rconPlutoniumVersion).mockClear(); + expect(await probeVersion(servers[0], { maintenance: true })).toBeNull(); + expect(rconPlutoniumVersion).not.toHaveBeenCalled(); + expect(db().prepare('SELECT * FROM plutonium_versions WHERE server=?').get(servers[0])).toEqual(saved); + }); + it.each(['another-owner', 'another-server', 'unsubmitted', 'finished'] as const)('does not bypass recovery for %s', async scenario => { + const { db } = await import('../../src/lib/server/db/index.js'); + const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js'); + const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); + const submittedAt = new Date(Date.now() - 1000).toISOString(); + db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by) + VALUES ('rebuild', ?, 'rebuild', ?, 'Verifying', ?, ?, 'test')`).run(scenario === 'another-server' ? servers[1] : servers[0], + scenario === 'finished' ? 'done' : 'awaiting-health', submittedAt, scenario === 'unsubmitted' ? null : submittedAt); + db().prepare('INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)') + .run(servers[0], submittedAt, scenario === 'another-owner' ? 'other-rebuild' : 'rebuild'); + expect(await probeVersion(servers[0], { operationId: 'rebuild' })).toBeNull(); + expect(rconPlutoniumVersion).not.toHaveBeenCalled(); + expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE server=?').get(servers[0])).toEqual({ healthy_at: null }); + }); + it('discards version evidence if its verifying operation finishes during the probe', async () => { + const { db } = await import('../../src/lib/server/db/index.js'); + const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js'); + const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); + const submittedAt = new Date(Date.now() - 1000).toISOString(); + db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by) + VALUES ('rebuild', ?, 'rebuild', 'awaiting-health', 'Verifying', ?, ?, 'test')`).run(servers[0], submittedAt, submittedAt); + db().prepare('INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)').run(servers[0], submittedAt, 'rebuild'); + vi.mocked(rconPlutoniumVersion).mockImplementationOnce(async () => { + db().prepare("UPDATE server_deployments SET state='failed' WHERE id='rebuild'").run(); + return { revision: 5354, error: null }; + }); + expect(await probeVersion(servers[0], { operationId: 'rebuild' })).toBeNull(); + expect(db().prepare('SELECT * FROM plutonium_versions WHERE server=?').get(servers[0])).toBeUndefined(); + }); it('refreshes hourly fleet versions even when provisioning recently refreshed the release manifest', async () => { const m = await api(); const { checkRelease, HOUR } = await import('../../src/lib/server/integrations/plutonium.js');