josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Fix native rebuild runtime verification deadlock

Browse Source
This commit is contained in:
josh
2026-10-04 14:15:05 -04:00
parent f8c60fd101
commit 1bbee98b62
4 changed files with 111 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/native-provisioning.test.ts
+53
View File
@@ -159,6 +159,59 @@ describe('native instance provisioning', () => {
await advance();
expect(nativeInstance(job.name)?.generation).toBe(2);
});
it('finishes native rebuild verification without waiting for its own recovery slot to be released', async () => {
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
const rebuild = queueNativeRebuild(job.name, 'test'); if (!rebuild.ok) throw new Error(rebuild.message);
await advance();
const { db } = await import('../../src/lib/server/db/index.js');
const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
const actual = await vi.importActual<typeof import('../../src/lib/server/integrations/plutonium.js')>('../../src/lib/server/integrations/plutonium.js');
const komodo = await import('../../src/lib/server/integrations/komodo.js');
vi.spyOn(komodo, 'komodoInspectHealthy').mockResolvedValue({ ok: true, state: 'running' });
vi.mocked(probeVersion).mockImplementationOnce(actual.probeVersion);
// Version evidence must follow the submitted command, even on a fast test host.
db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(new Date(Date.now() - 1000).toISOString(), rebuild.deploymentId);
db().prepare('UPDATE restart_rollout SET requested_at=? WHERE operation_id=?').run(new Date(Date.now() - 1000).toISOString(), rebuild.deploymentId);
await acknowledge(rebuild.deploymentId);
expect((await detail(job.deploymentId)).state).toBe('done');
expect((await detail(rebuild.deploymentId)).state).toBe('done');
expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE operation_id=?').get(rebuild.deploymentId))
.toEqual({ healthy_at: expect.any(String) });
expect(actual.runningVersions().find(version => version.server === job.name)?.revision).toBe(5354);
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
const { serverObservation } = await import('../../src/lib/server/fleet/status.js');
const { fleetOperations } = await import('../../src/lib/server/fleet/overview.js');
expect(serverObservation(getServer(job.name)!).health).not.toBe('provisioning');
expect(fleetOperations().filter(operation => operation.server === job.name && operation.state !== 'done')).toEqual([]);
});
it('resumes a rebuild waiting on version evidence after controller restart without rebuilding again', async () => {
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
const rebuild = queueNativeRebuild(job.name, 'test'); if (!rebuild.ok) throw new Error(rebuild.message);
await advance();
const operation = (await operationFor(rebuild.deploymentId))!;
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
const actual = await vi.importActual<typeof import('../../src/lib/server/integrations/plutonium.js')>('../../src/lib/server/integrations/plutonium.js');
const komodo = await import('../../src/lib/server/integrations/komodo.js');
vi.spyOn(komodo, 'komodoInspectHealthy').mockResolvedValue({ ok: true, state: 'running' });
const submittedAt = new Date(Date.now() - 1000).toISOString();
db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(submittedAt, rebuild.deploymentId);
db().prepare('UPDATE restart_rollout SET requested_at=? WHERE operation_id=?').run(submittedAt, rebuild.deploymentId);
vi.mocked(rconPlutoniumVersion).mockResolvedValueOnce({ revision: null, error: 'Version response unavailable' });
vi.mocked(probeVersion).mockImplementationOnce(actual.probeVersion);
await acknowledge(rebuild.deploymentId);
expect(await detail(rebuild.deploymentId)).toMatchObject({ state: 'awaiting-health', stage: 'Waiting to verify the Plutonium runtime revision.' });
expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE operation_id=?').get(rebuild.deploymentId)).toEqual({ healthy_at: null });
closeDb();
vi.mocked(probeVersion).mockImplementationOnce(actual.probeVersion);
await advance();
expect((await detail(rebuild.deploymentId)).state).toBe('done');
expect((await operationFor(rebuild.deploymentId))!.id).toBe(operation.id);
expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'").get()).toEqual({ count: 1 });
});
it('blocks new admission on draining hosts and retires existing instances only when empty', async () => {
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
const { updateHost } = await import('../../src/lib/server/hosts/index.js'); updateHost(hostId, { mode: 'draining' });
tests/unit/plutonium.test.ts
+47
View File
@@ -118,6 +118,53 @@ describe('Plutonium versions', () => {
await checkRelease();
expect(fetch).toHaveBeenCalledTimes(2);
});
it('lets the verifying rebuild probe its own recovery while maintenance still yields', async () => {
const { db } = await import('../../src/lib/server/db/index.js');
const { probeVersion, runningVersions } = await import('../../src/lib/server/integrations/plutonium.js');
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
const submittedAt = new Date(Date.now() - 1000).toISOString();
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by)
VALUES ('rebuild', ?, 'rebuild', 'awaiting-health', 'Verifying', ?, ?, 'test')`).run(servers[0], submittedAt, submittedAt);
db().prepare('INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)').run(servers[0], submittedAt, 'rebuild');
expect(await probeVersion(servers[0], { operationId: 'rebuild' })).toBe(5354);
expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE server=?').get(servers[0])).toEqual({ healthy_at: null });
// A successful revision check alone cannot advertise completed recovery.
expect(runningVersions().find(version => version.server === servers[0])?.revision).toBeNull();
const saved = db().prepare('SELECT * FROM plutonium_versions WHERE server=?').get(servers[0]);
vi.mocked(rconPlutoniumVersion).mockClear();
expect(await probeVersion(servers[0], { maintenance: true })).toBeNull();
expect(rconPlutoniumVersion).not.toHaveBeenCalled();
expect(db().prepare('SELECT * FROM plutonium_versions WHERE server=?').get(servers[0])).toEqual(saved);
});
it.each(['another-owner', 'another-server', 'unsubmitted', 'finished'] as const)('does not bypass recovery for %s', async scenario => {
const { db } = await import('../../src/lib/server/db/index.js');
const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
const submittedAt = new Date(Date.now() - 1000).toISOString();
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by)
VALUES ('rebuild', ?, 'rebuild', ?, 'Verifying', ?, ?, 'test')`).run(scenario === 'another-server' ? servers[1] : servers[0],
scenario === 'finished' ? 'done' : 'awaiting-health', submittedAt, scenario === 'unsubmitted' ? null : submittedAt);
db().prepare('INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)')
.run(servers[0], submittedAt, scenario === 'another-owner' ? 'other-rebuild' : 'rebuild');
expect(await probeVersion(servers[0], { operationId: 'rebuild' })).toBeNull();
expect(rconPlutoniumVersion).not.toHaveBeenCalled();
expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE server=?').get(servers[0])).toEqual({ healthy_at: null });
});
it('discards version evidence if its verifying operation finishes during the probe', async () => {
const { db } = await import('../../src/lib/server/db/index.js');
const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
const submittedAt = new Date(Date.now() - 1000).toISOString();
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by)
VALUES ('rebuild', ?, 'rebuild', 'awaiting-health', 'Verifying', ?, ?, 'test')`).run(servers[0], submittedAt, submittedAt);
db().prepare('INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)').run(servers[0], submittedAt, 'rebuild');
vi.mocked(rconPlutoniumVersion).mockImplementationOnce(async () => {
db().prepare("UPDATE server_deployments SET state='failed' WHERE id='rebuild'").run();
return { revision: 5354, error: null };
});
expect(await probeVersion(servers[0], { operationId: 'rebuild' })).toBeNull();
expect(db().prepare('SELECT * FROM plutonium_versions WHERE server=?').get(servers[0])).toBeUndefined();
});
it('refreshes hourly fleet versions even when provisioning recently refreshed the release manifest', async () => {
const m = await api();
const { checkRelease, HOUR } = await import('../../src/lib/server/integrations/plutonium.js');