josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Fix native rebuild runtime verification deadlock

Browse Source
This commit is contained in:
josh
2026-10-04 14:15:05 -04:00
parent f8c60fd101
commit 1bbee98b62
4 changed files with 111 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/plutonium.test.ts
+47
View File
@@ -118,6 +118,53 @@ describe('Plutonium versions', () => {
await checkRelease();
expect(fetch).toHaveBeenCalledTimes(2);
});
it('lets the verifying rebuild probe its own recovery while maintenance still yields', async () => {
const { db } = await import('../../src/lib/server/db/index.js');
const { probeVersion, runningVersions } = await import('../../src/lib/server/integrations/plutonium.js');
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
const submittedAt = new Date(Date.now() - 1000).toISOString();
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by)
VALUES ('rebuild', ?, 'rebuild', 'awaiting-health', 'Verifying', ?, ?, 'test')`).run(servers[0], submittedAt, submittedAt);
db().prepare('INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)').run(servers[0], submittedAt, 'rebuild');
expect(await probeVersion(servers[0], { operationId: 'rebuild' })).toBe(5354);
expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE server=?').get(servers[0])).toEqual({ healthy_at: null });
// A successful revision check alone cannot advertise completed recovery.
expect(runningVersions().find(version => version.server === servers[0])?.revision).toBeNull();
const saved = db().prepare('SELECT * FROM plutonium_versions WHERE server=?').get(servers[0]);
vi.mocked(rconPlutoniumVersion).mockClear();
expect(await probeVersion(servers[0], { maintenance: true })).toBeNull();
expect(rconPlutoniumVersion).not.toHaveBeenCalled();
expect(db().prepare('SELECT * FROM plutonium_versions WHERE server=?').get(servers[0])).toEqual(saved);
});
it.each(['another-owner', 'another-server', 'unsubmitted', 'finished'] as const)('does not bypass recovery for %s', async scenario => {
const { db } = await import('../../src/lib/server/db/index.js');
const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
const submittedAt = new Date(Date.now() - 1000).toISOString();
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by)
VALUES ('rebuild', ?, 'rebuild', ?, 'Verifying', ?, ?, 'test')`).run(scenario === 'another-server' ? servers[1] : servers[0],
scenario === 'finished' ? 'done' : 'awaiting-health', submittedAt, scenario === 'unsubmitted' ? null : submittedAt);
db().prepare('INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)')
.run(servers[0], submittedAt, scenario === 'another-owner' ? 'other-rebuild' : 'rebuild');
expect(await probeVersion(servers[0], { operationId: 'rebuild' })).toBeNull();
expect(rconPlutoniumVersion).not.toHaveBeenCalled();
expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE server=?').get(servers[0])).toEqual({ healthy_at: null });
});
it('discards version evidence if its verifying operation finishes during the probe', async () => {
const { db } = await import('../../src/lib/server/db/index.js');
const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
const submittedAt = new Date(Date.now() - 1000).toISOString();
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by)
VALUES ('rebuild', ?, 'rebuild', 'awaiting-health', 'Verifying', ?, ?, 'test')`).run(servers[0], submittedAt, submittedAt);
db().prepare('INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)').run(servers[0], submittedAt, 'rebuild');
vi.mocked(rconPlutoniumVersion).mockImplementationOnce(async () => {
db().prepare("UPDATE server_deployments SET state='failed' WHERE id='rebuild'").run();
return { revision: 5354, error: null };
});
expect(await probeVersion(servers[0], { operationId: 'rebuild' })).toBeNull();
expect(db().prepare('SELECT * FROM plutonium_versions WHERE server=?').get(servers[0])).toBeUndefined();
});
it('refreshes hourly fleet versions even when provisioning recently refreshed the release manifest', async () => {
const m = await api();
const { checkRelease, HOUR } = await import('../../src/lib/server/integrations/plutonium.js');