josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Verify Plutonium revisions during provisioning

Browse Source
This commit is contained in:
josh
2026-09-28 14:50:06 -04:00
parent 3fb59cd598
commit a4a5298408
19 changed files with 414 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/plutonium.test.ts
+101
View File
@@ -31,6 +31,35 @@ afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.unstubAl
const api = () => import('../../src/lib/server/fleet/plutonium.js');
describe('Plutonium versions', () => {
it('coalesces published-release checks without probing the fleet', async () => {
const { checkRelease, release } = await import('../../src/lib/server/integrations/plutonium.js');
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
const first = checkRelease();
const second = checkRelease();
expect(second).toBe(first);
await Promise.all([first, second]);
expect(fetch).toHaveBeenCalledTimes(1);
expect(release()).toMatchObject({ revision: 5354, error: null });
expect(komodoInspectHealthy).not.toHaveBeenCalled();
expect(rconPlutoniumVersion).not.toHaveBeenCalled();
await checkRelease();
expect(fetch).toHaveBeenCalledTimes(2);
});
it('refreshes hourly fleet versions even when provisioning recently refreshed the release manifest', async () => {
const m = await api();
const { checkRelease, HOUR } = await import('../../src/lib/server/integrations/plutonium.js');
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
await m.checkPlutonium();
vi.setSystemTime(Date.now() + HOUR);
await checkRelease();
vi.mocked(rconPlutoniumVersion).mockClear();
await m.maintenanceTick();
expect(rconPlutoniumVersion).toHaveBeenCalledTimes(servers.length);
for (const server of servers) expect(rconPlutoniumVersion).toHaveBeenCalledWith(server);
await m.maintenanceTick();
expect(rconPlutoniumVersion).toHaveBeenCalledTimes(servers.length);
});
it('uses the official production manifest without requesting the challenged changelog page', async () => {
const m = await api(); await m.checkPlutonium();
expect(fetch).toHaveBeenCalledTimes(1);
@@ -68,6 +97,78 @@ describe('Plutonium versions', () => {
m.queueMaintenance([servers[0]]);
expect(m.plutoniumStatus().jobs[0].target_revision).toBeNull();
});
it('rejects a version observed at the startup submission timestamp, then accepts a later observation', async () => {
const { db } = await import('../../src/lib/server/db/index.js');
const { probeVersion, runningVersions, checkRelease } = await import('../../src/lib/server/integrations/plutonium.js');
const submittedAt = new Date().toISOString();
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by)
VALUES ('startup', ?, 'create', 'awaiting-health', 'Waiting for health', ?, ?, 'test')`).run(servers[0], submittedAt, submittedAt);
await checkRelease();
expect(await probeVersion(servers[0])).toBeNull();
expect(runningVersions()[0]).toMatchObject({ revision: null, checked_at: submittedAt, update_status: 'unknown', error: expect.stringContaining('after startup') });
vi.setSystemTime(Date.now() + 1);
expect(await probeVersion(servers[0])).toBe(5354);
expect(runningVersions()[0]).toMatchObject({ revision: 5354, update_status: 'current', error: null });
});
it('discards a running revision when a new lifecycle submission arrives during its probe', async () => {
const { db } = await import('../../src/lib/server/db/index.js');
const { probeVersion, runningVersions } = await import('../../src/lib/server/integrations/plutonium.js');
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
vi.mocked(rconPlutoniumVersion).mockImplementationOnce(async () => {
const now = new Date().toISOString();
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by)
VALUES ('restart', ?, 'restart', 'awaiting-health', 'Waiting for health', ?, ?, 'test')`).run(servers[0], now, now);
return { revision: 5354, error: null };
});
expect(await probeVersion(servers[0])).toBeNull();
expect(runningVersions()[0]).toMatchObject({ revision: null, update_status: 'unknown' });
expect(db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])).toBeUndefined();
});
it.each(['startup', 'replacement'] as const)('preserves a newer successful revision when an old probe finishes after %s', async change => {
const { db } = await import('../../src/lib/server/db/index.js');
const { probeVersion, runningVersions, checkRelease } = await import('../../src/lib/server/integrations/plutonium.js');
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
await checkRelease();
db().prepare('UPDATE servers SET created_at = ? WHERE name = ?').run('2026-09-21T05:58:00.000Z', servers[0]);
let finishOld!: (value: { revision: number; error: null }) => void;
let started!: () => void;
const probing = new Promise<void>(resolve => { started = resolve; });
vi.mocked(rconPlutoniumVersion).mockImplementationOnce(() => {
started();
return new Promise(resolve => { finishOld = resolve; });
});
const oldProbe = probeVersion(servers[0]);
await probing;
const now = new Date().toISOString();
if (change === 'startup') {
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by)
VALUES ('new-startup', ?, 'start', 'awaiting-health', 'Waiting for health', ?, ?, 'test')`).run(servers[0], now, now);
} else {
// Both registrations have no lifecycle submission, so only their
// registry identity distinguishes the old and replacement instances.
db().prepare('DELETE FROM servers WHERE name = ?').run(servers[0]);
db().prepare('INSERT INTO servers (name, komodo_name, created_at) VALUES (?, ?, ?)').run(servers[0], servers[0], now);
}
vi.setSystemTime(Date.now() + 1);
expect(await probeVersion(servers[0])).toBe(5354);
const saved = db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]);
vi.setSystemTime(Date.now() + 1);
finishOld({ revision: 5300, error: null });
expect(await oldProbe).toBeNull();
expect(db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])).toEqual(saved);
expect(runningVersions()[0]).toMatchObject({ revision: 5354, error: null, update_status: 'current' });
});
it('discards a running revision when the server is stopped during its probe', async () => {
const { db } = await import('../../src/lib/server/db/index.js');
const { probeVersion, runningVersions } = await import('../../src/lib/server/integrations/plutonium.js');
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
vi.mocked(rconPlutoniumVersion).mockImplementationOnce(async () => {
db().prepare("UPDATE servers SET desired_state = 'stopped' WHERE name = ?").run(servers[0]);
return { revision: 5354, error: null };
});
expect(await probeVersion(servers[0])).toBeNull();
expect(runningVersions()[0]).toMatchObject({ revision: null, error: 'Server is intentionally stopped.', update_status: 'stopped' });
});
});
describe('maintenance timezones', () => {