josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Verify Plutonium revisions during provisioning

Browse Source
This commit is contained in:
josh
2026-09-28 14:50:06 -04:00
parent 3fb59cd598
commit a4a5298408
19 changed files with 414 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/provisioning-evidence.test.ts
+123 -2
View File
@@ -8,7 +8,7 @@ vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
komodoCreateDeployment: vi.fn(), komodoDeploy: vi.fn(), komodoDestroyDeployment: vi.fn(), komodoDeleteDeployment: vi.fn(),
komodoEnsureTag: vi.fn(), komodoUpdateTagsOnResource: vi.fn()
}));
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn() }));
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn(), rconPlutoniumVersion: vi.fn() }));
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
let fixture: Fixture;
@@ -20,6 +20,7 @@ beforeEach(async () => {
vi.resetModules(); vi.resetAllMocks();
vi.useFakeTimers({ toFake: ['Date'] });
vi.setSystemTime(new Date('2026-09-28T12:00:00Z'));
vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354 })));
fixture = await newFixture([server]);
for (const [key, value] of Object.entries({ KOMODO_ADDRESS: 'http://komodo.test', KOMODO_API_KEY: 'test', KOMODO_API_SECRET: 'test',
KOMODO_TARGET_SERVER: 'host', RCON_PASSWORD: 'test', DEPLOYMENT_CREATE_CONCURRENCY: '1', DEPLOYMENT_RESTART_CONCURRENCY: '1',
@@ -34,8 +35,9 @@ beforeEach(async () => {
const rcon = await import('../../src/lib/server/integrations/rcon.js');
vi.mocked(rcon.rconGetPlayerCount).mockResolvedValue(0);
vi.mocked(rcon.rconStatusResult).mockResolvedValue(status());
vi.mocked(rcon.rconPlutoniumVersion).mockResolvedValue({ revision: 5354, error: null });
});
afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); vi.unstubAllEnvs(); });
afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); });
async function register() {
const { db } = await import('../../src/lib/server/db/index.js');
@@ -57,6 +59,125 @@ async function telemetry(players: number, map = 'zombie_theater') {
return receivedAt;
}
async function launch(game: 't5' | 't6' = 't5') {
const { queueCreateServer, processCreations } = await import('../../src/lib/server/provisioning/instances.js');
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
vi.mocked(rconStatusResult).mockResolvedValue(status(game === 't5' ? 'zombie_theater' : 'zm_transit'));
const job = queueCreateServer({ game, gameMode: 'zombie', mapSlug: game === 't5' ? 'theater' : 'town', plutoKey: 'saved-key' });
if (!job.ok) throw new Error(job.message);
await processCreations();
vi.setSystemTime(Date.now() + 15_000);
return { ...job, processCreations };
}
describe('Plutonium provisioning verification', () => {
it.each(['t5', 't6'] as const)('verifies production and persists the running %s revision before completing', async game => {
const job = await launch(game);
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
const { runningVersions } = await import('../../src/lib/server/integrations/plutonium.js');
const { PLUTONIUM_MANIFEST_URL } = await import('../../src/lib/plutonium.js');
expect(rconPlutoniumVersion).not.toHaveBeenCalled();
await job.processCreations();
const completed = await detail(job.deploymentId);
expect(completed.deployment.state).toBe('done');
expect(completed.progress).toMatchObject({ phase: 'verify-plutonium', blocker: null });
expect(completed.progress?.checks).toEqual(expect.arrayContaining([
expect.objectContaining({ key: 'plutonium-release', source: 'manifest', result: 'pass', actual: 5354 }),
expect.objectContaining({ key: 'plutonium-version', source: 'rcon', result: 'pass', expected: 5354, actual: 5354 })
]));
expect(runningVersions().find(version => version.server === job.name)).toMatchObject({ revision: 5354, update_status: 'current' });
expect(fetch).toHaveBeenCalledWith(PLUTONIUM_MANIFEST_URL, expect.objectContaining({ redirect: 'error' }));
expect(rconPlutoniumVersion).toHaveBeenCalledTimes(1);
expect(rconPlutoniumVersion).toHaveBeenCalledWith(job.name);
const { isProvisioningDetail, provisioningGroups } = await import('../../src/lib/ui/provisioning-presentation.js');
expect(isProvisioningDetail(completed, job.deploymentId)).toBe(true);
expect(provisioningGroups(completed).find(group => group.key === 'verify')?.state).toBe('done');
});
it('retries unknown runtime evidence while retaining startup capacity and never replays creation', async () => {
const job = await launch();
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
const { komodoCreateDeployment, komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js');
const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js');
const next = queueCreateServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'second-key' });
if (!next.ok) throw new Error(next.message);
// A pre-start observation is not proof of the newly started process.
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare('INSERT INTO plutonium_versions (server, revision, checked_at) VALUES (?, 5354, ?)')
.run(job.name, '2026-09-28T11:59:00.000Z');
vi.mocked(rconPlutoniumVersion).mockResolvedValueOnce({ revision: null, error: 'Version reply timed out.' });
await job.processCreations();
expect((await detail(job.deploymentId)).deployment.state).toBe('awaiting-health');
expect((await detail(job.deploymentId)).progress?.blocker).toMatchObject({ code: 'plutonium-version', automatic: true });
expect((await detail(next.deploymentId)).deployment.state).toBe('queued');
expect(db().prepare('SELECT revision FROM plutonium_versions WHERE server = ?').get(job.name)).toEqual({ revision: null });
await job.processCreations();
expect((await detail(job.deploymentId)).deployment.state).toBe('done');
expect((await detail(next.deploymentId)).deployment.state).toBe('awaiting-health');
expect(komodoCreateDeployment).toHaveBeenCalledTimes(2);
expect(komodoDeploy).toHaveBeenCalledTimes(2);
});
it('requires a successful manifest check even with a previously cached revision, then resumes without recreating', async () => {
const job = await launch();
const { writeSetting } = await import('../../src/lib/server/maintenance/settings.js');
writeSetting('plutonium_release', { revision: 5300, checkedAt: new Date().toISOString(), error: null });
vi.mocked(fetch).mockRejectedValueOnce(new Error('Manifest offline'));
await job.processCreations();
expect((await detail(job.deploymentId)).deployment.state).toBe('awaiting-health');
expect((await detail(job.deploymentId)).progress?.checks).toContainEqual(expect.objectContaining({
key: 'plutonium-release', result: 'unknown', actual: null, message: 'Manifest offline'
}));
await job.processCreations();
expect((await detail(job.deploymentId)).deployment.state).toBe('done');
const { komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js');
expect(komodoDeploy).toHaveBeenCalledTimes(1);
});
it('reports a confirmed outdated runtime without automatically restarting or recreating it', async () => {
const job = await launch();
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: 5300, error: null });
await job.processCreations();
const failed = await detail(job.deploymentId);
expect(failed.deployment).toMatchObject({ state: 'failed', error: expect.stringContaining('running r5300, production is r5354') });
expect(failed.progress?.checks).toContainEqual(expect.objectContaining({ key: 'plutonium-version', result: 'fail', expected: 5354, actual: 5300 }));
const { runningVersions } = await import('../../src/lib/server/integrations/plutonium.js');
expect(runningVersions().find(version => version.server === job.name)).toMatchObject({ revision: 5300, update_status: 'outdated' });
await job.processCreations();
const { komodoCreateDeployment, komodoDeploy, komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
expect(komodoDeploy).toHaveBeenCalledTimes(1);
expect(komodoRestart).not.toHaveBeenCalled();
});
it('accepts a runtime ahead of the published release without requesting a downgrade', async () => {
const job = await launch();
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: 5355, error: null });
await job.processCreations();
expect((await detail(job.deploymentId)).deployment.state).toBe('done');
const { runningVersions } = await import('../../src/lib/server/integrations/plutonium.js');
expect(runningVersions().find(version => version.server === job.name)?.update_status).toBe('ahead');
});
it('discards a late version response when the instance was removed during the probe', async () => {
const job = await launch();
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
const { db } = await import('../../src/lib/server/db/index.js');
vi.mocked(rconPlutoniumVersion).mockImplementationOnce(async () => {
db().prepare('DELETE FROM servers WHERE name = ?').run(job.name);
return { revision: 5354, error: null };
});
await job.processCreations();
const removed = await detail(job.deploymentId);
expect(removed.deployment.server_deleted_at).toBeTruthy();
expect(removed.deployment.state).not.toBe('done');
expect(removed.progress?.checkpoints.some(check => check.key === 'verify-plutonium')).toBe(false);
expect(db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(job.name)).toBeUndefined();
});
});
describe('provisioning decision evidence', () => {
it('distinguishes queued recipes, attempted files, transient map checks, and telemetry completion without replaying startup', async () => {
const { queueCreateServer, processCreations } = await import('../../src/lib/server/provisioning/instances.js');