Verify Plutonium revisions during provisioning
This commit is contained in:
@@ -16,6 +16,7 @@ vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
||||
}));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
||||
rconGetPlayerCount: vi.fn(async () => 0),
|
||||
rconPlutoniumVersion: vi.fn(async () => ({ revision: 5354, error: null })),
|
||||
rconStatusResult: vi.fn(async () => ({ ok: true, endpoint: { host: 'localhost', port: 27016 }, rawReply: '',
|
||||
status: { players: 0, map: 'zombie_cosmodrome', gametype: null, roster: [] } }))
|
||||
}));
|
||||
@@ -29,6 +30,7 @@ beforeEach(async () => {
|
||||
vi.resetModules();
|
||||
vi.clearAllMocks();
|
||||
fixture = await newFixture([]);
|
||||
vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] })));
|
||||
vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test');
|
||||
vi.stubEnv('KOMODO_API_KEY', 'test');
|
||||
vi.stubEnv('KOMODO_API_SECRET', 'test');
|
||||
@@ -46,6 +48,7 @@ beforeEach(async () => {
|
||||
afterEach(async () => {
|
||||
await fixture.cleanup();
|
||||
vi.unstubAllEnvs();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
describe('queued removal cancellation', () => {
|
||||
@@ -486,7 +489,7 @@ describe('server creation map selection', () => {
|
||||
expect(getDeployment(id)?.deployment).toMatchObject({ state: 'awaiting-health' });
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: true, state: 'running' });
|
||||
await processCreations();
|
||||
expect(getDeployment(id)?.deployment).toMatchObject({ state: 'done', finished_at: expect.any(String), stage: 'Verified Komodo healthy, RCON healthy, and map zombie_cosmodrome' });
|
||||
expect(getDeployment(id)?.deployment).toMatchObject({ state: 'done', finished_at: expect.any(String), stage: 'Verified Komodo healthy, RCON healthy, map zombie_cosmodrome, and Plutonium r5354 (production r5354)' });
|
||||
});
|
||||
|
||||
it('returns a deployment URL identifier immediately and serves its lifecycle detail', async () => {
|
||||
|
||||
@@ -14,6 +14,7 @@ vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
||||
}));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
||||
rconGetPlayerCount: vi.fn(),
|
||||
rconPlutoniumVersion: vi.fn(),
|
||||
rconStatusResult: vi.fn()
|
||||
}));
|
||||
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
|
||||
@@ -28,6 +29,7 @@ function status(server: string, map = server.includes('_moon_') ? 'zombie_moon'
|
||||
beforeEach(async () => {
|
||||
vi.resetModules(); vi.resetAllMocks();
|
||||
fixture = await newFixture([]);
|
||||
vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] })));
|
||||
vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test');
|
||||
vi.stubEnv('KOMODO_API_KEY', 'test-key');
|
||||
vi.stubEnv('KOMODO_API_SECRET', 'test-secret');
|
||||
@@ -46,11 +48,12 @@ beforeEach(async () => {
|
||||
vi.mocked(komodo.komodoUpdateTagsOnResource).mockResolvedValue({ ok: true });
|
||||
const rcon = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rcon.rconGetPlayerCount).mockResolvedValue(0);
|
||||
vi.mocked(rcon.rconPlutoniumVersion).mockResolvedValue({ revision: 5354, error: null });
|
||||
vi.mocked(rcon.rconStatusResult).mockImplementation(async server => status(server));
|
||||
const { kick } = await import('../../src/lib/server/pipeline/worker.js');
|
||||
vi.mocked(kick).mockResolvedValue(undefined);
|
||||
});
|
||||
afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.restoreAllMocks(); });
|
||||
afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); vi.restoreAllMocks(); });
|
||||
|
||||
async function queueServers(count: number) {
|
||||
const { queueCreateServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
|
||||
@@ -15,6 +15,7 @@ vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
||||
}));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
||||
rconGetPlayerCount: vi.fn(async () => 0),
|
||||
rconPlutoniumVersion: vi.fn(async () => ({ revision: 5354, error: null })),
|
||||
rconStatusResult: vi.fn(async (server: string) => ({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: '',
|
||||
status: { players: 0, map: server.startsWith('t6_') ? 'zm_transit' : server.includes('_moon_') ? 'zombie_moon' : 'zombie_theater', gametype: null, roster: [] } }))
|
||||
}));
|
||||
@@ -29,6 +30,7 @@ const entries = [
|
||||
beforeEach(async () => {
|
||||
vi.resetModules(); vi.clearAllMocks();
|
||||
fixture = await newFixture([]);
|
||||
vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] })));
|
||||
vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test');
|
||||
vi.stubEnv('KOMODO_API_KEY', 'private-api-key');
|
||||
vi.stubEnv('KOMODO_API_SECRET', 'private-api-secret');
|
||||
@@ -38,7 +40,7 @@ beforeEach(async () => {
|
||||
vi.stubEnv('SERVER_PORT_MAX', '28980');
|
||||
vi.stubEnv('DEPLOYMENT_CREATE_CONCURRENCY', '1');
|
||||
});
|
||||
afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); });
|
||||
afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); });
|
||||
|
||||
async function previewLineup() {
|
||||
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
||||
|
||||
@@ -14,6 +14,7 @@ vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
||||
}));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
||||
rconGetPlayerCount: vi.fn(async () => 0),
|
||||
rconPlutoniumVersion: vi.fn(async () => ({ revision: 5354, error: null })),
|
||||
rconStatusResult: vi.fn(async () => ({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: '',
|
||||
status: { players: 0, map: 'zombie_theater', gametype: null, roster: [] } }))
|
||||
}));
|
||||
@@ -26,13 +27,14 @@ const input = { ...selection, gameMode: 'zombie' as const, plutoKey: 'private-pl
|
||||
beforeEach(async () => {
|
||||
vi.resetModules(); vi.clearAllMocks();
|
||||
fixture = await newFixture([]);
|
||||
vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] })));
|
||||
vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test');
|
||||
vi.stubEnv('KOMODO_API_KEY', 'private-api-key');
|
||||
vi.stubEnv('KOMODO_API_SECRET', 'private-api-secret');
|
||||
vi.stubEnv('KOMODO_TARGET_SERVER', 'test-host');
|
||||
vi.stubEnv('RCON_PASSWORD', 'private-rcon-password');
|
||||
});
|
||||
afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); });
|
||||
afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); });
|
||||
|
||||
async function seedArtifacts() {
|
||||
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
|
||||
@@ -31,6 +31,35 @@ afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.unstubAl
|
||||
const api = () => import('../../src/lib/server/fleet/plutonium.js');
|
||||
|
||||
describe('Plutonium versions', () => {
|
||||
it('coalesces published-release checks without probing the fleet', async () => {
|
||||
const { checkRelease, release } = await import('../../src/lib/server/integrations/plutonium.js');
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const first = checkRelease();
|
||||
const second = checkRelease();
|
||||
expect(second).toBe(first);
|
||||
await Promise.all([first, second]);
|
||||
expect(fetch).toHaveBeenCalledTimes(1);
|
||||
expect(release()).toMatchObject({ revision: 5354, error: null });
|
||||
expect(komodoInspectHealthy).not.toHaveBeenCalled();
|
||||
expect(rconPlutoniumVersion).not.toHaveBeenCalled();
|
||||
await checkRelease();
|
||||
expect(fetch).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
it('refreshes hourly fleet versions even when provisioning recently refreshed the release manifest', async () => {
|
||||
const m = await api();
|
||||
const { checkRelease, HOUR } = await import('../../src/lib/server/integrations/plutonium.js');
|
||||
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
await m.checkPlutonium();
|
||||
vi.setSystemTime(Date.now() + HOUR);
|
||||
await checkRelease();
|
||||
vi.mocked(rconPlutoniumVersion).mockClear();
|
||||
await m.maintenanceTick();
|
||||
expect(rconPlutoniumVersion).toHaveBeenCalledTimes(servers.length);
|
||||
for (const server of servers) expect(rconPlutoniumVersion).toHaveBeenCalledWith(server);
|
||||
await m.maintenanceTick();
|
||||
expect(rconPlutoniumVersion).toHaveBeenCalledTimes(servers.length);
|
||||
});
|
||||
it('uses the official production manifest without requesting the challenged changelog page', async () => {
|
||||
const m = await api(); await m.checkPlutonium();
|
||||
expect(fetch).toHaveBeenCalledTimes(1);
|
||||
@@ -68,6 +97,78 @@ describe('Plutonium versions', () => {
|
||||
m.queueMaintenance([servers[0]]);
|
||||
expect(m.plutoniumStatus().jobs[0].target_revision).toBeNull();
|
||||
});
|
||||
it('rejects a version observed at the startup submission timestamp, then accepts a later observation', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { probeVersion, runningVersions, checkRelease } = await import('../../src/lib/server/integrations/plutonium.js');
|
||||
const submittedAt = new Date().toISOString();
|
||||
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by)
|
||||
VALUES ('startup', ?, 'create', 'awaiting-health', 'Waiting for health', ?, ?, 'test')`).run(servers[0], submittedAt, submittedAt);
|
||||
await checkRelease();
|
||||
expect(await probeVersion(servers[0])).toBeNull();
|
||||
expect(runningVersions()[0]).toMatchObject({ revision: null, checked_at: submittedAt, update_status: 'unknown', error: expect.stringContaining('after startup') });
|
||||
vi.setSystemTime(Date.now() + 1);
|
||||
expect(await probeVersion(servers[0])).toBe(5354);
|
||||
expect(runningVersions()[0]).toMatchObject({ revision: 5354, update_status: 'current', error: null });
|
||||
});
|
||||
it('discards a running revision when a new lifecycle submission arrives during its probe', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { probeVersion, runningVersions } = await import('../../src/lib/server/integrations/plutonium.js');
|
||||
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rconPlutoniumVersion).mockImplementationOnce(async () => {
|
||||
const now = new Date().toISOString();
|
||||
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by)
|
||||
VALUES ('restart', ?, 'restart', 'awaiting-health', 'Waiting for health', ?, ?, 'test')`).run(servers[0], now, now);
|
||||
return { revision: 5354, error: null };
|
||||
});
|
||||
expect(await probeVersion(servers[0])).toBeNull();
|
||||
expect(runningVersions()[0]).toMatchObject({ revision: null, update_status: 'unknown' });
|
||||
expect(db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])).toBeUndefined();
|
||||
});
|
||||
it.each(['startup', 'replacement'] as const)('preserves a newer successful revision when an old probe finishes after %s', async change => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { probeVersion, runningVersions, checkRelease } = await import('../../src/lib/server/integrations/plutonium.js');
|
||||
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
await checkRelease();
|
||||
db().prepare('UPDATE servers SET created_at = ? WHERE name = ?').run('2026-09-21T05:58:00.000Z', servers[0]);
|
||||
let finishOld!: (value: { revision: number; error: null }) => void;
|
||||
let started!: () => void;
|
||||
const probing = new Promise<void>(resolve => { started = resolve; });
|
||||
vi.mocked(rconPlutoniumVersion).mockImplementationOnce(() => {
|
||||
started();
|
||||
return new Promise(resolve => { finishOld = resolve; });
|
||||
});
|
||||
const oldProbe = probeVersion(servers[0]);
|
||||
await probing;
|
||||
const now = new Date().toISOString();
|
||||
if (change === 'startup') {
|
||||
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by)
|
||||
VALUES ('new-startup', ?, 'start', 'awaiting-health', 'Waiting for health', ?, ?, 'test')`).run(servers[0], now, now);
|
||||
} else {
|
||||
// Both registrations have no lifecycle submission, so only their
|
||||
// registry identity distinguishes the old and replacement instances.
|
||||
db().prepare('DELETE FROM servers WHERE name = ?').run(servers[0]);
|
||||
db().prepare('INSERT INTO servers (name, komodo_name, created_at) VALUES (?, ?, ?)').run(servers[0], servers[0], now);
|
||||
}
|
||||
vi.setSystemTime(Date.now() + 1);
|
||||
expect(await probeVersion(servers[0])).toBe(5354);
|
||||
const saved = db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]);
|
||||
vi.setSystemTime(Date.now() + 1);
|
||||
finishOld({ revision: 5300, error: null });
|
||||
expect(await oldProbe).toBeNull();
|
||||
expect(db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])).toEqual(saved);
|
||||
expect(runningVersions()[0]).toMatchObject({ revision: 5354, error: null, update_status: 'current' });
|
||||
});
|
||||
it('discards a running revision when the server is stopped during its probe', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { probeVersion, runningVersions } = await import('../../src/lib/server/integrations/plutonium.js');
|
||||
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rconPlutoniumVersion).mockImplementationOnce(async () => {
|
||||
db().prepare("UPDATE servers SET desired_state = 'stopped' WHERE name = ?").run(servers[0]);
|
||||
return { revision: 5354, error: null };
|
||||
});
|
||||
expect(await probeVersion(servers[0])).toBeNull();
|
||||
expect(runningVersions()[0]).toMatchObject({ revision: null, error: 'Server is intentionally stopped.', update_status: 'stopped' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('maintenance timezones', () => {
|
||||
|
||||
@@ -8,7 +8,7 @@ vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
||||
komodoCreateDeployment: vi.fn(), komodoDeploy: vi.fn(), komodoDestroyDeployment: vi.fn(), komodoDeleteDeployment: vi.fn(),
|
||||
komodoEnsureTag: vi.fn(), komodoUpdateTagsOnResource: vi.fn()
|
||||
}));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn() }));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn(), rconPlutoniumVersion: vi.fn() }));
|
||||
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
|
||||
|
||||
let fixture: Fixture;
|
||||
@@ -20,6 +20,7 @@ beforeEach(async () => {
|
||||
vi.resetModules(); vi.resetAllMocks();
|
||||
vi.useFakeTimers({ toFake: ['Date'] });
|
||||
vi.setSystemTime(new Date('2026-09-28T12:00:00Z'));
|
||||
vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354 })));
|
||||
fixture = await newFixture([server]);
|
||||
for (const [key, value] of Object.entries({ KOMODO_ADDRESS: 'http://komodo.test', KOMODO_API_KEY: 'test', KOMODO_API_SECRET: 'test',
|
||||
KOMODO_TARGET_SERVER: 'host', RCON_PASSWORD: 'test', DEPLOYMENT_CREATE_CONCURRENCY: '1', DEPLOYMENT_RESTART_CONCURRENCY: '1',
|
||||
@@ -34,8 +35,9 @@ beforeEach(async () => {
|
||||
const rcon = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rcon.rconGetPlayerCount).mockResolvedValue(0);
|
||||
vi.mocked(rcon.rconStatusResult).mockResolvedValue(status());
|
||||
vi.mocked(rcon.rconPlutoniumVersion).mockResolvedValue({ revision: 5354, error: null });
|
||||
});
|
||||
afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); vi.unstubAllEnvs(); });
|
||||
afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); });
|
||||
|
||||
async function register() {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
@@ -57,6 +59,125 @@ async function telemetry(players: number, map = 'zombie_theater') {
|
||||
return receivedAt;
|
||||
}
|
||||
|
||||
async function launch(game: 't5' | 't6' = 't5') {
|
||||
const { queueCreateServer, processCreations } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rconStatusResult).mockResolvedValue(status(game === 't5' ? 'zombie_theater' : 'zm_transit'));
|
||||
const job = queueCreateServer({ game, gameMode: 'zombie', mapSlug: game === 't5' ? 'theater' : 'town', plutoKey: 'saved-key' });
|
||||
if (!job.ok) throw new Error(job.message);
|
||||
await processCreations();
|
||||
vi.setSystemTime(Date.now() + 15_000);
|
||||
return { ...job, processCreations };
|
||||
}
|
||||
|
||||
describe('Plutonium provisioning verification', () => {
|
||||
it.each(['t5', 't6'] as const)('verifies production and persists the running %s revision before completing', async game => {
|
||||
const job = await launch(game);
|
||||
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { runningVersions } = await import('../../src/lib/server/integrations/plutonium.js');
|
||||
const { PLUTONIUM_MANIFEST_URL } = await import('../../src/lib/plutonium.js');
|
||||
expect(rconPlutoniumVersion).not.toHaveBeenCalled();
|
||||
await job.processCreations();
|
||||
const completed = await detail(job.deploymentId);
|
||||
expect(completed.deployment.state).toBe('done');
|
||||
expect(completed.progress).toMatchObject({ phase: 'verify-plutonium', blocker: null });
|
||||
expect(completed.progress?.checks).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ key: 'plutonium-release', source: 'manifest', result: 'pass', actual: 5354 }),
|
||||
expect.objectContaining({ key: 'plutonium-version', source: 'rcon', result: 'pass', expected: 5354, actual: 5354 })
|
||||
]));
|
||||
expect(runningVersions().find(version => version.server === job.name)).toMatchObject({ revision: 5354, update_status: 'current' });
|
||||
expect(fetch).toHaveBeenCalledWith(PLUTONIUM_MANIFEST_URL, expect.objectContaining({ redirect: 'error' }));
|
||||
expect(rconPlutoniumVersion).toHaveBeenCalledTimes(1);
|
||||
expect(rconPlutoniumVersion).toHaveBeenCalledWith(job.name);
|
||||
const { isProvisioningDetail, provisioningGroups } = await import('../../src/lib/ui/provisioning-presentation.js');
|
||||
expect(isProvisioningDetail(completed, job.deploymentId)).toBe(true);
|
||||
expect(provisioningGroups(completed).find(group => group.key === 'verify')?.state).toBe('done');
|
||||
});
|
||||
|
||||
it('retries unknown runtime evidence while retaining startup capacity and never replays creation', async () => {
|
||||
const job = await launch();
|
||||
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoCreateDeployment, komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
const next = queueCreateServer({ gameMode: 'zombie', mapSlug: 'theater', plutoKey: 'second-key' });
|
||||
if (!next.ok) throw new Error(next.message);
|
||||
// A pre-start observation is not proof of the newly started process.
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare('INSERT INTO plutonium_versions (server, revision, checked_at) VALUES (?, 5354, ?)')
|
||||
.run(job.name, '2026-09-28T11:59:00.000Z');
|
||||
vi.mocked(rconPlutoniumVersion).mockResolvedValueOnce({ revision: null, error: 'Version reply timed out.' });
|
||||
await job.processCreations();
|
||||
expect((await detail(job.deploymentId)).deployment.state).toBe('awaiting-health');
|
||||
expect((await detail(job.deploymentId)).progress?.blocker).toMatchObject({ code: 'plutonium-version', automatic: true });
|
||||
expect((await detail(next.deploymentId)).deployment.state).toBe('queued');
|
||||
expect(db().prepare('SELECT revision FROM plutonium_versions WHERE server = ?').get(job.name)).toEqual({ revision: null });
|
||||
await job.processCreations();
|
||||
expect((await detail(job.deploymentId)).deployment.state).toBe('done');
|
||||
expect((await detail(next.deploymentId)).deployment.state).toBe('awaiting-health');
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(2);
|
||||
expect(komodoDeploy).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('requires a successful manifest check even with a previously cached revision, then resumes without recreating', async () => {
|
||||
const job = await launch();
|
||||
const { writeSetting } = await import('../../src/lib/server/maintenance/settings.js');
|
||||
writeSetting('plutonium_release', { revision: 5300, checkedAt: new Date().toISOString(), error: null });
|
||||
vi.mocked(fetch).mockRejectedValueOnce(new Error('Manifest offline'));
|
||||
await job.processCreations();
|
||||
expect((await detail(job.deploymentId)).deployment.state).toBe('awaiting-health');
|
||||
expect((await detail(job.deploymentId)).progress?.checks).toContainEqual(expect.objectContaining({
|
||||
key: 'plutonium-release', result: 'unknown', actual: null, message: 'Manifest offline'
|
||||
}));
|
||||
await job.processCreations();
|
||||
expect((await detail(job.deploymentId)).deployment.state).toBe('done');
|
||||
const { komodoDeploy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(komodoDeploy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('reports a confirmed outdated runtime without automatically restarting or recreating it', async () => {
|
||||
const job = await launch();
|
||||
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: 5300, error: null });
|
||||
await job.processCreations();
|
||||
const failed = await detail(job.deploymentId);
|
||||
expect(failed.deployment).toMatchObject({ state: 'failed', error: expect.stringContaining('running r5300, production is r5354') });
|
||||
expect(failed.progress?.checks).toContainEqual(expect.objectContaining({ key: 'plutonium-version', result: 'fail', expected: 5354, actual: 5300 }));
|
||||
const { runningVersions } = await import('../../src/lib/server/integrations/plutonium.js');
|
||||
expect(runningVersions().find(version => version.server === job.name)).toMatchObject({ revision: 5300, update_status: 'outdated' });
|
||||
await job.processCreations();
|
||||
const { komodoCreateDeployment, komodoDeploy, komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(komodoCreateDeployment).toHaveBeenCalledTimes(1);
|
||||
expect(komodoDeploy).toHaveBeenCalledTimes(1);
|
||||
expect(komodoRestart).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('accepts a runtime ahead of the published release without requesting a downgrade', async () => {
|
||||
const job = await launch();
|
||||
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: 5355, error: null });
|
||||
await job.processCreations();
|
||||
expect((await detail(job.deploymentId)).deployment.state).toBe('done');
|
||||
const { runningVersions } = await import('../../src/lib/server/integrations/plutonium.js');
|
||||
expect(runningVersions().find(version => version.server === job.name)?.update_status).toBe('ahead');
|
||||
});
|
||||
|
||||
it('discards a late version response when the instance was removed during the probe', async () => {
|
||||
const job = await launch();
|
||||
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
vi.mocked(rconPlutoniumVersion).mockImplementationOnce(async () => {
|
||||
db().prepare('DELETE FROM servers WHERE name = ?').run(job.name);
|
||||
return { revision: 5354, error: null };
|
||||
});
|
||||
await job.processCreations();
|
||||
const removed = await detail(job.deploymentId);
|
||||
expect(removed.deployment.server_deleted_at).toBeTruthy();
|
||||
expect(removed.deployment.state).not.toBe('done');
|
||||
expect(removed.progress?.checkpoints.some(check => check.key === 'verify-plutonium')).toBe(false);
|
||||
expect(db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(job.name)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('provisioning decision evidence', () => {
|
||||
it('distinguishes queued recipes, attempted files, transient map checks, and telemetry completion without replaying startup', async () => {
|
||||
const { queueCreateServer, processCreations } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
|
||||
@@ -81,6 +81,36 @@ describe('provisioning outcome and evidence presentation', () => {
|
||||
expect(provisioningGroups(detail).flatMap(group => group.steps).find(step => step.key === 'verify-game')).toMatchObject({ state: 'unknown', at: '2026-09-28T12:01:00Z' });
|
||||
});
|
||||
|
||||
it.each(['create', 'rebuild'] as const)('shows %s Plutonium verification from runtime evidence, separately from the release manifest', action => {
|
||||
const detail = fixture();
|
||||
detail.deployment.action = action;
|
||||
detail.progress = progress({ phase: 'verify-plutonium',
|
||||
checks: [
|
||||
{ key: 'plutonium-release', label: 'Production Plutonium release', result: 'pass', source: 'manifest', checkedAt: at, receivedAt: at, expected: null, actual: 5200, message: null },
|
||||
{ key: 'plutonium-version', label: 'Running Plutonium revision', result: 'unknown', source: 'rcon', checkedAt: at, receivedAt: null, expected: 5200, actual: null, message: 'Waiting for a runtime revision' }
|
||||
] });
|
||||
expect(isProvisioningDetail(detail, detail.deployment.id)).toBe(true);
|
||||
for (const [result, state] of [['unknown', 'unknown'], ['waiting', 'waiting'], ['fail', 'failed'], ['pass', 'done']] as const) {
|
||||
detail.progress.checks[1].result = result;
|
||||
const steps = provisioningGroups(detail).find(group => group.key === 'verify')!.steps;
|
||||
expect(steps.at(-1)).toMatchObject({ key: 'verify-plutonium', label: 'Verify Plutonium revision', state, at });
|
||||
}
|
||||
});
|
||||
|
||||
it.each(['start', 'restart', 'stop', 'destroy'] as const)('does not add Plutonium verification to %s', action => {
|
||||
const detail = fixture();
|
||||
detail.deployment.action = action;
|
||||
expect(provisioningGroups(detail).flatMap(group => group.steps).some(step => step.key === 'verify-plutonium')).toBe(false);
|
||||
});
|
||||
|
||||
it('does not claim historical creations verified Plutonium without evidence', () => {
|
||||
const detail = fixture();
|
||||
detail.deployment.state = 'done';
|
||||
detail.progress = progress({ checkpoints: [{ key: 'verify-map', label: 'Selected map verified', at, phase: 'verify-map' }] });
|
||||
expect(provisioningGroups(detail).flatMap(group => group.steps).find(step => step.key === 'verify-plutonium'))
|
||||
.toMatchObject({ state: 'unknown', at: null });
|
||||
});
|
||||
|
||||
it('preserves the failed request when later recovery checks pass', () => {
|
||||
const detail = fixture();
|
||||
detail.deployment.action = 'restart'; detail.deployment.state = 'failed';
|
||||
|
||||
@@ -54,6 +54,22 @@ describe('provisioning evidence persistence', () => {
|
||||
expect(handle.prepare("SELECT message FROM server_deployment_events WHERE kind = 'command'").get()).toEqual({ message: 'Reply was lost' });
|
||||
});
|
||||
|
||||
it('preserves Plutonium release and runtime evidence when later progress is recorded', async () => {
|
||||
const { recordProgress, read } = await setup();
|
||||
recordProgress('operation', { phase: 'verify-plutonium',
|
||||
checks: [
|
||||
{ key: 'plutonium-release', label: 'Production Plutonium release', result: 'pass', source: 'manifest', actual: 5200 },
|
||||
{ key: 'plutonium-version', label: 'Running Plutonium revision', result: 'pass', source: 'rcon', expected: 5200, actual: 5200 }
|
||||
], checkpoint: { key: 'verify-plutonium', label: 'Plutonium revision verified' } });
|
||||
vi.setSystemTime(Date.now() + 15_000);
|
||||
recordProgress('operation', { blocker: null });
|
||||
expect(read()).toMatchObject({ phase: 'verify-plutonium', phaseStartedAt: '2026-09-28T12:00:00.000Z',
|
||||
checks: [
|
||||
{ key: 'plutonium-release', result: 'pass', source: 'manifest', actual: 5200 },
|
||||
{ key: 'plutonium-version', result: 'pass', source: 'rcon', expected: 5200, actual: 5200 }
|
||||
], checkpoints: [{ key: 'verify-plutonium', phase: 'verify-plutonium' }] });
|
||||
});
|
||||
|
||||
it('keeps operation-owned recovery evidence after slot/server removal and does not rewrite the failed outcome', async () => {
|
||||
const { handle, recordProgress, read } = await setup();
|
||||
handle.prepare("UPDATE server_deployments SET state = 'failed', finished_at = '2026-09-28T12:00:00Z' WHERE id = 'operation'").run();
|
||||
|
||||
@@ -10,7 +10,7 @@ vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
||||
komodoEnsureTag: vi.fn(async (name: string) => ({ ok: true, id: name })),
|
||||
komodoUpdateTagsOnResource: vi.fn(async () => ({ ok: true }))
|
||||
}));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn() }));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn(), rconPlutoniumVersion: vi.fn() }));
|
||||
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
|
||||
|
||||
let fixture: Fixture;
|
||||
@@ -23,6 +23,7 @@ beforeEach(async () => {
|
||||
vi.useFakeTimers({ toFake: ['Date'] });
|
||||
vi.setSystemTime(new Date('2026-09-27T12:00:00Z'));
|
||||
fixture = await newFixture([server]);
|
||||
vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] })));
|
||||
for (const [key, value] of Object.entries({ KOMODO_ADDRESS: 'http://komodo.test', KOMODO_API_KEY: 'test', KOMODO_API_SECRET: 'test',
|
||||
KOMODO_TARGET_SERVER: 'host', RCON_PASSWORD: 'test', DEPLOYMENT_CREATE_CONCURRENCY: '1', DEPLOYMENT_RESTART_CONCURRENCY: '1',
|
||||
SERVER_PORT_MIN: '28960', SERVER_PORT_MAX: '28970' })) vi.stubEnv(key, value);
|
||||
@@ -52,8 +53,9 @@ beforeEach(async () => {
|
||||
const rcon = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rcon.rconGetPlayerCount).mockResolvedValue(0);
|
||||
vi.mocked(rcon.rconStatusResult).mockResolvedValue(healthyRcon);
|
||||
vi.mocked(rcon.rconPlutoniumVersion).mockResolvedValue({ revision: 5354, error: null });
|
||||
});
|
||||
afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); vi.unstubAllEnvs(); });
|
||||
afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); });
|
||||
|
||||
async function operation(id: string) {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
@@ -174,8 +176,11 @@ describe('provisioning operations', () => {
|
||||
expect(await operation(id)).toMatchObject({ state: 'awaiting-health' });
|
||||
expect(await readFile(join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp/fresh.gsc'), 'utf8')).toBe('fresh content');
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
vi.setSystemTime(Date.now() + 1);
|
||||
await processCreations();
|
||||
expect(await operation(id)).toMatchObject({ state: 'done' });
|
||||
const { runningVersions } = await import('../../src/lib/server/integrations/plutonium.js');
|
||||
expect(runningVersions().find(version => version.server === server)).toMatchObject({ revision: 5354, update_status: 'current' });
|
||||
});
|
||||
|
||||
it('rejects a rebuild without a saved key before touching the existing instance', async () => {
|
||||
|
||||
@@ -9,7 +9,7 @@ vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
||||
komodoCreateDeployment: vi.fn(), komodoDeploy: vi.fn(), komodoDestroyDeployment: vi.fn(), komodoDeleteDeployment: vi.fn(),
|
||||
komodoEnsureTag: vi.fn(), komodoUpdateTagsOnResource: vi.fn()
|
||||
}));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn() }));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn(), rconPlutoniumVersion: vi.fn() }));
|
||||
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
|
||||
|
||||
const server = 't5_zombie_theater_1';
|
||||
@@ -28,6 +28,7 @@ beforeEach(async () => {
|
||||
vi.useFakeTimers({ toFake: ['Date'] });
|
||||
vi.setSystemTime(new Date('2026-09-27T12:00:00Z'));
|
||||
fixture = await newFixture([server]);
|
||||
vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] })));
|
||||
for (const [key, value] of Object.entries({ KOMODO_ADDRESS: 'http://komodo.test', KOMODO_API_KEY: 'test', KOMODO_API_SECRET: 'test',
|
||||
KOMODO_TARGET_SERVER: 'host', RCON_PASSWORD: 'test', DEPLOYMENT_CREATE_CONCURRENCY: '1', DEPLOYMENT_RESTART_CONCURRENCY: '1',
|
||||
SERVER_PORT_MIN: '28960', SERVER_PORT_MAX: '28970' })) vi.stubEnv(key, value);
|
||||
@@ -47,9 +48,10 @@ beforeEach(async () => {
|
||||
rcon = await import('../../src/lib/server/integrations/rcon.js');
|
||||
vi.mocked(rcon.rconGetPlayerCount).mockResolvedValue(0);
|
||||
vi.mocked(rcon.rconStatusResult).mockResolvedValue(healthyRcon);
|
||||
vi.mocked(rcon.rconPlutoniumVersion).mockResolvedValue({ revision: 5354, error: null });
|
||||
watchdog = await import('../../src/lib/server/watchdog/index.js');
|
||||
});
|
||||
afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); vi.unstubAllEnvs(); });
|
||||
afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); });
|
||||
|
||||
const incidents = () => watchdog.watchdogOverview().incidents;
|
||||
const onlyIncident = () => { expect(incidents()).toHaveLength(1); return incidents()[0]; };
|
||||
@@ -375,6 +377,7 @@ describe('Watchdog drift campaigns', () => {
|
||||
.toEqual({ provisioning_id: action.operation_id, restart_state: 'pending' });
|
||||
await processCreations();
|
||||
vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
advance(1);
|
||||
await processCreations(); await sweep(); await watchdog.watchdogTick(true);
|
||||
expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved');
|
||||
expect(database.db().prepare('SELECT restart_state FROM deployment_servers WHERE deployment_id = ?').get(action.campaign_id)).toEqual({ restart_state: 'done' });
|
||||
|
||||
Reference in New Issue
Block a user