152 lines
10 KiB
TypeScript
152 lines
10 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
import { nextMaintenanceAt, parseChangelogRevision, parsePlutoniumRevision } from '../../src/lib/plutonium.js';
|
|
|
|
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoRestart: vi.fn(), komodoInspectHealthy: vi.fn() }));
|
|
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn(), rconPlutoniumVersion: vi.fn() }));
|
|
|
|
let fixture: Fixture;
|
|
const servers = ['t5_zombie_theater_1', 't6_zombie_town_1'];
|
|
beforeEach(async () => {
|
|
vi.resetModules(); vi.resetAllMocks();
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
vi.setSystemTime(new Date('2026-09-21T05:59:00Z'));
|
|
fixture = await newFixture(servers);
|
|
vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test');
|
|
vi.stubEnv('KOMODO_API_KEY', 'test'); vi.stubEnv('KOMODO_API_SECRET', 'test');
|
|
vi.stubEnv('RCON_PASSWORD', 'test'); vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '1');
|
|
vi.stubGlobal('fetch', vi.fn().mockImplementation(async () => new Response('<h2 id="r5354">r5354</h2><h2>r5300</h2>')));
|
|
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
|
await seedServerRegistry();
|
|
const k = await import('../../src/lib/server/integrations/komodo.js');
|
|
vi.mocked(k.komodoRestart).mockResolvedValue({ ok: true });
|
|
vi.mocked(k.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
|
const r = await import('../../src/lib/server/integrations/rcon.js');
|
|
vi.mocked(r.rconGetPlayerCount).mockResolvedValue(0);
|
|
vi.mocked(r.rconPlutoniumVersion).mockResolvedValue({ revision: 5354, error: null });
|
|
vi.mocked(r.rconStatusResult).mockResolvedValue({ ok: true, endpoint: { host: 'test', port: 28960 }, rawReply: '', status: { players: 0, map: null, gametype: null, roster: [] } });
|
|
});
|
|
afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); vi.useRealTimers(); });
|
|
const api = () => import('../../src/lib/server/fleet/plutonium.js');
|
|
|
|
describe('Plutonium versions', () => {
|
|
it('parses explicit revisions and release headings without interpreting engine numbers or links as releases', () => {
|
|
expect(parsePlutoniumRevision('print\nPlutonium T6 r5354 win-x86')).toBe(5354);
|
|
expect(parsePlutoniumRevision('version 1.0 build 20260921')).toBeNull();
|
|
expect(parseChangelogRevision('<a href="#r9999">r9999</a><h2><a>r5354</a></h2>')).toBe(5354);
|
|
expect(() => parseChangelogRevision('<h1>Service unavailable</h1>')).toThrow();
|
|
});
|
|
it('preserves the cached release with a visible error and clears failed running-version observations', async () => {
|
|
const m = await api(); await m.checkPlutonium();
|
|
vi.mocked(fetch).mockRejectedValue(new Error('Offline'));
|
|
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
|
vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: null, error: 'Timeout' });
|
|
await m.checkPlutonium();
|
|
expect(m.plutoniumStatus().release).toMatchObject({ revision: 5354, error: 'Offline' });
|
|
expect(m.plutoniumStatus().servers.every(s => s.revision === null && s.error === 'Timeout')).toBe(true);
|
|
});
|
|
});
|
|
|
|
describe('persistent maintenance', () => {
|
|
it('waits for occupied and unreachable servers, deduplicates requests, and verifies after recovery', async () => {
|
|
const m = await api(); await m.checkPlutonium();
|
|
expect(m.queueMaintenance([servers[0]])).toBe(1);
|
|
expect(m.queueMaintenance([servers[0]])).toBe(0);
|
|
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(2).mockResolvedValueOnce(null);
|
|
await m.maintenanceTick(); await m.maintenanceTick();
|
|
expect(komodoRestart).not.toHaveBeenCalled();
|
|
await Promise.all([m.maintenanceTick(), m.maintenanceTick()]);
|
|
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
|
expect(m.plutoniumStatus().jobs[0].state).toBe('restarting');
|
|
await m.maintenanceTick();
|
|
expect(m.plutoniumStatus().jobs[0].state).toBe('restarting');
|
|
vi.setSystemTime(Date.now() + 15_000);
|
|
await m.maintenanceTick();
|
|
expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'done', detail: 'Server recovered; running r5354.' });
|
|
});
|
|
it('shares rolling capacity across servers and survives closing the database', async () => {
|
|
const m = await api(); await m.checkPlutonium(); m.queueMaintenance(servers);
|
|
await m.maintenanceTick();
|
|
const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb();
|
|
expect(m.plutoniumStatus().jobs.map(j => j.state)).toEqual(['queued', 'restarting']);
|
|
vi.setSystemTime(Date.now() + 15_000); await m.maintenanceTick();
|
|
expect(m.plutoniumStatus().jobs.map(j => j.state)).toEqual(['restarting', 'done']);
|
|
});
|
|
it('reports a healthy but outdated server as failed without repeatedly restarting it', async () => {
|
|
const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
|
|
await m.maintenanceTick();
|
|
const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
|
|
vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: 5300, error: null });
|
|
vi.setSystemTime(Date.now() + 15_000); await m.maintenanceTick(); await m.maintenanceTick();
|
|
expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'failed', target_revision: 5354 });
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
|
});
|
|
it('does not retry an ambiguous restart timeout', async () => {
|
|
const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
vi.mocked(komodoRestart).mockResolvedValue({ ok: false, kind: 'http-error', message: 'timeout' });
|
|
await m.maintenanceTick(); vi.setSystemTime(Date.now() + 15_000); await m.maintenanceTick();
|
|
expect(m.plutoniumStatus().jobs[0].state).toBe('failed');
|
|
expect(komodoRestart).toHaveBeenCalledTimes(1);
|
|
});
|
|
it('honors cancellation arriving during the player check', async () => {
|
|
const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
|
|
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
|
vi.mocked(rconGetPlayerCount).mockImplementationOnce(async () => {
|
|
m.cancelMaintenance(m.plutoniumStatus().jobs[0].id); return 0;
|
|
});
|
|
await m.maintenanceTick();
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
expect(komodoRestart).not.toHaveBeenCalled();
|
|
expect(m.plutoniumStatus().jobs[0].state).toBe('cancelled');
|
|
});
|
|
it('holds maintenance while lifecycle work is queued', async () => {
|
|
const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
|
|
VALUES ('destroy-1', ?, 'destroy', 'queued', 'queued', ?, 'test')`).run(servers[0], new Date().toISOString());
|
|
await m.maintenanceTick();
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
expect(komodoRestart).not.toHaveBeenCalled();
|
|
expect(m.plutoniumStatus().jobs[0].detail).toContain('deployment or beta');
|
|
});
|
|
it('does not repeat an interrupted restart intent that has no matching rollout slot', async () => {
|
|
const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db().prepare(`UPDATE maintenance_jobs SET state = 'restarting', restarted_at = requested_at`).run();
|
|
await m.maintenanceTick();
|
|
expect(m.plutoniumStatus().jobs[0].state).toBe('failed');
|
|
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
|
expect(komodoRestart).not.toHaveBeenCalled();
|
|
});
|
|
it('validates API requests without changing the schedule or queue', async () => {
|
|
const { POST } = await import('../../src/routes/api/plutonium/+server.js');
|
|
const post = (body: unknown) => POST({ request: new Request('http://localhost/api/plutonium', {
|
|
method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body)
|
|
}) });
|
|
expect((await post({ action: 'schedule', enabled: true, time: '26:99' })).status).toBe(400);
|
|
expect((await post({ action: 'restart', servers: ['missing'] })).status).toBe(400);
|
|
expect((await post({ action: 'cancel', id: 100 })).status).toBe(409);
|
|
expect((await post({ action: 'schedule', enabled: true, time: '06:00' })).status).toBe(200);
|
|
const m = await api();
|
|
expect(m.plutoniumStatus().jobs).toEqual([]);
|
|
expect(m.plutoniumStatus().schedule.enabled).toBe(true);
|
|
});
|
|
it('runs a daily UTC schedule once and coalesces missed days', async () => {
|
|
expect(nextMaintenanceAt('06:00', new Date('2026-09-21T06:00:00Z'))).toBe('2026-09-22T06:00:00.000Z');
|
|
expect(() => nextMaintenanceAt('24:00')).toThrow();
|
|
const m = await api(); await m.checkPlutonium();
|
|
expect(m.plutoniumStatus().schedule.enabled).toBe(false);
|
|
m.saveMaintenanceSchedule(true, '06:00');
|
|
vi.setSystemTime(new Date('2026-09-24T07:00:00Z'));
|
|
await m.maintenanceTick(); await m.maintenanceTick();
|
|
expect(m.plutoniumStatus().jobs).toHaveLength(2);
|
|
expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-09-25T06:00:00.000Z');
|
|
m.saveMaintenanceSchedule(false, '06:00');
|
|
expect(m.plutoniumStatus().schedule.nextAt).toBeNull();
|
|
});
|
|
});
|