import { maintenanceTick } from '../helpers/provisioning.js'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; import { maintenanceTimeInZone, nextMaintenanceAt, parsePlutoniumManifest, parsePlutoniumRevision, PLUTONIUM_MANIFEST_URL } from '../../src/lib/plutonium.js'; vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoRestart: vi.fn(), komodoInspectHealthy: vi.fn() })); vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn(), rconPlutoniumVersion: vi.fn() })); let fixture: Fixture; const servers = ['t5_zombie_theater_1', 't6_zombie_town_1']; beforeEach(async () => { vi.resetModules(); vi.resetAllMocks(); vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(new Date('2026-09-21T05:59:00Z')); fixture = await newFixture(servers); vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test'); vi.stubEnv('KOMODO_API_KEY', 'test'); vi.stubEnv('KOMODO_API_SECRET', 'test'); vi.stubEnv('RCON_PASSWORD', 'test'); vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '1'); vi.stubGlobal('fetch', vi.fn().mockImplementation(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] }))); const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js'); await seedServerRegistry(); const k = await import('../../src/lib/server/integrations/komodo.js'); vi.mocked(k.komodoRestart).mockResolvedValue({ ok: true }); vi.mocked(k.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); const r = await import('../../src/lib/server/integrations/rcon.js'); vi.mocked(r.rconGetPlayerCount).mockResolvedValue(0); vi.mocked(r.rconPlutoniumVersion).mockResolvedValue({ revision: 5354, error: null }); vi.mocked(r.rconStatusResult).mockResolvedValue({ ok: true, endpoint: { host: 'test', port: 28960 }, rawReply: '', status: { players: 0, map: null, gametype: null, roster: [] } }); }); afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); vi.useRealTimers(); }); const api = () => import('../../src/lib/server/fleet/plutonium.js'); describe('Plutonium versions', () => { it('uses the official production manifest without requesting the challenged changelog page', async () => { const m = await api(); await m.checkPlutonium(); expect(fetch).toHaveBeenCalledTimes(1); expect(fetch).toHaveBeenCalledWith(PLUTONIUM_MANIFEST_URL, expect.any(Object)); expect(m.plutoniumStatus().release).toMatchObject({ revision: 5354, error: null }); expect(parsePlutoniumManifest({ product: 'plutonium-core-prod', revision: 5354 })).toBe(5354); for (const value of [null, {}, { ref: '1.0.221' }, { product: 'plutonium-core-dev', revision: 6000 }, { product: 'plutonium-core-prod', revision: '5354' }, { product: 'plutonium-core-prod', revision: -1 }]) { expect(() => parsePlutoniumManifest(value)).toThrow(); } }); it('parses explicit running revisions without interpreting engine numbers as releases', () => { expect(parsePlutoniumRevision('print\nPlutonium T6 r5354 win-x86')).toBe(5354); expect(parsePlutoniumRevision('version 1.0 build 20260921')).toBeNull(); }); it('preserves the cached release with a visible error and clears failed running-version observations', async () => { const m = await api(); await m.checkPlutonium(); vi.mocked(fetch).mockRejectedValue(new Error('Offline')); const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: null, error: 'Timeout' }); await m.checkPlutonium(); expect(m.plutoniumStatus().release).toMatchObject({ revision: 5354, error: 'Offline' }); expect(m.plutoniumStatus().servers.every(s => s.revision === null && s.error === 'Timeout')).toBe(true); }); it.each([ () => new Response('Blocked', { status: 403 }), () => new Response('Browser challenge'), () => Response.json({ product: 'plutonium-core-dev', revision: 9999 }) ])('keeps the cached revision when the manifest is unavailable or invalid', async response => { const m = await api(); await m.checkPlutonium(); vi.mocked(fetch).mockImplementation(async () => response()); await m.checkPlutonium(); expect(m.plutoniumStatus().release.revision).toBe(5354); expect(m.plutoniumStatus().release.error).toBeTruthy(); m.queueMaintenance([servers[0]]); expect(m.plutoniumStatus().jobs[0].target_revision).toBeNull(); }); }); describe('maintenance timezones', () => { it.each([ ['06:00', '2026-09-21T05:59:00Z', 'America/New_York', '2026-09-21T10:00:00.000Z'], ['06:00', '2026-12-21T05:59:00Z', 'America/New_York', '2026-12-21T11:00:00.000Z'], ['06:00', '2026-09-21T00:00:00Z', 'Asia/Kathmandu', '2026-09-21T00:15:00.000Z'], ['06:00', '2026-09-21T00:15:01Z', 'Asia/Kathmandu', '2026-09-22T00:15:00.000Z'], ['02:30', '2026-03-08T05:00:00Z', 'America/New_York', '2026-03-08T07:00:00.000Z'], ['01:30', '2026-11-01T04:00:00Z', 'America/New_York', '2026-11-01T05:30:00.000Z'], ['01:30', '2026-11-01T05:31:00Z', 'America/New_York', '2026-11-02T06:30:00.000Z'], ['02:15', '2026-10-03T13:00:00Z', 'Australia/Lord_Howe', '2026-10-03T15:30:00.000Z'] ])('schedules %s after %s in %s', (time, now, zone, expected) => { expect(nextMaintenanceAt(time, new Date(now), zone)).toBe(expected); }); it('preserves legacy UTC schedules and displays their next occurrence in the user timezone', async () => { const { db } = await import('../../src/lib/server/db/index.js'); db().prepare('INSERT INTO settings (key, value) VALUES (?, ?)').run('plutonium_schedule', JSON.stringify({ enabled: true, time: '06:00', nextAt: '2026-09-21T06:00:00.000Z' })); const m = await api(); const schedule = m.plutoniumStatus().schedule; expect(schedule.timeZone).toBe('UTC'); expect(maintenanceTimeInZone(schedule, 'America/New_York')).toBe('02:00'); expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-09-21T06:00:00.000Z'); }); it('persists the zone and advances in local calendar days across DST', async () => { vi.setSystemTime(new Date('2026-03-07T10:59:00Z')); const m = await api(); await m.checkPlutonium(); m.saveMaintenanceSchedule(true, '06:00', 'America/New_York'); const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb(); expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-03-07T11:00:00.000Z'); vi.setSystemTime(new Date('2026-03-07T11:00:00Z')); await maintenanceTick(); expect(m.plutoniumStatus().schedule).toMatchObject({ timeZone: 'America/New_York', nextAt: '2026-03-08T10:00:00.000Z' }); }); }); describe('persistent maintenance', () => { it('submits lifecycle work without executing it and rejects cancellation after Provisioning submits', async () => { const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]); const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); await m.maintenanceTick(); expect(komodoRestart).not.toHaveBeenCalled(); expect(rconGetPlayerCount).not.toHaveBeenCalled(); const job = m.plutoniumStatus().jobs[0]; expect(job.provisioning_id).toBeTruthy(); const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); await processProvisioning(); // The source has not mirrored submission yet; cancellation must still // consult the authoritative operation, not its stale queued state. expect(m.plutoniumStatus().jobs[0].state).toBe('queued'); expect(m.cancelMaintenance(job.id)).toBe(false); expect(komodoRestart).toHaveBeenCalledTimes(1); vi.setSystemTime(Date.now() + 15_000); await processProvisioning(); await m.maintenanceTick(); expect(m.plutoniumStatus().jobs[0].state).toBe('done'); }); it('waits for occupied and unreachable servers, deduplicates requests, and verifies after recovery', async () => { const m = await api(); await m.checkPlutonium(); expect(m.queueMaintenance([servers[0]])).toBe(1); expect(m.queueMaintenance([servers[0]])).toBe(0); const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(2).mockResolvedValueOnce(null); await maintenanceTick(); await maintenanceTick(); expect(komodoRestart).not.toHaveBeenCalled(); await Promise.all([maintenanceTick(), maintenanceTick()]); expect(komodoRestart).toHaveBeenCalledTimes(1); expect(m.plutoniumStatus().jobs[0].state).toBe('restarting'); await maintenanceTick(); expect(m.plutoniumStatus().jobs[0].state).toBe('restarting'); vi.setSystemTime(Date.now() + 15_000); await maintenanceTick(); expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'done', detail: 'Server recovered; running r5354.' }); }); it('shares rolling capacity across servers and survives closing the database', async () => { const m = await api(); await m.checkPlutonium(); m.queueMaintenance(servers); await maintenanceTick(); const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb(); expect(m.plutoniumStatus().jobs.map(j => j.state)).toEqual(['queued', 'restarting']); vi.setSystemTime(Date.now() + 15_000); await maintenanceTick(); expect(m.plutoniumStatus().jobs.map(j => j.state)).toEqual(['restarting', 'done']); }); it('reports a healthy but outdated server as failed without repeatedly restarting it', async () => { const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]); await maintenanceTick(); const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: 5300, error: null }); vi.setSystemTime(Date.now() + 15_000); await maintenanceTick(); await maintenanceTick(); expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'failed', target_revision: 5354 }); const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); expect(komodoRestart).toHaveBeenCalledTimes(1); }); it('does not retry an ambiguous restart timeout', async () => { const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]); const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); vi.mocked(komodoRestart).mockResolvedValue({ ok: false, kind: 'http-error', message: 'timeout' }); await maintenanceTick(); vi.setSystemTime(Date.now() + 15_000); await maintenanceTick(); expect(m.plutoniumStatus().jobs[0].state).toBe('failed'); expect(komodoRestart).toHaveBeenCalledTimes(1); }); it('honors cancellation arriving during the player check', async () => { const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]); const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); vi.mocked(rconGetPlayerCount).mockImplementationOnce(async () => { m.cancelMaintenance(m.plutoniumStatus().jobs[0].id); return 0; }); await maintenanceTick(); const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); expect(komodoRestart).not.toHaveBeenCalled(); expect(m.plutoniumStatus().jobs[0].state).toBe('cancelled'); }); it('holds maintenance while lifecycle work is queued', async () => { const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]); const { db } = await import('../../src/lib/server/db/index.js'); db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) VALUES ('destroy-1', ?, 'destroy', 'queued', 'queued', ?, 'test')`).run(servers[0], new Date().toISOString()); await maintenanceTick(); const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); expect(komodoRestart).not.toHaveBeenCalled(); expect(m.plutoniumStatus().jobs[0].detail).toContain('another provisioning operation'); }); it('does not repeat an interrupted restart intent that has no matching rollout slot', async () => { const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]); const { db } = await import('../../src/lib/server/db/index.js'); db().prepare(`UPDATE maintenance_jobs SET state = 'restarting', restarted_at = requested_at`).run(); await maintenanceTick(); expect(m.plutoniumStatus().jobs[0].state).toBe('restarting'); const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); expect(komodoRestart).not.toHaveBeenCalled(); vi.setSystemTime(Date.now() + 15_000); await maintenanceTick(); expect(m.plutoniumStatus().jobs[0].state).toBe('done'); expect(komodoRestart).not.toHaveBeenCalled(); }); it('validates API requests without changing the schedule or queue', async () => { const { POST } = await import('../../src/routes/api/plutonium/+server.js'); const post = (body: unknown) => POST({ request: new Request('http://localhost/api/plutonium', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) }) }); expect((await post({ action: 'schedule', enabled: true, time: '26:99' })).status).toBe(400); expect((await post({ action: 'restart', servers: ['missing'] })).status).toBe(400); expect((await post({ action: 'cancel', id: 100 })).status).toBe(409); expect((await post({ action: 'schedule', enabled: true, time: '06:00' })).status).toBe(400); expect((await post({ action: 'schedule', enabled: true, time: '06:00', timeZone: 'invalid' })).status).toBe(400); expect((await post({ action: 'schedule', enabled: true, time: '06:00', timeZone: 'America/New_York' })).status).toBe(200); const m = await api(); expect(m.plutoniumStatus().jobs).toEqual([]); expect(m.plutoniumStatus().schedule.enabled).toBe(true); expect(m.plutoniumStatus().schedule).toMatchObject({ timeZone: 'America/New_York', nextAt: '2026-09-21T10:00:00.000Z' }); }); it('runs a daily UTC schedule once and coalesces missed days', async () => { expect(nextMaintenanceAt('06:00', new Date('2026-09-21T06:00:00Z'))).toBe('2026-09-22T06:00:00.000Z'); expect(() => nextMaintenanceAt('24:00')).toThrow(); const m = await api(); await m.checkPlutonium(); expect(m.plutoniumStatus().schedule.enabled).toBe(false); m.saveMaintenanceSchedule(true, '06:00', 'UTC'); vi.setSystemTime(new Date('2026-09-24T07:00:00Z')); await maintenanceTick(); await maintenanceTick(); expect(m.plutoniumStatus().jobs).toHaveLength(2); expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-09-25T06:00:00.000Z'); m.saveMaintenanceSchedule(false, '06:00', 'UTC'); expect(m.plutoniumStatus().schedule.nextAt).toBeNull(); }); }); describe('Maintenance policies and runtime updates', () => { async function outdated() { const rcon = await import('../../src/lib/server/integrations/rcon.js'); vi.mocked(rcon.rconPlutoniumVersion).mockResolvedValue({ revision: 5300, error: null }); const m = await api(); await m.checkPlutonium(); return m; } it('preserves the legacy recovery preference without enabling updates', async () => { const { db, closeDb } = await import('../../src/lib/server/db/index.js'); db().prepare("INSERT INTO settings(key,value) VALUES ('watchdog_enabled','0')").run(); const p = await import('../../src/lib/server/maintenance/policies.js'); expect(p.maintenancePolicies()).toEqual({ paused: false, health: false, drift: false, updates: false }); p.setMaintenancePolicy('health', true); p.pauseMaintenance(true); closeDb(); expect(p.maintenancePolicies()).toEqual({ paused: true, health: true, drift: false, updates: false }); expect(p.maintenanceAllowed('health')).toBe(false); }); it('only automatically updates verified outdated servers and requires opt-in', async () => { const m = await outdated(); const { db } = await import('../../src/lib/server/db/index.js'); db().prepare('UPDATE plutonium_versions SET revision = NULL WHERE server = ?').run(servers[1]); await m.maintenanceTick(); expect(m.plutoniumStatus().jobs).toEqual([]); const p = await import('../../src/lib/server/maintenance/policies.js'); p.setMaintenancePolicy('updates', true); await m.maintenanceTick(); await m.maintenanceTick(); expect(m.plutoniumStatus().jobs).toEqual([expect.objectContaining({ server: servers[0], kind: 'update', origin: 'automatic', target_revision: 5354 })]); expect(m.plutoniumStatus().servers.find(s => s.server === servers[1])?.update_status).toBe('unknown'); }); it('holds queued automatic updates when their policy is disabled, but finishes submitted recovery while paused', async () => { const m = await outdated(); const p = await import('../../src/lib/server/maintenance/policies.js'); p.setMaintenancePolicy('updates', true); const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); m.queueUpdates([servers[0]], true); p.setMaintenancePolicy('updates', false); await processProvisioning(); expect(komodoRestart).not.toHaveBeenCalled(); p.setMaintenancePolicy('updates', true); p.pauseMaintenance(true); await processProvisioning(); expect(komodoRestart).not.toHaveBeenCalled(); p.pauseMaintenance(false); await processProvisioning(); expect(komodoRestart).toHaveBeenCalledTimes(1); p.pauseMaintenance(true); vi.setSystemTime(Date.now() + 15_000); await processProvisioning(); await m.maintenanceTick(); expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'failed', operation_state: 'done' }); expect(komodoRestart).toHaveBeenCalledTimes(1); }); it('latches failed update verification across restarts and retention, and supports an explicit retry', async () => { const m = await outdated(); const p = await import('../../src/lib/server/maintenance/policies.js'); p.setMaintenancePolicy('updates', true); const { db, closeDb } = await import('../../src/lib/server/db/index.js'); db().prepare("UPDATE servers SET desired_state = 'stopped' WHERE name = ?").run(servers[1]); await maintenanceTick(); vi.setSystemTime(Date.now() + 15_000); await maintenanceTick(); const failed = m.plutoniumStatus().jobs[0]; expect(failed).toMatchObject({ kind: 'update', state: 'failed', operation_state: 'done' }); closeDb(); vi.setSystemTime(Date.now() + 31 * 86_400_000); await maintenanceTick(); expect(m.plutoniumStatus().jobs).toHaveLength(1); expect(m.retryMaintenance(failed.id)).toBe(1); expect(m.plutoniumStatus().jobs).toHaveLength(2); const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: 5354, error: null }); await maintenanceTick(); vi.setSystemTime(Date.now() + 15_000); await maintenanceTick(); expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'done', origin: 'manual' }); expect(db().prepare("SELECT 1 FROM server_deployments WHERE action = 'rebuild'").get()).toBeUndefined(); }); it('does not restart intentionally stopped servers, including jobs queued before the intent changed', async () => { const m = await outdated(); m.queueUpdates([servers[0]]); const { db } = await import('../../src/lib/server/db/index.js'); db().prepare("UPDATE servers SET desired_state = 'stopped'").run(); await maintenanceTick(); const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); expect(komodoRestart).not.toHaveBeenCalled(); expect(m.plutoniumStatus().jobs[0].state).toBe('cancelled'); expect(m.queueUpdates()).toBe(0); expect(m.plutoniumStatus().servers.every(s => s.update_status === 'stopped')).toBe(true); }); it('separates scheduled restart success from revision verification and coalesces paused runs', async () => { const m = await outdated(); m.saveMaintenanceSchedule(true, '06:00', 'UTC'); const p = await import('../../src/lib/server/maintenance/policies.js'); p.pauseMaintenance(true); vi.setSystemTime(new Date('2026-09-24T07:00:00Z')); await m.maintenanceTick(); expect(m.plutoniumStatus().jobs).toHaveLength(0); p.pauseMaintenance(false); await maintenanceTick(); expect(m.plutoniumStatus().jobs).toHaveLength(2); expect(m.plutoniumStatus().jobs.every(j => j.kind === 'schedule' && j.target_revision === null)).toBe(true); const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: null, error: 'Unknown reply' }); vi.setSystemTime(Date.now() + 15_000); await maintenanceTick(); expect(m.plutoniumStatus().jobs.find(j => j.server === servers[0])?.state).toBe('done'); expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-09-25T06:00:00.000Z'); }); it('does not treat stale, pre-restart, or unhealthy observations as current runtime evidence', async () => { const m = await outdated(); const { db } = await import('../../src/lib/server/db/index.js'); db().prepare('INSERT INTO restart_rollout(server,requested_at,healthy_at) VALUES (?,?,?)').run(servers[0], new Date().toISOString(), new Date().toISOString()); expect(m.plutoniumStatus().servers[0].update_status).toBe('unknown'); vi.setSystemTime(Date.now() + 3_600_000); expect(() => m.queueUpdates()).toThrow('published revision'); const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js'); vi.mocked(rconPlutoniumVersion).mockClear(); vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'starting', reason: 'Starting' }); await m.checkPlutonium(); expect(rconPlutoniumVersion).not.toHaveBeenCalled(); expect(m.plutoniumStatus().servers.every(s => s.update_status === 'unknown')).toBe(true); }); it('adopts existing maintenance jobs without changing their IDs, schedules, or submitted operations', async () => { const m = await outdated(); m.saveMaintenanceSchedule(true, '06:00', 'America/New_York'); m.queueMaintenance([servers[0]]); await maintenanceTick(); const before = m.plutoniumStatus(); const { db, closeDb } = await import('../../src/lib/server/db/index.js'); db().exec('ALTER TABLE maintenance_jobs DROP COLUMN kind; ALTER TABLE maintenance_jobs DROP COLUMN origin;'); closeDb(); const after = m.plutoniumStatus(); expect(after.schedule).toEqual(before.schedule); expect(after.jobs[0]).toMatchObject({ id: before.jobs[0].id, provisioning_id: before.jobs[0].provisioning_id, kind: 'legacy', origin: 'manual' }); vi.setSystemTime(Date.now() + 15_000); await maintenanceTick(); const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); expect(komodoRestart).toHaveBeenCalledTimes(1); }); });