import { maintenanceTick } from '../helpers/provisioning.js';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import { maintenanceTimeInZone, nextMaintenanceAt, parsePlutoniumManifest, parsePlutoniumRevision, PLUTONIUM_MANIFEST_URL } from '../../src/lib/plutonium.js';

vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoRestart: vi.fn(), komodoInspectHealthy: vi.fn() }));
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn(), rconPlutoniumVersion: vi.fn() }));

let fixture: Fixture;
const servers = ['t5_zombie_theater_1', 't6_zombie_town_1'];
beforeEach(async () => {
    vi.resetModules(); vi.resetAllMocks();
    vi.useFakeTimers({ toFake: ['Date'] });
    vi.setSystemTime(new Date('2026-09-21T05:59:00Z'));
    fixture = await newFixture(servers);
    vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test');
    vi.stubEnv('KOMODO_API_KEY', 'test'); vi.stubEnv('KOMODO_API_SECRET', 'test');
    vi.stubEnv('RCON_PASSWORD', 'test'); vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '1');
    vi.stubGlobal('fetch', vi.fn().mockImplementation(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] })));
    const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
    await seedServerRegistry();
    const k = await import('../../src/lib/server/integrations/komodo.js');
    vi.mocked(k.komodoRestart).mockResolvedValue({ ok: true });
    vi.mocked(k.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
    const r = await import('../../src/lib/server/integrations/rcon.js');
    vi.mocked(r.rconGetPlayerCount).mockResolvedValue(0);
    vi.mocked(r.rconPlutoniumVersion).mockResolvedValue({ revision: 5354, error: null });
    vi.mocked(r.rconStatusResult).mockResolvedValue({ ok: true, endpoint: { host: 'test', port: 28960 }, rawReply: '', status: { players: 0, map: null, gametype: null, roster: [] } });
});
afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); vi.useRealTimers(); });
const api = () => import('../../src/lib/server/fleet/plutonium.js');

describe('Plutonium versions', () => {
    it.each([
        ['create', 'queued'], ['rebuild', 'waiting'], ['start', 'running'],
        ['restart', 'awaiting-health'], ['stop', 'waiting'], ['destroy', 'running']
    ])('suppresses fleet probes during %s (%s), preserves evidence, and resumes afterwards', async (action, state) => {
        const m = await api();
        await m.checkPlutonium();
        const { db, closeDb } = await import('../../src/lib/server/db/index.js');
        const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
        const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
        const saved = db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]);
        db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
            VALUES ('busy', ?, ?, ?, 'Provisioning', ?, 'test')`).run(servers[0], action, state, new Date().toISOString());
        closeDb();
        vi.mocked(komodoInspectHealthy).mockClear();
        vi.mocked(rconPlutoniumVersion).mockClear();
        vi.setSystemTime(Date.now() + 1_000);
        await m.checkPlutonium();
        expect(komodoInspectHealthy).toHaveBeenCalledTimes(1);
        expect(komodoInspectHealthy).toHaveBeenCalledWith(servers[1]);
        expect(rconPlutoniumVersion).toHaveBeenCalledTimes(1);
        expect(rconPlutoniumVersion).toHaveBeenCalledWith(servers[1]);
        expect(db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])).toEqual(saved);
        db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'busy'").run();
        await m.checkPlutonium();
        expect(rconPlutoniumVersion).toHaveBeenCalledWith(servers[0]);
        expect(db().prepare('SELECT checked_at FROM plutonium_versions WHERE server = ?').get(servers[0]))
            .toEqual({ checked_at: new Date().toISOString() });
    });
    it('suppresses fleet probes while a failed restart retains recovery capacity', async () => {
        const m = await api();
        await m.checkPlutonium();
        const { db } = await import('../../src/lib/server/db/index.js');
        const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
        const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
        const saved = db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]);
        db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
            VALUES ('failed-restart', ?, 'restart', 'failed', 'Timed out', ?, 'test')`).run(servers[0], new Date().toISOString());
        db().prepare('INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)')
            .run(servers[0], new Date().toISOString(), 'failed-restart');
        vi.mocked(komodoInspectHealthy).mockClear();
        vi.mocked(rconPlutoniumVersion).mockClear();
        await m.checkPlutonium();
        expect(komodoInspectHealthy).not.toHaveBeenCalledWith(servers[0]);
        expect(rconPlutoniumVersion).not.toHaveBeenCalledWith(servers[0]);
        expect(db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])).toEqual(saved);
        db().prepare('UPDATE restart_rollout SET healthy_at = ? WHERE server = ?').run(new Date().toISOString(), servers[0]);
        vi.setSystemTime(Date.now() + 1_000);
        await m.checkPlutonium();
        expect(rconPlutoniumVersion).toHaveBeenCalledWith(servers[0]);
    });
    it.each(['health', 'version'] as const)('discards maintenance observations when provisioning is queued during the %s probe', async phase => {
        const m = await api();
        await m.checkPlutonium();
        const { db } = await import('../../src/lib/server/db/index.js');
        const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
        const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
        const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
        const saved = db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]);
        const queue = () => db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
            VALUES ('new-rebuild', ?, 'rebuild', 'waiting', 'Waiting', ?, 'test')`).run(servers[0], new Date().toISOString());
        vi.mocked(rconPlutoniumVersion).mockClear();
        if (phase === 'health') vi.mocked(komodoInspectHealthy).mockImplementationOnce(async () => {
            queue(); return { ok: true, state: 'running' };
        });
        else vi.mocked(rconPlutoniumVersion).mockImplementationOnce(async () => {
            queue(); return { revision: null, error: 'Restarting' };
        });
        vi.setSystemTime(Date.now() + 1_000);
        expect(await probeVersion(servers[0], { maintenance: true })).toBeNull();
        if (phase === 'health') expect(rconPlutoniumVersion).not.toHaveBeenCalled();
        expect(db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])).toEqual(saved);
    });
    it('coalesces published-release checks without probing the fleet', async () => {
        const { checkRelease, release } = await import('../../src/lib/server/integrations/plutonium.js');
        const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
        const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
        const first = checkRelease();
        const second = checkRelease();
        expect(second).toBe(first);
        await Promise.all([first, second]);
        expect(fetch).toHaveBeenCalledTimes(1);
        expect(release()).toMatchObject({ revision: 5354, error: null });
        expect(komodoInspectHealthy).not.toHaveBeenCalled();
        expect(rconPlutoniumVersion).not.toHaveBeenCalled();
        await checkRelease();
        expect(fetch).toHaveBeenCalledTimes(2);
    });
    it('refreshes hourly fleet versions even when provisioning recently refreshed the release manifest', async () => {
        const m = await api();
        const { checkRelease, HOUR } = await import('../../src/lib/server/integrations/plutonium.js');
        const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
        await m.checkPlutonium();
        vi.setSystemTime(Date.now() + HOUR);
        await checkRelease();
        vi.mocked(rconPlutoniumVersion).mockClear();
        await m.maintenanceTick();
        expect(rconPlutoniumVersion).toHaveBeenCalledTimes(servers.length);
        for (const server of servers) expect(rconPlutoniumVersion).toHaveBeenCalledWith(server);
        await m.maintenanceTick();
        expect(rconPlutoniumVersion).toHaveBeenCalledTimes(servers.length);
    });
    it('uses the official production manifest without requesting the challenged changelog page', async () => {
        const m = await api(); await m.checkPlutonium();
        expect(fetch).toHaveBeenCalledTimes(1);
        expect(fetch).toHaveBeenCalledWith(PLUTONIUM_MANIFEST_URL, expect.any(Object));
        expect(m.plutoniumStatus().release).toMatchObject({ revision: 5354, error: null });
        expect(parsePlutoniumManifest({ product: 'plutonium-core-prod', revision: 5354 })).toBe(5354);
        for (const value of [null, {}, { ref: '1.0.221' }, { product: 'plutonium-core-dev', revision: 6000 },
            { product: 'plutonium-core-prod', revision: '5354' }, { product: 'plutonium-core-prod', revision: -1 }]) {
            expect(() => parsePlutoniumManifest(value)).toThrow();
        }
    });
    it('parses explicit running revisions without interpreting engine numbers as releases', () => {
        expect(parsePlutoniumRevision('print\nPlutonium T6 r5354 win-x86')).toBe(5354);
        expect(parsePlutoniumRevision('version 1.0 build 20260921')).toBeNull();
    });
    it('preserves the cached release with a visible error and clears failed running-version observations', async () => {
        const m = await api(); await m.checkPlutonium();
        vi.mocked(fetch).mockRejectedValue(new Error('Offline'));
        const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
        vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: null, error: 'Timeout' });
        await m.checkPlutonium();
        expect(m.plutoniumStatus().release).toMatchObject({ revision: 5354, error: 'Offline' });
        expect(m.plutoniumStatus().servers.every(s => s.revision === null && s.error === 'Timeout')).toBe(true);
    });
    it.each([
        () => new Response('Blocked', { status: 403 }),
        () => new Response('<html>Browser challenge</html>'),
        () => Response.json({ product: 'plutonium-core-dev', revision: 9999 })
    ])('keeps the cached revision when the manifest is unavailable or invalid', async response => {
        const m = await api(); await m.checkPlutonium();
        vi.mocked(fetch).mockImplementation(async () => response());
        await m.checkPlutonium();
        expect(m.plutoniumStatus().release.revision).toBe(5354);
        expect(m.plutoniumStatus().release.error).toBeTruthy();
        m.queueMaintenance([servers[0]]);
        expect(m.plutoniumStatus().jobs[0].target_revision).toBeNull();
    });
    it('rejects a version observed at the startup submission timestamp, then accepts a later observation', async () => {
        const { db } = await import('../../src/lib/server/db/index.js');
        const { probeVersion, runningVersions, checkRelease } = await import('../../src/lib/server/integrations/plutonium.js');
        const submittedAt = new Date().toISOString();
        db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by)
            VALUES ('startup', ?, 'create', 'awaiting-health', 'Waiting for health', ?, ?, 'test')`).run(servers[0], submittedAt, submittedAt);
        await checkRelease();
        expect(await probeVersion(servers[0])).toBeNull();
        expect(runningVersions()[0]).toMatchObject({ revision: null, checked_at: submittedAt, update_status: 'unknown', error: expect.stringContaining('after startup') });
        vi.setSystemTime(Date.now() + 1);
        expect(await probeVersion(servers[0])).toBe(5354);
        expect(runningVersions()[0]).toMatchObject({ revision: 5354, update_status: 'current', error: null });
    });
    it('discards a running revision when a new lifecycle submission arrives during its probe', async () => {
        const { db } = await import('../../src/lib/server/db/index.js');
        const { probeVersion, runningVersions } = await import('../../src/lib/server/integrations/plutonium.js');
        const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
        vi.mocked(rconPlutoniumVersion).mockImplementationOnce(async () => {
            const now = new Date().toISOString();
            db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by)
                VALUES ('restart', ?, 'restart', 'awaiting-health', 'Waiting for health', ?, ?, 'test')`).run(servers[0], now, now);
            return { revision: 5354, error: null };
        });
        expect(await probeVersion(servers[0])).toBeNull();
        expect(runningVersions()[0]).toMatchObject({ revision: null, update_status: 'unknown' });
        expect(db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])).toBeUndefined();
    });
    it.each(['startup', 'replacement'] as const)('preserves a newer successful revision when an old probe finishes after %s', async change => {
        const { db } = await import('../../src/lib/server/db/index.js');
        const { probeVersion, runningVersions, checkRelease } = await import('../../src/lib/server/integrations/plutonium.js');
        const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
        await checkRelease();
        db().prepare('UPDATE servers SET created_at = ? WHERE name = ?').run('2026-09-21T05:58:00.000Z', servers[0]);
        let finishOld!: (value: { revision: number; error: null }) => void;
        let started!: () => void;
        const probing = new Promise<void>(resolve => { started = resolve; });
        vi.mocked(rconPlutoniumVersion).mockImplementationOnce(() => {
            started();
            return new Promise(resolve => { finishOld = resolve; });
        });
        const oldProbe = probeVersion(servers[0]);
        await probing;
        const now = new Date().toISOString();
        if (change === 'startup') {
            db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by)
                VALUES ('new-startup', ?, 'start', 'awaiting-health', 'Waiting for health', ?, ?, 'test')`).run(servers[0], now, now);
        } else {
            // Both registrations have no lifecycle submission, so only their
            // registry identity distinguishes the old and replacement instances.
            db().prepare('DELETE FROM servers WHERE name = ?').run(servers[0]);
            db().prepare('INSERT INTO servers (name, komodo_name, created_at) VALUES (?, ?, ?)').run(servers[0], servers[0], now);
        }
        vi.setSystemTime(Date.now() + 1);
        expect(await probeVersion(servers[0])).toBe(5354);
        const saved = db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]);
        vi.setSystemTime(Date.now() + 1);
        finishOld({ revision: 5300, error: null });
        expect(await oldProbe).toBeNull();
        expect(db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])).toEqual(saved);
        expect(runningVersions()[0]).toMatchObject({ revision: 5354, error: null, update_status: 'current' });
    });
    it('discards a running revision when the server is stopped during its probe', async () => {
        const { db } = await import('../../src/lib/server/db/index.js');
        const { probeVersion, runningVersions } = await import('../../src/lib/server/integrations/plutonium.js');
        const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
        vi.mocked(rconPlutoniumVersion).mockImplementationOnce(async () => {
            db().prepare("UPDATE servers SET desired_state = 'stopped' WHERE name = ?").run(servers[0]);
            return { revision: 5354, error: null };
        });
        expect(await probeVersion(servers[0])).toBeNull();
        expect(runningVersions()[0]).toMatchObject({ revision: null, error: 'Server is intentionally stopped.', update_status: 'stopped' });
    });
});

describe('maintenance timezones', () => {
    it.each([
        ['06:00', '2026-09-21T05:59:00Z', 'America/New_York', '2026-09-21T10:00:00.000Z'],
        ['06:00', '2026-12-21T05:59:00Z', 'America/New_York', '2026-12-21T11:00:00.000Z'],
        ['06:00', '2026-09-21T00:00:00Z', 'Asia/Kathmandu', '2026-09-21T00:15:00.000Z'],
        ['06:00', '2026-09-21T00:15:01Z', 'Asia/Kathmandu', '2026-09-22T00:15:00.000Z'],
        ['02:30', '2026-03-08T05:00:00Z', 'America/New_York', '2026-03-08T07:00:00.000Z'],
        ['01:30', '2026-11-01T04:00:00Z', 'America/New_York', '2026-11-01T05:30:00.000Z'],
        ['01:30', '2026-11-01T05:31:00Z', 'America/New_York', '2026-11-02T06:30:00.000Z'],
        ['02:15', '2026-10-03T13:00:00Z', 'Australia/Lord_Howe', '2026-10-03T15:30:00.000Z']
    ])('schedules %s after %s in %s', (time, now, zone, expected) => {
        expect(nextMaintenanceAt(time, new Date(now), zone)).toBe(expected);
    });
    it('preserves legacy UTC schedules and displays their next occurrence in the user timezone', async () => {
        const { db } = await import('../../src/lib/server/db/index.js');
        db().prepare('INSERT INTO settings (key, value) VALUES (?, ?)').run('plutonium_schedule', JSON.stringify({
            enabled: true, time: '06:00', nextAt: '2026-09-21T06:00:00.000Z'
        }));
        const m = await api();
        const schedule = m.plutoniumStatus().schedule;
        expect(schedule.timeZone).toBe('UTC');
        expect(maintenanceTimeInZone(schedule, 'America/New_York')).toBe('02:00');
        expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-09-21T06:00:00.000Z');
    });
    it('persists the zone and advances in local calendar days across DST', async () => {
        vi.setSystemTime(new Date('2026-03-07T10:59:00Z'));
        const m = await api(); await m.checkPlutonium();
        m.saveMaintenanceSchedule(true, '06:00', 'America/New_York');
        const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb();
        expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-03-07T11:00:00.000Z');
        vi.setSystemTime(new Date('2026-03-07T11:00:00Z'));
        await maintenanceTick();
        expect(m.plutoniumStatus().schedule).toMatchObject({ timeZone: 'America/New_York', nextAt: '2026-03-08T10:00:00.000Z' });
    });
});

describe('persistent maintenance', () => {
    it('submits lifecycle work without executing it and rejects cancellation after Provisioning submits', async () => {
        const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
        await m.maintenanceTick();
        expect(komodoRestart).not.toHaveBeenCalled(); expect(rconGetPlayerCount).not.toHaveBeenCalled();
        const job = m.plutoniumStatus().jobs[0];
        expect(job.provisioning_id).toBeTruthy();
        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
        await processProvisioning();
        // The source has not mirrored submission yet; cancellation must still
        // consult the authoritative operation, not its stale queued state.
        expect(m.plutoniumStatus().jobs[0].state).toBe('queued');
        expect(m.cancelMaintenance(job.id)).toBe(false);
        expect(komodoRestart).toHaveBeenCalledTimes(1);
        vi.setSystemTime(Date.now() + 15_000); await processProvisioning();
        await m.maintenanceTick();
        expect(m.plutoniumStatus().jobs[0].state).toBe('done');
    });
    it('waits for occupied and unreachable servers, deduplicates requests, and verifies after recovery', async () => {
        const m = await api(); await m.checkPlutonium();
        expect(m.queueMaintenance([servers[0]])).toBe(1);
        expect(m.queueMaintenance([servers[0]])).toBe(0);
        const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(2).mockResolvedValueOnce(null);
        await maintenanceTick(); await maintenanceTick();
        expect(komodoRestart).not.toHaveBeenCalled();
        await Promise.all([maintenanceTick(), maintenanceTick()]);
        expect(komodoRestart).toHaveBeenCalledTimes(1);
        expect(m.plutoniumStatus().jobs[0].state).toBe('restarting');
        await maintenanceTick();
        expect(m.plutoniumStatus().jobs[0].state).toBe('restarting');
        vi.setSystemTime(Date.now() + 15_000);
        await maintenanceTick();
        expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'done', detail: 'Server recovered; running r5354.' });
    });
    it('shares rolling capacity across servers and survives closing the database', async () => {
        const m = await api(); await m.checkPlutonium(); m.queueMaintenance(servers);
        await maintenanceTick();
        const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb();
        expect(m.plutoniumStatus().jobs.map(j => j.state)).toEqual(['queued', 'restarting']);
        vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
        expect(m.plutoniumStatus().jobs.map(j => j.state)).toEqual(['restarting', 'done']);
    });
    it('reports a healthy but outdated server as failed without repeatedly restarting it', async () => {
        const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
        await maintenanceTick();
        const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
        vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: 5300, error: null });
        vi.setSystemTime(Date.now() + 15_000); await maintenanceTick(); await maintenanceTick();
        expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'failed', target_revision: 5354 });
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        expect(komodoRestart).toHaveBeenCalledTimes(1);
    });
    it('does not retry an ambiguous restart timeout', async () => {
        const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        vi.mocked(komodoRestart).mockResolvedValue({ ok: false, kind: 'http-error', message: 'timeout' });
        await maintenanceTick(); vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
        expect(m.plutoniumStatus().jobs[0].state).toBe('failed');
        expect(komodoRestart).toHaveBeenCalledTimes(1);
    });
    it('honors cancellation arriving during the player check', async () => {
        const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
        const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
        vi.mocked(rconGetPlayerCount).mockImplementationOnce(async () => {
            m.cancelMaintenance(m.plutoniumStatus().jobs[0].id); return 0;
        });
        await maintenanceTick();
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        expect(komodoRestart).not.toHaveBeenCalled();
        expect(m.plutoniumStatus().jobs[0].state).toBe('cancelled');
    });
    it('holds maintenance while lifecycle work is queued', async () => {
        const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
        const { db } = await import('../../src/lib/server/db/index.js');
        db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
            VALUES ('destroy-1', ?, 'destroy', 'queued', 'queued', ?, 'test')`).run(servers[0], new Date().toISOString());
        await maintenanceTick();
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        expect(komodoRestart).not.toHaveBeenCalled();
        expect(m.plutoniumStatus().jobs[0].detail).toContain('another provisioning operation');
    });
    it('waits to attach legacy maintenance while another operation owns the server', async () => {
        const m = await api(); await m.checkPlutonium();
        const { db } = await import('../../src/lib/server/db/index.js');
        db().prepare(`INSERT INTO maintenance_jobs (server, requested_at) VALUES (?, ?)`).run(servers[0], new Date().toISOString());
        db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
            VALUES ('existing-start', ?, 'start', 'waiting', 'Waiting to start', ?, 'test')`).run(servers[0], new Date().toISOString());
        await m.maintenanceTick();
        expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'queued', provisioning_id: null, detail: expect.stringContaining('another provisioning operation') });
        expect(db().prepare('SELECT COUNT(*) AS count FROM server_deployments').get()).toEqual({ count: 1 });
        db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'existing-start'").run();
        await m.maintenanceTick();
        expect(m.plutoniumStatus().jobs[0].provisioning_id).toBeTruthy();
        expect(db().prepare('SELECT COUNT(*) AS count FROM server_deployments').get()).toEqual({ count: 2 });
    });
    it.each([false, true])('adopts a submitted legacy restart with retained recovery capacity (linked operation: %s)', async linked => {
        const m = await api(); await m.checkPlutonium();
        const { db } = await import('../../src/lib/server/db/index.js');
        const submittedAt = new Date().toISOString();
        db().prepare(`INSERT INTO maintenance_jobs (server, requested_at, restarted_at, state)
            VALUES (?, ?, ?, 'restarting')`).run(servers[0], submittedAt, submittedAt);
        if (linked) {
            db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by)
                VALUES ('legacy-restart', ?, 'restart', 'awaiting-health', 'Waiting for health', ?, ?, 'test')`).run(servers[0], submittedAt, submittedAt);
        }
        db().prepare('INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)')
            .run(servers[0], submittedAt, linked ? 'legacy-restart' : null);
        await m.maintenanceTick();
        const job = m.plutoniumStatus().jobs[0];
        expect(job).toMatchObject({ state: 'restarting', operation_state: 'awaiting-health' });
        expect(job.provisioning_id).toBeTruthy();
        if (linked) expect(job.provisioning_id).toBe('legacy-restart');
        expect(db().prepare('SELECT operation_id FROM restart_rollout WHERE server = ?').get(servers[0])).toEqual({ operation_id: job.provisioning_id });
        vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
        expect(m.plutoniumStatus().jobs[0].state).toBe('done');
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        expect(komodoRestart).not.toHaveBeenCalled();
    });
    it('defers completed maintenance verification while a later operation owns the server, then resumes', async () => {
        const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
        const { db } = await import('../../src/lib/server/db/index.js');
        const job = m.plutoniumStatus().jobs[0];
        db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = ?").run(job.provisioning_id);
        db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
            VALUES ('later-restart', ?, 'restart', 'waiting', 'Waiting to restart', ?, 'test')`).run(servers[0], new Date().toISOString());
        const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
        const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
        vi.mocked(komodoInspectHealthy).mockClear(); vi.mocked(rconPlutoniumVersion).mockClear();
        await m.maintenanceTick();
        expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'queued', detail: expect.stringContaining('before verifying') });
        expect(komodoInspectHealthy).not.toHaveBeenCalled(); expect(rconPlutoniumVersion).not.toHaveBeenCalled();
        db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'later-restart'").run();
        await m.maintenanceTick();
        expect(m.plutoniumStatus().jobs[0].state).toBe('done');
        expect(rconPlutoniumVersion).toHaveBeenCalledWith(servers[0]);
    });
    it('does not fail maintenance verification when another operation starts during its version probe', async () => {
        const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
        const { db } = await import('../../src/lib/server/db/index.js');
        db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = ?").run(m.plutoniumStatus().jobs[0].provisioning_id);
        const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
        vi.mocked(rconPlutoniumVersion).mockImplementationOnce(async () => {
            db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
                VALUES ('later-rebuild', ?, 'rebuild', 'running', 'Rebuilding', ?, 'test')`).run(servers[0], new Date().toISOString());
            return { revision: null, error: 'Restarting' };
        });
        const before = db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0]);
        await m.maintenanceTick();
        expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'queued', detail: expect.stringContaining('before verifying') });
        expect(db().prepare('SELECT * FROM plutonium_versions WHERE server = ?').get(servers[0])).toEqual(before);
        db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'later-rebuild'").run();
        await m.maintenanceTick();
        expect(m.plutoniumStatus().jobs[0].state).toBe('done');
    });
    it('mirrors its own failed operation even while another operation owns the server', async () => {
        const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
        const { db } = await import('../../src/lib/server/db/index.js');
        db().prepare("UPDATE server_deployments SET state = 'failed', error = 'Restart failed' WHERE id = ?").run(m.plutoniumStatus().jobs[0].provisioning_id);
        db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
            VALUES ('later-start', ?, 'start', 'waiting', 'Waiting to start', ?, 'test')`).run(servers[0], new Date().toISOString());
        await m.maintenanceTick();
        expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'failed', detail: 'Restart failed' });
    });
    it('does not repeat an interrupted restart intent that has no matching rollout slot', async () => {
        const m = await api(); await m.checkPlutonium(); m.queueMaintenance([servers[0]]);
        const { db } = await import('../../src/lib/server/db/index.js');
        db().prepare(`UPDATE maintenance_jobs SET state = 'restarting', restarted_at = requested_at`).run();
        await maintenanceTick();
        expect(m.plutoniumStatus().jobs[0].state).toBe('restarting');
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        expect(komodoRestart).not.toHaveBeenCalled();
        vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
        expect(m.plutoniumStatus().jobs[0].state).toBe('done');
        expect(komodoRestart).not.toHaveBeenCalled();
    });
    it('validates API requests without changing the schedule or queue', async () => {
        const { POST } = await import('../../src/routes/api/plutonium/+server.js');
        const post = (body: unknown) => POST({ request: new Request('http://localhost/api/plutonium', {
            method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body)
        }) });
        expect((await post({ action: 'schedule', enabled: true, time: '26:99' })).status).toBe(400);
        expect((await post({ action: 'restart', servers: ['missing'] })).status).toBe(400);
        expect((await post({ action: 'cancel', id: 100 })).status).toBe(409);
        expect((await post({ action: 'schedule', enabled: true, time: '06:00' })).status).toBe(400);
        expect((await post({ action: 'schedule', enabled: true, time: '06:00', timeZone: 'invalid' })).status).toBe(400);
        expect((await post({ action: 'schedule', enabled: true, time: '06:00', timeZone: 'America/New_York' })).status).toBe(200);
        const m = await api();
        expect(m.plutoniumStatus().jobs).toEqual([]);
        expect(m.plutoniumStatus().schedule.enabled).toBe(true);
        expect(m.plutoniumStatus().schedule).toMatchObject({ timeZone: 'America/New_York', nextAt: '2026-09-21T10:00:00.000Z' });
    });
    it('runs a daily UTC schedule once and coalesces missed days', async () => {
        expect(nextMaintenanceAt('06:00', new Date('2026-09-21T06:00:00Z'))).toBe('2026-09-22T06:00:00.000Z');
        expect(() => nextMaintenanceAt('24:00')).toThrow();
        const m = await api(); await m.checkPlutonium();
        expect(m.plutoniumStatus().schedule.enabled).toBe(false);
        m.saveMaintenanceSchedule(true, '06:00', 'UTC');
        vi.setSystemTime(new Date('2026-09-24T07:00:00Z'));
        await maintenanceTick(); await maintenanceTick();
        expect(m.plutoniumStatus().jobs).toHaveLength(2);
        expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-09-25T06:00:00.000Z');
        m.saveMaintenanceSchedule(false, '06:00', 'UTC');
        expect(m.plutoniumStatus().schedule.nextAt).toBeNull();
    });
});

describe('Maintenance policies and runtime updates', () => {
    async function outdated() {
        const rcon = await import('../../src/lib/server/integrations/rcon.js');
        vi.mocked(rcon.rconPlutoniumVersion).mockResolvedValue({ revision: 5300, error: null });
        const m = await api(); await m.checkPlutonium(); return m;
    }
    it('preserves the legacy recovery preference without enabling updates', async () => {
        const { db, closeDb } = await import('../../src/lib/server/db/index.js');
        db().prepare("INSERT INTO settings(key,value) VALUES ('watchdog_enabled','0')").run();
        const p = await import('../../src/lib/server/maintenance/policies.js');
        expect(p.maintenancePolicies()).toEqual({ paused: false, health: false, drift: false, updates: false });
        p.setMaintenancePolicy('health', true); p.pauseMaintenance(true); closeDb();
        expect(p.maintenancePolicies()).toEqual({ paused: true, health: true, drift: false, updates: false });
        expect(p.maintenanceAllowed('health')).toBe(false);
    });
    it('only automatically updates verified outdated servers and requires opt-in', async () => {
        const m = await outdated();
        const { db } = await import('../../src/lib/server/db/index.js');
        db().prepare('UPDATE plutonium_versions SET revision = NULL WHERE server = ?').run(servers[1]);
        await m.maintenanceTick();
        expect(m.plutoniumStatus().jobs).toEqual([]);
        const p = await import('../../src/lib/server/maintenance/policies.js'); p.setMaintenancePolicy('updates', true);
        await m.maintenanceTick(); await m.maintenanceTick();
        expect(m.plutoniumStatus().jobs).toEqual([expect.objectContaining({ server: servers[0], kind: 'update', origin: 'automatic', target_revision: 5354 })]);
        expect(m.plutoniumStatus().servers.find(s => s.server === servers[1])?.update_status).toBe('unknown');
    });
    it.each(['queued', 'waiting', 'running', 'awaiting-health'])('skips automatic updates during %s provisioning and resumes after completion', async state => {
        const m = await outdated();
        const { db } = await import('../../src/lib/server/db/index.js');
        db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
            VALUES ('existing-restart', ?, 'restart', ?, 'Restarting', ?, 'test')`).run(servers[0], state, new Date().toISOString());
        const p = await import('../../src/lib/server/maintenance/policies.js'); p.setMaintenancePolicy('updates', true);
        await m.maintenanceTick();
        expect(m.plutoniumStatus().jobs).toEqual([expect.objectContaining({ server: servers[1], kind: 'update', origin: 'automatic' })]);
        expect(db().prepare('SELECT COUNT(*) AS count FROM server_deployments WHERE server = ?').get(servers[0])).toEqual({ count: 1 });
        db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'existing-restart'").run();
        await m.maintenanceTick();
        expect(m.plutoniumStatus().jobs).toHaveLength(2);
        expect(m.plutoniumStatus().jobs[0]).toMatchObject({ server: servers[0], kind: 'update', origin: 'automatic' });
    });
    it('skips a busy server for the scheduled occurrence while other servers proceed', async () => {
        const m = await outdated(); m.saveMaintenanceSchedule(true, '06:00', 'UTC');
        const { db } = await import('../../src/lib/server/db/index.js');
        db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
            VALUES ('existing-start', ?, 'start', 'waiting', 'Waiting to start', ?, 'test')`).run(servers[0], new Date().toISOString());
        vi.setSystemTime(new Date('2026-09-21T06:00:00Z'));
        await m.maintenanceTick();
        expect(m.plutoniumStatus().jobs).toEqual([expect.objectContaining({ server: servers[1], kind: 'schedule' })]);
        expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-09-22T06:00:00.000Z');
        db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'existing-start'").run();
        await m.maintenanceTick();
        expect(m.plutoniumStatus().jobs).toHaveLength(1);
        vi.setSystemTime(new Date('2026-09-22T06:00:00Z'));
        await m.maintenanceTick();
        expect(m.plutoniumStatus().jobs).toHaveLength(2);
        expect(m.plutoniumStatus().jobs[0]).toMatchObject({ server: servers[0], kind: 'schedule' });
    });
    it('skips automatic and scheduled requests while a failed operation retains recovery capacity', async () => {
        const m = await outdated(); m.saveMaintenanceSchedule(true, '06:00', 'UTC');
        const { db } = await import('../../src/lib/server/db/index.js');
        db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
            VALUES ('failed-restart', ?, 'restart', 'failed', 'Restart failed', ?, 'test')`).run(servers[0], new Date().toISOString());
        db().prepare(`INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, 'failed-restart')`)
            .run(servers[0], new Date().toISOString());
        expect(m.queueMaintenance([servers[0]], 'update', 'automatic')).toBe(0);
        vi.setSystemTime(new Date('2026-09-21T06:00:00Z')); await m.maintenanceTick();
        expect(m.plutoniumStatus().jobs).toEqual([expect.objectContaining({ server: servers[1], kind: 'schedule' })]);
        db().prepare('UPDATE restart_rollout SET healthy_at = ? WHERE server = ?').run(new Date().toISOString(), servers[0]);
        expect(m.queueMaintenance([servers[0]], 'update', 'automatic')).toBe(1);
    });
    it('preserves explicit update requests while another operation owns the server', async () => {
        const m = await outdated();
        const { db } = await import('../../src/lib/server/db/index.js');
        db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
            VALUES ('existing-start', ?, 'start', 'waiting', 'Waiting to start', ?, 'test')`).run(servers[0], new Date().toISOString());
        expect(m.queueUpdates([servers[0]])).toBe(1);
        expect(m.plutoniumStatus().jobs[0]).toMatchObject({ server: servers[0], kind: 'update', origin: 'manual' });
    });
    it('holds queued automatic updates when their policy is disabled, but finishes submitted recovery while paused', async () => {
        const m = await outdated();
        const p = await import('../../src/lib/server/maintenance/policies.js'); p.setMaintenancePolicy('updates', true);
        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        m.queueUpdates([servers[0]], true); p.setMaintenancePolicy('updates', false);
        await processProvisioning(); expect(komodoRestart).not.toHaveBeenCalled();
        p.setMaintenancePolicy('updates', true); p.pauseMaintenance(true);
        await processProvisioning(); expect(komodoRestart).not.toHaveBeenCalled();
        p.pauseMaintenance(false); await processProvisioning(); expect(komodoRestart).toHaveBeenCalledTimes(1);
        p.pauseMaintenance(true); vi.setSystemTime(Date.now() + 15_000);
        await processProvisioning(); await m.maintenanceTick();
        expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'failed', operation_state: 'done' });
        expect(komodoRestart).toHaveBeenCalledTimes(1);
    });
    it('latches failed update verification across restarts and retention, and supports an explicit retry', async () => {
        const m = await outdated();
        const p = await import('../../src/lib/server/maintenance/policies.js'); p.setMaintenancePolicy('updates', true);
        const { db, closeDb } = await import('../../src/lib/server/db/index.js');
        db().prepare("UPDATE servers SET desired_state = 'stopped' WHERE name = ?").run(servers[1]);
        await maintenanceTick(); vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
        const failed = m.plutoniumStatus().jobs[0];
        expect(failed).toMatchObject({ kind: 'update', state: 'failed', operation_state: 'done' });
        closeDb(); vi.setSystemTime(Date.now() + 31 * 86_400_000); await maintenanceTick();
        expect(m.plutoniumStatus().jobs).toHaveLength(1);
        expect(m.retryMaintenance(failed.id)).toBe(1);
        expect(m.plutoniumStatus().jobs).toHaveLength(2);
        const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
        vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: 5354, error: null });
        await maintenanceTick(); vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
        expect(m.plutoniumStatus().jobs[0]).toMatchObject({ state: 'done', origin: 'manual' });
        expect(db().prepare("SELECT 1 FROM server_deployments WHERE action = 'rebuild'").get()).toBeUndefined();
    });
    it('does not restart intentionally stopped servers, including jobs queued before the intent changed', async () => {
        const m = await outdated(); m.queueUpdates([servers[0]]);
        const { db } = await import('../../src/lib/server/db/index.js');
        db().prepare("UPDATE servers SET desired_state = 'stopped'").run();
        await maintenanceTick();
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        expect(komodoRestart).not.toHaveBeenCalled();
        expect(m.plutoniumStatus().jobs[0].state).toBe('cancelled');
        expect(m.queueUpdates()).toBe(0);
        expect(m.plutoniumStatus().servers.every(s => s.update_status === 'stopped')).toBe(true);
    });
    it('separates scheduled restart success from revision verification and coalesces paused runs', async () => {
        const m = await outdated(); m.saveMaintenanceSchedule(true, '06:00', 'UTC');
        const p = await import('../../src/lib/server/maintenance/policies.js'); p.pauseMaintenance(true);
        vi.setSystemTime(new Date('2026-09-24T07:00:00Z')); await m.maintenanceTick();
        expect(m.plutoniumStatus().jobs).toHaveLength(0);
        p.pauseMaintenance(false); await maintenanceTick();
        expect(m.plutoniumStatus().jobs).toHaveLength(2);
        expect(m.plutoniumStatus().jobs.every(j => j.kind === 'schedule' && j.target_revision === null)).toBe(true);
        const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
        vi.mocked(rconPlutoniumVersion).mockResolvedValue({ revision: null, error: 'Unknown reply' });
        vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
        expect(m.plutoniumStatus().jobs.find(j => j.server === servers[0])?.state).toBe('done');
        expect(m.plutoniumStatus().schedule.nextAt).toBe('2026-09-25T06:00:00.000Z');
    });
    it('does not treat stale, pre-restart, or unhealthy observations as current runtime evidence', async () => {
        const m = await outdated();
        const { db } = await import('../../src/lib/server/db/index.js');
        db().prepare('INSERT INTO restart_rollout(server,requested_at,healthy_at) VALUES (?,?,?)').run(servers[0], new Date().toISOString(), new Date().toISOString());
        expect(m.plutoniumStatus().servers[0].update_status).toBe('unknown');
        vi.setSystemTime(Date.now() + 3_600_000);
        expect(() => m.queueUpdates()).toThrow('published revision');
        const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
        const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
        vi.mocked(rconPlutoniumVersion).mockClear();
        vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'starting', reason: 'Starting' });
        await m.checkPlutonium();
        expect(rconPlutoniumVersion).not.toHaveBeenCalled();
        expect(m.plutoniumStatus().servers.every(s => s.update_status === 'unknown')).toBe(true);
    });
    it('adopts existing maintenance jobs without changing their IDs, schedules, or submitted operations', async () => {
        const m = await outdated(); m.saveMaintenanceSchedule(true, '06:00', 'America/New_York');
        m.queueMaintenance([servers[0]]); await maintenanceTick();
        const before = m.plutoniumStatus();
        const { db, closeDb } = await import('../../src/lib/server/db/index.js');
        db().exec('ALTER TABLE maintenance_jobs DROP COLUMN kind; ALTER TABLE maintenance_jobs DROP COLUMN origin;'); closeDb();
        const after = m.plutoniumStatus();
        expect(after.schedule).toEqual(before.schedule);
        expect(after.jobs[0]).toMatchObject({ id: before.jobs[0].id, provisioning_id: before.jobs[0].provisioning_id, kind: 'legacy', origin: 'manual' });
        vi.setSystemTime(Date.now() + 15_000); await maintenanceTick();
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        expect(komodoRestart).toHaveBeenCalledTimes(1);
    });
});