import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';

vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
    komodoRestart: vi.fn(), komodoInspectHealthy: vi.fn()
}));
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
    rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn()
}));

let fixture: Fixture;
const servers = ['t5_zombie_theater_1', 't5_zombie_theater_2', 't5_zombie_theater_3'];
const healthyRcon = {
    ok: true as const, endpoint: { host: 'test', port: 28960 }, rawReply: '',
    status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] }
};

beforeEach(async () => {
    vi.resetModules();
    vi.resetAllMocks();
    vi.useFakeTimers({ toFake: ['Date'] });
    vi.setSystemTime(new Date('2026-09-17T15:00:00Z'));
    fixture = await newFixture(servers);
    vi.stubEnv('KOMODO_ADDRESS', 'http://komodo.test');
    vi.stubEnv('KOMODO_API_KEY', 'test');
    vi.stubEnv('KOMODO_API_SECRET', 'test');
    vi.stubEnv('RCON_PASSWORD', 'test');
    vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '1');
    const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
    await seedServerRegistry();
    const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
    const { rconGetPlayerCount, rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
    vi.mocked(komodoRestart).mockResolvedValue({ ok: true });
    vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
    vi.mocked(rconGetPlayerCount).mockResolvedValue(0);
    vi.mocked(rconStatusResult).mockResolvedValue(healthyRcon);
});

afterEach(async () => {
    await fixture.cleanup();
    vi.unstubAllEnvs();
    vi.useRealTimers();
});

function nextTick() { vi.setSystemTime(Date.now() + 15_000); }

async function deployment(id = 'dep-1', targets = servers, action = 'promote') {
    const { db } = await import('../../src/lib/server/db/index.js');
    db().prepare('INSERT INTO deployments (id, action, started_at) VALUES (?, ?, ?)')
        .run(id, action, new Date().toISOString());
    for (const server of targets) {
        db().prepare(`INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state)
            VALUES (?, ?, 1, 'pending')`).run(id, server);
    }
}

async function rows(id = 'dep-1') {
    const { db } = await import('../../src/lib/server/db/index.js');
    return db().prepare('SELECT server, restart_state, restarted_at, error FROM deployment_servers WHERE deployment_id = ? ORDER BY server')
        .all(id) as Array<{ server: string; restart_state: string; restarted_at: string | null; error: string | null }>;
}

async function beta(targets: string[], waitForHealth = true) {
    const { createScript, getLatestVersion } = await import('../../src/lib/server/fleet/scripts.js');
    const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
    const { script } = createScript({ name: 'test script', targetRelPath: 'rollout.gsc', content: Buffer.from('main(){}') });
    return openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: getLatestVersion(script.id)!.id }],
        servers: targets, waitForHealth, waitForMatch: false });
}

describe('rolling deployment restarts', () => {
    it('restarts one server per recovery and finishes only after the last server is healthy', async () => {
        await deployment();
        const { sweep } = await import('../helpers/provisioning.js');
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        expect((await sweep()).restarted).toBe(1);
        expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]]]);
        expect((await rows()).map((row) => !!row.restarted_at)).toEqual([true, false, false]);
        await sweep(); // Repeated kick can't bypass the grace period.
        expect(komodoRestart).toHaveBeenCalledTimes(1);
        nextTick();
        expect(await sweep()).toMatchObject({ restarted: 1, healthy: 1 });
        expect((await rows()).map((row) => row.restart_state)).toEqual(['done', 'pending', 'pending']);
        nextTick();
        expect(await sweep()).toMatchObject({ restarted: 1, healthy: 1 });
        nextTick();
        expect(await sweep()).toMatchObject({ restarted: 0, healthy: 1 });
        expect((await rows()).every((row) => row.restart_state === 'done')).toBe(true);
        expect(vi.mocked(komodoRestart).mock.calls).toEqual(servers.map((server) => [server]));
        expect(db().prepare('SELECT finished_at FROM deployments WHERE id = ?').get('dep-1'))
            .toMatchObject({ finished_at: expect.any(String) });
    });

    it('shares capacity across promote, rollback, and drift deployments in oldest-first order', async () => {
        await deployment('z-oldest', [servers[2]], 'rollback');
        nextTick();
        await deployment('a-next', [servers[0]], 'drift');
        nextTick();
        await deployment('b-latest', [servers[1]], 'promote');
        const { sweep } = await import('../helpers/provisioning.js');
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        await sweep();
        expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[2]]]);
        nextTick();
        await sweep();
        expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[2]], [servers[0]]]);
    });

    it('skips occupied and unreachable servers without consuming capacity', async () => {
        await deployment();
        const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
        vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(4).mockResolvedValueOnce(null).mockResolvedValueOnce(0);
        const { sweep } = await import('../helpers/provisioning.js');
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        await sweep();
        expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[2]]]);
    });

    it('keeps the fleet queued while either container or RCON health is failing', async () => {
        await deployment();
        const { sweep } = await import('../helpers/provisioning.js');
        const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
        const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
        await sweep();
        nextTick();
        vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'starting' });
        await sweep();
        expect(komodoRestart).toHaveBeenCalledTimes(1);
        nextTick();
        vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: false, endpoint: null, reason: 'timeout', detail: 'starting' });
        await sweep();
        expect(komodoRestart).toHaveBeenCalledTimes(1);
        nextTick();
        await sweep();
        expect(komodoRestart).toHaveBeenCalledTimes(2);
    });

    it('coalesces overlapping sweeps while the remote restart is unresolved', async () => {
        await deployment();
        const { sweep } = await import('../helpers/provisioning.js');
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        let finish!: () => void;
        const entered = new Promise<void>((resolve) => {
            vi.mocked(komodoRestart).mockImplementationOnce(() => {
                resolve();
                return new Promise((done) => { finish = () => done({ ok: true }); });
            });
        });
        const first = sweep();
        await entered;
        const second = sweep();
        expect(second).toBe(first);
        // Even a long-running restart request cannot be freed by a health poll.
        nextTick();
        const { refreshRestartRollout, rolloutWaitReason } = await import('../../src/lib/server/pipeline/restart-rollout.js');
        await refreshRestartRollout();
        expect(rolloutWaitReason(servers[1])).toMatch(/Queued/);
        finish();
        await Promise.all([first, second]);
        expect(komodoRestart).toHaveBeenCalledTimes(1);
    });

    it('deduplicates simultaneous deployment rows for the same server', async () => {
        await deployment('dep-1', [servers[0]]);
        await deployment('dep-2', [servers[0], servers[1]]);
        const { sweep } = await import('../helpers/provisioning.js');
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        await sweep();
        expect(komodoRestart).toHaveBeenCalledTimes(1);
        expect((await rows('dep-1'))[0].restarted_at).toBeTruthy();
        expect((await rows('dep-2'))[0].restarted_at).toBeTruthy();
    });

    it('does not count a later deployment as having been applied by an earlier restart', async () => {
        await deployment('dep-1', [servers[0]]);
        const { sweep } = await import('../helpers/provisioning.js');
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        await sweep();
        await deployment('dep-2', [servers[0]]);
        nextTick();
        await sweep();
        expect((await rows('dep-1'))[0].restart_state).toBe('done');
        expect((await rows('dep-2'))[0].restart_state).toBe('pending');
        // Provisioning can submit the next job in the pass that verifies the
        // first, but it must issue a distinct restart and await its recovery.
        expect(komodoRestart).toHaveBeenCalledTimes(2);
        expect((await rows('dep-2'))[0].restarted_at).not.toBe((await rows('dep-1'))[0].restarted_at);
        await sweep();
        expect(komodoRestart).toHaveBeenCalledTimes(2);
        expect((await rows('dep-2'))[0].restarted_at).toBeTruthy();
    });

    it('preserves occupied slots across app/module restarts', async () => {
        await deployment();
        const { sweep } = await import('../helpers/provisioning.js');
        await sweep();
        const { closeDb } = await import('../../src/lib/server/db/index.js');
        closeDb();
        vi.resetModules();
        const fresh = await import('../helpers/provisioning.js');
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        await fresh.sweep();
        expect(komodoRestart).toHaveBeenCalledTimes(1);
        nextTick();
        await fresh.sweep();
        expect(komodoRestart).toHaveBeenCalledTimes(2);
    });

    it('adopts pre-upgrade restarts instead of starting a second server', async () => {
        await deployment();
        const { db } = await import('../../src/lib/server/db/index.js');
        db().prepare('UPDATE deployment_servers SET restarted_at = ? WHERE server = ?')
            .run(new Date().toISOString(), servers[0]);
        const { sweep } = await import('../helpers/provisioning.js');
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        await sweep();
        expect(komodoRestart).not.toHaveBeenCalled();
    });

    it('retains capacity after a timeout because the remote restart may have been accepted', async () => {
        await deployment();
        const { sweep } = await import('../helpers/provisioning.js');
        const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
        vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'timed out' });
        await sweep();
        expect((await rows())[0].restart_state).toBe('failed');
        expect(komodoRestart).toHaveBeenCalledTimes(1);
        nextTick();
        vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'restarting' });
        await sweep();
        expect(komodoRestart).toHaveBeenCalledTimes(1);
    });

    it('does not restart a server whose file deployment failed', async () => {
        await deployment();
        const { db } = await import('../../src/lib/server/db/index.js');
        db().prepare("UPDATE deployment_servers SET files_changed = 0, restart_state = 'failed', error = 'write failed' WHERE server = ?")
            .run(servers[0]);
        const { sweep } = await import('../helpers/provisioning.js');
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        await sweep();
        expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[1]]]);
        expect((await rows())[0].error).toBe('write failed');
    });

    it('releases slots for definite local failures that could not have started a restart', async () => {
        await deployment();
        const { sweep } = await import('../helpers/provisioning.js');
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'no-name', message: 'no komodo_name' });
        await sweep();
        expect((await rows())[0].restart_state).toBe('failed');
        expect(vi.mocked(komodoRestart).mock.calls).toEqual([[servers[0]], [servers[1]]]);
    });
});

describe('beta and deployment coordination', () => {
    it('limits beta restarts and releases capacity only after recovery', async () => {
        await beta(servers);
        const { betaTick } = await import('../helpers/provisioning.js');
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        const first = betaTick();
        expect(betaTick()).toBe(first);
        await first;
        expect(komodoRestart).toHaveBeenCalledTimes(1);
        nextTick();
        await betaTick();
        expect(komodoRestart).toHaveBeenCalledTimes(2);
    });

    it('shares the same capacity when beta and deployment ticks run concurrently', async () => {
        await beta([servers[1]]);
        await deployment('dep-1', [servers[0]]);
        const { betaTick } = await import('../helpers/provisioning.js');
        const { sweep } = await import('../helpers/provisioning.js');
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        await Promise.all([sweep(), betaTick()]);
        expect(komodoRestart).toHaveBeenCalledTimes(1);
    });

    it('retains beta recovery slots even when the beta skips health checks or is closed', async () => {
        const { betaId } = await beta([servers[0]], false);
        const { betaTick } = await import('../helpers/provisioning.js');
        const { sweep } = await import('../helpers/provisioning.js');
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        await betaTick();
        expect(db().prepare('SELECT stage FROM beta_servers WHERE beta_id = ?').get(betaId)).toEqual({ stage: 'ready' });
        db().prepare("UPDATE betas SET state = 'promoted' WHERE id = ?").run(betaId);
        await deployment('dep-1', [servers[1]]);
        await sweep();
        expect(komodoRestart).toHaveBeenCalledTimes(1);
        nextTick();
        await sweep();
        expect(komodoRestart).toHaveBeenCalledTimes(2);
    });

    it('checks player occupancy again when resuming a legacy restarting beta', async () => {
        const { betaId } = await beta([servers[0]]);
        const { db } = await import('../../src/lib/server/db/index.js');
        db().prepare("UPDATE beta_servers SET stage = 'restarting', updated_at = ? WHERE beta_id = ?")
            .run(new Date(Date.now() - 30_000).toISOString(), betaId);
        const { betaTick } = await import('../helpers/provisioning.js');
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
        vi.mocked(rconGetPlayerCount).mockResolvedValue(3);
        await betaTick();
        expect(komodoRestart).not.toHaveBeenCalled();
    });
});

describe('Provisioning ownership', () => {
    it('persists caller requests before probes and completes them without another caller tick', async () => {
        await deployment('campaign', [servers[0]]);
        const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
        const { provisioningOperation, getDeployment } = await import('../../src/lib/server/provisioning/index.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
        const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
        await sweep(); await sweep();
        const source = db().prepare('SELECT provisioning_id FROM deployment_servers WHERE deployment_id = ?').get('campaign') as { provisioning_id: string };
        const id = source.provisioning_id;
        expect(provisioningOperation(id)).toMatchObject({ action: 'restart', state: 'waiting', submitted_at: null });
        expect(getDeployment(id)?.sources).toEqual([{ kind: 'campaign', source_id: 'campaign', server: servers[0] }]);
        expect(komodoRestart).not.toHaveBeenCalled();
        expect(komodoInspectHealthy).not.toHaveBeenCalled();
        expect(rconGetPlayerCount).not.toHaveBeenCalled();
        expect(db().prepare('SELECT COUNT(*) n FROM server_deployments').get()).toEqual({ n: 1 });
        await processProvisioning(); nextTick(); await processProvisioning();
        expect(provisioningOperation(id)?.state).toBe('done');
        expect((await rows('campaign'))[0].restart_state).toBe('pending');
        await sweep();
        expect((await rows('campaign'))[0].restart_state).toBe('done');
        expect(komodoRestart).toHaveBeenCalledTimes(1);
    });

    it('lets concurrent callers submit while only the owner admits restart capacity', async () => {
        await deployment('campaign', [servers[0]]); await beta([servers[1]]);
        const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
        const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
        await Promise.all([sweep(), betaTick()]);
        expect(komodoRestart).not.toHaveBeenCalled(); expect(rconGetPlayerCount).not.toHaveBeenCalled();
        const first = processProvisioning();
        expect(processProvisioning()).toBe(first);
        await first;
        expect(komodoRestart).toHaveBeenCalledTimes(1);
    });

    it('does not replay an ambiguous campaign restart after recovery or app restart', async () => {
        await deployment('campaign', [servers[0]]);
        const { sweep } = await import('../helpers/provisioning.js');
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'timeout' });
        await sweep(); nextTick(); await sweep();
        const { closeDb } = await import('../../src/lib/server/db/index.js');
        closeDb(); vi.resetModules();
        const fresh = await import('../helpers/provisioning.js');
        nextTick(); await fresh.sweep();
        expect((await rows('campaign'))[0].restart_state).toBe('failed');
        expect(komodoRestart).toHaveBeenCalledTimes(1);
    });

    it('holds a paused beta request and cancels closed work without a beta runner tick', async () => {
        const { betaId } = await beta([servers[0]]);
        const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
        await betaTick();
        const { db } = await import('../../src/lib/server/db/index.js');
        db().prepare("INSERT INTO beta_test_plans (beta_id, plan, outcome) VALUES (?, '{}', 'paused')").run(betaId);
        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
        await processProvisioning();
        expect(db().prepare('SELECT state, stage FROM server_deployments').get()).toEqual({ state: 'waiting', stage: 'Waiting for the requester to resume' });
        expect(rconGetPlayerCount).not.toHaveBeenCalled();
        db().prepare("UPDATE betas SET state = 'rolled-back' WHERE id = ?").run(betaId);
        await processProvisioning();
        expect(db().prepare('SELECT state FROM server_deployments').get()).toEqual({ state: 'cancelled' });
        expect(komodoRestart).not.toHaveBeenCalled();
    });

    it('allows an explicit new operation after failed campaign recovery without changing the failed history', async () => {
        await deployment('campaign', [servers[0]]);
        const { sweep } = await import('../helpers/provisioning.js');
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'timeout' });
        await sweep(); nextTick(); await sweep();
        const { requestLifecycle } = await import('../../src/lib/server/provisioning/index.js');
        expect(requestLifecycle(servers[0], 'restart').ok).toBe(true);
        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
        await processProvisioning();
        expect(komodoRestart).toHaveBeenCalledTimes(2);
        expect((await rows('campaign'))[0].restart_state).toBe('failed');
    });

    it('does not credit a legacy submission with an older completed recovery slot', async () => {
        await deployment('campaign', [servers[0]]);
        const { db } = await import('../../src/lib/server/db/index.js');
        const old = new Date().toISOString(); nextTick();
        const submitted = new Date().toISOString();
        db().prepare('INSERT INTO restart_rollout (server, requested_at, healthy_at) VALUES (?, ?, ?)').run(servers[0], old, old);
        db().prepare('UPDATE deployment_servers SET restarted_at = ?').run(submitted);
        const { sweep } = await import('../helpers/provisioning.js');
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        await sweep();
        expect(db().prepare('SELECT state FROM server_deployments').get()).toEqual({ state: 'awaiting-health' });
        expect(db().prepare('SELECT requested_at, healthy_at FROM restart_rollout').get()).toEqual({ requested_at: submitted, healthy_at: null });
        nextTick(); await sweep();
        expect((await rows('campaign'))[0].restart_state).toBe('done');
        expect(komodoRestart).not.toHaveBeenCalled();
    });

    it('preserves a queued request across app restarts and links explicit beta retries to new history', async () => {
        const { betaId } = await beta([servers[0]]);
        const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
        await betaTick();
        const { db, closeDb } = await import('../../src/lib/server/db/index.js');
        const previous = db().prepare('SELECT id FROM server_deployments').get() as { id: string };
        closeDb(); vi.resetModules();
        const { requestRestart, releaseRestartRequest, provisioningOperation } = await import('../../src/lib/server/provisioning/index.js');
        const source = { kind: 'beta' as const, id: String(betaId) };
        expect(requestRestart(servers[0], source).id).toBe(previous.id);
        releaseRestartRequest(servers[0], source);
        const next = requestRestart(servers[0], source);
        expect(next.id).not.toBe(previous.id);
        expect(provisioningOperation(previous.id)?.state).toBe('cancelled');
        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
        await processProvisioning();
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        expect(komodoRestart).toHaveBeenCalledTimes(1);
        expect(provisioningOperation(next.id)?.state).toBe('awaiting-health');
    });
});

describe('rollout configuration', () => {
    it('honors a configured limit larger than one', async () => {
        const { closeDb } = await import('../../src/lib/server/db/index.js');
        closeDb();
        vi.resetModules();
        vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '2');
        await deployment();
        const { sweep } = await import('../helpers/provisioning.js');
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        await sweep();
        expect(komodoRestart).toHaveBeenCalledTimes(2);
        await sweep();
        expect(komodoRestart).toHaveBeenCalledTimes(2);
    });

    it.each(['0', '-1', '1.5', 'abc'])('rejects invalid concurrency %s', async (value) => {
        const { closeDb } = await import('../../src/lib/server/db/index.js');
        closeDb();
        vi.resetModules();
        vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', value);
        await expect(import('../../src/lib/server/config.js')).rejects.toThrow(/DEPLOYMENT_RESTART_CONCURRENCY/);
        vi.stubEnv('DEPLOYMENT_RESTART_CONCURRENCY', '1');
        vi.resetModules();
    });
});