import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import Database from 'better-sqlite3';
import { readFile, writeFile, access } from 'node:fs/promises';
import { join } from 'node:path';
import { newFixture, type Fixture } from '../helpers/fixture.js';

vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
    komodoRestart: vi.fn(), komodoRuntimeAction: vi.fn(), komodoInspectHealthy: vi.fn(),
    komodoCreateDeployment: vi.fn(), komodoDeploy: vi.fn(), komodoDestroyDeployment: vi.fn(), komodoDeleteDeployment: vi.fn(),
    komodoEnsureTag: vi.fn(async (name: string) => ({ ok: true, id: name })),
    komodoUpdateTagsOnResource: vi.fn(async () => ({ ok: true }))
}));
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn(), rconPlutoniumVersion: vi.fn() }));
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));

let fixture: Fixture;
const server = 't5_zombie_theater_2';
const healthyRcon = { ok: true as const, endpoint: { host: 'test', port: 28960 }, rawReply: '',
    status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } };

beforeEach(async () => {
    vi.resetModules(); vi.resetAllMocks();
    vi.useFakeTimers({ toFake: ['Date'] });
    vi.setSystemTime(new Date('2026-09-27T12:00:00Z'));
    fixture = await newFixture([server]);
    vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] })));
    for (const [key, value] of Object.entries({ KOMODO_ADDRESS: 'http://komodo.test', KOMODO_API_KEY: 'test', KOMODO_API_SECRET: 'test',
        KOMODO_TARGET_SERVER: 'host', RCON_PASSWORD: 'test', DEPLOYMENT_CREATE_CONCURRENCY: '1', DEPLOYMENT_RESTART_CONCURRENCY: '1',
        SERVER_PORT_MIN: '28960', SERVER_PORT_MAX: '28970' })) vi.stubEnv(key, value);
    // Exercise the upgrade path, with history and references present before migration.
    const old = new Database(fixture.dbPath);
    const schema = await readFile(new URL('../../src/lib/server/db/schema.sql', import.meta.url), 'utf8');
    old.exec(schema.replace("'create', 'start', 'restart', 'rebuild', 'stop', 'destroy'", "'create', 'destroy'")
        .replace("'campaign','beta','maintenance','watchdog'", "'campaign','beta','maintenance'")
        .replace("server        TEXT NOT NULL, -- Campaign observation survives a rebuild's registry gap.", 'server        TEXT NOT NULL REFERENCES servers(name),'));
    old.prepare(`INSERT INTO server_deployments (rowid,id,server,action,state,stage,started_at,triggered_by) VALUES (42,'old','old-server','create','done','Complete','2026-09-01','test')`).run();
    old.prepare("INSERT INTO server_deployment_events (deployment_id,created_at,message) VALUES ('old','2026-09-01','Old activity')").run();
    old.prepare("INSERT INTO server_creation_jobs VALUES ('old','old-server',28961,'zombie','theater','zombie_theater')").run();
    old.prepare("INSERT INTO provisioning_sources VALUES ('old','campaign','legacy-campaign','old-server')").run();
    old.close();
    const { db } = await import('../../src/lib/server/db/index.js');
    db().prepare(`INSERT INTO servers (name,komodo_type,komodo_name,game,map,game_mode,map_slug,game_port,host_dir)
        VALUES (?, 'deployment', ?, 't5','zombie_theater','zombie','theater',28960,?)`).run(server, server, join(fixture.t5, server));
    const { saveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
    saveServerKey(server, 'saved-key');
    const komodo = await import('../../src/lib/server/integrations/komodo.js');
    vi.mocked(komodo.komodoEnsureTag).mockImplementation(async name => ({ ok: true, id: name }));
    vi.mocked(komodo.komodoUpdateTagsOnResource).mockResolvedValue({ ok: true });
    const { kick } = await import('../../src/lib/server/pipeline/worker.js');
    vi.mocked(kick).mockResolvedValue(undefined);
    for (const name of ['komodoRestart', 'komodoRuntimeAction', 'komodoCreateDeployment', 'komodoDeploy', 'komodoDestroyDeployment', 'komodoDeleteDeployment'] as const) vi.mocked(komodo[name]).mockResolvedValue({ ok: true });
    vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
    const rcon = await import('../../src/lib/server/integrations/rcon.js');
    vi.mocked(rcon.rconGetPlayerCount).mockResolvedValue(0);
    vi.mocked(rcon.rconStatusResult).mockResolvedValue(healthyRcon);
    vi.mocked(rcon.rconPlutoniumVersion).mockResolvedValue({ revision: 5354, error: null });
});
afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); });

async function operation(id: string) {
    const { db } = await import('../../src/lib/server/db/index.js');
    return db().prepare('SELECT * FROM server_deployments WHERE id = ?').get(id);
}

async function queue(action: 'start' | 'restart' | 'stop' | 'rebuild') {
    const { requestLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
    const result = requestLifecycle(server, action);
    expect(result.ok).toBe(true);
    if (!result.ok) throw new Error(result.error);
    return result.deploymentId;
}

describe('provisioning operations', () => {
    it('migrates the action constraint without losing row ordering, activity, or foreign keys', async () => {
        const { db, closeDb } = await import('../../src/lib/server/db/index.js');
        await queue('start');
        closeDb();
        expect(db().prepare("SELECT rowid FROM server_deployments WHERE id = 'old'").get()).toEqual({ rowid: 42 });
        expect(db().prepare("SELECT message FROM server_deployment_events WHERE deployment_id = 'old'").get()).toEqual({ message: 'Old activity' });
        expect(db().prepare("SELECT server FROM server_creation_jobs WHERE deployment_id = 'old'").get()).toEqual({ server: 'old-server' });
        expect(db().prepare("SELECT kind, source_id FROM provisioning_sources WHERE operation_id = 'old'").get()).toEqual({ kind: 'campaign', source_id: 'legacy-campaign' });
        expect(db().pragma('foreign_key_check')).toEqual([]);
        expect(db().pragma('foreign_keys', { simple: true })).toBe(1);
    });

    it('starts without an RCON empty check and verifies recovery after the grace period', async () => {
        const { processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
        const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
        const { komodoRuntimeAction } = await import('../../src/lib/server/integrations/komodo.js');
        const id = await queue('start');
        await processLifecycle();
        expect(komodoRuntimeAction).toHaveBeenCalledWith(server, 'start');
        expect(rconGetPlayerCount).not.toHaveBeenCalled();
        expect(await operation(id)).toMatchObject({ action: 'start', state: 'awaiting-health' });
        await processLifecycle();
        expect(await operation(id)).toMatchObject({ state: 'awaiting-health' });
        vi.setSystemTime(Date.now() + 15_000);
        await processLifecycle();
        expect(await operation(id)).toMatchObject({ state: 'done' });
        expect(komodoRuntimeAction).toHaveBeenCalledTimes(1);
    });

    it('deduplicates requests, rejects conflicting actions, and waits for a fresh empty reading', async () => {
        const { processLifecycle, requestLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
        const { destroyServer } = await import('../../src/lib/server/fleet/create.js');
        const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        const id = await queue('restart');
        expect(requestLifecycle(server, 'restart')).toEqual({ ok: true, deploymentId: id });
        expect(requestLifecycle(server, 'stop')).toMatchObject({ ok: false, status: 409 });
        expect(destroyServer(server)).toMatchObject({ ok: false, kind: 'conflict' });
        for (const players of [4, null]) {
            vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(players);
            await processLifecycle();
            expect(await operation(id)).toMatchObject({ state: 'waiting' });
            expect(komodoRestart).not.toHaveBeenCalled();
        }
        await processLifecycle();
        expect(komodoRestart).toHaveBeenCalledTimes(1);
    });

    it('does not repeat an uncertain restart and retains its recovery slot', async () => {
        const { processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        vi.mocked(komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'timeout' });
        const id = await queue('restart');
        await processLifecycle(); await processLifecycle();
        expect(await operation(id)).toMatchObject({ state: 'failed', error: 'komodo: timeout' });
        expect(komodoRestart).toHaveBeenCalledTimes(1);
        expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server)).toEqual({ healthy_at: null });
    });

    it('keeps stop pending on unknown health and completes only on a confirmed stopped state', async () => {
        const { processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
        const { komodoInspectHealthy, komodoRuntimeAction } = await import('../../src/lib/server/integrations/komodo.js');
        const { closeDb } = await import('../../src/lib/server/db/index.js');
        const id = await queue('stop');
        await processLifecycle();
        closeDb();
        vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, reason: 'network error' });
        await processLifecycle();
        expect(await operation(id)).toMatchObject({ state: 'awaiting-health' });
        vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: false, state: 'exited', reason: 'stopped' });
        await processLifecycle();
        expect(await operation(id)).toMatchObject({ state: 'done' });
        expect(komodoRuntimeAction).toHaveBeenCalledTimes(1);
        await expect(access(join(fixture.t5, server))).resolves.toBeUndefined();
    });

    it('rebuilds from nothing in the same slot and port, then verifies the fresh instance', async () => {
        const { processLifecycle, requestLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
        const { processCreations, nextLobbyNumber, nextAvailablePort } = await import('../../src/lib/server/fleet/create.js');
        const { komodoInspectHealthy, komodoDestroyDeployment, komodoDeleteDeployment, komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        const sentinel = join(fixture.t5, server, 'old-instance.dat');
        await writeFile(sentinel, 'must not survive');
        const id = await queue('rebuild');
        // Promotions after queueing must be included in the fresh instance.
        const { createScript, getLatestVersion } = await import('../../src/lib/server/fleet/scripts.js');
        const { script } = createScript({ name: 'Fresh promoted script', targetRelPath: 'fresh.gsc', scopeType: 'game', scopeValue: 't5', content: Buffer.from('fresh content') });
        db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(getLatestVersion(script.id)!.id, script.id);
        vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'stopped' });
        await processLifecycle();
        expect(komodoDestroyDeployment).toHaveBeenCalledWith(server);
        expect(komodoDeleteDeployment).toHaveBeenCalledWith(server);
        await expect(access(sentinel)).rejects.toThrow();
        expect(db().prepare('SELECT name FROM servers WHERE name = ?').get(server)).toBeUndefined();
        expect(await operation(id)).toMatchObject({ state: 'queued', action: 'rebuild' });
        expect(nextAvailablePort()).toBe(28961);
        expect(nextLobbyNumber('t5_zombie_theater', ['t5_zombie_theater_1'])).toBe(3);
        expect(requestLifecycle(server, 'rebuild')).toEqual({ ok: true, deploymentId: id });
        await processCreations();
        expect(komodoCreateDeployment).toHaveBeenCalledWith(expect.objectContaining({ name: server, ports: ['28960:28960/udp', '28960:28960/tcp'] }));
        await expect(access(sentinel)).rejects.toThrow();
        expect(await operation(id)).toMatchObject({ state: 'awaiting-health' });
        expect(await readFile(join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp/fresh.gsc'), 'utf8')).toBe('fresh content');
        vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
        vi.setSystemTime(Date.now() + 1);
        await processCreations();
        expect(await operation(id)).toMatchObject({ state: 'done' });
        const { runningVersions } = await import('../../src/lib/server/integrations/plutonium.js');
        expect(runningVersions().find(version => version.server === server)).toMatchObject({ revision: 5354, update_status: 'current' });
    });

    it('rejects a rebuild without a saved key before touching the existing instance', async () => {
        const { requestLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
        const { removeServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
        const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js');
        removeServerKey(server);
        expect(requestLifecycle(server, 'rebuild')).toMatchObject({ ok: false, error: expect.stringContaining('Save a server key') });
        expect(komodoDestroyDeployment).not.toHaveBeenCalled();
    });

    it('retains the rebuild cleanup checkpoint and reservation while waiting for capacity', async () => {
        const { processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
        const { beginDeployment, deploymentStage, finishDeployment, resumeRebuild, REMOVING_SERVER } = await import('../../src/lib/server/fleet/deployments.js');
        const { nextAvailablePort } = await import('../../src/lib/server/fleet/create.js');
        const { db, closeDb } = await import('../../src/lib/server/db/index.js');
        const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js');
        const id = await queue('rebuild');
        // Model a crash after confirmed remote deletion and registry cleanup.
        deploymentStage(id, REMOVING_SERVER);
        db().prepare('DELETE FROM servers WHERE name = ?').run(server);
        finishDeployment(id, { ok: false, message: 'cleanup interrupted' });
        expect(nextAvailablePort()).toBe(28961);
        expect(resumeRebuild(id)).toBe(true);
        const occupied = beginDeployment('another-server', 'create');
        await processLifecycle();
        expect(await operation(id)).toMatchObject({ state: 'waiting', stage: REMOVING_SERVER });
        closeDb();
        finishDeployment(occupied, { ok: true });
        await processLifecycle();
        expect(await operation(id)).toMatchObject({ state: 'queued' });
        expect(komodoDestroyDeployment).not.toHaveBeenCalled();
    });

    it('allows explicit destruction of a stalled startup and records the interrupted verification', async () => {
        const { processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
        const { destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
        const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
        const id = await queue('start');
        await processLifecycle();
        expect(destroyServer(server)).toMatchObject({ ok: true });
        vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'stopped' });
        await processDestructions();
        expect(await operation(id)).toMatchObject({ state: 'cancelled', error: 'Server deleted', server_deleted_at: expect.any(String) });
    });

    it('validates lifecycle API input and returns the durable operation ID', async () => {
        const { POST, GET } = await import('../../src/routes/api/provisioning/+server.js');
        const submit = (body: unknown) => POST({ request: new Request('http://localhost/api/provisioning', {
            method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body)
        }) });
        for (const body of [null, { server, action: 'erase' }, { server: 5, action: 'start' }]) expect((await submit(body)).status).toBe(400);
        const result = await submit({ server, action: 'start' });
        expect(result.status).toBe(202);
        const { deploymentId } = await result.json();
        expect((await GET().json()).deployments).toContainEqual(expect.objectContaining({ id: deploymentId, action: 'start', state: 'waiting' }));
    });

    it('does not create a replacement when teardown fails or resume verification of the old instance', async () => {
        const { processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
        const { processCreations } = await import('../../src/lib/server/fleet/create.js');
        const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
        const { komodoDestroyDeployment, komodoCreateDeployment } = await import('../../src/lib/server/integrations/komodo.js');
        vi.mocked(komodoDestroyDeployment).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'timeout' });
        const id = await queue('rebuild');
        await processLifecycle(); await processCreations(); await processLifecycle();
        expect(await operation(id)).toMatchObject({ state: 'failed' });
        expect(getDeployment(id)?.canResumeVerification).toBe(false);
        expect(komodoDestroyDeployment).toHaveBeenCalledTimes(1);
        expect(komodoCreateDeployment).not.toHaveBeenCalled();
    });
});

async function incident(id = 'incident') {
    const { db } = await import('../../src/lib/server/db/index.js');
    const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
    // These lifecycle tests exercise an actual container failure. Healthy
    // containers with fresh game evidence now cancel obsolete health work.
    vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'unhealthy', reason: 'Container unhealthy' });
    const now = new Date().toISOString();
    db().prepare(`INSERT INTO watchdog_incidents (id,server,kind,state,summary,first_seen_at,last_seen_at)
        VALUES (?,?,'health','open','Server down',?,?)`).run(id, server, now, now);
    return id;
}

async function report(players: number, timestamp = new Date().toISOString(), map = 'zombie_theater') {
    const { db } = await import('../../src/lib/server/db/index.js');
    const { gameEvent } = await import('../helpers/telemetry.js');
    db().prepare(`INSERT INTO server_telemetry (server,round,players,received_at,payload) VALUES (?,1,?,?,?)
        ON CONFLICT(server) DO UPDATE SET players=excluded.players, received_at=excluded.received_at, payload=excluded.payload`)
        .run(server, players, timestamp, JSON.stringify({ ...gameEvent('heartbeat', 1, players, server, map), receivedAt: timestamp }));
}

describe('Watchdog provisioning and telemetry evidence', () => {
    it('keeps the original running intent when a queued removal is cancelled', async () => {
        const { destroyServer, cancelRemoval } = await import('../../src/lib/server/provisioning/index.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        const removal = destroyServer(server);
        if (!removal.ok || !removal.deploymentId) throw new Error('Could not queue removal');
        expect(cancelRemoval(removal.deploymentId)).toBe(true);
        expect(db().prepare('SELECT desired_state FROM servers WHERE name = ?').get(server)).toEqual({ desired_state: 'running' });
    });

    it('preserves stop intent across process restarts, including an uncertain stop request', async () => {
        const { db, closeDb } = await import('../../src/lib/server/db/index.js');
        const { finishDeployment } = await import('../../src/lib/server/provisioning/history.js');
        const id = await queue('stop');
        finishDeployment(id, { ok: false, message: 'lost response' });
        expect(db().prepare('SELECT desired_state FROM servers WHERE name = ?').get(server)).toEqual({ desired_state: 'stopped' });
        // Exercise a legacy upgrade where the stop exists but intent was not stored.
        db().prepare("UPDATE servers SET desired_state = 'running'").run();
        db().prepare("DELETE FROM settings WHERE key = 'watchdog_desired_state_v1'").run();
        closeDb();
        expect(db().prepare('SELECT desired_state FROM servers WHERE name = ?').get(server)).toEqual({ desired_state: 'stopped' });
        const { requestWatchdogLifecycle } = await import('../../src/lib/server/provisioning/index.js');
        await incident();
        expect(requestWatchdogLifecycle(server, 'restart', 'incident')).toMatchObject({ ok: false, error: 'This server is intentionally stopped.' });
        await queue('start');
        expect(db().prepare('SELECT desired_state FROM servers WHERE name = ?').get(server)).toEqual({ desired_state: 'running' });
    });

    it('deduplicates Watchdog requests, records attribution, and holds paused work', async () => {
        const { requestWatchdogLifecycle, getDeployment } = await import('../../src/lib/server/provisioning/index.js');
        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        await incident();
        const result = requestWatchdogLifecycle(server, 'restart', 'incident');
        expect(result.ok).toBe(true);
        if (!result.ok) throw new Error(result.error);
        expect(requestWatchdogLifecycle(server, 'restart', 'incident')).toEqual(result);
        expect(getDeployment(result.deploymentId)?.sources).toEqual([{ kind: 'watchdog', source_id: 'incident', server,
            label: 'Server down', href: '/maintenance/incident' }]);
        expect(await operation(result.deploymentId)).toMatchObject({ triggered_by: 'watchdog:incident' });
        db().prepare("INSERT INTO settings (key,value) VALUES ('watchdog_enabled','0')").run();
        await processProvisioning();
        expect(komodoRestart).not.toHaveBeenCalled();
        db().prepare("UPDATE settings SET value = '1' WHERE key = 'watchdog_enabled'").run();
        await processProvisioning();
        expect(komodoRestart).toHaveBeenCalledTimes(1);
    });

    it('accepts fresh telemetry for occupancy and restart recovery without RCON, while rejecting old-process reports', async () => {
        const { requestWatchdogLifecycle } = await import('../../src/lib/server/provisioning/index.js');
        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
        const { rconGetPlayerCount, rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
        const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
        const { config } = await import('../../src/lib/server/config.js');
        vi.spyOn(config.rcon, 'password', 'get').mockReturnValue('');
        vi.mocked(rconGetPlayerCount).mockResolvedValue(null);
        vi.mocked(rconStatusResult).mockResolvedValue({ ok: false, reason: 'no-config', detail: 'No RCON', endpoint: null });
        await incident(); await report(0);
        const result = requestWatchdogLifecycle(server, 'restart', 'incident');
        if (!result.ok) throw new Error(result.error);
        await processProvisioning();
        expect(komodoRestart).toHaveBeenCalledTimes(1);
        vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
        vi.setSystemTime(Date.now() + 15_000);
        await processProvisioning();
        expect(await operation(result.deploymentId)).toMatchObject({ state: 'awaiting-health' });
        await report(0);
        vi.setSystemTime(Date.now() + 15_000);
        await processProvisioning();
        expect(await operation(result.deploymentId)).toMatchObject({ state: 'done' });
    });

    it('uses fresh occupied telemetry arriving during RCON to veto a zero-player restart', async () => {
        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
        const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
        const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        const id = await queue('restart');
        vi.mocked(rconGetPlayerCount).mockImplementationOnce(async () => { await report(2); return 0; });
        await processProvisioning();
        expect(komodoRestart).not.toHaveBeenCalled();
        expect(await operation(id)).toMatchObject({ state: 'waiting', players: 2 });
        await report(0);
        vi.mocked(rconGetPlayerCount).mockResolvedValueOnce(3);
        await processProvisioning();
        expect(komodoRestart).not.toHaveBeenCalled();
        expect(await operation(id)).toMatchObject({ state: 'waiting', players: 3 });
    });

    it('rejects stale, future, pre-submission, and invalid telemetry evidence', async () => {
        const { telemetryEvidence, probePlayerCount } = await import('../../src/lib/server/fleet/status-evidence.js');
        const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        vi.mocked(rconGetPlayerCount).mockResolvedValue(null);
        for (const offset of [-45_001, 1]) {
            await report(0, new Date(Date.now() + offset).toISOString());
            expect(telemetryEvidence(server)).toBeNull();
            expect(await probePlayerCount(server)).toBeNull();
        }
        await report(0);
        expect(telemetryEvidence(server, new Date().toISOString())).toBeNull();
        db().prepare("UPDATE server_telemetry SET payload = '{}' WHERE server = ?").run(server);
        expect(telemetryEvidence(server)).toBeNull();
    });

    it('reserves a rebuild before transferring its exact restart slot and requires fresh emptiness', async () => {
        const { requestWatchdogLifecycle, escalateWatchdogRestart } = await import('../../src/lib/server/provisioning/index.js');
        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
        const { komodoInspectHealthy, komodoDestroyDeployment, komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        const { rconGetPlayerCount, rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
        const { db, closeDb } = await import('../../src/lib/server/db/index.js');
        await incident();
        const restart = requestWatchdogLifecycle(server, 'restart', 'incident');
        if (!restart.ok) throw new Error(restart.error);
        await processProvisioning();
        expect(escalateWatchdogRestart('incident', restart.deploymentId)).toMatchObject({ ok: false });
        vi.setSystemTime(Date.now() + 300_000);
        const rebuild = escalateWatchdogRestart('incident', restart.deploymentId);
        if (!rebuild.ok) throw new Error(rebuild.error);
        expect(escalateWatchdogRestart('incident', restart.deploymentId)).toEqual(rebuild);
        expect(await operation(restart.deploymentId)).toMatchObject({ state: 'failed' });
        expect(db().prepare('SELECT operation_id, healthy_at FROM restart_rollout WHERE server = ?').get(server))
            .toEqual({ operation_id: rebuild.deploymentId, healthy_at: null });
        closeDb();
        vi.mocked(rconGetPlayerCount).mockResolvedValue(null);
        vi.mocked(rconStatusResult).mockResolvedValue({ ok: false, reason: 'timeout', detail: 'No reply', endpoint: null });
        await processProvisioning();
        expect(komodoDestroyDeployment).not.toHaveBeenCalled();
        expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'waiting' });
        expect(komodoRestart).toHaveBeenCalledTimes(1);
        vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'Stopped' });
        await processProvisioning();
        expect(komodoDestroyDeployment).toHaveBeenCalledTimes(1);
        expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'queued' });
        expect(db().prepare('SELECT 1 FROM restart_rollout WHERE server = ?').get(server)).toBeUndefined();
    });

    it('retains recovery capacity when a rebuild prerequisite fails or a different operation owns the slot', async () => {
        const { requestWatchdogLifecycle, escalateWatchdogRestart } = await import('../../src/lib/server/provisioning/index.js');
        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
        const { removeServerKey, saveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        await incident();
        const restart = requestWatchdogLifecycle(server, 'restart', 'incident');
        if (!restart.ok) throw new Error(restart.error);
        await processProvisioning(); vi.setSystemTime(Date.now() + 300_000);
        removeServerKey(server);
        expect(escalateWatchdogRestart('incident', restart.deploymentId)).toMatchObject({ ok: false });
        expect(await operation(restart.deploymentId)).toMatchObject({ state: 'awaiting-health' });
        expect(db().prepare('SELECT operation_id FROM restart_rollout WHERE server = ?').get(server)).toEqual({ operation_id: restart.deploymentId });
        saveServerKey(server, 'saved-key');
        db().prepare("UPDATE restart_rollout SET operation_id = 'another-operation' WHERE server = ?").run(server);
        expect(escalateWatchdogRestart('incident', restart.deploymentId)).toMatchObject({ ok: false });
        expect(db().prepare('SELECT operation_id FROM restart_rollout WHERE server = ?').get(server)).toEqual({ operation_id: 'another-operation' });
        expect(db().prepare("SELECT COUNT(*) AS n FROM server_deployments WHERE action = 'rebuild'").get()).toEqual({ n: 0 });
    });

    it('does not let an in-flight old restart observation release a transferred rebuild slot', async () => {
        const { requestWatchdogLifecycle, escalateWatchdogRestart } = await import('../../src/lib/server/provisioning/index.js');
        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
        const { refreshRestartRollout } = await import('../../src/lib/server/provisioning/restarts.js');
        const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        await incident();
        const restart = requestWatchdogLifecycle(server, 'restart', 'incident');
        if (!restart.ok) throw new Error(restart.error);
        await processProvisioning(); vi.setSystemTime(Date.now() + 300_000);
        let reportHealthy!: (value: { ok: true; state: string }) => void;
        vi.mocked(komodoInspectHealthy).mockImplementationOnce(() => new Promise(resolve => { reportHealthy = resolve; }));
        const observation = refreshRestartRollout();
        const rebuild = escalateWatchdogRestart('incident', restart.deploymentId);
        if (!rebuild.ok) throw new Error(rebuild.error);
        reportHealthy({ ok: true, state: 'running' });
        await observation;
        expect(db().prepare('SELECT operation_id,healthy_at FROM restart_rollout WHERE server = ?').get(server))
            .toEqual({ operation_id: rebuild.deploymentId, healthy_at: null });
        expect(await operation(restart.deploymentId)).toMatchObject({ state: 'failed' });
        expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'waiting' });
    });

    it('resumes explicitly requested rebuild cleanup after the incident needs attention, even when Watchdog is paused', async () => {
        const { requestWatchdogLifecycle, resumeRebuild } = await import('../../src/lib/server/provisioning/index.js');
        const { deploymentStage, finishDeployment, REMOVING_SERVER } = await import('../../src/lib/server/provisioning/history.js');
        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
        const { komodoDestroyDeployment } = await import('../../src/lib/server/integrations/komodo.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        await incident();
        const rebuild = requestWatchdogLifecycle(server, 'rebuild', 'incident');
        if (!rebuild.ok) throw new Error(rebuild.error);
        deploymentStage(rebuild.deploymentId, REMOVING_SERVER);
        finishDeployment(rebuild.deploymentId, { ok: false, message: 'Cleanup interrupted' });
        db().prepare("UPDATE watchdog_incidents SET state = 'attention' WHERE id = 'incident'").run();
        db().prepare("INSERT INTO settings (key,value) VALUES ('watchdog_enabled','0')").run();
        db().prepare('DELETE FROM servers WHERE name = ?').run(server);
        expect(resumeRebuild(rebuild.deploymentId)).toBe(true);
        await processProvisioning();
        expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'queued' });
        expect(komodoDestroyDeployment).not.toHaveBeenCalled();
    });

    it.each(['restart','rebuild'] as const)('hands retained failed-rebuild capacity to an explicit new-cycle %s attempt', async action => {
        const { requestWatchdogLifecycle, escalateWatchdogRestart } = await import('../../src/lib/server/provisioning/index.js');
        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
        const { refreshRestartRollout } = await import('../../src/lib/server/provisioning/restarts.js');
        const { komodoDestroyDeployment, komodoInspectHealthy, komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        await incident();
        const restart = requestWatchdogLifecycle(server, 'restart', 'incident');
        if (!restart.ok) throw new Error(restart.error);
        await processProvisioning(); vi.setSystemTime(Date.now() + 300_000);
        const rebuild = escalateWatchdogRestart('incident', restart.deploymentId);
        if (!rebuild.ok) throw new Error(rebuild.error);
        db().prepare("INSERT INTO watchdog_actions (id,incident_id,action,operation_id,created_at) VALUES ('failed-rebuild','incident','rebuild',?,?)")
            .run(rebuild.deploymentId, new Date().toISOString());
        vi.mocked(komodoDestroyDeployment).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'Teardown failed' });
        await processProvisioning();
        expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'failed' });
        db().prepare("UPDATE watchdog_incidents SET cycle = 2, state = 'open' WHERE id = 'incident'").run();
        const retry = requestWatchdogLifecycle(server, action, 'incident');
        if (!retry.ok) throw new Error(retry.error);
        expect(db().prepare('SELECT operation_id,healthy_at FROM restart_rollout WHERE server = ?').get(server))
            .toEqual({ operation_id: retry.deploymentId, healthy_at: null });
        await refreshRestartRollout();
        // Old process health cannot release capacity before this retry submits.
        expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server)).toEqual({ healthy_at: null });
        if (action === 'rebuild') vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'Stopped' });
        await processProvisioning();
        expect(await operation(retry.deploymentId)).toMatchObject({ state: action === 'restart' ? 'awaiting-health' : 'queued' });
        expect(komodoRestart).toHaveBeenCalledTimes(action === 'restart' ? 2 : 1);
    });

    it('keeps its linked drift campaign and file audit throughout rebuild, and verifies the replacement through telemetry', async () => {
        const { requestWatchdogLifecycle } = await import('../../src/lib/server/provisioning/index.js');
        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
        const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
        const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        await incident();
        const rebuild = requestWatchdogLifecycle(server, 'rebuild', 'incident');
        if (!rebuild.ok) throw new Error(rebuild.error);
        const now = new Date().toISOString();
        db().prepare("INSERT INTO deployments (id,action,started_at) VALUES ('drift-audit','drift',?)").run(now);
        db().prepare("INSERT INTO deployment_servers (deployment_id,server,files_changed,restart_state,provisioning_id) VALUES ('drift-audit',?,1,'pending',?)").run(server, rebuild.deploymentId);
        db().prepare("INSERT INTO deployment_files (deployment_id,server,verb,rel_path) VALUES ('drift-audit',?,'updated','script.gsc')").run(server);
        db().prepare("INSERT INTO provisioning_sources VALUES (?,'campaign','drift-audit',?)").run(rebuild.deploymentId, server);
        db().prepare("INSERT INTO watchdog_actions (id,incident_id,action,operation_id,campaign_id,created_at) VALUES ('action','incident','rebuild',?,'drift-audit',?)").run(rebuild.deploymentId, now);
        vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'Stopped' });
        await processProvisioning();
        expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'queued' });
        expect(db().prepare("SELECT provisioning_id FROM deployment_servers WHERE deployment_id = 'drift-audit'").get()).toEqual({ provisioning_id: rebuild.deploymentId });
        expect(db().prepare("SELECT COUNT(*) AS n FROM deployment_files WHERE deployment_id = 'drift-audit'").get()).toEqual({ n: 1 });
        expect(db().pragma('foreign_key_check')).toEqual([]);
        await processProvisioning();
        expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'awaiting-health' });
        vi.setSystemTime(Date.now() + 15_000); await report(0);
        vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
        vi.mocked(rconStatusResult).mockResolvedValue({ ok: false, reason: 'timeout', detail: 'No reply', endpoint: null });
        await processProvisioning();
        expect(await operation(rebuild.deploymentId)).toMatchObject({ state: 'done' });
    });
});