import { prepareNativeHost } from '../helpers/native-host.js';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { access, mkdir, readFile, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import { gameEvent } from '../helpers/telemetry.js';

vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn(), rconPlutoniumVersion: vi.fn() }));
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));

const server = 't5_zombie_theater_1';
const healthyRcon = { ok: true as const, endpoint: { host: 'test', port: 28960 }, rawReply: '',
    status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } };
const missingRcon = { ok: false as const, endpoint: { host: 'test', port: 28960 }, reason: 'timeout' as const, detail: 'No reply' };
let fixture: Fixture;
type Engine = typeof import('../../src/lib/server/watchdog/index.js');
let watchdog: Engine;
let database: typeof import('../../src/lib/server/db/index.js');
let backend: typeof import('../../src/lib/server/backends/native.js');
let refreshHost: () => unknown;
let rcon: typeof import('../../src/lib/server/integrations/rcon.js');

beforeEach(async () => {
    vi.resetModules(); vi.resetAllMocks();
    vi.useFakeTimers({ toFake: ['Date'] });
    vi.setSystemTime(new Date('2026-09-27T12:00:00Z'));
    fixture = await newFixture([server]);
    vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] })));
    for (const [key, value] of Object.entries({ RCON_PASSWORD: 'test', DEPLOYMENT_CREATE_CONCURRENCY: '1', DEPLOYMENT_RESTART_CONCURRENCY: '1' })) vi.stubEnv(key, value);
    database = await import('../../src/lib/server/db/index.js');
    const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
    await seedServerRegistry();
    database.db().prepare('UPDATE servers SET game_port = 28960 WHERE name = ?').run(server);
    const { reserveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
    reserveServerKey(server, 't5', 'saved-key');
    refreshHost = (await prepareNativeHost([server])).heartbeat;
    backend = await import('../../src/lib/server/backends/native.js');
    vi.spyOn(backend, 'nativeHealth').mockResolvedValue({ ok: true, state: 'running' });
    const { mockAgentFiles } = await import('../helpers/agent-files.js');
    await mockAgentFiles();
    rcon = await import('../../src/lib/server/integrations/rcon.js');
    vi.mocked(rcon.rconGetPlayerCount).mockResolvedValue(0);
    vi.mocked(rcon.rconStatusResult).mockResolvedValue(healthyRcon);
    vi.mocked(rcon.rconPlutoniumVersion).mockResolvedValue({ revision: 5354, error: null });
    watchdog = await import('../../src/lib/server/watchdog/index.js');
});
afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); vi.unstubAllEnvs(); vi.unstubAllGlobals(); });

const incidents = () => watchdog.watchdogOverview().incidents;
const onlyIncident = () => { expect(incidents()).toHaveLength(1); return incidents()[0]; };
const operations = () => database.db().prepare('SELECT id, action, state FROM server_deployments ORDER BY rowid').all() as { id: string; action: string; state: string }[];
function advance(ms: number) { vi.setSystemTime(Date.now() + ms); refreshHost(); }
function down() { vi.mocked(backend.nativeHealth).mockResolvedValue({ ok: false, state: 'exited', reason: 'Container exited' }); }
async function pollStatus() {
    const worker = await vi.importActual<typeof import('../../src/lib/server/pipeline/worker.js')>('../../src/lib/server/pipeline/worker.js');
    await worker.fanoutStatus();
}
async function sustainedFailure() {
    down();
    await watchdog.watchdogTick(true);
    advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
    advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
    expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'waiting' })]);
    return onlyIncident();
}

describe('Watchdog detection and ownership', () => {
    it('waits for status-monitor evidence without issuing its own routine probes', async () => {
        down();
        for (let minute = 0; minute < 4; minute++) {
            await watchdog.watchdogTick();
            advance(watchdog.WATCHDOG_SCAN_MS);
        }
        expect(backend.nativeHealth).not.toHaveBeenCalled();
        expect(rcon.rconStatusResult).not.toHaveBeenCalled();
        expect(incidents()).toEqual([]);
        expect(operations()).toEqual([]);
    });

    it.each(['container', 'game'])('requests recovery from sustained %s failures collected by status polling', async kind => {
        if (kind === 'container') down();
        else vi.mocked(rcon.rconStatusResult).mockResolvedValue(missingRcon);
        for (let minute = 0; minute < 3; minute++) {
            await pollStatus();
            await watchdog.watchdogTick();
            if (minute < 2) {
                expect(onlyIncident().failures).toBe(minute + 1);
                expect(operations()).toEqual([]);
                advance(watchdog.WATCHDOG_SCAN_MS);
            }
        }
        expect(onlyIncident()).toMatchObject({ failures: 3, state: 'recovering' });
        expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'waiting' })]);
        expect(backend.nativeHealth).toHaveBeenCalledTimes(3);
        expect(rcon.rconStatusResult).toHaveBeenCalledTimes(kind === 'game' ? 3 : 0);
    });

    it('does not count the same sample again or use stale data, and resets after a monitoring gap', async () => {
        down();
        await pollStatus();
        await watchdog.watchdogTick();
        const first = onlyIncident();
        advance(30_000);
        database.db().prepare("DELETE FROM settings WHERE key='watchdog_last_scan'").run();
        await watchdog.watchdogTick();
        advance(60_000);
        await watchdog.watchdogTick();
        expect(onlyIncident()).toMatchObject({ failures: 1, last_seen_at: first.last_seen_at });
        expect(backend.nativeHealth).toHaveBeenCalledTimes(1);
        advance(90_001);
        await pollStatus();
        await watchdog.watchdogTick();
        expect(onlyIncident()).toMatchObject({ failures: 1, first_seen_at: new Date().toISOString() });
        expect(operations()).toEqual([]);
    });

    it('uses sample times across polling offsets without counting early or losing sustained failures', async () => {
        down();
        await pollStatus();
        advance(20_000);
        await watchdog.watchdogTick();
        advance(10_000);
        await pollStatus();
        advance(50_000);
        // The first failure is stale until the monitor supplies another sample.
        await pollStatus();
        await watchdog.watchdogTick();
        expect(onlyIncident().failures).toBe(2);
        advance(40_000);
        await pollStatus();
        advance(20_000);
        await watchdog.watchdogTick();
        expect(onlyIncident()).toMatchObject({ failures: 3, state: 'recovering' });
        expect(operations()).toHaveLength(1);
    });

    it('breaks a routine failure streak on a successful status poll between watchdog passes', async () => {
        vi.mocked(rcon.rconStatusResult).mockResolvedValue(missingRcon);
        await pollStatus();
        await watchdog.watchdogTick();
        advance(30_000);
        vi.mocked(rcon.rconStatusResult).mockResolvedValueOnce(healthyRcon);
        await pollStatus();
        advance(30_000);
        await pollStatus();
        await watchdog.watchdogTick();
        expect(onlyIncident()).toMatchObject({ failures: 1, first_seen_at: new Date().toISOString() });
        expect(operations()).toEqual([]);
        expect(rcon.rconStatusResult).toHaveBeenCalledTimes(3);
    });

    it('uses fresh telemetry alongside the polled container and waits for polled RCON when telemetry expires', async () => {
        const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js');
        saveTelemetry({ ...gameEvent('heartbeat', 17, 4, server), receivedAt: new Date().toISOString() });
        await pollStatus();
        await watchdog.watchdogTick();
        expect(incidents()).toEqual([]);
        expect(rcon.rconStatusResult).not.toHaveBeenCalled();
        advance(watchdog.WATCHDOG_SCAN_MS);
        await watchdog.watchdogTick();
        expect(incidents()).toEqual([]);
        expect(rcon.rconStatusResult).not.toHaveBeenCalled();
        vi.mocked(rcon.rconStatusResult).mockResolvedValue(missingRcon);
        advance(30_001);
        await pollStatus();
        advance(30_000);
        database.db().prepare("DELETE FROM settings WHERE key='watchdog_last_scan'").run();
        await watchdog.watchdogTick();
        expect(onlyIncident()).toMatchObject({ failures: 1, state: 'open' });
        expect(rcon.rconStatusResult).toHaveBeenCalledTimes(1);
    });

    it('starts a new failure window for a replacement registration', async () => {
        down();
        await pollStatus();
        await watchdog.watchdogTick();
        advance(watchdog.WATCHDOG_SCAN_MS);
        await pollStatus();
        await watchdog.watchdogTick();
        expect(onlyIncident().failures).toBe(2);
        advance(watchdog.WATCHDOG_SCAN_MS);
        database.db().prepare('UPDATE servers SET created_at=? WHERE name=?').run(new Date().toISOString(), server);
        await pollStatus();
        await watchdog.watchdogTick();
        expect(onlyIncident()).toMatchObject({ failures: 1, first_seen_at: new Date().toISOString() });
        expect(operations()).toEqual([]);
    });

    it('scans files every five minutes while retaining minute health checks and immediate manual scans', async () => {
        const drift = await import('../../src/lib/server/fleet/drift.js');
        const scan = vi.spyOn(drift, 'prepareDriftCampaign');
        await pollStatus();
        await watchdog.watchdogTick();
        expect(scan).toHaveBeenCalledTimes(1);
        for (let minute = 1; minute < 5; minute++) {
            advance(watchdog.WATCHDOG_SCAN_MS);
            await pollStatus();
            await watchdog.watchdogTick();
        }
        expect(scan).toHaveBeenCalledTimes(1);
        expect(backend.nativeHealth).toHaveBeenCalledTimes(5);
        advance(watchdog.WATCHDOG_SCAN_MS);
        await pollStatus();
        await watchdog.watchdogTick();
        expect(scan).toHaveBeenCalledTimes(2);
        await watchdog.watchdogTick(true);
        expect(scan).toHaveBeenCalledTimes(3);
        advance(watchdog.WATCHDOG_SCAN_MS);
        database.db().prepare('UPDATE servers SET created_at=? WHERE name=?').run(new Date().toISOString(), server);
        await pollStatus();
        await watchdog.watchdogTick();
        expect(scan).toHaveBeenCalledTimes(4);
        scan.mockRestore();
    });

    it('retries failed file scans on the next health scan instead of retaining successful freshness', async () => {
        const drift = await import('../../src/lib/server/fleet/drift.js');
        const scan = vi.spyOn(drift, 'prepareDriftCampaign').mockRejectedValueOnce(new Error('Agent unavailable'));
        await pollStatus();
        await watchdog.watchdogTick();
        expect(watchdog.watchdogOverview().scanError).toContain('Agent unavailable');
        advance(watchdog.WATCHDOG_SCAN_MS);
        await pollStatus();
        await watchdog.watchdogTick();
        expect(scan).toHaveBeenCalledTimes(2);
        expect(watchdog.watchdogOverview().scanError).toBeNull();
        scan.mockRestore();
    });

    it('defaults to automatic recovery and debounces manual scans without accelerating the failure window', async () => {
        expect(watchdog.watchdogOverview().enabled).toBe(true);
        down();
        await watchdog.watchdogTick(true);
        expect(onlyIncident()).toMatchObject({ state: 'open', failures: 1 });
        for (let n = 0; n < 5; n++) await watchdog.watchdogTick(true);
        expect(onlyIncident().failures).toBe(1);
        expect(operations()).toEqual([]);
        advance(watchdog.WATCHDOG_SCAN_MS);
        await pollStatus();
        await watchdog.watchdogTick();
        expect(onlyIncident().failures).toBe(2);
        advance(watchdog.WATCHDOG_SCAN_MS - 1);
        await watchdog.watchdogTick(true);
        expect(operations()).toEqual([]);
        advance(1);
        await watchdog.watchdogTick(true);
        const row = onlyIncident();
        expect(row).toMatchObject({ state: 'recovering', failures: 3 });
        expect(watchdog.watchdogDetail(row.id)?.actions).toEqual([expect.objectContaining({ action: 'restart', operation_state: 'waiting' })]);
        await watchdog.watchdogTick(true);
        expect(operations()).toHaveLength(1);
        expect(database.db().prepare("SELECT id FROM ec_host_operations WHERE kind LIKE 'instance.%'").all()).toEqual([]);
    });

    it('does not restart an unknown host agent state even after repeated sustained observations', async () => {
        vi.mocked(backend.nativeHealth).mockResolvedValue({ ok: false, reason: 'host agent request timed out' });
        for (let n = 0; n < 8; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
        expect(onlyIncident()).toMatchObject({ kind: 'health', state: 'open' });
        expect(operations()).toEqual([]);
        expect(rcon.rconStatusResult).not.toHaveBeenCalled();
        down();
        await watchdog.watchdogTick(true);
        expect(onlyIncident().failures).toBe(1);
        expect(operations()).toEqual([]);
        advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
        expect(operations()).toEqual([]);
        advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
        expect(operations()).toEqual([expect.objectContaining({ action: 'restart' })]);
    });

    it('allows a longer initial startup grace period before requesting recovery', async () => {
        vi.mocked(backend.nativeHealth).mockResolvedValue({ ok: false, state: 'starting', reason: 'Starting' });
        for (let n = 0; n < 5; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
        expect(operations()).toEqual([]);
        await watchdog.watchdogTick(true);
        expect(operations()).toEqual([expect.objectContaining({ action: 'restart' })]);
    });

    it('accepts fresh validated telemetry when RCON is unavailable and rechecks after it becomes stale', async () => {
        const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js');
        vi.mocked(rcon.rconStatusResult).mockResolvedValue(missingRcon);
        saveTelemetry({ ...gameEvent('heartbeat', 17, 4, server), receivedAt: new Date().toISOString() });
        await watchdog.watchdogTick(true);
        expect(incidents()).toEqual([]);
        expect(rcon.rconStatusResult).not.toHaveBeenCalled();
        advance(90_001);
        await watchdog.watchdogTick(true);
        expect(onlyIncident()).toMatchObject({ kind: 'health', state: 'open', failures: 1 });
        saveTelemetry({ ...gameEvent('heartbeat', 18, 4, server), receivedAt: new Date().toISOString() });
        await watchdog.watchdogTick(true);
        expect(onlyIncident().state).toBe('resolved');
        expect(operations()).toEqual([]);
    });

    it.each(['no-config', 'no-port', 'bad-reply', 'no-players'] as const)('keeps %s RCON failures observational', async reason => {
        vi.mocked(rcon.rconStatusResult).mockResolvedValue({ ...missingRcon, reason });
        for (let n = 0; n < 4; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
        expect(onlyIncident()).toMatchObject({ state: 'open' });
        expect(operations()).toEqual([]);
    });

    it('breaks the sustained failure window on a successful probe between watchdog samples', async () => {
        vi.mocked(rcon.rconStatusResult).mockResolvedValue(missingRcon);
        await watchdog.watchdogTick(true);
        advance(watchdog.WATCHDOG_SCAN_MS);
        await watchdog.watchdogTick(true);
        advance(20_000);
        const { POST } = await import('../../src/routes/api/servers/[name]/probe-rcon/+server.js');
        vi.mocked(rcon.rconStatusResult).mockResolvedValueOnce(healthyRcon);
        await POST({ params: { name: server } } as Parameters<typeof POST>[0]);
        advance(40_000);
        await watchdog.watchdogTick(true);
        expect(onlyIncident()).toMatchObject({ failures: 1, state: 'open', first_seen_at: new Date().toISOString() });
        expect(operations()).toEqual([]);
        advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
        expect(operations()).toEqual([]);
        advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
        expect(operations()).toEqual([expect.objectContaining({ action: 'restart' })]);
    });

    it('preserves intentional stop intent after completion and database reopening', async () => {
        const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
        const stop = requestLifecycle(server, 'stop');
        expect(stop.ok).toBe(true);
        if (!stop.ok) throw new Error(stop.error);
        database.db().prepare("UPDATE server_deployments SET state = 'done', finished_at = ? WHERE id = ?").run(new Date().toISOString(), stop.deploymentId);
        down();
        database.closeDb();
        for (let n = 0; n < 4; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
        expect(incidents()).toEqual([]);
        expect(operations()).toEqual([expect.objectContaining({ action: 'stop' })]);
        expect(backend.nativeHealth).not.toHaveBeenCalled();
    });

    it.each(['beta', 'campaign'] as const)('defers a server owned by %s work', async (owner) => {
        if (owner === 'beta') {
            const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
            const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
            const { script, version } = createScript({ name: 'beta', content: Buffer.from('experimental'), targetRelPath: 'beta.gsc' });
            await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: version.id }], servers: [server], waitForHealth: false, waitForMatch: false });
        } else {
            database.db().prepare("INSERT INTO deployments (id, action, started_at) VALUES ('campaign', 'promote', ?)").run(new Date().toISOString());
            database.db().prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES ('campaign', ?, 1, 'held')").run(server);
        }
        down();
        for (let n = 0; n < 4; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); }
        expect(incidents()).toEqual([]);
        expect(operations()).toEqual([]);
        expect(backend.nativeHealth).not.toHaveBeenCalled();
    });

    it('rechecks lifecycle ownership after a health probe yields to an operator stop', async () => {
        const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
        vi.mocked(backend.nativeHealth).mockImplementationOnce(async () => {
            expect(requestLifecycle(server, 'stop').ok).toBe(true);
            return { ok: false, state: 'exited', reason: 'Container exited' };
        });
        await watchdog.watchdogTick(true);
        expect(incidents()).toEqual([]);
        expect(operations()).toEqual([expect.objectContaining({ action: 'stop' })]);
    });

    it.each(['queued', 'waiting', 'running', 'awaiting-health', 'retained recovery'])(
        'suppresses health and drift throughout %s provisioning, then resumes', async state => {
            const retained = state === 'retained recovery';
            const now = new Date().toISOString();
            database.db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
                VALUES ('manual-operation', ?, ?, ?, 'Operator work', ?, 'manual')`)
                .run(server, retained ? 'restart' : 'rebuild', retained ? 'failed' : state, now);
            if (retained) database.db().prepare(`INSERT INTO restart_rollout (server, operation_id, requested_at)
                VALUES (?, 'manual-operation', ?)`).run(server, now);
            await writeFile(join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc'), 'unexpected');
            down();
            for (let n = 0; n < 4; n++) {
                await watchdog.watchdogTick(true);
                advance(watchdog.WATCHDOG_SCAN_MS);
            }
            expect(incidents()).toEqual([]);
            expect(backend.nativeHealth).not.toHaveBeenCalled();
            expect(rcon.rconStatusResult).not.toHaveBeenCalled();
            expect(operations()).toHaveLength(1);

            database.db().prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'manual-operation'").run();
            if (retained) database.db().prepare('UPDATE restart_rollout SET healthy_at = ? WHERE server = ?').run(new Date().toISOString(), server);
            await watchdog.watchdogTick(true);
            expect(incidents()).toEqual(expect.arrayContaining([
                expect.objectContaining({ kind: 'health', state: 'open', failures: 1 }),
                expect.objectContaining({ kind: 'drift', state: 'recovering' })
            ]));
            expect(operations()).toHaveLength(2);
            expect(operations()[1]).toMatchObject({ action: 'rebuild', state: 'waiting' });
        }
    );

    it('stops probing when provisioning takes ownership during the container inspection', async () => {
        const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
        vi.mocked(backend.nativeHealth).mockImplementationOnce(async () => {
            expect(requestLifecycle(server, 'restart').ok).toBe(true);
            return { ok: true, state: 'running' };
        });
        await watchdog.watchdogTick(true);
        expect(incidents()).toEqual([]);
        expect(rcon.rconStatusResult).not.toHaveBeenCalled();
        expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'waiting' })]);
    });
});

describe('Watchdog recovery persistence', () => {

    it('resumes a persisted detection window and does not duplicate queued work after reopening', async () => {
        down();
        await watchdog.watchdogTick(true);
        advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
        const id = onlyIncident().id;
        database.closeDb();
        advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
        expect(onlyIncident().id).toBe(id);
        const queued = operations();
        expect(queued).toHaveLength(1);
        database.closeDb();
        advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
        expect(operations()).toEqual(queued);
        expect(watchdog.watchdogDetail(id)?.actions).toHaveLength(1);
    });

    it('permits an explicit new attempt after dismissal without silently rearming a sustained condition', async () => {
        down();
        await watchdog.watchdogTick(true);
        const id = onlyIncident().id;
        expect(watchdog.updateWatchdogIncident(id, 'dismiss')).toEqual({ ok: true });
        for (let n = 0; n < 4; n++) { advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true); }
        expect(operations()).toEqual([]);
        expect(onlyIncident().state).toBe('dismissed');
        expect(watchdog.updateWatchdogIncident(id, 'retry')).toEqual({ ok: true });
        await watchdog.watchdogTick(true);
        expect(operations()).toEqual([]);
        advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
        advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
        expect(onlyIncident()).toMatchObject({ id, state: 'recovering', cycle: 2 });
        expect(watchdog.watchdogDetail(id)?.actions).toEqual([expect.objectContaining({ action: 'restart', cycle: 2 })]);
    });
});

describe('Watchdog APIs', () => {
    const request = (body: unknown, raw = false) => new Request('http://localhost/api/watchdog', {
        method: 'POST', headers: { 'Content-Type': 'application/json' }, body: raw ? String(body) : JSON.stringify(body)
    });

    it('returns overview state and validates scan actions and pause booleans', async () => {
        const { GET, POST } = await import('../../src/routes/api/watchdog/+server.js');
        expect(await GET().json()).toEqual({ enabled: true, lastScanAt: null, scanError: null, incidents: [] });
        expect((await POST({ request: request('{', true) })).status).toBe(400);
        for (const body of [null, [], {}, { action: 'restart' }, { enabled: 'false' }, { enabled: 0 }, { action: 'scan', enabled: 'false' }]) {
            expect((await POST({ request: request(body) })).status).toBe(400);
        }
        expect(await (await POST({ request: request({ enabled: false }) })).json()).toMatchObject({ enabled: false });
        database.closeDb();
        expect(await GET().json()).toMatchObject({ enabled: false });
        down();
        const scanned = await POST({ request: request({ action: 'scan' }) });
        expect(scanned.status).toBe(200);
        expect(await scanned.json()).toMatchObject({ enabled: false, lastScanAt: expect.any(String), incidents: [expect.objectContaining({ kind: 'health', state: 'open' })] });
        expect(operations()).toEqual([]);
    });

    it('validates incident actions and refuses retries while provisioning is pending', async () => {
        const { GET, POST } = await import('../../src/routes/api/watchdog/[id]/+server.js');
        expect(GET({ params: { id: 'missing' } }).status).toBe(404);
        expect((await POST({ params: { id: 'missing' }, request: request({ action: 'retry' }) })).status).toBe(404);
        for (const body of [null, {}, [], { action: 'restart' }]) {
            expect((await POST({ params: { id: 'missing' }, request: request(body) })).status).toBe(400);
        }
        expect((await POST({ params: { id: 'missing' }, request: request('{', true) })).status).toBe(400);
        const row = await sustainedFailure();
        expect(await GET({ params: { id: row.id } }).json()).toMatchObject({
            incident: { id: row.id, state: 'recovering' }, actions: [expect.objectContaining({ action: 'restart' })], events: expect.any(Array)
        });
        for (const action of ['retry', 'dismiss']) {
            const response = await POST({ params: { id: row.id }, request: request({ action }) });
            expect(response.status).toBe(409);
            expect(await response.json()).toMatchObject({ error: expect.stringContaining('Provisioning is still active') });
        }
        expect(operations()).toHaveLength(1);
    });

    it('keeps legacy drift previews readable and directs mutations to Watchdog without changing files', async () => {
        const { GET, POST } = await import('../../src/routes/api/drift/+server.js');
        const path = join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc');
        await writeFile(path, 'unexpected content');
        expect(await (await GET()).json()).toMatchObject({ totalDrift: 1, servers: [{ server, files: [expect.objectContaining({ relPath: 'stray.gsc' })] }] });
        const preview = await POST({ request: request({ dryRun: true, servers: [server] }) });
        expect(preview.status).toBe(200);
        const body = await preview.json();
        expect(body).toMatchObject({ ok: true, plan: { servers: [server], fingerprint: expect.any(String), files: [expect.objectContaining({ relPath: 'stray.gsc' })] } });
        const mutation = await POST({ request: request({ servers: [server], expectedFingerprint: body.plan.fingerprint }) });
        expect(mutation.status).toBe(410);
        expect(await mutation.json()).toMatchObject({ ok: false, watchdogUrl: '/watchdog', error: expect.stringContaining('Watchdog') });
        expect(await readFile(path, 'utf8')).toBe('unexpected content');
        expect(operations()).toEqual([]);
        expect(database.db().prepare('SELECT id FROM deployments').all()).toEqual([]);
    });
});

describe('Watchdog drift campaigns', () => {

    it('keeps a missing rebuild key visible as attention without deleting files or retrying automatically', async () => {
        // Simulate legacy registry damage; administrators cannot remove a leased key.
        database.db().prepare('DELETE FROM server_key_leases WHERE server_id=?').run(server);
        const path = join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp', 'stray.gsc');
        await writeFile(path, 'unexpected');
        await watchdog.watchdogTick(true);
        expect(onlyIncident()).toMatchObject({ kind: 'drift', state: 'attention', summary: expect.stringContaining('server key are required') });
        advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true);
        expect(operations()).toEqual([]);
        expect(await readFile(path, 'utf8')).toBe('unexpected');
        expect(database.db().prepare("SELECT id FROM ec_host_operations WHERE kind LIKE 'instance.%'").all()).toEqual([]);
    });
});