import { putFileBlob } from '../../src/lib/server/db/file-blobs.js';
import { createHash } from 'node:crypto';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { join } from 'node:path';
import { newFixture, type Fixture } from '../helpers/fixture.js';

const remote = vi.hoisted(() => ({ state: 'running', players: 0, map: 'zombie_theater' }));
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
    rconStatusResult: vi.fn(async () => ({ ok: true, endpoint: { host: 'agent', port: 28960 }, rawReply: '', status: { players: remote.players, map: remote.map, gametype: 'zom', roster: [] } })),
    rconGetPlayerCount: vi.fn(async () => remote.players), rconPlutoniumVersion: vi.fn(async () => ({ revision: 5354, error: null }))
}));
vi.mock('../../src/lib/server/integrations/plutonium.js', () => ({ checkRelease: vi.fn(async () => {}), freshRelease: () => 5354, probeVersion: vi.fn(async () => 5354) }));
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) }));
vi.mock('../../src/lib/server/rebalancing/admission.js', () => ({
    rebalanceAdmission: vi.fn(() => null), requestRebalanceStopForOperation: vi.fn()
}));

let queuedKeys = 0;
let fixture: Fixture; let hostId: string; let libraryId: string;
const image = 'example.test/plutainer:test';
const inventory = () => ({ cpuCount: 16, cpuUsagePercent: 10, memoryTotal: 64 * 1024 ** 3, memoryAvailable: 60 * 1024 ** 3, protocolVersion: 1, capabilities: ['instance.unhealthy-recovery'], dockerRunning: true, directoriesWritable: true, storageAvailable: 100_000_000_000, runtimeImages: [image], libraries: libraryId ? [{ id: libraryId, state: 'ready' as const }] : [] });
beforeEach(async () => {
    vi.resetModules(); vi.clearAllMocks(); remote.state = 'running'; remote.players = 0; remote.map = 'zombie_theater'; libraryId = ''; queuedKeys = 0;
    fixture = await newFixture([]);
    vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries'));
    vi.stubEnv('PLUTAINER_IMAGE', image); vi.stubEnv('RCON_PASSWORD', 'test-password');

    const { createHost, createEnrollment, enrollHost } = await import('../../src/lib/server/hosts/index.js');
    hostId = createHost({ name: 'Native Debian', portMin: 28960, portMax: 28970 }).id;
    enrollHost(createEnrollment(hostId).token, inventory());
    const backend = await import('../../src/lib/server/backends/native.js');
    vi.spyOn(backend, 'nativeRequest').mockImplementation(async (_instance, kind) => {
        if (kind !== 'instance.inspect') throw new Error(`Unexpected operation: ${kind}`);
        return { healthy: remote.state === 'running', state: remote.state, exists: true } as never;
    });
});
afterEach(async () => { vi.useRealTimers(); await fixture.cleanup(); vi.unstubAllEnvs(); });

async function prepareLibrary(game: 't5' | 't6' = 't5') {
    const libs = await import('../../src/lib/server/game-files/index.js');
    const version = libs.createLibraryImport({ hostId, game });
    const bytes = Buffer.from('verified base game');
    const file = { path: 'zone/base.ff', size: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex') };
    libs.setImportManifest(hostId, version.id, [file]);
    await libs.uploadImportFile(hostId, version.id, file.path, new Response(bytes).body);
    const ready = await libs.publishImport(hostId, version.id); libraryId = ready.id;
    libs.updateHostLibraryCaches(hostId, [{ libraryVersionId: ready.id, status: 'ready', manifestSha256: ready.manifestSha256 }]);
    const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); heartbeat(hostId, inventory());
    return ready;
}
async function queue(game: 't5' | 't6' = 't5', placement: 'pinned' | 'automatic' = 'pinned') {
    await prepareLibrary(game); remote.map = game === 't5' ? 'zombie_theater' : 'zm_transit';
    const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js');
    const result = queueCreateServer({ game, gameMode: 'zombie', mapSlug: game === 't5' ? 'theater' : 'town',
        hostId: placement === 'pinned' ? hostId : undefined, libraryId, image, plutoKey: ++queuedKeys === 1 ? 'saved-key' : `saved-key-${queuedKeys}` });
    if (!result.ok) throw new Error(result.message);
    return result;
}
async function operationFor(deploymentId: string) {
    const { db } = await import('../../src/lib/server/db/index.js');
    const row = db().prepare('SELECT operation_id FROM native_provisioning WHERE deployment_id=?').get(deploymentId) as { operation_id: string | null };
    const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
    return row?.operation_id ? getHostOperation(row.operation_id)! : null;
}
async function advance() {
    const { processNativeProvisioning } = await import('../../src/lib/server/provisioning/native.js');
    await processNativeProvisioning(() => null);
}
async function acknowledge(deploymentId: string) {
    const operation = await operationFor(deploymentId); if (!operation) throw new Error('Missing durable native operation');
    const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
    reportHostOperation(hostId, operation.id, { state: 'completed', result: { state: remote.state, healthy: remote.state === 'running' } });
    await advance();
    return operation;
}
async function detail(id: string) {
    const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
    return getDeployment(id)!.deployment;
}


async function observations(server: string) {
    const { getServer } = await import('../../src/lib/server/fleet/servers.js');
    const { serverObservation, observationReader } = await import('../../src/lib/server/fleet/status.js');
    const { MonitoringSnapshot } = await import('../../src/lib/server/fleet/monitoring-snapshot.js');
    const row = getServer(server)!;
    return [serverObservation(row), observationReader()(row), new MonitoringSnapshot().observe(row)];
}

async function moveTarget() {
    const hosts = await import('../../src/lib/server/hosts/index.js');
    const libraries = await import('../../src/lib/server/game-files/index.js');
    const target = hosts.createHost({ name: 'Rebalance destination', portMin: 29000, portMax: 29005 });
    hosts.enrollHost(hosts.createEnrollment(target.id).token, inventory());
    libraries.updateHostLibraryCaches(target.id, [{ libraryVersionId: libraryId, status: 'ready',
        manifestSha256: libraries.getLibraryVersion(libraryId)!.manifestSha256 }]);
    return target;
}

describe('rebalance instance migrations', () => {
    it('preserves deployed configuration, patched file versions, identity and credentials on an active source', async () => {
        const { config } = await import('../../src/lib/server/config.js');
        Object.assign(config.rcon, { password: '' });
        const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        const baseline = createScript({ name: 'Deployed baseline', kind: 'baseline', scopeType: 'game', scopeValue: 't5',
            content: Buffer.from('set sv_voice "1"\n') });
        const script = createScript({ name: 'Deployed script', kind: 'script', targetRelPath: 'preserved.gsc',
            scopeType: 'game', scopeValue: 't5', content: Buffer.from('original deployed script') });
        for (const artifact of [baseline, script]) db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(artifact.version.id, artifact.script.id);
        const job = await queue(); await advance();
        const launch = (await operationFor(job.deploymentId))!.payload as { rconPassword: string; files: { path: string; sha256: string }[] };
        await acknowledge(job.deploymentId);
        const patched = uploadNewVersion(script.script.id, Buffer.from('confirmed patch after initial launch'));
        const { recordScript } = await import('../../src/lib/server/fleet/inventory.js');
        recordScript(job.name, 'preserved.gsc', patched.id, patched.sha256, patched.size, 'campaign');
        const newest = uploadNewVersion(script.script.id, Buffer.from('new promotion must not be installed'));
        db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(newest.id, script.script.id);
        const newBaseline = uploadNewVersion(baseline.script.id, Buffer.from('set sv_voice "0"\n'));
        db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(newBaseline.id, baseline.script.id);
        Object.assign(config.rcon, { password: 'new-controller-password' });
        const target = await moveTarget();
        const { captureMoveRecipe, requestInstanceMove, hostMigration } = await import('../../src/lib/server/provisioning/host-drain.js');
        const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
        const original = nativeInstance(job.name)!;
        const recipe = captureMoveRecipe(job.name);
        expect(recipe.files.every(file => file.content === undefined)).toBe(true);
        const id = requestInstanceMove(job.name, target.id, { triggeredBy: 'rebalance:test', recipe });
        await advance();
        const removal = hostMigration(id)!.remove_operation_id!;
        expect(removal).toBeTruthy();
        const { cancelProvisioningJob, canCancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
        expect(canCancelProvisioningJob(id)).toBe(false);
        expect(cancelProvisioningJob(id)).toBe(false);
        const { getDeployment, listDeployments } = await import('../../src/lib/server/provisioning/history.js');
        expect(getDeployment(id)?.canCancel).toBe(false);
        expect(listDeployments().find(item => item.id === id)?.canCancel).toBe(0);
        const { POST } = await import('../../src/routes/api/deployments/[id]/cancel/+server.js');
        const response = POST({ params: { id } });
        expect(response.status).toBe(409);
        expect(await response.json()).toMatchObject({ error: expect.stringContaining('must finish recreating') });
        const { reportHostOperation, getHost } = await import('../../src/lib/server/hosts/index.js');
        reportHostOperation(hostId, removal, { state: 'completed' });
        await advance(); await advance();
        const creation = (await operationFor(id))!;
        expect(creation).toMatchObject({ hostId: target.id, kind: 'instance.create', generation: 2,
            payload: { name: job.name, libraryId, image, rconPassword: launch.rconPassword, env: { PLUTO_SERVER_KEY: 'saved-key' } } });
        const payload = creation.payload as { files: { path: string; sha256: string }[] };
        expect(payload.files).toContainEqual(expect.objectContaining({ path: expect.stringContaining('preserved.gsc'), sha256: patched.sha256 }));
        expect(payload.files.find(file => file.path === 'configs/dedicated_sp.cfg')?.sha256)
            .toBe(launch.files.find(file => file.path === 'configs/dedicated_sp.cfg')?.sha256);
        reportHostOperation(target.id, creation.id, { state: 'completed', result: { state: 'running', healthy: true } });
        await advance();
        expect((await detail(id)).state).toBe('done');
        expect(nativeInstance(job.name)).toMatchObject({ id: original.id, host_id: target.id, generation: 2 });
        expect(getHost(hostId)?.mode).toBe('active');
        expect(JSON.stringify(getDeployment(id)?.recipe)).not.toContain(launch.rconPassword);
    });

    it('rechecks occupancy, target capacity and run admission before removing the source', async () => {
        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
        const target = await moveTarget();
        const { requestInstanceMove, hostMigration } = await import('../../src/lib/server/provisioning/host-drain.js');
        expect(() => requestInstanceMove(job.name, hostId, { triggeredBy: 'rebalance:test' })).toThrow('different destination');
        const id = requestInstanceMove(job.name, target.id, { triggeredBy: 'rebalance:test' });
        remote.players = 2; await advance();
        expect(hostMigration(id)?.remove_operation_id).toBeNull();
        remote.players = 0;
        const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
        heartbeat(target.id, { ...inventory(), memoryAvailable: 1 });
        await advance();
        expect(hostMigration(id)?.remove_operation_id).toBeNull();
        heartbeat(target.id, inventory());
        const { rebalanceAdmission } = await import('../../src/lib/server/rebalancing/admission.js');
        vi.mocked(rebalanceAdmission).mockReturnValueOnce('Run stopped during occupancy check.');
        await advance();
        expect(hostMigration(id)?.remove_operation_id).toBeNull();
        await advance();
        expect(hostMigration(id)?.remove_operation_id).toEqual(expect.any(String));
    });

    it('withdraws only uncommitted moves and preserves their server keys', async () => {
        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
        const target = await moveTarget();
        const { requestInstanceMove } = await import('../../src/lib/server/provisioning/host-drain.js');
        const id = requestInstanceMove(job.name, target.id, { triggeredBy: 'rebalance:test' });
        const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
        expect(cancelProvisioningJob(id)).toBe(true);
        const { requestRebalanceStopForOperation } = await import('../../src/lib/server/rebalancing/admission.js');
        expect(requestRebalanceStopForOperation).toHaveBeenCalledWith(id);
        await advance();
        const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
        expect(getServerKey(job.name)).toBe('saved-key');
        const { db } = await import('../../src/lib/server/db/index.js');
        expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all()).toEqual([]);
    });

    it('refuses to move when recorded deployed bytes cannot be restored', async () => {
        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
        const { captureMoveRecipe } = await import('../../src/lib/server/provisioning/host-drain.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        db().prepare("UPDATE server_scripts SET sha256=? WHERE server=?").run('0'.repeat(64), job.name);
        expect(() => captureMoveRecipe(job.name)).toThrow('deployed bytes are unavailable');
    });
});

describe('runtime configuration at dispatch', () => {
    it.each([
        ['t5', 'shared'], ['t5', 'generated'], ['t6', 'shared'], ['t6', 'generated']
    ] as const)('dispatches %s with a promoted baseline and %s RCON password', async (game, source) => {
        const { config, configFileForGame } = await import('../../src/lib/server/config.js');
        if (source === 'generated') Object.assign(config.rcon, { password: '' });
        const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        const { script, version } = createScript({ name: 'Launch baseline', kind: 'baseline', scopeType: 'game', scopeValue: game,
            content: Buffer.from('set rcon_password "password"\nset sv_voice "1"\n') });
        db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(version.id, script.id);

        const job = await queue(game);
        await advance();
        const operation = await operationFor(job.deploymentId);
        expect(operation).toMatchObject({ kind: 'instance.create', state: 'pending' });
        const payload = operation!.payload as { rconPassword: string; files: { path: string; contentBase64: string; sha256: string }[] };
        expect(payload.rconPassword).toEqual(source === 'shared' ? 'test-password' : expect.stringMatching(/^[a-f0-9]{64}$/));
        const configFile = configFileForGame(game);
        const file = payload.files.find(file => file.path === `configs/${configFile}`)!;
        const content = Buffer.from(file.contentBase64, 'base64');
        expect(content.toString()).toContain(`set rcon_password "${payload.rconPassword}"`);
        expect(content.toString()).not.toContain('set rcon_password "password"');
        const hash = createHash('sha256').update(content).digest('hex');
        expect(file.sha256).toBe(hash);
        const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
        expect(getServerScript(job.name, configFile)).toMatchObject({ sha256: hash, size: content.length, source: 'native-create' });
        const { recipeForDeployment } = await import('../../src/lib/server/fleet/launch-recipes.js');
        expect(recipeForDeployment(job.deploymentId)!.recipe.files).toContainEqual(expect.objectContaining({
            path: configFile, sha256: hash, size: content.length, content: content.toString('base64')
        }));
    });
});

describe('provisioning health handoff', () => {
    it.each(['create', 'start', 'restart', 'rebuild'] as const)('%s publishes verified game health before completion becomes visible', async action => {
        const job = await queue(); await advance();
        const { db } = await import('../../src/lib/server/db/index.js');
        const { getServer } = await import('../../src/lib/server/fleet/servers.js');
        let id = job.deploymentId;
        if (action !== 'create') {
            await acknowledge(id);
            db().prepare('UPDATE servers SET container_state=? WHERE name=?').run(action === 'start' ? 'stopped' : 'running', job.name);
            const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
            const request = requestLifecycle(job.name, action);
            if (!request.ok) throw new Error(request.error);
            id = request.deploymentId;
            await advance();
        }
        const old = new Date(Date.now() - 120_000).toISOString();
        db().prepare(`UPDATE servers SET last_seen_at=?,last_checked_at=?,last_players=0,last_roster='[]',
            last_error='timeout:old process',rcon_failures=2,active_since=NULL,inactive_since=? WHERE name=?`).run(old, old, old, job.name);
        vi.useFakeTimers({ toFake: ['Date'] });
        vi.setSystemTime(Date.now() + 1000);
        const observedAt = new Date().toISOString();
        const roster = [{ num: 0, name: 'Player', guid: '12345678901234567', score: 500, ping: 20 }];
        const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
        vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: true, endpoint: { host: 'agent', port: 28960 }, rawReply: 'verified status',
            status: { players: 1, map: remote.map, gametype: 'zom', roster } });
        const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
        const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
        const { serverObservation } = await import('../../src/lib/server/fleet/status.js');
        const visible: string[] = [];
        const unsubscribe = subscribe(event => {
            if (event.type === 'fleet-invalidated' && getDeployment(id)?.deployment.state === 'done') {
                visible.push(serverObservation(getServer(job.name)!).health);
            }
        });
        try { await acknowledge(id); } finally { unsubscribe(); }
        expect((await detail(id)).state).toBe('done');
        expect(visible).toEqual(['online']);
        expect(getServer(job.name)).toMatchObject({ last_checked_at: observedAt, last_seen_at: observedAt,
            last_players: 1, last_roster: JSON.stringify(roster), last_reply_snippet: 'verified status',
            last_error: null, rcon_failures: 0, active_since: observedAt, inactive_since: null });
        for (const status of await observations(job.name)) expect(status).toMatchObject({ health: 'online', last_players: 1, roster });
    });

    it('completes directly from persisted telemetry without inventing an RCON observation', async () => {
        const job = await queue(); await advance();
        const operation = (await operationFor(job.deploymentId))!;
        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
        reportHostOperation(hostId, operation.id, { state: 'completed', result: { state: 'running', healthy: true } });
        vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(Date.now() + 1000);
        const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js');
        const { gameEvent } = await import('../helpers/telemetry.js');
        const receivedAt = new Date().toISOString();
        saveTelemetry({ ...gameEvent('heartbeat', 0, 0, job.name), receivedAt });
        await advance();
        expect((await detail(job.deploymentId)).state).toBe('done');
        const { getServer } = await import('../../src/lib/server/fleet/servers.js');
        expect(getServer(job.name)).toMatchObject({ last_seen_at: null, inactive_since: receivedAt });
        for (const status of await observations(job.name)) expect(status).toMatchObject({ health: 'online', status_source: 'telemetry', last_seen_at: receivedAt });
    });

    it('rechecks expired verification evidence instead of renewing its timestamps', async () => {
        const job = await queue(); await advance();
        vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(Date.now() + 1000);
        const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
        const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
        vi.mocked(probeVersion).mockImplementationOnce(async () => {
            vi.setSystemTime(Date.now() + 91_000);
            heartbeat(hostId, inventory());
            return 5354;
        });
        await acknowledge(job.deploymentId);
        expect((await detail(job.deploymentId)).state).toBe('awaiting-health');
        const { getServer } = await import('../../src/lib/server/fleet/servers.js');
        expect(getServer(job.name)?.last_seen_at).toBeNull();
        for (const status of await observations(job.name)) expect(status.health).toBe('provisioning');
        await advance();
        expect((await detail(job.deploymentId)).state).toBe('done');
        expect(getServer(job.name)?.last_seen_at).toBe(new Date().toISOString());
    });

    it.each(['instance', 'lifecycle'] as const)('discards verification when the %s changes during later checks', async change => {
        const job = await queue(); await advance();
        const { db } = await import('../../src/lib/server/db/index.js');
        const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
        vi.mocked(probeVersion).mockImplementationOnce(async () => {
            if (change === 'instance') db().prepare('UPDATE native_instances SET generation=generation+1 WHERE server=?').run(job.name);
            else db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(new Date(Date.now() + 1000).toISOString(), job.deploymentId);
            return 5354;
        });
        await acknowledge(job.deploymentId);
        expect((await detail(job.deploymentId)).state).toBe('awaiting-health');
        const { getServer } = await import('../../src/lib/server/fleet/servers.js');
        expect(getServer(job.name)).toMatchObject({ last_seen_at: null, container_checked_at: null });
    });

    it.each(['online', 'offline'] as const)('preserves newer %s monitoring evidence received during version verification', async health => {
        const job = await queue(); await advance();
        vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(Date.now() + 1000);
        const { db } = await import('../../src/lib/server/db/index.js');
        const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
        let newer: string;
        vi.mocked(probeVersion).mockImplementationOnce(async () => {
            vi.setSystemTime(Date.now() + 1000); newer = new Date().toISOString();
            db().prepare(`UPDATE servers SET container_state=?,container_checked_at=?,last_seen_at=?,last_checked_at=?,
                last_players=2,last_roster='[]',last_error=NULL WHERE name=?`).run(health === 'online' ? 'running' : 'stopped', newer, newer, newer, job.name);
            return 5354;
        });
        await acknowledge(job.deploymentId);
        const { getServer } = await import('../../src/lib/server/fleet/servers.js');
        expect(getServer(job.name)).toMatchObject({ last_seen_at: newer!, container_checked_at: newer!, last_players: 2 });
        expect((await detail(job.deploymentId)).state).toBe(health === 'online' ? 'done' : 'awaiting-health');
        if (health === 'online') for (const status of await observations(job.name)) expect(status).toMatchObject({ health: 'online', last_players: 2 });
    });

    it('keeps a slow restart Starting until verification succeeds and surfaces terminal failures', async () => {
        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
        const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
        const request = requestLifecycle(job.name, 'restart');
        if (!request.ok) throw new Error(request.error);
        await advance();
        vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(Date.now() + 20_000);
        for (const status of await observations(job.name)) expect(status.health).toBe('starting');
        const { db } = await import('../../src/lib/server/db/index.js');
        for (const state of ['failed', 'cancelled', 'interrupted']) {
            db().prepare('UPDATE server_deployments SET state=? WHERE id=?').run(state, request.deploymentId);
            for (const status of await observations(job.name)) expect(status.health).toBe('offline');
        }
        db().prepare("UPDATE server_deployments SET state='running' WHERE id=?").run(request.deploymentId);
        await acknowledge(request.deploymentId);
        expect((await detail(request.deploymentId)).state).toBe('done');
        for (const status of await observations(job.name)) expect(status.health).toBe('online');
    });
});

describe('native instance provisioning', () => {
    it('recognizes an unplaced native intent and resumes it once after a controller restart', async () => {
        const job = await queue();
        const { db, closeDb } = await import('../../src/lib/server/db/index.js');
        const { nativeDeployment, nativeInstance } = await import('../../src/lib/server/backends/native.js');
        const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
        expect(job.port).toBeNull();
        expect(nativeDeployment(job.deploymentId)).toBe(true);
        expect(nativeInstance(job.name)).toBeNull();
        expect(getDeployment(job.deploymentId)).toMatchObject({ backend: 'native', canCancelCreation: true });
        expect(db().prepare('SELECT deployment_id FROM native_creation_intents').all()).toEqual([{ deployment_id: job.deploymentId }]);
        expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);

        closeDb(); vi.resetModules();
        const backend = await import('../../src/lib/server/backends/native.js');
        expect(backend.nativeDeployment(job.deploymentId)).toBe(true);
        await advance();
        const original = await operationFor(job.deploymentId);
        expect(original).toMatchObject({ kind: 'instance.create', payload: { name: job.name, port: 28960 } });
        await advance();
        expect((await operationFor(job.deploymentId))?.id).toBe(original!.id);
        const restartedDb = (await import('../../src/lib/server/db/index.js')).db();
        expect(restartedDb.prepare("SELECT id FROM ec_host_operations WHERE kind='instance.create'").all()).toHaveLength(1);
        expect(restartedDb.prepare('SELECT protocol FROM ec_host_ports ORDER BY protocol').all()).toEqual([{ protocol: 'tcp' }, { protocol: 'udp' }]);
    });

    it('still rejects missing keys and stale lobby identities before creating an intent', async () => {
        await prepareLibrary();
        const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
        const input = { game: 't5' as const, gameMode: 'zombie' as const, mapSlug: 'theater', hostId, libraryId, image };
        expect(queueNativeCreation(input)).toMatchObject({ ok: false, message: expect.stringContaining('server key') });
        expect(queueNativeCreation({ ...input, plutoKey: 'saved-key', expectedName: 't5_zombie_theater_2' }))
            .toMatchObject({ ok: false });
        const { db } = await import('../../src/lib/server/db/index.js');
        expect(db().prepare('SELECT deployment_id FROM native_creation_intents').all()).toEqual([]);
        expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]);
        expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
    });

    it('accepts creation cleanup through the provisioning API and rejects an already completed creation', async () => {
        const job = await queue();
        const { POST } = await import('../../src/routes/api/provisioning/[id]/cancel-creation/+server.js');
        const response = POST({ params: { id: job.deploymentId } });
        expect(response.status).toBe(200);
        expect(await response.json()).toMatchObject({ ok: true, cleanupId: expect.any(String) });
        const next = await queue(); await advance(); await acknowledge(next.deploymentId);
        expect(POST({ params: { id: next.deploymentId } }).status).toBe(409);
        expect(POST({ params: { id: job.deploymentId } }).status).toBe(200);
    });
    it('cancels an unsubmitted creation locally and returns the same cleanup on repeated requests', async () => {
        const job = await queue();
        const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
        const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        expect(getDeployment(job.deploymentId)!.canCancelCreation).toBe(true);
        const cleanup = cancelCreation(job.deploymentId)!;
        expect(cancelCreation(job.deploymentId)).toBe(cleanup);
        expect((await detail(job.deploymentId)).state).toBe('cancelled');
        expect((await detail(cleanup)).state).toBe('done');
        expect(getDeployment(job.deploymentId)!.cancellation?.state).toBe('done');
        expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
        await advance();
        expect(await operationFor(job.deploymentId)).toBeNull();
        expect(await operationFor(cleanup)).toBeNull();
    });

    it('cancels a submitted creation without occupancy checks, suppresses replay, and releases resources only after cleanup', async () => {
        const job = await queue(); await advance(); remote.players = 4;
        const original = (await operationFor(job.deploymentId))!;
        const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
        const { heartbeat, pendingHostOperations, reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
        heartbeat(hostId, { ...inventory(), capabilities: ['instance.cancel-create'] });
        // A previously delivered command might still be executing when cancellation arrives.
        expect(pendingHostOperations(hostId).map(op => op.id)).toContain(original.id);
        const cleanup = cancelCreation(job.deploymentId)!;
        const removal = (await operationFor(cleanup))!;
        expect(removal.kind).toBe('instance.cancel-create');
        expect(removal.payload).toEqual({ createOperationId: original.id });
        expect(cancelCreation(job.deploymentId)).toBe(cleanup);
        expect(pendingHostOperations(hostId).map(op => op.id)).toContain(removal.id);
        expect(pendingHostOperations(hostId).map(op => op.id)).not.toContain(original.id);
        await advance();
        expect(nativeRequest).not.toHaveBeenCalled();
        expect(db().prepare('SELECT * FROM ec_host_ports').all()).toHaveLength(2);
        reportHostOperation(hostId, original.id, { state: 'completed', result: { healthy: true } });
        await advance();
        expect((await detail(job.deploymentId)).state).toBe('cancelled');
        expect(nativeRequest).not.toHaveBeenCalled();
        reportHostOperation(hostId, removal.id, { state: 'completed', result: { exists: false } });
        await advance();
        expect((await detail(cleanup)).state).toBe('done');
        expect((await detail(job.deploymentId)).server_deleted_at).toBeTruthy();
        expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
        expect(db().prepare('SELECT * FROM servers WHERE name=?').get(job.name)).toBeUndefined();
        expect(db().prepare('SELECT retired_at FROM native_instances WHERE id=?').get(original.instanceId)).toMatchObject({ retired_at: expect.any(String) });
    });

    it('holds cancellation for an old or offline host without delivering an unsupported command or losing reservations', async () => {
        const job = await queue(); await advance();
        const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
        const { heartbeat, pendingHostOperations } = await import('../../src/lib/server/hosts/index.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        const cleanup = cancelCreation(job.deploymentId)!;
        expect(pendingHostOperations(hostId).filter(op => op.kind.startsWith('instance.'))).toEqual([]);
        await advance();
        expect((await detail(cleanup)).stage).toContain('Update the host agent');
        expect((await detail(cleanup)).stage).toContain('Automatic updates are not enabled');
        heartbeat(hostId, { ...inventory(), capabilities: ['agent.update'],
            agentUpdate: { enabled: true, state: 'deferred', currentVersion: '1.2.2', targetVersion: '1.3.0' } });
        await advance();
        expect((await detail(cleanup)).stage).toContain('New command delivery is paused');
        expect((await detail(cleanup)).stage).toContain('automatic updater can run');
        db().prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?').run('2000-01-01T00:00:00.000Z', hostId);
        await advance();
        expect((await detail(cleanup)).stage).toContain('unavailable');
        expect(db().prepare('SELECT * FROM ec_host_ports').all()).toHaveLength(2);
        heartbeat(hostId, { ...inventory(), capabilities: ['instance.cancel-create'] });
        expect(pendingHostOperations(hostId).filter(op => op.kind === 'instance.cancel-create')).toHaveLength(1);
        await acknowledge(cleanup);
        expect((await detail(cleanup)).state).toBe('done');
    });

    it('replaces an occupancy-blocked removal and ignores an outstanding creation health reply', async () => {
        const job = await queue(); await advance();
        const operation = (await operationFor(job.deploymentId))!;
        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
        reportHostOperation(hostId, operation.id, { state: 'completed', result: { healthy: true } });
        const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
        let release!: (value: never) => void;
        vi.mocked(nativeRequest).mockImplementationOnce(() => new Promise(resolve => { release = resolve; }));
        const pending = advance();
        try {
            const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
            const waitingRemoval = destroyServer(job.name).deploymentId!;
            const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
            const cleanup = cancelCreation(job.deploymentId)!;
            expect(cleanup).toBeTruthy();
            expect((await detail(waitingRemoval)).state).toBe('cancelled');
            release({ healthy: true, state: 'running' } as never);
            await pending;
            expect((await detail(job.deploymentId)).state).toBe('cancelled');
            expect((await detail(cleanup)).state).toBe('running');
        } finally { release({ healthy: true, state: 'running' } as never); await pending; }
    });

    it('does not offer creation cancellation after success or against a replacement instance', async () => {
        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
        const { canCancelCreation, cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
        expect(canCancelCreation(job.deploymentId)).toBe(false);
        expect(cancelCreation(job.deploymentId)).toBeNull();
        const { db } = await import('../../src/lib/server/db/index.js');
        db().prepare("UPDATE server_deployments SET state='awaiting-health' WHERE id=?").run(job.deploymentId);
        db().prepare('UPDATE native_instances SET generation=generation+1 WHERE server=?').run(job.name);
        expect(cancelCreation(job.deploymentId)).toBeNull();
    });

    it('retains every native launch checkpoint and readiness check through completion and UI validation', async () => {
        const job = await queue();
        const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
        const { isProvisioningDetail, provisioningGroups } = await import('../../src/lib/ui/provisioning-presentation.js');
        const queued = getDeployment(job.deploymentId)!;
        expect(queued.progress?.phase).toBe('eligibility');
        expect(queued.progress?.blocker).toBeNull();
        expect(queued.backend).toBe('native');
        await advance();
        expect(getDeployment(job.deploymentId)!.progress?.recipeStatus).toBe('attempted');
        await acknowledge(job.deploymentId);
        const result = getDeployment(job.deploymentId)!;
        expect(result.deployment.state).toBe('done');
        expect(isProvisioningDetail(result, job.deploymentId)).toBe(true);
        expect(result.progress?.checks.map(check => check.key)).toEqual(['container', 'game', 'map', 'plutonium-release', 'plutonium-version']);
        expect(result.progress?.checks.every(check => check.result === 'pass')).toBe(true);
        expect(result.progress?.checks[0].source).toBe('agent');
        const steps = provisioningGroups(result).flatMap(group => group.steps);
        expect(steps.every(step => step.state === 'done')).toBe(true);
        expect(result.agentOperation?.state).toBe('completed');
    });

    it('records agent receipts immediately and only once, even before the provisioning worker runs', async () => {
        const job = await queue(); await advance();
        const operation = (await operationFor(job.deploymentId))!;
        const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
        for (const state of ['accepted', 'accepted', 'running', 'completed', 'completed']) {
            reportHostOperation(hostId, operation.id, { state, result: { state: 'running', healthy: true } });
        }
        const result = getDeployment(job.deploymentId)!;
        expect(result.deployment.state).toBe('running');
        expect(result.agentOperation?.state).toBe('completed');
        expect(result.events.filter(event => event.kind === 'agent')).toHaveLength(3);
        expect(result.progress?.checkpoints.map(point => point.key)).toEqual(expect.arrayContaining(['agent-accepted', 'agent-running', 'agent-completed']));
        expect(result.progress?.command?.state).toBe('accepted');
    });

    it('records an outstanding container check before awaiting the host and admits other jobs meanwhile', async () => {
        const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
        saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 });
        const first = await queue(); await advance();
        const operation = (await operationFor(first.deploymentId))!;
        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
        reportHostOperation(hostId, operation.id, { state: 'completed', result: { healthy: true } });
        const second = await queue();
        const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
        let release!: (value: never) => void;
        vi.mocked(nativeRequest).mockImplementationOnce(() => new Promise(resolve => { release = resolve; }));
        const pending = advance();
        try {
            const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
            expect(getDeployment(first.deploymentId)!.progress?.phase).toBe('verify-container');
            await vi.waitFor(async () => {
                expect((await operationFor(second.deploymentId))?.kind).toBe('instance.create');
            }, { timeout: 5_000 });
        } finally {
            release({ healthy: true, state: 'running' } as never);
            await pending;
        }
    });

    it('admits a later request on the next scheduler pass while an earlier native probe is pending', async () => {
        const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
        saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 });
        const first = await queue(); await advance();
        const operation = (await operationFor(first.deploymentId))!;
        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
        reportHostOperation(hostId, operation.id, { state: 'completed', result: { healthy: true } });
        const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
        let release!: (value: never) => void;
        vi.mocked(nativeRequest).mockImplementationOnce(() => new Promise(resolve => { release = resolve; }));
        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
        await processProvisioning({ wait: false });
        try {
            const second = await queue();
            await processProvisioning({ wait: false });
            await vi.waitFor(async () => {
                expect((await operationFor(second.deploymentId))?.kind).toBe('instance.create');
            }, { timeout: 5_000 });
            expect(vi.mocked(nativeRequest)).toHaveBeenCalledTimes(1);
        } finally {
            release({ healthy: true, state: 'running' } as never);
            await processProvisioning();
        }
    });

    it('ignores a late health result after removal retires the operation', async () => {
        const job = await queue(); await advance();
        const operation = (await operationFor(job.deploymentId))!;
        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
        reportHostOperation(hostId, operation.id, { state: 'completed', result: { healthy: true } });
        const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
        let release!: (value: never) => void;
        vi.mocked(nativeRequest).mockImplementationOnce(() => new Promise(resolve => { release = resolve; }));
        const pending = advance();
        try {
            await vi.waitFor(() => expect(release).toBeTypeOf('function'));
            const { db } = await import('../../src/lib/server/db/index.js');
            db().prepare("UPDATE server_deployments SET state='cancelled',server_deleted_at=? WHERE id=?")
                .run(new Date().toISOString(), job.deploymentId);
            const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
            const before = getDeployment(job.deploymentId)!;
            release({ healthy: true, state: 'running' } as never);
            await pending;
            const after = getDeployment(job.deploymentId)!;
            expect(after.deployment.state).toBe('cancelled');
            expect(after.progress).toEqual(before.progress);
            expect(after.activity).toEqual(before.activity);
        } finally { release({ healthy: true, state: 'running' } as never); await pending; }
    });

    it('records map mismatch evidence while retaining passed container and game checks', async () => {
        const job = await queue(); await advance(); remote.map = 'zombie_moon';
        await acknowledge(job.deploymentId);
        const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
        const result = getDeployment(job.deploymentId)!;
        expect(result.deployment.state).toBe('awaiting-health');
        expect(result.progress?.phase).toBe('verify-map');
        expect(result.progress?.checks).toEqual(expect.arrayContaining([
            expect.objectContaining({ key: 'container', result: 'pass' }),
            expect.objectContaining({ key: 'game', result: 'pass' }),
            expect.objectContaining({ key: 'map', result: 'waiting', actual: 'zombie_moon', expected: 'zombie_theater' })
        ]));
    });

    it('selects the latest game files automatically and rejects stale explicit selections', async () => {
        const { nativeSelection } = await import('../../src/lib/server/provisioning/native.js');
        expect(() => nativeSelection({ hostId, image }, 't5')).toThrow('Upload T5 game files');
        const old = await prepareLibrary('t5');
        expect(nativeSelection({ hostId, image }, 't5').libraryId).toBe(old.id);
        const latest = await prepareLibrary('t5');
        expect(nativeSelection({ hostId, image }, 't5').libraryId).toBe(latest.id);
        expect(() => nativeSelection({ hostId, image, libraryId: old.id }, 't5')).toThrow('Refresh the launch preview');
    });

    it('rejects a launch preview made before newer game files were published without reserving a server', async () => {
        await prepareLibrary('t5');
        const { previewNativeCreation, queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
        const preview = previewNativeCreation({ hostId, image, game: 't5', mapSlug: 'theater' });
        await prepareLibrary('t5');
        const result = queueNativeCreation({ game: 't5', gameMode: 'zombie', mapSlug: 'theater', previewId: preview.id, plutoKey: 'new-key' });
        expect(result).toMatchObject({ ok: false, message: expect.stringContaining('Refresh the launch preview') });
        const { db } = await import('../../src/lib/server/db/index.js');
        expect(db().prepare('SELECT COUNT(*) AS count FROM native_instances').get()).toEqual({ count: 0 });
        expect(db().prepare('SELECT COUNT(*) AS count FROM ec_host_ports').get()).toEqual({ count: 0 });
    });

    it.each(['pinned', 'automatic'] as const)('keeps an unplaced %s launch pinned to its accepted files during cleanup', async placement => {
        const job = await queue('t5', placement);
        const acceptedId = libraryId;
        const latest = await prepareLibrary('t5');
        expect(latest.id).not.toBe(acceptedId);
        const { db } = await import('../../src/lib/server/db/index.js');
        const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
        const { reconcileHostLibraryCleanup, cleanupControllerLibraries, getLibraryVersion, updateHostLibraryCaches }
            = await import('../../src/lib/server/game-files/index.js');
        const accepted = getLibraryVersion(acceptedId)!;
        heartbeat(hostId, { ...inventory(), capabilities: ['library.remove'] });
        expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]);
        reconcileHostLibraryCleanup(hostId);
        expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='library.remove' AND json_extract(payload,'$.libraryVersionId')=?")
            .all(acceptedId)).toEqual([]);

        // The unplaced intent also protects the controller's copy when every
        // host reports it absent; its accepted version remains downloadable.
        updateHostLibraryCaches(hostId, [{ libraryVersionId: acceptedId, status: 'missing' }]);
        await cleanupControllerLibraries();
        expect(db().prepare('SELECT version_id FROM game_file_cleanup WHERE version_id=?').get(acceptedId)).toBeUndefined();
        const { stat } = await import('node:fs/promises');
        expect((await stat(join(fixture.root, 'libraries', 't5', acceptedId, 'zone/base.ff'))).isFile()).toBe(true);
        updateHostLibraryCaches(hostId, [{ libraryVersionId: acceptedId, status: 'ready', manifestSha256: accepted.manifestSha256 }]);
        await advance();
        expect((await operationFor(job.deploymentId))?.payload.libraryId).toBe(acceptedId);
        const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
        expect(nativeInstance(job.name)?.library_id).toBe(acceptedId);
    });

    it.each(['t5', 't6'] as const)('queues and verifies a %s deployment through its host agent', async game => {
        const job = await queue(game); await advance();
        const operation = await operationFor(job.deploymentId);
        expect(job.port).toBeNull();
        expect(operation).toMatchObject({ kind: 'instance.create', hostId, generation: 1, state: 'pending', payload: { game, libraryId, image, port: 28960 } });
        expect(operation?.payload.files).toEqual(expect.arrayContaining([expect.objectContaining({ path: 'configs/electric_cherry_map.cfg' })]));
        await acknowledge(job.deploymentId);
        expect((await detail(job.deploymentId)).state).toBe('done');
        const { db } = await import('../../src/lib/server/db/index.js');
        expect(db().prepare('SELECT protocol FROM ec_host_ports WHERE instance_id=? ORDER BY protocol').all(operation!.instanceId)).toEqual([{ protocol: 'tcp' }, { protocol: 'udp' }]);
    });
    it('reconciles the same operation after controller restart and an ambiguous remote result', async () => {
        const job = await queue(); await advance();
        const original = (await operationFor(job.deploymentId))!;
        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
        reportHostOperation(hostId, original.id, { state: 'unknown', error: 'Reply lost after execution' });
        const { closeDb, db } = await import('../../src/lib/server/db/index.js'); closeDb();
        const { interruptDeployments } = await import('../../src/lib/server/provisioning/history.js'); interruptDeployments();
        await advance(); await advance();
        expect((await operationFor(job.deploymentId))!.id).toBe(original.id);
        expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.create'").get()).toEqual({ count: 1 });
        expect((await detail(job.deploymentId)).state).toBe('running');
        await acknowledge(job.deploymentId);
        expect((await detail(job.deploymentId)).state).toBe('done');
    });
    it('holds disconnected instances as unknown without replacement and resumes the original operation', async () => {
        const job = await queue(); await advance();
        const original = (await operationFor(job.deploymentId))!;
        const { db } = await import('../../src/lib/server/db/index.js');
        db().prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?').run('2000-01-01T00:00:00Z', hostId);
        await advance();
        expect(db().prepare('SELECT container_state FROM servers WHERE name=?').get(job.name)).toEqual({ container_state: 'unknown' });
        expect((await operationFor(job.deploymentId))!.id).toBe(original.id);
        const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); heartbeat(hostId, inventory());
        await acknowledge(job.deploymentId);
        expect((await detail(job.deploymentId)).state).toBe('done');
    });
    it('rebuilds only an empty instance and advances its generation after the acknowledged command', async () => {
        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
        const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
        const rebuild = queueNativeRebuild(job.name, 'test'); if (!rebuild.ok) throw new Error(rebuild.message);
        remote.players = 2; await advance();
        expect(await operationFor(rebuild.deploymentId)).toBeNull();
        remote.players = 0; await advance();
        const operation = (await operationFor(rebuild.deploymentId))!;
        expect(operation).toMatchObject({ kind: 'instance.rebuild', generation: 1, payload: { nextGeneration: 2, libraryId } });
        const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
        expect(nativeInstance(job.name)?.generation).toBe(1);
        await acknowledge(rebuild.deploymentId);
        expect(nativeInstance(job.name)?.generation).toBe(2);
        expect((await detail(rebuild.deploymentId)).state).toBe('done');
        await advance();
        expect(nativeInstance(job.name)?.generation).toBe(2);
    });
    it('carries confirmed admin override through serialized rebuild preparation', async () => {
        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
        const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
        heartbeat(hostId, { ...inventory(), capabilities: ['instance.safety-override'] });
        const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
        const rebuild = queueNativeRebuild(job.name, 'test'); if (!rebuild.ok) throw new Error(rebuild.message);
        remote.players = 2; await advance();
        expect(await operationFor(rebuild.deploymentId)).toBeNull();
        const { enableSafetyOverride } = await import('../../src/lib/server/provisioning/safety-override.js');
        expect(enableSafetyOverride(rebuild.deploymentId, { id: 'admin', username: 'admin' })).toEqual({ ok: true });
        await advance();
        expect(await operationFor(rebuild.deploymentId)).toMatchObject({ kind: 'instance.rebuild', generation: 1,
            payload: { nextGeneration: 2, libraryId, safetyOverride: true } });
        await acknowledge(rebuild.deploymentId);
        expect((await detail(rebuild.deploymentId)).state).toBe('done');
    });

    it('holds a rebuild when players join while its launch files are being prepared', async () => {
        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
        const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
        const rebuild = queueNativeRebuild(job.name, 'test'); if (!rebuild.ok) throw new Error(rebuild.message);
        const preparation = await import('../../src/lib/server/provisioning/launch-preparation.js');
        const prepare = preparation.prepareNativeLaunch;
        let started!: () => void;
        let release!: () => void;
        const ready = new Promise<void>(resolve => { started = resolve; });
        const gate = new Promise<void>(resolve => { release = resolve; });
        const spy = vi.spyOn(preparation, 'prepareNativeLaunch').mockImplementationOnce(async input => {
            const result = await prepare(input);
            started(); await gate;
            return result;
        });
        const pending = advance();
        try {
            await ready;
            remote.players = 2;
            release(); await pending;
            expect(await operationFor(rebuild.deploymentId)).toBeNull();
            remote.players = 0;
            await advance();
            expect((await operationFor(rebuild.deploymentId))?.kind).toBe('instance.rebuild');
        } finally { release(); await pending; spy.mockRestore(); }
    });
    it('finishes native rebuild verification without waiting for its own recovery slot to be released', async () => {
        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
        const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
        const rebuild = queueNativeRebuild(job.name, 'test'); if (!rebuild.ok) throw new Error(rebuild.message);
        await advance();
        const { db } = await import('../../src/lib/server/db/index.js');
        const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
        const actual = await vi.importActual<typeof import('../../src/lib/server/integrations/plutonium.js')>('../../src/lib/server/integrations/plutonium.js');
        const backend = await import('../../src/lib/server/backends/native.js');
        vi.spyOn(backend, 'nativeHealth').mockResolvedValue({ ok: true, state: 'running' });
        vi.mocked(probeVersion).mockImplementationOnce(actual.probeVersion);
        // Version evidence must follow the submitted command, even on a fast test host.
        db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(new Date(Date.now() - 1000).toISOString(), rebuild.deploymentId);
        db().prepare('UPDATE restart_rollout SET requested_at=? WHERE operation_id=?').run(new Date(Date.now() - 1000).toISOString(), rebuild.deploymentId);
        await acknowledge(rebuild.deploymentId);
        expect((await detail(job.deploymentId)).state).toBe('done');
        expect((await detail(rebuild.deploymentId)).state).toBe('done');
        expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE operation_id=?').get(rebuild.deploymentId))
            .toEqual({ healthy_at: expect.any(String) });
        expect(actual.runningVersions().find(version => version.server === job.name)?.revision).toBe(5354);
        const { getServer } = await import('../../src/lib/server/fleet/servers.js');
        const { serverObservation } = await import('../../src/lib/server/fleet/status.js');
        const { fleetOperations } = await import('../../src/lib/server/fleet/overview.js');
        expect(serverObservation(getServer(job.name)!).health).not.toBe('provisioning');
        expect(fleetOperations().filter(operation => operation.server === job.name && operation.state !== 'done')).toEqual([]);
    });
    it('resumes a rebuild waiting on version evidence after controller restart without rebuilding again', async () => {
        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
        const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
        const rebuild = queueNativeRebuild(job.name, 'test'); if (!rebuild.ok) throw new Error(rebuild.message);
        await advance();
        const operation = (await operationFor(rebuild.deploymentId))!;
        const { db, closeDb } = await import('../../src/lib/server/db/index.js');
        const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
        const { rconPlutoniumVersion } = await import('../../src/lib/server/integrations/rcon.js');
        const actual = await vi.importActual<typeof import('../../src/lib/server/integrations/plutonium.js')>('../../src/lib/server/integrations/plutonium.js');
        const backend = await import('../../src/lib/server/backends/native.js');
        vi.spyOn(backend, 'nativeHealth').mockResolvedValue({ ok: true, state: 'running' });
        const submittedAt = new Date(Date.now() - 1000).toISOString();
        db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(submittedAt, rebuild.deploymentId);
        db().prepare('UPDATE restart_rollout SET requested_at=? WHERE operation_id=?').run(submittedAt, rebuild.deploymentId);
        vi.mocked(rconPlutoniumVersion).mockResolvedValueOnce({ revision: null, error: 'Version response unavailable' });
        vi.mocked(probeVersion).mockImplementationOnce(actual.probeVersion);
        await acknowledge(rebuild.deploymentId);
        expect(await detail(rebuild.deploymentId)).toMatchObject({ state: 'awaiting-health', stage: 'Waiting to verify the Plutonium runtime revision.' });
        expect(db().prepare('SELECT healthy_at FROM restart_rollout WHERE operation_id=?').get(rebuild.deploymentId)).toEqual({ healthy_at: null });
        closeDb();
        vi.mocked(probeVersion).mockImplementationOnce(actual.probeVersion);
        await advance();
        expect((await detail(rebuild.deploymentId)).state).toBe('done');
        expect((await operationFor(rebuild.deploymentId))!.id).toBe(operation.id);
        expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'").get()).toEqual({ count: 1 });
    });
    it('drains by reserving destination capacity, waiting for empty lobbies, and rebuilding once on another host', async () => {
        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
        const { updateHost, createHost, createEnrollment, enrollHost, reportHostOperation, getHost } = await import('../../src/lib/server/hosts/index.js');
        updateHost(hostId, { mode: 'draining' });
        const { nativeSelection } = await import('../../src/lib/server/provisioning/native.js');
        expect(() => nativeSelection({ hostId, libraryId, image }, 't5')).toThrow(/draining/);
        const { db } = await import('../../src/lib/server/db/index.js');
        const { getLibraryVersion, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js');
        await processProvisioning();
        const migration = db().prepare("SELECT id FROM server_deployments WHERE triggered_by='host-drain' AND server=?").get(job.name) as { id: string };
        expect((await detail(migration.id)).stage).toContain('No host has capacity');
        expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all()).toEqual([]);
        const target = createHost({ name: 'Destination', portMin: 29000, portMax: 29005 });
        enrollHost(createEnrollment(target.id).token, inventory());
        updateHostLibraryCaches(target.id, [{ libraryVersionId: libraryId, status: 'ready', manifestSha256: getLibraryVersion(libraryId)!.manifestSha256 }]);
        remote.players = 2; await processProvisioning();
        expect((await detail(migration.id)).stage).toContain('2 player(s)');
        expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all()).toEqual([]);
        remote.players = 0; await processProvisioning();
        const remove = db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").get() as { id: string };
        expect(remove).toBeTruthy();
        expect(db().prepare('SELECT protocol FROM ec_host_ports WHERE host_id=?').all(target.id)).toHaveLength(2);
        expect(() => updateHost(target.id, { mode: 'draining' })).toThrow('incoming');
        reportHostOperation(hostId, remove.id, { state: 'unknown' });
        const { closeDb } = await import('../../src/lib/server/db/index.js');
        closeDb();
        await processProvisioning();
        expect(db().prepare('SELECT host_id FROM native_instances WHERE server=?').get(job.name)).toEqual({ host_id: hostId });
        reportHostOperation(hostId, remove.id, { state: 'completed' });
        await processProvisioning();
        expect(db().prepare('SELECT host_id,generation FROM native_instances WHERE server=?').get(job.name)).toEqual({ host_id: target.id, generation: 2 });
        expect(db().prepare('SELECT * FROM ec_host_ports WHERE host_id=?').all(hostId)).toEqual([]);
        expect(getHost(hostId)?.mode).toBe('draining');
        const { cancelProvisioning } = await import('../../src/lib/server/provisioning/requests.js');
        expect(cancelProvisioning(migration.id)).toBe(false);
        await processProvisioning();
        const create = (await operationFor(migration.id))!;
        expect(create).toMatchObject({ hostId: target.id, kind: 'instance.create', generation: 2, payload: { name: job.name, port: 29000 } });
        reportHostOperation(target.id, create.id, { state: 'completed', result: { state: 'running', healthy: true } });
        await processProvisioning();
        expect((await detail(migration.id)).state).toBe('done');
        expect(getHost(hostId)?.mode).toBe('disabled');
        expect(db().prepare('SELECT generation FROM native_instances WHERE server=?').get(job.name)).toEqual({ generation: 2 });
        await advance();
        expect((await operationFor(migration.id))?.id).toBe(create.id);
        expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all()).toHaveLength(1);
        expect(getLibraryVersion(libraryId)?.state).toBe('ready');
    });

    it('reserves migration capacity before source removal and retains it while a replacement awaits startup', async () => {
        const first = await queue(); await advance(); await acknowledge(first.deploymentId);
        const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js');
        const second = queueCreateServer({ game: 't5', gameMode: 'zombie', mapSlug: 'theater', hostId, libraryId, image, plutoKey: 'second-key' });
        if (!second.ok) throw new Error(second.message);
        await advance(); await acknowledge(second.deploymentId);
        const { createHost, createEnrollment, enrollHost, updateHost, reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
        const { getLibraryVersion, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js');
        const target = createHost({ name: 'One free slot' });
        enrollHost(createEnrollment(target.id).token, { ...inventory(), memoryTotal: 4 * 1024 ** 3, memoryAvailable: 1.75 * 1024 ** 3 });
        updateHostLibraryCaches(target.id, [{ libraryVersionId: libraryId, status: 'ready', manifestSha256: getLibraryVersion(libraryId)!.manifestSha256 }]);
        updateHost(hostId, { mode: 'draining' });
        await advance();
        const { db } = await import('../../src/lib/server/db/index.js');
        const removals = () => db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all() as { id: string }[];
        expect(removals()).toHaveLength(1);
        expect(db().prepare('SELECT instance_id FROM ec_host_ports WHERE host_id=?').all(target.id)).toHaveLength(2);
        const { choosePlacement } = await import('../../src/lib/server/hosts/capacity.js');
        expect(() => choosePlacement({ hostId: target.id, libraryId, image })).toThrow('memory capacity');
        reportHostOperation(hostId, removals()[0].id, { state: 'completed' });
        await advance();
        expect(() => choosePlacement({ hostId: target.id, libraryId, image })).toThrow('memory capacity');
        expect(removals()).toHaveLength(1);
    });

    it('retries definite migration failures without replaying successful source removal', async () => {
        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
        const { createHost, createEnrollment, enrollHost, updateHost, reportHostOperation, getHost } = await import('../../src/lib/server/hosts/index.js');
        const { getLibraryVersion, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js');
        const target = createHost({ name: 'Destination' });
        enrollHost(createEnrollment(target.id).token, inventory());
        updateHostLibraryCaches(target.id, [{ libraryVersionId: libraryId, status: 'ready', manifestSha256: getLibraryVersion(libraryId)!.manifestSha256 }]);
        updateHost(hostId, { mode: 'draining' });
        await advance();
        const { db } = await import('../../src/lib/server/db/index.js');
        const migration = db().prepare('SELECT deployment_id,remove_operation_id FROM ec_host_migrations').get() as { deployment_id: string; remove_operation_id: string };
        const expireRetry = (id: string) => db().prepare('UPDATE ec_host_operations SET updated_at=? WHERE id=?').run(new Date(Date.now() - 61_000).toISOString(), id);
        reportHostOperation(hostId, migration.remove_operation_id, { state: 'failed', error: 'Docker unavailable before removal' });
        await advance();
        expect(db().prepare('SELECT host_id FROM native_instances WHERE server=?').get(job.name)).toEqual({ host_id: hostId });
        expireRetry(migration.remove_operation_id);
        await advance(); await advance();
        const retry = db().prepare('SELECT remove_operation_id FROM ec_host_migrations').get() as { remove_operation_id: string };
        expect(retry.remove_operation_id).not.toBe(migration.remove_operation_id);
        reportHostOperation(hostId, retry.remove_operation_id, { state: 'completed' });
        await advance(); await advance();
        const creation = (await operationFor(migration.deployment_id))!;
        reportHostOperation(target.id, creation.id, { state: 'failed', error: 'Docker unavailable before creation' });
        expireRetry(creation.id);
        await advance(); await advance();
        const replacement = (await operationFor(migration.deployment_id))!;
        expect(replacement.id).not.toBe(creation.id);
        expect(replacement).toMatchObject({ kind: 'instance.create', hostId: target.id, generation: 2 });
        reportHostOperation(target.id, replacement.id, { state: 'completed' });
        await advance();
        expect(getHost(hostId)?.mode).toBe('disabled');
        expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.remove'").all()).toHaveLength(2);
    });
    it('disables an empty draining host and redirects its unplaced launches to another host', async () => {
        const job = await queue();
        const { createHost, createEnrollment, enrollHost, updateHost, getHost } = await import('../../src/lib/server/hosts/index.js');
        const { getLibraryVersion, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js');
        const target = createHost({ name: 'Destination' });
        enrollHost(createEnrollment(target.id).token, inventory());
        updateHostLibraryCaches(target.id, [{ libraryVersionId: libraryId, status: 'ready', manifestSha256: getLibraryVersion(libraryId)!.manifestSha256 }]);
        updateHost(hostId, { mode: 'draining' });
        expect(() => updateHost(hostId, { mode: 'active' })).toThrow('finish draining');
        await advance();
        expect(getHost(hostId)?.mode).toBe('disabled');
        expect(await operationFor(job.deploymentId)).toMatchObject({ hostId: target.id, kind: 'instance.create' });
    });

    it('rejects the removed manual disable API action', async () => {
        const { PATCH } = await import('../../src/routes/api/hosts/[id]/+server.js');
        const response = await PATCH({ params: { id: hostId }, request: new Request(`https://ec.test/api/hosts/${hostId}`, {
            method: 'PATCH', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ mode: 'disabled' })
        }) });
        expect(response.status).toBe(400);
        expect(await response.json()).toMatchObject({ error: expect.stringContaining('disable automatically') });
        const { getHost } = await import('../../src/lib/server/hosts/index.js');
        expect(getHost(hostId)?.mode).toBe('active');
    });
    it('stages campaign bytes while occupied and records agent before images only after safe activation', async () => {
        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
        const { db } = await import('../../src/lib/server/db/index.js');
        const before = Buffer.from('original bytes on the host'); const after = Buffer.from('new campaign bytes');
        db().prepare("INSERT INTO deployments(id,action,started_at) VALUES('campaign','promote',?)").run(new Date().toISOString());
        db().prepare("INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES('campaign',?,1,'pending')").run(job.name);
        const fileId = Number(db().prepare("INSERT INTO deployment_files(deployment_id,server,verb,rel_path,content_after_hash,snapshot_pending) VALUES('campaign',?,'updated','hud.gsc',?,1)").run(job.name, putFileBlob(after, db())).lastInsertRowid);
        const { queueFileChange, stagedFileChanges } = await import('../../src/lib/server/fleet/staged-files.js');
        queueFileChange({ sourceKind: 'campaign', sourceId: 'campaign', server: job.name, kind: 'script', relPath: 'hud.gsc', content: after, inventorySource: 'test', deploymentFileId: fileId });
        const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
        const restart = requestRestart(job.name, { kind: 'campaign', id: 'campaign' });
        remote.players = 2; await advance();
        const staged = db().prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?').get(restart.id) as { stage_operation_id: string };
        expect(staged.stage_operation_id).toBeTruthy();
        const { getHostOperation, reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
        const stage = getHostOperation(staged.stage_operation_id)!;
        expect(stage.kind).toBe('files.stage'); expect(await operationFor(restart.id)).toBeNull();
        const routing = db().prepare('SELECT payload FROM ec_host_operations WHERE id=?').get(stage.id) as { payload: string };
        expect(JSON.parse(routing.payload)).toEqual({ stageId: restart.id });
        const plan = db().prepare('SELECT files_json FROM native_provisioning WHERE deployment_id=?').get(restart.id) as { files_json: string };
        expect(JSON.parse(plan.files_json)[0]).not.toHaveProperty('contentBase64');
        const storedPayload = db().prepare('SELECT payload FROM ec_host_operation_payloads WHERE operation_id=?').get(stage.id) as { payload: string };
        expect(JSON.parse(storedPayload.payload).files[0]).toMatchObject({ contentSha256: createHash('sha256').update(after).digest('hex') });
        expect(JSON.parse(storedPayload.payload).files[0]).not.toHaveProperty('contentBase64');
        const { closeDb } = await import('../../src/lib/server/db/index.js');
        closeDb();
        const { pendingHostOperations } = await import('../../src/lib/server/hosts/index.js');
        expect(pendingHostOperations(hostId).find(operation => operation.id === stage.id)?.payload).toEqual(stage.payload);
        await advance();
        expect((db().prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?').get(restart.id) as { stage_operation_id: string }).stage_operation_id).toBe(stage.id);
        expect(stagedFileChanges('campaign', 'campaign')[0].state).toBe('pending');
        reportHostOperation(hostId, stage.id, { state: 'completed', result: { state: 'staged' } });
        await advance(); expect(await operationFor(restart.id)).toBeNull();
        remote.players = 0; await advance();
        const activation = (await operationFor(restart.id))!;
        expect(activation).toMatchObject({ kind: 'instance.restart', generation: 1, payload: { stageId: restart.id } });
        expect(db().prepare('SELECT (SELECT content FROM file_blobs WHERE sha256=content_before_hash) AS content_before,snapshot_pending FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: null, snapshot_pending: 1 });
        const files = stage.payload.files as Array<{ id: number; path: string }>;
        reportHostOperation(hostId, activation.id, { state: 'completed', result: { before: [{ id: files[0].id, path: files[0].path, contentBase64: before.toString('base64') }] } });
        await advance();
        expect((await detail(restart.id)).state).toBe('done');
        expect(stagedFileChanges('campaign', 'campaign')[0].state).toBe('applied');
        expect(db().prepare('SELECT (SELECT content FROM file_blobs WHERE sha256=content_before_hash) AS content_before,sha256_before,snapshot_pending FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: before, sha256_before: createHash('sha256').update(before).digest('hex'), snapshot_pending: 0 });
        const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
        expect(getServerScript(job.name, 'hud.gsc')?.sha256).toBe(createHash('sha256').update(after).digest('hex'));
    });
    it('atomically queues mixed-game collections and reserves per-host ports when dispatched', async () => {
        const t5 = await prepareLibrary('t5'); const t6 = await prepareLibrary('t6');
        const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
        const blueprint = saveBlueprint({ name: 'Both games', entries: [{ game: 't5', mapSlug: 'theater', count: 2 }, { game: 't6', mapSlug: 'town', count: 1 }] });
        const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
        const preview = previewCollection(blueprint.id, 'fill', { hostId, libraryIds: { t5: t5.id, t6: t6.id }, image });
        expect(preview.ready).toBe(true);
        expect(preview.servers.map(server => server.port)).toEqual([null, null, null]);
        const queued = queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key-' + server.name])) });
        expect(queueCollection({ previewId: preview.id }).collectionId).toBe(queued.collectionId);
        const { db } = await import('../../src/lib/server/db/index.js');
        expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]);
        expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
        const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
        saveFleetSettings({ ...fleetSettings(), createConcurrency: 4 });
        await advance();
        const instances = db().prepare('SELECT host_id,library_id,port FROM native_instances ORDER BY port').all();
        expect(instances).toEqual([{ host_id: hostId, library_id: t5.id, port: 28960 }, { host_id: hostId, library_id: t5.id, port: 28961 }, { host_id: hostId, library_id: t6.id, port: 28962 }]);
        const { createHost, createEnrollment, enrollHost, hostPortAvailable } = await import('../../src/lib/server/hosts/index.js');
        const secondHost = createHost({ name: 'Second Debian' }); enrollHost(createEnrollment(secondHost.id).token, inventory());
        const { updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js');
        updateHostLibraryCaches(secondHost.id, [{ libraryVersionId: t5.id, status: 'ready', manifestSha256: t5.manifestSha256 }]);
        const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js');
        const second = queueCreateServer({ game: 't5', gameMode: 'zombie', mapSlug: 'moon', hostId: secondHost.id, libraryId: t5.id, image, plutoKey: 'second-host-key' });
        expect(second.ok).toBe(true); if (!second.ok) throw new Error(second.message);
        expect(second.port).toBeNull();
        await advance();
        expect((await operationFor(second.deploymentId))?.payload.port).toBe(28960);
        expect(hostPortAvailable(hostId, 28960)).toBe(false);
        expect(hostPortAvailable(secondHost.id, 28960)).toBe(false);
        expect(db().prepare('SELECT COUNT(*) AS count FROM ec_host_ports').get()).toEqual({ count: 8 });
    });
    it('reads remote instance files through the agent including missing files and epoch timestamps', async () => {
        const job = await queue(); await advance();
        const backend = await import('../../src/lib/server/backends/native.js');
        const remoteRead = vi.mocked(backend.nativeRequest);
        remoteRead.mockImplementation(async (_instance, kind, payload) => {
            if (payload?.path === 'missing') return (kind === 'files.read' ? { contentBase64: null } : null) as never;
            if (kind === 'files.stat') return { size: 15, mtime: 1_790_000_000, isFile: true, isDirectory: false } as never;
            if (kind === 'files.read') return { contentBase64: Buffer.from('agent file bytes').toString('base64') } as never;
            if (kind === 'files.list') return { entries: [{ name: 'configs', isDirectory: true, isFile: false }] } as never;
            throw new Error('Unexpected file operation');
        });
        const files = await import('../../src/lib/server/backends/files.js');
        const root = join(fixture.t5, job.name);
        expect((await files.readFile(join(root, 'configs/server.cfg'))).toString()).toBe('agent file bytes');
        const info = await files.stat(join(root, 'configs/server.cfg'));
        expect(info.mtime.getTime()).toBe(1_790_000_000_000); expect(info.isFile()).toBe(true);
        expect(await files.exists(join(root, 'missing'))).toBe(false);
        await expect(files.readFile(join(root, 'missing'))).rejects.toMatchObject({ code: 'ENOENT' });
        const entries = await files.readdir(root, { withFileTypes: true });
        expect(entries[0].name).toBe('configs'); expect(entries[0].isDirectory()).toBe(true);
        expect(remoteRead).toHaveBeenCalledWith(expect.objectContaining({ server: job.name }), 'files.read', { path: 'configs/server.cfg' });
        const { stat } = await import('node:fs/promises');
        await expect(stat(root)).rejects.toMatchObject({ code: 'ENOENT' });
    });
    it('retires an agent-confirmed missing container without waiting on nonexistent RCON', async () => {
        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
        const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
        const removal = destroyServer(job.name); if (!removal.ok || !removal.deploymentId) throw new Error('Removal was not queued');
        const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); vi.mocked(rconStatusResult).mockClear();
        remote.state = 'missing'; await advance();
        expect((await operationFor(removal.deploymentId))?.kind).toBe('instance.remove');
        expect(rconStatusResult).not.toHaveBeenCalled();
        await acknowledge(removal.deploymentId);
        const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
        expect(nativeInstance(job.name)).toBeNull();
        expect((await detail(removal.deploymentId)).state).toBe('done');
    });
    it('releases a failed activation slot without recording fictitious file changes', async () => {
        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
        const { db } = await import('../../src/lib/server/db/index.js');
        db().prepare("INSERT INTO deployments(id,action,started_at) VALUES('preflight-failed','promote',?)").run(new Date().toISOString());
        db().prepare("INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state) VALUES('preflight-failed',?,1,'pending')").run(job.name);
        const fileId = Number(db().prepare("INSERT INTO deployment_files(deployment_id,server,verb,rel_path) VALUES('preflight-failed',?,'updated','hud.gsc')").run(job.name).lastInsertRowid);
        const { queueFileChange, stagedFileChanges } = await import('../../src/lib/server/fleet/staged-files.js');
        queueFileChange({ sourceKind: 'campaign', sourceId: 'preflight-failed', server: job.name, kind: 'script', relPath: 'hud.gsc', content: Buffer.from('new bytes'), inventorySource: 'test', deploymentFileId: fileId });
        const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
        const restart = requestRestart(job.name, { kind: 'campaign', id: 'preflight-failed' });
        await advance();
        const stage = db().prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?').get(restart.id) as { stage_operation_id: string };
        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
        reportHostOperation(hostId, stage.stage_operation_id, { state: 'completed', result: { state: 'staged' } });
        await advance();
        const activation = (await operationFor(restart.id))!;
        expect(stagedFileChanges('campaign', 'preflight-failed')[0].state).toBe('applying');
        reportHostOperation(hostId, activation.id, { state: 'failed', error: 'Target changed after campaign review; no activation occurred' });
        await advance();
        expect((await detail(restart.id)).state).toBe('failed');
        expect(stagedFileChanges('campaign', 'preflight-failed')[0].state).toBe('pending');
        expect(db().prepare('SELECT COUNT(*) AS count FROM restart_rollout WHERE healthy_at IS NULL').get()).toEqual({ count: 0 });
        expect(db().prepare('SELECT (SELECT content FROM file_blobs WHERE sha256=content_before_hash) AS content_before,sha256_before FROM deployment_files WHERE id=?').get(fileId)).toEqual({ content_before: null, sha256_before: null });
        const { getServerScript } = await import('../../src/lib/server/fleet/inventory.js');
        expect(getServerScript(job.name, 'hud.gsc')).toBeUndefined();
    });
    it('cancels an unplaced native launch locally without reserving ports or submitting a create', async () => {
        const job = await queue();
        const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
        expect(nativeInstance(job.name)).toBeNull();
        const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
        const cancelled = destroyServer(job.name); expect(cancelled.ok).toBe(true);
        await advance();
        const { db } = await import('../../src/lib/server/db/index.js');
        expect((await detail(job.deploymentId)).state).toBe('cancelled');
        expect(nativeInstance(job.name)).toBeNull();
        expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
        expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind LIKE 'instance.%'").get()).toEqual({ count: 0 });
        const { getLibraryVersion } = await import('../../src/lib/server/game-files/index.js');
        expect(getLibraryVersion(libraryId)?.state).toBe('ready');
        const { getServerKey, serverKeyPool } = await import('../../src/lib/server/fleet/server-keys.js');
        expect(getServerKey(job.name)).toBeUndefined();
        expect(serverKeyPool('t5')).toEqual({ total: 1, available: 1 });
    });
});

describe('automatic capacity admission and placement', () => {
    const input = { game: 't5' as const, gameMode: 'zombie' as const, mapSlug: 'theater', plutoKey: 'key' };
    async function secondHost(overrides = {}) {
        const api = await import('../../src/lib/server/hosts/index.js');
        const libs = await import('../../src/lib/server/game-files/index.js');
        const host = api.createHost({ name: 'Second host', publicAddress: 'second.example', portMin: 28960, portMax: 28970 });
        api.updateHost(host.id, { publicAddress: 'second.example' });
        api.enrollHost(api.createEnrollment(host.id).token, { ...inventory(), ...overrides });
        const version = libs.getLibraryVersion(libraryId)!;
        libs.updateHostLibraryCaches(host.id, [{ libraryVersionId: libraryId, status: 'ready', manifestSha256: version.manifestSha256 }]);
        return host;
    }
    it('admits without a host, then selects the least loaded eligible host and its network defaults', async () => {
        await prepareLibrary();
        const other = await secondHost({ cpuUsagePercent: 0 });
        const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js');
        const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
        const { recipeForDeployment } = await import('../../src/lib/server/fleet/launch-recipes.js');
        const result = queueCreateServer(input);
        expect(result.ok).toBe(true); if (!result.ok) return;
        expect(nativeInstance(result.name)).toBeNull();
        expect(result.port).toBeNull();
        await advance();
        expect((await operationFor(result.deploymentId))?.hostId).toBe(other.id);
        expect(nativeInstance(result.name)).toMatchObject({ host_id: other.id, public_address: 'second.example' });
        expect(recipeForDeployment(result.deploymentId)?.recipe.native?.hostId).toBe(other.id);
        // Once submitted, even a now-offline host cannot trigger another placement.
        const { db } = await import('../../src/lib/server/db/index.js');
        db().prepare("UPDATE ec_hosts SET last_heartbeat='2000-01-01' WHERE id=?").run(other.id);
        await advance();
        expect((await operationFor(result.deploymentId))?.hostId).toBe(other.id);
        expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.create'").all()).toHaveLength(1);
    });
    it('moves an unsubmitted reservation off an offline host and preserves explicit network overrides', async () => {
        await prepareLibrary(); await secondHost();
        const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
        const result = queueNativeCreation({ ...input, publicAddress: 'nat.example', publicPort: 30000 });
        expect(result.ok).toBe(true); if (!result.ok) return;
        db().prepare("UPDATE ec_hosts SET last_heartbeat='2000-01-01' WHERE id=?").run(hostId);
        await advance();
        expect(nativeInstance(result.name)).toMatchObject({ public_address: 'nat.example', public_port: 30000 });
        expect((await operationFor(result.deploymentId))?.hostId).not.toBe(hostId);
        expect(db().prepare('SELECT * FROM ec_host_ports WHERE host_id=?').all(hostId)).toEqual([]);
    });
    it.each([
        { cpuUsagePercent: undefined }, { memoryAvailable: undefined }, { cpuCount: 0 },
        { cpuUsagePercent: 99 }, { memoryAvailable: 1024 ** 3 }
    ])('accepts an unplaced job and waits for unavailable resources: %j', async overrides => {
        await prepareLibrary();
        const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
        heartbeat(hostId, { ...inventory(), ...overrides });
        const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        const queued = queueNativeCreation(input);
        expect(queued).toMatchObject({ ok: true, port: null });
        if (!queued.ok) throw new Error(queued.message);
        await advance();
        expect(await operationFor(queued.deploymentId)).toBeNull();
        expect((await detail(queued.deploymentId)).stage).toContain('capacity');
        expect(db().prepare('SELECT * FROM native_instances').all()).toEqual([]);
        expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
    });
    it('rechecks preview admission without binding the request to the preview host or port', async () => {
        await prepareLibrary();
        const other = await secondHost();
        const { updateHost } = await import('../../src/lib/server/hosts/index.js');
        updateHost(other.id, { portMin: 30000, portMax: 30002 });
        const { previewNativeCreation, queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
        const preview = previewNativeCreation(input);
        const { db } = await import('../../src/lib/server/db/index.js');
        db().prepare("UPDATE ec_hosts SET last_heartbeat='2000-01-01' WHERE id=?").run(hostId);
        const result = queueNativeCreation({ ...input, previewId: preview.id });
        expect(result).toMatchObject({ ok: true, name: preview.name, port: null });
        expect(queueNativeCreation({ ...input, previewId: preview.id })).toEqual(result);
        if (!result.ok) throw new Error(result.message);
        await advance();
        expect((await operationFor(result.deploymentId))?.payload.port).toBe(30000);
    });
    it('queues the whole collection and waits when capacity shrinks after its preview', async () => {
        await prepareLibrary();
        const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
        const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
        const blueprint = saveBlueprint({ name: 'Two lobbies', entries: [{ game: 't5', mapSlug: 'theater', count: 2 }] });
        const preview = previewCollection(blueprint.id);
        const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
        heartbeat(hostId, { ...inventory(), cpuUsagePercent: 100 });
        const result = queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key-' + server.name])) });
        expect(result).toMatchObject({ ok: true, collectionId: expect.any(String) });
        await advance();
        const { db } = await import('../../src/lib/server/db/index.js');
        expect(db().prepare('SELECT * FROM native_instances').all()).toEqual([]);
        expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
        expect(db().prepare('SELECT * FROM server_launch_collections').all()).toHaveLength(1);
        expect(db().prepare('SELECT * FROM native_creation_intents').all()).toHaveLength(2);
    });
    it.each([
        { resource: 'CPU', measurements: { cpuCount: 4, cpuUsagePercent: 82.5 }, over: { cpuUsagePercent: 82.51 } },
        { resource: 'memory', measurements: { memoryAvailable: 1024 ** 3 + 500_000_000 }, over: { memoryAvailable: 1024 ** 3 + 499_999_999 } }
    ])('reserves only the requested host $resource headroom', async ({ measurements, over }) => {
        await prepareLibrary();
        const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
        const { choosePlacement } = await import('../../src/lib/server/hosts/capacity.js');
        const options = { hostId, libraryId, image };
        heartbeat(hostId, { ...inventory(), ...measurements });
        // Includes the new server's 0.5-core / 1-GiB bootstrap reservation.
        expect(choosePlacement(options).host.id).toBe(hostId);
        expect(choosePlacement({ ...options, thorough: true }).host.id).toBe(hostId);
        heartbeat(hostId, { ...inventory(), ...measurements, ...over });
        expect(() => choosePlacement(options)).toThrow('capacity');
        expect(() => choosePlacement({ ...options, thorough: true })).toThrow('capacity');
    });
    it('dispatches a waiting instance after calibration lowers the cold-start allowance', async () => {
        const first = await queue();
        await advance(); await acknowledge(first.deploymentId);
        const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
        const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
        const { recordInstanceResources, capacityCalibration } = await import('../../src/lib/server/hosts/calibration.js');
        const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
        const measured = { ...inventory(), cpuCount: 1, cpuUsagePercent: 50 };
        heartbeat(hostId, measured);
        const next = queueNativeCreation({ ...input, plutoKey: 'next-key' });
        expect(next.ok).toBe(true); if (!next.ok) throw new Error(next.message);
        await advance();
        expect(await operationFor(next.deploymentId)).toBeNull();
        const instance = nativeInstance(first.name)!;
        for (let minute = 0; minute < 25; minute++) {
            recordInstanceResources(hostId, { ...measured, instanceResources: [{ instanceId: instance.id, generation: 1,
                uptimeSeconds: minute < 4 ? minute * 60 : 600, cpuCores: 0.1, memoryBytes: 256 * 1024 ** 2 }] },
                new Date(Date.now() - (25 - minute) * 60_000).toISOString());
        }
        expect(capacityCalibration(hostId).startup.source).toBe('host');
        await advance();
        expect(await operationFor(next.deploymentId)).not.toBeNull();
    });
    it('queues a collection at running demand and releases startup reservations as live usage arrives', async () => {
        await prepareLibrary();
        const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
        const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
        const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
        const GiB = 1024 ** 3;
        const measured = { ...inventory(), cpuCount: 4, cpuUsagePercent: 5, memoryTotal: 4 * GiB, memoryAvailable: 3.75 * GiB };
        heartbeat(hostId, measured);
        saveFleetSettings({ ...fleetSettings(), createConcurrency: 3 });
        const insert = db().prepare(`INSERT INTO ec_resource_allowances
            (scope,phase,cpu_cores,memory_bytes,samples,window_start,observed_at,computed_at) VALUES(?,?,?,?,30,?,?,?)`);
        const now = new Date().toISOString();
        const start = new Date(Date.now() - 30 * 60_000).toISOString();
        insert.run(hostId, 'startup', 2, 2 * GiB, start, now, now);
        insert.run(hostId, 'running', 0.2, GiB / 4, start, now, now);
        const blueprint = saveBlueprint({ name: 'Staggered lobbies', entries: [{ game: 't5', mapSlug: 'theater', count: 3 }] });
        const preview = previewCollection(blueprint.id);
        expect(preview.ready).toBe(true);
        expect(queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key-' + server.name])) }).ok).toBe(true);
        await advance();
        const creates = () => db().prepare("SELECT instance_id FROM ec_host_operations WHERE kind='instance.create'").all() as { instance_id: string }[];
        expect(creates()).toHaveLength(1);
        await advance();
        expect(creates()).toHaveLength(1); // No heartbeat yet: the first launch still reserves startup resources.
        heartbeat(hostId, { ...measured, cpuUsagePercent: 10, memoryAvailable: 3.5 * GiB,
            instanceResources: [{ instanceId: creates()[0].instance_id, generation: 1, uptimeSeconds: 60, cpuCores: 0.2, memoryBytes: GiB / 4 }] });
        await advance();
        expect(creates()).toHaveLength(2); // Its actual usage replaces the peak even before health verification.
    });
    it('reserves queued lobby identities without host resources and releases them on cancellation', async () => {
        await prepareLibrary();
        const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
        heartbeat(hostId, { ...inventory(), cpuCount: 1, cpuUsagePercent: 0 });
        const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
        const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
        const first = queueNativeCreation(input);
        expect(first.ok).toBe(true); if (!first.ok) return;
        const second = queueNativeCreation({ ...input, plutoKey: 'second-key' });
        expect(second).toMatchObject({ ok: true, name: 't5_zombie_theater_2', port: null });
        const { db } = await import('../../src/lib/server/db/index.js');
        expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]);
        expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
        cancelCreation(first.deploymentId);
        expect(queueNativeCreation(input)).toMatchObject({ ok: true, name: first.name, port: null });
    });
    it.each(['memory', 'cpu'] as const)('dispatches immediately after %s recovers instead of waiting for five-minute history', async resource => {
        await prepareLibrary();
        const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
        const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        const GiB = 1024 ** 3;
        // A recently busy 4 GiB host is now mostly empty, as in the reported
        // ec-2 inventory. Admission must not pass only to stall in dispatch.
        heartbeat(hostId, { ...inventory(), cpuCount: 4, memoryTotal: 4 * GiB, memoryAvailable: Math.floor(3.8 * GiB) });
        for (let minute = 1; minute <= 4; minute++) db().prepare(`INSERT INTO ec_host_resource_samples
            (host_id,sampled_at,cpu_count,cpu_usage_percent,memory_total,memory_available) VALUES(?,?,4,?,?,?)`)
            .run(hostId, new Date(Date.now() - minute * 60_000).toISOString(), resource === 'cpu' ? 100 : 10,
                4 * GiB, resource === 'memory' ? GiB / 8 : Math.floor(3.8 * GiB));
        const jobs = Array.from({ length: 3 }, (_, index) => queueNativeCreation({ ...input, plutoKey: `key-${index}` }));
        expect(jobs.every(job => job.ok)).toBe(true);
        await advance();
        expect(await operationFor(jobs[0].ok ? jobs[0].deploymentId : 'missing')).not.toBeNull();
    });
    it('dispatches one queued launch when learned startup costs outgrow the original batch reservation', async () => {
        await prepareLibrary();
        const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
        saveFleetSettings({ ...fleetSettings(), createConcurrency: 3 });
        const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
        const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        const GiB = 1024 ** 3;
        heartbeat(hostId, { ...inventory(), cpuCount: 4, memoryTotal: 4 * GiB, memoryAvailable: Math.floor(3.8 * GiB) });
        const jobs = Array.from({ length: 3 }, (_, index) => queueNativeCreation({ ...input, plutoKey: `key-${index}` }));
        expect(jobs.every(job => job.ok)).toBe(true);
        // Learning increases the startup allowance after admission. All three
        // no longer fit concurrently, but this otherwise empty host fits one.
        db().prepare(`INSERT INTO ec_resource_allowances
            (scope,phase,cpu_cores,memory_bytes,samples,window_start,observed_at,computed_at)
            VALUES(?,'startup',0.5,?,3,?,?,?)`).run(hostId, 2 * GiB,
                new Date(Date.now() - 180_000).toISOString(), new Date().toISOString(), new Date().toISOString());
        await advance();
        const operations = await Promise.all(jobs.map(job => operationFor(job.ok ? job.deploymentId : 'missing')));
        expect(operations.filter(Boolean)).toHaveLength(1);
        expect(operations[0]?.kind).toBe('instance.create');
        // The dispatched operation now consumes capacity and blocks the others,
        // even though the next host heartbeat has not observed it yet.
        await advance();
        expect(db().prepare("SELECT * FROM ec_host_operations WHERE kind='instance.create'").all()).toHaveLength(1);
        const later = queueNativeCreation(input);
        expect(later.ok).toBe(true); if (!later.ok) throw new Error(later.message);
        await advance();
        expect(await operationFor(later.deploymentId)).toBeNull();
    });
    it('does not charge a starting container twice when its usage is already in the heartbeat', async () => {
        await prepareLibrary();
        const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
        saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 });
        const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
        const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
        const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
        const GiB = 1024 ** 3;
        const host = { ...inventory(), cpuCount: 4, memoryTotal: 3 * GiB, memoryAvailable: 2.8 * GiB };
        heartbeat(hostId, { ...host, memoryAvailable: Math.floor(host.memoryAvailable) });
        const first = queueNativeCreation(input);
        expect(first.ok).toBe(true); if (!first.ok) return;
        await advance();
        const instance = nativeInstance(first.name)!;
        // 0.2 GiB OS + 0.8 GiB container are already in host usage.
        // Adding a new 1 GiB server fits below 2.53 GiB usable memory.
        heartbeat(hostId, { ...host, memoryAvailable: 2 * GiB, instanceResources: [{ instanceId: instance.id,
            generation: instance.generation, uptimeSeconds: 60, cpuCores: 0.1, memoryBytes: Math.floor(0.8 * GiB) }] });
        const second = queueNativeCreation({ ...input, plutoKey: 'second-key' });
        expect(second).toMatchObject({ ok: true });
        await advance();
        expect(await operationFor(second.ok ? second.deploymentId : 'missing')).not.toBeNull();
    });
    it.each(['missing', 'old-generation'] as const)('keeps full reservations for a submitted launch with %s measurements', async evidence => {
        await prepareLibrary();
        const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
        saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 });
        const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
        const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
        const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
        const GiB = 1024 ** 3;
        heartbeat(hostId, { ...inventory(), memoryTotal: 3 * GiB, memoryAvailable: Math.floor(2.8 * GiB) });
        const first = queueNativeCreation(input);
        expect(first.ok).toBe(true); if (!first.ok) return;
        await advance();
        const instance = nativeInstance(first.name)!;
        heartbeat(hostId, { ...inventory(), memoryTotal: 3 * GiB, memoryAvailable: 2 * GiB,
            instanceResources: evidence === 'missing' ? [] : [{ instanceId: instance.id, generation: instance.generation + 1,
                uptimeSeconds: 60, cpuCores: 0.1, memoryBytes: Math.floor(0.8 * GiB) }] });
        const next = queueNativeCreation({ ...input, plutoKey: 'next-key' });
        expect(next.ok).toBe(true); if (!next.ok) throw new Error(next.message);
        await advance();
        expect(await operationFor(next.deploymentId)).toBeNull();
        expect((await detail(next.deploymentId)).stage).toContain('3.00 GiB projected, 2.53 GiB usable');
    });
    it('waits when capacity disappears after admission without sending a create command', async () => {
        await prepareLibrary();
        const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
        const first = queueNativeCreation(input);
        expect(first.ok).toBe(true); if (!first.ok) return;
        const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
        heartbeat(hostId, { ...inventory(), cpuUsagePercent: 100 });
        await advance();
        expect(await operationFor(first.deploymentId)).toBeNull();
        expect((await detail(first.deploymentId)).stage).toContain('capacity');
    });
    it('uses recent history at placement while keeping admission to current inventory', async () => {
        await prepareLibrary();
        const other = await secondHost({ cpuUsagePercent: 30 });
        const { db } = await import('../../src/lib/server/db/index.js');
        db().prepare(`INSERT INTO ec_host_resource_samples(host_id,sampled_at,cpu_count,cpu_usage_percent,memory_total,memory_available)
            VALUES(?,?,16,95,?,?)`).run(hostId, new Date(Date.now() - 60_000).toISOString(), 64 * 1024 ** 3, 60 * 1024 ** 3);
        const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
        const first = queueNativeCreation(input);
        expect(first.ok).toBe(true); if (!first.ok) return;
        await advance();
        expect((await operationFor(first.deploymentId))?.hostId).toBe(other.id);
    });
    it('persists nullable CPU/memory samples and prunes old history per host', async () => {
        await prepareLibrary();
        const { db, closeDb } = await import('../../src/lib/server/db/index.js');
        const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
        db().prepare("INSERT INTO ec_host_resource_samples(host_id,sampled_at) VALUES(?,'2000-01-01')").run(hostId);
        heartbeat(hostId, { ...inventory(), cpuUsagePercent: undefined });
        closeDb();
        const rows = db().prepare('SELECT * FROM ec_host_resource_samples WHERE host_id=? ORDER BY sampled_at').all(hostId) as { cpu_usage_percent: number | null; memory_available: number; sampled_at: string }[];
        expect(rows.at(-1)).toMatchObject({ cpu_usage_percent: null, memory_available: 60 * 1024 ** 3 });
        expect(rows.some(row => row.sampled_at === '2000-01-01')).toBe(false);
    });
    it('checks both port protocols and skips draining hosts', async () => {
        await prepareLibrary();
        const other = await secondHost();
        const { reserveHostPort, updateHost } = await import('../../src/lib/server/hosts/index.js');
        const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
        for (let port = 28960; port <= 28970; port++) reserveHostPort(hostId, 'external-' + port, { port, protocol: 'tcp' });
        const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
        const result = queueNativeCreation(input);
        expect(result.ok).toBe(true); if (!result.ok) return;
        expect(nativeInstance(result.name)).toBeNull();
        await advance();
        expect(nativeInstance(result.name)?.host_id).toBe(other.id);
        updateHost(other.id, { mode: 'draining' });
        const next = queueNativeCreation({ ...input, plutoKey: 'next-key' });
        expect(next.ok).toBe(true); if (!next.ok) throw new Error(next.message);
        await advance();
        expect(await operationFor(next.deploymentId)).toBeNull();
    });
    it('spreads collection dispatch across hosts, allowing the same port on different hosts', async () => {
        await prepareLibrary(); await secondHost({ cpuCount: 1, cpuUsagePercent: 0 });
        const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
        heartbeat(hostId, { ...inventory(), cpuCount: 1, cpuUsagePercent: 0 });
        const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
        const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
        const blueprint = saveBlueprint({ name: 'Two lobbies', entries: [{ game: 't5', mapSlug: 'theater', count: 2 }] });
        const preview = previewCollection(blueprint.id);
        expect(preview.ready).toBe(true);
        expect(preview.servers.map(server => server.port)).toEqual([null, null]);
        expect(queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key-' + server.name])) }).ok).toBe(true);
        const { db } = await import('../../src/lib/server/db/index.js');
        expect(db().prepare('SELECT id FROM native_instances').all()).toEqual([]);
        const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
        saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 });
        await advance();
        // Concurrent ranking can select the same host before the first
        // reservation commits. The held job retries against the new usage.
        await advance();
        expect(db().prepare('SELECT DISTINCT host_id FROM native_instances').all()).toHaveLength(2);
        expect(db().prepare('SELECT port FROM native_instances').all()).toEqual([{ port: 28960 }, { port: 28960 }]);
    });
});

describe('native verification resume and explicit failed-creation retry', () => {
    async function failedCreation(state: 'failed' | 'interrupted' = 'failed') {
        const job = await queue(); await advance();
        const original = (await operationFor(job.deploymentId))!;
        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
        reportHostOperation(hostId, original.id, { state: 'failed', error: 'Host runtime preflight failed' });
        await advance();
        if (state === 'interrupted') {
            const { db } = await import('../../src/lib/server/db/index.js');
            db().prepare("UPDATE server_deployments SET state='interrupted' WHERE id=?").run(job.deploymentId);
        }
        return { job, original };
    }

    it.each(['failed', 'interrupted'] as const)('retries a %s creation through a new rebuild while retaining the original failure', async state => {
        const { job, original } = await failedCreation(state);
        const history = await import('../../src/lib/server/provisioning/history.js');
        expect(history.getDeployment(job.deploymentId)).toMatchObject({ canResumeVerification: false, canRetryWithRebuild: true });
        const { POST: resume } = await import('../../src/routes/api/provisioning/[id]/resume/+server.js');
        expect(resume({ params: { id: job.deploymentId } }).status).toBe(409);
        const { POST: retry } = await import('../../src/routes/api/provisioning/[id]/retry-rebuild/+server.js');
        const response = await retry({ params: { id: job.deploymentId } });
        expect(response.status).toBe(202);
        const next = await response.json() as { ok: boolean; deploymentId: string };
        expect(next.ok).toBe(true);
        expect(next.deploymentId).not.toBe(job.deploymentId);
        expect(await detail(next.deploymentId)).toMatchObject({ action: 'rebuild', state: 'waiting', server: job.name });
        expect(history.getDeployment(job.deploymentId)).toMatchObject({
            canRetryWithRebuild: false,
            deployment: { state, error: 'Host runtime preflight failed' },
            progress: { recovery: { state: 'transferred', operationId: next.deploymentId } }
        });
        expect((await retry({ params: { id: job.deploymentId } })).status).toBe(409);
        const { db } = await import('../../src/lib/server/db/index.js');
        expect(db().prepare('SELECT COUNT(*) AS count FROM server_deployments WHERE server=?').get(job.name)).toEqual({ count: 2 });
        const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
        expect(nativeInstance(job.name)).toMatchObject({ id: original.instanceId, generation: 1, port: original.payload.port });
        remote.state = 'missing'; await advance();
        const rebuilding = (await operationFor(next.deploymentId))!;
        expect(rebuilding).toMatchObject({ kind: 'instance.rebuild', generation: 1, instanceId: original.instanceId,
            payload: { name: job.name, port: original.payload.port, nextGeneration: 2, env: { PLUTO_SERVER_KEY: 'saved-key' } } });
        expect(rebuilding.id).not.toBe(original.id);
        expect(nativeInstance(job.name)?.generation).toBe(1);
        remote.state = 'running'; await acknowledge(next.deploymentId);
        expect((await detail(next.deploymentId)).state).toBe('done');
        expect(nativeInstance(job.name)?.generation).toBe(2);
        // An interrupted original is reconciled to its already-failed receipt
        // on the next worker pass; the failure itself is never cleared.
        expect(await detail(job.deploymentId)).toMatchObject({ state: 'failed', error: 'Host runtime preflight failed' });
        expect(await operationFor(job.deploymentId)).toMatchObject({ id: original.id, state: 'failed' });
        expect(db().prepare("SELECT kind FROM ec_host_operations WHERE kind IN ('instance.create','instance.rebuild','instance.restart') ORDER BY rowid").all())
            .toEqual([{ kind: 'instance.create' }, { kind: 'instance.rebuild' }]);
    });

    it.each([
        ['create', false], ['create', true], ['rebuild', false], ['rebuild', true],
        ['recovery-restart', false], ['recovery-restart', true], ['recovery-rebuild', false], ['recovery-rebuild', true]
    ] as const)('resumes completed %s verification with the same command (receipt applied=%s)', async (action, applied) => {
        let job = await queue(); await advance();
        if (action === 'rebuild') {
            await acknowledge(job.deploymentId);
            const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
            const rebuild = queueNativeRebuild(job.name, 'test');
            if (!rebuild.ok) throw new Error(rebuild.message);
            job = rebuild; await advance();
        } else if (action.startsWith('recovery')) {
            remote.state = 'unhealthy'; await acknowledge(job.deploymentId);
            if (action === 'recovery-rebuild') await acknowledge(job.deploymentId);
        }
        const operation = (await operationFor(job.deploymentId))!;
        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
        remote.state = 'starting'; reportHostOperation(hostId, operation.id, { state: 'completed' });
        if (applied) await advance();
        const history = await import('../../src/lib/server/provisioning/history.js');
        history.finishDeployment(job.deploymentId, { ok: false, message: 'Verification interrupted for inspection' });
        expect(history.getDeployment(job.deploymentId)).toMatchObject({ canResumeVerification: true, canRetryWithRebuild: false });
        const { db } = await import('../../src/lib/server/db/index.js');
        const commands = db().prepare("SELECT id FROM ec_host_operations WHERE kind LIKE 'instance.%' ORDER BY rowid").all();
        const { POST } = await import('../../src/routes/api/provisioning/[id]/resume/+server.js');
        expect(POST({ params: { id: job.deploymentId } }).status).toBe(202);
        remote.state = 'running'; await advance();
        expect((await detail(job.deploymentId)).state).toBe('done');
        expect((await operationFor(job.deploymentId))!.id).toBe(operation.id);
        expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind LIKE 'instance.%' ORDER BY rowid").all()).toEqual(commands);
        const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
        expect(nativeInstance(job.name)?.generation).toBe(operation.generation! + (operation.kind === 'instance.rebuild' ? 1 : 0));
        expect(history.getDeployment(job.deploymentId)!.events).toContainEqual(expect.objectContaining({
            kind: 'outcome', message: 'Failed: Verification interrupted for inspection'
        }));
    });

    it.each(['pending', 'accepted', 'running', 'unknown', 'failed'] as const)('does not offer verification of a %s command or replay an ambiguous result', async state => {
        const job = await queue(); await advance();
        const operation = (await operationFor(job.deploymentId))!;
        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
        if (state !== 'pending') reportHostOperation(hostId, operation.id, { state });
        const history = await import('../../src/lib/server/provisioning/history.js');
        history.finishDeployment(job.deploymentId, { ok: false, message: 'Controller observation interrupted' });
        expect(history.getDeployment(job.deploymentId)).toMatchObject({ canResumeVerification: false, canRetryWithRebuild: state === 'failed' });
        expect(history.resumeVerification(job.deploymentId)).toBe(false);
        if (state !== 'failed') {
            const { requestRetryWithRebuild } = await import('../../src/lib/server/provisioning/requests.js');
            expect(requestRetryWithRebuild(job.deploymentId)).toMatchObject({ ok: false, status: 409 });
        }
        const { db } = await import('../../src/lib/server/db/index.js');
        if (state === 'unknown') db().prepare("UPDATE server_deployments SET state='interrupted' WHERE id=?").run(job.deploymentId);
        await advance();
        expect((await operationFor(job.deploymentId))!.id).toBe(operation.id);
        expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind IN ('instance.create','instance.restart','instance.rebuild')").all())
            .toEqual([{ id: operation.id }]);
    });

    it.each(['instance generation', 'command generation', 'command instance', 'command host', 'retired instance', 'server port', 'newer work', 'creation cleanup', 'recovery attempt'])
        ('refuses to retry a failed creation after its %s changes', async change => {
            const { job, original } = await failedCreation();
            const { db } = await import('../../src/lib/server/db/index.js');
            if (change === 'instance generation') db().prepare('UPDATE native_instances SET generation=generation+1 WHERE server=?').run(job.name);
            if (change === 'command generation') db().prepare('UPDATE ec_host_operations SET generation=generation+1 WHERE id=?').run(original.id);
            if (change === 'command instance') db().prepare("UPDATE ec_host_operations SET instance_id='different-instance' WHERE id=?").run(original.id);
            if (change === 'command host') {
                const { createHost } = await import('../../src/lib/server/hosts/index.js');
                db().prepare('UPDATE ec_host_operations SET host_id=? WHERE id=?').run(createHost({ name: 'Different host' }).id, original.id);
            }
            if (change === 'retired instance') db().prepare('UPDATE native_instances SET retired_at=? WHERE server=?').run(new Date().toISOString(), job.name);
            if (change === 'server port') db().prepare('UPDATE servers SET game_port=game_port+1 WHERE name=?').run(job.name);
            if (change === 'newer work') {
                const { beginDeployment } = await import('../../src/lib/server/provisioning/history.js');
                beginDeployment(job.name, 'restart');
            }
            if (change === 'creation cleanup') {
                const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
                expect(cancelCreation(job.deploymentId)).toEqual(expect.any(String));
            }
            if (change === 'recovery attempt') db().prepare("UPDATE native_provisioning SET recovery_step='restart' WHERE deployment_id=?").run(job.deploymentId);
            const history = await import('../../src/lib/server/provisioning/history.js');
            const { requestRetryWithRebuild } = await import('../../src/lib/server/provisioning/requests.js');
            expect(history.getDeployment(job.deploymentId)?.canRetryWithRebuild).toBe(false);
            expect(requestRetryWithRebuild(job.deploymentId)).toMatchObject({ ok: false, status: 409 });
            expect(db().prepare("SELECT id FROM server_deployments WHERE action='rebuild'").all()).toEqual([]);
        });

    it.each(['instance generation', 'command generation', 'command instance', 'command host', 'retired instance', 'server port', 'newer work'])
        ('refuses to resume a completed creation after its %s changes', async change => {
            const job = await queue(); await advance();
            const original = (await operationFor(job.deploymentId))!;
            const { reportHostOperation, createHost } = await import('../../src/lib/server/hosts/index.js');
            reportHostOperation(hostId, original.id, { state: 'completed' });
            const history = await import('../../src/lib/server/provisioning/history.js');
            history.finishDeployment(job.deploymentId, { ok: false, message: 'Verification interrupted' });
            const { db } = await import('../../src/lib/server/db/index.js');
            if (change === 'instance generation') db().prepare('UPDATE native_instances SET generation=generation+1 WHERE server=?').run(job.name);
            if (change === 'command generation') db().prepare('UPDATE ec_host_operations SET generation=generation+1 WHERE id=?').run(original.id);
            if (change === 'command instance') db().prepare("UPDATE ec_host_operations SET instance_id='different-instance' WHERE id=?").run(original.id);
            if (change === 'command host') db().prepare('UPDATE ec_host_operations SET host_id=? WHERE id=?').run(createHost({ name: 'Different host' }).id, original.id);
            if (change === 'retired instance') db().prepare('UPDATE native_instances SET retired_at=? WHERE server=?').run(new Date().toISOString(), job.name);
            if (change === 'server port') db().prepare('UPDATE servers SET game_port=game_port+1 WHERE name=?').run(job.name);
            if (change === 'newer work') history.beginDeployment(job.name, 'restart');
            expect(history.getDeployment(job.deploymentId)?.canResumeVerification).toBe(false);
            expect(history.resumeVerification(job.deploymentId)).toBe(false);
            expect((await operationFor(job.deploymentId))!.id).toBe(original.id);
        });
});

describe('unhealthy recovery within native create and rebuild operations', () => {
    async function launch(action: 'create' | 'rebuild') {
        const job = await queue(); await advance();
        if (action === 'create') return job;
        await acknowledge(job.deploymentId);
        const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
        const rebuild = queueNativeRebuild(job.name, 'test');
        if (!rebuild.ok) throw new Error(rebuild.message);
        await advance();
        return rebuild;
    }

    it.each(['create', 'rebuild'] as const)('recovers an unhealthy %s with one restart then a rebuild using the original launch payload', async action => {
        const { config } = await import('../../src/lib/server/config.js');
        Object.assign(config.rcon, { password: '' });
        const job = await launch(action);
        const original = (await operationFor(job.deploymentId))!;
        const currentGeneration = action === 'rebuild' ? 2 : 1;
        const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
        const { db, closeDb } = await import('../../src/lib/server/db/index.js');
        const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
        vi.mocked(rconStatusResult).mockClear();
        remote.state = 'unhealthy'; remote.players = 4;
        Object.assign(config.rcon, { password: 'changed-after-original-launch' });
        await acknowledge(job.deploymentId);
        const restart = (await operationFor(job.deploymentId))!;
        const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
        expect(restart).toMatchObject({ kind: 'instance.restart', state: 'pending', generation: currentGeneration,
            instanceId: original.instanceId, payload: { unhealthyRecovery: true } });
        expect(getDeployment(job.deploymentId)!.capacity.pools).toEqual(expect.arrayContaining([
            expect.objectContaining({ kind: 'startup', ownsSlot: true }),
            expect.objectContaining({ kind: 'restart', ownsSlot: true })
        ]));
        expect(restart.id).not.toBe(original.id);
        expect(nativeInstance(job.name)?.generation).toBe(currentGeneration);
        expect(db().prepare('SELECT recovery_step,generation FROM native_provisioning WHERE deployment_id=?').get(job.deploymentId))
            .toEqual({ recovery_step: 'restart', generation: currentGeneration });
        expect(rconStatusResult).not.toHaveBeenCalled();

        // Restarting the controller must preserve the current recovery command.
        closeDb(); await advance();
        expect((await operationFor(job.deploymentId))!.id).toBe(restart.id);
        await acknowledge(job.deploymentId);
        const rebuild = (await operationFor(job.deploymentId))!;
        expect(rebuild).toMatchObject({ kind: 'instance.rebuild', state: 'pending', generation: currentGeneration,
            instanceId: original.instanceId, payload: { unhealthyRecovery: true, nextGeneration: currentGeneration + 1,
                files: original.payload.files, env: original.payload.env, rconPassword: original.payload.rconPassword,
                libraryId: original.payload.libraryId, image: original.payload.image } });
        expect(rebuild.id).not.toBe(restart.id);
        expect(nativeInstance(job.name)?.generation).toBe(currentGeneration);
        expect(rconStatusResult).not.toHaveBeenCalled();
        expect(db().prepare('SELECT recovery_step,generation FROM native_provisioning WHERE deployment_id=?').get(job.deploymentId))
            .toEqual({ recovery_step: 'rebuild', generation: currentGeneration });
        remote.state = 'running'; remote.players = 0;
        await acknowledge(job.deploymentId);
        expect(nativeInstance(job.name)?.generation).toBe(currentGeneration + 1);
        expect(await detail(job.deploymentId)).toMatchObject({ action, state: 'done' });
        expect(getDeployment(job.deploymentId)!.progress?.recovery?.state).toBe('released');
        expect(getDeployment(job.deploymentId)!.capacity.pools.every(pool => !pool.ownsSlot)).toBe(true);
        await advance();
        expect((await operationFor(job.deploymentId))!.id).toBe(rebuild.id);
        expect(nativeInstance(job.name)?.generation).toBe(currentGeneration + 1);
    });

    it.each(['create', 'rebuild'] as const)('finishes %s after a successful recovery restart without another rebuild', async action => {
        const job = await launch(action);
        remote.state = 'unhealthy'; await acknowledge(job.deploymentId);
        const restart = (await operationFor(job.deploymentId))!;
        expect(restart.kind).toBe('instance.restart');
        remote.state = 'running'; await acknowledge(job.deploymentId); await advance();
        expect(await detail(job.deploymentId)).toMatchObject({ action, state: 'done' });
        expect((await operationFor(job.deploymentId))!.id).toBe(restart.id);
        const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
        expect(nativeInstance(job.name)?.generation).toBe(action === 'rebuild' ? 2 : 1);
        const { db } = await import('../../src/lib/server/db/index.js');
        expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'").get())
            .toEqual({ count: action === 'rebuild' ? 1 : 0 });
    });

    it.each(['restart', 'rebuild'] as const)('retains an unknown recovery %s receipt without replaying or advancing recovery', async step => {
        const job = await launch('create');
        remote.state = 'unhealthy'; await acknowledge(job.deploymentId);
        if (step === 'rebuild') await acknowledge(job.deploymentId);
        const operation = (await operationFor(job.deploymentId))!;
        expect(operation.kind).toBe(`instance.${step}`);
        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
        reportHostOperation(hostId, operation.id, { state: 'unknown', error: 'Reply lost during recovery' });
        const { closeDb, db } = await import('../../src/lib/server/db/index.js'); closeDb();
        const { interruptDeployments } = await import('../../src/lib/server/provisioning/history.js'); interruptDeployments();
        await advance(); await advance();
        expect((await operationFor(job.deploymentId))!.id).toBe(operation.id);
        expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind IN ('instance.create','instance.restart','instance.rebuild')").get())
            .toEqual({ count: step === 'restart' ? 2 : 3 });
        expect((await detail(job.deploymentId)).state).not.toBe('failed');
        remote.state = 'running'; await acknowledge(job.deploymentId);
        expect((await detail(job.deploymentId)).state).toBe('done');
    });

    it.each(['starting', 'restarting', 'unknown'])('waits for %s container readiness without forcing recovery', async state => {
        const job = await launch('create');
        const original = (await operationFor(job.deploymentId))!;
        remote.state = state; await acknowledge(job.deploymentId); await advance();
        expect((await operationFor(job.deploymentId))!.id).toBe(original.id);
        expect(await detail(job.deploymentId)).toMatchObject({ state: 'awaiting-health', stage: `Waiting for container readiness: ${state}` });
        const { db } = await import('../../src/lib/server/db/index.js');
        expect(db().prepare('SELECT recovery_step FROM native_provisioning WHERE deployment_id=?').get(job.deploymentId))
            .toEqual({ recovery_step: null });
    });

    it.each(['create', 'rebuild'] as const)('does not submit recovery if %s is cancelled during its health probe', async action => {
        const job = await launch(action);
        const original = (await operationFor(job.deploymentId))!;
        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
        reportHostOperation(hostId, original.id, { state: 'completed' });
        const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
        const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
        let release!: (value: never) => void;
        vi.mocked(nativeRequest).mockImplementationOnce(() => new Promise(resolve => { release = resolve; }));
        const pending = advance();
        await vi.waitFor(() => expect(release).toBeTypeOf('function'));
        expect(cancelProvisioningJob(job.deploymentId)).toBe(true);
        release({ healthy: false, state: 'unhealthy' } as never);
        await pending; await advance();
        expect((await detail(job.deploymentId)).state).toBe('cancelled');
        expect((await operationFor(job.deploymentId))!.id).toBe(original.id);
        const { db } = await import('../../src/lib/server/db/index.js');
        expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.restart'").all()).toEqual([]);
    });

    it.each(['create', 'rebuild'] as const)('reconciles a delivered recovery rebuild after its %s flow is cancelled', async action => {
        const job = await launch(action);
        remote.state = 'unhealthy'; await acknowledge(job.deploymentId); await acknowledge(job.deploymentId);
        const rebuild = (await operationFor(job.deploymentId))!;
        expect(rebuild.kind).toBe('instance.rebuild');
        const { pendingHostOperations, reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
        const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
        pendingHostOperations(hostId);
        expect(cancelProvisioningJob(job.deploymentId)).toBe(true);
        reportHostOperation(hostId, rebuild.id, { state: 'completed' });
        await advance(); await advance();
        const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
        expect(nativeInstance(job.name)?.generation).toBe(action === 'rebuild' ? 3 : 2);
        expect((await detail(job.deploymentId)).state).toBe('cancelled');
        expect((await operationFor(job.deploymentId))!.id).toBe(rebuild.id);
    });

    it('fails clearly after an unhealthy recovery rebuild instead of looping', async () => {
        const job = await launch('create');
        remote.state = 'unhealthy';
        await acknowledge(job.deploymentId);
        await acknowledge(job.deploymentId);
        const rebuild = (await operationFor(job.deploymentId))!;
        expect(rebuild.kind).toBe('instance.rebuild');
        await acknowledge(job.deploymentId); await advance();
        expect(await detail(job.deploymentId)).toMatchObject({ state: 'failed', error: expect.stringMatching(/unhealthy.*restart.*rebuild/i) });
        expect((await operationFor(job.deploymentId))!.id).toBe(rebuild.id);
        const { db } = await import('../../src/lib/server/db/index.js');
        expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind IN ('instance.create','instance.restart','instance.rebuild')").get())
            .toEqual({ count: 3 });
    });

    it.each(['create', 'rebuild'] as const)('holds unhealthy %s verification until the agent supports recovery', async action => {
        const job = await launch(action);
        const original = (await operationFor(job.deploymentId))!;
        const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
        heartbeat(hostId, { ...inventory(), capabilities: [] });
        remote.state = 'unhealthy'; await acknowledge(job.deploymentId); await advance();
        expect((await operationFor(job.deploymentId))!.id).toBe(original.id);
        expect(await detail(job.deploymentId)).toMatchObject({ state: 'awaiting-health', stage: expect.stringMatching(/agent/i) });
        heartbeat(hostId, inventory()); await advance();
        expect(await operationFor(job.deploymentId)).toMatchObject({ kind: 'instance.restart', payload: { unhealthyRecovery: true } });
    });

    it.each([false, true])('rebuilds an already unhealthy container without requiring RCON emptiness once supported (supported=%s)', async supported => {
        const created = await launch('create'); await acknowledge(created.deploymentId);
        const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
        if (!supported) heartbeat(hostId, { ...inventory(), capabilities: [] });
        const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
        const rebuild = queueNativeRebuild(created.name, 'test');
        if (!rebuild.ok) throw new Error(rebuild.message);
        const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
        vi.mocked(rconStatusResult).mockClear();
        remote.state = 'unhealthy'; remote.players = 3;
        await advance();
        if (!supported) {
            expect(await operationFor(rebuild.deploymentId)).toBeNull();
            expect((await detail(rebuild.deploymentId)).stage).toMatch(/agent/i);
            heartbeat(hostId, inventory()); await advance();
        }
        expect(await operationFor(rebuild.deploymentId)).toMatchObject({ kind: 'instance.rebuild', generation: 1,
            payload: { unhealthyRecovery: true, nextGeneration: 2 } });
        expect(rconStatusResult).not.toHaveBeenCalled();
    });
});

describe('native provisioning cancellation and rollback', () => {
    it('cancels an unhealthy restart and prevents a late health response from reviving it', async () => {
        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
        const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
        const { POST } = await import('../../src/routes/api/provisioning/[id]/cancel/+server.js');
        const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
        const restart = requestLifecycle(job.name, 'restart'); if (!restart.ok) throw new Error(restart.error);
        await advance();
        const command = (await operationFor(restart.deploymentId))!;
        reportHostOperation(hostId, command.id, { state: 'completed' });
        let release!: (value: never) => void;
        vi.mocked(nativeRequest).mockImplementationOnce(() => new Promise(resolve => { release = resolve; }));
        const pending = advance();
        await vi.waitFor(() => expect(release).toBeTypeOf('function'));
        expect(POST({ params: { id: restart.deploymentId } }).status).toBe(200);
        release({ healthy: true, state: 'running' } as never);
        await pending;
        expect(await detail(restart.deploymentId)).toMatchObject({ state: 'cancelled' });
        const replacement = requestLifecycle(job.name, 'restart'); if (!replacement.ok) throw new Error(replacement.error);
        await advance();
        expect(await operationFor(replacement.deploymentId)).toMatchObject({ kind: 'instance.restart', state: 'pending' });
    });

    it.each([false, true])('withdraws an undelivered restart or waits for a delivered one before replacement (delivered=%s)', async delivered => {
        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
        const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
        const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
        const { pendingHostOperations, reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
        const first = requestLifecycle(job.name, 'restart'); if (!first.ok) throw new Error(first.error);
        await advance(); const command = (await operationFor(first.deploymentId))!;
        if (delivered) pendingHostOperations(hostId);
        expect(cancelProvisioningJob(first.deploymentId)).toBe(true);
        const next = requestLifecycle(job.name, 'restart'); if (!next.ok) throw new Error(next.error);
        await advance();
        if (delivered) {
            expect(await operationFor(next.deploymentId)).toBeNull();
            reportHostOperation(hostId, command.id, { state: 'completed' });
            await advance();
        } else {
            expect((await operationFor(first.deploymentId))?.state).toBe('failed');
            expect(pendingHostOperations(hostId).some(op => op.id === command.id)).toBe(false);
        }
        expect(await operationFor(next.deploymentId)).not.toBeNull();
        expect((await detail(first.deploymentId)).state).toBe('cancelled');
    });

    it('rolls back a completed native activation even while its restart is stuck starting', async () => {
        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
        const { db } = await import('../../src/lib/server/db/index.js');
        const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
        const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
        const { queueFileChange } = await import('../../src/lib/server/fleet/staged-files.js');
        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
        const { rollbackBeta } = await import('../../src/lib/server/fleet/beta.js');
        const { script, version } = createScript({ name: 'Broken beta', kind: 'script', targetRelPath: 'beta.gsc', content: Buffer.from('broken'), scopeType: 'game', scopeValue: 't5' });
        const betaId = Number(db().prepare("INSERT INTO betas(script_id,script_version_id,trigger_kind,state) VALUES(?,?,'manual','running')").run(script.id,version.id).lastInsertRowid);
        db().prepare('INSERT INTO beta_scripts(beta_id,script_id,script_version_id) VALUES(?,?,?)').run(betaId,script.id,version.id);
        db().prepare("INSERT INTO beta_servers(beta_id,server,had_original,stage) VALUES(?,?,0,'awaiting-empty')").run(betaId,job.name);
        db().prepare('INSERT INTO beta_server_scripts(beta_id,server,script_id,had_original) VALUES(?,?,?,0)').run(betaId,job.name,script.id);
        queueFileChange({ sourceKind: 'beta', sourceId: String(betaId), server: job.name, kind: 'script', relPath: 'beta.gsc', content: Buffer.from('broken'), inventorySource: 'beta' });
        const restart = requestRestart(job.name,{kind:'beta',id:String(betaId)});
        await advance();
        const staging = db().prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?').get(restart.id) as {stage_operation_id:string};
        reportHostOperation(hostId,staging.stage_operation_id,{state:'completed'});
        await advance();
        const activation = (await operationFor(restart.id))!;
        const files = JSON.parse((db().prepare('SELECT files_json FROM native_provisioning WHERE deployment_id=?').get(restart.id) as {files_json:string}).files_json);
        remote.state = 'starting';
        reportHostOperation(hostId,activation.id,{state:'completed',result:{before:files.map((f:{id:number;path:string})=>({...f,contentBase64:Buffer.from('working').toString('base64')}))}});
        // Rollback must consume the receipt itself; the health worker has not caught up.
        const rollback = await rollbackBeta(betaId);
        expect((await detail(restart.id)).state).toBe('cancelled');
        const replacement = db().prepare('SELECT provisioning_id FROM deployment_servers WHERE deployment_id=?').get(rollback.deploymentId) as {provisioning_id:string};
        expect(replacement.provisioning_id).not.toBe(restart.id);
        expect(db().prepare("SELECT (SELECT content FROM file_blobs WHERE sha256=content_hash) AS content FROM staged_file_changes WHERE source_kind='campaign' AND source_id=?").get(rollback.deploymentId))
            .toEqual({content:Buffer.from('working')});
        await advance();
        const stagedRollback = db().prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?').get(replacement.provisioning_id) as {stage_operation_id:string};
        reportHostOperation(hostId,stagedRollback.stage_operation_id,{state:'completed'});
        await advance();
        expect(await operationFor(replacement.provisioning_id)).toMatchObject({state:'pending'});
    });
});

describe('partial rebuild failure reconciliation', () => {
    const originalError = 'Replacement game library could not be prepared';
    const failure = (generation = 1) => ({ reconciled: true, originalError, rebuildFailure: {
        phase: 'disrupting', generation, nextGeneration: generation + 1,
        oldContainerMissing: true, replacementContainerMissing: true, replacementFilesRemoved: true
    } });

    async function cancelledRebuild() {
        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
        const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
        const first = requestLifecycle(job.name, 'rebuild');
        if (!first.ok) throw new Error(first.error);
        await advance();
        const command = (await operationFor(first.deploymentId))!;
        const { pendingHostOperations, reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
        pendingHostOperations(hostId);
        reportHostOperation(hostId, command.id, { state: 'unknown', error: originalError });
        const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
        expect(cancelProvisioningJob(first.deploymentId)).toBe(true);
        const replacement = requestLifecycle(job.name, 'rebuild');
        if (!replacement.ok) throw new Error(replacement.error);
        return { job, first, command, replacement };
    }

    it('records missing containers before releasing a queued rebuild without replaying the cancelled command', async () => {
        const { job, first, command, replacement } = await cancelledRebuild();
        const { db, closeDb } = await import('../../src/lib/server/db/index.js');
        const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
        const { getHostOperation, reportHostOperation, pendingHostOperations } = await import('../../src/lib/server/hosts/index.js');
        const { nativeInstance, nativeRequest } = await import('../../src/lib/server/backends/native.js');
        const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
        const { unsettledCancelledCommand } = await import('../../src/lib/server/provisioning/cancellation.js');
        const ports = db().prepare('SELECT host_id,instance_id,port,protocol FROM ec_host_ports ORDER BY protocol').all();
        await advance();
        expect(await operationFor(replacement.deploymentId)).toBeNull();
        expect(getDeployment(replacement.deploymentId)?.progress?.blocker).toMatchObject({ code: 'cancelled-command',
            href: `/provisioning/${first.deploymentId}`, label: 'View cancelled operation',
            message: expect.stringContaining(originalError) });
        expect(getDeployment(replacement.deploymentId)?.progress?.blocker?.message).toContain('unknown');
        reportHostOperation(hostId, command.id, { state: 'failed',
            error: `Old container removed; replacement creation failed. ${originalError}`, result: failure() });
        expect(unsettledCancelledCommand(job.name, replacement.deploymentId)).not.toBeNull();
        expect(db().prepare('SELECT applied_at FROM native_provisioning WHERE deployment_id=?').get(first.deploymentId))
            .toEqual({ applied_at: null });
        remote.state = 'missing';
        vi.mocked(nativeRequest).mockImplementationOnce(async () => {
            expect(db().prepare('SELECT container_state,last_players FROM servers WHERE name=?').get(job.name))
                .toEqual({ container_state: 'missing', last_players: null });
            expect(db().prepare('SELECT applied_at FROM native_provisioning WHERE deployment_id=?').get(first.deploymentId))
                .toEqual({ applied_at: expect.any(String) });
            expect(unsettledCancelledCommand(job.name, replacement.deploymentId)).toBeNull();
            return { exists: false, healthy: false, state: 'missing' } as never;
        });
        closeDb(); await advance();
        const next = (await operationFor(replacement.deploymentId))!;
        expect(next).toMatchObject({ kind: 'instance.rebuild', generation: 1, instanceId: command.instanceId,
            payload: { nextGeneration: 2, env: { PLUTO_SERVER_KEY: 'saved-key' } } });
        expect(next.id).not.toBe(command.id);
        expect(nativeInstance(job.name)).toMatchObject({ id: command.instanceId, generation: 1, retired_at: null });
        expect(getServerKey(job.name)).toBe('saved-key');
        expect(db().prepare('SELECT host_id,instance_id,port,protocol FROM ec_host_ports ORDER BY protocol').all()).toEqual(ports);
        expect((await detail(first.deploymentId)).state).toBe('cancelled');
        expect(getHostOperation(command.id)).toMatchObject({ state: 'failed', result: failure() });
        expect(pendingHostOperations(hostId).some(operation => operation.id === command.id)).toBe(false);
        await advance();
        expect((await operationFor(replacement.deploymentId))!.id).toBe(next.id);
        expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'").get()).toEqual({ count: 2 });
        remote.state = 'running'; await acknowledge(replacement.deploymentId);
        expect((await detail(replacement.deploymentId)).state).toBe('done');
        expect(nativeInstance(job.name)?.generation).toBe(2);
        expect((await detail(first.deploymentId)).state).toBe('cancelled');
    });

    it.each(['disrupting', 'creating'])('keeps an ambiguous %s rebuild blocked even when inspections currently see no containers', async phase => {
        const { job, first, command, replacement } = await cancelledRebuild();
        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
        reportHostOperation(hostId, command.id, { state: 'unknown', error: originalError,
            result: { checkpoint: { phase }, state: 'missing', oldContainerMissing: true, replacementContainerMissing: true } });
        remote.state = 'missing';
        const { nativeRequest, nativeInstance } = await import('../../src/lib/server/backends/native.js');
        vi.mocked(nativeRequest).mockClear();
        const { closeDb, db } = await import('../../src/lib/server/db/index.js'); closeDb();
        await advance(); await advance();
        expect(await operationFor(replacement.deploymentId)).toBeNull();
        expect(vi.mocked(nativeRequest)).not.toHaveBeenCalled();
        expect((await operationFor(first.deploymentId))!.id).toBe(command.id);
        expect(nativeInstance(job.name)?.generation).toBe(1);
        expect(db().prepare('SELECT applied_at FROM native_provisioning WHERE deployment_id=?').get(first.deploymentId))
            .toEqual({ applied_at: null });
        expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'").get()).toEqual({ count: 1 });
    });

    it.each(['creation started', 'replacement files remain', 'wrong target generation', 'command target mismatch'])('does not release a replacement from invalid partial evidence: %s', async evidence => {
        const { first, command, replacement } = await cancelledRebuild();
        const result = failure();
        if (evidence === 'creation started') result.rebuildFailure.phase = 'creating';
        if (evidence === 'replacement files remain') result.rebuildFailure.replacementFilesRemoved = false;
        if (evidence === 'wrong target generation') result.rebuildFailure.nextGeneration = 3;
        const { db } = await import('../../src/lib/server/db/index.js');
        if (evidence === 'command target mismatch') db().prepare("UPDATE ec_host_operation_payloads SET payload=json_set(payload,'$.nextGeneration',3) WHERE operation_id=?")
            .run(command.id);
        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
        reportHostOperation(hostId, command.id, { state: 'failed', error: originalError, result });
        remote.state = 'missing'; await advance();
        expect(await operationFor(replacement.deploymentId)).toBeNull();
        expect(db().prepare('SELECT applied_at FROM native_provisioning WHERE deployment_id=?').get(first.deploymentId))
            .toEqual({ applied_at: null });
        expect(db().prepare("SELECT COUNT(*) AS count FROM ec_host_operations WHERE kind='instance.rebuild'").get()).toEqual({ count: 1 });
    });

    it('records an uncancelled partial rebuild failure without claiming replacement generation or successful health', async () => {
        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
        const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
        const first = requestLifecycle(job.name, 'rebuild');
        if (!first.ok) throw new Error(first.error);
        await advance();
        const command = (await operationFor(first.deploymentId))!;
        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
        reportHostOperation(hostId, command.id, { state: 'failed', result: failure(),
            error: `Old container removed; replacement creation failed. ${originalError}` });
        await advance();
        const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        expect((await detail(first.deploymentId)).state).toBe('failed');
        expect(nativeInstance(job.name)).toMatchObject({ generation: 1, retired_at: null });
        expect(db().prepare('SELECT container_state,last_players FROM servers WHERE name=?').get(job.name))
            .toEqual({ container_state: 'missing', last_players: null });
        expect(db().prepare('SELECT applied_at,verified_at FROM native_provisioning WHERE deployment_id=?').get(first.deploymentId))
            .toEqual({ applied_at: expect.any(String), verified_at: null });
        expect(db().prepare('SELECT COUNT(*) AS count FROM ec_host_ports WHERE instance_id=?').get(command.instanceId)).toEqual({ count: 2 });
        const { getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
        expect(getServerKey(job.name)).toBe('saved-key');
    });
});

describe('cancelled native mutation bookkeeping', () => {
    it.each([null, 'before beta'])('keeps the original host receipt across beta revisions (original=%s)', async original => {
        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
        const { db } = await import('../../src/lib/server/db/index.js');
        const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
        const { openBeta, reviseBeta, rollbackBeta } = await import('../../src/lib/server/fleet/beta.js');
        const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
        const { reportHostOperation, getHostOperation } = await import('../../src/lib/server/hosts/index.js');
        const { script, version } = createScript({ name: 'Revised beta', kind: 'script', targetRelPath: 'revision.gsc',
            content: Buffer.from('revision one'), scopeType: 'game', scopeValue: 't5' });
        const { betaId } = await openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: version.id }],
            servers: [job.name], waitForHealth: true, waitForMatch: false });
        async function activate(before: string | null) {
            const restart = requestRestart(job.name, { kind: 'beta', id: String(betaId) });
            await advance();
            const staged = db().prepare('SELECT stage_operation_id FROM native_provisioning WHERE deployment_id = ?').get(restart.id) as { stage_operation_id: string };
            const files = getHostOperation(staged.stage_operation_id)!.payload.files as Array<{ id: number; path: string }>;
            reportHostOperation(hostId, staged.stage_operation_id, { state: 'completed' });
            await advance();
            // A dispatched activation must settle before revision submission.
            await expect(reviseBeta(betaId, 1, [{ scriptId: script.id, content: Buffer.from('premature update') }])).rejects.toThrow();
            const activation = (await operationFor(restart.id))!;
            reportHostOperation(hostId, activation.id, { state: 'completed', result: {
                before: files.map(file => ({ ...file, contentBase64: before === null ? null : Buffer.from(before).toString('base64') }))
            } });
            await advance();
            expect((await detail(restart.id)).state).toBe('done');
        }
        await activate(original);
        await reviseBeta(betaId, 1, [{ scriptId: script.id, content: Buffer.from('revision two') }]);
        await activate('revision one');
        const rollback = await rollbackBeta(betaId);
        expect(db().prepare("SELECT (SELECT content FROM file_blobs WHERE sha256=content_hash) AS content FROM staged_file_changes WHERE source_kind = 'campaign' AND source_id = ?").get(rollback.deploymentId))
            .toEqual({ content: original === null ? null : Buffer.from(original) });
    });

    it('keeps explicit cleanup available after cancelling an unsubmitted creation', async () => {
        const job = await queue();
        const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
        const { cancelCreation, canCancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
        expect(cancelProvisioningJob(job.deploymentId)).toBe(true);
        expect(canCancelCreation(job.deploymentId)).toBe(true);
        const cleanup = cancelCreation(job.deploymentId)!;
        expect((await detail(cleanup)).state).toBe('done');
    });

    it.each(['rebuild','destroy'] as const)('reconciles a delivered %s receipt without resuming the cancelled job', async action => {
        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
        const { db } = await import('../../src/lib/server/db/index.js');
        const { serverKeyPool, getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
        const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
        const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
        const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
        const { pendingHostOperations, reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
        const result = action === 'rebuild' ? requestLifecycle(job.name,'rebuild') : destroyServer(job.name);
        if (!result.ok || !result.deploymentId) throw new Error('Job was not queued');
        const id = result.deploymentId;
        await advance(); const operation = (await operationFor(id))!;
        pendingHostOperations(hostId);
        expect(cancelProvisioningJob(id)).toBe(true);
        expect(serverKeyPool('t5').available).toBe(0);
        const replacement = action === 'rebuild' ? requestLifecycle(job.name, 'restart') : null;
        if (replacement && !replacement.ok) throw new Error(replacement.error);
        await advance();
        reportHostOperation(hostId,operation.id,{state:'completed'});
        await advance(); await advance();
        if (replacement?.ok) expect(await operationFor(replacement.deploymentId)).toMatchObject({ kind: 'instance.restart', generation: 2 });
        expect((await detail(id)).state).toBe('cancelled');
        expect(serverKeyPool('t5').available).toBe(action === 'destroy' ? 1 : 0);
        expect(getServerKey(job.name)).toBe(action === 'destroy' ? undefined : 'saved-key');
        if(action === 'rebuild') expect(db().prepare('SELECT generation,retired_at FROM native_instances WHERE server=?').get(job.name)).toEqual({generation:2,retired_at:null});
        else {
            expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
            expect(db().prepare('SELECT * FROM servers WHERE name=?').get(job.name)).toBeUndefined();
        }
    });

    it.each([false,true])('cancels a host migration without restarting it or leaking destination reservations (delivered=%s)', async delivered => {
        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
        const hosts = await import('../../src/lib/server/hosts/index.js');
        const { serverKeyPool, getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
        const { getLibraryVersion, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
        const { cancelProvisioningJob } = await import('../../src/lib/server/provisioning/cancellation.js');
        const destination = hosts.createHost({name:'Destination',portMin:29000,portMax:29005});
        hosts.enrollHost(hosts.createEnrollment(destination.id).token,inventory());
        updateHostLibraryCaches(destination.id,[{libraryVersionId:libraryId,status:'ready',manifestSha256:getLibraryVersion(libraryId)!.manifestSha256}]);
        hosts.updateHost(hostId,{mode:'draining'});
        await processProvisioning();
        const migration = db().prepare('SELECT deployment_id,remove_operation_id FROM ec_host_migrations').get() as {deployment_id:string;remove_operation_id:string};
        expect(migration.remove_operation_id).toBeTruthy();
        if(delivered) hosts.pendingHostOperations(hostId);
        expect(cancelProvisioningJob(migration.deployment_id)).toBe(true);
        expect(serverKeyPool('t5').available).toBe(0);
        if(delivered) hosts.reportHostOperation(hostId,migration.remove_operation_id,{state:'completed'});
        await processProvisioning(); await processProvisioning();
        expect((await detail(migration.deployment_id)).state).toBe('cancelled');
        expect(db().prepare('SELECT * FROM ec_host_ports WHERE host_id=?').all(destination.id)).toEqual([]);
        expect(db().prepare('SELECT * FROM ec_host_migrations').all()).toHaveLength(1);
        expect(db().prepare('SELECT * FROM ec_host_ports WHERE host_id=?').all(hostId)).toHaveLength(delivered ? 0 : 2);
        expect(serverKeyPool('t5').available).toBe(delivered ? 1 : 0);
        expect(getServerKey(job.name)).toBe(delivered ? undefined : 'saved-key');
    });
});

describe('additional concurrency admission', () => {
    it.each(['createConcurrencyPerHost', 'lifecycleConcurrency'] as const)('holds simultaneous startups with %s until recovery completes', async key => {
        const first = await queue();
        const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
        saveFleetSettings({ ...fleetSettings(), createConcurrency: 5, [key]: 1 });
        const { queueCreateServer } = await import('../../src/lib/server/provisioning/instances.js');
        const second = queueCreateServer({ game: 't5', gameMode: 'zombie', mapSlug: 'theater', hostId, libraryId, image, plutoKey: 'second-key' });
        if (!second.ok) throw new Error(second.message);
        await advance();
        const operations = [await operationFor(first.deploymentId), await operationFor(second.deploymentId)];
        expect(operations.filter(Boolean)).toHaveLength(1);
        expect((await detail(second.deploymentId)).stage).toContain('capacity');
        saveFleetSettings({ ...fleetSettings(), [key]: 0 });
        await advance();
        expect(await operationFor(second.deploymentId)).not.toBeNull();
    });
});

describe('game key pool launch lifecycle', () => {
    it('reserves across maps atomically, rejects duplicate credentials, and reuses a cancelled launch key', async () => {
        await prepareLibrary();
        const { saveServerKey, serverKeyPool, getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
        const { queueCreateServer, destroyServer, previewCreation } = await import('../../src/lib/server/provisioning/instances.js');
        saveServerKey('t5', 'shared-t5-key', 'Any T5 map');
        saveServerKey('t6', 'shared-t6-key');
        const input = { game: 't5' as const, gameMode: 'zombie' as const, mapSlug: 'moon', hostId, libraryId, image };
        expect(previewCreation(input).hasSavedKey).toBe(true);
        const first = queueCreateServer(input);
        if (!first.ok) throw new Error(first.message);
        expect(first.name).toBe('t5_zombie_moon_1');
        expect(getServerKey(first.name)).toBe('shared-t5-key');
        expect(serverKeyPool('t5')).toEqual({ total: 1, available: 0 });
        expect(serverKeyPool('t6')).toEqual({ total: 1, available: 1 });
        expect(previewCreation({ ...input, mapSlug: 'theater' }).hasSavedKey).toBe(false);
        expect(queueCreateServer({ ...input, mapSlug: 'theater' }).ok).toBe(false);
        expect(queueCreateServer({ ...input, mapSlug: 'theater', plutoKey: 'shared-t5-key' }).ok).toBe(false);
        const { db, closeDb } = await import('../../src/lib/server/db/index.js');
        closeDb();
        expect(serverKeyPool('t5').available).toBe(0);
        expect(db().prepare('SELECT * FROM server_creation_jobs').all()).toHaveLength(1);
        expect(destroyServer(first.name).ok).toBe(true);
        expect(serverKeyPool('t5').available).toBe(1);
        const second = queueCreateServer({ ...input, mapSlug: 'theater' });
        if (!second.ok) throw new Error(second.message);
        expect(getServerKey(second.name)).toBe('shared-t5-key');
        expect(getServerKey(first.name)).toBeUndefined();
    });

    it('keeps a key through stop and rebuild, then releases it only after confirmed removal', async () => {
        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
        const { serverKeyPool, getServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
        const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
        const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
        const stopped = requestLifecycle(job.name, 'stop');
        if (!stopped.ok) throw new Error(stopped.error);
        await advance(); remote.state = 'stopped'; await acknowledge(stopped.deploymentId);
        expect(serverKeyPool('t5').available).toBe(0);
        const rebuilt = requestLifecycle(job.name, 'rebuild');
        if (!rebuilt.ok) throw new Error(rebuilt.error);
        await advance(); remote.state = 'running'; await acknowledge(rebuilt.deploymentId);
        expect(getServerKey(job.name)).toBe('saved-key');
        expect(serverKeyPool('t5').available).toBe(0);
        const removal = destroyServer(job.name);
        if (!removal.ok || !removal.deploymentId) throw new Error('Expected removal');
        await advance();
        expect(serverKeyPool('t5').available).toBe(0);
        const operation = (await operationFor(removal.deploymentId))!;
        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
        reportHostOperation(hostId, operation.id, { state: 'unknown', error: 'Receipt lost' });
        await advance();
        expect(serverKeyPool('t5').available).toBe(0);
        await acknowledge(removal.deploymentId);
        expect(serverKeyPool('t5')).toEqual({ total: 1, available: 1 });
        expect(getServerKey(job.name)).toBeUndefined();
    });

    it('keeps the reservation for a submitted cancellation until the host confirms cleanup', async () => {
        const job = await queue(); await advance();
        const { serverKeyPool } = await import('../../src/lib/server/fleet/server-keys.js');
        const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
        const cleanup = cancelCreation(job.deploymentId)!;
        expect(cleanup).toEqual(expect.any(String));
        expect(serverKeyPool('t5').available).toBe(0);
        await acknowledge(cleanup);
        expect(serverKeyPool('t5')).toEqual({ total: 1, available: 1 });
    });
});