electric-cherry/tests/unit/beta-destruction.test.ts

import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { access } from 'node:fs/promises';
import { join } from 'node:path';
import { newFixture, type Fixture } from '../helpers/fixture.js';

vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
    komodoCreateDeployment: vi.fn(async () => ({ ok: true })),
    komodoDeploy: vi.fn(async () => ({ ok: true })),
    komodoDestroyDeployment: vi.fn(async () => ({ ok: true })),
    komodoDeleteDeployment: vi.fn(async () => ({ ok: true })),
    komodoInspectHealthy: vi.fn(),
    komodoEnsureTag: vi.fn(async (name: string) => ({ ok: true, id: name })),
    komodoUpdateTagsOnResource: vi.fn(async () => ({ ok: true }))
}));
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
    rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn()
}));
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));

let fixture: Fixture;
const otherServer = 't5_zombie_cosmodrome_1';
const server = 't5_zombie_theater_1';

beforeEach(async () => {
    vi.resetModules(); vi.clearAllMocks();
    fixture = await newFixture([otherServer, server]);
    for (const [key, value] of Object.entries({ KOMODO_ADDRESS: 'http://komodo.test', KOMODO_API_KEY: 'test',
        KOMODO_API_SECRET: 'test', KOMODO_TARGET_SERVER: 'test-host', RCON_PASSWORD: 'test',
        SERVER_PORT_MIN: '28960', SERVER_PORT_MAX: '28980' })) vi.stubEnv(key, value);
    const { db } = await import('../../src/lib/server/db/index.js');
    for (const [index, name] of [otherServer, server].entries()) {
        const map = name === server ? 'theater' : 'cosmodrome';
        db().prepare(`INSERT INTO servers (name, komodo_type, komodo_name, game, map, game_mode, map_slug, game_port, host_dir)
            VALUES (?, 'deployment', ?, 't5', ?, 'zombie', ?, ?, ?)`).run(name, name, 'zombie_' + map, map, 28960 + index, join(fixture.t5, name));
    }
    const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
    const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
    vi.mocked(rconGetPlayerCount).mockReset().mockResolvedValue(0);
    vi.mocked(komodoInspectHealthy).mockReset().mockResolvedValue({ ok: false, state: 'exited', reason: 'stopped' });
});
afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); });

async function attachBeta(state = 'running', outcome?: string, deleted = false) {
    const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
    const { db } = await import('../../src/lib/server/db/index.js');
    const { script, version } = createScript({ name: 'Protected beta', targetRelPath: 'protected.gsc',
        scopeType: 'game', scopeValue: 't5', content: Buffer.from('beta content') });
    const id = Number(db().prepare(`INSERT INTO betas (script_id, script_version_id, trigger_kind, state)
        VALUES (?, ?, 'manual', ?)`).run(script.id, version.id, state).lastInsertRowid);
    db().prepare(`INSERT INTO beta_servers (beta_id, server, had_original, stage, server_deleted_at)
        VALUES (?, ?, 0, ?, ?)`).run(id, server, outcome === 'failed' ? 'failed' : 'ready', deleted ? '2026-09-01' : null);
    if (outcome) db().prepare('INSERT INTO beta_test_plans (beta_id, plan, outcome) VALUES (?, ?, ?)').run(id, '{}', outcome);
    return id;
}

async function expectServerPreserved() {
    const { db } = await import('../../src/lib/server/db/index.js');
    const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js');
    expect(komodoDestroyDeployment).not.toHaveBeenCalled();
    expect(komodoDeleteDeployment).not.toHaveBeenCalled();
    expect(db().prepare('SELECT name FROM servers WHERE name = ?').get(server)).toEqual({ name: server });
    await expect(access(join(fixture.t5, server))).resolves.toBeUndefined();
}

describe('active beta destruction guard', () => {
    it.each([
        { state: 'running', outcome: undefined },
        { state: 'ready-promote', outcome: 'passed' },
        { state: 'running', outcome: 'paused' },
        { state: 'running', outcome: 'failed' }
    ])('rejects destruction for $state with test outcome $outcome', async ({ state, outcome }) => {
        const id = await attachBeta(state, outcome);
        const { destroyServer } = await import('../../src/lib/server/fleet/create.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        const result = destroyServer(server);
        expect(result).toMatchObject({ ok: false, kind: 'conflict' });
        if (result.ok) throw new Error('Expected active beta conflict');
        expect(result.message).toContain(String(id));
        expect(result.message).toMatch(/promot/i);
        expect(result.message).toMatch(/roll.?back/i);
        expect(db().prepare("SELECT id FROM server_deployments WHERE action = 'destroy'").all()).toEqual([]);
        await expectServerPreserved();
    });

    it('returns HTTP 409 with actionable beta information and does not kick the worker', async () => {
        const id = await attachBeta();
        const { DELETE } = await import('../../src/routes/api/servers/[name]/+server.js');
        const { kick } = await import('../../src/lib/server/pipeline/worker.js');
        const response = await DELETE({ params: { name: server } });
        expect(response.status).toBe(409);
        expect(await response.json()).toMatchObject({ ok: false, kind: 'conflict', error: expect.stringContaining(String(id)) });
        expect(kick).not.toHaveBeenCalled();
        await expectServerPreserved();
    });

    it('rejects a rebuild request with HTTP 409 before reserving a replacement', async () => {
        const id = await attachBeta();
        const { POST } = await import('../../src/routes/api/provisioning/+server.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        const response = await POST({ request: new Request('http://localhost/api/provisioning', {
            method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ server, action: 'rebuild' })
        }) });
        expect(response.status).toBe(409);
        expect(await response.json()).toMatchObject({ error: expect.stringContaining(String(id)) });
        expect(db().prepare('SELECT * FROM server_creation_jobs').all()).toEqual([]);
        await expectServerPreserved();
    });

    it.each([
        { state: 'promoted', deleted: false },
        { state: 'rolled-back', deleted: false },
        { state: 'running', deleted: true },
        { state: 'ready-promote', deleted: true }
    ])('allows removal for $state with historical deletion $deleted', async ({ state, deleted }) => {
        await attachBeta(state, undefined, deleted);
        const { destroyServer } = await import('../../src/lib/server/fleet/create.js');
        expect(destroyServer(server)).toMatchObject({ ok: true, deploymentId: expect.any(String) });
    });

    it('holds a queued destruction until its beta closes, then completes the same request', async () => {
        const { destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
        const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
        const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js');
        const queued = destroyServer(server);
        expect(queued.ok).toBe(true);
        const id = await attachBeta();
        await processDestructions();
        expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'waiting', stage: expect.stringMatching(/beta/i) });
        expect(rconGetPlayerCount).not.toHaveBeenCalled();
        await expectServerPreserved();
        db().prepare("UPDATE betas SET state = 'rolled-back', closed_at = ? WHERE id = ?").run(new Date().toISOString(), id);
        await processDestructions();
        expect(komodoDestroyDeployment).toHaveBeenCalledOnce();
        expect(komodoDeleteDeployment).toHaveBeenCalledOnce();
        expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'done', server_exists: 0 });
    });

    it('rechecks beta ownership after an in-flight empty-server probe', async () => {
        const { destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
        const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
        const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
        let resolveCount!: (players: number) => void;
        vi.mocked(rconGetPlayerCount).mockImplementationOnce(() => new Promise(resolve => { resolveCount = resolve; }));
        const queued = destroyServer(server);
        const work = processDestructions();
        await vi.waitFor(() => expect(resolveCount).toBeTypeOf('function'));
        await attachBeta();
        resolveCount(0);
        await work;
        expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'waiting', stage: expect.stringMatching(/beta/i) });
        await expectServerPreserved();
    });

    it('also holds rebuild destruction when a beta appears during the occupancy probe', async () => {
        const { requestLifecycle, processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
        const { saveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
        const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
        const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
        const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
        saveServerKey(server, 'saved-key');
        vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: true, state: 'running' });
        let resolveCount!: (players: number) => void;
        vi.mocked(rconGetPlayerCount).mockImplementationOnce(() => new Promise(resolve => { resolveCount = resolve; }));
        const queued = requestLifecycle(server, 'rebuild');
        if (!queued.ok) throw new Error(queued.error);
        const work = processLifecycle();
        await vi.waitFor(() => expect(resolveCount).toBeTypeOf('function'));
        await attachBeta();
        resolveCount(0);
        await work;
        expect(getDeployment(queued.deploymentId)?.deployment).toMatchObject({ state: 'waiting', stage: expect.stringMatching(/beta/i) });
        await expectServerPreserved();
    });

    it('preserves the cleanup checkpoint and files while an active beta blocks a legacy cleanup resume', async () => {
        const { destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
        const { deploymentStage, finishDeployment, resumeCleanup, getDeployment, REMOVING_SERVER } = await import('../../src/lib/server/fleet/deployments.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        const queued = destroyServer(server);
        deploymentStage(queued.deploymentId!, REMOVING_SERVER);
        finishDeployment(queued.deploymentId!, { ok: false, message: 'Interrupted local cleanup' });
        const id = await attachBeta();
        expect(resumeCleanup(queued.deploymentId!)).toBe(true);
        await processDestructions();
        expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'failed', stage: REMOVING_SERVER,
            error: expect.stringMatching(/beta/i) });
        expect(getDeployment(queued.deploymentId!)?.canResumeCleanup).toBe(true);
        await expectServerPreserved();
        db().prepare("UPDATE betas SET state = 'promoted' WHERE id = ?").run(id);
        expect(resumeCleanup(queued.deploymentId!)).toBe(true);
        await processDestructions();
        expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'done', server_exists: 0 });
        const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js');
        expect(komodoDestroyDeployment).not.toHaveBeenCalled();
        expect(komodoDeleteDeployment).not.toHaveBeenCalled();
    });

    it.each(['destroy', 'rebuild'] as const)('requires cancelling a pending %s before opening a beta', async action => {
        const { destroyServer } = await import('../../src/lib/server/fleet/create.js');
        const { requestLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
        const { cancelRemoval } = await import('../../src/lib/server/fleet/deployments.js');
        const { cancelProvisioning } = await import('../../src/lib/server/provisioning/requests.js');
        const { saveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
        const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
        const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        saveServerKey(server, 'saved-key');
        const queued = action === 'destroy' ? destroyServer(server) : requestLifecycle(server, 'rebuild');
        expect(queued.ok).toBe(true);
        if (!queued.ok) throw new Error('Expected queued removal');
        const { script, version } = createScript({ name: 'Beta admission', targetRelPath: 'admission.gsc',
            scopeType: 'game', scopeValue: 't5', content: Buffer.from('beta content') });
        const input = { scripts: [{ scriptId: script.id, scriptVersionId: version.id }], servers: [server],
            waitForHealth: false, waitForMatch: false };
        await expect(openBeta(input)).rejects.toMatchObject({ kind: 'conflict', message: expect.stringMatching(/cancel or finish/i) });
        expect(db().prepare('SELECT id FROM betas').all()).toEqual([]);
        expect(action === 'destroy' ? cancelRemoval(queued.deploymentId!) : cancelProvisioning(queued.deploymentId!)).toBe(true);
        await expect(openBeta(input)).resolves.toMatchObject({ servers: [server], betaId: expect.any(Number) });
        await expectServerPreserved();
    });

    it.each((['destroy', 'rebuild'] as const).flatMap(action => ['failed', 'interrupted'].flatMap(state =>
        ['Stopping and removing container', 'Deleting Komodo deployment', 'Removing server files and registry'].map(stage => ({ action, state, stage }))
    )))('rejects beta admission after $action is $state at $stage', async ({ action, state, stage }) => {
        const { beginDeployment } = await import('../../src/lib/server/fleet/deployments.js');
        const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
        const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        const operation = beginDeployment(server, action);
        db().prepare('UPDATE server_deployments SET state = ?, stage = ? WHERE id = ?').run(state, stage, operation);
        const { script, version } = createScript({ name: 'Beta admission', targetRelPath: 'admission.gsc',
            scopeType: 'game', scopeValue: 't5', content: Buffer.from('beta content') });
        await expect(openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: version.id }], servers: [server],
            waitForHealth: false, waitForMatch: false })).rejects.toMatchObject({ kind: 'conflict' });
        expect(db().prepare('SELECT id FROM betas').all()).toEqual([]);
        expect(db().prepare('SELECT * FROM staged_file_changes').all()).toEqual([]);
        await expectServerPreserved();
    });
});

describe('bulk removal beta protection', () => {
    it('rolls back earlier House Party removals when another target has an active beta', async () => {
        const id = await attachBeta();
        const { housePartyTargets, queueHouseParty } = await import('../../src/lib/server/fleet/house-party.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        expect(housePartyTargets()).toEqual([otherServer, server]);
        expect(() => queueHouseParty(housePartyTargets())).toThrow(new RegExp(String(id)));
        expect(db().prepare("SELECT id FROM server_deployments WHERE action = 'destroy'").all()).toEqual([]);
        await expectServerPreserved();
    });

    it('returns an actionable HTTP 409 for a blocked House Party without queuing partial removals', async () => {
        const id = await attachBeta();
        const { POST } = await import('../../src/routes/api/provisioning/house-party/+server.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        const response = await POST({ request: new Request('http://localhost/api/provisioning/house-party', {
            method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ servers: [otherServer, server] })
        }) });
        expect(response.status).toBe(409);
        expect(await response.json()).toMatchObject({ ok: false, error: expect.stringContaining(String(id)) });
        expect(db().prepare("SELECT id FROM server_deployments WHERE action = 'destroy'").all()).toEqual([]);
        await expectServerPreserved();
    });

    it('shows the active beta as an exact collection blocker while fill mode remains available', async () => {
        const id = await attachBeta();
        const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
        const { previewCollection } = await import('../../src/lib/server/fleet/collections.js');
        const blueprint = saveBlueprint({ name: 'Replace fleet', entries: [{ game: 't5', mapSlug: 'moon', count: 1 }] });
        const preview = previewCollection(blueprint.id, 'exact');
        expect(preview.ready).toBe(false);
        expect(preview.blockers).toContainEqual(expect.stringContaining(String(id)));
        expect(preview.blockers.join(' ')).toMatch(/promot.*roll.?back/i);
        expect(previewCollection(blueprint.id, 'fill').ready).toBe(true);
    });

    it('rolls back an exact collection if a beta starts after its preview', async () => {
        const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
        const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
        const { db } = await import('../../src/lib/server/db/index.js');
        const blueprint = saveBlueprint({ name: 'Replace fleet', entries: [{ game: 't5', mapSlug: 'moon', count: 1 }] });
        const preview = previewCollection(blueprint.id, 'exact');
        expect(preview.removals.map(target => target.name)).toEqual([otherServer, server]);
        const id = await attachBeta();
        expect(() => queueCollection({ previewId: preview.id, keys: { t5_zombie_moon_1: 'new-key' } })).toThrow(new RegExp(String(id)));
        for (const table of ['server_deployments', 'server_launch_collections', 'server_collection_removals', 'server_creation_jobs', 'server_keys']) {
            expect(db().prepare(`SELECT * FROM ${table}`).all()).toEqual([]);
        }
        expect(db().prepare('SELECT collection_id FROM server_collection_previews WHERE id = ?').get(preview.id)).toEqual({ collection_id: null });
        await expectServerPreserved();
    });
});