Block server destruction while linked to an active beta
Protect individual, bulk, and queued removals and rebuilds until beta closure. Recheck ownership before teardown and reject beta admission during pending or unresolved destruction. Fixes #36
This commit is contained in:
@@ -0,0 +1,294 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { access } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
|
||||
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
||||
komodoCreateDeployment: vi.fn(async () => ({ ok: true })),
|
||||
komodoDeploy: vi.fn(async () => ({ ok: true })),
|
||||
komodoDestroyDeployment: vi.fn(async () => ({ ok: true })),
|
||||
komodoDeleteDeployment: vi.fn(async () => ({ ok: true })),
|
||||
komodoInspectHealthy: vi.fn(),
|
||||
komodoEnsureTag: vi.fn(async (name: string) => ({ ok: true, id: name })),
|
||||
komodoUpdateTagsOnResource: vi.fn(async () => ({ ok: true }))
|
||||
}));
|
||||
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
||||
rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn()
|
||||
}));
|
||||
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
|
||||
|
||||
let fixture: Fixture;
|
||||
const otherServer = 't5_zombie_cosmodrome_1';
|
||||
const server = 't5_zombie_theater_1';
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.resetModules(); vi.clearAllMocks();
|
||||
fixture = await newFixture([otherServer, server]);
|
||||
for (const [key, value] of Object.entries({ KOMODO_ADDRESS: 'http://komodo.test', KOMODO_API_KEY: 'test',
|
||||
KOMODO_API_SECRET: 'test', KOMODO_TARGET_SERVER: 'test-host', RCON_PASSWORD: 'test',
|
||||
SERVER_PORT_MIN: '28960', SERVER_PORT_MAX: '28980' })) vi.stubEnv(key, value);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
for (const [index, name] of [otherServer, server].entries()) {
|
||||
const map = name === server ? 'theater' : 'cosmodrome';
|
||||
db().prepare(`INSERT INTO servers (name, komodo_type, komodo_name, game, map, game_mode, map_slug, game_port, host_dir)
|
||||
VALUES (?, 'deployment', ?, 't5', ?, 'zombie', ?, ?, ?)`).run(name, name, 'zombie_' + map, map, 28960 + index, join(fixture.t5, name));
|
||||
}
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(rconGetPlayerCount).mockReset().mockResolvedValue(0);
|
||||
vi.mocked(komodoInspectHealthy).mockReset().mockResolvedValue({ ok: false, state: 'exited', reason: 'stopped' });
|
||||
});
|
||||
afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); });
|
||||
|
||||
async function attachBeta(state = 'running', outcome?: string, deleted = false) {
|
||||
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { script, version } = createScript({ name: 'Protected beta', targetRelPath: 'protected.gsc',
|
||||
scopeType: 'game', scopeValue: 't5', content: Buffer.from('beta content') });
|
||||
const id = Number(db().prepare(`INSERT INTO betas (script_id, script_version_id, trigger_kind, state)
|
||||
VALUES (?, ?, 'manual', ?)`).run(script.id, version.id, state).lastInsertRowid);
|
||||
db().prepare(`INSERT INTO beta_servers (beta_id, server, had_original, stage, server_deleted_at)
|
||||
VALUES (?, ?, 0, ?, ?)`).run(id, server, outcome === 'failed' ? 'failed' : 'ready', deleted ? '2026-09-01' : null);
|
||||
if (outcome) db().prepare('INSERT INTO beta_test_plans (beta_id, plan, outcome) VALUES (?, ?, ?)').run(id, '{}', outcome);
|
||||
return id;
|
||||
}
|
||||
|
||||
async function expectServerPreserved() {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
expect(komodoDeleteDeployment).not.toHaveBeenCalled();
|
||||
expect(db().prepare('SELECT name FROM servers WHERE name = ?').get(server)).toEqual({ name: server });
|
||||
await expect(access(join(fixture.t5, server))).resolves.toBeUndefined();
|
||||
}
|
||||
|
||||
describe('active beta destruction guard', () => {
|
||||
it.each([
|
||||
{ state: 'running', outcome: undefined },
|
||||
{ state: 'ready-promote', outcome: 'passed' },
|
||||
{ state: 'running', outcome: 'paused' },
|
||||
{ state: 'running', outcome: 'failed' }
|
||||
])('rejects destruction for $state with test outcome $outcome', async ({ state, outcome }) => {
|
||||
const id = await attachBeta(state, outcome);
|
||||
const { destroyServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const result = destroyServer(server);
|
||||
expect(result).toMatchObject({ ok: false, kind: 'conflict' });
|
||||
if (result.ok) throw new Error('Expected active beta conflict');
|
||||
expect(result.message).toContain(String(id));
|
||||
expect(result.message).toMatch(/promot/i);
|
||||
expect(result.message).toMatch(/roll.?back/i);
|
||||
expect(db().prepare("SELECT id FROM server_deployments WHERE action = 'destroy'").all()).toEqual([]);
|
||||
await expectServerPreserved();
|
||||
});
|
||||
|
||||
it('returns HTTP 409 with actionable beta information and does not kick the worker', async () => {
|
||||
const id = await attachBeta();
|
||||
const { DELETE } = await import('../../src/routes/api/servers/[name]/+server.js');
|
||||
const { kick } = await import('../../src/lib/server/pipeline/worker.js');
|
||||
const response = await DELETE({ params: { name: server } });
|
||||
expect(response.status).toBe(409);
|
||||
expect(await response.json()).toMatchObject({ ok: false, kind: 'conflict', error: expect.stringContaining(String(id)) });
|
||||
expect(kick).not.toHaveBeenCalled();
|
||||
await expectServerPreserved();
|
||||
});
|
||||
|
||||
it('rejects a rebuild request with HTTP 409 before reserving a replacement', async () => {
|
||||
const id = await attachBeta();
|
||||
const { POST } = await import('../../src/routes/api/provisioning/+server.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const response = await POST({ request: new Request('http://localhost/api/provisioning', {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ server, action: 'rebuild' })
|
||||
}) });
|
||||
expect(response.status).toBe(409);
|
||||
expect(await response.json()).toMatchObject({ error: expect.stringContaining(String(id)) });
|
||||
expect(db().prepare('SELECT * FROM server_creation_jobs').all()).toEqual([]);
|
||||
await expectServerPreserved();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ state: 'promoted', deleted: false },
|
||||
{ state: 'rolled-back', deleted: false },
|
||||
{ state: 'running', deleted: true },
|
||||
{ state: 'ready-promote', deleted: true }
|
||||
])('allows removal for $state with historical deletion $deleted', async ({ state, deleted }) => {
|
||||
await attachBeta(state, undefined, deleted);
|
||||
const { destroyServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
expect(destroyServer(server)).toMatchObject({ ok: true, deploymentId: expect.any(String) });
|
||||
});
|
||||
|
||||
it('holds a queued destruction until its beta closes, then completes the same request', async () => {
|
||||
const { destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
const queued = destroyServer(server);
|
||||
expect(queued.ok).toBe(true);
|
||||
const id = await attachBeta();
|
||||
await processDestructions();
|
||||
expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'waiting', stage: expect.stringMatching(/beta/i) });
|
||||
expect(rconGetPlayerCount).not.toHaveBeenCalled();
|
||||
await expectServerPreserved();
|
||||
db().prepare("UPDATE betas SET state = 'rolled-back', closed_at = ? WHERE id = ?").run(new Date().toISOString(), id);
|
||||
await processDestructions();
|
||||
expect(komodoDestroyDeployment).toHaveBeenCalledOnce();
|
||||
expect(komodoDeleteDeployment).toHaveBeenCalledOnce();
|
||||
expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'done', server_exists: 0 });
|
||||
});
|
||||
|
||||
it('rechecks beta ownership after an in-flight empty-server probe', async () => {
|
||||
const { destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
let resolveCount!: (players: number) => void;
|
||||
vi.mocked(rconGetPlayerCount).mockImplementationOnce(() => new Promise(resolve => { resolveCount = resolve; }));
|
||||
const queued = destroyServer(server);
|
||||
const work = processDestructions();
|
||||
await vi.waitFor(() => expect(resolveCount).toBeTypeOf('function'));
|
||||
await attachBeta();
|
||||
resolveCount(0);
|
||||
await work;
|
||||
expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'waiting', stage: expect.stringMatching(/beta/i) });
|
||||
await expectServerPreserved();
|
||||
});
|
||||
|
||||
it('also holds rebuild destruction when a beta appears during the occupancy probe', async () => {
|
||||
const { requestLifecycle, processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
|
||||
const { saveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
saveServerKey(server, 'saved-key');
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: true, state: 'running' });
|
||||
let resolveCount!: (players: number) => void;
|
||||
vi.mocked(rconGetPlayerCount).mockImplementationOnce(() => new Promise(resolve => { resolveCount = resolve; }));
|
||||
const queued = requestLifecycle(server, 'rebuild');
|
||||
if (!queued.ok) throw new Error(queued.error);
|
||||
const work = processLifecycle();
|
||||
await vi.waitFor(() => expect(resolveCount).toBeTypeOf('function'));
|
||||
await attachBeta();
|
||||
resolveCount(0);
|
||||
await work;
|
||||
expect(getDeployment(queued.deploymentId)?.deployment).toMatchObject({ state: 'waiting', stage: expect.stringMatching(/beta/i) });
|
||||
await expectServerPreserved();
|
||||
});
|
||||
|
||||
it('preserves the cleanup checkpoint and files while an active beta blocks a legacy cleanup resume', async () => {
|
||||
const { destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { deploymentStage, finishDeployment, resumeCleanup, getDeployment, REMOVING_SERVER } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const queued = destroyServer(server);
|
||||
deploymentStage(queued.deploymentId!, REMOVING_SERVER);
|
||||
finishDeployment(queued.deploymentId!, { ok: false, message: 'Interrupted local cleanup' });
|
||||
const id = await attachBeta();
|
||||
expect(resumeCleanup(queued.deploymentId!)).toBe(true);
|
||||
await processDestructions();
|
||||
expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'failed', stage: REMOVING_SERVER,
|
||||
error: expect.stringMatching(/beta/i) });
|
||||
expect(getDeployment(queued.deploymentId!)?.canResumeCleanup).toBe(true);
|
||||
await expectServerPreserved();
|
||||
db().prepare("UPDATE betas SET state = 'promoted' WHERE id = ?").run(id);
|
||||
expect(resumeCleanup(queued.deploymentId!)).toBe(true);
|
||||
await processDestructions();
|
||||
expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'done', server_exists: 0 });
|
||||
const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
expect(komodoDeleteDeployment).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(['destroy', 'rebuild'] as const)('requires cancelling a pending %s before opening a beta', async action => {
|
||||
const { destroyServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { requestLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js');
|
||||
const { cancelRemoval } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { cancelProvisioning } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
const { saveServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
saveServerKey(server, 'saved-key');
|
||||
const queued = action === 'destroy' ? destroyServer(server) : requestLifecycle(server, 'rebuild');
|
||||
expect(queued.ok).toBe(true);
|
||||
if (!queued.ok) throw new Error('Expected queued removal');
|
||||
const { script, version } = createScript({ name: 'Beta admission', targetRelPath: 'admission.gsc',
|
||||
scopeType: 'game', scopeValue: 't5', content: Buffer.from('beta content') });
|
||||
const input = { scripts: [{ scriptId: script.id, scriptVersionId: version.id }], servers: [server],
|
||||
waitForHealth: false, waitForMatch: false };
|
||||
await expect(openBeta(input)).rejects.toMatchObject({ kind: 'conflict', message: expect.stringMatching(/cancel or finish/i) });
|
||||
expect(db().prepare('SELECT id FROM betas').all()).toEqual([]);
|
||||
expect(action === 'destroy' ? cancelRemoval(queued.deploymentId!) : cancelProvisioning(queued.deploymentId!)).toBe(true);
|
||||
await expect(openBeta(input)).resolves.toMatchObject({ servers: [server], betaId: expect.any(Number) });
|
||||
await expectServerPreserved();
|
||||
});
|
||||
|
||||
it.each((['destroy', 'rebuild'] as const).flatMap(action => ['failed', 'interrupted'].flatMap(state =>
|
||||
['Stopping and removing container', 'Deleting Komodo deployment', 'Removing server files and registry'].map(stage => ({ action, state, stage }))
|
||||
)))('rejects beta admission after $action is $state at $stage', async ({ action, state, stage }) => {
|
||||
const { beginDeployment } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const operation = beginDeployment(server, action);
|
||||
db().prepare('UPDATE server_deployments SET state = ?, stage = ? WHERE id = ?').run(state, stage, operation);
|
||||
const { script, version } = createScript({ name: 'Beta admission', targetRelPath: 'admission.gsc',
|
||||
scopeType: 'game', scopeValue: 't5', content: Buffer.from('beta content') });
|
||||
await expect(openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: version.id }], servers: [server],
|
||||
waitForHealth: false, waitForMatch: false })).rejects.toMatchObject({ kind: 'conflict' });
|
||||
expect(db().prepare('SELECT id FROM betas').all()).toEqual([]);
|
||||
expect(db().prepare('SELECT * FROM staged_file_changes').all()).toEqual([]);
|
||||
await expectServerPreserved();
|
||||
});
|
||||
});
|
||||
|
||||
describe('bulk removal beta protection', () => {
|
||||
it('rolls back earlier House Party removals when another target has an active beta', async () => {
|
||||
const id = await attachBeta();
|
||||
const { housePartyTargets, queueHouseParty } = await import('../../src/lib/server/fleet/house-party.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
expect(housePartyTargets()).toEqual([otherServer, server]);
|
||||
expect(() => queueHouseParty(housePartyTargets())).toThrow(new RegExp(String(id)));
|
||||
expect(db().prepare("SELECT id FROM server_deployments WHERE action = 'destroy'").all()).toEqual([]);
|
||||
await expectServerPreserved();
|
||||
});
|
||||
|
||||
it('returns an actionable HTTP 409 for a blocked House Party without queuing partial removals', async () => {
|
||||
const id = await attachBeta();
|
||||
const { POST } = await import('../../src/routes/api/provisioning/house-party/+server.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const response = await POST({ request: new Request('http://localhost/api/provisioning/house-party', {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ servers: [otherServer, server] })
|
||||
}) });
|
||||
expect(response.status).toBe(409);
|
||||
expect(await response.json()).toMatchObject({ ok: false, error: expect.stringContaining(String(id)) });
|
||||
expect(db().prepare("SELECT id FROM server_deployments WHERE action = 'destroy'").all()).toEqual([]);
|
||||
await expectServerPreserved();
|
||||
});
|
||||
|
||||
it('shows the active beta as an exact collection blocker while fill mode remains available', async () => {
|
||||
const id = await attachBeta();
|
||||
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
||||
const { previewCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const blueprint = saveBlueprint({ name: 'Replace fleet', entries: [{ game: 't5', mapSlug: 'moon', count: 1 }] });
|
||||
const preview = previewCollection(blueprint.id, 'exact');
|
||||
expect(preview.ready).toBe(false);
|
||||
expect(preview.blockers).toContainEqual(expect.stringContaining(String(id)));
|
||||
expect(preview.blockers.join(' ')).toMatch(/promot.*roll.?back/i);
|
||||
expect(previewCollection(blueprint.id, 'fill').ready).toBe(true);
|
||||
});
|
||||
|
||||
it('rolls back an exact collection if a beta starts after its preview', async () => {
|
||||
const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
|
||||
const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const blueprint = saveBlueprint({ name: 'Replace fleet', entries: [{ game: 't5', mapSlug: 'moon', count: 1 }] });
|
||||
const preview = previewCollection(blueprint.id, 'exact');
|
||||
expect(preview.removals.map(target => target.name)).toEqual([otherServer, server]);
|
||||
const id = await attachBeta();
|
||||
expect(() => queueCollection({ previewId: preview.id, keys: { t5_zombie_moon_1: 'new-key' } })).toThrow(new RegExp(String(id)));
|
||||
for (const table of ['server_deployments', 'server_launch_collections', 'server_collection_removals', 'server_creation_jobs', 'server_keys']) {
|
||||
expect(db().prepare(`SELECT * FROM ${table}`).all()).toEqual([]);
|
||||
}
|
||||
expect(db().prepare('SELECT collection_id FROM server_collection_previews WHERE id = ?').get(preview.id)).toEqual({ collection_id: null });
|
||||
await expectServerPreserved();
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user