From b904ddbc486598c33ccff73ff4d6c4601ae3be6d Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Sun, 27 Sep 2026 23:21:37 -0400 Subject: [PATCH] Block server destruction while linked to an active beta Protect individual, bulk, and queued removals and rebuilds until beta closure. Recheck ownership before teardown and reject beta admission during pending or unresolved destruction. Fixes #36 --- README.md | 8 +- docs/provisioning.md | 8 + src/lib/server/fleet/beta.ts | 8 + src/lib/server/fleet/collections.ts | 24 +- src/lib/server/fleet/servers.ts | 9 + src/lib/server/provisioning/instances.ts | 23 +- src/lib/server/provisioning/requests.ts | 8 +- src/lib/server/provisioning/worker.ts | 4 +- .../api/deployments/house-party/+server.ts | 7 +- tests/unit/beta-destruction.test.ts | 294 ++++++++++++++++++ tests/unit/create-server.test.ts | 4 +- 11 files changed, 380 insertions(+), 17 deletions(-) create mode 100644 tests/unit/beta-destruction.test.ts diff --git a/README.md b/README.md index 350f9bf..5b46edc 100644 --- a/README.md +++ b/README.md @@ -172,6 +172,8 @@ or unreachable servers remain queued. Containers, server files, and registration are removed, while saved keys and collections are retained. The preview must be reviewed again if fleet membership changes before confirmation. Servers created after confirmation are not included. Progress appears in deployment history. +If any target belongs to an active beta, no removals are queued. Promote or roll +back that beta before retrying fleet removal or an exact collection launch. If deletion fails after Komodo removal, the operation detail page offers **Resume cleanup** to retry local file and database removal without RCON or @@ -405,8 +407,10 @@ Deleting a server preserves its name, recorded beta progress and test evidence, campaign results and file diffs, provisioning and maintenance history, and match records. Historical entries show **Deleted** and omit live server links and actions. Reusing the same server name starts a new instance; old records remain -marked deleted, and old work cannot target the replacement. Deleting a participant -pauses an open beta; roll it back and open a new beta to test the replacement. +marked deleted, and old work cannot target the replacement. Servers linked to an +active beta cannot be destroyed or rebuilt until that beta is promoted or rolled +back. A participant removed outside provisioning still pauses its open beta; roll +it back and open a new beta to test the replacement. Normal history retention still applies. Records erased by earlier versions cannot be restored by this upgrade. Existing history is marked deleted where the server is already absent; instances that reused a name before upgrading cannot always be diff --git a/docs/provisioning.md b/docs/provisioning.md index 5ff27a0..6c99a21 100644 --- a/docs/provisioning.md +++ b/docs/provisioning.md @@ -14,6 +14,14 @@ server, choose **Manage server** or open its **Server actions** menu. | Stop | Wait until empty, stop the container, and verify that Komodo reports it stopped. Keep the instance for a later Start. | | Destroy | Wait until empty, remove the container and Komodo deployment, then delete server files and registration. | +Servers linked to a running or ready-to-promote beta cannot be destroyed or +rebuilt. Promote or roll back the beta first, including paused or failed test +plans. Fleet removal and exact collection launches queue nothing if any server +selected for removal belongs to an active beta. Provisioning checks again before +teardown; already queued removals wait while their server belongs to an active +beta. Opening a beta also requires cancelling or finishing any pending destruction +or rebuild on its selected servers. + Rebuild requires confirmation because **no instance data is preserved**. Its fresh files come from the latest promoted setup when creation begins, including changes promoted while waiting. The independently saved key in **Keys** must exist before diff --git a/src/lib/server/fleet/beta.ts b/src/lib/server/fleet/beta.ts index 398c29b..f4e3cd8 100644 --- a/src/lib/server/fleet/beta.ts +++ b/src/lib/server/fleet/beta.ts @@ -357,6 +357,14 @@ export async function openBeta(input: OpenBetaInput): Promise { for (const name of input.servers) { const row = getServer(name); if (!row) throw new BetaError(`unknown server: ${name}`, 'unknown-server'); + // Once teardown is queued or in flight, opening a beta cannot safely + // claim the instance. A failed remote request may still be running. + const removal = db().prepare(`SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL + AND action IN ('destroy', 'rebuild') AND (state IN ('waiting', 'queued', 'running', 'awaiting-health') + OR (state IN ('failed', 'interrupted') AND stage IN ( + 'Stopping and removing container', 'Deleting Komodo deployment', 'Removing server files and registry'))) + LIMIT 1`).get(name); + if (removal) throw new BetaError(`${name}: cancel or finish the pending destruction or rebuild before opening a beta.`, 'conflict'); serverRows.push({ name, game: row.game, map: row.map }); } diff --git a/src/lib/server/fleet/collections.ts b/src/lib/server/fleet/collections.ts index 5196178..dc0355a 100644 --- a/src/lib/server/fleet/collections.ts +++ b/src/lib/server/fleet/collections.ts @@ -9,7 +9,7 @@ import { deploymentStage, finishDeployment, recordDeploymentEvent } from './depl import { housePartyTargets } from './house-party.js'; import { getServerKey } from './server-keys.js'; import { serverObservation } from './status.js'; -import { getServer } from './servers.js'; +import { activeBetaRemovalBlocker, getServer } from './servers.js'; import { parseServerName, zombieMapBySlug, zombieMaps } from './map-aliases.js'; function coverage(entries: BlueprintEntry[]) { @@ -93,6 +93,16 @@ function blockersFor(servers: CollectionPreview['servers']): string[] { return blockers; } +function removalBlockersFor(removals: CollectionPreview['removals']): string[] { + const blockers: string[] = []; + if (removals.length && (!komodoConfigured() || !rconConfigured())) blockers.push('Configure Komodo and RCON before removing servers.'); + for (const server of removals) { + const blocker = activeBetaRemovalBlocker(server.name); + if (blocker) blockers.push(blocker); + } + return blockers; +} + export function previewCollection(blueprintId: unknown, mode: unknown = 'fill'): CollectionPreview { if (typeof blueprintId !== 'string' || !blueprintId) throw new Error('Choose a collection to launch.'); if (mode !== 'fill' && mode !== 'exact') throw new Error('Choose fill or exact collection mode.'); @@ -100,8 +110,7 @@ export function previewCollection(blueprintId: unknown, mode: unknown = 'fill'): const { entries, blueprintName } = blueprintEntries(blueprintId); const { counts, retainedServers, removals } = collectionPlan(entries, mode); const servers = allocate(entries, removals.map(server => server.name)); - const blockers = blockersFor(servers); - if (removals.length && (!komodoConfigured() || !rconConfigured())) blockers.push('Configure Komodo and RCON before removing servers.'); + const blockers = [...blockersFor(servers), ...removalBlockersFor(removals)]; const id = randomUUID(); const expiresAt = new Date(Date.now() + 30 * 60_000).toISOString(); db().prepare('DELETE FROM server_collection_previews WHERE expires_at < ? AND collection_id IS NULL').run(new Date().toISOString()); @@ -132,8 +141,7 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }): const { retainedServers, removals } = collectionPlan(entries, preview.mode); const releasingServers = removals.map(server => server.name); const servers = allocate(entries, releasingServers); - const blockers = blockersFor(servers); - if (removals.length && (!komodoConfigured() || !rconConfigured())) blockers.push('Configure Komodo and RCON before removing servers.'); + const blockers = [...blockersFor(servers), ...removalBlockersFor(removals)]; if (blockers.length) throw new Error(blockers.join(' ')); if (servers.length !== reviewed.length || servers.some((server, i) => server.entryIndex !== reviewed[i]?.entryIndex || server.name !== reviewed[i]?.name || server.port !== reviewed[i]?.port)) { throw new Error('Fleet counts, lobby or port availability changed. Nothing was queued. Refresh the preview and review each server key.'); @@ -155,7 +163,8 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }): if (queued && !getServer(server.name)) { // No container or files exist yet: cancel the reservation instead of booting just to remove it. const result = destroyServer(server.name, 'blueprint-exact'); - if (!result.ok || !result.deploymentId) throw new Error('Could not queue removal for ' + server.name); + if (!result.ok) throw new Error(result.message); + if (!result.deploymentId) throw new Error('Could not queue removal for ' + server.name); id = result.deploymentId; db().prepare("UPDATE server_deployments SET state = 'cancelled', stage = 'Cancelled by exact collection', finished_at = ? WHERE id = ?").run(new Date().toISOString(), queued.id); recordDeploymentEvent(queued.id, 'Queued creation cancelled by exact collection; saved key retained.'); @@ -163,7 +172,8 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }): finishDeployment(id, { ok: true }); } else { const result = destroyServer(server.name, 'blueprint-exact'); - if (!result.ok || !result.deploymentId) throw new Error('Could not queue removal for ' + server.name); + if (!result.ok) throw new Error(result.message); + if (!result.deploymentId) throw new Error('Could not queue removal for ' + server.name); id = result.deploymentId; } db().prepare('INSERT INTO server_collection_removals (collection_id, deployment_id, position) VALUES (?, ?, ?)').run(collectionId, id, position); diff --git a/src/lib/server/fleet/servers.ts b/src/lib/server/fleet/servers.ts index 020a750..bfa7bd1 100644 --- a/src/lib/server/fleet/servers.ts +++ b/src/lib/server/fleet/servers.ts @@ -153,6 +153,15 @@ export function listServers(): ServerRow[] { return db().prepare('SELECT * FROM servers WHERE destroyed_at IS NULL ORDER BY name').all() as ServerRow[]; } +/** Beta ownership lasts until explicit closure, regardless of test outcome or stage. */ +export function activeBetaRemovalBlocker(server: string): string | null { + const betas = db().prepare(`SELECT b.id FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id + WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running', 'ready-promote') + ORDER BY b.id`).all(server) as { id: number }[]; + if (!betas.length) return null; + return `${server} is linked to active beta${betas.length === 1 ? '' : 's'} ${betas.map(beta => '#' + beta.id).join(', ')}. Promote or roll back the beta${betas.length === 1 ? '' : 's'} before destroying or rebuilding this server.`; +} + export function setServerField( name: string, field: keyof ServerRow, diff --git a/src/lib/server/provisioning/instances.ts b/src/lib/server/provisioning/instances.ts index d0b1908..2bcd389 100644 --- a/src/lib/server/provisioning/instances.ts +++ b/src/lib/server/provisioning/instances.ts @@ -9,7 +9,7 @@ import { dirname, join } from 'node:path'; import { config, destOfKind, serverDir, gameOfServer, configFileForGame } from '../config.js'; import { db } from '../db/index.js'; import { getServerKey, saveServerKey } from '../fleet/server-keys.js'; -import { getServer } from '../fleet/servers.js'; +import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js'; import { zombieMapBySlug, type GameMode, type ZombieMapEntry, type ZombieGame } from '../fleet/map-aliases.js'; import { komodoCreateDeployment, @@ -28,6 +28,7 @@ import { getVersion } from '../fleet/scripts.js'; export type CreateServerFailure = | 'bad-request' + | 'conflict' | 'unconfigured' // Komodo creds / target server missing | 'unknown-map' // slug isn't in the catalog | 'no-slot' // all lobby slots for this map are taken @@ -231,6 +232,8 @@ export function queueCreateServer(input: CreateServerInput, releasingServers: st /** Reserve the same identity and port before removing any part of the instance. */ export function queueRebuildServer(name: string, triggeredBy = 'ui'): CreateServerResult { + const blocker = activeBetaRemovalBlocker(name); + if (blocker) return { ok: false, kind: 'conflict', message: blocker }; const server = getServer(name); const game = gameOfServer(name); const map = server?.map_slug && zombieMapBySlug(game, server.map_slug); @@ -259,6 +262,13 @@ export async function rebuildServer(id: string): Promise { const job = db().prepare(`SELECT j.*, d.stage FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.action = 'rebuild' AND d.state = 'waiting'`).get(id) as (CreationJob & { stage: string }) | undefined; if (!job) return; + const blocker = activeBetaRemovalBlocker(job.server); + if (blocker) { + // Keep the cleanup checkpoint intact when resuming after remote deletion. + if (job.stage === REMOVING_SERVER) finishDeployment(id, { ok: false, message: blocker }); + else deploymentStage(id, blocker); + return; + } if (creating || destroying.size || creationQueue().active.length >= fleetSettings().createConcurrency) { // The cleanup checkpoint must survive capacity waits and app restarts. if (job.stage !== REMOVING_SERVER) deploymentStage(id, 'Waiting for a server creation slot before rebuilding'); @@ -491,6 +501,8 @@ function hasActiveCreation(name: string): boolean { } export function destroyServer(name: string, triggeredBy = 'ui'): DestroyServerResult & { deploymentId?: string } { + const blocker = activeBetaRemovalBlocker(name); + if (blocker) return { ok: false, kind: 'conflict', message: blocker, warnings: [] }; if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('start','restart','stop','rebuild') AND state IN ('waiting','queued','running')`).get(name)) { return { ok: false, kind: 'conflict', message: 'Wait for the current provisioning operation before destroying this server.', warnings: [] }; @@ -547,6 +559,8 @@ async function runDestructions(): Promise { finishDeployment(operation.id, { ok: false, message: 'Server is no longer registered' }); continue; } + const blocker = activeBetaRemovalBlocker(operation.server); + if (blocker) { deploymentStage(operation.id, blocker); continue; } // A fresh successful zero is required; cached occupancy and timeouts are not evidence of emptiness. let players = await probePlayerCount(operation.server); if (players === null) { @@ -554,6 +568,9 @@ async function runDestructions(): Promise { if (['stopped','exited','created','dead'].includes(health.state?.toLowerCase() ?? '')) players = 0; } if (!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND state = 'waiting'").get(operation.id)) continue; + // Beta ownership may have changed while the network probes were in flight. + const latestBlocker = activeBetaRemovalBlocker(operation.server); + if (latestBlocker) { deploymentStage(operation.id, latestBlocker); continue; } if (players !== 0) { deploymentStage(operation.id, players === null ? 'Waiting for player count: server could not be reached' @@ -573,6 +590,8 @@ async function runDestructions(): Promise { } async function destroyServerUnlocked(name: string, operation: string): Promise { + const blocker = activeBetaRemovalBlocker(name); + if (blocker) return { ok: false, kind: 'conflict', message: blocker, warnings: [] }; const row = getServer(name); if (!row) { return { ok: false, kind: 'not-found', message: `no server named '${name}'`, warnings: [] }; @@ -610,6 +629,8 @@ async function destroyServerUnlocked(name: string, operation: string): Promise { + const blocker = activeBetaRemovalBlocker(name); + if (blocker) return { ok: false, kind: 'conflict', message: blocker, warnings: [] }; const row = getServer(name); const warnings: string[] = []; // Persist the checkpoint before local work so failed cleanup can resume without RCON. diff --git a/src/lib/server/provisioning/requests.ts b/src/lib/server/provisioning/requests.ts index 10dd5e1..9e14149 100644 --- a/src/lib/server/provisioning/requests.ts +++ b/src/lib/server/provisioning/requests.ts @@ -1,7 +1,7 @@ import { maintenanceAllowed } from '../maintenance/policies.js'; import { db } from '../db/index.js'; import { komodoConfigured, rconConfigured } from '../config.js'; -import { getServer } from '../fleet/servers.js'; +import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js'; import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent } from './history.js'; import { queueRebuildServer } from './instances.js'; import { restartRequestInFlight } from './restarts.js'; @@ -122,6 +122,8 @@ export function requestLifecycle(server: string, action: RuntimeAction) { if (!runtimeActions.includes(action)) return { ok: false as const, status: 400, error: 'Choose start, restart, rebuild, or stop.' }; if (!komodoConfigured() || !rconConfigured()) return { ok: false as const, status: 503, error: 'Configure Komodo and RCON before provisioning.' }; return db().transaction(() => { + const blocker = action === 'rebuild' ? activeBetaRemovalBlocker(server) : null; + if (blocker) return { ok: false as const, status: 409, error: blocker }; const existing = db().prepare(`SELECT id, action FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND state IN ('queued','waiting','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(server) as { id: string; action: string } | undefined; @@ -136,7 +138,7 @@ export function requestLifecycle(server: string, action: RuntimeAction) { const result = queueRebuildServer(server); if (result.ok) db().prepare("UPDATE servers SET desired_state = 'running' WHERE name = ?").run(server); return result.ok ? { ok: true as const, deploymentId: result.deploymentId } - : { ok: false as const, status: 400, error: result.message }; + : { ok: false as const, status: result.kind === 'conflict' ? 409 : 400, error: result.message }; } const id = beginDeployment(server, action); db().prepare('UPDATE servers SET desired_state = ? WHERE name = ?').run(action === 'stop' ? 'stopped' : 'running', server); @@ -170,7 +172,7 @@ function queueWatchdogAction(server: string, action: 'restart' | 'rebuild', inci let id: string; if (action === 'rebuild') { const result = queueRebuildServer(server, `watchdog:${incidentId}`); - if (!result.ok) return { ok: false, status: 400, error: result.message }; + if (!result.ok) return { ok: false, status: result.kind === 'conflict' ? 409 : 400, error: result.message }; id = result.deploymentId; } else { id = beginDeployment(server, action, `watchdog:${incidentId}`); diff --git a/src/lib/server/provisioning/worker.ts b/src/lib/server/provisioning/worker.ts index ea68768..712e165 100644 --- a/src/lib/server/provisioning/worker.ts +++ b/src/lib/server/provisioning/worker.ts @@ -140,7 +140,9 @@ async function processOperation(operation: Operation): Promise { } } // Requests can arrive during the probes. Recheck before claiming the action. - if (blocker(server, id) || sourceEligibility(operation) !== 'ready') return; + const latestReason = blocker(server, id); + if (latestReason) { deploymentStage(id, latestReason); return; } + if (sourceEligibility(operation) !== 'ready') return; if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return; if (action === 'rebuild') { await rebuildServer(id); return; } const claim = db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").run(id); diff --git a/src/routes/api/deployments/house-party/+server.ts b/src/routes/api/deployments/house-party/+server.ts index d337161..604043d 100644 --- a/src/routes/api/deployments/house-party/+server.ts +++ b/src/routes/api/deployments/house-party/+server.ts @@ -11,7 +11,12 @@ export async function POST({ request }: { request: Request }) { if (!Array.isArray(body?.servers) || !body.servers.every((name: unknown) => typeof name === 'string')) { return json({ error: 'A reviewed server list is required.' }, { status: 400 }); } - const result = queueHouseParty(body.servers); + let result: ReturnType; + try { + result = queueHouseParty(body.servers); + } catch (err) { + return json({ ok: false, error: (err as Error).message }, { status: 409 }); + } if (!result.ok) return json(result, { status: 409 }); void kick().catch(err => console.error('[house-party] worker kick failed:', err)); return json(result, { status: 202 }); diff --git a/tests/unit/beta-destruction.test.ts b/tests/unit/beta-destruction.test.ts new file mode 100644 index 0000000..0645b55 --- /dev/null +++ b/tests/unit/beta-destruction.test.ts @@ -0,0 +1,294 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { access } from 'node:fs/promises'; +import { join } from 'node:path'; +import { newFixture, type Fixture } from '../helpers/fixture.js'; + +vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ + komodoCreateDeployment: vi.fn(async () => ({ ok: true })), + komodoDeploy: vi.fn(async () => ({ ok: true })), + komodoDestroyDeployment: vi.fn(async () => ({ ok: true })), + komodoDeleteDeployment: vi.fn(async () => ({ ok: true })), + komodoInspectHealthy: vi.fn(), + komodoEnsureTag: vi.fn(async (name: string) => ({ ok: true, id: name })), + komodoUpdateTagsOnResource: vi.fn(async () => ({ ok: true })) +})); +vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ + rconGetPlayerCount: vi.fn(), rconStatusResult: vi.fn() +})); +vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) })); + +let fixture: Fixture; +const otherServer = 't5_zombie_cosmodrome_1'; +const server = 't5_zombie_theater_1'; + +beforeEach(async () => { + vi.resetModules(); vi.clearAllMocks(); + fixture = await newFixture([otherServer, server]); + for (const [key, value] of Object.entries({ KOMODO_ADDRESS: 'http://komodo.test', KOMODO_API_KEY: 'test', + KOMODO_API_SECRET: 'test', KOMODO_TARGET_SERVER: 'test-host', RCON_PASSWORD: 'test', + SERVER_PORT_MIN: '28960', SERVER_PORT_MAX: '28980' })) vi.stubEnv(key, value); + const { db } = await import('../../src/lib/server/db/index.js'); + for (const [index, name] of [otherServer, server].entries()) { + const map = name === server ? 'theater' : 'cosmodrome'; + db().prepare(`INSERT INTO servers (name, komodo_type, komodo_name, game, map, game_mode, map_slug, game_port, host_dir) + VALUES (?, 'deployment', ?, 't5', ?, 'zombie', ?, ?, ?)`).run(name, name, 'zombie_' + map, map, 28960 + index, join(fixture.t5, name)); + } + const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); + const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); + vi.mocked(rconGetPlayerCount).mockReset().mockResolvedValue(0); + vi.mocked(komodoInspectHealthy).mockReset().mockResolvedValue({ ok: false, state: 'exited', reason: 'stopped' }); +}); +afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); }); + +async function attachBeta(state = 'running', outcome?: string, deleted = false) { + const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + const { script, version } = createScript({ name: 'Protected beta', targetRelPath: 'protected.gsc', + scopeType: 'game', scopeValue: 't5', content: Buffer.from('beta content') }); + const id = Number(db().prepare(`INSERT INTO betas (script_id, script_version_id, trigger_kind, state) + VALUES (?, ?, 'manual', ?)`).run(script.id, version.id, state).lastInsertRowid); + db().prepare(`INSERT INTO beta_servers (beta_id, server, had_original, stage, server_deleted_at) + VALUES (?, ?, 0, ?, ?)`).run(id, server, outcome === 'failed' ? 'failed' : 'ready', deleted ? '2026-09-01' : null); + if (outcome) db().prepare('INSERT INTO beta_test_plans (beta_id, plan, outcome) VALUES (?, ?, ?)').run(id, '{}', outcome); + return id; +} + +async function expectServerPreserved() { + const { db } = await import('../../src/lib/server/db/index.js'); + const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js'); + expect(komodoDestroyDeployment).not.toHaveBeenCalled(); + expect(komodoDeleteDeployment).not.toHaveBeenCalled(); + expect(db().prepare('SELECT name FROM servers WHERE name = ?').get(server)).toEqual({ name: server }); + await expect(access(join(fixture.t5, server))).resolves.toBeUndefined(); +} + +describe('active beta destruction guard', () => { + it.each([ + { state: 'running', outcome: undefined }, + { state: 'ready-promote', outcome: 'passed' }, + { state: 'running', outcome: 'paused' }, + { state: 'running', outcome: 'failed' } + ])('rejects destruction for $state with test outcome $outcome', async ({ state, outcome }) => { + const id = await attachBeta(state, outcome); + const { destroyServer } = await import('../../src/lib/server/fleet/create.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + const result = destroyServer(server); + expect(result).toMatchObject({ ok: false, kind: 'conflict' }); + if (result.ok) throw new Error('Expected active beta conflict'); + expect(result.message).toContain(String(id)); + expect(result.message).toMatch(/promot/i); + expect(result.message).toMatch(/roll.?back/i); + expect(db().prepare("SELECT id FROM server_deployments WHERE action = 'destroy'").all()).toEqual([]); + await expectServerPreserved(); + }); + + it('returns HTTP 409 with actionable beta information and does not kick the worker', async () => { + const id = await attachBeta(); + const { DELETE } = await import('../../src/routes/api/servers/[name]/+server.js'); + const { kick } = await import('../../src/lib/server/pipeline/worker.js'); + const response = await DELETE({ params: { name: server } }); + expect(response.status).toBe(409); + expect(await response.json()).toMatchObject({ ok: false, kind: 'conflict', error: expect.stringContaining(String(id)) }); + expect(kick).not.toHaveBeenCalled(); + await expectServerPreserved(); + }); + + it('rejects a rebuild request with HTTP 409 before reserving a replacement', async () => { + const id = await attachBeta(); + const { POST } = await import('../../src/routes/api/provisioning/+server.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + const response = await POST({ request: new Request('http://localhost/api/provisioning', { + method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ server, action: 'rebuild' }) + }) }); + expect(response.status).toBe(409); + expect(await response.json()).toMatchObject({ error: expect.stringContaining(String(id)) }); + expect(db().prepare('SELECT * FROM server_creation_jobs').all()).toEqual([]); + await expectServerPreserved(); + }); + + it.each([ + { state: 'promoted', deleted: false }, + { state: 'rolled-back', deleted: false }, + { state: 'running', deleted: true }, + { state: 'ready-promote', deleted: true } + ])('allows removal for $state with historical deletion $deleted', async ({ state, deleted }) => { + await attachBeta(state, undefined, deleted); + const { destroyServer } = await import('../../src/lib/server/fleet/create.js'); + expect(destroyServer(server)).toMatchObject({ ok: true, deploymentId: expect.any(String) }); + }); + + it('holds a queued destruction until its beta closes, then completes the same request', async () => { + const { destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); + const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); + const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js'); + const queued = destroyServer(server); + expect(queued.ok).toBe(true); + const id = await attachBeta(); + await processDestructions(); + expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'waiting', stage: expect.stringMatching(/beta/i) }); + expect(rconGetPlayerCount).not.toHaveBeenCalled(); + await expectServerPreserved(); + db().prepare("UPDATE betas SET state = 'rolled-back', closed_at = ? WHERE id = ?").run(new Date().toISOString(), id); + await processDestructions(); + expect(komodoDestroyDeployment).toHaveBeenCalledOnce(); + expect(komodoDeleteDeployment).toHaveBeenCalledOnce(); + expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'done', server_exists: 0 }); + }); + + it('rechecks beta ownership after an in-flight empty-server probe', async () => { + const { destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); + const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js'); + const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); + let resolveCount!: (players: number) => void; + vi.mocked(rconGetPlayerCount).mockImplementationOnce(() => new Promise(resolve => { resolveCount = resolve; })); + const queued = destroyServer(server); + const work = processDestructions(); + await vi.waitFor(() => expect(resolveCount).toBeTypeOf('function')); + await attachBeta(); + resolveCount(0); + await work; + expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'waiting', stage: expect.stringMatching(/beta/i) }); + await expectServerPreserved(); + }); + + it('also holds rebuild destruction when a beta appears during the occupancy probe', async () => { + const { requestLifecycle, processLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js'); + const { saveServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); + const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js'); + const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js'); + const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); + saveServerKey(server, 'saved-key'); + vi.mocked(komodoInspectHealthy).mockResolvedValueOnce({ ok: true, state: 'running' }); + let resolveCount!: (players: number) => void; + vi.mocked(rconGetPlayerCount).mockImplementationOnce(() => new Promise(resolve => { resolveCount = resolve; })); + const queued = requestLifecycle(server, 'rebuild'); + if (!queued.ok) throw new Error(queued.error); + const work = processLifecycle(); + await vi.waitFor(() => expect(resolveCount).toBeTypeOf('function')); + await attachBeta(); + resolveCount(0); + await work; + expect(getDeployment(queued.deploymentId)?.deployment).toMatchObject({ state: 'waiting', stage: expect.stringMatching(/beta/i) }); + await expectServerPreserved(); + }); + + it('preserves the cleanup checkpoint and files while an active beta blocks a legacy cleanup resume', async () => { + const { destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); + const { deploymentStage, finishDeployment, resumeCleanup, getDeployment, REMOVING_SERVER } = await import('../../src/lib/server/fleet/deployments.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + const queued = destroyServer(server); + deploymentStage(queued.deploymentId!, REMOVING_SERVER); + finishDeployment(queued.deploymentId!, { ok: false, message: 'Interrupted local cleanup' }); + const id = await attachBeta(); + expect(resumeCleanup(queued.deploymentId!)).toBe(true); + await processDestructions(); + expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'failed', stage: REMOVING_SERVER, + error: expect.stringMatching(/beta/i) }); + expect(getDeployment(queued.deploymentId!)?.canResumeCleanup).toBe(true); + await expectServerPreserved(); + db().prepare("UPDATE betas SET state = 'promoted' WHERE id = ?").run(id); + expect(resumeCleanup(queued.deploymentId!)).toBe(true); + await processDestructions(); + expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'done', server_exists: 0 }); + const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js'); + expect(komodoDestroyDeployment).not.toHaveBeenCalled(); + expect(komodoDeleteDeployment).not.toHaveBeenCalled(); + }); + + it.each(['destroy', 'rebuild'] as const)('requires cancelling a pending %s before opening a beta', async action => { + const { destroyServer } = await import('../../src/lib/server/fleet/create.js'); + const { requestLifecycle } = await import('../../src/lib/server/fleet/lifecycle.js'); + const { cancelRemoval } = await import('../../src/lib/server/fleet/deployments.js'); + const { cancelProvisioning } = await import('../../src/lib/server/provisioning/requests.js'); + const { saveServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); + const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); + const { openBeta } = await import('../../src/lib/server/fleet/beta.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + saveServerKey(server, 'saved-key'); + const queued = action === 'destroy' ? destroyServer(server) : requestLifecycle(server, 'rebuild'); + expect(queued.ok).toBe(true); + if (!queued.ok) throw new Error('Expected queued removal'); + const { script, version } = createScript({ name: 'Beta admission', targetRelPath: 'admission.gsc', + scopeType: 'game', scopeValue: 't5', content: Buffer.from('beta content') }); + const input = { scripts: [{ scriptId: script.id, scriptVersionId: version.id }], servers: [server], + waitForHealth: false, waitForMatch: false }; + await expect(openBeta(input)).rejects.toMatchObject({ kind: 'conflict', message: expect.stringMatching(/cancel or finish/i) }); + expect(db().prepare('SELECT id FROM betas').all()).toEqual([]); + expect(action === 'destroy' ? cancelRemoval(queued.deploymentId!) : cancelProvisioning(queued.deploymentId!)).toBe(true); + await expect(openBeta(input)).resolves.toMatchObject({ servers: [server], betaId: expect.any(Number) }); + await expectServerPreserved(); + }); + + it.each((['destroy', 'rebuild'] as const).flatMap(action => ['failed', 'interrupted'].flatMap(state => + ['Stopping and removing container', 'Deleting Komodo deployment', 'Removing server files and registry'].map(stage => ({ action, state, stage })) + )))('rejects beta admission after $action is $state at $stage', async ({ action, state, stage }) => { + const { beginDeployment } = await import('../../src/lib/server/fleet/deployments.js'); + const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); + const { openBeta } = await import('../../src/lib/server/fleet/beta.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + const operation = beginDeployment(server, action); + db().prepare('UPDATE server_deployments SET state = ?, stage = ? WHERE id = ?').run(state, stage, operation); + const { script, version } = createScript({ name: 'Beta admission', targetRelPath: 'admission.gsc', + scopeType: 'game', scopeValue: 't5', content: Buffer.from('beta content') }); + await expect(openBeta({ scripts: [{ scriptId: script.id, scriptVersionId: version.id }], servers: [server], + waitForHealth: false, waitForMatch: false })).rejects.toMatchObject({ kind: 'conflict' }); + expect(db().prepare('SELECT id FROM betas').all()).toEqual([]); + expect(db().prepare('SELECT * FROM staged_file_changes').all()).toEqual([]); + await expectServerPreserved(); + }); +}); + +describe('bulk removal beta protection', () => { + it('rolls back earlier House Party removals when another target has an active beta', async () => { + const id = await attachBeta(); + const { housePartyTargets, queueHouseParty } = await import('../../src/lib/server/fleet/house-party.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + expect(housePartyTargets()).toEqual([otherServer, server]); + expect(() => queueHouseParty(housePartyTargets())).toThrow(new RegExp(String(id))); + expect(db().prepare("SELECT id FROM server_deployments WHERE action = 'destroy'").all()).toEqual([]); + await expectServerPreserved(); + }); + + it('returns an actionable HTTP 409 for a blocked House Party without queuing partial removals', async () => { + const id = await attachBeta(); + const { POST } = await import('../../src/routes/api/provisioning/house-party/+server.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + const response = await POST({ request: new Request('http://localhost/api/provisioning/house-party', { + method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ servers: [otherServer, server] }) + }) }); + expect(response.status).toBe(409); + expect(await response.json()).toMatchObject({ ok: false, error: expect.stringContaining(String(id)) }); + expect(db().prepare("SELECT id FROM server_deployments WHERE action = 'destroy'").all()).toEqual([]); + await expectServerPreserved(); + }); + + it('shows the active beta as an exact collection blocker while fill mode remains available', async () => { + const id = await attachBeta(); + const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); + const { previewCollection } = await import('../../src/lib/server/fleet/collections.js'); + const blueprint = saveBlueprint({ name: 'Replace fleet', entries: [{ game: 't5', mapSlug: 'moon', count: 1 }] }); + const preview = previewCollection(blueprint.id, 'exact'); + expect(preview.ready).toBe(false); + expect(preview.blockers).toContainEqual(expect.stringContaining(String(id))); + expect(preview.blockers.join(' ')).toMatch(/promot.*roll.?back/i); + expect(previewCollection(blueprint.id, 'fill').ready).toBe(true); + }); + + it('rolls back an exact collection if a beta starts after its preview', async () => { + const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); + const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + const blueprint = saveBlueprint({ name: 'Replace fleet', entries: [{ game: 't5', mapSlug: 'moon', count: 1 }] }); + const preview = previewCollection(blueprint.id, 'exact'); + expect(preview.removals.map(target => target.name)).toEqual([otherServer, server]); + const id = await attachBeta(); + expect(() => queueCollection({ previewId: preview.id, keys: { t5_zombie_moon_1: 'new-key' } })).toThrow(new RegExp(String(id))); + for (const table of ['server_deployments', 'server_launch_collections', 'server_collection_removals', 'server_creation_jobs', 'server_keys']) { + expect(db().prepare(`SELECT * FROM ${table}`).all()).toEqual([]); + } + expect(db().prepare('SELECT collection_id FROM server_collection_previews WHERE id = ?').get(preview.id)).toEqual({ collection_id: null }); + await expectServerPreserved(); + }); +}); diff --git a/tests/unit/create-server.test.ts b/tests/unit/create-server.test.ts index c4b670f..9938de9 100644 --- a/tests/unit/create-server.test.ts +++ b/tests/unit/create-server.test.ts @@ -103,7 +103,7 @@ describe('queued removal cancellation', () => { }); describe('destruction cleanup recovery', () => { - it('removes T6 servers while preserving beta progress and test evidence', async () => { + it('removes T6 servers while preserving closed beta progress and test evidence', async () => { const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js'); const { createScript } = await import('../../src/lib/server/fleet/scripts.js'); const { db } = await import('../../src/lib/server/db/index.js'); @@ -117,7 +117,7 @@ describe('destruction cleanup recovery', () => { expect(await createServer({ ...input, game: 't6', mapSlug: 'town' })).toMatchObject({ ok: true, name }); expect(await createServer(input)).toMatchObject({ ok: true }); const { script, version } = createScript({ name: 'test script', kind: 'script', scopeType: 'game', scopeValue: 't6', targetRelPath: 'test.gsc', content: Buffer.from('main(){}') }); - const beta = db().prepare("INSERT INTO betas (script_id, script_version_id, trigger_kind, state) VALUES (?, ?, 'manual', 'running')").run(script.id, version.id).lastInsertRowid; + const beta = db().prepare("INSERT INTO betas (script_id, script_version_id, trigger_kind, state) VALUES (?, ?, 'manual', 'promoted')").run(script.id, version.id).lastInsertRowid; for (const server of [name, serverName]) { db().prepare("INSERT INTO beta_servers (beta_id, server, had_original, stage, healthy_at) VALUES (?, ?, 0, 'ready', '2026-09-18')").run(beta, server); db().prepare("INSERT INTO beta_test_runs (id, beta_id, server, created_at) VALUES (?, ?, ?, '2026-09-18')").run(server, beta, server);