Block server destruction while linked to an active beta

Protect individual, bulk, and queued removals and rebuilds until beta closure. Recheck ownership before teardown and reject beta admission during pending or unresolved destruction.

Fixes #36

src/lib/server/fleet/beta.ts

@@ -357,6 +357,14 @@ export async function openBeta(input: OpenBetaInput): Promise<OpenBetaResult> {
     for (const name of input.servers) {
         const row = getServer(name);
         if (!row) throw new BetaError(`unknown server: ${name}`, 'unknown-server');
+        // Once teardown is queued or in flight, opening a beta cannot safely
+        // claim the instance. A failed remote request may still be running.
+        const removal = db().prepare(`SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL
+            AND action IN ('destroy', 'rebuild') AND (state IN ('waiting', 'queued', 'running', 'awaiting-health')
+                OR (state IN ('failed', 'interrupted') AND stage IN (
+                    'Stopping and removing container', 'Deleting Komodo deployment', 'Removing server files and registry')))
+            LIMIT 1`).get(name);
+        if (removal) throw new BetaError(`${name}: cancel or finish the pending destruction or rebuild before opening a beta.`, 'conflict');
         serverRows.push({ name, game: row.game, map: row.map });
     }
 

src/lib/server/fleet/collections.ts

@@ -9,7 +9,7 @@ import { deploymentStage, finishDeployment, recordDeploymentEvent } from './depl
 import { housePartyTargets } from './house-party.js';
 import { getServerKey } from './server-keys.js';
 import { serverObservation } from './status.js';
-import { getServer } from './servers.js';
+import { activeBetaRemovalBlocker, getServer } from './servers.js';
 import { parseServerName, zombieMapBySlug, zombieMaps } from './map-aliases.js';
 
 function coverage(entries: BlueprintEntry[]) {
@@ -93,6 +93,16 @@ function blockersFor(servers: CollectionPreview['servers']): string[] {
     return blockers;
 }
 
+function removalBlockersFor(removals: CollectionPreview['removals']): string[] {
+    const blockers: string[] = [];
+    if (removals.length && (!komodoConfigured() || !rconConfigured())) blockers.push('Configure Komodo and RCON before removing servers.');
+    for (const server of removals) {
+        const blocker = activeBetaRemovalBlocker(server.name);
+        if (blocker) blockers.push(blocker);
+    }
+    return blockers;
+}
+
 export function previewCollection(blueprintId: unknown, mode: unknown = 'fill'): CollectionPreview {
     if (typeof blueprintId !== 'string' || !blueprintId) throw new Error('Choose a collection to launch.');
     if (mode !== 'fill' && mode !== 'exact') throw new Error('Choose fill or exact collection mode.');
@@ -100,8 +110,7 @@ export function previewCollection(blueprintId: unknown, mode: unknown = 'fill'):
         const { entries, blueprintName } = blueprintEntries(blueprintId);
         const { counts, retainedServers, removals } = collectionPlan(entries, mode);
         const servers = allocate(entries, removals.map(server => server.name));
-        const blockers = blockersFor(servers);
-        if (removals.length && (!komodoConfigured() || !rconConfigured())) blockers.push('Configure Komodo and RCON before removing servers.');
+        const blockers = [...blockersFor(servers), ...removalBlockersFor(removals)];
         const id = randomUUID();
         const expiresAt = new Date(Date.now() + 30 * 60_000).toISOString();
         db().prepare('DELETE FROM server_collection_previews WHERE expires_at < ? AND collection_id IS NULL').run(new Date().toISOString());
@@ -132,8 +141,7 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
         const { retainedServers, removals } = collectionPlan(entries, preview.mode);
         const releasingServers = removals.map(server => server.name);
         const servers = allocate(entries, releasingServers);
-        const blockers = blockersFor(servers);
-        if (removals.length && (!komodoConfigured() || !rconConfigured())) blockers.push('Configure Komodo and RCON before removing servers.');
+        const blockers = [...blockersFor(servers), ...removalBlockersFor(removals)];
         if (blockers.length) throw new Error(blockers.join(' '));
         if (servers.length !== reviewed.length || servers.some((server, i) => server.entryIndex !== reviewed[i]?.entryIndex || server.name !== reviewed[i]?.name || server.port !== reviewed[i]?.port)) {
             throw new Error('Fleet counts, lobby or port availability changed. Nothing was queued. Refresh the preview and review each server key.');
@@ -155,7 +163,8 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
             if (queued && !getServer(server.name)) {
                 // No container or files exist yet: cancel the reservation instead of booting just to remove it.
                 const result = destroyServer(server.name, 'blueprint-exact');
-                if (!result.ok || !result.deploymentId) throw new Error('Could not queue removal for ' + server.name);
+                if (!result.ok) throw new Error(result.message);
+                if (!result.deploymentId) throw new Error('Could not queue removal for ' + server.name);
                 id = result.deploymentId;
                 db().prepare("UPDATE server_deployments SET state = 'cancelled', stage = 'Cancelled by exact collection', finished_at = ? WHERE id = ?").run(new Date().toISOString(), queued.id);
                 recordDeploymentEvent(queued.id, 'Queued creation cancelled by exact collection; saved key retained.');
@@ -163,7 +172,8 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
                 finishDeployment(id, { ok: true });
             } else {
                 const result = destroyServer(server.name, 'blueprint-exact');
-                if (!result.ok || !result.deploymentId) throw new Error('Could not queue removal for ' + server.name);
+                if (!result.ok) throw new Error(result.message);
+                if (!result.deploymentId) throw new Error('Could not queue removal for ' + server.name);
                 id = result.deploymentId;
             }
             db().prepare('INSERT INTO server_collection_removals (collection_id, deployment_id, position) VALUES (?, ?, ?)').run(collectionId, id, position);

src/lib/server/fleet/servers.ts

@@ -153,6 +153,15 @@ export function listServers(): ServerRow[] {
     return db().prepare('SELECT * FROM servers WHERE destroyed_at IS NULL ORDER BY name').all() as ServerRow[];
 }
 
+/** Beta ownership lasts until explicit closure, regardless of test outcome or stage. */
+export function activeBetaRemovalBlocker(server: string): string | null {
+    const betas = db().prepare(`SELECT b.id FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
+        WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running', 'ready-promote')
+        ORDER BY b.id`).all(server) as { id: number }[];
+    if (!betas.length) return null;
+    return `${server} is linked to active beta${betas.length === 1 ? '' : 's'} ${betas.map(beta => '#' + beta.id).join(', ')}. Promote or roll back the beta${betas.length === 1 ? '' : 's'} before destroying or rebuilding this server.`;
+}
+
 export function setServerField(
     name: string,
     field: keyof ServerRow,

src/lib/server/provisioning/instances.ts

@@ -9,7 +9,7 @@ import { dirname, join } from 'node:path';
 import { config, destOfKind, serverDir, gameOfServer, configFileForGame } from '../config.js';
 import { db } from '../db/index.js';
 import { getServerKey, saveServerKey } from '../fleet/server-keys.js';
-import { getServer } from '../fleet/servers.js';
+import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
 import { zombieMapBySlug, type GameMode, type ZombieMapEntry, type ZombieGame } from '../fleet/map-aliases.js';
 import {
     komodoCreateDeployment,
@@ -28,6 +28,7 @@ import { getVersion } from '../fleet/scripts.js';
 
 export type CreateServerFailure =
     | 'bad-request'
+    | 'conflict'
     | 'unconfigured'    // Komodo creds / target server missing
     | 'unknown-map'     // slug isn't in the catalog
     | 'no-slot'         // all lobby slots for this map are taken
@@ -231,6 +232,8 @@ export function queueCreateServer(input: CreateServerInput, releasingServers: st
 
 /** Reserve the same identity and port before removing any part of the instance. */
 export function queueRebuildServer(name: string, triggeredBy = 'ui'): CreateServerResult {
+    const blocker = activeBetaRemovalBlocker(name);
+    if (blocker) return { ok: false, kind: 'conflict', message: blocker };
     const server = getServer(name);
     const game = gameOfServer(name);
     const map = server?.map_slug && zombieMapBySlug(game, server.map_slug);
@@ -259,6 +262,13 @@ export async function rebuildServer(id: string): Promise<void> {
     const job = db().prepare(`SELECT j.*, d.stage FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
         WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.action = 'rebuild' AND d.state = 'waiting'`).get(id) as (CreationJob & { stage: string }) | undefined;
     if (!job) return;
+    const blocker = activeBetaRemovalBlocker(job.server);
+    if (blocker) {
+        // Keep the cleanup checkpoint intact when resuming after remote deletion.
+        if (job.stage === REMOVING_SERVER) finishDeployment(id, { ok: false, message: blocker });
+        else deploymentStage(id, blocker);
+        return;
+    }
     if (creating || destroying.size || creationQueue().active.length >= fleetSettings().createConcurrency) {
         // The cleanup checkpoint must survive capacity waits and app restarts.
         if (job.stage !== REMOVING_SERVER) deploymentStage(id, 'Waiting for a server creation slot before rebuilding');
@@ -491,6 +501,8 @@ function hasActiveCreation(name: string): boolean {
 }
 
 export function destroyServer(name: string, triggeredBy = 'ui'): DestroyServerResult & { deploymentId?: string } {
+    const blocker = activeBetaRemovalBlocker(name);
+    if (blocker) return { ok: false, kind: 'conflict', message: blocker, warnings: [] };
     if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('start','restart','stop','rebuild')
         AND state IN ('waiting','queued','running')`).get(name)) {
         return { ok: false, kind: 'conflict', message: 'Wait for the current provisioning operation before destroying this server.', warnings: [] };
@@ -547,6 +559,8 @@ async function runDestructions(): Promise<void> {
                 finishDeployment(operation.id, { ok: false, message: 'Server is no longer registered' });
                 continue;
             }
+            const blocker = activeBetaRemovalBlocker(operation.server);
+            if (blocker) { deploymentStage(operation.id, blocker); continue; }
             // A fresh successful zero is required; cached occupancy and timeouts are not evidence of emptiness.
             let players = await probePlayerCount(operation.server);
             if (players === null) {
@@ -554,6 +568,9 @@ async function runDestructions(): Promise<void> {
                 if (['stopped','exited','created','dead'].includes(health.state?.toLowerCase() ?? '')) players = 0;
             }
             if (!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND state = 'waiting'").get(operation.id)) continue;
+            // Beta ownership may have changed while the network probes were in flight.
+            const latestBlocker = activeBetaRemovalBlocker(operation.server);
+            if (latestBlocker) { deploymentStage(operation.id, latestBlocker); continue; }
             if (players !== 0) {
                 deploymentStage(operation.id, players === null
                     ? 'Waiting for player count: server could not be reached'
@@ -573,6 +590,8 @@ async function runDestructions(): Promise<void> {
 }
 
 async function destroyServerUnlocked(name: string, operation: string): Promise<DestroyServerResult> {
+    const blocker = activeBetaRemovalBlocker(name);
+    if (blocker) return { ok: false, kind: 'conflict', message: blocker, warnings: [] };
     const row = getServer(name);
     if (!row) {
         return { ok: false, kind: 'not-found', message: `no server named '${name}'`, warnings: [] };
@@ -610,6 +629,8 @@ async function destroyServerUnlocked(name: string, operation: string): Promise<D
 }
 
 async function cleanupDestroyedServer(name: string, operation: string): Promise<DestroyServerResult> {
+    const blocker = activeBetaRemovalBlocker(name);
+    if (blocker) return { ok: false, kind: 'conflict', message: blocker, warnings: [] };
     const row = getServer(name);
     const warnings: string[] = [];
     // Persist the checkpoint before local work so failed cleanup can resume without RCON.

src/lib/server/provisioning/requests.ts

@@ -1,7 +1,7 @@
 import { maintenanceAllowed } from '../maintenance/policies.js';
 import { db } from '../db/index.js';
 import { komodoConfigured, rconConfigured } from '../config.js';
-import { getServer } from '../fleet/servers.js';
+import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
 import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent } from './history.js';
 import { queueRebuildServer } from './instances.js';
 import { restartRequestInFlight } from './restarts.js';
@@ -122,6 +122,8 @@ export function requestLifecycle(server: string, action: RuntimeAction) {
     if (!runtimeActions.includes(action)) return { ok: false as const, status: 400, error: 'Choose start, restart, rebuild, or stop.' };
     if (!komodoConfigured() || !rconConfigured()) return { ok: false as const, status: 503, error: 'Configure Komodo and RCON before provisioning.' };
     return db().transaction(() => {
+        const blocker = action === 'rebuild' ? activeBetaRemovalBlocker(server) : null;
+        if (blocker) return { ok: false as const, status: 409, error: blocker };
         const existing = db().prepare(`SELECT id, action FROM server_deployments WHERE server = ?
             AND server_deleted_at IS NULL
             AND state IN ('queued','waiting','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(server) as { id: string; action: string } | undefined;
@@ -136,7 +138,7 @@ export function requestLifecycle(server: string, action: RuntimeAction) {
             const result = queueRebuildServer(server);
             if (result.ok) db().prepare("UPDATE servers SET desired_state = 'running' WHERE name = ?").run(server);
             return result.ok ? { ok: true as const, deploymentId: result.deploymentId }
-                : { ok: false as const, status: 400, error: result.message };
+                : { ok: false as const, status: result.kind === 'conflict' ? 409 : 400, error: result.message };
         }
         const id = beginDeployment(server, action);
         db().prepare('UPDATE servers SET desired_state = ? WHERE name = ?').run(action === 'stop' ? 'stopped' : 'running', server);
@@ -170,7 +172,7 @@ function queueWatchdogAction(server: string, action: 'restart' | 'rebuild', inci
     let id: string;
     if (action === 'rebuild') {
         const result = queueRebuildServer(server, `watchdog:${incidentId}`);
-        if (!result.ok) return { ok: false, status: 400, error: result.message };
+        if (!result.ok) return { ok: false, status: result.kind === 'conflict' ? 409 : 400, error: result.message };
         id = result.deploymentId;
     } else {
         id = beginDeployment(server, action, `watchdog:${incidentId}`);

src/lib/server/provisioning/worker.ts

@@ -140,7 +140,9 @@ async function processOperation(operation: Operation): Promise<void> {
         }
     }
     // Requests can arrive during the probes. Recheck before claiming the action.
-    if (blocker(server, id) || sourceEligibility(operation) !== 'ready') return;
+    const latestReason = blocker(server, id);
+    if (latestReason) { deploymentStage(id, latestReason); return; }
+    if (sourceEligibility(operation) !== 'ready') return;
     if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return;
     if (action === 'rebuild') { await rebuildServer(id); return; }
     const claim = db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").run(id);