Preserve deleted server history

Retain historical server identities, results, and evidence after deletion. Mark archived entries as Deleted and prevent old work from targeting servers that reuse the same name.

Fixes #16.

src/lib/server/fleet/beta-tests.ts

@@ -9,6 +9,8 @@ import { applySample, applyGameEvent, emptyMetrics, evaluateRequirements, type T
 export interface TestRun { id: string; beta_id: number; server: string; phase: number; created_at: string; observing_at: string | null; metrics: string }
 export interface StoredPlan { beta_id: number; plan: string; outcome: string; reason: string | null }
 const nowIso = () => new Date().toISOString();
+const activeParticipant = (id: number, server: string) => !!db().prepare(`SELECT 1 FROM beta_servers
+    WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL`).get(id, server);
 export const storedPlan = (id: number) => db().prepare('SELECT * FROM beta_test_plans WHERE beta_id = ?').get(id) as StoredPlan | undefined;
 export const currentRun = (id: number, server: string) => db().prepare('SELECT * FROM beta_test_runs WHERE beta_id = ? AND server = ? AND retired_at IS NULL').get(id, server) as TestRun | undefined;
 export function activity(id: number, message: string) { db().prepare('INSERT INTO beta_events (beta_id, message) VALUES (?, ?)').run(id, message); }
@@ -27,6 +29,7 @@ export function evidence(run: TestRun, eventId: string, kind: string, payload: u
 
 /** Internal progress record; never sent to or required from the game script. */
 export async function prepareRun(betaId: number, server: string, phase = 0): Promise<TestRun> {
+    if (!activeParticipant(betaId, server)) throw new Error('Original test server was deleted; start a new beta');
     db().transaction(() => {
         db().prepare('UPDATE beta_test_runs SET retired_at = ? WHERE beta_id = ? AND server = ? AND retired_at IS NULL').run(nowIso(), betaId, server);
         db().prepare('INSERT INTO beta_test_runs (id, beta_id, server, phase, created_at, metrics) VALUES (?, ?, ?, ?, ?, ?)')
@@ -41,6 +44,7 @@ export function recordGameEvent(event: GameTelemetry): void {
         JOIN beta_test_plans p ON p.beta_id = b.id
         JOIN beta_servers s ON s.beta_id = b.id AND s.server = r.server
         WHERE r.server = ? AND r.retired_at IS NULL AND b.state IN ('running','ready-promote')
+        AND s.server_deleted_at IS NULL
         AND p.outcome IN ('active','passed') AND s.stage IN ('awaiting-health','observing','ready')
         AND s.restarted_at IS NOT NULL`).all(event.server) as TestRun[];
     for (const run of runs) {
@@ -53,6 +57,7 @@ export function recordGameEvent(event: GameTelemetry): void {
 }
 
 export async function observeRun(run: TestRun, plan: BetaPlan): Promise<void> {
+    if (!activeParticipant(run.beta_id, run.server)) return;
     const m = readMetrics(run);
     const row = getServer(run.server);
     const server = row ? serverObservation(row) : null;
@@ -74,19 +79,22 @@ export function testView(betaId: number): BetaTestView | null {
     const stored = storedPlan(betaId);
     if (!stored) return null;
     const plan = parseBetaPlan(JSON.parse(stored.plan));
-    const runs = db().prepare('SELECT * FROM beta_test_runs WHERE beta_id = ? AND retired_at IS NULL ORDER BY server').all(betaId) as TestRun[];
+    const runs = db().prepare(`SELECT r.*, s.server_deleted_at FROM beta_test_runs r
+        JOIN beta_servers s ON s.beta_id = r.beta_id AND s.server = r.server
+        WHERE r.beta_id = ? AND r.retired_at IS NULL ORDER BY r.server`).all(betaId) as Array<TestRun & { server_deleted_at: string | null }>;
     return { plan, outcome: stored.outcome, reason: stored.reason,
         servers: runs.map(run => {
             const metrics = readMetrics(run);
             const progress = evaluateRequirements(plan, metrics, run.phase);
             const latest = db().prepare("SELECT received_at, kind, payload FROM beta_test_evidence WHERE run_id = ? AND kind IN ('game_start','round_change','heartbeat','game_end') ORDER BY id DESC LIMIT 1").get(run.id) as { received_at: string; kind: string; payload: string } | undefined;
             const event = latest ? JSON.parse(latest.payload) : null;
-            return { server: run.server, phase: run.phase, passed: progress.every(r => r.passed), progress, observingAt: run.observing_at,
+            return { server: run.server, server_deleted_at: run.server_deleted_at, phase: run.phase, passed: progress.every(r => r.passed), progress, observingAt: run.observing_at,
                 observation: { lastGameplayAt: latest?.received_at ?? null, lastEvent: latest?.kind ?? null,
                     players: metrics.match?.players ?? null, round: event?.round ?? null,
                     matchStarted: metrics.match?.started !== undefined, gameEnded: metrics.match?.ended ?? false,
                     healthy: metrics.wasHealthy ?? null, healthCheckedAt: metrics.lastSample ? new Date(metrics.lastSample).toISOString() : null } };
         }),
-        evidence: db().prepare(`SELECT e.id, r.server, e.received_at, e.kind, e.payload FROM beta_test_evidence e
-            JOIN beta_test_runs r ON r.id = e.run_id WHERE r.beta_id = ? ORDER BY e.id DESC LIMIT 200`).all(betaId) as BetaTestView['evidence'] };
+        evidence: db().prepare(`SELECT e.id, r.server, s.server_deleted_at, e.received_at, e.kind, e.payload FROM beta_test_evidence e
+            JOIN beta_test_runs r ON r.id = e.run_id JOIN beta_servers s ON s.beta_id = r.beta_id AND s.server = r.server
+            WHERE r.beta_id = ? ORDER BY e.id DESC LIMIT 200`).all(betaId) as BetaTestView['evidence'] };
 }

src/lib/server/fleet/beta.ts

@@ -148,6 +148,7 @@ export interface BetaRow {
 export interface BetaServerRow {
     beta_id: number;
     server: string;
+    server_deleted_at: string | null;
     had_original: number;                 // deprecated; per-script data lives in beta_server_scripts
     prior_version_id: number | null;      // deprecated
     stage: BetaStage;
@@ -225,6 +226,13 @@ export function listBetaServers(betaId: number): BetaServerRow[] {
         .all(betaId) as BetaServerRow[];
 }
 
+/** History retains names, so operational callers must also check the participant's identity. */
+export function isActiveBetaServer(betaId: number, server: string): boolean {
+    return !!db().prepare(`SELECT 1 FROM beta_servers bs JOIN servers s ON s.name = bs.server
+        WHERE bs.beta_id = ? AND bs.server = ? AND bs.server_deleted_at IS NULL AND s.destroyed_at IS NULL`)
+        .get(betaId, server);
+}
+
 export function listBetaScripts(betaId: number): BetaScriptRow[] {
     return db()
         .prepare('SELECT * FROM beta_scripts WHERE beta_id = ? ORDER BY script_id')
@@ -456,6 +464,9 @@ export async function deployBetaServers(betaId: number, names: string[]): Promis
     return exclusive(betaId, () => deployBetaServersInternal(betaId, names));
 }
 async function deployBetaServersInternal(betaId: number, names: string[]): Promise<void> {
+    for (const name of names) {
+        if (!isActiveBetaServer(betaId, name)) throw new BetaError(`${name}: original beta server was deleted; start a new beta`, 'unknown-server');
+    }
     const resolved = listBetaScripts(betaId).map(r => ({ script: getScript(r.script_id)!, version: getVersion(r.script_version_id)! }));
     const serverRows = names.map(name => getServer(name)!);
     // Split resolved artifacts into non-collapsing (each writes its own file)
@@ -484,10 +495,11 @@ async function deployBetaServersInternal(betaId: number, names: string[]): Promi
     const updateStage = db().prepare(
         `UPDATE beta_servers
          SET stage = ?, error = ?, deployed_at = ?, updated_at = ?
-         WHERE beta_id = ? AND server = ?`
+         WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL`
     );
 
     for (const name of names) {
+        if (!isActiveBetaServer(betaId, name)) continue;
         db().prepare("UPDATE beta_servers SET stage = 'deploying', deployed_at = ? WHERE beta_id = ? AND server = ?")
             .run(nowIso(), betaId, name);
         const serverRow = serverRows.find((r) => r.name === name)!;
@@ -503,6 +515,7 @@ async function deployBetaServersInternal(betaId: number, names: string[]): Promi
                 await mkdir(join(dest, dirname(script.target_rel_path)), { recursive: true });
                 if (existsSync(targetPath)) {
                     const priorBytes = await readFile(targetPath);
+                    if (!isActiveBetaServer(betaId, name)) break;
                     insertSnapshot.run(
                         betaId,
                         name,
@@ -514,7 +527,9 @@ async function deployBetaServersInternal(betaId: number, names: string[]): Promi
                     db().prepare('UPDATE beta_server_scripts SET had_original = 1 WHERE beta_id = ? AND server = ? AND script_id = ?').run(betaId, name, script.id);
                 }
                 db().prepare('INSERT OR IGNORE INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, ?)').run(betaId, name, script.target_rel_path);
+                if (!isActiveBetaServer(betaId, name)) break;
                 await writeFile(targetPath, version.content);
+                if (!isActiveBetaServer(betaId, name)) break;
                 recordScript(name, script.target_rel_path, version.id, version.sha256, version.size, `beta:${betaId}`);
             } catch (err) {
                 firstErr = `${script.name}: ${(err as Error).message}`;
@@ -541,6 +556,7 @@ async function deployBetaServersInternal(betaId: number, names: string[]): Promi
                 const targetPath = join(dest, targetRel);
                 if (existsSync(targetPath)) {
                     const priorBytes = await readFile(targetPath);
+                    if (!isActiveBetaServer(betaId, name)) continue;
                     insertSnapshot.run(
                         betaId,
                         name,
@@ -552,7 +568,9 @@ async function deployBetaServersInternal(betaId: number, names: string[]): Promi
                 }
                 const composedBuf = Buffer.from(composedResult.text, 'utf8');
                 db().prepare('INSERT OR IGNORE INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, ?)').run(betaId, name, targetRel);
+                if (!isActiveBetaServer(betaId, name)) continue;
                 await writeFile(targetPath, composedBuf);
+                if (!isActiveBetaServer(betaId, name)) continue;
                 recordScript(
                     name,
                     targetRel,
@@ -568,6 +586,7 @@ async function deployBetaServersInternal(betaId: number, names: string[]): Promi
             }
         }
         const t = nowIso();
+        if (!isActiveBetaServer(betaId, name)) continue;
         if (firstErr) {
             updateStage.run('failed', `deploy: ${firstErr}`, t, t, betaId, name);
             emit({ type: 'beta-server-stage', betaId, server: name, stage: 'failed', error: `deploy: ${firstErr}` });
@@ -619,8 +638,13 @@ async function rollbackBetaInternal(betaId: number): Promise<RollbackResult> {
     );
 
     for (const bs of servers) {
+        if (!isActiveBetaServer(betaId, bs.server)) {
+            warnings.push(`${bs.server}: original server was deleted; retained history was skipped`);
+            continue;
+        }
         if (!bs.deployed_at && bs.stage === 'queued') continue;
         for (const { script } of bundle) {
+            if (!isActiveBetaServer(betaId, bs.server)) break;
             if (isCollapsingKind(script.kind)) continue; // handled once per server below
             if (storedPlan(betaId) && !db().prepare('SELECT 1 FROM beta_file_changes WHERE beta_id = ? AND server = ? AND rel_path = ?').get(betaId, bs.server, script.target_rel_path)) continue;
             const dest = destOfKind(bs.server, script.kind);
@@ -633,7 +657,9 @@ async function rollbackBetaInternal(betaId: number): Promise<RollbackResult> {
                     | undefined;
                 if (snap) {
                     await mkdir(join(dest, dirname(script.target_rel_path)), { recursive: true });
+                    if (!isActiveBetaServer(betaId, bs.server)) break;
                     await writeFile(targetPath, snap.prior_content);
+                    if (!isActiveBetaServer(betaId, bs.server)) break;
                     if (bss.prior_version_id) {
                         recordScript(bs.server, script.target_rel_path, bss.prior_version_id, snap.prior_sha256, snap.prior_size, `rollback:${betaId}`);
                     } else {
@@ -646,12 +672,14 @@ async function rollbackBetaInternal(betaId: number): Promise<RollbackResult> {
             } else {
                 // No prior file — the beta wrote a fresh file. Drop it.
                 if (existsSync(targetPath)) await rm(targetPath, { force: true });
+                if (!isActiveBetaServer(betaId, bs.server)) break;
                 removeScript(bs.server, script.target_rel_path);
                 dropped.add(bs.server);
             }
         }
 
         // Collapsing config layer: single restore for the shared cfg file.
+        if (!isActiveBetaServer(betaId, bs.server)) continue;
         if (hasCollapsing && (!storedPlan(betaId) || db().prepare('SELECT 1 FROM beta_file_changes WHERE beta_id = ? AND server = ? AND rel_path = ?').get(betaId, bs.server, configFileOf(bs.server)))) {
             const dest = destOfKind(bs.server, 'config');
             const targetRel = configFileOf(bs.server);
@@ -661,7 +689,9 @@ async function rollbackBetaInternal(betaId: number): Promise<RollbackResult> {
                 | undefined;
             if (snap) {
                 await mkdir(dest, { recursive: true });
+                if (!isActiveBetaServer(betaId, bs.server)) continue;
                 await writeFile(targetPath, snap.prior_content);
+                if (!isActiveBetaServer(betaId, bs.server)) continue;
                 recordScript(bs.server, targetRel, null, snap.prior_sha256, snap.prior_size, `rollback:${betaId}`);
                 restored.add(bs.server);
             } else if (existsSync(targetPath)) {
@@ -669,6 +699,7 @@ async function rollbackBetaInternal(betaId: number): Promise<RollbackResult> {
                 // no pre-existing file. Remove the beta-written cfg to leave
                 // the server clean.
                 await rm(targetPath, { force: true });
+                if (!isActiveBetaServer(betaId, bs.server)) continue;
                 removeScript(bs.server, targetRel);
                 dropped.add(bs.server);
             }
@@ -738,7 +769,8 @@ function promotionPlan(betaId: number): { preview: BetaPromotionPreview; configC
         return { script, version };
     });
     if (!bundle.length) throw new BetaError('This beta has no files.', 'no-scripts');
-    const tested = new Set(listBetaServers(betaId).map(row => row.server));
+    const participants = listBetaServers(betaId);
+    const tested = new Set(participants.filter(row => isActiveBetaServer(betaId, row.server)).map(row => row.server));
     const targets = (script: ScriptRow) => [...new Set([...tested, ...(script.scope_type && script.scope_value
         ? serversMatchingScope(script.scope_type, script.scope_value) : [])])].sort();
     const files = bundle.map(({ script, version }) => ({ id: script.id, name: script.name, kind: script.kind,
@@ -746,7 +778,8 @@ function promotionPlan(betaId: number): { preview: BetaPromotionPreview; configC
         release_version: version.release_version, sha256: version.sha256,
         scope: script.scope_type ? `${script.scope_type}: ${script.scope_value}` : 'Selected test servers only', servers: targets(script) }));
     const names = [...new Set(files.flatMap(file => file.servers))].sort();
-    const blockers: string[] = [];
+    const blockers: string[] = participants.filter(row => !isActiveBetaServer(betaId, row.server))
+        .map(row => `${row.server}: original test server was deleted; start a new beta before promoting.`);
     const servers = names.map(name => {
         const server = getServer(name);
         if (!server || server.destroyed_at) blockers.push(`${name}: server is no longer available.`);
@@ -810,7 +843,7 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
         version: getVersion(br.script_version_id)!
     }));
 
-    const betaServerSet = new Set(listBetaServers(betaId).map((r) => r.server));
+    const betaServerSet = new Set(listBetaServers(betaId).filter(row => isActiveBetaServer(betaId, row.server)).map((r) => r.server));
 
     // Split bundle into non-collapsing (per-file) and collapsing (composed cfg).
     const nonCollapsing = bundle.filter((b) => !isCollapsingKind(b.script.kind));
@@ -944,12 +977,16 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
     // the captured target lists/content so later membership changes never expand it.
     checkReview();
     for (const p of plans) {
+        checkReview();
         const dest = destOfKind(p.server, p.script.kind);
         try {
             await mkdir(join(dest, dirname(p.script.target_rel_path)), { recursive: true });
+            checkReview();
             await writeFile(join(dest, p.script.target_rel_path), p.version.content);
+            checkReview();
             planWritten.set(p, true);
         } catch (err) {
+            if (err instanceof BetaError) throw err;
             planWritten.set(p, false);
             if (!serverErr.has(p.server)) serverErr.set(p.server, `${p.script.name}: ${(err as Error).message}`);
         }
@@ -957,16 +994,20 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
     // Write composed cfg to each target.
     const cfgWritten = new Map<CfgPlan, boolean>();
     for (const cp of cfgPlans) {
+        checkReview();
         try {
             const dest = destOfKind(cp.server, 'config');
             await mkdir(dest, { recursive: true });
+            checkReview();
             const targetRel = configFileOf(cp.server);
             const targetPath = join(dest, targetRel);
             // Beta server already has composed content at targetPath; overwrite
             // idempotently. Scope-expansion server: write in place.
             await writeFile(targetPath, cp.composedContent);
+            checkReview();
             cfgWritten.set(cp, true);
         } catch (err) {
+            if (err instanceof BetaError) throw err;
             cfgWritten.set(cp, false);
             const msg = `configs: ${(err as Error).message}`;
             if (!serverErr.has(cp.server)) serverErr.set(cp.server, msg);

src/lib/server/fleet/collections.ts

@@ -17,8 +17,8 @@ function coverage(entries: BlueprintEntry[]) {
     const registered = db().prepare('SELECT name, game, map_slug, map FROM servers WHERE destroyed_at IS NULL').all() as Server[];
     const queued = db().prepare(`SELECT j.server AS name, substr(j.server, 1, 2) AS game, j.map_slug, j.expected_map AS map
         FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
-        WHERE d.state IN ('queued', 'running', 'awaiting-health')`).all() as Server[];
-    const removing = new Set((db().prepare("SELECT server FROM server_deployments WHERE action = 'destroy' AND state IN ('waiting', 'running')").all() as { server: string }[]).map(row => row.server));
+        WHERE d.server_deleted_at IS NULL AND d.state IN ('queued', 'running', 'awaiting-health')`).all() as Server[];
+    const removing = new Set((db().prepare("SELECT server FROM server_deployments WHERE server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')").all() as { server: string }[]).map(row => row.server));
     // A provisioning server can appear in both tables; count its identity once.
     const servers = [...new Map([...queued, ...registered].map(server => [server.name, server])).values()];
     return entries.map(entry => {
@@ -35,7 +35,7 @@ function coverage(entries: BlueprintEntry[]) {
             return matches.length === 1 && matches[0].slug === entry.recipe.mapSlug;
         }).map(server => server.name).sort();
         const coverage = existingServers.map(name => {
-            const operation = db().prepare("SELECT id, state FROM server_deployments WHERE server = ? AND action IN ('create','rebuild') AND state IN ('queued', 'running', 'awaiting-health') ORDER BY rowid DESC LIMIT 1").get(name) as { id: string; state: string } | undefined;
+            const operation = db().prepare("SELECT id, state FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('create','rebuild') AND state IN ('queued', 'running', 'awaiting-health') ORDER BY rowid DESC LIMIT 1").get(name) as { id: string; state: string } | undefined;
             const server = getServer(name);
             return { name, health: operation ? operation.state === 'queued' ? 'queued' : 'starting'
                 : server?.container_state && ['stopped', 'exited', 'created', 'dead'].includes(server.container_state) ? 'stopped'
@@ -49,8 +49,8 @@ function coverage(entries: BlueprintEntry[]) {
 function fleetSnapshot() {
     return housePartyTargets().map(name => ({ name,
         server: db().prepare('SELECT game, map_slug, map, game_port FROM servers WHERE name = ?').get(name) ?? null,
-        creation: db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action IN ('create','rebuild') ORDER BY rowid DESC LIMIT 1").get(name) ?? null,
-        removal: db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) ?? null
+        creation: db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('create','rebuild') ORDER BY rowid DESC LIMIT 1").get(name) ?? null,
+        removal: db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) ?? null
     }));
 }
 
@@ -149,8 +149,8 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
         const removalJobs = removals.map((server, position) => {
             const ports = (db().prepare(`SELECT game_port AS port FROM servers WHERE name = ? UNION
                 SELECT j.port FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
-                WHERE j.server = ? AND d.state IN ('queued', 'running', 'awaiting-health')`).all(server.name, server.name) as { port: number | null }[]).map(row => row.port);
-            const queued = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'create' AND state = 'queued'").get(server.name) as { id: string } | undefined;
+                WHERE j.server = ? AND d.server_deleted_at IS NULL AND d.state IN ('queued', 'running', 'awaiting-health')`).all(server.name, server.name) as { port: number | null }[]).map(row => row.port);
+            const queued = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'create' AND state = 'queued'").get(server.name) as { id: string } | undefined;
             let id: string;
             if (queued && !getServer(server.name)) {
                 // No container or files exist yet: cancel the reservation instead of booting just to remove it.
@@ -210,13 +210,13 @@ export function listCollections(): CollectionSummary[] {
 export function getCollection(id: string): CollectionDetail | null {
     const collection = db().prepare(summarySql + ' WHERE c.id = ? GROUP BY c.id').get(id) as CollectionSummary | undefined;
     if (!collection) return null;
-    const servers = db().prepare(`SELECT d.id, d.server, d.action, d.state, d.stage, d.started_at, d.finished_at, d.error, j.port, j.map_slug,
-        EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server) AS server_exists
+    const servers = db().prepare(`SELECT d.id, d.server, d.server_deleted_at, d.action, d.state, d.stage, d.started_at, d.finished_at, d.error, j.port, j.map_slug,
+        (d.server_deleted_at IS NULL AND EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server AND s.destroyed_at IS NULL)) AS server_exists
         FROM (${membershipSql}) m JOIN server_deployments d ON d.id = m.deployment_id
         LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE m.collection_id = ? ORDER BY d.action DESC, m.position`).all(id) as CollectionDetail['servers'];
     const queue = creationQueue();
     const positions = new Map(queue.queued.map((job, index) => [job.id, index + 1]));
-    const stored = db().prepare('SELECT retained_servers FROM server_launch_collections WHERE id = ?').get(id) as { retained_servers: string };
-    return { collection, retainedServers: JSON.parse(stored.retained_servers), queue: { concurrency: fleetSettings().createConcurrency, active: queue.active, queued: queue.queued.length },
+    const stored = db().prepare('SELECT retained_servers, retained_server_deletions FROM server_launch_collections WHERE id = ?').get(id) as { retained_servers: string; retained_server_deletions: string };
+    return { collection, retainedServers: JSON.parse(stored.retained_servers), retainedServerDeletedAt: JSON.parse(stored.retained_server_deletions), queue: { concurrency: fleetSettings().createConcurrency, active: queue.active, queued: queue.queued.length },
         servers: servers.map(server => ({ ...server, queuePosition: positions.get(server.id) ?? null })) };
 }

src/lib/server/fleet/drift.ts

@@ -54,7 +54,7 @@ function toPosix(p: string): string {
 
 function hasActiveBeta(server: string): boolean {
     return !!db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
-        WHERE bs.server = ? AND b.state IN ('running', 'ready-promote') LIMIT 1`).get(server);
+        WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running', 'ready-promote') LIMIT 1`).get(server);
 }
 
 async function requireServerDirectory(server: string): Promise<void> {

src/lib/server/fleet/leaderboard-ingest.ts

@@ -51,7 +51,7 @@ function fingerprint(report: GameTelemetry): string {
 export function recordLeaderboardTelemetry(handle: Database.Database, report: GameTelemetry): void {
     const identity = mapIdentity(handle, report);
     const signature = fingerprint(report);
-    let match = handle.prepare('SELECT * FROM leaderboard_matches WHERE server = ? ORDER BY id DESC LIMIT 1')
+    let match = handle.prepare('SELECT * FROM leaderboard_matches WHERE server = ? AND server_deleted_at IS NULL ORDER BY id DESC LIMIT 1')
         .get(report.server) as Match | undefined;
     if (match?.last_fingerprint === signature && match.map_key === identity.key) return;
 

src/lib/server/fleet/match-history.ts

@@ -6,7 +6,7 @@ export function serverMatchHistory(server?: string, limit = 25): { matches: Serv
     const where = `WHERE m.highest_round >= ${MIN_RECORDED_ROUND}${server === undefined ? '' : ' AND m.server = ?'}`;
     const parameters = server === undefined ? [] : [server];
     const total = (handle.prepare(`SELECT COUNT(*) AS total FROM leaderboard_matches m ${where}`).get(...parameters) as { total: number }).total;
-    const matches = handle.prepare(`SELECT m.id, m.server, m.map_name AS map, m.started_at AS startedAt, m.status,
+    const matches = handle.prepare(`SELECT m.id, m.server, m.server_deleted_at, m.map_name AS map, m.started_at AS startedAt, m.status,
         CASE WHEN m.status = 'completed' THEN m.ended_at END AS endedAt,
         m.start_players AS startPlayers,
         (SELECT COUNT(*) FROM leaderboard_match_players p WHERE p.match_id = m.id) AS uniquePlayers,

src/lib/server/fleet/overview.ts

@@ -14,7 +14,7 @@ export function fleetOperations(now = Date.now()): FleetOperation[] {
     const campaigns = handle.prepare(`SELECT d.id, d.action, d.started_at, ds.server,
         ds.restart_state, ds.error, ds.restarted_at, ds.players
         FROM deployments d JOIN deployment_servers ds ON ds.deployment_id = d.id
-        WHERE ds.restart_state != 'done' OR ds.checked_at >= ? OR d.finished_at >= ? OR d.started_at >= ?`).all(since, since, since) as {
+        WHERE ds.server_deleted_at IS NULL AND (ds.restart_state != 'done' OR ds.checked_at >= ? OR d.finished_at >= ? OR d.started_at >= ?)`).all(since, since, since) as {
         id: string; action: string; started_at: string; server: string; restart_state: string;
         error: string | null; restarted_at: string | null; players: number | null;
     }[];
@@ -35,10 +35,10 @@ export function fleetOperations(now = Date.now()): FleetOperation[] {
     }
     // Only the newest lifecycle operation can still require action on a server.
     const lifecycle = handle.prepare(`SELECT d.* FROM server_deployments d
-        WHERE d.state IN ('queued', 'running', 'waiting', 'awaiting-health')
+        WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued', 'running', 'waiting', 'awaiting-health')
         OR (d.state IN ('done', 'warning', 'cancelled') AND d.finished_at >= ?)
         OR (d.state IN ('failed', 'interrupted', 'warning') AND NOT EXISTS
-            (SELECT 1 FROM server_deployments n WHERE n.server = d.server AND n.rowid > d.rowid))`)
+            (SELECT 1 FROM server_deployments n WHERE n.server = d.server AND n.rowid > d.rowid)))`)
         .all(since) as { id: string; server: string; action: string; state: string; stage: string;
             error: string | null; started_at: string }[];
     for (const row of lifecycle) {
@@ -52,7 +52,7 @@ export function fleetOperations(now = Date.now()): FleetOperation[] {
             detail: row.error ?? row.stage, href: `/provisioning/${encodeURIComponent(row.id)}` });
     }
     const betas = handle.prepare(`SELECT b.id, b.opened_at, bs.server, bs.stage, bs.error
-        FROM betas b JOIN beta_servers bs ON bs.beta_id = b.id WHERE b.state IN ('running', 'ready-promote')`)
+        FROM betas b JOIN beta_servers bs ON bs.beta_id = b.id WHERE bs.server_deleted_at IS NULL AND b.state IN ('running', 'ready-promote')`)
         .all() as { id: number; opened_at: string; server: string; stage: string; error: string | null }[];
     const stages: Record<string, string> = { queued: 'Waiting for an earlier test wave', deploying: 'Deploying beta',
         'awaiting-empty': 'Waiting for an empty lobby', restarting: 'Restarting', 'awaiting-health': 'Recovering',
@@ -64,7 +64,7 @@ export function fleetOperations(now = Date.now()): FleetOperation[] {
             state: row.stage === 'failed' ? 'attention' : 'active', label: stages[row.stage] ?? row.stage,
             detail: row.error ?? recovering.get(row.server)?.error ?? `Beta ${row.id}`, href: `/betas/${row.id}` });
     }
-    const maintenance = handle.prepare(`SELECT m.* FROM maintenance_jobs m WHERE m.state = 'failed'
+    const maintenance = handle.prepare(`SELECT m.* FROM maintenance_jobs m WHERE m.server_deleted_at IS NULL AND m.state = 'failed'
         AND NOT EXISTS (SELECT 1 FROM maintenance_jobs n WHERE n.server = m.server AND n.kind = m.kind AND n.id > m.id)`)
         .all() as { id: number; server: string; kind: string; detail: string | null; requested_at: string }[];
     for (const job of maintenance) {

src/lib/server/fleet/player-profile.ts

@@ -25,7 +25,7 @@ export function getPlayerProfile(game: string, guid: string, requestedOffset = 0
         MAX(mp.last_seen_at) AS lastSeen, ${totals} ${scope}
         GROUP BY m.map_key ORDER BY highestRound DESC, kills DESC, name COLLATE NOCASE, key`)
         .all(game, guid) as PlayerMapRecord[];
-    const matches = handle.prepare(`SELECT m.id, m.server, m.map_name AS mapName,
+    const matches = handle.prepare(`SELECT m.id, m.server, m.server_deleted_at, m.map_name AS mapName,
         m.started_at AS startedAt, mp.last_seen_at AS lastSeen, m.status, mp.completed,
         mp.highest_round AS highestRound, mp.rounds_survived AS roundsSurvived,
         mp.kills, mp.headshots, mp.revives, mp.downs, mp.best_score AS bestScore

src/lib/server/fleet/provisioning.ts

@@ -3,7 +3,7 @@ import { db } from '../db/index.js';
 /** Only the latest lifecycle operation can describe the current server. */
 export function isProvisioning(server: string): boolean {
     const operation = db().prepare(`SELECT action, state FROM server_deployments
-        WHERE server = ? ORDER BY rowid DESC LIMIT 1`).get(server) as
+        WHERE server = ? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1`).get(server) as
         { action: string; state: string } | undefined;
     return !!operation && ['create', 'rebuild', 'start'].includes(operation.action) && ['queued', 'running', 'awaiting-health'].includes(operation.state);
 }

src/lib/server/fleet/rollback.ts

@@ -100,8 +100,10 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
 
     const fileRows = db()
         .prepare(
-            `SELECT id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after
-             FROM deployment_files WHERE deployment_id = ? ORDER BY server, rel_path`
+            `SELECT f.id, f.server, f.verb, f.rel_path, f.sha256_before, f.sha256_after, f.content_before, f.content_after
+             FROM deployment_files f JOIN deployment_servers ds ON ds.deployment_id = f.deployment_id AND ds.server = f.server
+             JOIN servers s ON s.name = f.server
+             WHERE f.deployment_id = ? AND ds.server_deleted_at IS NULL AND s.destroyed_at IS NULL ORDER BY f.server, f.rel_path`
         )
         .all(promoteId) as FileRow[];
 
@@ -148,20 +150,27 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
 
     const affectedServers: string[] = [];
     const perServerFailures = new Map<string, string>();
+    const active = (server: string) => !!db().prepare(`SELECT 1 FROM deployment_servers ds JOIN servers s ON s.name = ds.server
+        WHERE ds.deployment_id = ? AND ds.server = ? AND ds.server_deleted_at IS NULL AND s.destroyed_at IS NULL`).get(promoteId, server);
 
     for (const [server, files] of perServer) {
+        if (!active(server)) continue;
         try {
             for (const f of files) {
+                if (!active(server)) break;
                 const kind = kindForRelPath(f.rel_path);
                 const dest = destOfKind(server, kind);
                 const abs = join(dest, f.rel_path);
                 await mkdir(join(dest, dirname(f.rel_path)), { recursive: true });
+                if (!active(server)) break;
                 const owningScript = scriptByTarget.get(f.rel_path);
                 if (f.verb === 'added') {
                     if (existsSync(abs)) await rm(abs, { force: true });
+                    if (!active(server)) break;
                     removeScript(server, f.rel_path);
                 } else if (f.verb === 'updated' && f.content_before) {
                     await writeFile(abs, f.content_before);
+                    if (!active(server)) break;
                     const priorVer = owningScript?.prior_version_id
                         ? getVersion(owningScript.prior_version_id)
                         : null;
@@ -175,10 +184,11 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
                     );
                 } else if (f.verb === 'removed' && f.content_before) {
                     await writeFile(abs, f.content_before);
+                    if (!active(server)) break;
                     recordScript(server, f.rel_path, null, f.sha256_before!, f.content_before.byteLength, `rollback:${newDepId}`);
                 }
             }
-            affectedServers.push(server);
+            if (active(server)) affectedServers.push(server);
         } catch (err) {
             perServerFailures.set(server, (err as Error).message);
         }
@@ -220,6 +230,7 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
             );
         }
         for (const [server, files] of perServer) {
+            if (!active(server)) continue;
             if (perServerFailures.has(server)) {
                 insertDepServer.run(newDepId, server, 0, 'failed', perServerFailures.get(server)!);
                 continue;

src/lib/server/fleet/servers.ts

@@ -207,7 +207,7 @@ export function heldByBeta(server: string): Map<string, number> {
              JOIN betas b ON b.id = bsv.beta_id
              JOIN beta_scripts bs ON bs.beta_id = b.id
              JOIN scripts s ON s.id = bs.script_id
-             WHERE bsv.server = ? AND b.state IN ('running','ready-promote')`
+             WHERE bsv.server = ? AND bsv.server_deleted_at IS NULL AND b.state IN ('running','ready-promote')`
         )
         .all(server) as Array<{ target: string; beta_id: number }>;
     const map = new Map<string, number>();
@@ -226,7 +226,7 @@ export function ownerOf(server: string, target: string): number | null {
              JOIN betas b ON b.id = bsv.beta_id
              JOIN beta_scripts bs ON bs.beta_id = b.id
              JOIN scripts s ON s.id = bs.script_id
-             WHERE bsv.server = ? AND s.target_rel_path = ? AND b.state IN ('running','ready-promote')
+             WHERE bsv.server = ? AND bsv.server_deleted_at IS NULL AND s.target_rel_path = ? AND b.state IN ('running','ready-promote')
              LIMIT 1`
         )
         .get(server, target) as { beta_id: number } | undefined;

src/lib/server/fleet/status-evidence.ts

@@ -8,7 +8,7 @@ export interface TelemetryEvidence { players: number; map: string; receivedAt: s
 /** Reports from before the latest lifecycle submission describe the old process. */
 export function latestGameSubmission(server: string): string | null {
     return (db().prepare(`SELECT MAX(at) AS at FROM (
-        SELECT submitted_at AS at FROM server_deployments WHERE server=? AND action IN ('create','start','restart','rebuild')
+        SELECT submitted_at AS at FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild')
         UNION ALL SELECT requested_at AS at FROM restart_rollout WHERE server=?)`).get(server, server) as { at: string | null }).at;
 }
 

src/lib/server/fleet/telemetry.ts

@@ -37,7 +37,7 @@ export function saveTelemetry(report: GameTelemetry): GameTelemetry {
         } else if (report.event === 'game_end') {
             handle.prepare('UPDATE servers SET game_ended_at = ? WHERE name = ?').run(report.receivedAt, report.server);
             // Legacy betas also take match completion directly from the script.
-            if (report.round >= MIN_RECORDED_ROUND) handle.prepare(`UPDATE beta_servers SET match_seen_at = ? WHERE server = ? AND stage = 'awaiting-match'
+            if (report.round >= MIN_RECORDED_ROUND) handle.prepare(`UPDATE beta_servers SET match_seen_at = ? WHERE server = ? AND server_deleted_at IS NULL AND stage = 'awaiting-match'
                 AND healthy_at <= ? AND beta_id IN (SELECT id FROM betas WHERE state = 'running')`)
                 .run(report.receivedAt, report.server, report.receivedAt);
         }