diff --git a/README.md b/README.md index 59855dc..d4cd0d0 100644 --- a/README.md +++ b/README.md @@ -382,6 +382,17 @@ RCON parsing (fake UDP socket). Campaigns (`/campaigns`) tracks script/config updates, drift reconciliation, restart progress, file diffs, and rollbacks. Existing campaign records are preserved in the original database tables. The campaign API lives at `/api/campaigns`; legacy detail API endpoints remain available for compatibility. +Deleting a server preserves its name, recorded beta progress and test evidence, +campaign results and file diffs, provisioning and maintenance history, and match +records. Historical entries show **Deleted** and omit live server links and +actions. Reusing the same server name starts a new instance; old records remain +marked deleted, and old work cannot target the replacement. Deleting a participant +pauses an open beta; roll it back and open a new beta to test the replacement. +Normal history retention still applies. Records erased by earlier versions cannot +be restored by this upgrade. Existing history is marked deleted where the server +is already absent; instances that reused a name before upgrading cannot always be +distinguished. + Provisioning (`/provisioning`, `/api/provisioning`) owns Create, Start, Restart, Rebuild, Stop, and Destroy: request persistence, eligibility and occupancy checks, execution, recovery, and history. Campaigns, betas, and scheduled maintenance request lifecycle work through Provisioning and link to the resulting operation. They retain their file, test-evidence, and version-verification workflows. Pending automated restarts appear in Provisioning before execution. History survives server removal. New creation requests return HTTP 202 with an operation ID and open the provisioning detail page. Creation is complete only when Komodo reports healthy and RCON or fresh telemetry verifies game status with an engine map that exactly matches the selected map. Queued requests reserve their server name and port; queued creation and pending health checks survive application restarts. The detail page records startup progress and explains any health or map mismatch that is still blocking completion. Each server deployment opens a detail page with its flow and timestamped activity. Destroy uses an in-app confirmation modal and immediately returns a queued deployment (`DELETE /api/servers/:name` returns HTTP 202 with `deploymentId`). The worker waits for a fresh zero-player RCON or telemetry report, or a verified stopped container, before teardown. Occupied reports from either source block teardown; servers without current verification remain queued and are checked each worker tick. Waiting requests survive application restarts, while interrupted teardown is marked for inspection. Completed destruction stays on the provisioning detail page. diff --git a/src/lib/beta-plan.ts b/src/lib/beta-plan.ts index 150a987..3c94436 100644 --- a/src/lib/beta-plan.ts +++ b/src/lib/beta-plan.ts @@ -82,9 +82,9 @@ export interface BetaTestView { plan: BetaPlan; outcome: string; reason: string | null; - servers: Array<{ server: string; phase: number; passed: boolean; progress: RequirementProgress[]; observingAt?: string | null; + servers: Array<{ server: string; server_deleted_at: string | null; phase: number; passed: boolean; progress: RequirementProgress[]; observingAt?: string | null; observation?: { lastGameplayAt: string | null; lastEvent: string | null; players: number | null; round: number | null; matchStarted: boolean; gameEnded: boolean; healthy: boolean | null; healthCheckedAt: string | null } }>; - evidence: Array<{ id: number; server: string; received_at: string; kind: string; payload: string }>; + evidence: Array<{ id: number; server: string; server_deleted_at: string | null; received_at: string; kind: string; payload: string }>; } export interface BetaPlanIssue { field: string; message: string; requirementId?: string } diff --git a/src/lib/deployment-plan.ts b/src/lib/deployment-plan.ts index c8efba8..86fb651 100644 --- a/src/lib/deployment-plan.ts +++ b/src/lib/deployment-plan.ts @@ -66,9 +66,10 @@ export interface CollectionSummary { export interface CollectionDetail { collection: CollectionSummary; retainedServers: string[]; + retainedServerDeletedAt?: Record; queue: { concurrency: number; active: { id: string; server: string; state: string; stage: string }[]; queued: number }; servers: { id: string; server: string; action: 'create' | 'destroy'; port: number | null; map_slug: string | null; state: string; stage: string; - started_at: string; finished_at: string | null; error: string | null; server_exists: number; queuePosition: number | null }[]; + started_at: string; finished_at: string | null; error: string | null; server_exists: number; server_deleted_at?: string | null; queuePosition: number | null }[]; } export interface LaunchPreview { diff --git a/src/lib/leaderboard.ts b/src/lib/leaderboard.ts index 6997069..4963c82 100644 --- a/src/lib/leaderboard.ts +++ b/src/lib/leaderboard.ts @@ -22,6 +22,7 @@ export interface PlayerMapRecord extends PlayerStats { export interface PlayerMatch { id: number; server: string; + server_deleted_at?: string | null; mapName: string; startedAt: string; lastSeen: string; diff --git a/src/lib/match-history.ts b/src/lib/match-history.ts index 66318ac..994ae9a 100644 --- a/src/lib/match-history.ts +++ b/src/lib/match-history.ts @@ -4,6 +4,7 @@ export const MIN_RECORDED_ROUND = 2; export interface ServerMatch { id: number; server: string; + server_deleted_at?: string | null; map: string; startedAt: string; endedAt: string | null; diff --git a/src/lib/plutonium.ts b/src/lib/plutonium.ts index 4b93c2a..0e6c7c4 100644 --- a/src/lib/plutonium.ts +++ b/src/lib/plutonium.ts @@ -22,6 +22,7 @@ export interface PlutoniumVersion { export interface MaintenanceJob { id: number; server: string; + server_deleted_at?: string | null; state: 'queued' | 'restarting' | 'done' | 'failed' | 'cancelled'; target_revision: number | null; requested_at: string; diff --git a/src/lib/server/db/index.ts b/src/lib/server/db/index.ts index 7692bb7..f5b62d5 100644 --- a/src/lib/server/db/index.ts +++ b/src/lib/server/db/index.ts @@ -7,6 +7,7 @@ import { requirement, type BetaPlan, type BetaRequirement } from '../../beta-pla import { emptyMetrics } from '../fleet/beta-evaluation.js'; import { backfillLeaderboard } from '../fleet/leaderboard-ingest.js'; import { initializeFleetSettings } from './fleet-settings.js'; +import { migrateServerHistory } from './server-history.js'; let _db: Database.Database | null = null; @@ -124,6 +125,8 @@ function backfillMultiScriptJoins(handle: Database.Database): void { FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id WHERE b.script_id IS NOT NULL + AND bs.server_deleted_at IS NULL + AND EXISTS (SELECT 1 FROM servers WHERE name = bs.server) AND NOT EXISTS ( SELECT 1 FROM beta_server_scripts x WHERE x.beta_id = bs.beta_id AND x.server = bs.server AND x.script_id = b.script_id @@ -329,6 +332,7 @@ export function db(): Database.Database { { name: 'mode', ddl: "ALTER TABLE server_launch_collections ADD COLUMN mode TEXT NOT NULL DEFAULT 'fill'" }, { name: 'retained_servers', ddl: "ALTER TABLE server_launch_collections ADD COLUMN retained_servers TEXT NOT NULL DEFAULT '[]'" } ]); + migrateServerHistory(handle); migrateTable(handle, 'server_collection_previews', [ { name: 'mode', ddl: "ALTER TABLE server_collection_previews ADD COLUMN mode TEXT NOT NULL DEFAULT 'fill'" }, { name: 'fleet_snapshot', ddl: 'ALTER TABLE server_collection_previews ADD COLUMN fleet_snapshot TEXT' } diff --git a/src/lib/server/db/schema.sql b/src/lib/server/db/schema.sql index dc1f1a5..047c80b 100644 --- a/src/lib/server/db/schema.sql +++ b/src/lib/server/db/schema.sql @@ -11,6 +11,7 @@ PRAGMA foreign_keys = ON; CREATE TABLE IF NOT EXISTS leaderboard_matches ( id INTEGER PRIMARY KEY AUTOINCREMENT, server TEXT NOT NULL, + server_deleted_at TEXT, game TEXT NOT NULL, map TEXT NOT NULL, map_key TEXT NOT NULL, @@ -181,7 +182,8 @@ CREATE INDEX IF NOT EXISTS idx_betas_script ON betas(script_id); CREATE TABLE IF NOT EXISTS beta_servers ( beta_id INTEGER NOT NULL REFERENCES betas(id) ON DELETE CASCADE, - server TEXT NOT NULL REFERENCES servers(name), + server TEXT NOT NULL, + server_deleted_at TEXT, had_original INTEGER NOT NULL, -- deprecated; see beta_server_scripts prior_version_id INTEGER REFERENCES script_versions(id), -- deprecated; see beta_server_scripts stage TEXT NOT NULL, -- 'deploying'|'awaiting-empty'|'restarting'|'awaiting-health'|'awaiting-match'|'ready'|'failed' @@ -206,7 +208,7 @@ CREATE TABLE IF NOT EXISTS beta_test_plans ( CREATE TABLE IF NOT EXISTS beta_test_runs ( id TEXT PRIMARY KEY, beta_id INTEGER NOT NULL REFERENCES betas(id) ON DELETE CASCADE, - server TEXT NOT NULL REFERENCES servers(name), + server TEXT NOT NULL, phase INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL, observing_at TEXT, @@ -261,7 +263,7 @@ CREATE INDEX IF NOT EXISTS idx_beta_scripts_script ON beta_scripts(script_id); -- lives in beta_file_snapshots. CREATE TABLE IF NOT EXISTS beta_server_scripts ( beta_id INTEGER NOT NULL REFERENCES betas(id) ON DELETE CASCADE, - server TEXT NOT NULL REFERENCES servers(name), + server TEXT NOT NULL, script_id INTEGER NOT NULL REFERENCES scripts(id) ON DELETE CASCADE, had_original INTEGER NOT NULL, prior_version_id INTEGER REFERENCES script_versions(id), @@ -302,6 +304,7 @@ CREATE INDEX IF NOT EXISTS idx_deployments_started ON deployments(started_at DES CREATE TABLE IF NOT EXISTS deployment_servers ( deployment_id TEXT NOT NULL REFERENCES deployments(id) ON DELETE CASCADE, server TEXT NOT NULL, -- Campaign observation survives a rebuild's registry gap. + server_deleted_at TEXT, files_changed INTEGER NOT NULL, restart_state TEXT NOT NULL, -- pending | done | failed | manual | held players INTEGER, @@ -373,6 +376,7 @@ CREATE TABLE IF NOT EXISTS plutonium_versions ( CREATE TABLE IF NOT EXISTS maintenance_jobs ( id INTEGER PRIMARY KEY AUTOINCREMENT, server TEXT NOT NULL, + server_deleted_at TEXT, state TEXT NOT NULL DEFAULT 'queued', target_revision INTEGER, requested_at TEXT NOT NULL, @@ -411,6 +415,7 @@ CREATE INDEX IF NOT EXISTS idx_telemetry_requests_received ON telemetry_requests CREATE TABLE IF NOT EXISTS server_deployments ( id TEXT PRIMARY KEY, server TEXT NOT NULL, + server_deleted_at TEXT, action TEXT NOT NULL CHECK (action IN ('create', 'start', 'restart', 'rebuild', 'stop', 'destroy')), state TEXT NOT NULL, stage TEXT NOT NULL, @@ -486,7 +491,8 @@ CREATE TABLE IF NOT EXISTS server_launch_collections ( name TEXT NOT NULL, created_at TEXT NOT NULL, mode TEXT NOT NULL DEFAULT 'fill', - retained_servers TEXT NOT NULL DEFAULT '[]' + retained_servers TEXT NOT NULL DEFAULT '[]', + retained_server_deletions TEXT NOT NULL DEFAULT '{}' ); CREATE TABLE IF NOT EXISTS server_collection_members ( @@ -629,6 +635,7 @@ CREATE TABLE IF NOT EXISTS metric_samples ( CREATE TABLE IF NOT EXISTS watchdog_incidents ( id TEXT PRIMARY KEY, server TEXT NOT NULL, + server_deleted_at TEXT, kind TEXT NOT NULL CHECK (kind IN ('health','drift')), state TEXT NOT NULL CHECK (state IN ('open','recovering','attention','resolved','dismissed')), summary TEXT NOT NULL, diff --git a/src/lib/server/db/server-history.ts b/src/lib/server/db/server-history.ts new file mode 100644 index 0000000..419d06e --- /dev/null +++ b/src/lib/server/db/server-history.ts @@ -0,0 +1,130 @@ +import type Database from 'better-sqlite3'; + +const historyTables = ['beta_servers', 'deployment_servers', 'server_deployments', 'leaderboard_matches', 'watchdog_incidents', 'maintenance_jobs']; +const now = "strftime('%Y-%m-%dT%H:%M:%fZ','now')"; + +/** Detach durable beta records without dropping their evidence, indexes, or activity triggers. */ +function detachBetaHistory(handle: Database.Database): void { + for (const name of ['beta_servers', 'beta_test_runs', 'beta_server_scripts']) { + const { sql } = handle.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?").get(name) as { sql: string }; + const replacement = sql.replace(/server\s+TEXT NOT NULL REFERENCES servers\(name\)/, 'server TEXT NOT NULL'); + if (replacement === sql) continue; + const dependents = handle.prepare("SELECT sql FROM sqlite_master WHERE tbl_name = ? AND type IN ('index','trigger') AND sql IS NOT NULL") + .all(name) as { sql: string }[]; + const columns = (handle.prepare(`PRAGMA table_info(${name})`).all() as { name: string }[]).map(col => col.name).join(', '); + handle.pragma('foreign_keys = OFF'); + try { + handle.transaction(() => { + handle.exec(replacement.replace(name, name + '_history_new')); + handle.exec(`INSERT INTO ${name}_history_new (rowid, ${columns}) SELECT rowid, ${columns} FROM ${name}; + DROP TABLE ${name}; ALTER TABLE ${name}_history_new RENAME TO ${name};`); + for (const dependent of dependents) handle.exec(dependent.sql); + if ((handle.pragma('foreign_key_check') as unknown[]).length) throw new Error('Server history migration found broken references'); + })(); + } finally { handle.pragma('foreign_keys = ON'); } + } +} + +// A rebuild owns the replacement instance. Only its current operation and the +// Watchdog records observing it cross the deliberate gap in the live registry. +const removingRebuild = `SELECT d.id FROM server_deployments d + JOIN server_rebuild_jobs r ON r.deployment_id = d.id + WHERE d.server = OLD.name AND d.server_deleted_at IS NULL AND d.action = 'rebuild' + AND d.state IN ('waiting','running','failed','interrupted') + AND d.stage = 'Removing server files and registry' AND r.removed_at IS NULL + AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server + AND newer.rowid > d.rowid)`; + +/** Installed after additive migrations so existing databases can boot safely. */ +export function migrateServerHistory(handle: Database.Database): void { + detachBetaHistory(handle); + handle.transaction(() => { + for (const table of historyTables) { + const columns = handle.prepare(`PRAGMA table_info(${table})`).all() as { name: string }[]; + if (!columns.some(column => column.name === 'server_deleted_at')) handle.exec(`ALTER TABLE ${table} ADD COLUMN server_deleted_at TEXT`); + } + const collectionColumns = handle.prepare('PRAGMA table_info(server_launch_collections)').all() as { name: string }[]; + if (!collectionColumns.some(column => column.name === 'retained_server_deletions')) { + handle.exec("ALTER TABLE server_launch_collections ADD COLUMN retained_server_deletions TEXT NOT NULL DEFAULT '{}'"); + } + // Historical unfinished work cannot claim the next server with this name. + handle.exec(`DROP INDEX IF EXISTS idx_maintenance_pending; + CREATE UNIQUE INDEX idx_maintenance_pending ON maintenance_jobs(server) + WHERE state IN ('queued', 'restarting') AND server_deleted_at IS NULL; + DROP INDEX IF EXISTS idx_watchdog_active; + CREATE UNIQUE INDEX idx_watchdog_active ON watchdog_incidents(server,kind) + WHERE state != 'resolved' AND server_deleted_at IS NULL; + DROP TRIGGER IF EXISTS preserve_deleted_server_history; + CREATE TRIGGER preserve_deleted_server_history BEFORE DELETE ON servers + BEGIN + UPDATE server_launch_collections SET retained_server_deletions = json_patch(retained_server_deletions, json_object(OLD.name, ${now})) + WHERE EXISTS (SELECT 1 FROM json_each(retained_servers) WHERE value = OLD.name) + AND NOT EXISTS (SELECT 1 FROM json_each(retained_server_deletions) WHERE key = OLD.name); + UPDATE beta_servers SET server_deleted_at = ${now} WHERE server = OLD.name AND server_deleted_at IS NULL; + UPDATE deployment_servers SET server_deleted_at = ${now} + WHERE server = OLD.name AND server_deleted_at IS NULL AND deployment_id NOT IN ( + SELECT campaign_id FROM watchdog_actions WHERE campaign_id IS NOT NULL AND operation_id IN (${removingRebuild})); + UPDATE watchdog_incidents SET server_deleted_at = ${now} + WHERE server = OLD.name AND server_deleted_at IS NULL AND id NOT IN ( + SELECT incident_id FROM watchdog_actions WHERE operation_id IN (${removingRebuild})); + UPDATE maintenance_jobs SET server_deleted_at = ${now}, + finished_at = CASE WHEN state IN ('queued','restarting') THEN ${now} ELSE finished_at END, + detail = CASE WHEN state IN ('queued','restarting') THEN COALESCE(detail || '; ', '') || 'Server deleted' ELSE detail END, + state = CASE WHEN state IN ('queued','restarting') THEN 'cancelled' ELSE state END + WHERE server = OLD.name AND server_deleted_at IS NULL; + UPDATE leaderboard_matches SET server_deleted_at = ${now}, + ended_at = CASE WHEN status = 'active' THEN COALESCE(ended_at, ${now}) ELSE ended_at END, + status = CASE WHEN status = 'active' THEN 'interrupted' ELSE status END + WHERE server = OLD.name AND server_deleted_at IS NULL; + INSERT INTO server_deployment_events (deployment_id, created_at, message) + SELECT id, ${now}, 'Cancelled: server deleted' FROM server_deployments + WHERE server = OLD.name AND server_deleted_at IS NULL AND state IN ('queued','waiting','running','awaiting-health') + AND id NOT IN (${removingRebuild}) + AND NOT (action = 'destroy' AND state = 'running' AND stage = 'Removing server files and registry'); + UPDATE server_deployments SET state = 'cancelled', error = 'Server deleted', finished_at = ${now} + WHERE server = OLD.name AND server_deleted_at IS NULL AND state IN ('queued','waiting','running','awaiting-health') + AND id NOT IN (${removingRebuild}) + AND NOT (action = 'destroy' AND state = 'running' AND stage = 'Removing server files and registry'); + UPDATE server_deployments SET server_deleted_at = ${now} + WHERE server = OLD.name AND server_deleted_at IS NULL AND id NOT IN (${removingRebuild}); + END;`); + if (!handle.prepare("SELECT 1 FROM settings WHERE key = 'server_history_v1'").get()) { + // Recover the availability of history already orphaned before this + // migration. Queued launches and rebuild registry gaps are not deletions. + const liveRebuilds = `SELECT d.id FROM server_deployments d JOIN server_rebuild_jobs r ON r.deployment_id = d.id + WHERE d.server_deleted_at IS NULL AND d.action = 'rebuild' AND (d.state IN ('waiting','queued','running','awaiting-health') + OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry')) + AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`; + for (const table of historyTables) { + const exception = table === 'server_deployments' + ? `AND (action != 'create' OR state IN ('done','warning') OR EXISTS ( + SELECT 1 FROM server_deployments removal WHERE removal.server = server_deployments.server + AND removal.rowid > server_deployments.rowid AND removal.action = 'destroy' AND removal.state IN ('done','warning'))) + AND id NOT IN (${liveRebuilds})` + : table === 'deployment_servers' + ? `AND deployment_id NOT IN (SELECT campaign_id FROM watchdog_actions WHERE campaign_id IS NOT NULL AND operation_id IN (${liveRebuilds}))` + : table === 'watchdog_incidents' + ? `AND id NOT IN (SELECT incident_id FROM watchdog_actions WHERE operation_id IN (${liveRebuilds}))` : ''; + handle.exec(`UPDATE ${table} SET server_deleted_at = ${now} WHERE server_deleted_at IS NULL + AND NOT EXISTS (SELECT 1 FROM servers WHERE name = ${table}.server AND destroyed_at IS NULL) ${exception}`); + } + handle.exec(`UPDATE server_launch_collections SET retained_server_deletions = json_patch(retained_server_deletions, + (SELECT json_group_object(value, ${now}) FROM json_each(retained_servers) retained + WHERE NOT EXISTS (SELECT 1 FROM servers WHERE name = retained.value AND destroyed_at IS NULL) + AND NOT EXISTS (SELECT 1 FROM server_deployments d WHERE d.server = retained.value + AND d.server_deleted_at IS NULL AND d.action IN ('create','rebuild') + AND d.state IN ('queued','running','awaiting-health','waiting')) + AND NOT EXISTS (SELECT 1 FROM json_each(retained_server_deletions) WHERE key = retained.value)));`); + handle.exec(`UPDATE maintenance_jobs SET state = 'cancelled', finished_at = ${now}, detail = COALESCE(detail || '; ', '') || 'Server deleted' + WHERE server_deleted_at IS NOT NULL AND state IN ('queued','restarting'); + UPDATE leaderboard_matches SET status = 'interrupted', ended_at = COALESCE(ended_at, server_deleted_at) + WHERE server_deleted_at IS NOT NULL AND status = 'active'; + INSERT INTO server_deployment_events (deployment_id, created_at, message) + SELECT id, ${now}, 'Cancelled: server deleted' FROM server_deployments + WHERE server_deleted_at IS NOT NULL AND state IN ('queued','waiting','running','awaiting-health'); + UPDATE server_deployments SET state = 'cancelled', error = 'Server deleted', finished_at = ${now} + WHERE server_deleted_at IS NOT NULL AND state IN ('queued','waiting','running','awaiting-health'); + INSERT INTO settings (key, value) VALUES ('server_history_v1', '1');`); + } + })(); +} diff --git a/src/lib/server/fleet/beta-tests.ts b/src/lib/server/fleet/beta-tests.ts index f06947c..6c29d8e 100644 --- a/src/lib/server/fleet/beta-tests.ts +++ b/src/lib/server/fleet/beta-tests.ts @@ -9,6 +9,8 @@ import { applySample, applyGameEvent, emptyMetrics, evaluateRequirements, type T export interface TestRun { id: string; beta_id: number; server: string; phase: number; created_at: string; observing_at: string | null; metrics: string } export interface StoredPlan { beta_id: number; plan: string; outcome: string; reason: string | null } const nowIso = () => new Date().toISOString(); +const activeParticipant = (id: number, server: string) => !!db().prepare(`SELECT 1 FROM beta_servers + WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL`).get(id, server); export const storedPlan = (id: number) => db().prepare('SELECT * FROM beta_test_plans WHERE beta_id = ?').get(id) as StoredPlan | undefined; export const currentRun = (id: number, server: string) => db().prepare('SELECT * FROM beta_test_runs WHERE beta_id = ? AND server = ? AND retired_at IS NULL').get(id, server) as TestRun | undefined; export function activity(id: number, message: string) { db().prepare('INSERT INTO beta_events (beta_id, message) VALUES (?, ?)').run(id, message); } @@ -27,6 +29,7 @@ export function evidence(run: TestRun, eventId: string, kind: string, payload: u /** Internal progress record; never sent to or required from the game script. */ export async function prepareRun(betaId: number, server: string, phase = 0): Promise { + if (!activeParticipant(betaId, server)) throw new Error('Original test server was deleted; start a new beta'); db().transaction(() => { db().prepare('UPDATE beta_test_runs SET retired_at = ? WHERE beta_id = ? AND server = ? AND retired_at IS NULL').run(nowIso(), betaId, server); db().prepare('INSERT INTO beta_test_runs (id, beta_id, server, phase, created_at, metrics) VALUES (?, ?, ?, ?, ?, ?)') @@ -41,6 +44,7 @@ export function recordGameEvent(event: GameTelemetry): void { JOIN beta_test_plans p ON p.beta_id = b.id JOIN beta_servers s ON s.beta_id = b.id AND s.server = r.server WHERE r.server = ? AND r.retired_at IS NULL AND b.state IN ('running','ready-promote') + AND s.server_deleted_at IS NULL AND p.outcome IN ('active','passed') AND s.stage IN ('awaiting-health','observing','ready') AND s.restarted_at IS NOT NULL`).all(event.server) as TestRun[]; for (const run of runs) { @@ -53,6 +57,7 @@ export function recordGameEvent(event: GameTelemetry): void { } export async function observeRun(run: TestRun, plan: BetaPlan): Promise { + if (!activeParticipant(run.beta_id, run.server)) return; const m = readMetrics(run); const row = getServer(run.server); const server = row ? serverObservation(row) : null; @@ -74,19 +79,22 @@ export function testView(betaId: number): BetaTestView | null { const stored = storedPlan(betaId); if (!stored) return null; const plan = parseBetaPlan(JSON.parse(stored.plan)); - const runs = db().prepare('SELECT * FROM beta_test_runs WHERE beta_id = ? AND retired_at IS NULL ORDER BY server').all(betaId) as TestRun[]; + const runs = db().prepare(`SELECT r.*, s.server_deleted_at FROM beta_test_runs r + JOIN beta_servers s ON s.beta_id = r.beta_id AND s.server = r.server + WHERE r.beta_id = ? AND r.retired_at IS NULL ORDER BY r.server`).all(betaId) as Array; return { plan, outcome: stored.outcome, reason: stored.reason, servers: runs.map(run => { const metrics = readMetrics(run); const progress = evaluateRequirements(plan, metrics, run.phase); const latest = db().prepare("SELECT received_at, kind, payload FROM beta_test_evidence WHERE run_id = ? AND kind IN ('game_start','round_change','heartbeat','game_end') ORDER BY id DESC LIMIT 1").get(run.id) as { received_at: string; kind: string; payload: string } | undefined; const event = latest ? JSON.parse(latest.payload) : null; - return { server: run.server, phase: run.phase, passed: progress.every(r => r.passed), progress, observingAt: run.observing_at, + return { server: run.server, server_deleted_at: run.server_deleted_at, phase: run.phase, passed: progress.every(r => r.passed), progress, observingAt: run.observing_at, observation: { lastGameplayAt: latest?.received_at ?? null, lastEvent: latest?.kind ?? null, players: metrics.match?.players ?? null, round: event?.round ?? null, matchStarted: metrics.match?.started !== undefined, gameEnded: metrics.match?.ended ?? false, healthy: metrics.wasHealthy ?? null, healthCheckedAt: metrics.lastSample ? new Date(metrics.lastSample).toISOString() : null } }; }), - evidence: db().prepare(`SELECT e.id, r.server, e.received_at, e.kind, e.payload FROM beta_test_evidence e - JOIN beta_test_runs r ON r.id = e.run_id WHERE r.beta_id = ? ORDER BY e.id DESC LIMIT 200`).all(betaId) as BetaTestView['evidence'] }; + evidence: db().prepare(`SELECT e.id, r.server, s.server_deleted_at, e.received_at, e.kind, e.payload FROM beta_test_evidence e + JOIN beta_test_runs r ON r.id = e.run_id JOIN beta_servers s ON s.beta_id = r.beta_id AND s.server = r.server + WHERE r.beta_id = ? ORDER BY e.id DESC LIMIT 200`).all(betaId) as BetaTestView['evidence'] }; } diff --git a/src/lib/server/fleet/beta.ts b/src/lib/server/fleet/beta.ts index 7030c90..8d11972 100644 --- a/src/lib/server/fleet/beta.ts +++ b/src/lib/server/fleet/beta.ts @@ -148,6 +148,7 @@ export interface BetaRow { export interface BetaServerRow { beta_id: number; server: string; + server_deleted_at: string | null; had_original: number; // deprecated; per-script data lives in beta_server_scripts prior_version_id: number | null; // deprecated stage: BetaStage; @@ -225,6 +226,13 @@ export function listBetaServers(betaId: number): BetaServerRow[] { .all(betaId) as BetaServerRow[]; } +/** History retains names, so operational callers must also check the participant's identity. */ +export function isActiveBetaServer(betaId: number, server: string): boolean { + return !!db().prepare(`SELECT 1 FROM beta_servers bs JOIN servers s ON s.name = bs.server + WHERE bs.beta_id = ? AND bs.server = ? AND bs.server_deleted_at IS NULL AND s.destroyed_at IS NULL`) + .get(betaId, server); +} + export function listBetaScripts(betaId: number): BetaScriptRow[] { return db() .prepare('SELECT * FROM beta_scripts WHERE beta_id = ? ORDER BY script_id') @@ -456,6 +464,9 @@ export async function deployBetaServers(betaId: number, names: string[]): Promis return exclusive(betaId, () => deployBetaServersInternal(betaId, names)); } async function deployBetaServersInternal(betaId: number, names: string[]): Promise { + for (const name of names) { + if (!isActiveBetaServer(betaId, name)) throw new BetaError(`${name}: original beta server was deleted; start a new beta`, 'unknown-server'); + } const resolved = listBetaScripts(betaId).map(r => ({ script: getScript(r.script_id)!, version: getVersion(r.script_version_id)! })); const serverRows = names.map(name => getServer(name)!); // Split resolved artifacts into non-collapsing (each writes its own file) @@ -484,10 +495,11 @@ async function deployBetaServersInternal(betaId: number, names: string[]): Promi const updateStage = db().prepare( `UPDATE beta_servers SET stage = ?, error = ?, deployed_at = ?, updated_at = ? - WHERE beta_id = ? AND server = ?` + WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL` ); for (const name of names) { + if (!isActiveBetaServer(betaId, name)) continue; db().prepare("UPDATE beta_servers SET stage = 'deploying', deployed_at = ? WHERE beta_id = ? AND server = ?") .run(nowIso(), betaId, name); const serverRow = serverRows.find((r) => r.name === name)!; @@ -503,6 +515,7 @@ async function deployBetaServersInternal(betaId: number, names: string[]): Promi await mkdir(join(dest, dirname(script.target_rel_path)), { recursive: true }); if (existsSync(targetPath)) { const priorBytes = await readFile(targetPath); + if (!isActiveBetaServer(betaId, name)) break; insertSnapshot.run( betaId, name, @@ -514,7 +527,9 @@ async function deployBetaServersInternal(betaId: number, names: string[]): Promi db().prepare('UPDATE beta_server_scripts SET had_original = 1 WHERE beta_id = ? AND server = ? AND script_id = ?').run(betaId, name, script.id); } db().prepare('INSERT OR IGNORE INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, ?)').run(betaId, name, script.target_rel_path); + if (!isActiveBetaServer(betaId, name)) break; await writeFile(targetPath, version.content); + if (!isActiveBetaServer(betaId, name)) break; recordScript(name, script.target_rel_path, version.id, version.sha256, version.size, `beta:${betaId}`); } catch (err) { firstErr = `${script.name}: ${(err as Error).message}`; @@ -541,6 +556,7 @@ async function deployBetaServersInternal(betaId: number, names: string[]): Promi const targetPath = join(dest, targetRel); if (existsSync(targetPath)) { const priorBytes = await readFile(targetPath); + if (!isActiveBetaServer(betaId, name)) continue; insertSnapshot.run( betaId, name, @@ -552,7 +568,9 @@ async function deployBetaServersInternal(betaId: number, names: string[]): Promi } const composedBuf = Buffer.from(composedResult.text, 'utf8'); db().prepare('INSERT OR IGNORE INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, ?)').run(betaId, name, targetRel); + if (!isActiveBetaServer(betaId, name)) continue; await writeFile(targetPath, composedBuf); + if (!isActiveBetaServer(betaId, name)) continue; recordScript( name, targetRel, @@ -568,6 +586,7 @@ async function deployBetaServersInternal(betaId: number, names: string[]): Promi } } const t = nowIso(); + if (!isActiveBetaServer(betaId, name)) continue; if (firstErr) { updateStage.run('failed', `deploy: ${firstErr}`, t, t, betaId, name); emit({ type: 'beta-server-stage', betaId, server: name, stage: 'failed', error: `deploy: ${firstErr}` }); @@ -619,8 +638,13 @@ async function rollbackBetaInternal(betaId: number): Promise { ); for (const bs of servers) { + if (!isActiveBetaServer(betaId, bs.server)) { + warnings.push(`${bs.server}: original server was deleted; retained history was skipped`); + continue; + } if (!bs.deployed_at && bs.stage === 'queued') continue; for (const { script } of bundle) { + if (!isActiveBetaServer(betaId, bs.server)) break; if (isCollapsingKind(script.kind)) continue; // handled once per server below if (storedPlan(betaId) && !db().prepare('SELECT 1 FROM beta_file_changes WHERE beta_id = ? AND server = ? AND rel_path = ?').get(betaId, bs.server, script.target_rel_path)) continue; const dest = destOfKind(bs.server, script.kind); @@ -633,7 +657,9 @@ async function rollbackBetaInternal(betaId: number): Promise { | undefined; if (snap) { await mkdir(join(dest, dirname(script.target_rel_path)), { recursive: true }); + if (!isActiveBetaServer(betaId, bs.server)) break; await writeFile(targetPath, snap.prior_content); + if (!isActiveBetaServer(betaId, bs.server)) break; if (bss.prior_version_id) { recordScript(bs.server, script.target_rel_path, bss.prior_version_id, snap.prior_sha256, snap.prior_size, `rollback:${betaId}`); } else { @@ -646,12 +672,14 @@ async function rollbackBetaInternal(betaId: number): Promise { } else { // No prior file — the beta wrote a fresh file. Drop it. if (existsSync(targetPath)) await rm(targetPath, { force: true }); + if (!isActiveBetaServer(betaId, bs.server)) break; removeScript(bs.server, script.target_rel_path); dropped.add(bs.server); } } // Collapsing config layer: single restore for the shared cfg file. + if (!isActiveBetaServer(betaId, bs.server)) continue; if (hasCollapsing && (!storedPlan(betaId) || db().prepare('SELECT 1 FROM beta_file_changes WHERE beta_id = ? AND server = ? AND rel_path = ?').get(betaId, bs.server, configFileOf(bs.server)))) { const dest = destOfKind(bs.server, 'config'); const targetRel = configFileOf(bs.server); @@ -661,7 +689,9 @@ async function rollbackBetaInternal(betaId: number): Promise { | undefined; if (snap) { await mkdir(dest, { recursive: true }); + if (!isActiveBetaServer(betaId, bs.server)) continue; await writeFile(targetPath, snap.prior_content); + if (!isActiveBetaServer(betaId, bs.server)) continue; recordScript(bs.server, targetRel, null, snap.prior_sha256, snap.prior_size, `rollback:${betaId}`); restored.add(bs.server); } else if (existsSync(targetPath)) { @@ -669,6 +699,7 @@ async function rollbackBetaInternal(betaId: number): Promise { // no pre-existing file. Remove the beta-written cfg to leave // the server clean. await rm(targetPath, { force: true }); + if (!isActiveBetaServer(betaId, bs.server)) continue; removeScript(bs.server, targetRel); dropped.add(bs.server); } @@ -738,7 +769,8 @@ function promotionPlan(betaId: number): { preview: BetaPromotionPreview; configC return { script, version }; }); if (!bundle.length) throw new BetaError('This beta has no files.', 'no-scripts'); - const tested = new Set(listBetaServers(betaId).map(row => row.server)); + const participants = listBetaServers(betaId); + const tested = new Set(participants.filter(row => isActiveBetaServer(betaId, row.server)).map(row => row.server)); const targets = (script: ScriptRow) => [...new Set([...tested, ...(script.scope_type && script.scope_value ? serversMatchingScope(script.scope_type, script.scope_value) : [])])].sort(); const files = bundle.map(({ script, version }) => ({ id: script.id, name: script.name, kind: script.kind, @@ -746,7 +778,8 @@ function promotionPlan(betaId: number): { preview: BetaPromotionPreview; configC release_version: version.release_version, sha256: version.sha256, scope: script.scope_type ? `${script.scope_type}: ${script.scope_value}` : 'Selected test servers only', servers: targets(script) })); const names = [...new Set(files.flatMap(file => file.servers))].sort(); - const blockers: string[] = []; + const blockers: string[] = participants.filter(row => !isActiveBetaServer(betaId, row.server)) + .map(row => `${row.server}: original test server was deleted; start a new beta before promoting.`); const servers = names.map(name => { const server = getServer(name); if (!server || server.destroyed_at) blockers.push(`${name}: server is no longer available.`); @@ -810,7 +843,7 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom version: getVersion(br.script_version_id)! })); - const betaServerSet = new Set(listBetaServers(betaId).map((r) => r.server)); + const betaServerSet = new Set(listBetaServers(betaId).filter(row => isActiveBetaServer(betaId, row.server)).map((r) => r.server)); // Split bundle into non-collapsing (per-file) and collapsing (composed cfg). const nonCollapsing = bundle.filter((b) => !isCollapsingKind(b.script.kind)); @@ -944,12 +977,16 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom // the captured target lists/content so later membership changes never expand it. checkReview(); for (const p of plans) { + checkReview(); const dest = destOfKind(p.server, p.script.kind); try { await mkdir(join(dest, dirname(p.script.target_rel_path)), { recursive: true }); + checkReview(); await writeFile(join(dest, p.script.target_rel_path), p.version.content); + checkReview(); planWritten.set(p, true); } catch (err) { + if (err instanceof BetaError) throw err; planWritten.set(p, false); if (!serverErr.has(p.server)) serverErr.set(p.server, `${p.script.name}: ${(err as Error).message}`); } @@ -957,16 +994,20 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom // Write composed cfg to each target. const cfgWritten = new Map(); for (const cp of cfgPlans) { + checkReview(); try { const dest = destOfKind(cp.server, 'config'); await mkdir(dest, { recursive: true }); + checkReview(); const targetRel = configFileOf(cp.server); const targetPath = join(dest, targetRel); // Beta server already has composed content at targetPath; overwrite // idempotently. Scope-expansion server: write in place. await writeFile(targetPath, cp.composedContent); + checkReview(); cfgWritten.set(cp, true); } catch (err) { + if (err instanceof BetaError) throw err; cfgWritten.set(cp, false); const msg = `configs: ${(err as Error).message}`; if (!serverErr.has(cp.server)) serverErr.set(cp.server, msg); diff --git a/src/lib/server/fleet/collections.ts b/src/lib/server/fleet/collections.ts index c7f65b1..5196178 100644 --- a/src/lib/server/fleet/collections.ts +++ b/src/lib/server/fleet/collections.ts @@ -17,8 +17,8 @@ function coverage(entries: BlueprintEntry[]) { const registered = db().prepare('SELECT name, game, map_slug, map FROM servers WHERE destroyed_at IS NULL').all() as Server[]; const queued = db().prepare(`SELECT j.server AS name, substr(j.server, 1, 2) AS game, j.map_slug, j.expected_map AS map FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id - WHERE d.state IN ('queued', 'running', 'awaiting-health')`).all() as Server[]; - const removing = new Set((db().prepare("SELECT server FROM server_deployments WHERE action = 'destroy' AND state IN ('waiting', 'running')").all() as { server: string }[]).map(row => row.server)); + WHERE d.server_deleted_at IS NULL AND d.state IN ('queued', 'running', 'awaiting-health')`).all() as Server[]; + const removing = new Set((db().prepare("SELECT server FROM server_deployments WHERE server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')").all() as { server: string }[]).map(row => row.server)); // A provisioning server can appear in both tables; count its identity once. const servers = [...new Map([...queued, ...registered].map(server => [server.name, server])).values()]; return entries.map(entry => { @@ -35,7 +35,7 @@ function coverage(entries: BlueprintEntry[]) { return matches.length === 1 && matches[0].slug === entry.recipe.mapSlug; }).map(server => server.name).sort(); const coverage = existingServers.map(name => { - const operation = db().prepare("SELECT id, state FROM server_deployments WHERE server = ? AND action IN ('create','rebuild') AND state IN ('queued', 'running', 'awaiting-health') ORDER BY rowid DESC LIMIT 1").get(name) as { id: string; state: string } | undefined; + const operation = db().prepare("SELECT id, state FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('create','rebuild') AND state IN ('queued', 'running', 'awaiting-health') ORDER BY rowid DESC LIMIT 1").get(name) as { id: string; state: string } | undefined; const server = getServer(name); return { name, health: operation ? operation.state === 'queued' ? 'queued' : 'starting' : server?.container_state && ['stopped', 'exited', 'created', 'dead'].includes(server.container_state) ? 'stopped' @@ -49,8 +49,8 @@ function coverage(entries: BlueprintEntry[]) { function fleetSnapshot() { return housePartyTargets().map(name => ({ name, server: db().prepare('SELECT game, map_slug, map, game_port FROM servers WHERE name = ?').get(name) ?? null, - creation: db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action IN ('create','rebuild') ORDER BY rowid DESC LIMIT 1").get(name) ?? null, - removal: db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) ?? null + creation: db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('create','rebuild') ORDER BY rowid DESC LIMIT 1").get(name) ?? null, + removal: db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) ?? null })); } @@ -149,8 +149,8 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }): const removalJobs = removals.map((server, position) => { const ports = (db().prepare(`SELECT game_port AS port FROM servers WHERE name = ? UNION SELECT j.port FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id - WHERE j.server = ? AND d.state IN ('queued', 'running', 'awaiting-health')`).all(server.name, server.name) as { port: number | null }[]).map(row => row.port); - const queued = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'create' AND state = 'queued'").get(server.name) as { id: string } | undefined; + WHERE j.server = ? AND d.server_deleted_at IS NULL AND d.state IN ('queued', 'running', 'awaiting-health')`).all(server.name, server.name) as { port: number | null }[]).map(row => row.port); + const queued = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'create' AND state = 'queued'").get(server.name) as { id: string } | undefined; let id: string; if (queued && !getServer(server.name)) { // No container or files exist yet: cancel the reservation instead of booting just to remove it. @@ -210,13 +210,13 @@ export function listCollections(): CollectionSummary[] { export function getCollection(id: string): CollectionDetail | null { const collection = db().prepare(summarySql + ' WHERE c.id = ? GROUP BY c.id').get(id) as CollectionSummary | undefined; if (!collection) return null; - const servers = db().prepare(`SELECT d.id, d.server, d.action, d.state, d.stage, d.started_at, d.finished_at, d.error, j.port, j.map_slug, - EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server) AS server_exists + const servers = db().prepare(`SELECT d.id, d.server, d.server_deleted_at, d.action, d.state, d.stage, d.started_at, d.finished_at, d.error, j.port, j.map_slug, + (d.server_deleted_at IS NULL AND EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server AND s.destroyed_at IS NULL)) AS server_exists FROM (${membershipSql}) m JOIN server_deployments d ON d.id = m.deployment_id LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE m.collection_id = ? ORDER BY d.action DESC, m.position`).all(id) as CollectionDetail['servers']; const queue = creationQueue(); const positions = new Map(queue.queued.map((job, index) => [job.id, index + 1])); - const stored = db().prepare('SELECT retained_servers FROM server_launch_collections WHERE id = ?').get(id) as { retained_servers: string }; - return { collection, retainedServers: JSON.parse(stored.retained_servers), queue: { concurrency: fleetSettings().createConcurrency, active: queue.active, queued: queue.queued.length }, + const stored = db().prepare('SELECT retained_servers, retained_server_deletions FROM server_launch_collections WHERE id = ?').get(id) as { retained_servers: string; retained_server_deletions: string }; + return { collection, retainedServers: JSON.parse(stored.retained_servers), retainedServerDeletedAt: JSON.parse(stored.retained_server_deletions), queue: { concurrency: fleetSettings().createConcurrency, active: queue.active, queued: queue.queued.length }, servers: servers.map(server => ({ ...server, queuePosition: positions.get(server.id) ?? null })) }; } diff --git a/src/lib/server/fleet/drift.ts b/src/lib/server/fleet/drift.ts index 2d46cc2..e7aa6ef 100644 --- a/src/lib/server/fleet/drift.ts +++ b/src/lib/server/fleet/drift.ts @@ -54,7 +54,7 @@ function toPosix(p: string): string { function hasActiveBeta(server: string): boolean { return !!db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id - WHERE bs.server = ? AND b.state IN ('running', 'ready-promote') LIMIT 1`).get(server); + WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running', 'ready-promote') LIMIT 1`).get(server); } async function requireServerDirectory(server: string): Promise { diff --git a/src/lib/server/fleet/leaderboard-ingest.ts b/src/lib/server/fleet/leaderboard-ingest.ts index a757299..84b0072 100644 --- a/src/lib/server/fleet/leaderboard-ingest.ts +++ b/src/lib/server/fleet/leaderboard-ingest.ts @@ -51,7 +51,7 @@ function fingerprint(report: GameTelemetry): string { export function recordLeaderboardTelemetry(handle: Database.Database, report: GameTelemetry): void { const identity = mapIdentity(handle, report); const signature = fingerprint(report); - let match = handle.prepare('SELECT * FROM leaderboard_matches WHERE server = ? ORDER BY id DESC LIMIT 1') + let match = handle.prepare('SELECT * FROM leaderboard_matches WHERE server = ? AND server_deleted_at IS NULL ORDER BY id DESC LIMIT 1') .get(report.server) as Match | undefined; if (match?.last_fingerprint === signature && match.map_key === identity.key) return; diff --git a/src/lib/server/fleet/match-history.ts b/src/lib/server/fleet/match-history.ts index fce773a..2c25beb 100644 --- a/src/lib/server/fleet/match-history.ts +++ b/src/lib/server/fleet/match-history.ts @@ -6,7 +6,7 @@ export function serverMatchHistory(server?: string, limit = 25): { matches: Serv const where = `WHERE m.highest_round >= ${MIN_RECORDED_ROUND}${server === undefined ? '' : ' AND m.server = ?'}`; const parameters = server === undefined ? [] : [server]; const total = (handle.prepare(`SELECT COUNT(*) AS total FROM leaderboard_matches m ${where}`).get(...parameters) as { total: number }).total; - const matches = handle.prepare(`SELECT m.id, m.server, m.map_name AS map, m.started_at AS startedAt, m.status, + const matches = handle.prepare(`SELECT m.id, m.server, m.server_deleted_at, m.map_name AS map, m.started_at AS startedAt, m.status, CASE WHEN m.status = 'completed' THEN m.ended_at END AS endedAt, m.start_players AS startPlayers, (SELECT COUNT(*) FROM leaderboard_match_players p WHERE p.match_id = m.id) AS uniquePlayers, diff --git a/src/lib/server/fleet/overview.ts b/src/lib/server/fleet/overview.ts index 3aa18d1..fe4f4f5 100644 --- a/src/lib/server/fleet/overview.ts +++ b/src/lib/server/fleet/overview.ts @@ -14,7 +14,7 @@ export function fleetOperations(now = Date.now()): FleetOperation[] { const campaigns = handle.prepare(`SELECT d.id, d.action, d.started_at, ds.server, ds.restart_state, ds.error, ds.restarted_at, ds.players FROM deployments d JOIN deployment_servers ds ON ds.deployment_id = d.id - WHERE ds.restart_state != 'done' OR ds.checked_at >= ? OR d.finished_at >= ? OR d.started_at >= ?`).all(since, since, since) as { + WHERE ds.server_deleted_at IS NULL AND (ds.restart_state != 'done' OR ds.checked_at >= ? OR d.finished_at >= ? OR d.started_at >= ?)`).all(since, since, since) as { id: string; action: string; started_at: string; server: string; restart_state: string; error: string | null; restarted_at: string | null; players: number | null; }[]; @@ -35,10 +35,10 @@ export function fleetOperations(now = Date.now()): FleetOperation[] { } // Only the newest lifecycle operation can still require action on a server. const lifecycle = handle.prepare(`SELECT d.* FROM server_deployments d - WHERE d.state IN ('queued', 'running', 'waiting', 'awaiting-health') + WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued', 'running', 'waiting', 'awaiting-health') OR (d.state IN ('done', 'warning', 'cancelled') AND d.finished_at >= ?) OR (d.state IN ('failed', 'interrupted', 'warning') AND NOT EXISTS - (SELECT 1 FROM server_deployments n WHERE n.server = d.server AND n.rowid > d.rowid))`) + (SELECT 1 FROM server_deployments n WHERE n.server = d.server AND n.rowid > d.rowid)))`) .all(since) as { id: string; server: string; action: string; state: string; stage: string; error: string | null; started_at: string }[]; for (const row of lifecycle) { @@ -52,7 +52,7 @@ export function fleetOperations(now = Date.now()): FleetOperation[] { detail: row.error ?? row.stage, href: `/provisioning/${encodeURIComponent(row.id)}` }); } const betas = handle.prepare(`SELECT b.id, b.opened_at, bs.server, bs.stage, bs.error - FROM betas b JOIN beta_servers bs ON bs.beta_id = b.id WHERE b.state IN ('running', 'ready-promote')`) + FROM betas b JOIN beta_servers bs ON bs.beta_id = b.id WHERE bs.server_deleted_at IS NULL AND b.state IN ('running', 'ready-promote')`) .all() as { id: number; opened_at: string; server: string; stage: string; error: string | null }[]; const stages: Record = { queued: 'Waiting for an earlier test wave', deploying: 'Deploying beta', 'awaiting-empty': 'Waiting for an empty lobby', restarting: 'Restarting', 'awaiting-health': 'Recovering', @@ -64,7 +64,7 @@ export function fleetOperations(now = Date.now()): FleetOperation[] { state: row.stage === 'failed' ? 'attention' : 'active', label: stages[row.stage] ?? row.stage, detail: row.error ?? recovering.get(row.server)?.error ?? `Beta ${row.id}`, href: `/betas/${row.id}` }); } - const maintenance = handle.prepare(`SELECT m.* FROM maintenance_jobs m WHERE m.state = 'failed' + const maintenance = handle.prepare(`SELECT m.* FROM maintenance_jobs m WHERE m.server_deleted_at IS NULL AND m.state = 'failed' AND NOT EXISTS (SELECT 1 FROM maintenance_jobs n WHERE n.server = m.server AND n.kind = m.kind AND n.id > m.id)`) .all() as { id: number; server: string; kind: string; detail: string | null; requested_at: string }[]; for (const job of maintenance) { diff --git a/src/lib/server/fleet/player-profile.ts b/src/lib/server/fleet/player-profile.ts index 7db5663..bff8d5d 100644 --- a/src/lib/server/fleet/player-profile.ts +++ b/src/lib/server/fleet/player-profile.ts @@ -25,7 +25,7 @@ export function getPlayerProfile(game: string, guid: string, requestedOffset = 0 MAX(mp.last_seen_at) AS lastSeen, ${totals} ${scope} GROUP BY m.map_key ORDER BY highestRound DESC, kills DESC, name COLLATE NOCASE, key`) .all(game, guid) as PlayerMapRecord[]; - const matches = handle.prepare(`SELECT m.id, m.server, m.map_name AS mapName, + const matches = handle.prepare(`SELECT m.id, m.server, m.server_deleted_at, m.map_name AS mapName, m.started_at AS startedAt, mp.last_seen_at AS lastSeen, m.status, mp.completed, mp.highest_round AS highestRound, mp.rounds_survived AS roundsSurvived, mp.kills, mp.headshots, mp.revives, mp.downs, mp.best_score AS bestScore diff --git a/src/lib/server/fleet/provisioning.ts b/src/lib/server/fleet/provisioning.ts index cbbac8e..3f8eff2 100644 --- a/src/lib/server/fleet/provisioning.ts +++ b/src/lib/server/fleet/provisioning.ts @@ -3,7 +3,7 @@ import { db } from '../db/index.js'; /** Only the latest lifecycle operation can describe the current server. */ export function isProvisioning(server: string): boolean { const operation = db().prepare(`SELECT action, state FROM server_deployments - WHERE server = ? ORDER BY rowid DESC LIMIT 1`).get(server) as + WHERE server = ? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1`).get(server) as { action: string; state: string } | undefined; return !!operation && ['create', 'rebuild', 'start'].includes(operation.action) && ['queued', 'running', 'awaiting-health'].includes(operation.state); } diff --git a/src/lib/server/fleet/rollback.ts b/src/lib/server/fleet/rollback.ts index 527bc3a..d832aa6 100644 --- a/src/lib/server/fleet/rollback.ts +++ b/src/lib/server/fleet/rollback.ts @@ -100,8 +100,10 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise(); + const active = (server: string) => !!db().prepare(`SELECT 1 FROM deployment_servers ds JOIN servers s ON s.name = ds.server + WHERE ds.deployment_id = ? AND ds.server = ? AND ds.server_deleted_at IS NULL AND s.destroyed_at IS NULL`).get(promoteId, server); for (const [server, files] of perServer) { + if (!active(server)) continue; try { for (const f of files) { + if (!active(server)) break; const kind = kindForRelPath(f.rel_path); const dest = destOfKind(server, kind); const abs = join(dest, f.rel_path); await mkdir(join(dest, dirname(f.rel_path)), { recursive: true }); + if (!active(server)) break; const owningScript = scriptByTarget.get(f.rel_path); if (f.verb === 'added') { if (existsSync(abs)) await rm(abs, { force: true }); + if (!active(server)) break; removeScript(server, f.rel_path); } else if (f.verb === 'updated' && f.content_before) { await writeFile(abs, f.content_before); + if (!active(server)) break; const priorVer = owningScript?.prior_version_id ? getVersion(owningScript.prior_version_id) : null; @@ -175,10 +184,11 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise { JOIN betas b ON b.id = bsv.beta_id JOIN beta_scripts bs ON bs.beta_id = b.id JOIN scripts s ON s.id = bs.script_id - WHERE bsv.server = ? AND b.state IN ('running','ready-promote')` + WHERE bsv.server = ? AND bsv.server_deleted_at IS NULL AND b.state IN ('running','ready-promote')` ) .all(server) as Array<{ target: string; beta_id: number }>; const map = new Map(); @@ -226,7 +226,7 @@ export function ownerOf(server: string, target: string): number | null { JOIN betas b ON b.id = bsv.beta_id JOIN beta_scripts bs ON bs.beta_id = b.id JOIN scripts s ON s.id = bs.script_id - WHERE bsv.server = ? AND s.target_rel_path = ? AND b.state IN ('running','ready-promote') + WHERE bsv.server = ? AND bsv.server_deleted_at IS NULL AND s.target_rel_path = ? AND b.state IN ('running','ready-promote') LIMIT 1` ) .get(server, target) as { beta_id: number } | undefined; diff --git a/src/lib/server/fleet/status-evidence.ts b/src/lib/server/fleet/status-evidence.ts index dca6c70..545ef70 100644 --- a/src/lib/server/fleet/status-evidence.ts +++ b/src/lib/server/fleet/status-evidence.ts @@ -8,7 +8,7 @@ export interface TelemetryEvidence { players: number; map: string; receivedAt: s /** Reports from before the latest lifecycle submission describe the old process. */ export function latestGameSubmission(server: string): string | null { return (db().prepare(`SELECT MAX(at) AS at FROM ( - SELECT submitted_at AS at FROM server_deployments WHERE server=? AND action IN ('create','start','restart','rebuild') + SELECT submitted_at AS at FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild') UNION ALL SELECT requested_at AS at FROM restart_rollout WHERE server=?)`).get(server, server) as { at: string | null }).at; } diff --git a/src/lib/server/fleet/telemetry.ts b/src/lib/server/fleet/telemetry.ts index d364a02..63dac5a 100644 --- a/src/lib/server/fleet/telemetry.ts +++ b/src/lib/server/fleet/telemetry.ts @@ -37,7 +37,7 @@ export function saveTelemetry(report: GameTelemetry): GameTelemetry { } else if (report.event === 'game_end') { handle.prepare('UPDATE servers SET game_ended_at = ? WHERE name = ?').run(report.receivedAt, report.server); // Legacy betas also take match completion directly from the script. - if (report.round >= MIN_RECORDED_ROUND) handle.prepare(`UPDATE beta_servers SET match_seen_at = ? WHERE server = ? AND stage = 'awaiting-match' + if (report.round >= MIN_RECORDED_ROUND) handle.prepare(`UPDATE beta_servers SET match_seen_at = ? WHERE server = ? AND server_deleted_at IS NULL AND stage = 'awaiting-match' AND healthy_at <= ? AND beta_id IN (SELECT id FROM betas WHERE state = 'running')`) .run(report.receivedAt, report.server, report.receivedAt); } diff --git a/src/lib/server/maintenance/recovery.ts b/src/lib/server/maintenance/recovery.ts index 5ca503c..6e0d850 100644 --- a/src/lib/server/maintenance/recovery.ts +++ b/src/lib/server/maintenance/recovery.ts @@ -26,7 +26,7 @@ export function setWatchdogEnabled(enabled: boolean) { } function incident(id: string) { return db().prepare('SELECT * FROM watchdog_incidents WHERE id = ?').get(id) as WatchdogIncident | undefined; } -function active(server: string, kind: 'health' | 'drift') { return db().prepare("SELECT * FROM watchdog_incidents WHERE server = ? AND kind = ? AND state != 'resolved'").get(server, kind) as WatchdogIncident | undefined; } +function active(server: string, kind: 'health' | 'drift') { return db().prepare("SELECT * FROM watchdog_incidents WHERE server = ? AND server_deleted_at IS NULL AND kind = ? AND state != 'resolved'").get(server, kind) as WatchdogIncident | undefined; } function event(id: string, message: string) { db().prepare('INSERT INTO watchdog_events(incident_id,created_at,message) VALUES (?,?,?)').run(id, iso(), message); } function transition(id: string, state: WatchdogState, summary: string) { const previous = incident(id); @@ -74,11 +74,11 @@ export function watchdogDetail(id: string): WatchdogDetail | null { return row ? { incident: row, actions: actions(id), events: db().prepare('SELECT id,created_at,message FROM watchdog_events WHERE incident_id=? ORDER BY id').all(id) as WatchdogDetail['events'] } : null; } function hasPending(server: string) { - return !!db().prepare("SELECT 1 FROM server_deployments WHERE server=? AND state IN ('queued','waiting','running','awaiting-health') LIMIT 1").get(server); + return !!db().prepare("SELECT 1 FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND state IN ('queued','waiting','running','awaiting-health') LIMIT 1").get(server); } function protectedWork(server: string): boolean { - return !!db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id=bs.beta_id WHERE bs.server=? AND b.state IN ('running','ready-promote') LIMIT 1`).get(server) - || !!db().prepare(`SELECT 1 FROM deployment_servers ds WHERE ds.server=? AND ds.restart_state IN ('pending','held','failed','manual') + return !!db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id=bs.beta_id WHERE bs.server=? AND bs.server_deleted_at IS NULL AND b.state IN ('running','ready-promote') LIMIT 1`).get(server) + || !!db().prepare(`SELECT 1 FROM deployment_servers ds WHERE ds.server=? AND ds.server_deleted_at IS NULL AND ds.restart_state IN ('pending','held','failed','manual') AND NOT EXISTS (SELECT 1 FROM watchdog_actions a WHERE a.campaign_id=ds.deployment_id) LIMIT 1`).get(server); } function addAction(row: WatchdogIncident, operationId: string, action: 'restart' | 'rebuild', campaignId: string | null = null) { @@ -91,6 +91,7 @@ export function updateWatchdogIncident(id: string, action: 'retry' | 'dismiss') return db().transaction(() => { const row = incident(id); if (!row) return { ok: false as const, status: 404, error: 'Watchdog incident not found.' }; + if (row.server_deleted_at) return { ok: false as const, status: 409, error: 'This server was deleted; its recovery history cannot be changed.' }; if (actions(id).some(a => pendingStates.includes(a.operation_state ?? ''))) return { ok: false as const, status: 409, error: 'Provisioning is still active. Complete or cancel that operation first.' }; if (action === 'dismiss') { if (!['open','attention'].includes(row.state)) return { ok: false as const, status: 409, error: 'Only detected incidents or incidents needing attention can be dismissed.' }; @@ -106,7 +107,7 @@ export function updateWatchdogIncident(id: string, action: 'retry' | 'dismiss') } async function reconcileActions(): Promise { - const rows = db().prepare("SELECT * FROM watchdog_incidents WHERE state IN ('recovering','attention')").all() as WatchdogIncident[]; + const rows = db().prepare("SELECT * FROM watchdog_incidents WHERE server_deleted_at IS NULL AND state IN ('recovering','attention')").all() as WatchdogIncident[]; for (const row of rows) { const action = actions(row.id).filter(a => a.cycle === row.cycle).at(-1); if (!action) continue; @@ -229,7 +230,7 @@ async function run(forceScan: boolean): Promise { } await Promise.all(Array.from({ length: Math.min(4, servers.length) }, pump)); // Preserve incidents for temporarily absent rebuild instances, but close retired registrations. - const abandoned = db().prepare("SELECT * FROM watchdog_incidents WHERE state != 'resolved'").all() as WatchdogIncident[]; + const abandoned = db().prepare("SELECT * FROM watchdog_incidents WHERE server_deleted_at IS NULL AND state != 'resolved'").all() as WatchdogIncident[]; for (const row of abandoned) { if (getServer(row.server) || hasPending(row.server)) continue; const last = db().prepare('SELECT action,state FROM server_deployments WHERE server=? ORDER BY rowid DESC LIMIT 1').get(row.server) as { action: string; state: string } | undefined; diff --git a/src/lib/server/maintenance/runtime.ts b/src/lib/server/maintenance/runtime.ts index 1b0e3a4..0a3aae9 100644 --- a/src/lib/server/maintenance/runtime.ts +++ b/src/lib/server/maintenance/runtime.ts @@ -39,7 +39,7 @@ export function queueMaintenance(servers: string[], kind: MaintenanceJob['kind'] if (getServer(server)?.desired_state === 'stopped') continue; added += db().prepare(`INSERT OR IGNORE INTO maintenance_jobs (server, target_revision, requested_at, kind, origin) VALUES (?, ?, ?, ?, ?)`).run(server, target, nowIso(), kind, origin).changes; - const job = db().prepare("SELECT id FROM maintenance_jobs WHERE server = ? AND state IN ('queued','restarting')").get(server) as { id: number }; + const job = db().prepare("SELECT id FROM maintenance_jobs WHERE server = ? AND server_deleted_at IS NULL AND state IN ('queued','restarting')").get(server) as { id: number }; requestRestart(server, { kind: 'maintenance', id: String(job.id) }); } return added; @@ -55,13 +55,14 @@ export function queueUpdates(servers?: string[], automatic = false): number { if (servers && (!servers.length || servers.some(server => !versions.some(v => v.server === server)))) throw new Error('Choose existing fleet servers.'); const candidates = versions.filter(v => v.update_status === 'outdated' && (!servers || servers.includes(v.server))); const names = candidates.filter(v => !automatic || !db().prepare(`SELECT 1 FROM maintenance_jobs - WHERE server = ? AND target_revision = ? AND kind = 'update'`).get(v.server, freshRelease())).map(v => v.server); + WHERE server = ? AND server_deleted_at IS NULL AND target_revision = ? AND kind = 'update'`).get(v.server, freshRelease())).map(v => v.server); return names.length ? queueMaintenance(names, 'update', automatic ? 'automatic' : 'manual') : 0; } export function retryMaintenance(id: number): number { const job = db().prepare("SELECT * FROM maintenance_jobs WHERE id = ? AND state = 'failed'").get(id) as MaintenanceJob | undefined; if (!job) throw new Error('Only failed maintenance can be retried.'); + if (job.server_deleted_at) throw new Error('This server was deleted; its maintenance history cannot be retried.'); if (!maintenanceAllowed()) throw new Error('Maintenance is paused.'); if (getServer(job.server)?.desired_state === 'stopped') throw new Error('This server is intentionally stopped.'); // Ambiguous lifecycle failures retain their slot and require diagnostics, not another request. @@ -82,11 +83,11 @@ export function cancelMaintenance(id: number): boolean { } function finish(job: MaintenanceJob, state: 'done' | 'failed' | 'cancelled', detail: string) { - db().prepare('UPDATE maintenance_jobs SET state = ?, detail = ?, finished_at = ? WHERE id = ?') + db().prepare('UPDATE maintenance_jobs SET state = ?, detail = ?, finished_at = ? WHERE id = ? AND server_deleted_at IS NULL') .run(state, detail, nowIso(), job.id); } function hold(job: MaintenanceJob, detail: string) { - db().prepare('UPDATE maintenance_jobs SET detail = ? WHERE id = ?').run(detail, job.id); + db().prepare('UPDATE maintenance_jobs SET detail = ? WHERE id = ? AND server_deleted_at IS NULL').run(detail, job.id); } let ticking: Promise | null = null; @@ -108,7 +109,7 @@ async function runTick(): Promise { writeSetting('plutonium_schedule', { ...plan, nextAt: nextMaintenanceAt(plan.time, new Date(), plan.timeZone) }); } })(); - const jobs = db().prepare(`SELECT * FROM maintenance_jobs WHERE state IN ('queued','restarting') ORDER BY id`).all() as MaintenanceJob[]; + const jobs = db().prepare(`SELECT * FROM maintenance_jobs WHERE server_deleted_at IS NULL AND state IN ('queued','restarting') ORDER BY id`).all() as MaintenanceJob[]; for (const job of jobs) { try { await processJob(job); } catch { hold(job, 'Maintenance check failed; waiting for the next check.'); } @@ -119,7 +120,7 @@ async function runTick(): Promise { } async function processJob(job: MaintenanceJob): Promise { const server = getServer(job.server); - if (!server || server.destroyed_at) return finish(job, 'cancelled', 'Server removed.'); + if (job.server_deleted_at || !server || server.destroyed_at) return finish(job, 'cancelled', 'Server removed.'); const operation = requestRestart(job.server, { kind: 'maintenance', id: String(job.id) }); if (['failed','interrupted','cancelled'].includes(operation.state)) { return finish(job, operation.state === 'cancelled' ? 'cancelled' : 'failed', operation.error ?? operation.stage); diff --git a/src/lib/server/pipeline/beta-plan-runner.ts b/src/lib/server/pipeline/beta-plan-runner.ts index 84c0a94..26d5df9 100644 --- a/src/lib/server/pipeline/beta-plan-runner.ts +++ b/src/lib/server/pipeline/beta-plan-runner.ts @@ -1,5 +1,5 @@ import { db } from '../db/index.js'; -import { betaOperationInFlight, deployBetaServers, getBeta, listBetaServers, rollbackBeta } from '../fleet/beta.js'; +import { betaOperationInFlight, deployBetaServers, getBeta, isActiveBetaServer, listBetaServers, rollbackBeta } from '../fleet/beta.js'; import { activity, currentRun, observeRun, prepareRun, readMetrics, saveMetrics, setOutcome, storedPlan, type StoredPlan } from '../fleet/beta-tests.js'; import { coveragePassed, evaluateRequirements } from '../fleet/beta-evaluation.js'; import { parseBetaPlan, type BetaPlan } from '../../beta-plan.js'; @@ -7,9 +7,9 @@ import { emit, type BetaStage } from './bus.js'; import { requestRestart, releaseRestartRequest } from '../provisioning/index.js'; function stage(id: number, server: string, next: BetaStage, error: string | null = null): void { - db().prepare('UPDATE beta_servers SET stage = ?, error = ?, updated_at = ? WHERE beta_id = ? AND server = ?') + const result = db().prepare('UPDATE beta_servers SET stage = ?, error = ?, updated_at = ? WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL') .run(next, error, new Date().toISOString(), id, server); - emit({ type: 'beta-server-stage', betaId: id, server, stage: next, error }); + if (result.changes) emit({ type: 'beta-server-stage', betaId: id, server, stage: next, error }); } export function betaState(id: number, ready: boolean): void { @@ -25,14 +25,16 @@ export function betaState(id: number, ready: boolean): void { } export async function rollbackTestPlan(id: number, plan: BetaPlan): Promise { - const servers = listBetaServers(id).filter(s => s.deployed_at); + const servers = listBetaServers(id).filter(s => s.deployed_at && isActiveBetaServer(id, s.server)); const result = await rollbackBeta(id); if (plan.rollbackRestart === 'when-empty' && servers.length) { const deploymentId = `dep-beta-rollback-${id}-${Date.now()}`; db().transaction(() => { db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by, beta_id) VALUES (?, 'rollback', ?, 'beta-test-plan', ?)") .run(deploymentId, new Date().toISOString(), id); - for (const server of servers) db().prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES (?, ?, 1, 'pending')").run(deploymentId, server.server); + for (const server of servers) { + if (isActiveBetaServer(id, server.server)) db().prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES (?, ?, 1, 'pending')").run(deploymentId, server.server); + } })(); activity(id, 'Restored files; rolling restarts will wait for empty servers'); } else activity(id, 'Restored files; manual restarts are required to load them'); @@ -73,12 +75,21 @@ async function advancePlan(stored: StoredPlan): Promise { const plan = parseBetaPlan(JSON.parse(stored.plan)); const beta = getBeta(id)!; const servers = listBetaServers(id); + // Preserve the original coverage and evidence. A replacement with the same + // name cannot complete a test that began on a different server. + const missing = servers.filter(server => !isActiveBetaServer(id, server.server)); + if (missing.length) { + betaState(id, false); + setOutcome(id, 'paused', `Test server deleted: ${missing.map(server => server.server).join(', ')}. Roll back this beta and start a new test.`); + return; + } if (stored.outcome !== 'passed' && Date.now() - Date.parse(beta.opened_at) >= plan.timeoutHours * 3_600_000) { betaState(id, false); setOutcome(id, 'inconclusive', 'Deadline reached without enough qualifying evidence'); return; } for (const server of servers) { + if (!isActiveBetaServer(id, server.server)) return; const freshPlan = storedPlan(id)!; if (!['active', 'passed'].includes(freshPlan.outcome)) return; if (server.stage === 'failed') { await fail(id, plan, `${server.server}: ${server.error ?? 'server failed'}`); return; } @@ -108,6 +119,7 @@ async function advancePlan(stored: StoredPlan): Promise { } } } + if (!isActiveBetaServer(id, server.server)) return; if (server.stage === 'awaiting-empty' || server.stage === 'awaiting-health') { const operation = requestRestart(server.server, { kind: 'beta', id: String(id) }); if (['failed','interrupted','cancelled'].includes(operation.state)) { @@ -129,6 +141,7 @@ async function advancePlan(stored: StoredPlan): Promise { const run = currentRun(id, server.server); if (!run?.observing_at) continue; await observeRun(run, plan); + if (!isActiveBetaServer(id, server.server)) return; if (!['active', 'passed'].includes(storedPlan(id)!.outcome)) { betaState(id, false); return; } const metrics = readMetrics(currentRun(id, server.server)!); const progress = evaluateRequirements(plan, metrics, run.phase); @@ -143,6 +156,7 @@ async function advancePlan(stored: StoredPlan): Promise { } else stage(id, server.server, progress.every(p => p.passed) ? 'ready' : 'observing'); } const updated = listBetaServers(id); + if (updated.some(server => !isActiveBetaServer(id, server.server))) return; const queued = updated.filter(s => s.stage === 'queued'); if (queued.length && updated.filter(s => s.stage !== 'queued').every(s => s.stage === 'ready')) { const wave = queued.slice(0, plan.waveSize || queued.length).map(s => s.server); diff --git a/src/lib/server/pipeline/beta-runner.ts b/src/lib/server/pipeline/beta-runner.ts index 1cb90a2..accac78 100644 --- a/src/lib/server/pipeline/beta-runner.ts +++ b/src/lib/server/pipeline/beta-runner.ts @@ -25,7 +25,7 @@ const setStage = () => restarted_at = CASE WHEN ? = 'awaiting-health' OR ? = 'ready' THEN COALESCE(restarted_at, ?) ELSE restarted_at END, healthy_at = CASE WHEN ? = 'awaiting-match' OR (? = 'ready' AND healthy_at IS NULL) THEN COALESCE(healthy_at, ?) ELSE healthy_at END, match_seen_at = CASE WHEN ? = 'ready' AND match_seen_at IS NULL THEN ? ELSE match_seen_at END - WHERE beta_id = ? AND server = ?` + WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL` ); function transition( @@ -35,7 +35,7 @@ function transition( error: string | null = null ): void { const t = nowIso(); - setStage().run( + const result = setStage().run( stage, error, t, @@ -50,17 +50,17 @@ function transition( betaId, server ); - emit({ type: 'beta-server-stage', betaId, server, stage, error }); + if (result.changes) emit({ type: 'beta-server-stage', betaId, server, stage, error }); } // Bump beta.state to ready-promote when every one of its beta_servers is ready. // A single failed row is enough to keep the beta in 'running' (operator retries). function reconcileBetaState(betaId: number): void { const rows = db() - .prepare(`SELECT stage FROM beta_servers WHERE beta_id = ?`) - .all(betaId) as Array<{ stage: BetaStage }>; + .prepare(`SELECT stage, server_deleted_at FROM beta_servers WHERE beta_id = ?`) + .all(betaId) as Array<{ stage: BetaStage; server_deleted_at: string | null }>; if (rows.length === 0) return; - if (rows.every((r) => r.stage === 'ready')) { + if (rows.every((r) => r.stage === 'ready' && !r.server_deleted_at)) { const upd = db() .prepare(`UPDATE betas SET state = 'ready-promote' WHERE id = ? AND state = 'running'`) .run(betaId); @@ -98,6 +98,7 @@ async function runBetaTick(): Promise<{ processed: number }> { FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id WHERE b.state = 'running' + AND bs.server_deleted_at IS NULL AND NOT EXISTS (SELECT 1 FROM beta_test_plans p WHERE p.beta_id = b.id) AND bs.stage IN ('awaiting-empty','restarting','awaiting-health','awaiting-match') ORDER BY b.id, bs.server` diff --git a/src/lib/server/pipeline/sweep.ts b/src/lib/server/pipeline/sweep.ts index 1c92ff6..5c7a330 100644 --- a/src/lib/server/pipeline/sweep.ts +++ b/src/lib/server/pipeline/sweep.ts @@ -19,7 +19,7 @@ export function sweep(): Promise { } async function runSweep(): Promise { const pending = db().prepare(`SELECT ds.* FROM deployment_servers ds JOIN deployments d ON d.id = ds.deployment_id - WHERE ds.restart_state IN ('pending','failed') ORDER BY d.started_at, d.id, ds.server`).all() as PendingRow[]; + WHERE ds.server_deleted_at IS NULL AND ds.restart_state IN ('pending','failed') ORDER BY d.started_at, d.id, ds.server`).all() as PendingRow[]; const result: SweepResult = { checked: 0, restarted: 0, healthy: 0, held: 0, failed: 0 }; for (const row of pending) { if (!row.files_changed) continue; // A failed file write is not a restart request. @@ -40,10 +40,10 @@ async function runSweep(): Promise { emit({ type: 'server-state', deploymentId: row.deployment_id, server: row.server, state, players: operation.players, error }); } db().prepare(`UPDATE deployments SET finished_at = ? WHERE finished_at IS NULL AND NOT EXISTS ( - SELECT 1 FROM deployment_servers WHERE deployment_id = deployments.id AND restart_state IN ('pending','failed','held'))`).run(nowIso()); + SELECT 1 FROM deployment_servers WHERE deployment_id = deployments.id AND server_deleted_at IS NULL AND restart_state IN ('pending','failed','held'))`).run(nowIso()); db().prepare(`DELETE FROM deployments WHERE started_at < ? AND NOT EXISTS (SELECT 1 FROM watchdog_actions WHERE campaign_id = deployments.id) AND NOT EXISTS ( - SELECT 1 FROM deployment_servers WHERE deployment_id = deployments.id AND restart_state IN ('pending','failed','held'))`) + SELECT 1 FROM deployment_servers WHERE deployment_id = deployments.id AND server_deleted_at IS NULL AND restart_state IN ('pending','failed','held'))`) .run(new Date(Date.now() - fleetSettings().deployKeepDays * 86_400_000).toISOString()); emit({ type: 'sweep-tick', frozen: false }); return result; diff --git a/src/lib/server/provisioning/history.ts b/src/lib/server/provisioning/history.ts index 3c7f4d8..731f4d8 100644 --- a/src/lib/server/provisioning/history.ts +++ b/src/lib/server/provisioning/history.ts @@ -24,7 +24,7 @@ export function recordDeploymentEvent(id: string, message: string): void { } export function getDeployment(id: string) { - const deployment = db().prepare(`SELECT d.*, j.expected_map, EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server) AS server_exists + const deployment = db().prepare(`SELECT d.*, j.expected_map, (d.server_deleted_at IS NULL AND EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server AND s.destroyed_at IS NULL)) AS server_exists FROM server_deployments d LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE d.id = ?`).get(id); if (!deployment) return null; const events = db().prepare('SELECT id, created_at, message FROM server_deployment_events WHERE deployment_id = ? ORDER BY id').all(id); @@ -42,14 +42,14 @@ export function getDeployment(id: string) { export const REMOVING_SERVER = 'Removing server files and registry'; export function canCancelRemoval(id: string): boolean { - return !!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND action = 'destroy' AND state = 'waiting' AND stage != ?").get(id, REMOVING_SERVER); + return !!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state = 'waiting' AND stage != ?").get(id, REMOVING_SERVER); } /** Competes atomically with the worker's claim, before any external removal. */ export function cancelRemoval(id: string): boolean { return db().transaction(() => { const result = db().prepare(`UPDATE server_deployments SET state = 'cancelled', stage = 'Removal cancelled', finished_at = ?, error = NULL - WHERE id = ? AND action = 'destroy' AND state = 'waiting' AND stage != ?`).run(new Date().toISOString(), id, REMOVING_SERVER); + WHERE id = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state = 'waiting' AND stage != ?`).run(new Date().toISOString(), id, REMOVING_SERVER); if (!result.changes) return false; recordDeploymentEvent(id, 'Queued removal cancelled. The server and its files were kept.'); return true; @@ -57,12 +57,18 @@ export function cancelRemoval(id: string): boolean { } export function claimRemoval(id: string): boolean { - return db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND action = 'destroy' AND state = 'waiting'").run(id).changes > 0; + return db().prepare(`UPDATE server_deployments SET state = 'running' + WHERE id = ? AND action = 'destroy' AND state = 'waiting' + AND (server_deleted_at IS NULL OR (stage = ? + AND NOT EXISTS (SELECT 1 FROM servers s WHERE s.name = server_deployments.server) + AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = server_deployments.server + AND newer.rowid > server_deployments.rowid)))`).run(id, REMOVING_SERVER).changes > 0; } export function canResumeCleanup(id: string): boolean { return !!db().prepare(`SELECT d.id FROM server_deployments d - WHERE d.id = ? AND d.action = 'destroy' AND d.state IN ('failed', 'interrupted') AND d.stage = ? + WHERE d.id = ? AND (d.server_deleted_at IS NULL OR NOT EXISTS (SELECT 1 FROM servers s WHERE s.name = d.server)) + AND d.action = 'destroy' AND d.state IN ('failed', 'interrupted') AND d.stage = ? AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`) .get(id, REMOVING_SERVER); } @@ -80,7 +86,7 @@ export function canResumeVerification(id: string): boolean { return !!db().prepare(`SELECT d.id FROM server_deployments d JOIN server_creation_jobs j ON j.deployment_id = d.id JOIN servers s ON s.name = j.server AND s.game_port = j.port - WHERE d.id = ? AND (d.action = 'create' OR (d.action = 'rebuild' AND EXISTS ( + WHERE d.id = ? AND d.server_deleted_at IS NULL AND (d.action = 'create' OR (d.action = 'rebuild' AND EXISTS ( SELECT 1 FROM server_rebuild_jobs r WHERE r.deployment_id = d.id AND r.removed_at IS NOT NULL))) AND d.state IN ('failed', 'interrupted') AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`) .get(id); @@ -88,7 +94,7 @@ export function canResumeVerification(id: string): boolean { export function canResumeRebuild(id: string): boolean { return !!db().prepare(`SELECT d.id FROM server_deployments d JOIN server_rebuild_jobs r ON r.deployment_id = d.id - WHERE d.id = ? AND d.state IN ('failed','interrupted') AND d.stage = ? AND r.removed_at IS NULL + WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.state IN ('failed','interrupted') AND d.stage = ? AND r.removed_at IS NULL AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`) .get(id, REMOVING_SERVER); } @@ -120,8 +126,8 @@ export function finishDeployment(id: string, result: { ok: boolean; message?: st } export function listDeployments() { - return db().prepare(`SELECT d.*, EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server) AS server_exists, - (d.action = 'destroy' AND d.state = 'waiting' AND d.stage != ?) AS canCancelRemoval + return db().prepare(`SELECT d.*, (d.server_deleted_at IS NULL AND EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server AND s.destroyed_at IS NULL)) AS server_exists, + (d.server_deleted_at IS NULL AND d.action = 'destroy' AND d.state = 'waiting' AND d.stage != ?) AS canCancelRemoval FROM server_deployments d ORDER BY started_at DESC, rowid DESC`).all(REMOVING_SERVER); } diff --git a/src/lib/server/provisioning/instances.ts b/src/lib/server/provisioning/instances.ts index 2c6683a..d0b1908 100644 --- a/src/lib/server/provisioning/instances.ts +++ b/src/lib/server/provisioning/instances.ts @@ -63,7 +63,7 @@ export function nextLobbyNumber(baseName: string, additionalNames: string[] = [] const rows = db() .prepare(`SELECT name FROM servers WHERE name LIKE ? UNION SELECT j.server AS name FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id - WHERE d.state IN ('queued', 'running', 'awaiting-health') OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry')))`) + WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued', 'running', 'awaiting-health') OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry'))))`) .all(`${baseName}_%`) as Array<{ name: string }>; const taken = new Set(); const prefix = `${baseName}_`; @@ -87,7 +87,7 @@ export function nextAvailablePort(additionalPorts: number[] = [], releasingServe const rows = db() .prepare(`SELECT name, game_port FROM servers WHERE game_port IS NOT NULL UNION SELECT j.server AS name, j.port AS game_port FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id - WHERE d.state IN ('queued', 'running', 'awaiting-health') OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry')))`) + WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued', 'running', 'awaiting-health') OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry'))))`) .all() as Array<{ name: string; game_port: number }>; const taken = new Set([...rows.filter(row => !releasingServers.includes(row.name)).map((r) => r.game_port), ...additionalPorts]); for (let p = config.portRange.min; p <= config.portRange.max; p++) { @@ -257,7 +257,7 @@ export function queueRebuildServer(name: string, triggeredBy = 'ui'): CreateServ /** Caller has just verified emptiness and excluded conflicting work. */ export async function rebuildServer(id: string): Promise { const job = db().prepare(`SELECT j.*, d.stage FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id - WHERE d.id = ? AND d.action = 'rebuild' AND d.state = 'waiting'`).get(id) as (CreationJob & { stage: string }) | undefined; + WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.action = 'rebuild' AND d.state = 'waiting'`).get(id) as (CreationJob & { stage: string }) | undefined; if (!job) return; if (creating || destroying.size || creationQueue().active.length >= fleetSettings().createConcurrency) { // The cleanup checkpoint must survive capacity waits and app restarts. @@ -487,17 +487,17 @@ const destroying = new Set(); // Persist the request before returning so navigation and client disconnects cannot lose it. function hasActiveCreation(name: string): boolean { - return !!db().prepare("SELECT 1 FROM server_deployments WHERE server = ? AND action = 'create' AND state IN ('queued', 'running', 'awaiting-health')").get(name); + return !!db().prepare("SELECT 1 FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'create' AND state IN ('queued', 'running', 'awaiting-health')").get(name); } export function destroyServer(name: string, triggeredBy = 'ui'): DestroyServerResult & { deploymentId?: string } { - if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND action IN ('start','restart','stop','rebuild') + if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('start','restart','stop','rebuild') AND state IN ('waiting','queued','running')`).get(name)) { return { ok: false, kind: 'conflict', message: 'Wait for the current provisioning operation before destroying this server.', warnings: [] }; } - const existing = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) as { id: string } | undefined; + const existing = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) as { id: string } | undefined; if (existing) return { ok: true, warnings: [], deploymentId: existing.id }; - const previous = db().prepare("SELECT id FROM server_deployments WHERE server = ? ORDER BY rowid DESC LIMIT 1").get(name) as { id: string } | undefined; + const previous = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1").get(name) as { id: string } | undefined; if (previous && resumeCleanup(previous.id)) return { ok: true, warnings: [], deploymentId: previous.id }; if (!getServer(name) && !hasActiveCreation(name)) return { ok: false, kind: 'not-found', message: 'Server not found', warnings: [] }; const operation = db().transaction(() => { @@ -517,13 +517,18 @@ export function processDestructions(): Promise { } async function runDestructions(): Promise { - const queued = db().prepare("SELECT id, server, stage FROM server_deployments WHERE action = 'destroy' AND state = 'waiting' ORDER BY started_at").all() as { id: string; server: string; stage: string }[]; + const queued = db().prepare(`SELECT id, server, stage, server_deleted_at FROM server_deployments WHERE action = 'destroy' AND state = 'waiting' + AND (server_deleted_at IS NULL OR stage = ?) ORDER BY started_at`).all(REMOVING_SERVER) as { id: string; server: string; stage: string; server_deleted_at: string | null }[]; for (const operation of queued) { if (creating || destroying.has(operation.server)) continue; if (!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND state = 'waiting'").get(operation.id)) continue; destroying.add(operation.server); try { if (operation.stage === REMOVING_SERVER) { + if (operation.server_deleted_at && getServer(operation.server)) { + finishDeployment(operation.id, { ok: false, message: 'A replacement server is registered; local cleanup cannot safely resume.' }); + continue; + } const newer = db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND action = 'create' AND rowid > (SELECT rowid FROM server_deployments WHERE id = ?)`).get(operation.server, operation.id); if (newer) { @@ -619,25 +624,10 @@ async function cleanupDestroyedServer(name: string, operation: string): Promise< warnings.push(`rm ${hostDir} failed: ${(err as Error).message}`); } - // Remove server-owned beta and deployment rows before the registry row. - // Test evidence cascades from runs; inventory and snapshots cascade from servers. + // The registry trigger preserves history and marks its old server identity. + // Live inventory, rollback snapshots, and restart slots still cascade away. try { - const tx = db().transaction((n: string) => { - // The Watchdog's rebuild campaign observes this exact operation - // through the registry gap and retains its before/after file audit. - for (const table of ['deployment_servers', 'deployment_files']) { - db().prepare(`DELETE FROM ${table} WHERE server = ? AND deployment_id NOT IN ( - SELECT campaign_id FROM watchdog_actions WHERE campaign_id IS NOT NULL)`) - .run(n); - } - db().prepare('DELETE FROM beta_servers WHERE server = ?').run(n); - db().prepare('DELETE FROM beta_server_scripts WHERE server = ?').run(n); - db().prepare('DELETE FROM beta_test_runs WHERE server = ?').run(n); - db().prepare('DELETE FROM beta_test_coverage WHERE server = ?').run(n); - db().prepare('DELETE FROM beta_file_changes WHERE server = ?').run(n); - db().prepare('DELETE FROM servers WHERE name = ?').run(n); - }); - tx(name); + db().prepare('DELETE FROM servers WHERE name = ?').run(name); } catch (err) { return { ok: false, @@ -706,7 +696,7 @@ export async function processCreations(): Promise { export function creationQueue() { type Entry = { id: string; server: string; state: string; stage: string }; const jobs = db().prepare(`SELECT id, server, state, stage FROM server_deployments - WHERE action IN ('create', 'rebuild') AND state IN ('queued', 'running', 'awaiting-health') + WHERE server_deleted_at IS NULL AND action IN ('create', 'rebuild') AND state IN ('queued', 'running', 'awaiting-health') ORDER BY started_at, rowid`).all() as Entry[]; return { active: jobs.filter(job => job.state !== 'queued'), queued: jobs.filter(job => job.state === 'queued') }; } @@ -726,7 +716,7 @@ async function runCreations(): Promise { if (creating || destroying.size > 0) return outcomes; // Restart recovery is observation-only: never repeat external creation or // deployment. A known interrupted startup must retain its readiness gate. - const interrupted = db().prepare("SELECT id FROM server_deployments WHERE action IN ('create', 'rebuild') AND state = 'interrupted' ORDER BY started_at, rowid").all() as { id: string }[]; + const interrupted = db().prepare("SELECT id FROM server_deployments WHERE server_deleted_at IS NULL AND action IN ('create', 'rebuild') AND state = 'interrupted' ORDER BY started_at, rowid").all() as { id: string }[]; for (const { id } of interrupted) { if (resumeVerification(id)) continue; // A queued removal does not make the interrupted container disappear. @@ -748,14 +738,14 @@ async function runCreations(): Promise { // Check only previously started servers. Newly launched servers keep their // slot until a later pass, even if a health probe would answer immediately. const checking = db().prepare(`SELECT j.*, COALESCE(d.submitted_at, d.started_at) AS submitted_at FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id - WHERE d.state = 'awaiting-health' ORDER BY d.started_at, d.rowid`).all() as CreationJob[]; + WHERE d.server_deleted_at IS NULL AND d.state = 'awaiting-health' ORDER BY d.started_at, d.rowid`).all() as CreationJob[]; for (const job of checking) { const id = job.deployment_id; if (getServer(job.server)?.game_port !== job.port) { finishDeployment(id, { ok: false, message: 'Server was removed or replaced before creation health checks completed' }); continue; } - if (db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'destroy' AND state IN ('waiting', 'running')").get(job.server)) continue; + if (db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')").get(job.server)) continue; try { const komodo = await komodoInspectHealthy(job.server); if (!komodo.ok) { deploymentStage(id, 'Waiting for Komodo health: ' + komodo.reason); continue; } @@ -787,7 +777,7 @@ async function runCreations(): Promise { // API calls can cancel a selected job or resume an older startup while // a previous provision awaits Komodo. Recheck without expanding this pass. if (creationQueue().active.length >= fleetSettings().createConcurrency) break; - if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND action IN ('create', 'rebuild') AND state = 'queued'").get(id)) continue; + if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND action IN ('create', 'rebuild') AND state = 'queued'").get(id)) continue; const blocker = creationBlocker(id); if (blocker) { deploymentStage(id, blocker); diff --git a/src/lib/server/provisioning/requests.ts b/src/lib/server/provisioning/requests.ts index 20d9c5f..47d157a 100644 --- a/src/lib/server/provisioning/requests.ts +++ b/src/lib/server/provisioning/requests.ts @@ -28,7 +28,7 @@ const sourceTables = { export function requestRestart(server: string, source: RestartSource): LifecycleOperation { return db().transaction(() => { const { table, key } = sourceTables[source.kind]; - const row = db().prepare(`SELECT provisioning_id, restarted_at FROM ${table} WHERE ${key} = ? AND server = ?`) + const row = db().prepare(`SELECT provisioning_id, restarted_at FROM ${table} WHERE ${key} = ? AND server = ? AND server_deleted_at IS NULL`) .get(source.id, server) as { provisioning_id: string | null; restarted_at: string | null } | undefined; if (!row) throw new Error('Restart source no longer exists'); if (row.provisioning_id) { @@ -45,14 +45,14 @@ export function requestRestart(server: string, source: RestartSource): Lifecycle // The previous release already recorded some submissions in history. // Bind to that exact interval, never to a newer restart of the server. id = (db().prepare(`SELECT d.id FROM restart_rollout r JOIN server_deployments d ON d.id = r.operation_id - WHERE r.server = ? AND d.action = 'restart' AND d.started_at <= ? AND r.requested_at >= ?`) + WHERE r.server = ? AND d.server_deleted_at IS NULL AND d.action = 'restart' AND d.started_at <= ? AND r.requested_at >= ?`) .get(server, row.restarted_at, row.restarted_at) as { id: string } | undefined)?.id; if (id) db().prepare('UPDATE server_deployments SET submitted_at = COALESCE(submitted_at, ?) WHERE id = ?').run(row.restarted_at, id); } // Only not-yet-submitted campaign changes can share an activation. A later // file deployment must never be credited to an already running restart. if (source.kind === 'campaign' && !row.restarted_at) { - id = (db().prepare(`SELECT d.id FROM server_deployments d WHERE d.server = ? AND d.action = 'restart' + id = (db().prepare(`SELECT d.id FROM server_deployments d WHERE d.server = ? AND d.server_deleted_at IS NULL AND d.action = 'restart' AND d.state = 'waiting' AND d.submitted_at IS NULL AND EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'campaign') AND NOT EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind != 'campaign') @@ -107,9 +107,9 @@ export function cancelProvisioning(id: string, reason = 'Cancelled by requester' /** Explicit caller retry starts a new job and keeps the old job as history. */ export function releaseRestartRequest(server: string, source: RestartSource): void { const { table, key } = sourceTables[source.kind]; - const row = db().prepare(`SELECT provisioning_id FROM ${table} WHERE ${key} = ? AND server = ?`).get(source.id, server) as { provisioning_id: string | null } | undefined; + const row = db().prepare(`SELECT provisioning_id FROM ${table} WHERE ${key} = ? AND server = ? AND server_deleted_at IS NULL`).get(source.id, server) as { provisioning_id: string | null } | undefined; if (row?.provisioning_id) cancelProvisioning(row.provisioning_id, 'Superseded by a fresh caller attempt'); - db().prepare(`UPDATE ${table} SET provisioning_id = NULL WHERE ${key} = ? AND server = ?`).run(source.id, server); + db().prepare(`UPDATE ${table} SET provisioning_id = NULL WHERE ${key} = ? AND server = ? AND server_deleted_at IS NULL`).run(source.id, server); } export function requestLifecycle(server: string, action: RuntimeAction) { @@ -117,6 +117,7 @@ export function requestLifecycle(server: string, action: RuntimeAction) { if (!komodoConfigured() || !rconConfigured()) return { ok: false as const, status: 503, error: 'Configure Komodo and RCON before provisioning.' }; return db().transaction(() => { const existing = db().prepare(`SELECT id, action FROM server_deployments WHERE server = ? + AND server_deleted_at IS NULL AND state IN ('queued','waiting','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(server) as { id: string; action: string } | undefined; if (existing) { if (existing.action !== action) return { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' }; @@ -142,7 +143,7 @@ export function requestLifecycle(server: string, action: RuntimeAction) { type LifecycleRequestResult = { ok: true; deploymentId: string } | { ok: false; status: number; error: string }; function watchdogRequestGuard(server: string, incidentId: string): Extract | null { - const incident = db().prepare("SELECT kind FROM watchdog_incidents WHERE id = ? AND server = ? AND state IN ('open','recovering')").get(incidentId, server) as { kind: 'health' | 'drift' } | undefined; + const incident = db().prepare("SELECT kind FROM watchdog_incidents WHERE id = ? AND server = ? AND server_deleted_at IS NULL AND state IN ('open','recovering')").get(incidentId, server) as { kind: 'health' | 'drift' } | undefined; if (!incident) return { ok: false, status: 409, error: 'The Watchdog incident is no longer active.' }; if (!maintenanceAllowed(incident.kind)) { return { ok: false, status: 409, error: 'This maintenance policy is paused.' }; @@ -183,7 +184,7 @@ export function requestWatchdogLifecycle(server: string, action: 'restart' | 're if (error) return error; const pending = db().prepare(`SELECT d.id, d.action, s.source_id FROM server_deployments d LEFT JOIN provisioning_sources s ON s.operation_id = d.id AND s.kind = 'watchdog' - WHERE d.server = ? AND d.state IN ('queued','waiting','running','awaiting-health') ORDER BY d.rowid DESC LIMIT 1`) + WHERE d.server = ? AND d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health') ORDER BY d.rowid DESC LIMIT 1`) .get(server) as { id: string; action: string; source_id: string | null } | undefined; if (pending) return pending.source_id === incidentId && pending.action === action ? { ok: true as const, deploymentId: pending.id } @@ -223,7 +224,7 @@ export function escalateWatchdogRestart(incidentId: string, operationId: string, if (error) return error; const pending = db().prepare(`SELECT d.id, d.action, s.source_id FROM server_deployments d LEFT JOIN provisioning_sources s ON s.operation_id = d.id AND s.kind = 'watchdog' - WHERE d.server = ? AND d.id != ? AND d.state IN ('queued','waiting','running','awaiting-health') LIMIT 1`) + WHERE d.server = ? AND d.server_deleted_at IS NULL AND d.id != ? AND d.state IN ('queued','waiting','running','awaiting-health') LIMIT 1`) .get(operation.server, operationId) as { id: string; action: string; source_id: string | null } | undefined; if (pending) return pending.action === 'rebuild' && pending.source_id === incidentId ? { ok: true as const, deploymentId: pending.id } diff --git a/src/lib/server/provisioning/restarts.ts b/src/lib/server/provisioning/restarts.ts index 4610acd..295912e 100644 --- a/src/lib/server/provisioning/restarts.ts +++ b/src/lib/server/provisioning/restarts.ts @@ -7,7 +7,7 @@ import { deploymentStage, finishDeployment } from './history.js'; // Komodo queues restart work. Don't mistake the still-running old container // for a completed restart when an API kick immediately follows the request. const HEALTH_POLL_MS = 15_000; -const requestsInFlight = new Set(); +const requestsInFlight = new Map(); export const restartRequestInFlight = (server: string) => requestsInFlight.has(server); interface RestartSlot { @@ -36,14 +36,14 @@ export function restartSettling(server: string): boolean { } export function rolloutWaitReason(server: string, operationId = ''): string | null { - if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND id != ? + if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL AND ((action IN ('start','stop','rebuild') AND state IN ('running','awaiting-health')) OR (action = 'destroy' AND state IN ('waiting','running')))`).get(server, operationId)) { return 'Waiting for the current provisioning operation'; } const slot = slotFor(server); const ownHandoff = slot?.operation_id === operationId && !!db().prepare(`SELECT 1 FROM server_deployments d - WHERE d.id = ? AND d.action IN ('restart','rebuild') AND EXISTS ( + WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.action IN ('restart','rebuild') AND EXISTS ( SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'watchdog')`).get(operationId); if (slot?.healthy_at === null && !ownHandoff) { return `Waiting for the previous restart of this server to recover${slot.error ? `: ${slot.error}` : ''}`; @@ -61,6 +61,7 @@ export type RolloutRestartResult = export async function restartInRollout(server: string, options: { operationId: string; action: 'start' | 'restart' }): Promise { const operationId = options.operationId; const waitReason = db().transaction(() => { + if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return 'Original server was deleted'; const reason = rolloutWaitReason(server, operationId); if (reason) return reason; db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), operationId); @@ -73,35 +74,37 @@ export async function restartInRollout(server: string, options: { operationId: s }).immediate(); if (waitReason) return { kind: 'queued', message: waitReason }; - requestsInFlight.add(server); + requestsInFlight.set(server, operationId); db().prepare(`UPDATE servers SET container_state = 'restarting', last_error = NULL, last_reply_snippet = NULL, game_started_at = NULL, game_ended_at = NULL WHERE name = ?`).run(server); try { const result = options.action === 'start' ? await komodoRuntimeAction(server, 'start') : await komodoRestart(server); + if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return { kind: 'failed', message: 'Original server was deleted' }; if (result.ok) { - db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(operationId); + db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ? AND server_deleted_at IS NULL").run(operationId); deploymentStage(operationId!, 'Waiting for container and game health (RCON or fresh telemetry)'); return { kind: 'started' }; } // These failures cannot have queued a remote restart. Other failures // (especially a timeout) may have: retain capacity until recovery. if (result.kind === 'no-name' || result.kind === 'unconfigured') { - db().prepare('DELETE FROM restart_rollout WHERE server = ?').run(server); + db().prepare('DELETE FROM restart_rollout WHERE server = ? AND operation_id = ?').run(server, operationId); } else { - db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ?').run(`komodo: ${result.message}`, server); + db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ? AND operation_id = ?').run(`komodo: ${result.message}`, server, operationId); } finishDeployment(operationId!, { ok: false, message: `komodo: ${result.message}` }); return { kind: 'failed', message: `komodo: ${result.message}` }; } catch (err) { + if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return { kind: 'failed', message: 'Original server was deleted' }; const message = `komodo: ${(err as Error).message}`; - db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ?').run(message, server); + db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ? AND operation_id = ?').run(message, server, operationId); finishDeployment(operationId!, { ok: false, message }); return { kind: 'failed', message }; } finally { // Start the grace period at completion, including slow/failed requests. - db().prepare('UPDATE restart_rollout SET requested_at = ? WHERE server = ?') - .run(new Date().toISOString(), server); - requestsInFlight.delete(server); + db().prepare('UPDATE restart_rollout SET requested_at = ? WHERE server = ? AND operation_id = ?') + .run(new Date().toISOString(), server, operationId); + if (requestsInFlight.get(server) === operationId) requestsInFlight.delete(server); } } @@ -121,7 +124,7 @@ async function refresh(): Promise { if (requestsInFlight.has(slot.server)) continue; // Recovery capacity transferred to a rebuild remains reserved until // teardown. Its health is verified by the creation worker, never here. - if (slot.operation_id && db().prepare(`SELECT 1 FROM server_deployments d WHERE d.id = ? AND ( + if (slot.operation_id && db().prepare(`SELECT 1 FROM server_deployments d WHERE d.id = ? AND d.server_deleted_at IS NULL AND ( (d.action = 'rebuild' AND d.state IN ('waiting','queued','running','awaiting-health')) OR (d.action = 'restart' AND d.state = 'waiting' AND d.submitted_at IS NULL AND EXISTS ( SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'watchdog')))`) @@ -146,7 +149,7 @@ async function refresh(): Promise { WHERE server = ? AND requested_at = ? AND operation_id IS ? AND healthy_at IS NULL`) .run(now, error ? null : now, players, error, slot.server, slot.requested_at, slot.operation_id); if (!checked.changes) continue; - if (slot.operation_id && db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND state = 'awaiting-health'").get(slot.operation_id)) { + if (slot.operation_id && db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'awaiting-health'").get(slot.operation_id)) { db().prepare('UPDATE server_deployments SET checked_at = ?, players = ? WHERE id = ?').run(now, players, slot.operation_id); if (error) deploymentStage(slot.operation_id, 'Waiting for health: ' + error); else { diff --git a/src/lib/server/provisioning/worker.ts b/src/lib/server/provisioning/worker.ts index a3d7549..576bb3c 100644 --- a/src/lib/server/provisioning/worker.ts +++ b/src/lib/server/provisioning/worker.ts @@ -23,7 +23,7 @@ function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled' if (!sources.length) return 'ready'; // Watchdog intent gates every linked source, including its drift campaign. for (const source of sources.filter(source => source.kind === 'watchdog')) { - if (!db().prepare("SELECT 1 FROM watchdog_incidents WHERE id = ? AND server = ? AND state IN ('open','recovering')").get(source.source_id, operation.server) + if (!db().prepare("SELECT 1 FROM watchdog_incidents WHERE id = ? AND server = ? AND server_deleted_at IS NULL AND state IN ('open','recovering')").get(source.source_id, operation.server) || getServer(operation.server)?.desired_state === 'stopped') return 'cancelled'; const incident = db().prepare('SELECT kind FROM watchdog_incidents WHERE id = ?').get(source.source_id) as { kind: 'health' | 'drift' }; if (!maintenanceAllowed(incident.kind)) return 'held'; @@ -31,21 +31,21 @@ function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled' let held = false; for (const source of sources) { if (source.kind === 'campaign') { - const row = db().prepare('SELECT restart_state, files_changed, provisioning_id FROM deployment_servers WHERE deployment_id = ? AND server = ?') + const row = db().prepare('SELECT restart_state, files_changed, provisioning_id FROM deployment_servers WHERE deployment_id = ? AND server = ? AND server_deleted_at IS NULL') .get(source.source_id, operation.server) as { restart_state: string; files_changed: number; provisioning_id: string } | undefined; if (row?.provisioning_id !== operation.id || !row.files_changed) continue; if (['pending','failed'].includes(row.restart_state)) return 'ready'; if (['held','manual'].includes(row.restart_state)) held = true; } else if (source.kind === 'beta') { const row = db().prepare(`SELECT b.state, bs.stage, bs.provisioning_id, p.outcome FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id - LEFT JOIN beta_test_plans p ON p.beta_id = b.id WHERE bs.beta_id = ? AND bs.server = ?`) + LEFT JOIN beta_test_plans p ON p.beta_id = b.id WHERE bs.beta_id = ? AND bs.server = ? AND bs.server_deleted_at IS NULL`) .get(source.source_id, operation.server) as { state: string; stage: string; provisioning_id: string; outcome: string | null } | undefined; if (row?.provisioning_id !== operation.id || !['running','ready-promote'].includes(row.state)) continue; if (!['awaiting-empty','restarting','awaiting-health'].includes(row.stage)) continue; if (!row.outcome || ['active','passed'].includes(row.outcome)) return 'ready'; if (row.outcome === 'paused') held = true; } else if (source.kind === 'maintenance') { - const row = db().prepare('SELECT state, provisioning_id, kind, origin FROM maintenance_jobs WHERE id = ?').get(source.source_id) as { state: string; provisioning_id: string; kind: string; origin: string } | undefined; + const row = db().prepare('SELECT state, provisioning_id, kind, origin FROM maintenance_jobs WHERE id = ? AND server_deleted_at IS NULL').get(source.source_id) as { state: string; provisioning_id: string; kind: string; origin: string } | undefined; if (row?.provisioning_id === operation.id && ['queued','restarting'].includes(row.state)) { if (getServer(operation.server)?.desired_state === 'stopped') return 'cancelled'; if (!maintenanceAllowed() || (row.kind === 'update' && row.origin === 'automatic' && !maintenanceAllowed('updates'))) return 'held'; @@ -59,7 +59,7 @@ function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled' function blocker(server: string, id: string): string | null { const sources = sourcesFor(id); const rank = priority(sources); - const others = db().prepare(`SELECT id, state, action, rowid FROM server_deployments WHERE server = ? AND id != ? + const others = db().prepare(`SELECT id, state, action, rowid FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL AND state IN ('queued','waiting','running','awaiting-health')`).all(server, id) as { id: string; state: string; action: string; rowid: number }[]; const own = db().prepare('SELECT rowid FROM server_deployments WHERE id = ?').get(id) as { rowid: number }; if (others.some(other => other.state !== 'waiting' || other.action === 'destroy' @@ -69,8 +69,8 @@ function blocker(server: string, id: string): string | null { // Beta activation must not wait on the campaign rows that the beta itself // holds. Campaign/maintenance callers likewise never block on their own job. if (rank > 0 && db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id - WHERE bs.server = ? AND b.state IN ('running','ready-promote') LIMIT 1`).get(server)) return 'Waiting for beta work on this server'; - if (rank > 1 && db().prepare(`SELECT 1 FROM deployment_servers ds WHERE ds.server = ? + WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running','ready-promote') LIMIT 1`).get(server)) return 'Waiting for beta work on this server'; + if (rank > 1 && db().prepare(`SELECT 1 FROM deployment_servers ds WHERE ds.server = ? AND ds.server_deleted_at IS NULL AND ds.restart_state IN ('pending','failed','held') AND (ds.provisioning_id IS NULL OR EXISTS ( SELECT 1 FROM server_deployments d WHERE d.id = ds.provisioning_id AND d.state IN ('queued','waiting','running','awaiting-health'))) AND NOT EXISTS (SELECT 1 FROM watchdog_actions a WHERE a.operation_id = ? AND a.campaign_id = ds.deployment_id) @@ -93,7 +93,7 @@ async function run(): Promise { await processDestructions(); await refreshRestartRollout(); const operations = db().prepare(`SELECT id, server, action, state, stage FROM server_deployments - WHERE action IN ('start','restart','stop','rebuild') AND (state = 'waiting' OR (action = 'stop' AND state = 'awaiting-health')) + WHERE server_deleted_at IS NULL AND action IN ('start','restart','stop','rebuild') AND (state = 'waiting' OR (action = 'stop' AND state = 'awaiting-health')) ORDER BY started_at, rowid`).all() as Operation[]; for (const operation of operations) { try { await processOperation(operation); } @@ -140,9 +140,9 @@ async function processOperation(operation: Operation): Promise { } // Requests can arrive during the probes. Recheck before claiming the action. if (blocker(server, id) || sourceEligibility(operation) !== 'ready') return; - if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND state = 'waiting'").get(id)) return; + if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return; if (action === 'rebuild') { await rebuildServer(id); return; } - const claim = db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND state = 'waiting'").run(id); + const claim = db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").run(id); if (!claim.changes) return; if (action === 'start' || action === 'restart') { const result = await restartInRollout(server, { operationId: id, action }); diff --git a/src/lib/ui/components/BetaTestProgress.svelte b/src/lib/ui/components/BetaTestProgress.svelte index 906c971..1011735 100644 --- a/src/lib/ui/components/BetaTestProgress.svelte +++ b/src/lib/ui/components/BetaTestProgress.svelte @@ -1,7 +1,7 @@ + + + {#if href && !deletedAt} + {:else}{/if} + {#if deletedAt}Deleted{/if} + + + diff --git a/src/lib/ui/components/MatchHistory.svelte b/src/lib/ui/components/MatchHistory.svelte index cf2dd81..0c0a81d 100644 --- a/src/lib/ui/components/MatchHistory.svelte +++ b/src/lib/ui/components/MatchHistory.svelte @@ -3,7 +3,7 @@ import type { ServerMatch } from '$lib/match-history.js'; import { pollVisible } from '$ui/poll.js'; import { formatDuration } from '$ui/duration.js'; - import { serverTitle } from '$ui/server-labels.js'; + import HistoricalServerIdentity from './HistoricalServerIdentity.svelte'; import Card from './primitives/Card.svelte'; import Button from './primitives/Button.svelte'; import EmptyState from './primitives/EmptyState.svelte'; @@ -60,7 +60,7 @@ {#if match.endedAt}{:else}—{/if} - {#if server === undefined}{serverTitle(match.server)}{/if} + {#if server === undefined}{/if} {match.map} {match.status === 'active' ? 'In progress' : match.status === 'completed' ? 'Completed' : 'Interrupted'} {match.startPlayers ?? '—'}{match.uniquePlayers}{match.endPlayers ?? '—'} @@ -84,5 +84,4 @@ th, td { padding: var(--space-3); text-align: left; border-bottom: 1px solid var(--border-subtle); white-space: nowrap; } th { color: var(--text-secondary); font-size: var(--text-xs); } td { font-variant-numeric: tabular-nums; } - a { color: var(--accent-primary-hover); } diff --git a/src/lib/watchdog.ts b/src/lib/watchdog.ts index d2b8e52..5a232c9 100644 --- a/src/lib/watchdog.ts +++ b/src/lib/watchdog.ts @@ -4,6 +4,7 @@ export type WatchdogState = 'open' | 'recovering' | 'attention' | 'resolved' | ' export interface WatchdogIncident { id: string; server: string; + server_deleted_at?: string | null; kind: 'health' | 'drift'; state: WatchdogState; summary: string; diff --git a/src/routes/api/betas/[id]/+server.ts b/src/routes/api/betas/[id]/+server.ts index 8439b4e..3a70f7e 100644 --- a/src/routes/api/betas/[id]/+server.ts +++ b/src/routes/api/betas/[id]/+server.ts @@ -33,7 +33,7 @@ export async function GET({ params }: { params: { id: string } }) { const events = db().prepare('SELECT id, created_at, message FROM beta_events WHERE beta_id = ? ORDER BY id').all(id); const campaign = db().prepare("SELECT id FROM deployments WHERE beta_id = ? AND action = 'promote' ORDER BY started_at DESC LIMIT 1").get(id) as { id: string } | undefined; const promotion = campaign ? { ...campaign, - pending: (db().prepare("SELECT COUNT(*) n FROM deployment_servers WHERE deployment_id = ? AND restart_state NOT IN ('done','failed','cancelled')").get(campaign.id) as { n: number }).n, - failed: (db().prepare("SELECT COUNT(*) n FROM deployment_servers WHERE deployment_id = ? AND restart_state = 'failed'").get(campaign.id) as { n: number }).n } : null; + pending: (db().prepare("SELECT COUNT(*) n FROM deployment_servers WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state NOT IN ('done','failed','cancelled')").get(campaign.id) as { n: number }).n, + failed: (db().prepare("SELECT COUNT(*) n FROM deployment_servers WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state = 'failed'").get(campaign.id) as { n: number }).n } : null; return json({ beta, scripts, servers, events, test: testView(id), promotion }); } diff --git a/src/routes/api/betas/[id]/retry/+server.ts b/src/routes/api/betas/[id]/retry/+server.ts index cc55d61..267460d 100644 --- a/src/routes/api/betas/[id]/retry/+server.ts +++ b/src/routes/api/betas/[id]/retry/+server.ts @@ -19,11 +19,12 @@ export async function POST({ params, request }: { params: { id: string }; reques if (!body.server) return json({ ok: false, error: 'server is required' }, { status: 400 }); const row = db() - .prepare('SELECT stage, restarted_at, healthy_at FROM beta_servers WHERE beta_id = ? AND server = ?') + .prepare('SELECT stage, restarted_at, healthy_at, server_deleted_at FROM beta_servers WHERE beta_id = ? AND server = ?') .get(betaId, body.server) as - | { stage: string; restarted_at: string | null; healthy_at: string | null } + | { stage: string; restarted_at: string | null; healthy_at: string | null; server_deleted_at: string | null } | undefined; if (!row) return json({ ok: false, error: 'no such beta_server' }, { status: 404 }); + if (row.server_deleted_at) return json({ ok: false, error: 'This beta server was deleted; open a new beta for its replacement.' }, { status: 409 }); if (row.stage !== 'failed') return json({ ok: false, error: `stage is ${row.stage}, not failed` }, { status: 409 }); // Best-effort resume point: if we already got past restart, go back to diff --git a/src/routes/api/betas/[id]/test/+server.ts b/src/routes/api/betas/[id]/test/+server.ts index f622d10..3a254a4 100644 --- a/src/routes/api/betas/[id]/test/+server.ts +++ b/src/routes/api/betas/[id]/test/+server.ts @@ -17,6 +17,9 @@ export async function POST({ params, request }: { params: { id: string }; reques if (!['running', 'ready-promote'].includes(beta.state)) return json({ error: 'Beta is closed' }, { status: 409 }); try { const body = await request.json(); + if (db().prepare('SELECT 1 FROM beta_servers WHERE beta_id = ? AND server_deleted_at IS NOT NULL').get(id)) { + return json({ ok: false, error: 'A beta server was deleted. Roll back this beta and open a new one to collect fresh evidence.' }, { status: 409 }); + } const plan = parseBetaPlan(JSON.parse(stored.plan)); if (body.action === 'sign-off') { if (!['active', 'passed'].includes(stored.outcome)) throw new Error('Resume or retry the test before signing off'); diff --git a/src/routes/api/campaigns/+server.ts b/src/routes/api/campaigns/+server.ts index 0f0035f..b937727 100644 --- a/src/routes/api/campaigns/+server.ts +++ b/src/routes/api/campaigns/+server.ts @@ -40,12 +40,13 @@ export async function GET({ url }: { url: URL }) { const placeholders = ids.map(() => '?').join(','); const serversByDep = db() .prepare( - `SELECT deployment_id, server, files_changed, restart_state, players, checked_at, restarted_at, attempts, error + `SELECT deployment_id, server, server_deleted_at, files_changed, restart_state, players, checked_at, restarted_at, attempts, error FROM deployment_servers WHERE deployment_id IN (${placeholders})` ) .all(...ids) as Array<{ deployment_id: string; server: string; + server_deleted_at: string | null; files_changed: number; restart_state: string; players: number | null; @@ -92,7 +93,7 @@ export async function GET({ url }: { url: URL }) { servers: grouped.get(d.id) ?? [], scripts: scriptsByDep.get(d.id) ?? [] })) - .filter((d) => (activeOnly ? d.servers.some((s) => s.restart_state === 'pending' || s.restart_state === 'failed') : true)); + .filter((d) => (activeOnly ? d.servers.some((s) => !s.server_deleted_at && (s.restart_state === 'pending' || s.restart_state === 'failed')) : true)); return json({ deployments: rows }); } diff --git a/src/routes/betas/[id]/+page.svelte b/src/routes/betas/[id]/+page.svelte index 6f77501..c7240e8 100644 --- a/src/routes/betas/[id]/+page.svelte +++ b/src/routes/betas/[id]/+page.svelte @@ -11,6 +11,7 @@ import { confirmAction } from '$ui/confirm.js'; import BetaTestProgress from '$ui/components/BetaTestProgress.svelte'; import ServerIdentity from '$ui/components/ServerIdentity.svelte'; + import HistoricalServerIdentity from '$ui/components/HistoricalServerIdentity.svelte'; import OperationHistory from '$ui/components/OperationHistory.svelte'; import StatusBadge from '$ui/components/primitives/StatusBadge.svelte'; import Card from '$ui/components/primitives/Card.svelte'; @@ -21,7 +22,7 @@ import EmptyState from '$ui/components/primitives/EmptyState.svelte'; interface BetaRow { id:number; state:string; opened_at:string; closed_at:string|null; wait_for_health:number; wait_for_match:number; notes?:string|null; } - interface BetaServer { server:string; stage:string; error:string|null; deployed_at:string|null; restarted_at:string|null; healthy_at:string|null; match_seen_at:string|null; provisioning_id:string|null; } + interface BetaServer { server:string; server_deleted_at?:string|null; stage:string; error:string|null; deployed_at:string|null; restarted_at:string|null; healthy_at:string|null; match_seen_at:string|null; provisioning_id:string|null; } interface BetaFile { id:number; name:string; kind:string; target_rel_path:string; version:{id:number; version:number; release_version:string|null; sha256:string}|null; } interface Promotion { id:string; pending:number; failed:number; } let test = $state(null); @@ -36,6 +37,7 @@ let now = $state(Date.now()); const id = $derived(Number($page.params.id)); const closed = $derived(beta?.state === 'promoted' || beta?.state === 'rolled-back'); + const hasDeletedServers = $derived(servers.some(server => server.server_deleted_at)); const deadline = $derived(beta && test ? new Date(Date.parse(beta.opened_at) + test.plan.timeoutHours * 3600000) : null); const remaining = $derived(deadline ? Math.max(0, Math.ceil((deadline.getTime() - now) / 60000)) : 0); const deadlineText = $derived(remaining > 0 ? `${Math.floor(remaining / 60)}h ${remaining % 60}m remaining` : 'Testing window ended'); @@ -43,6 +45,7 @@ const ranked = $derived([...servers].sort((a,b) => (a.stage === 'failed' ? 0 : a.stage === 'ready' ? 2 : 1) - (b.stage === 'failed' ? 0 : b.stage === 'ready' ? 2 : 1) || a.server.localeCompare(b.server))); const waiting = $derived(ranked.filter(server => server.stage !== 'ready')); const nextAction = $derived(closed ? beta?.state === 'promoted' ? 'Follow the campaign to check activation on additional servers.' : 'Reuse these files and this plan when you are ready to test again.' + : hasDeletedServers ? 'A participating server was deleted. Its results remain below. Roll back, then reuse this plan to test the current servers.' : test?.outcome === 'inconclusive' ? 'The deadline passed without enough evidence. Roll back, then reuse this plan with a longer testing window.' : test?.outcome === 'paused' ? 'Resume to continue collecting evidence, or start a fresh attempt. The original deadline still applies.' : test?.outcome === 'failed' ? 'Investigate the failed check before a fresh attempt, or roll back.' @@ -91,7 +94,7 @@ async function rollback() { const reject = beta?.state === 'ready-promote'; const restarts = test?.plan.rollbackRestart === 'manual' ? 'Restart servers manually to load the restored files.' : 'Restarts wait for empty servers.'; - if (!await confirmAction({title:reject?'Reject beta?':'Roll back beta?',message:`Restore the original files on ${servers.length} beta servers${reject ? ' and mark these versions as test-failed' : ''}. ${restarts}`,confirmLabel:reject?'Reject beta':'Roll back',danger:true})) return; + if (!await confirmAction({title:reject?'Reject beta?':'Roll back beta?',message:`Restore the original files on ${servers.filter(server => !server.server_deleted_at).length} beta servers${reject ? ' and mark these versions as test-failed' : ''}. ${restarts}`,confirmLabel:reject?'Reject beta':'Roll back',danger:true})) return; acting = 'rollback'; flash = ''; success = ''; try { const response = await fetch(`/api/betas/${id}/rollback`,{method:'POST'}), body = await response.json(); @@ -136,7 +139,7 @@ {#if !closed && test}

Waiting for empty servers uses this window. Pausing or retrying does not extend the deadline.

{/if}
- {#if !closed}{/if} + {#if !closed}{/if} {#if closed}{/if} {#if promotion}{/if}
@@ -146,13 +149,13 @@ {#if !closed && waiting.length}

Outstanding conditions

- {#each waiting as server}

{server.error ?? outstanding(server.server)}

{server.error ? 'Diagnose' : 'View checks'}
{/each} + {#each waiting as server}

{server.error ?? (server.server_deleted_at ? 'This server was deleted. Recorded progress is retained.' : outstanding(server.server))}

{#if server.error && !server.server_deleted_at}Diagnose{:else if test}View checks{/if}
{/each}
{/if} - {#if test}
server.error?.startsWith('deploy:')) ? 'Deployment was incomplete. Roll back before opening a fresh beta.' : undefined} onUpdate={load} />
{/if} + {#if test}
server.error?.startsWith('deploy:')) ? 'Deployment was incomplete. Roll back before opening a fresh beta.' : undefined} onUpdate={load} />
{/if}
Files and versions ({scripts.length})
{#each scripts as file}
{file.name}{artifactVersionLabel(file.version)}{file.target_rel_path}{file.version?.sha256.slice(0,12)}
{/each}
Deployment progress ({servers.length} servers)
- {#each ranked as server}

Files {server.deployed_at ? 'installed' : 'pending'} · Restart {server.restarted_at ? 'submitted' : 'pending'}{#if beta.wait_for_health} · Health {server.healthy_at ? 'verified' : 'pending'}{/if}{#if beta.wait_for_match} · Match {server.match_seen_at ? 'observed' : 'pending'}{/if}

{#if server.provisioning_id}View provisioning{/if}{#if server.error}

{server.error}

{/if}{#if server.stage === 'failed' && !test && !closed}{/if}
{/each} + {#each ranked as server}

Files {server.deployed_at ? 'installed' : 'pending'} · Restart {server.restarted_at ? 'submitted' : 'pending'}{#if beta.wait_for_health} · Health {server.healthy_at ? 'verified' : 'pending'}{/if}{#if beta.wait_for_match} · Match {server.match_seen_at ? 'observed' : 'pending'}{/if}

{#if server.provisioning_id}View provisioning{/if}{#if server.error}

{server.error}

{/if}{#if server.stage === 'failed' && !server.server_deleted_at && !test && !closed}{/if}
{/each}
diff --git a/src/routes/campaigns/+page.svelte b/src/routes/campaigns/+page.svelte index d50b121..33865c2 100644 --- a/src/routes/campaigns/+page.svelte +++ b/src/routes/campaigns/+page.svelte @@ -6,6 +6,7 @@ import { pollVisible } from '$ui/poll.js'; import { confirmAction } from '$ui/confirm.js'; import { serverTitle } from '$ui/server-labels.js'; + import HistoricalServerIdentity from '$ui/components/HistoricalServerIdentity.svelte'; import StatusBadge from '$ui/components/primitives/StatusBadge.svelte'; import ListFooter from '$ui/components/primitives/ListFooter.svelte'; import Badge from '$ui/components/primitives/Badge.svelte'; @@ -15,6 +16,7 @@ interface ServerRow { server: string; + server_deleted_at?: string | null; restart_state: string; files_changed: number; players: number | null; @@ -76,7 +78,7 @@ onMount(() => pollVisible(loadCampaigns, 5000)); function isActive(d: Deployment): boolean { - return d.servers.some((s) => s.restart_state === 'pending' || s.restart_state === 'failed'); + return d.servers.some((s) => !s.server_deleted_at && (s.restart_state === 'pending' || s.restart_state === 'failed')); } function ago(iso: string): string { const d = Date.now() - new Date(iso).getTime(); @@ -108,7 +110,7 @@ ev.preventDefault(); ev.stopPropagation(); const summary = scriptSummary(d) || 'this promote'; - if (!await confirmAction({ title:'Roll back promotion?', message:`Restore the previous version of ${summary} on ${d.servers.length} servers.`, confirmLabel:'Roll back', danger:true })) return; + if (!await confirmAction({ title:'Roll back promotion?', message:`Restore the previous version of ${summary} on ${d.servers.filter(server => !server.server_deleted_at).length} servers.`, confirmLabel:'Roll back', danger:true })) return; rollingBack = d.id; flash = null; flashId = d.id; flashError = false; try { @@ -211,7 +213,7 @@ {/if}
- {#if d.action === 'promote' && !d.rolled_back_by} + {#if d.action === 'promote' && !d.rolled_back_by && d.servers.some(server => !server.server_deleted_at)}
diff --git a/src/routes/campaigns/[id]/+page.svelte b/src/routes/campaigns/[id]/+page.svelte index 6a345cd..053e3dd 100644 --- a/src/routes/campaigns/[id]/+page.svelte +++ b/src/routes/campaigns/[id]/+page.svelte @@ -3,7 +3,7 @@ import { onMount } from 'svelte'; import { pollVisible } from '$ui/poll.js'; import Dialog from '$ui/components/primitives/Dialog.svelte'; - import ServerIdentity from '$ui/components/ServerIdentity.svelte'; + import HistoricalServerIdentity from '$ui/components/HistoricalServerIdentity.svelte'; import StatusBadge from '$ui/components/primitives/StatusBadge.svelte'; import { page } from '$app/stores'; import DiffViewer from '$ui/components/DiffViewer.svelte'; @@ -22,7 +22,7 @@ size_before: number | null; size_after: number | null; } interface ServerRow { - deployment_id: string; server: string; files_changed: number; + deployment_id: string; server: string; server_deleted_at?: string | null; files_changed: number; restart_state: string; players: number | null; checked_at: string | null; restarted_at: string | null; provisioning_id: string | null; attempts: number; error: string | null; @@ -48,7 +48,8 @@ let openFile = $state<{ id: number; before: string | null; after: string | null; verb: string; relPath: string } | null>(null); let retryFile = $state(null); const completed = $derived(detail?.servers.filter(server => server.restart_state === 'done').length ?? 0); - const unresolved = $derived(detail?.servers.filter(server => server.restart_state !== 'done').length ?? 0); + const unresolved = $derived(detail?.servers.filter(server => !server.server_deleted_at && server.restart_state !== 'done').length ?? 0); + const deletedIncomplete = $derived(detail?.servers.filter(server => server.server_deleted_at && server.restart_state !== 'done').length ?? 0); const rankedServers = $derived([...(detail?.servers ?? [])].sort((a, b) => { const rank: Record = { failed: 0, manual: 1, held: 2, pending: 3, done: 4 }; return (rank[a.restart_state] ?? 3) - (rank[b.restart_state] ?? 3); @@ -122,7 +123,7 @@ {/snippet} -

{completed} / {detail.servers.length} servers activated{unresolved ? ` · ${unresolved} unresolved` : ''}

{detail.watchdogIncidentId ? 'Maintenance repairs drift by deleting the existing instance files and rebuilding with the latest promoted setup after verifying a safe server state.' : 'Files are staged before activation. Servers restart when empty, then their health is verified.'}

{#if unresolved}

Review failed or waiting servers below. Diagnose connectivity or inspect startup logs from the affected row.

{:else}

Every recorded server completed activation.

{/if}
+

{completed} / {detail.servers.length} servers activated{unresolved ? ` · ${unresolved} unresolved` : ''}{deletedIncomplete ? ` · ${deletedIncomplete} deleted before activation` : ''}

{detail.watchdogIncidentId ? 'Maintenance repairs drift by deleting the existing instance files and rebuilding with the latest promoted setup after verifying a safe server state.' : 'Files are staged before activation. Servers restart when empty, then their health is verified.'}

{#if unresolved || deletedIncomplete}

Review the recorded results below.{#if deletedIncomplete} Deleted servers retain their last recorded progress.{/if}{#if unresolved} Diagnose connectivity or inspect startup logs from the affected row.{/if}

{:else}

Every recorded server completed activation.

{/if}
{#if detail.watchdogIncidentId}

Started by Maintenance · View incident and recovery history

{/if} @@ -144,7 +145,7 @@ {#each rankedServers as s (s.server)}
- + {#if s.error} {s.error} {/if} @@ -159,7 +160,7 @@ {#if s.restarted_at} restarted {fmt(s.restarted_at)} {/if} - {#if s.restart_state !== 'done'}DiagnoseOpen logs{/if} + {#if s.restart_state !== 'done' && !s.server_deleted_at}DiagnoseOpen logs{/if}
{/each} @@ -176,7 +177,7 @@ {:else} {#each [...byServer.entries()] as [srv, files] (srv)}
-

+

server.server === srv)?.server_deleted_at} />

{#each files as f (f.id)} {:else if diffLoading} diff --git a/src/routes/leaderboard/players/[game]/[guid]/+page.svelte b/src/routes/leaderboard/players/[game]/[guid]/+page.svelte index 217262b..e64c789 100644 --- a/src/routes/leaderboard/players/[game]/[guid]/+page.svelte +++ b/src/routes/leaderboard/players/[game]/[guid]/+page.svelte @@ -5,6 +5,7 @@ import SectionHeader from '$ui/components/primitives/SectionHeader.svelte'; import Button from '$ui/components/primitives/Button.svelte'; import CopyValue from '$ui/components/CopyValue.svelte'; + import Badge from '$ui/components/primitives/Badge.svelte'; let { data }: { data: { profile: PlayerProfile } } = $props(); const profile = $derived(data.profile); @@ -76,7 +77,7 @@ {#each historyColumns as key}{/each}{#each profile.matches as match (match.id)} - + {#each historyColumns as key}{/each} diff --git a/src/routes/maintenance/+page.svelte b/src/routes/maintenance/+page.svelte index bf6bab0..47816fc 100644 --- a/src/routes/maintenance/+page.svelte +++ b/src/routes/maintenance/+page.svelte @@ -9,6 +9,7 @@ import { timeFormat, timeZone } from '$ui/stores/time-zone.js'; import { serverTitle } from '$ui/server-labels.js'; import ServerIdentity from '$ui/components/ServerIdentity.svelte'; + import HistoricalServerIdentity from '$ui/components/HistoricalServerIdentity.svelte'; import SectionHeader from '$ui/components/primitives/SectionHeader.svelte'; import Tabs from '$ui/components/primitives/Tabs.svelte'; import Badge from '$ui/components/primitives/Badge.svelte'; @@ -40,16 +41,16 @@ const versions = $derived(data?.runtime.servers.filter(v => !server || `${v.server} ${serverTitle(v.server)}`.toLowerCase().includes(server.toLowerCase())) ?? []); const outdated = $derived(versions.filter(v => v.update_status === 'outdated')); const unknown = $derived(versions.filter(v => v.update_status === 'unknown')); - const activeJobs = $derived(data?.runtime.jobs.filter(j => ['queued', 'restarting'].includes(j.state)) ?? []); + const activeJobs = $derived(data?.runtime.jobs.filter(j => !j.server_deleted_at && ['queued', 'restarting'].includes(j.state)) ?? []); const activities = $derived([ ...(data?.recovery.incidents ?? []).map(i => ({ - key: i.id, kind: i.kind, server: i.server, at: i.first_seen_at, + key: i.id, kind: i.kind, server: i.server, server_deleted_at: i.server_deleted_at, at: i.first_seen_at, title: i.kind === 'health' ? 'Health recovery' : 'Configuration drift', state: ['open', 'recovering'].includes(i.state) ? 'active' : i.state === 'attention' ? 'attention' : 'done', label: watchdogStateLabels[i.state], detail: i.state === 'recovering' ? i.operation_stage ?? i.summary : i.summary, href: `/maintenance/${encodeURIComponent(i.id)}`, job: null })), ...(data?.runtime.jobs ?? []).map(j => ({ - key: `job-${j.id}`, kind: j.kind === 'schedule' ? 'schedule' : 'updates', server: j.server, at: j.requested_at, + key: `job-${j.id}`, kind: j.kind === 'schedule' ? 'schedule' : 'updates', server: j.server, server_deleted_at: j.server_deleted_at, at: j.requested_at, title: j.kind === 'schedule' ? 'Scheduled restart' : j.kind === 'update' ? 'Plutonium update' : 'Legacy maintenance', state: ['queued', 'restarting'].includes(j.state) ? 'active' : j.state === 'failed' ? 'attention' : 'done', label: j.state === 'failed' ? 'Needs attention' : j.state === 'done' ? 'Completed' : j.state === 'cancelled' ? 'Cancelled' : j.state === 'restarting' ? 'Recovering' : 'Queued', @@ -58,12 +59,12 @@ })) ].sort((a, b) => b.at.localeCompare(a.at))); const isLatest = (item: (typeof activities)[number]) => !item.job || !data?.runtime.jobs.some(j => j.server === item.server && j.kind === item.job!.kind && j.id > item.job!.id); - const attention = $derived(activities.filter(a => a.state === 'attention' && isLatest(a)).length); - const waiting = $derived(activities.filter(a => a.state === 'active').length); + const attention = $derived(activities.filter(a => !a.server_deleted_at && a.state === 'attention' && isLatest(a)).length); + const waiting = $derived(activities.filter(a => !a.server_deleted_at && a.state === 'active').length); const matching = $derived(activities.filter(a => (kind === 'all' || a.kind === kind) && (!server || `${a.server} ${serverTitle(a.server)}`.toLowerCase().includes(server.toLowerCase())) && - (view === 'overview' ? a.state !== 'done' && isLatest(a) : status === 'all' || a.state === status) + (view === 'overview' ? !a.server_deleted_at && a.state !== 'done' && isLatest(a) : status === 'all' || a.state === status) )); $effect(() => { const params = $page.url.searchParams; @@ -186,12 +187,12 @@

{view === 'overview' ? 'Active work & attention' : 'Maintenance activity'}

{#each matching.slice(0, limit) as item (item.key)}
-

{item.title} ·

{item.label}
+

{item.title} ·

{item.label}
{#if item.detail}

{item.detail}

{/if}

{$timeFormat.dateTime(item.at)}{item.job?.target_revision ? ` · Expected r${item.job.target_revision} or newer` : ''}

{#if item.href}{item.job ? 'View restart operation' : 'View evidence and recovery'}{/if} - {#if item.job?.state === 'queued'}{/if} - {#if item.job?.state === 'failed' && item.job.operation_state === 'done'}{/if} + {#if item.job?.state === 'queued' && !item.server_deleted_at}{/if} + {#if item.job?.state === 'failed' && !item.server_deleted_at && item.job.operation_state === 'done'}{/if}
{:else}{/each}
diff --git a/src/routes/maintenance/[id]/+page.svelte b/src/routes/maintenance/[id]/+page.svelte index f0cf845..9869ddd 100644 --- a/src/routes/maintenance/[id]/+page.svelte +++ b/src/routes/maintenance/[id]/+page.svelte @@ -33,10 +33,11 @@ const actions = $derived(detail?.actions.slice().reverse() ?? []); const events = $derived(detail?.events.slice().reverse() ?? []); const activeOperation = $derived(actions.some(action => ['queued', 'waiting', 'running', 'awaiting-health'].includes(action.operation_state ?? ''))); - const canRetry = $derived(incident?.state === 'attention' || incident?.state === 'dismissed'); - const canDismiss = $derived(incident?.state === 'open' || incident?.state === 'attention'); + const canRetry = $derived(!incident?.server_deleted_at && (incident?.state === 'attention' || incident?.state === 'dismissed')); + const canDismiss = $derived(!incident?.server_deleted_at && (incident?.state === 'open' || incident?.state === 'attention')); const explanation = $derived.by(() => { if (!incident) return ''; + if (incident.server_deleted_at) return 'This server was deleted. Its recorded finding, evidence, and recovery results are preserved below.'; if (incident.state === 'resolved') return incident.kind === 'drift' ? 'The observed file drift has cleared.' : 'The server passed the checks for this incident.'; if (incident.state === 'dismissed') return 'This incident was dismissed. Retry to allow automatic recovery again.'; if (incident.state === 'attention') return 'Automatic recovery needs attention. Review the failed operation and server diagnostics before retrying.'; @@ -107,11 +108,11 @@ {#if incident && detail}
-
View server
+
{#if incident.server_deleted_at}Deleted{:else}View server{/if}

{incident.summary}

{explanation}

{#if activeOperation && (canRetry || canDismiss)}

Complete or cancel the active provisioning operation before retrying or dismissing this incident.

{/if} - {#if incident.kind === 'drift'}

Drift repair deletes the existing instance and all of its files, then rebuilds it with the latest promoted setup. The repair is recorded as a campaign.

{:else if incident.state !== 'resolved'}

Automatic health recovery first tries a restart, then rebuilds if needed. A rebuild deletes the existing instance and all of its files and installs the latest promoted setup.

{/if} + {#if !incident.server_deleted_at}{#if incident.kind === 'drift'}

Drift repair deletes the existing instance and all of its files, then rebuilds it with the latest promoted setup. The repair is recorded as a campaign.

{:else if incident.state !== 'resolved'}

Automatic health recovery first tries a restart, then rebuilds if needed. A rebuild deletes the existing instance and all of its files and installs the latest promoted setup.

{/if}{/if}
First detected
{$timeFormat.dateTime(incident.first_seen_at)}
Last observed
{$timeFormat.dateTime(incident.last_seen_at)}
@@ -119,7 +120,7 @@ {#if evidence.containerState}
Observed Komodo state
{evidence.containerState}
{/if}
{#if evidence.reason && evidence.reason !== incident.summary}

Observed reason: {evidence.reason}

{/if} -
Diagnose connectionOpen server logs
+ {#if !incident.server_deleted_at}
Diagnose connectionOpen server logs
{/if}
diff --git a/src/routes/provisioning/+page.svelte b/src/routes/provisioning/+page.svelte index 856fd73..0851c4f 100644 --- a/src/routes/provisioning/+page.svelte +++ b/src/routes/provisioning/+page.svelte @@ -7,7 +7,7 @@ import Tabs from '$ui/components/primitives/Tabs.svelte'; import StatusBadge from '$ui/components/primitives/StatusBadge.svelte'; import ListFooter from '$ui/components/primitives/ListFooter.svelte'; - import ServerIdentity from '$ui/components/ServerIdentity.svelte'; + import HistoricalServerIdentity from '$ui/components/HistoricalServerIdentity.svelte'; import { serverTitle } from '$ui/server-labels.js'; import SectionHeader from '$ui/components/primitives/SectionHeader.svelte'; import Badge from '$ui/components/primitives/Badge.svelte'; @@ -22,7 +22,7 @@ let lifecycleOpen = $state(false); interface Deployment { - id: string; server: string; action: ProvisioningAction; + id: string; server: string; server_deleted_at?: string | null; action: ProvisioningAction; state: 'queued' | 'awaiting-health' | 'waiting' | 'running' | 'done' | 'failed' | 'warning' | 'interrupted' | 'cancelled'; stage: string; started_at: string; finished_at: string | null; triggered_by: string; error: string | null; warnings: string; server_exists: number; @@ -72,10 +72,10 @@ let action = $state('all'); let search = $state(''); let refreshing = false; - let active = $derived(deployments.filter(d => ['queued', 'awaiting-health', 'running', 'waiting'].includes(d.state))); + let active = $derived(deployments.filter(d => !d.server_deleted_at && ['queued', 'awaiting-health', 'running', 'waiting'].includes(d.state))); const needsAttention = (deployment: Deployment) => ['failed', 'interrupted', 'warning'].includes(deployment.state); const attention = $derived(deployments.filter(needsAttention)); - let history = $derived(deployments.filter(d => !['queued', 'awaiting-health', 'running', 'waiting'].includes(d.state) && + let history = $derived(deployments.filter(d => (d.server_deleted_at || !['queued', 'awaiting-health', 'running', 'waiting'].includes(d.state)) && (historyState === 'all' || needsAttention(d)) && (action === 'all' || d.action === action) && (d.server + ' ' + serverTitle(d.server)).toLowerCase().includes(search.toLowerCase()))); async function load() { @@ -151,7 +151,7 @@ + {#if d.canCancelRemoval && !d.server_deleted_at}
{/if} {/snippet}
Map / serverMatch startedMatch status{leaderboardMetrics.find(metric => metric.key === key)!.label}
{match.mapName}{match.server}{date(match.startedAt)}
{match.mapName}{match.server} {#if match.server_deleted_at}Deleted{/if}{date(match.startedAt)} {match.status}{match.completed ? 'Completion credited' : 'No completion credit'}{number.format(match[key])}