josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Preserve deleted server history

Browse Source 
Retain historical server identities, results, and evidence after deletion. Mark archived entries as Deleted and prevent old work from targeting servers that reuse the same name.

Fixes #16.
This commit is contained in:
josh
2026-09-27 21:48:55 -04:00
parent 53b29f726c
commit 981fb27c1c
61 changed files with 929 additions and 223 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/beta-plan.ts
+2 -2
View File
@@ -82,9 +82,9 @@ export interface BetaTestView {
plan: BetaPlan;
outcome: string;
reason: string | null;
servers: Array<{ server: string; phase: number; passed: boolean; progress: RequirementProgress[]; observingAt?: string | null;
servers: Array<{ server: string; server_deleted_at: string | null; phase: number; passed: boolean; progress: RequirementProgress[]; observingAt?: string | null;
observation?: { lastGameplayAt: string | null; lastEvent: string | null; players: number | null; round: number | null; matchStarted: boolean; gameEnded: boolean; healthy: boolean | null; healthCheckedAt: string | null } }>;
evidence: Array<{ id: number; server: string; received_at: string; kind: string; payload: string }>;
evidence: Array<{ id: number; server: string; server_deleted_at: string | null; received_at: string; kind: string; payload: string }>;
}
export interface BetaPlanIssue { field: string; message: string; requirementId?: string }
src/lib/deployment-plan.ts
+2 -1
View File
@@ -66,9 +66,10 @@ export interface CollectionSummary {
export interface CollectionDetail {
collection: CollectionSummary;
retainedServers: string[];
retainedServerDeletedAt?: Record<string, string>;
queue: { concurrency: number; active: { id: string; server: string; state: string; stage: string }[]; queued: number };
servers: { id: string; server: string; action: 'create' | 'destroy'; port: number | null; map_slug: string | null; state: string; stage: string;
started_at: string; finished_at: string | null; error: string | null; server_exists: number; queuePosition: number | null }[];
started_at: string; finished_at: string | null; error: string | null; server_exists: number; server_deleted_at?: string | null; queuePosition: number | null }[];
}
export interface LaunchPreview {
src/lib/leaderboard.ts
+1
View File
@@ -22,6 +22,7 @@ export interface PlayerMapRecord extends PlayerStats {
export interface PlayerMatch {
id: number;
server: string;
server_deleted_at?: string | null;
mapName: string;
startedAt: string;
lastSeen: string;
src/lib/match-history.ts
+1
View File
@@ -4,6 +4,7 @@ export const MIN_RECORDED_ROUND = 2;
export interface ServerMatch {
id: number;
server: string;
server_deleted_at?: string | null;
map: string;
startedAt: string;
endedAt: string | null;
src/lib/plutonium.ts
+1
View File
@@ -22,6 +22,7 @@ export interface PlutoniumVersion {
export interface MaintenanceJob {
id: number;
server: string;
server_deleted_at?: string | null;
state: 'queued' | 'restarting' | 'done' | 'failed' | 'cancelled';
target_revision: number | null;
requested_at: string;
src/lib/server/db/index.ts
+4
View File
@@ -7,6 +7,7 @@ import { requirement, type BetaPlan, type BetaRequirement } from '../../beta-pla
import { emptyMetrics } from '../fleet/beta-evaluation.js';
import { backfillLeaderboard } from '../fleet/leaderboard-ingest.js';
import { initializeFleetSettings } from './fleet-settings.js';
import { migrateServerHistory } from './server-history.js';
let _db: Database.Database | null = null;
@@ -124,6 +125,8 @@ function backfillMultiScriptJoins(handle: Database.Database): void {
FROM beta_servers bs
JOIN betas b ON b.id = bs.beta_id
WHERE b.script_id IS NOT NULL
AND bs.server_deleted_at IS NULL
AND EXISTS (SELECT 1 FROM servers WHERE name = bs.server)
AND NOT EXISTS (
SELECT 1 FROM beta_server_scripts x
WHERE x.beta_id = bs.beta_id AND x.server = bs.server AND x.script_id = b.script_id
@@ -329,6 +332,7 @@ export function db(): Database.Database {
{ name: 'mode', ddl: "ALTER TABLE server_launch_collections ADD COLUMN mode TEXT NOT NULL DEFAULT 'fill'" },
{ name: 'retained_servers', ddl: "ALTER TABLE server_launch_collections ADD COLUMN retained_servers TEXT NOT NULL DEFAULT '[]'" }
]);
migrateServerHistory(handle);
migrateTable(handle, 'server_collection_previews', [
{ name: 'mode', ddl: "ALTER TABLE server_collection_previews ADD COLUMN mode TEXT NOT NULL DEFAULT 'fill'" },
{ name: 'fleet_snapshot', ddl: 'ALTER TABLE server_collection_previews ADD COLUMN fleet_snapshot TEXT' }
src/lib/server/db/schema.sql
+11 -4
View File
@@ -11,6 +11,7 @@ PRAGMA foreign_keys = ON;
CREATE TABLE IF NOT EXISTS leaderboard_matches (
id INTEGER PRIMARY KEY AUTOINCREMENT,
server TEXT NOT NULL,
server_deleted_at TEXT,
game TEXT NOT NULL,
map TEXT NOT NULL,
map_key TEXT NOT NULL,
@@ -181,7 +182,8 @@ CREATE INDEX IF NOT EXISTS idx_betas_script ON betas(script_id);
CREATE TABLE IF NOT EXISTS beta_servers (
beta_id INTEGER NOT NULL REFERENCES betas(id) ON DELETE CASCADE,
server TEXT NOT NULL REFERENCES servers(name),
server TEXT NOT NULL,
server_deleted_at TEXT,
had_original INTEGER NOT NULL, -- deprecated; see beta_server_scripts
prior_version_id INTEGER REFERENCES script_versions(id), -- deprecated; see beta_server_scripts
stage TEXT NOT NULL, -- 'deploying'|'awaiting-empty'|'restarting'|'awaiting-health'|'awaiting-match'|'ready'|'failed'
@@ -206,7 +208,7 @@ CREATE TABLE IF NOT EXISTS beta_test_plans (
CREATE TABLE IF NOT EXISTS beta_test_runs (
id TEXT PRIMARY KEY,
beta_id INTEGER NOT NULL REFERENCES betas(id) ON DELETE CASCADE,
server TEXT NOT NULL REFERENCES servers(name),
server TEXT NOT NULL,
phase INTEGER NOT NULL DEFAULT 0,
created_at TEXT NOT NULL,
observing_at TEXT,
@@ -261,7 +263,7 @@ CREATE INDEX IF NOT EXISTS idx_beta_scripts_script ON beta_scripts(script_id);
-- lives in beta_file_snapshots.
CREATE TABLE IF NOT EXISTS beta_server_scripts (
beta_id INTEGER NOT NULL REFERENCES betas(id) ON DELETE CASCADE,
server TEXT NOT NULL REFERENCES servers(name),
server TEXT NOT NULL,
script_id INTEGER NOT NULL REFERENCES scripts(id) ON DELETE CASCADE,
had_original INTEGER NOT NULL,
prior_version_id INTEGER REFERENCES script_versions(id),
@@ -302,6 +304,7 @@ CREATE INDEX IF NOT EXISTS idx_deployments_started ON deployments(started_at DES
CREATE TABLE IF NOT EXISTS deployment_servers (
deployment_id TEXT NOT NULL REFERENCES deployments(id) ON DELETE CASCADE,
server TEXT NOT NULL, -- Campaign observation survives a rebuild's registry gap.
server_deleted_at TEXT,
files_changed INTEGER NOT NULL,
restart_state TEXT NOT NULL, -- pending | done | failed | manual | held
players INTEGER,
@@ -373,6 +376,7 @@ CREATE TABLE IF NOT EXISTS plutonium_versions (
CREATE TABLE IF NOT EXISTS maintenance_jobs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
server TEXT NOT NULL,
server_deleted_at TEXT,
state TEXT NOT NULL DEFAULT 'queued',
target_revision INTEGER,
requested_at TEXT NOT NULL,
@@ -411,6 +415,7 @@ CREATE INDEX IF NOT EXISTS idx_telemetry_requests_received ON telemetry_requests
CREATE TABLE IF NOT EXISTS server_deployments (
id TEXT PRIMARY KEY,
server TEXT NOT NULL,
server_deleted_at TEXT,
action TEXT NOT NULL CHECK (action IN ('create', 'start', 'restart', 'rebuild', 'stop', 'destroy')),
state TEXT NOT NULL,
stage TEXT NOT NULL,
@@ -486,7 +491,8 @@ CREATE TABLE IF NOT EXISTS server_launch_collections (
name TEXT NOT NULL,
created_at TEXT NOT NULL,
mode TEXT NOT NULL DEFAULT 'fill',
retained_servers TEXT NOT NULL DEFAULT '[]'
retained_servers TEXT NOT NULL DEFAULT '[]',
retained_server_deletions TEXT NOT NULL DEFAULT '{}'
);
CREATE TABLE IF NOT EXISTS server_collection_members (
@@ -629,6 +635,7 @@ CREATE TABLE IF NOT EXISTS metric_samples (
CREATE TABLE IF NOT EXISTS watchdog_incidents (
id TEXT PRIMARY KEY,
server TEXT NOT NULL,
server_deleted_at TEXT,
kind TEXT NOT NULL CHECK (kind IN ('health','drift')),
state TEXT NOT NULL CHECK (state IN ('open','recovering','attention','resolved','dismissed')),
summary TEXT NOT NULL,
src/lib/server/db/server-history.ts
+130
View File
@@ -0,0 +1,130 @@
import type Database from 'better-sqlite3';
const historyTables = ['beta_servers', 'deployment_servers', 'server_deployments', 'leaderboard_matches', 'watchdog_incidents', 'maintenance_jobs'];
const now = "strftime('%Y-%m-%dT%H:%M:%fZ','now')";
/** Detach durable beta records without dropping their evidence, indexes, or activity triggers. */
function detachBetaHistory(handle: Database.Database): void {
for (const name of ['beta_servers', 'beta_test_runs', 'beta_server_scripts']) {
const { sql } = handle.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?").get(name) as { sql: string };
const replacement = sql.replace(/server\s+TEXT NOT NULL REFERENCES servers\(name\)/, 'server TEXT NOT NULL');
if (replacement === sql) continue;
const dependents = handle.prepare("SELECT sql FROM sqlite_master WHERE tbl_name = ? AND type IN ('index','trigger') AND sql IS NOT NULL")
.all(name) as { sql: string }[];
const columns = (handle.prepare(`PRAGMA table_info(${name})`).all() as { name: string }[]).map(col => col.name).join(', ');
handle.pragma('foreign_keys = OFF');
try {
handle.transaction(() => {
handle.exec(replacement.replace(name, name + '_history_new'));
handle.exec(`INSERT INTO ${name}_history_new (rowid, ${columns}) SELECT rowid, ${columns} FROM ${name};
DROP TABLE ${name}; ALTER TABLE ${name}_history_new RENAME TO ${name};`);
for (const dependent of dependents) handle.exec(dependent.sql);
if ((handle.pragma('foreign_key_check') as unknown[]).length) throw new Error('Server history migration found broken references');
})();
} finally { handle.pragma('foreign_keys = ON'); }
}
}
// A rebuild owns the replacement instance. Only its current operation and the
// Watchdog records observing it cross the deliberate gap in the live registry.
const removingRebuild = `SELECT d.id FROM server_deployments d
JOIN server_rebuild_jobs r ON r.deployment_id = d.id
WHERE d.server = OLD.name AND d.server_deleted_at IS NULL AND d.action = 'rebuild'
AND d.state IN ('waiting','running','failed','interrupted')
AND d.stage = 'Removing server files and registry' AND r.removed_at IS NULL
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server
AND newer.rowid > d.rowid)`;
/** Installed after additive migrations so existing databases can boot safely. */
export function migrateServerHistory(handle: Database.Database): void {
detachBetaHistory(handle);
handle.transaction(() => {
for (const table of historyTables) {
const columns = handle.prepare(`PRAGMA table_info(${table})`).all() as { name: string }[];
if (!columns.some(column => column.name === 'server_deleted_at')) handle.exec(`ALTER TABLE ${table} ADD COLUMN server_deleted_at TEXT`);
}
const collectionColumns = handle.prepare('PRAGMA table_info(server_launch_collections)').all() as { name: string }[];
if (!collectionColumns.some(column => column.name === 'retained_server_deletions')) {
handle.exec("ALTER TABLE server_launch_collections ADD COLUMN retained_server_deletions TEXT NOT NULL DEFAULT '{}'");
}
// Historical unfinished work cannot claim the next server with this name.
handle.exec(`DROP INDEX IF EXISTS idx_maintenance_pending;
CREATE UNIQUE INDEX idx_maintenance_pending ON maintenance_jobs(server)
WHERE state IN ('queued', 'restarting') AND server_deleted_at IS NULL;
DROP INDEX IF EXISTS idx_watchdog_active;
CREATE UNIQUE INDEX idx_watchdog_active ON watchdog_incidents(server,kind)
WHERE state != 'resolved' AND server_deleted_at IS NULL;
DROP TRIGGER IF EXISTS preserve_deleted_server_history;
CREATE TRIGGER preserve_deleted_server_history BEFORE DELETE ON servers
BEGIN
UPDATE server_launch_collections SET retained_server_deletions = json_patch(retained_server_deletions, json_object(OLD.name, ${now}))
WHERE EXISTS (SELECT 1 FROM json_each(retained_servers) WHERE value = OLD.name)
AND NOT EXISTS (SELECT 1 FROM json_each(retained_server_deletions) WHERE key = OLD.name);
UPDATE beta_servers SET server_deleted_at = ${now} WHERE server = OLD.name AND server_deleted_at IS NULL;
UPDATE deployment_servers SET server_deleted_at = ${now}
WHERE server = OLD.name AND server_deleted_at IS NULL AND deployment_id NOT IN (
SELECT campaign_id FROM watchdog_actions WHERE campaign_id IS NOT NULL AND operation_id IN (${removingRebuild}));
UPDATE watchdog_incidents SET server_deleted_at = ${now}
WHERE server = OLD.name AND server_deleted_at IS NULL AND id NOT IN (
SELECT incident_id FROM watchdog_actions WHERE operation_id IN (${removingRebuild}));
UPDATE maintenance_jobs SET server_deleted_at = ${now},
finished_at = CASE WHEN state IN ('queued','restarting') THEN ${now} ELSE finished_at END,
detail = CASE WHEN state IN ('queued','restarting') THEN COALESCE(detail || '; ', '') || 'Server deleted' ELSE detail END,
state = CASE WHEN state IN ('queued','restarting') THEN 'cancelled' ELSE state END
WHERE server = OLD.name AND server_deleted_at IS NULL;
UPDATE leaderboard_matches SET server_deleted_at = ${now},
ended_at = CASE WHEN status = 'active' THEN COALESCE(ended_at, ${now}) ELSE ended_at END,
status = CASE WHEN status = 'active' THEN 'interrupted' ELSE status END
WHERE server = OLD.name AND server_deleted_at IS NULL;
INSERT INTO server_deployment_events (deployment_id, created_at, message)
SELECT id, ${now}, 'Cancelled: server deleted' FROM server_deployments
WHERE server = OLD.name AND server_deleted_at IS NULL AND state IN ('queued','waiting','running','awaiting-health')
AND id NOT IN (${removingRebuild})
AND NOT (action = 'destroy' AND state = 'running' AND stage = 'Removing server files and registry');
UPDATE server_deployments SET state = 'cancelled', error = 'Server deleted', finished_at = ${now}
WHERE server = OLD.name AND server_deleted_at IS NULL AND state IN ('queued','waiting','running','awaiting-health')
AND id NOT IN (${removingRebuild})
AND NOT (action = 'destroy' AND state = 'running' AND stage = 'Removing server files and registry');
UPDATE server_deployments SET server_deleted_at = ${now}
WHERE server = OLD.name AND server_deleted_at IS NULL AND id NOT IN (${removingRebuild});
END;`);
if (!handle.prepare("SELECT 1 FROM settings WHERE key = 'server_history_v1'").get()) {
// Recover the availability of history already orphaned before this
// migration. Queued launches and rebuild registry gaps are not deletions.
const liveRebuilds = `SELECT d.id FROM server_deployments d JOIN server_rebuild_jobs r ON r.deployment_id = d.id
WHERE d.server_deleted_at IS NULL AND d.action = 'rebuild' AND (d.state IN ('waiting','queued','running','awaiting-health')
OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry'))
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`;
for (const table of historyTables) {
const exception = table === 'server_deployments'
? `AND (action != 'create' OR state IN ('done','warning') OR EXISTS (
SELECT 1 FROM server_deployments removal WHERE removal.server = server_deployments.server
AND removal.rowid > server_deployments.rowid AND removal.action = 'destroy' AND removal.state IN ('done','warning')))
AND id NOT IN (${liveRebuilds})`
: table === 'deployment_servers'
? `AND deployment_id NOT IN (SELECT campaign_id FROM watchdog_actions WHERE campaign_id IS NOT NULL AND operation_id IN (${liveRebuilds}))`
: table === 'watchdog_incidents'
? `AND id NOT IN (SELECT incident_id FROM watchdog_actions WHERE operation_id IN (${liveRebuilds}))` : '';
handle.exec(`UPDATE ${table} SET server_deleted_at = ${now} WHERE server_deleted_at IS NULL
AND NOT EXISTS (SELECT 1 FROM servers WHERE name = ${table}.server AND destroyed_at IS NULL) ${exception}`);
}
handle.exec(`UPDATE server_launch_collections SET retained_server_deletions = json_patch(retained_server_deletions,
(SELECT json_group_object(value, ${now}) FROM json_each(retained_servers) retained
WHERE NOT EXISTS (SELECT 1 FROM servers WHERE name = retained.value AND destroyed_at IS NULL)
AND NOT EXISTS (SELECT 1 FROM server_deployments d WHERE d.server = retained.value
AND d.server_deleted_at IS NULL AND d.action IN ('create','rebuild')
AND d.state IN ('queued','running','awaiting-health','waiting'))
AND NOT EXISTS (SELECT 1 FROM json_each(retained_server_deletions) WHERE key = retained.value)));`);
handle.exec(`UPDATE maintenance_jobs SET state = 'cancelled', finished_at = ${now}, detail = COALESCE(detail || '; ', '') || 'Server deleted'
WHERE server_deleted_at IS NOT NULL AND state IN ('queued','restarting');
UPDATE leaderboard_matches SET status = 'interrupted', ended_at = COALESCE(ended_at, server_deleted_at)
WHERE server_deleted_at IS NOT NULL AND status = 'active';
INSERT INTO server_deployment_events (deployment_id, created_at, message)
SELECT id, ${now}, 'Cancelled: server deleted' FROM server_deployments
WHERE server_deleted_at IS NOT NULL AND state IN ('queued','waiting','running','awaiting-health');
UPDATE server_deployments SET state = 'cancelled', error = 'Server deleted', finished_at = ${now}
WHERE server_deleted_at IS NOT NULL AND state IN ('queued','waiting','running','awaiting-health');
INSERT INTO settings (key, value) VALUES ('server_history_v1', '1');`);
}
})();
}
src/lib/server/fleet/beta-tests.ts
+12 -4
View File
@@ -9,6 +9,8 @@ import { applySample, applyGameEvent, emptyMetrics, evaluateRequirements, type T
export interface TestRun { id: string; beta_id: number; server: string; phase: number; created_at: string; observing_at: string | null; metrics: string }
export interface StoredPlan { beta_id: number; plan: string; outcome: string; reason: string | null }
const nowIso = () => new Date().toISOString();
const activeParticipant = (id: number, server: string) => !!db().prepare(`SELECT 1 FROM beta_servers
WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL`).get(id, server);
export const storedPlan = (id: number) => db().prepare('SELECT * FROM beta_test_plans WHERE beta_id = ?').get(id) as StoredPlan | undefined;
export const currentRun = (id: number, server: string) => db().prepare('SELECT * FROM beta_test_runs WHERE beta_id = ? AND server = ? AND retired_at IS NULL').get(id, server) as TestRun | undefined;
export function activity(id: number, message: string) { db().prepare('INSERT INTO beta_events (beta_id, message) VALUES (?, ?)').run(id, message); }
@@ -27,6 +29,7 @@ export function evidence(run: TestRun, eventId: string, kind: string, payload: u
/** Internal progress record; never sent to or required from the game script. */
export async function prepareRun(betaId: number, server: string, phase = 0): Promise<TestRun> {
if (!activeParticipant(betaId, server)) throw new Error('Original test server was deleted; start a new beta');
db().transaction(() => {
db().prepare('UPDATE beta_test_runs SET retired_at = ? WHERE beta_id = ? AND server = ? AND retired_at IS NULL').run(nowIso(), betaId, server);
db().prepare('INSERT INTO beta_test_runs (id, beta_id, server, phase, created_at, metrics) VALUES (?, ?, ?, ?, ?, ?)')
@@ -41,6 +44,7 @@ export function recordGameEvent(event: GameTelemetry): void {
JOIN beta_test_plans p ON p.beta_id = b.id
JOIN beta_servers s ON s.beta_id = b.id AND s.server = r.server
WHERE r.server = ? AND r.retired_at IS NULL AND b.state IN ('running','ready-promote')
AND s.server_deleted_at IS NULL
AND p.outcome IN ('active','passed') AND s.stage IN ('awaiting-health','observing','ready')
AND s.restarted_at IS NOT NULL`).all(event.server) as TestRun[];
for (const run of runs) {
@@ -53,6 +57,7 @@ export function recordGameEvent(event: GameTelemetry): void {
}
export async function observeRun(run: TestRun, plan: BetaPlan): Promise<void> {
if (!activeParticipant(run.beta_id, run.server)) return;
const m = readMetrics(run);
const row = getServer(run.server);
const server = row ? serverObservation(row) : null;
@@ -74,19 +79,22 @@ export function testView(betaId: number): BetaTestView | null {
const stored = storedPlan(betaId);
if (!stored) return null;
const plan = parseBetaPlan(JSON.parse(stored.plan));
const runs = db().prepare('SELECT * FROM beta_test_runs WHERE beta_id = ? AND retired_at IS NULL ORDER BY server').all(betaId) as TestRun[];
const runs = db().prepare(`SELECT r.*, s.server_deleted_at FROM beta_test_runs r
JOIN beta_servers s ON s.beta_id = r.beta_id AND s.server = r.server
WHERE r.beta_id = ? AND r.retired_at IS NULL ORDER BY r.server`).all(betaId) as Array<TestRun & { server_deleted_at: string | null }>;
return { plan, outcome: stored.outcome, reason: stored.reason,
servers: runs.map(run => {
const metrics = readMetrics(run);
const progress = evaluateRequirements(plan, metrics, run.phase);
const latest = db().prepare("SELECT received_at, kind, payload FROM beta_test_evidence WHERE run_id = ? AND kind IN ('game_start','round_change','heartbeat','game_end') ORDER BY id DESC LIMIT 1").get(run.id) as { received_at: string; kind: string; payload: string } | undefined;
const event = latest ? JSON.parse(latest.payload) : null;
return { server: run.server, phase: run.phase, passed: progress.every(r => r.passed), progress, observingAt: run.observing_at,
return { server: run.server, server_deleted_at: run.server_deleted_at, phase: run.phase, passed: progress.every(r => r.passed), progress, observingAt: run.observing_at,
observation: { lastGameplayAt: latest?.received_at ?? null, lastEvent: latest?.kind ?? null,
players: metrics.match?.players ?? null, round: event?.round ?? null,
matchStarted: metrics.match?.started !== undefined, gameEnded: metrics.match?.ended ?? false,
healthy: metrics.wasHealthy ?? null, healthCheckedAt: metrics.lastSample ? new Date(metrics.lastSample).toISOString() : null } };
}),
evidence: db().prepare(`SELECT e.id, r.server, e.received_at, e.kind, e.payload FROM beta_test_evidence e
JOIN beta_test_runs r ON r.id = e.run_id WHERE r.beta_id = ? ORDER BY e.id DESC LIMIT 200`).all(betaId) as BetaTestView['evidence'] };
evidence: db().prepare(`SELECT e.id, r.server, s.server_deleted_at, e.received_at, e.kind, e.payload FROM beta_test_evidence e
JOIN beta_test_runs r ON r.id = e.run_id JOIN beta_servers s ON s.beta_id = r.beta_id AND s.server = r.server
WHERE r.beta_id = ? ORDER BY e.id DESC LIMIT 200`).all(betaId) as BetaTestView['evidence'] };
}
src/lib/server/fleet/beta.ts
+45 -4
View File
@@ -148,6 +148,7 @@ export interface BetaRow {
export interface BetaServerRow {
beta_id: number;
server: string;
server_deleted_at: string | null;
had_original: number; // deprecated; per-script data lives in beta_server_scripts
prior_version_id: number | null; // deprecated
stage: BetaStage;
@@ -225,6 +226,13 @@ export function listBetaServers(betaId: number): BetaServerRow[] {
.all(betaId) as BetaServerRow[];
}
/** History retains names, so operational callers must also check the participant's identity. */
export function isActiveBetaServer(betaId: number, server: string): boolean {
return !!db().prepare(`SELECT 1 FROM beta_servers bs JOIN servers s ON s.name = bs.server
WHERE bs.beta_id = ? AND bs.server = ? AND bs.server_deleted_at IS NULL AND s.destroyed_at IS NULL`)
.get(betaId, server);
}
export function listBetaScripts(betaId: number): BetaScriptRow[] {
return db()
.prepare('SELECT * FROM beta_scripts WHERE beta_id = ? ORDER BY script_id')
@@ -456,6 +464,9 @@ export async function deployBetaServers(betaId: number, names: string[]): Promis
return exclusive(betaId, () => deployBetaServersInternal(betaId, names));
}
async function deployBetaServersInternal(betaId: number, names: string[]): Promise<void> {
for (const name of names) {
if (!isActiveBetaServer(betaId, name)) throw new BetaError(`${name}: original beta server was deleted; start a new beta`, 'unknown-server');
}
const resolved = listBetaScripts(betaId).map(r => ({ script: getScript(r.script_id)!, version: getVersion(r.script_version_id)! }));
const serverRows = names.map(name => getServer(name)!);
// Split resolved artifacts into non-collapsing (each writes its own file)
@@ -484,10 +495,11 @@ async function deployBetaServersInternal(betaId: number, names: string[]): Promi
const updateStage = db().prepare(
`UPDATE beta_servers
SET stage = ?, error = ?, deployed_at = ?, updated_at = ?
WHERE beta_id = ? AND server = ?`
WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL`
);
for (const name of names) {
if (!isActiveBetaServer(betaId, name)) continue;
db().prepare("UPDATE beta_servers SET stage = 'deploying', deployed_at = ? WHERE beta_id = ? AND server = ?")
.run(nowIso(), betaId, name);
const serverRow = serverRows.find((r) => r.name === name)!;
@@ -503,6 +515,7 @@ async function deployBetaServersInternal(betaId: number, names: string[]): Promi
await mkdir(join(dest, dirname(script.target_rel_path)), { recursive: true });
if (existsSync(targetPath)) {
const priorBytes = await readFile(targetPath);
if (!isActiveBetaServer(betaId, name)) break;
insertSnapshot.run(
betaId,
name,
@@ -514,7 +527,9 @@ async function deployBetaServersInternal(betaId: number, names: string[]): Promi
db().prepare('UPDATE beta_server_scripts SET had_original = 1 WHERE beta_id = ? AND server = ? AND script_id = ?').run(betaId, name, script.id);
}
db().prepare('INSERT OR IGNORE INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, ?)').run(betaId, name, script.target_rel_path);
if (!isActiveBetaServer(betaId, name)) break;
await writeFile(targetPath, version.content);
if (!isActiveBetaServer(betaId, name)) break;
recordScript(name, script.target_rel_path, version.id, version.sha256, version.size, `beta:${betaId}`);
} catch (err) {
firstErr = `${script.name}: ${(err as Error).message}`;
@@ -541,6 +556,7 @@ async function deployBetaServersInternal(betaId: number, names: string[]): Promi
const targetPath = join(dest, targetRel);
if (existsSync(targetPath)) {
const priorBytes = await readFile(targetPath);
if (!isActiveBetaServer(betaId, name)) continue;
insertSnapshot.run(
betaId,
name,
@@ -552,7 +568,9 @@ async function deployBetaServersInternal(betaId: number, names: string[]): Promi
}
const composedBuf = Buffer.from(composedResult.text, 'utf8');
db().prepare('INSERT OR IGNORE INTO beta_file_changes (beta_id, server, rel_path) VALUES (?, ?, ?)').run(betaId, name, targetRel);
if (!isActiveBetaServer(betaId, name)) continue;
await writeFile(targetPath, composedBuf);
if (!isActiveBetaServer(betaId, name)) continue;
recordScript(
name,
targetRel,
@@ -568,6 +586,7 @@ async function deployBetaServersInternal(betaId: number, names: string[]): Promi
}
}
const t = nowIso();
if (!isActiveBetaServer(betaId, name)) continue;
if (firstErr) {
updateStage.run('failed', `deploy: ${firstErr}`, t, t, betaId, name);
emit({ type: 'beta-server-stage', betaId, server: name, stage: 'failed', error: `deploy: ${firstErr}` });
@@ -619,8 +638,13 @@ async function rollbackBetaInternal(betaId: number): Promise<RollbackResult> {
);
for (const bs of servers) {
if (!isActiveBetaServer(betaId, bs.server)) {
warnings.push(`${bs.server}: original server was deleted; retained history was skipped`);
continue;
}
if (!bs.deployed_at && bs.stage === 'queued') continue;
for (const { script } of bundle) {
if (!isActiveBetaServer(betaId, bs.server)) break;
if (isCollapsingKind(script.kind)) continue; // handled once per server below
if (storedPlan(betaId) && !db().prepare('SELECT 1 FROM beta_file_changes WHERE beta_id = ? AND server = ? AND rel_path = ?').get(betaId, bs.server, script.target_rel_path)) continue;
const dest = destOfKind(bs.server, script.kind);
@@ -633,7 +657,9 @@ async function rollbackBetaInternal(betaId: number): Promise<RollbackResult> {
| undefined;
if (snap) {
await mkdir(join(dest, dirname(script.target_rel_path)), { recursive: true });
if (!isActiveBetaServer(betaId, bs.server)) break;
await writeFile(targetPath, snap.prior_content);
if (!isActiveBetaServer(betaId, bs.server)) break;
if (bss.prior_version_id) {
recordScript(bs.server, script.target_rel_path, bss.prior_version_id, snap.prior_sha256, snap.prior_size, `rollback:${betaId}`);
} else {
@@ -646,12 +672,14 @@ async function rollbackBetaInternal(betaId: number): Promise<RollbackResult> {
} else {
// No prior file — the beta wrote a fresh file. Drop it.
if (existsSync(targetPath)) await rm(targetPath, { force: true });
if (!isActiveBetaServer(betaId, bs.server)) break;
removeScript(bs.server, script.target_rel_path);
dropped.add(bs.server);
}
}
// Collapsing config layer: single restore for the shared cfg file.
if (!isActiveBetaServer(betaId, bs.server)) continue;
if (hasCollapsing && (!storedPlan(betaId) || db().prepare('SELECT 1 FROM beta_file_changes WHERE beta_id = ? AND server = ? AND rel_path = ?').get(betaId, bs.server, configFileOf(bs.server)))) {
const dest = destOfKind(bs.server, 'config');
const targetRel = configFileOf(bs.server);
@@ -661,7 +689,9 @@ async function rollbackBetaInternal(betaId: number): Promise<RollbackResult> {
| undefined;
if (snap) {
await mkdir(dest, { recursive: true });
if (!isActiveBetaServer(betaId, bs.server)) continue;
await writeFile(targetPath, snap.prior_content);
if (!isActiveBetaServer(betaId, bs.server)) continue;
recordScript(bs.server, targetRel, null, snap.prior_sha256, snap.prior_size, `rollback:${betaId}`);
restored.add(bs.server);
} else if (existsSync(targetPath)) {
@@ -669,6 +699,7 @@ async function rollbackBetaInternal(betaId: number): Promise<RollbackResult> {
// no pre-existing file. Remove the beta-written cfg to leave
// the server clean.
await rm(targetPath, { force: true });
if (!isActiveBetaServer(betaId, bs.server)) continue;
removeScript(bs.server, targetRel);
dropped.add(bs.server);
}
@@ -738,7 +769,8 @@ function promotionPlan(betaId: number): { preview: BetaPromotionPreview; configC
return { script, version };
});
if (!bundle.length) throw new BetaError('This beta has no files.', 'no-scripts');
const tested = new Set(listBetaServers(betaId).map(row => row.server));
const participants = listBetaServers(betaId);
const tested = new Set(participants.filter(row => isActiveBetaServer(betaId, row.server)).map(row => row.server));
const targets = (script: ScriptRow) => [...new Set([...tested, ...(script.scope_type && script.scope_value
? serversMatchingScope(script.scope_type, script.scope_value) : [])])].sort();
const files = bundle.map(({ script, version }) => ({ id: script.id, name: script.name, kind: script.kind,
@@ -746,7 +778,8 @@ function promotionPlan(betaId: number): { preview: BetaPromotionPreview; configC
release_version: version.release_version, sha256: version.sha256,
scope: script.scope_type ? `${script.scope_type}: ${script.scope_value}` : 'Selected test servers only', servers: targets(script) }));
const names = [...new Set(files.flatMap(file => file.servers))].sort();
const blockers: string[] = [];
const blockers: string[] = participants.filter(row => !isActiveBetaServer(betaId, row.server))
.map(row => `${row.server}: original test server was deleted; start a new beta before promoting.`);
const servers = names.map(name => {
const server = getServer(name);
if (!server || server.destroyed_at) blockers.push(`${name}: server is no longer available.`);
@@ -810,7 +843,7 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
version: getVersion(br.script_version_id)!
}));
const betaServerSet = new Set(listBetaServers(betaId).map((r) => r.server));
const betaServerSet = new Set(listBetaServers(betaId).filter(row => isActiveBetaServer(betaId, row.server)).map((r) => r.server));
// Split bundle into non-collapsing (per-file) and collapsing (composed cfg).
const nonCollapsing = bundle.filter((b) => !isCollapsingKind(b.script.kind));
@@ -944,12 +977,16 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
// the captured target lists/content so later membership changes never expand it.
checkReview();
for (const p of plans) {
checkReview();
const dest = destOfKind(p.server, p.script.kind);
try {
await mkdir(join(dest, dirname(p.script.target_rel_path)), { recursive: true });
checkReview();
await writeFile(join(dest, p.script.target_rel_path), p.version.content);
checkReview();
planWritten.set(p, true);
} catch (err) {
if (err instanceof BetaError) throw err;
planWritten.set(p, false);
if (!serverErr.has(p.server)) serverErr.set(p.server, `${p.script.name}: ${(err as Error).message}`);
}
@@ -957,16 +994,20 @@ async function promoteBetaInternal(betaId: number, reviewedToken?: string): Prom
// Write composed cfg to each target.
const cfgWritten = new Map<CfgPlan, boolean>();
for (const cp of cfgPlans) {
checkReview();
try {
const dest = destOfKind(cp.server, 'config');
await mkdir(dest, { recursive: true });
checkReview();
const targetRel = configFileOf(cp.server);
const targetPath = join(dest, targetRel);
// Beta server already has composed content at targetPath; overwrite
// idempotently. Scope-expansion server: write in place.
await writeFile(targetPath, cp.composedContent);
checkReview();
cfgWritten.set(cp, true);
} catch (err) {
if (err instanceof BetaError) throw err;
cfgWritten.set(cp, false);
const msg = `configs: ${(err as Error).message}`;
if (!serverErr.has(cp.server)) serverErr.set(cp.server, msg);
src/lib/server/fleet/collections.ts
+11 -11
View File
@@ -17,8 +17,8 @@ function coverage(entries: BlueprintEntry[]) {
const registered = db().prepare('SELECT name, game, map_slug, map FROM servers WHERE destroyed_at IS NULL').all() as Server[];
const queued = db().prepare(`SELECT j.server AS name, substr(j.server, 1, 2) AS game, j.map_slug, j.expected_map AS map
FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
WHERE d.state IN ('queued', 'running', 'awaiting-health')`).all() as Server[];
const removing = new Set((db().prepare("SELECT server FROM server_deployments WHERE action = 'destroy' AND state IN ('waiting', 'running')").all() as { server: string }[]).map(row => row.server));
WHERE d.server_deleted_at IS NULL AND d.state IN ('queued', 'running', 'awaiting-health')`).all() as Server[];
const removing = new Set((db().prepare("SELECT server FROM server_deployments WHERE server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')").all() as { server: string }[]).map(row => row.server));
// A provisioning server can appear in both tables; count its identity once.
const servers = [...new Map([...queued, ...registered].map(server => [server.name, server])).values()];
return entries.map(entry => {
@@ -35,7 +35,7 @@ function coverage(entries: BlueprintEntry[]) {
return matches.length === 1 && matches[0].slug === entry.recipe.mapSlug;
}).map(server => server.name).sort();
const coverage = existingServers.map(name => {
const operation = db().prepare("SELECT id, state FROM server_deployments WHERE server = ? AND action IN ('create','rebuild') AND state IN ('queued', 'running', 'awaiting-health') ORDER BY rowid DESC LIMIT 1").get(name) as { id: string; state: string } | undefined;
const operation = db().prepare("SELECT id, state FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('create','rebuild') AND state IN ('queued', 'running', 'awaiting-health') ORDER BY rowid DESC LIMIT 1").get(name) as { id: string; state: string } | undefined;
const server = getServer(name);
return { name, health: operation ? operation.state === 'queued' ? 'queued' : 'starting'
: server?.container_state && ['stopped', 'exited', 'created', 'dead'].includes(server.container_state) ? 'stopped'
@@ -49,8 +49,8 @@ function coverage(entries: BlueprintEntry[]) {
function fleetSnapshot() {
return housePartyTargets().map(name => ({ name,
server: db().prepare('SELECT game, map_slug, map, game_port FROM servers WHERE name = ?').get(name) ?? null,
creation: db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action IN ('create','rebuild') ORDER BY rowid DESC LIMIT 1").get(name) ?? null,
removal: db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) ?? null
creation: db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('create','rebuild') ORDER BY rowid DESC LIMIT 1").get(name) ?? null,
removal: db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) ?? null
}));
}
@@ -149,8 +149,8 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
const removalJobs = removals.map((server, position) => {
const ports = (db().prepare(`SELECT game_port AS port FROM servers WHERE name = ? UNION
SELECT j.port FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
WHERE j.server = ? AND d.state IN ('queued', 'running', 'awaiting-health')`).all(server.name, server.name) as { port: number | null }[]).map(row => row.port);
const queued = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'create' AND state = 'queued'").get(server.name) as { id: string } | undefined;
WHERE j.server = ? AND d.server_deleted_at IS NULL AND d.state IN ('queued', 'running', 'awaiting-health')`).all(server.name, server.name) as { port: number | null }[]).map(row => row.port);
const queued = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'create' AND state = 'queued'").get(server.name) as { id: string } | undefined;
let id: string;
if (queued && !getServer(server.name)) {
// No container or files exist yet: cancel the reservation instead of booting just to remove it.
@@ -210,13 +210,13 @@ export function listCollections(): CollectionSummary[] {
export function getCollection(id: string): CollectionDetail | null {
const collection = db().prepare(summarySql + ' WHERE c.id = ? GROUP BY c.id').get(id) as CollectionSummary | undefined;
if (!collection) return null;
const servers = db().prepare(`SELECT d.id, d.server, d.action, d.state, d.stage, d.started_at, d.finished_at, d.error, j.port, j.map_slug,
EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server) AS server_exists
const servers = db().prepare(`SELECT d.id, d.server, d.server_deleted_at, d.action, d.state, d.stage, d.started_at, d.finished_at, d.error, j.port, j.map_slug,
(d.server_deleted_at IS NULL AND EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server AND s.destroyed_at IS NULL)) AS server_exists
FROM (${membershipSql}) m JOIN server_deployments d ON d.id = m.deployment_id
LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE m.collection_id = ? ORDER BY d.action DESC, m.position`).all(id) as CollectionDetail['servers'];
const queue = creationQueue();
const positions = new Map(queue.queued.map((job, index) => [job.id, index + 1]));
const stored = db().prepare('SELECT retained_servers FROM server_launch_collections WHERE id = ?').get(id) as { retained_servers: string };
return { collection, retainedServers: JSON.parse(stored.retained_servers), queue: { concurrency: fleetSettings().createConcurrency, active: queue.active, queued: queue.queued.length },
const stored = db().prepare('SELECT retained_servers, retained_server_deletions FROM server_launch_collections WHERE id = ?').get(id) as { retained_servers: string; retained_server_deletions: string };
return { collection, retainedServers: JSON.parse(stored.retained_servers), retainedServerDeletedAt: JSON.parse(stored.retained_server_deletions), queue: { concurrency: fleetSettings().createConcurrency, active: queue.active, queued: queue.queued.length },
servers: servers.map(server => ({ ...server, queuePosition: positions.get(server.id) ?? null })) };
}
src/lib/server/fleet/drift.ts
+1 -1
View File
@@ -54,7 +54,7 @@ function toPosix(p: string): string {
function hasActiveBeta(server: string): boolean {
return !!db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
WHERE bs.server = ? AND b.state IN ('running', 'ready-promote') LIMIT 1`).get(server);
WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running', 'ready-promote') LIMIT 1`).get(server);
}
async function requireServerDirectory(server: string): Promise<void> {
src/lib/server/fleet/leaderboard-ingest.ts
+1 -1
View File
@@ -51,7 +51,7 @@ function fingerprint(report: GameTelemetry): string {
export function recordLeaderboardTelemetry(handle: Database.Database, report: GameTelemetry): void {
const identity = mapIdentity(handle, report);
const signature = fingerprint(report);
let match = handle.prepare('SELECT * FROM leaderboard_matches WHERE server = ? ORDER BY id DESC LIMIT 1')
let match = handle.prepare('SELECT * FROM leaderboard_matches WHERE server = ? AND server_deleted_at IS NULL ORDER BY id DESC LIMIT 1')
.get(report.server) as Match | undefined;
if (match?.last_fingerprint === signature && match.map_key === identity.key) return;
src/lib/server/fleet/match-history.ts
+1 -1
View File
@@ -6,7 +6,7 @@ export function serverMatchHistory(server?: string, limit = 25): { matches: Serv
const where = `WHERE m.highest_round >= ${MIN_RECORDED_ROUND}${server === undefined ? '' : ' AND m.server = ?'}`;
const parameters = server === undefined ? [] : [server];
const total = (handle.prepare(`SELECT COUNT(*) AS total FROM leaderboard_matches m ${where}`).get(...parameters) as { total: number }).total;
const matches = handle.prepare(`SELECT m.id, m.server, m.map_name AS map, m.started_at AS startedAt, m.status,
const matches = handle.prepare(`SELECT m.id, m.server, m.server_deleted_at, m.map_name AS map, m.started_at AS startedAt, m.status,
CASE WHEN m.status = 'completed' THEN m.ended_at END AS endedAt,
m.start_players AS startPlayers,
(SELECT COUNT(*) FROM leaderboard_match_players p WHERE p.match_id = m.id) AS uniquePlayers,
src/lib/server/fleet/overview.ts
+5 -5
View File
@@ -14,7 +14,7 @@ export function fleetOperations(now = Date.now()): FleetOperation[] {
const campaigns = handle.prepare(`SELECT d.id, d.action, d.started_at, ds.server,
ds.restart_state, ds.error, ds.restarted_at, ds.players
FROM deployments d JOIN deployment_servers ds ON ds.deployment_id = d.id
WHERE ds.restart_state != 'done' OR ds.checked_at >= ? OR d.finished_at >= ? OR d.started_at >= ?`).all(since, since, since) as {
WHERE ds.server_deleted_at IS NULL AND (ds.restart_state != 'done' OR ds.checked_at >= ? OR d.finished_at >= ? OR d.started_at >= ?)`).all(since, since, since) as {
id: string; action: string; started_at: string; server: string; restart_state: string;
error: string | null; restarted_at: string | null; players: number | null;
}[];
@@ -35,10 +35,10 @@ export function fleetOperations(now = Date.now()): FleetOperation[] {
}
// Only the newest lifecycle operation can still require action on a server.
const lifecycle = handle.prepare(`SELECT d.* FROM server_deployments d
WHERE d.state IN ('queued', 'running', 'waiting', 'awaiting-health')
WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued', 'running', 'waiting', 'awaiting-health')
OR (d.state IN ('done', 'warning', 'cancelled') AND d.finished_at >= ?)
OR (d.state IN ('failed', 'interrupted', 'warning') AND NOT EXISTS
(SELECT 1 FROM server_deployments n WHERE n.server = d.server AND n.rowid > d.rowid))`)
(SELECT 1 FROM server_deployments n WHERE n.server = d.server AND n.rowid > d.rowid)))`)
.all(since) as { id: string; server: string; action: string; state: string; stage: string;
error: string | null; started_at: string }[];
for (const row of lifecycle) {
@@ -52,7 +52,7 @@ export function fleetOperations(now = Date.now()): FleetOperation[] {
detail: row.error ?? row.stage, href: `/provisioning/${encodeURIComponent(row.id)}` });
}
const betas = handle.prepare(`SELECT b.id, b.opened_at, bs.server, bs.stage, bs.error
FROM betas b JOIN beta_servers bs ON bs.beta_id = b.id WHERE b.state IN ('running', 'ready-promote')`)
FROM betas b JOIN beta_servers bs ON bs.beta_id = b.id WHERE bs.server_deleted_at IS NULL AND b.state IN ('running', 'ready-promote')`)
.all() as { id: number; opened_at: string; server: string; stage: string; error: string | null }[];
const stages: Record<string, string> = { queued: 'Waiting for an earlier test wave', deploying: 'Deploying beta',
'awaiting-empty': 'Waiting for an empty lobby', restarting: 'Restarting', 'awaiting-health': 'Recovering',
@@ -64,7 +64,7 @@ export function fleetOperations(now = Date.now()): FleetOperation[] {
state: row.stage === 'failed' ? 'attention' : 'active', label: stages[row.stage] ?? row.stage,
detail: row.error ?? recovering.get(row.server)?.error ?? `Beta ${row.id}`, href: `/betas/${row.id}` });
}
const maintenance = handle.prepare(`SELECT m.* FROM maintenance_jobs m WHERE m.state = 'failed'
const maintenance = handle.prepare(`SELECT m.* FROM maintenance_jobs m WHERE m.server_deleted_at IS NULL AND m.state = 'failed'
AND NOT EXISTS (SELECT 1 FROM maintenance_jobs n WHERE n.server = m.server AND n.kind = m.kind AND n.id > m.id)`)
.all() as { id: number; server: string; kind: string; detail: string | null; requested_at: string }[];
for (const job of maintenance) {
src/lib/server/fleet/player-profile.ts
+1 -1
View File
@@ -25,7 +25,7 @@ export function getPlayerProfile(game: string, guid: string, requestedOffset = 0
MAX(mp.last_seen_at) AS lastSeen, ${totals} ${scope}
GROUP BY m.map_key ORDER BY highestRound DESC, kills DESC, name COLLATE NOCASE, key`)
.all(game, guid) as PlayerMapRecord[];
const matches = handle.prepare(`SELECT m.id, m.server, m.map_name AS mapName,
const matches = handle.prepare(`SELECT m.id, m.server, m.server_deleted_at, m.map_name AS mapName,
m.started_at AS startedAt, mp.last_seen_at AS lastSeen, m.status, mp.completed,
mp.highest_round AS highestRound, mp.rounds_survived AS roundsSurvived,
mp.kills, mp.headshots, mp.revives, mp.downs, mp.best_score AS bestScore
src/lib/server/fleet/provisioning.ts
+1 -1
View File
@@ -3,7 +3,7 @@ import { db } from '../db/index.js';
/** Only the latest lifecycle operation can describe the current server. */
export function isProvisioning(server: string): boolean {
const operation = db().prepare(`SELECT action, state FROM server_deployments
WHERE server = ? ORDER BY rowid DESC LIMIT 1`).get(server) as
WHERE server = ? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1`).get(server) as
{ action: string; state: string } | undefined;
return !!operation && ['create', 'rebuild', 'start'].includes(operation.action) && ['queued', 'running', 'awaiting-health'].includes(operation.state);
}
src/lib/server/fleet/rollback.ts
+14 -3
View File
@@ -100,8 +100,10 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
const fileRows = db()
.prepare(
`SELECT id, server, verb, rel_path, sha256_before, sha256_after, content_before, content_after
FROM deployment_files WHERE deployment_id = ? ORDER BY server, rel_path`
`SELECT f.id, f.server, f.verb, f.rel_path, f.sha256_before, f.sha256_after, f.content_before, f.content_after
FROM deployment_files f JOIN deployment_servers ds ON ds.deployment_id = f.deployment_id AND ds.server = f.server
JOIN servers s ON s.name = f.server
WHERE f.deployment_id = ? AND ds.server_deleted_at IS NULL AND s.destroyed_at IS NULL ORDER BY f.server, f.rel_path`
)
.all(promoteId) as FileRow[];
@@ -148,20 +150,27 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
const affectedServers: string[] = [];
const perServerFailures = new Map<string, string>();
const active = (server: string) => !!db().prepare(`SELECT 1 FROM deployment_servers ds JOIN servers s ON s.name = ds.server
WHERE ds.deployment_id = ? AND ds.server = ? AND ds.server_deleted_at IS NULL AND s.destroyed_at IS NULL`).get(promoteId, server);
for (const [server, files] of perServer) {
if (!active(server)) continue;
try {
for (const f of files) {
if (!active(server)) break;
const kind = kindForRelPath(f.rel_path);
const dest = destOfKind(server, kind);
const abs = join(dest, f.rel_path);
await mkdir(join(dest, dirname(f.rel_path)), { recursive: true });
if (!active(server)) break;
const owningScript = scriptByTarget.get(f.rel_path);
if (f.verb === 'added') {
if (existsSync(abs)) await rm(abs, { force: true });
if (!active(server)) break;
removeScript(server, f.rel_path);
} else if (f.verb === 'updated' && f.content_before) {
await writeFile(abs, f.content_before);
if (!active(server)) break;
const priorVer = owningScript?.prior_version_id
? getVersion(owningScript.prior_version_id)
: null;
@@ -175,10 +184,11 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
);
} else if (f.verb === 'removed' && f.content_before) {
await writeFile(abs, f.content_before);
if (!active(server)) break;
recordScript(server, f.rel_path, null, f.sha256_before!, f.content_before.byteLength, `rollback:${newDepId}`);
}
}
affectedServers.push(server);
if (active(server)) affectedServers.push(server);
} catch (err) {
perServerFailures.set(server, (err as Error).message);
}
@@ -220,6 +230,7 @@ export async function rollbackPromoteDeployment(promoteId: string): Promise<Roll
);
}
for (const [server, files] of perServer) {
if (!active(server)) continue;
if (perServerFailures.has(server)) {
insertDepServer.run(newDepId, server, 0, 'failed', perServerFailures.get(server)!);
continue;
src/lib/server/fleet/servers.ts
+2 -2
View File
@@ -207,7 +207,7 @@ export function heldByBeta(server: string): Map<string, number> {
JOIN betas b ON b.id = bsv.beta_id
JOIN beta_scripts bs ON bs.beta_id = b.id
JOIN scripts s ON s.id = bs.script_id
WHERE bsv.server = ? AND b.state IN ('running','ready-promote')`
WHERE bsv.server = ? AND bsv.server_deleted_at IS NULL AND b.state IN ('running','ready-promote')`
)
.all(server) as Array<{ target: string; beta_id: number }>;
const map = new Map<string, number>();
@@ -226,7 +226,7 @@ export function ownerOf(server: string, target: string): number | null {
JOIN betas b ON b.id = bsv.beta_id
JOIN beta_scripts bs ON bs.beta_id = b.id
JOIN scripts s ON s.id = bs.script_id
WHERE bsv.server = ? AND s.target_rel_path = ? AND b.state IN ('running','ready-promote')
WHERE bsv.server = ? AND bsv.server_deleted_at IS NULL AND s.target_rel_path = ? AND b.state IN ('running','ready-promote')
LIMIT 1`
)
.get(server, target) as { beta_id: number } | undefined;
src/lib/server/fleet/status-evidence.ts
+1 -1
View File
@@ -8,7 +8,7 @@ export interface TelemetryEvidence { players: number; map: string; receivedAt: s
/** Reports from before the latest lifecycle submission describe the old process. */
export function latestGameSubmission(server: string): string | null {
return (db().prepare(`SELECT MAX(at) AS at FROM (
SELECT submitted_at AS at FROM server_deployments WHERE server=? AND action IN ('create','start','restart','rebuild')
SELECT submitted_at AS at FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild')
UNION ALL SELECT requested_at AS at FROM restart_rollout WHERE server=?)`).get(server, server) as { at: string | null }).at;
}
src/lib/server/fleet/telemetry.ts
+1 -1
View File
@@ -37,7 +37,7 @@ export function saveTelemetry(report: GameTelemetry): GameTelemetry {
} else if (report.event === 'game_end') {
handle.prepare('UPDATE servers SET game_ended_at = ? WHERE name = ?').run(report.receivedAt, report.server);
// Legacy betas also take match completion directly from the script.
if (report.round >= MIN_RECORDED_ROUND) handle.prepare(`UPDATE beta_servers SET match_seen_at = ? WHERE server = ? AND stage = 'awaiting-match'
if (report.round >= MIN_RECORDED_ROUND) handle.prepare(`UPDATE beta_servers SET match_seen_at = ? WHERE server = ? AND server_deleted_at IS NULL AND stage = 'awaiting-match'
AND healthy_at <= ? AND beta_id IN (SELECT id FROM betas WHERE state = 'running')`)
.run(report.receivedAt, report.server, report.receivedAt);
}
src/lib/server/maintenance/recovery.ts
+7 -6
View File
@@ -26,7 +26,7 @@ export function setWatchdogEnabled(enabled: boolean) {
}
function incident(id: string) { return db().prepare('SELECT * FROM watchdog_incidents WHERE id = ?').get(id) as WatchdogIncident | undefined; }
function active(server: string, kind: 'health' | 'drift') { return db().prepare("SELECT * FROM watchdog_incidents WHERE server = ? AND kind = ? AND state != 'resolved'").get(server, kind) as WatchdogIncident | undefined; }
function active(server: string, kind: 'health' | 'drift') { return db().prepare("SELECT * FROM watchdog_incidents WHERE server = ? AND server_deleted_at IS NULL AND kind = ? AND state != 'resolved'").get(server, kind) as WatchdogIncident | undefined; }
function event(id: string, message: string) { db().prepare('INSERT INTO watchdog_events(incident_id,created_at,message) VALUES (?,?,?)').run(id, iso(), message); }
function transition(id: string, state: WatchdogState, summary: string) {
const previous = incident(id);
@@ -74,11 +74,11 @@ export function watchdogDetail(id: string): WatchdogDetail | null {
return row ? { incident: row, actions: actions(id), events: db().prepare('SELECT id,created_at,message FROM watchdog_events WHERE incident_id=? ORDER BY id').all(id) as WatchdogDetail['events'] } : null;
}
function hasPending(server: string) {
return !!db().prepare("SELECT 1 FROM server_deployments WHERE server=? AND state IN ('queued','waiting','running','awaiting-health') LIMIT 1").get(server);
return !!db().prepare("SELECT 1 FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND state IN ('queued','waiting','running','awaiting-health') LIMIT 1").get(server);
}
function protectedWork(server: string): boolean {
return !!db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id=bs.beta_id WHERE bs.server=? AND b.state IN ('running','ready-promote') LIMIT 1`).get(server)
|| !!db().prepare(`SELECT 1 FROM deployment_servers ds WHERE ds.server=? AND ds.restart_state IN ('pending','held','failed','manual')
return !!db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id=bs.beta_id WHERE bs.server=? AND bs.server_deleted_at IS NULL AND b.state IN ('running','ready-promote') LIMIT 1`).get(server)
|| !!db().prepare(`SELECT 1 FROM deployment_servers ds WHERE ds.server=? AND ds.server_deleted_at IS NULL AND ds.restart_state IN ('pending','held','failed','manual')
AND NOT EXISTS (SELECT 1 FROM watchdog_actions a WHERE a.campaign_id=ds.deployment_id) LIMIT 1`).get(server);
}
function addAction(row: WatchdogIncident, operationId: string, action: 'restart' | 'rebuild', campaignId: string | null = null) {
@@ -91,6 +91,7 @@ export function updateWatchdogIncident(id: string, action: 'retry' | 'dismiss')
return db().transaction(() => {
const row = incident(id);
if (!row) return { ok: false as const, status: 404, error: 'Watchdog incident not found.' };
if (row.server_deleted_at) return { ok: false as const, status: 409, error: 'This server was deleted; its recovery history cannot be changed.' };
if (actions(id).some(a => pendingStates.includes(a.operation_state ?? ''))) return { ok: false as const, status: 409, error: 'Provisioning is still active. Complete or cancel that operation first.' };
if (action === 'dismiss') {
if (!['open','attention'].includes(row.state)) return { ok: false as const, status: 409, error: 'Only detected incidents or incidents needing attention can be dismissed.' };
@@ -106,7 +107,7 @@ export function updateWatchdogIncident(id: string, action: 'retry' | 'dismiss')
}
async function reconcileActions(): Promise<void> {
const rows = db().prepare("SELECT * FROM watchdog_incidents WHERE state IN ('recovering','attention')").all() as WatchdogIncident[];
const rows = db().prepare("SELECT * FROM watchdog_incidents WHERE server_deleted_at IS NULL AND state IN ('recovering','attention')").all() as WatchdogIncident[];
for (const row of rows) {
const action = actions(row.id).filter(a => a.cycle === row.cycle).at(-1);
if (!action) continue;
@@ -229,7 +230,7 @@ async function run(forceScan: boolean): Promise<void> {
}
await Promise.all(Array.from({ length: Math.min(4, servers.length) }, pump));
// Preserve incidents for temporarily absent rebuild instances, but close retired registrations.
const abandoned = db().prepare("SELECT * FROM watchdog_incidents WHERE state != 'resolved'").all() as WatchdogIncident[];
const abandoned = db().prepare("SELECT * FROM watchdog_incidents WHERE server_deleted_at IS NULL AND state != 'resolved'").all() as WatchdogIncident[];
for (const row of abandoned) {
if (getServer(row.server) || hasPending(row.server)) continue;
const last = db().prepare('SELECT action,state FROM server_deployments WHERE server=? ORDER BY rowid DESC LIMIT 1').get(row.server) as { action: string; state: string } | undefined;
src/lib/server/maintenance/runtime.ts
+7 -6
View File
@@ -39,7 +39,7 @@ export function queueMaintenance(servers: string[], kind: MaintenanceJob['kind']
if (getServer(server)?.desired_state === 'stopped') continue;
added += db().prepare(`INSERT OR IGNORE INTO maintenance_jobs (server, target_revision, requested_at, kind, origin)
VALUES (?, ?, ?, ?, ?)`).run(server, target, nowIso(), kind, origin).changes;
const job = db().prepare("SELECT id FROM maintenance_jobs WHERE server = ? AND state IN ('queued','restarting')").get(server) as { id: number };
const job = db().prepare("SELECT id FROM maintenance_jobs WHERE server = ? AND server_deleted_at IS NULL AND state IN ('queued','restarting')").get(server) as { id: number };
requestRestart(server, { kind: 'maintenance', id: String(job.id) });
}
return added;
@@ -55,13 +55,14 @@ export function queueUpdates(servers?: string[], automatic = false): number {
if (servers && (!servers.length || servers.some(server => !versions.some(v => v.server === server)))) throw new Error('Choose existing fleet servers.');
const candidates = versions.filter(v => v.update_status === 'outdated' && (!servers || servers.includes(v.server)));
const names = candidates.filter(v => !automatic || !db().prepare(`SELECT 1 FROM maintenance_jobs
WHERE server = ? AND target_revision = ? AND kind = 'update'`).get(v.server, freshRelease())).map(v => v.server);
WHERE server = ? AND server_deleted_at IS NULL AND target_revision = ? AND kind = 'update'`).get(v.server, freshRelease())).map(v => v.server);
return names.length ? queueMaintenance(names, 'update', automatic ? 'automatic' : 'manual') : 0;
}
export function retryMaintenance(id: number): number {
const job = db().prepare("SELECT * FROM maintenance_jobs WHERE id = ? AND state = 'failed'").get(id) as MaintenanceJob | undefined;
if (!job) throw new Error('Only failed maintenance can be retried.');
if (job.server_deleted_at) throw new Error('This server was deleted; its maintenance history cannot be retried.');
if (!maintenanceAllowed()) throw new Error('Maintenance is paused.');
if (getServer(job.server)?.desired_state === 'stopped') throw new Error('This server is intentionally stopped.');
// Ambiguous lifecycle failures retain their slot and require diagnostics, not another request.
@@ -82,11 +83,11 @@ export function cancelMaintenance(id: number): boolean {
}
function finish(job: MaintenanceJob, state: 'done' | 'failed' | 'cancelled', detail: string) {
db().prepare('UPDATE maintenance_jobs SET state = ?, detail = ?, finished_at = ? WHERE id = ?')
db().prepare('UPDATE maintenance_jobs SET state = ?, detail = ?, finished_at = ? WHERE id = ? AND server_deleted_at IS NULL')
.run(state, detail, nowIso(), job.id);
}
function hold(job: MaintenanceJob, detail: string) {
db().prepare('UPDATE maintenance_jobs SET detail = ? WHERE id = ?').run(detail, job.id);
db().prepare('UPDATE maintenance_jobs SET detail = ? WHERE id = ? AND server_deleted_at IS NULL').run(detail, job.id);
}
let ticking: Promise<void> | null = null;
@@ -108,7 +109,7 @@ async function runTick(): Promise<void> {
writeSetting('plutonium_schedule', { ...plan, nextAt: nextMaintenanceAt(plan.time, new Date(), plan.timeZone) });
}
})();
const jobs = db().prepare(`SELECT * FROM maintenance_jobs WHERE state IN ('queued','restarting') ORDER BY id`).all() as MaintenanceJob[];
const jobs = db().prepare(`SELECT * FROM maintenance_jobs WHERE server_deleted_at IS NULL AND state IN ('queued','restarting') ORDER BY id`).all() as MaintenanceJob[];
for (const job of jobs) {
try { await processJob(job); }
catch { hold(job, 'Maintenance check failed; waiting for the next check.'); }
@@ -119,7 +120,7 @@ async function runTick(): Promise<void> {
}
async function processJob(job: MaintenanceJob): Promise<void> {
const server = getServer(job.server);
if (!server || server.destroyed_at) return finish(job, 'cancelled', 'Server removed.');
if (job.server_deleted_at || !server || server.destroyed_at) return finish(job, 'cancelled', 'Server removed.');
const operation = requestRestart(job.server, { kind: 'maintenance', id: String(job.id) });
if (['failed','interrupted','cancelled'].includes(operation.state)) {
return finish(job, operation.state === 'cancelled' ? 'cancelled' : 'failed', operation.error ?? operation.stage);
src/lib/server/pipeline/beta-plan-runner.ts
+19 -5
View File
@@ -1,5 +1,5 @@
import { db } from '../db/index.js';
import { betaOperationInFlight, deployBetaServers, getBeta, listBetaServers, rollbackBeta } from '../fleet/beta.js';
import { betaOperationInFlight, deployBetaServers, getBeta, isActiveBetaServer, listBetaServers, rollbackBeta } from '../fleet/beta.js';
import { activity, currentRun, observeRun, prepareRun, readMetrics, saveMetrics, setOutcome, storedPlan, type StoredPlan } from '../fleet/beta-tests.js';
import { coveragePassed, evaluateRequirements } from '../fleet/beta-evaluation.js';
import { parseBetaPlan, type BetaPlan } from '../../beta-plan.js';
@@ -7,9 +7,9 @@ import { emit, type BetaStage } from './bus.js';
import { requestRestart, releaseRestartRequest } from '../provisioning/index.js';
function stage(id: number, server: string, next: BetaStage, error: string | null = null): void {
db().prepare('UPDATE beta_servers SET stage = ?, error = ?, updated_at = ? WHERE beta_id = ? AND server = ?')
const result = db().prepare('UPDATE beta_servers SET stage = ?, error = ?, updated_at = ? WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL')
.run(next, error, new Date().toISOString(), id, server);
emit({ type: 'beta-server-stage', betaId: id, server, stage: next, error });
if (result.changes) emit({ type: 'beta-server-stage', betaId: id, server, stage: next, error });
}
export function betaState(id: number, ready: boolean): void {
@@ -25,14 +25,16 @@ export function betaState(id: number, ready: boolean): void {
}
export async function rollbackTestPlan(id: number, plan: BetaPlan): Promise<void> {
const servers = listBetaServers(id).filter(s => s.deployed_at);
const servers = listBetaServers(id).filter(s => s.deployed_at && isActiveBetaServer(id, s.server));
const result = await rollbackBeta(id);
if (plan.rollbackRestart === 'when-empty' && servers.length) {
const deploymentId = `dep-beta-rollback-${id}-${Date.now()}`;
db().transaction(() => {
db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by, beta_id) VALUES (?, 'rollback', ?, 'beta-test-plan', ?)")
.run(deploymentId, new Date().toISOString(), id);
for (const server of servers) db().prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES (?, ?, 1, 'pending')").run(deploymentId, server.server);
for (const server of servers) {
if (isActiveBetaServer(id, server.server)) db().prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES (?, ?, 1, 'pending')").run(deploymentId, server.server);
}
})();
activity(id, 'Restored files; rolling restarts will wait for empty servers');
} else activity(id, 'Restored files; manual restarts are required to load them');
@@ -73,12 +75,21 @@ async function advancePlan(stored: StoredPlan): Promise<void> {
const plan = parseBetaPlan(JSON.parse(stored.plan));
const beta = getBeta(id)!;
const servers = listBetaServers(id);
// Preserve the original coverage and evidence. A replacement with the same
// name cannot complete a test that began on a different server.
const missing = servers.filter(server => !isActiveBetaServer(id, server.server));
if (missing.length) {
betaState(id, false);
setOutcome(id, 'paused', `Test server deleted: ${missing.map(server => server.server).join(', ')}. Roll back this beta and start a new test.`);
return;
}
if (stored.outcome !== 'passed' && Date.now() - Date.parse(beta.opened_at) >= plan.timeoutHours * 3_600_000) {
betaState(id, false);
setOutcome(id, 'inconclusive', 'Deadline reached without enough qualifying evidence');
return;
}
for (const server of servers) {
if (!isActiveBetaServer(id, server.server)) return;
const freshPlan = storedPlan(id)!;
if (!['active', 'passed'].includes(freshPlan.outcome)) return;
if (server.stage === 'failed') { await fail(id, plan, `${server.server}: ${server.error ?? 'server failed'}`); return; }
@@ -108,6 +119,7 @@ async function advancePlan(stored: StoredPlan): Promise<void> {
}
}
}
if (!isActiveBetaServer(id, server.server)) return;
if (server.stage === 'awaiting-empty' || server.stage === 'awaiting-health') {
const operation = requestRestart(server.server, { kind: 'beta', id: String(id) });
if (['failed','interrupted','cancelled'].includes(operation.state)) {
@@ -129,6 +141,7 @@ async function advancePlan(stored: StoredPlan): Promise<void> {
const run = currentRun(id, server.server);
if (!run?.observing_at) continue;
await observeRun(run, plan);
if (!isActiveBetaServer(id, server.server)) return;
if (!['active', 'passed'].includes(storedPlan(id)!.outcome)) { betaState(id, false); return; }
const metrics = readMetrics(currentRun(id, server.server)!);
const progress = evaluateRequirements(plan, metrics, run.phase);
@@ -143,6 +156,7 @@ async function advancePlan(stored: StoredPlan): Promise<void> {
} else stage(id, server.server, progress.every(p => p.passed) ? 'ready' : 'observing');
}
const updated = listBetaServers(id);
if (updated.some(server => !isActiveBetaServer(id, server.server))) return;
const queued = updated.filter(s => s.stage === 'queued');
if (queued.length && updated.filter(s => s.stage !== 'queued').every(s => s.stage === 'ready')) {
const wave = queued.slice(0, plan.waveSize || queued.length).map(s => s.server);
src/lib/server/pipeline/beta-runner.ts
+7 -6
View File
@@ -25,7 +25,7 @@ const setStage = () =>
restarted_at = CASE WHEN ? = 'awaiting-health' OR ? = 'ready' THEN COALESCE(restarted_at, ?) ELSE restarted_at END,
healthy_at = CASE WHEN ? = 'awaiting-match' OR (? = 'ready' AND healthy_at IS NULL) THEN COALESCE(healthy_at, ?) ELSE healthy_at END,
match_seen_at = CASE WHEN ? = 'ready' AND match_seen_at IS NULL THEN ? ELSE match_seen_at END
WHERE beta_id = ? AND server = ?`
WHERE beta_id = ? AND server = ? AND server_deleted_at IS NULL`
);
function transition(
@@ -35,7 +35,7 @@ function transition(
error: string | null = null
): void {
const t = nowIso();
setStage().run(
const result = setStage().run(
stage,
error,
t,
@@ -50,17 +50,17 @@ function transition(
betaId,
server
);
emit({ type: 'beta-server-stage', betaId, server, stage, error });
if (result.changes) emit({ type: 'beta-server-stage', betaId, server, stage, error });
}
// Bump beta.state to ready-promote when every one of its beta_servers is ready.
// A single failed row is enough to keep the beta in 'running' (operator retries).
function reconcileBetaState(betaId: number): void {
const rows = db()
.prepare(`SELECT stage FROM beta_servers WHERE beta_id = ?`)
.all(betaId) as Array<{ stage: BetaStage }>;
.prepare(`SELECT stage, server_deleted_at FROM beta_servers WHERE beta_id = ?`)
.all(betaId) as Array<{ stage: BetaStage; server_deleted_at: string | null }>;
if (rows.length === 0) return;
if (rows.every((r) => r.stage === 'ready')) {
if (rows.every((r) => r.stage === 'ready' && !r.server_deleted_at)) {
const upd = db()
.prepare(`UPDATE betas SET state = 'ready-promote' WHERE id = ? AND state = 'running'`)
.run(betaId);
@@ -98,6 +98,7 @@ async function runBetaTick(): Promise<{ processed: number }> {
FROM beta_servers bs
JOIN betas b ON b.id = bs.beta_id
WHERE b.state = 'running'
AND bs.server_deleted_at IS NULL
AND NOT EXISTS (SELECT 1 FROM beta_test_plans p WHERE p.beta_id = b.id)
AND bs.stage IN ('awaiting-empty','restarting','awaiting-health','awaiting-match')
ORDER BY b.id, bs.server`
src/lib/server/pipeline/sweep.ts
+3 -3
View File
@@ -19,7 +19,7 @@ export function sweep(): Promise<SweepResult> {
}
async function runSweep(): Promise<SweepResult> {
const pending = db().prepare(`SELECT ds.* FROM deployment_servers ds JOIN deployments d ON d.id = ds.deployment_id
WHERE ds.restart_state IN ('pending','failed') ORDER BY d.started_at, d.id, ds.server`).all() as PendingRow[];
WHERE ds.server_deleted_at IS NULL AND ds.restart_state IN ('pending','failed') ORDER BY d.started_at, d.id, ds.server`).all() as PendingRow[];
const result: SweepResult = { checked: 0, restarted: 0, healthy: 0, held: 0, failed: 0 };
for (const row of pending) {
if (!row.files_changed) continue; // A failed file write is not a restart request.
@@ -40,10 +40,10 @@ async function runSweep(): Promise<SweepResult> {
emit({ type: 'server-state', deploymentId: row.deployment_id, server: row.server, state, players: operation.players, error });
}
db().prepare(`UPDATE deployments SET finished_at = ? WHERE finished_at IS NULL AND NOT EXISTS (
SELECT 1 FROM deployment_servers WHERE deployment_id = deployments.id AND restart_state IN ('pending','failed','held'))`).run(nowIso());
SELECT 1 FROM deployment_servers WHERE deployment_id = deployments.id AND server_deleted_at IS NULL AND restart_state IN ('pending','failed','held'))`).run(nowIso());
db().prepare(`DELETE FROM deployments WHERE started_at < ?
AND NOT EXISTS (SELECT 1 FROM watchdog_actions WHERE campaign_id = deployments.id) AND NOT EXISTS (
SELECT 1 FROM deployment_servers WHERE deployment_id = deployments.id AND restart_state IN ('pending','failed','held'))`)
SELECT 1 FROM deployment_servers WHERE deployment_id = deployments.id AND server_deleted_at IS NULL AND restart_state IN ('pending','failed','held'))`)
.run(new Date(Date.now() - fleetSettings().deployKeepDays * 86_400_000).toISOString());
emit({ type: 'sweep-tick', frozen: false });
return result;
src/lib/server/provisioning/history.ts
+15 -9
View File
@@ -24,7 +24,7 @@ export function recordDeploymentEvent(id: string, message: string): void {
}
export function getDeployment(id: string) {
const deployment = db().prepare(`SELECT d.*, j.expected_map, EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server) AS server_exists
const deployment = db().prepare(`SELECT d.*, j.expected_map, (d.server_deleted_at IS NULL AND EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server AND s.destroyed_at IS NULL)) AS server_exists
FROM server_deployments d LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE d.id = ?`).get(id);
if (!deployment) return null;
const events = db().prepare('SELECT id, created_at, message FROM server_deployment_events WHERE deployment_id = ? ORDER BY id').all(id);
@@ -42,14 +42,14 @@ export function getDeployment(id: string) {
export const REMOVING_SERVER = 'Removing server files and registry';
export function canCancelRemoval(id: string): boolean {
return !!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND action = 'destroy' AND state = 'waiting' AND stage != ?").get(id, REMOVING_SERVER);
return !!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state = 'waiting' AND stage != ?").get(id, REMOVING_SERVER);
}
/** Competes atomically with the worker's claim, before any external removal. */
export function cancelRemoval(id: string): boolean {
return db().transaction(() => {
const result = db().prepare(`UPDATE server_deployments SET state = 'cancelled', stage = 'Removal cancelled', finished_at = ?, error = NULL
WHERE id = ? AND action = 'destroy' AND state = 'waiting' AND stage != ?`).run(new Date().toISOString(), id, REMOVING_SERVER);
WHERE id = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state = 'waiting' AND stage != ?`).run(new Date().toISOString(), id, REMOVING_SERVER);
if (!result.changes) return false;
recordDeploymentEvent(id, 'Queued removal cancelled. The server and its files were kept.');
return true;
@@ -57,12 +57,18 @@ export function cancelRemoval(id: string): boolean {
}
export function claimRemoval(id: string): boolean {
return db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND action = 'destroy' AND state = 'waiting'").run(id).changes > 0;
return db().prepare(`UPDATE server_deployments SET state = 'running'
WHERE id = ? AND action = 'destroy' AND state = 'waiting'
AND (server_deleted_at IS NULL OR (stage = ?
AND NOT EXISTS (SELECT 1 FROM servers s WHERE s.name = server_deployments.server)
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = server_deployments.server
AND newer.rowid > server_deployments.rowid)))`).run(id, REMOVING_SERVER).changes > 0;
}
export function canResumeCleanup(id: string): boolean {
return !!db().prepare(`SELECT d.id FROM server_deployments d
WHERE d.id = ? AND d.action = 'destroy' AND d.state IN ('failed', 'interrupted') AND d.stage = ?
WHERE d.id = ? AND (d.server_deleted_at IS NULL OR NOT EXISTS (SELECT 1 FROM servers s WHERE s.name = d.server))
AND d.action = 'destroy' AND d.state IN ('failed', 'interrupted') AND d.stage = ?
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
.get(id, REMOVING_SERVER);
}
@@ -80,7 +86,7 @@ export function canResumeVerification(id: string): boolean {
return !!db().prepare(`SELECT d.id FROM server_deployments d
JOIN server_creation_jobs j ON j.deployment_id = d.id
JOIN servers s ON s.name = j.server AND s.game_port = j.port
WHERE d.id = ? AND (d.action = 'create' OR (d.action = 'rebuild' AND EXISTS (
WHERE d.id = ? AND d.server_deleted_at IS NULL AND (d.action = 'create' OR (d.action = 'rebuild' AND EXISTS (
SELECT 1 FROM server_rebuild_jobs r WHERE r.deployment_id = d.id AND r.removed_at IS NOT NULL))) AND d.state IN ('failed', 'interrupted')
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
.get(id);
@@ -88,7 +94,7 @@ export function canResumeVerification(id: string): boolean {
export function canResumeRebuild(id: string): boolean {
return !!db().prepare(`SELECT d.id FROM server_deployments d JOIN server_rebuild_jobs r ON r.deployment_id = d.id
WHERE d.id = ? AND d.state IN ('failed','interrupted') AND d.stage = ? AND r.removed_at IS NULL
WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.state IN ('failed','interrupted') AND d.stage = ? AND r.removed_at IS NULL
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
.get(id, REMOVING_SERVER);
}
@@ -120,8 +126,8 @@ export function finishDeployment(id: string, result: { ok: boolean; message?: st
}
export function listDeployments() {
return db().prepare(`SELECT d.*, EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server) AS server_exists,
(d.action = 'destroy' AND d.state = 'waiting' AND d.stage != ?) AS canCancelRemoval
return db().prepare(`SELECT d.*, (d.server_deleted_at IS NULL AND EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server AND s.destroyed_at IS NULL)) AS server_exists,
(d.server_deleted_at IS NULL AND d.action = 'destroy' AND d.state = 'waiting' AND d.stage != ?) AS canCancelRemoval
FROM server_deployments d ORDER BY started_at DESC, rowid DESC`).all(REMOVING_SERVER);
}
src/lib/server/provisioning/instances.ts
+21 -31
View File
@@ -63,7 +63,7 @@ export function nextLobbyNumber(baseName: string, additionalNames: string[] = []
const rows = db()
.prepare(`SELECT name FROM servers WHERE name LIKE ? UNION
SELECT j.server AS name FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
WHERE d.state IN ('queued', 'running', 'awaiting-health') OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry')))`)
WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued', 'running', 'awaiting-health') OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry'))))`)
.all(`${baseName}_%`) as Array<{ name: string }>;
const taken = new Set<number>();
const prefix = `${baseName}_`;
@@ -87,7 +87,7 @@ export function nextAvailablePort(additionalPorts: number[] = [], releasingServe
const rows = db()
.prepare(`SELECT name, game_port FROM servers WHERE game_port IS NOT NULL UNION
SELECT j.server AS name, j.port AS game_port FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
WHERE d.state IN ('queued', 'running', 'awaiting-health') OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry')))`)
WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued', 'running', 'awaiting-health') OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry'))))`)
.all() as Array<{ name: string; game_port: number }>;
const taken = new Set([...rows.filter(row => !releasingServers.includes(row.name)).map((r) => r.game_port), ...additionalPorts]);
for (let p = config.portRange.min; p <= config.portRange.max; p++) {
@@ -257,7 +257,7 @@ export function queueRebuildServer(name: string, triggeredBy = 'ui'): CreateServ
/** Caller has just verified emptiness and excluded conflicting work. */
export async function rebuildServer(id: string): Promise<void> {
const job = db().prepare(`SELECT j.*, d.stage FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
WHERE d.id = ? AND d.action = 'rebuild' AND d.state = 'waiting'`).get(id) as (CreationJob & { stage: string }) | undefined;
WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.action = 'rebuild' AND d.state = 'waiting'`).get(id) as (CreationJob & { stage: string }) | undefined;
if (!job) return;
if (creating || destroying.size || creationQueue().active.length >= fleetSettings().createConcurrency) {
// The cleanup checkpoint must survive capacity waits and app restarts.
@@ -487,17 +487,17 @@ const destroying = new Set<string>();
// Persist the request before returning so navigation and client disconnects cannot lose it.
function hasActiveCreation(name: string): boolean {
return !!db().prepare("SELECT 1 FROM server_deployments WHERE server = ? AND action = 'create' AND state IN ('queued', 'running', 'awaiting-health')").get(name);
return !!db().prepare("SELECT 1 FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'create' AND state IN ('queued', 'running', 'awaiting-health')").get(name);
}
export function destroyServer(name: string, triggeredBy = 'ui'): DestroyServerResult & { deploymentId?: string } {
if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND action IN ('start','restart','stop','rebuild')
if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('start','restart','stop','rebuild')
AND state IN ('waiting','queued','running')`).get(name)) {
return { ok: false, kind: 'conflict', message: 'Wait for the current provisioning operation before destroying this server.', warnings: [] };
}
const existing = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) as { id: string } | undefined;
const existing = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) as { id: string } | undefined;
if (existing) return { ok: true, warnings: [], deploymentId: existing.id };
const previous = db().prepare("SELECT id FROM server_deployments WHERE server = ? ORDER BY rowid DESC LIMIT 1").get(name) as { id: string } | undefined;
const previous = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL ORDER BY rowid DESC LIMIT 1").get(name) as { id: string } | undefined;
if (previous && resumeCleanup(previous.id)) return { ok: true, warnings: [], deploymentId: previous.id };
if (!getServer(name) && !hasActiveCreation(name)) return { ok: false, kind: 'not-found', message: 'Server not found', warnings: [] };
const operation = db().transaction(() => {
@@ -517,13 +517,18 @@ export function processDestructions(): Promise<void> {
}
async function runDestructions(): Promise<void> {
const queued = db().prepare("SELECT id, server, stage FROM server_deployments WHERE action = 'destroy' AND state = 'waiting' ORDER BY started_at").all() as { id: string; server: string; stage: string }[];
const queued = db().prepare(`SELECT id, server, stage, server_deleted_at FROM server_deployments WHERE action = 'destroy' AND state = 'waiting'
AND (server_deleted_at IS NULL OR stage = ?) ORDER BY started_at`).all(REMOVING_SERVER) as { id: string; server: string; stage: string; server_deleted_at: string | null }[];
for (const operation of queued) {
if (creating || destroying.has(operation.server)) continue;
if (!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND state = 'waiting'").get(operation.id)) continue;
destroying.add(operation.server);
try {
if (operation.stage === REMOVING_SERVER) {
if (operation.server_deleted_at && getServer(operation.server)) {
finishDeployment(operation.id, { ok: false, message: 'A replacement server is registered; local cleanup cannot safely resume.' });
continue;
}
const newer = db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND action = 'create'
AND rowid > (SELECT rowid FROM server_deployments WHERE id = ?)`).get(operation.server, operation.id);
if (newer) {
@@ -619,25 +624,10 @@ async function cleanupDestroyedServer(name: string, operation: string): Promise<
warnings.push(`rm ${hostDir} failed: ${(err as Error).message}`);
}
// Remove server-owned beta and deployment rows before the registry row.
// Test evidence cascades from runs; inventory and snapshots cascade from servers.
// The registry trigger preserves history and marks its old server identity.
// Live inventory, rollback snapshots, and restart slots still cascade away.
try {
const tx = db().transaction((n: string) => {
// The Watchdog's rebuild campaign observes this exact operation
// through the registry gap and retains its before/after file audit.
for (const table of ['deployment_servers', 'deployment_files']) {
db().prepare(`DELETE FROM ${table} WHERE server = ? AND deployment_id NOT IN (
SELECT campaign_id FROM watchdog_actions WHERE campaign_id IS NOT NULL)`)
.run(n);
}
db().prepare('DELETE FROM beta_servers WHERE server = ?').run(n);
db().prepare('DELETE FROM beta_server_scripts WHERE server = ?').run(n);
db().prepare('DELETE FROM beta_test_runs WHERE server = ?').run(n);
db().prepare('DELETE FROM beta_test_coverage WHERE server = ?').run(n);
db().prepare('DELETE FROM beta_file_changes WHERE server = ?').run(n);
db().prepare('DELETE FROM servers WHERE name = ?').run(n);
});
tx(name);
db().prepare('DELETE FROM servers WHERE name = ?').run(name);
} catch (err) {
return {
ok: false,
@@ -706,7 +696,7 @@ export async function processCreations(): Promise<void> {
export function creationQueue() {
type Entry = { id: string; server: string; state: string; stage: string };
const jobs = db().prepare(`SELECT id, server, state, stage FROM server_deployments
WHERE action IN ('create', 'rebuild') AND state IN ('queued', 'running', 'awaiting-health')
WHERE server_deleted_at IS NULL AND action IN ('create', 'rebuild') AND state IN ('queued', 'running', 'awaiting-health')
ORDER BY started_at, rowid`).all() as Entry[];
return { active: jobs.filter(job => job.state !== 'queued'), queued: jobs.filter(job => job.state === 'queued') };
}
@@ -726,7 +716,7 @@ async function runCreations(): Promise<CreationOutcomes> {
if (creating || destroying.size > 0) return outcomes;
// Restart recovery is observation-only: never repeat external creation or
// deployment. A known interrupted startup must retain its readiness gate.
const interrupted = db().prepare("SELECT id FROM server_deployments WHERE action IN ('create', 'rebuild') AND state = 'interrupted' ORDER BY started_at, rowid").all() as { id: string }[];
const interrupted = db().prepare("SELECT id FROM server_deployments WHERE server_deleted_at IS NULL AND action IN ('create', 'rebuild') AND state = 'interrupted' ORDER BY started_at, rowid").all() as { id: string }[];
for (const { id } of interrupted) {
if (resumeVerification(id)) continue;
// A queued removal does not make the interrupted container disappear.
@@ -748,14 +738,14 @@ async function runCreations(): Promise<CreationOutcomes> {
// Check only previously started servers. Newly launched servers keep their
// slot until a later pass, even if a health probe would answer immediately.
const checking = db().prepare(`SELECT j.*, COALESCE(d.submitted_at, d.started_at) AS submitted_at FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
WHERE d.state = 'awaiting-health' ORDER BY d.started_at, d.rowid`).all() as CreationJob[];
WHERE d.server_deleted_at IS NULL AND d.state = 'awaiting-health' ORDER BY d.started_at, d.rowid`).all() as CreationJob[];
for (const job of checking) {
const id = job.deployment_id;
if (getServer(job.server)?.game_port !== job.port) {
finishDeployment(id, { ok: false, message: 'Server was removed or replaced before creation health checks completed' });
continue;
}
if (db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'destroy' AND state IN ('waiting', 'running')").get(job.server)) continue;
if (db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')").get(job.server)) continue;
try {
const komodo = await komodoInspectHealthy(job.server);
if (!komodo.ok) { deploymentStage(id, 'Waiting for Komodo health: ' + komodo.reason); continue; }
@@ -787,7 +777,7 @@ async function runCreations(): Promise<CreationOutcomes> {
// API calls can cancel a selected job or resume an older startup while
// a previous provision awaits Komodo. Recheck without expanding this pass.
if (creationQueue().active.length >= fleetSettings().createConcurrency) break;
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND action IN ('create', 'rebuild') AND state = 'queued'").get(id)) continue;
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND action IN ('create', 'rebuild') AND state = 'queued'").get(id)) continue;
const blocker = creationBlocker(id);
if (blocker) {
deploymentStage(id, blocker);
src/lib/server/provisioning/requests.ts
+9 -8
View File
@@ -28,7 +28,7 @@ const sourceTables = {
export function requestRestart(server: string, source: RestartSource): LifecycleOperation {
return db().transaction(() => {
const { table, key } = sourceTables[source.kind];
const row = db().prepare(`SELECT provisioning_id, restarted_at FROM ${table} WHERE ${key} = ? AND server = ?`)
const row = db().prepare(`SELECT provisioning_id, restarted_at FROM ${table} WHERE ${key} = ? AND server = ? AND server_deleted_at IS NULL`)
.get(source.id, server) as { provisioning_id: string | null; restarted_at: string | null } | undefined;
if (!row) throw new Error('Restart source no longer exists');
if (row.provisioning_id) {
@@ -45,14 +45,14 @@ export function requestRestart(server: string, source: RestartSource): Lifecycle
// The previous release already recorded some submissions in history.
// Bind to that exact interval, never to a newer restart of the server.
id = (db().prepare(`SELECT d.id FROM restart_rollout r JOIN server_deployments d ON d.id = r.operation_id
WHERE r.server = ? AND d.action = 'restart' AND d.started_at <= ? AND r.requested_at >= ?`)
WHERE r.server = ? AND d.server_deleted_at IS NULL AND d.action = 'restart' AND d.started_at <= ? AND r.requested_at >= ?`)
.get(server, row.restarted_at, row.restarted_at) as { id: string } | undefined)?.id;
if (id) db().prepare('UPDATE server_deployments SET submitted_at = COALESCE(submitted_at, ?) WHERE id = ?').run(row.restarted_at, id);
}
// Only not-yet-submitted campaign changes can share an activation. A later
// file deployment must never be credited to an already running restart.
if (source.kind === 'campaign' && !row.restarted_at) {
id = (db().prepare(`SELECT d.id FROM server_deployments d WHERE d.server = ? AND d.action = 'restart'
id = (db().prepare(`SELECT d.id FROM server_deployments d WHERE d.server = ? AND d.server_deleted_at IS NULL AND d.action = 'restart'
AND d.state = 'waiting' AND d.submitted_at IS NULL
AND EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'campaign')
AND NOT EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind != 'campaign')
@@ -107,9 +107,9 @@ export function cancelProvisioning(id: string, reason = 'Cancelled by requester'
/** Explicit caller retry starts a new job and keeps the old job as history. */
export function releaseRestartRequest(server: string, source: RestartSource): void {
const { table, key } = sourceTables[source.kind];
const row = db().prepare(`SELECT provisioning_id FROM ${table} WHERE ${key} = ? AND server = ?`).get(source.id, server) as { provisioning_id: string | null } | undefined;
const row = db().prepare(`SELECT provisioning_id FROM ${table} WHERE ${key} = ? AND server = ? AND server_deleted_at IS NULL`).get(source.id, server) as { provisioning_id: string | null } | undefined;
if (row?.provisioning_id) cancelProvisioning(row.provisioning_id, 'Superseded by a fresh caller attempt');
db().prepare(`UPDATE ${table} SET provisioning_id = NULL WHERE ${key} = ? AND server = ?`).run(source.id, server);
db().prepare(`UPDATE ${table} SET provisioning_id = NULL WHERE ${key} = ? AND server = ? AND server_deleted_at IS NULL`).run(source.id, server);
}
export function requestLifecycle(server: string, action: RuntimeAction) {
@@ -117,6 +117,7 @@ export function requestLifecycle(server: string, action: RuntimeAction) {
if (!komodoConfigured() || !rconConfigured()) return { ok: false as const, status: 503, error: 'Configure Komodo and RCON before provisioning.' };
return db().transaction(() => {
const existing = db().prepare(`SELECT id, action FROM server_deployments WHERE server = ?
AND server_deleted_at IS NULL
AND state IN ('queued','waiting','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(server) as { id: string; action: string } | undefined;
if (existing) {
if (existing.action !== action) return { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' };
@@ -142,7 +143,7 @@ export function requestLifecycle(server: string, action: RuntimeAction) {
type LifecycleRequestResult = { ok: true; deploymentId: string } | { ok: false; status: number; error: string };
function watchdogRequestGuard(server: string, incidentId: string): Extract<LifecycleRequestResult, { ok: false }> | null {
const incident = db().prepare("SELECT kind FROM watchdog_incidents WHERE id = ? AND server = ? AND state IN ('open','recovering')").get(incidentId, server) as { kind: 'health' | 'drift' } | undefined;
const incident = db().prepare("SELECT kind FROM watchdog_incidents WHERE id = ? AND server = ? AND server_deleted_at IS NULL AND state IN ('open','recovering')").get(incidentId, server) as { kind: 'health' | 'drift' } | undefined;
if (!incident) return { ok: false, status: 409, error: 'The Watchdog incident is no longer active.' };
if (!maintenanceAllowed(incident.kind)) {
return { ok: false, status: 409, error: 'This maintenance policy is paused.' };
@@ -183,7 +184,7 @@ export function requestWatchdogLifecycle(server: string, action: 'restart' | 're
if (error) return error;
const pending = db().prepare(`SELECT d.id, d.action, s.source_id FROM server_deployments d
LEFT JOIN provisioning_sources s ON s.operation_id = d.id AND s.kind = 'watchdog'
WHERE d.server = ? AND d.state IN ('queued','waiting','running','awaiting-health') ORDER BY d.rowid DESC LIMIT 1`)
WHERE d.server = ? AND d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health') ORDER BY d.rowid DESC LIMIT 1`)
.get(server) as { id: string; action: string; source_id: string | null } | undefined;
if (pending) return pending.source_id === incidentId && pending.action === action
? { ok: true as const, deploymentId: pending.id }
@@ -223,7 +224,7 @@ export function escalateWatchdogRestart(incidentId: string, operationId: string,
if (error) return error;
const pending = db().prepare(`SELECT d.id, d.action, s.source_id FROM server_deployments d
LEFT JOIN provisioning_sources s ON s.operation_id = d.id AND s.kind = 'watchdog'
WHERE d.server = ? AND d.id != ? AND d.state IN ('queued','waiting','running','awaiting-health') LIMIT 1`)
WHERE d.server = ? AND d.server_deleted_at IS NULL AND d.id != ? AND d.state IN ('queued','waiting','running','awaiting-health') LIMIT 1`)
.get(operation.server, operationId) as { id: string; action: string; source_id: string | null } | undefined;
if (pending) return pending.action === 'rebuild' && pending.source_id === incidentId
? { ok: true as const, deploymentId: pending.id }
src/lib/server/provisioning/restarts.ts
+16 -13
View File
@@ -7,7 +7,7 @@ import { deploymentStage, finishDeployment } from './history.js';
// Komodo queues restart work. Don't mistake the still-running old container
// for a completed restart when an API kick immediately follows the request.
const HEALTH_POLL_MS = 15_000;
const requestsInFlight = new Set<string>();
const requestsInFlight = new Map<string, string>();
export const restartRequestInFlight = (server: string) => requestsInFlight.has(server);
interface RestartSlot {
@@ -36,14 +36,14 @@ export function restartSettling(server: string): boolean {
}
export function rolloutWaitReason(server: string, operationId = ''): string | null {
if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND id != ?
if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL
AND ((action IN ('start','stop','rebuild') AND state IN ('running','awaiting-health'))
OR (action = 'destroy' AND state IN ('waiting','running')))`).get(server, operationId)) {
return 'Waiting for the current provisioning operation';
}
const slot = slotFor(server);
const ownHandoff = slot?.operation_id === operationId && !!db().prepare(`SELECT 1 FROM server_deployments d
WHERE d.id = ? AND d.action IN ('restart','rebuild') AND EXISTS (
WHERE d.id = ? AND d.server_deleted_at IS NULL AND d.action IN ('restart','rebuild') AND EXISTS (
SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'watchdog')`).get(operationId);
if (slot?.healthy_at === null && !ownHandoff) {
return `Waiting for the previous restart of this server to recover${slot.error ? `: ${slot.error}` : ''}`;
@@ -61,6 +61,7 @@ export type RolloutRestartResult =
export async function restartInRollout(server: string, options: { operationId: string; action: 'start' | 'restart' }): Promise<RolloutRestartResult> {
const operationId = options.operationId;
const waitReason = db().transaction(() => {
if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return 'Original server was deleted';
const reason = rolloutWaitReason(server, operationId);
if (reason) return reason;
db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), operationId);
@@ -73,35 +74,37 @@ export async function restartInRollout(server: string, options: { operationId: s
}).immediate();
if (waitReason) return { kind: 'queued', message: waitReason };
requestsInFlight.add(server);
requestsInFlight.set(server, operationId);
db().prepare(`UPDATE servers SET container_state = 'restarting', last_error = NULL, last_reply_snippet = NULL,
game_started_at = NULL, game_ended_at = NULL WHERE name = ?`).run(server);
try {
const result = options.action === 'start' ? await komodoRuntimeAction(server, 'start') : await komodoRestart(server);
if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return { kind: 'failed', message: 'Original server was deleted' };
if (result.ok) {
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(operationId);
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ? AND server_deleted_at IS NULL").run(operationId);
deploymentStage(operationId!, 'Waiting for container and game health (RCON or fresh telemetry)');
return { kind: 'started' };
}
// These failures cannot have queued a remote restart. Other failures
// (especially a timeout) may have: retain capacity until recovery.
if (result.kind === 'no-name' || result.kind === 'unconfigured') {
db().prepare('DELETE FROM restart_rollout WHERE server = ?').run(server);
db().prepare('DELETE FROM restart_rollout WHERE server = ? AND operation_id = ?').run(server, operationId);
} else {
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ?').run(`komodo: ${result.message}`, server);
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ? AND operation_id = ?').run(`komodo: ${result.message}`, server, operationId);
}
finishDeployment(operationId!, { ok: false, message: `komodo: ${result.message}` });
return { kind: 'failed', message: `komodo: ${result.message}` };
} catch (err) {
if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return { kind: 'failed', message: 'Original server was deleted' };
const message = `komodo: ${(err as Error).message}`;
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ?').run(message, server);
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ? AND operation_id = ?').run(message, server, operationId);
finishDeployment(operationId!, { ok: false, message });
return { kind: 'failed', message };
} finally {
// Start the grace period at completion, including slow/failed requests.
db().prepare('UPDATE restart_rollout SET requested_at = ? WHERE server = ?')
.run(new Date().toISOString(), server);
requestsInFlight.delete(server);
db().prepare('UPDATE restart_rollout SET requested_at = ? WHERE server = ? AND operation_id = ?')
.run(new Date().toISOString(), server, operationId);
if (requestsInFlight.get(server) === operationId) requestsInFlight.delete(server);
}
}
@@ -121,7 +124,7 @@ async function refresh(): Promise<void> {
if (requestsInFlight.has(slot.server)) continue;
// Recovery capacity transferred to a rebuild remains reserved until
// teardown. Its health is verified by the creation worker, never here.
if (slot.operation_id && db().prepare(`SELECT 1 FROM server_deployments d WHERE d.id = ? AND (
if (slot.operation_id && db().prepare(`SELECT 1 FROM server_deployments d WHERE d.id = ? AND d.server_deleted_at IS NULL AND (
(d.action = 'rebuild' AND d.state IN ('waiting','queued','running','awaiting-health'))
OR (d.action = 'restart' AND d.state = 'waiting' AND d.submitted_at IS NULL AND EXISTS (
SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'watchdog')))`)
@@ -146,7 +149,7 @@ async function refresh(): Promise<void> {
WHERE server = ? AND requested_at = ? AND operation_id IS ? AND healthy_at IS NULL`)
.run(now, error ? null : now, players, error, slot.server, slot.requested_at, slot.operation_id);
if (!checked.changes) continue;
if (slot.operation_id && db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND state = 'awaiting-health'").get(slot.operation_id)) {
if (slot.operation_id && db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'awaiting-health'").get(slot.operation_id)) {
db().prepare('UPDATE server_deployments SET checked_at = ?, players = ? WHERE id = ?').run(now, players, slot.operation_id);
if (error) deploymentStage(slot.operation_id, 'Waiting for health: ' + error);
else {
src/lib/server/provisioning/worker.ts
+10 -10
View File
@@ -23,7 +23,7 @@ function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled'
if (!sources.length) return 'ready';
// Watchdog intent gates every linked source, including its drift campaign.
for (const source of sources.filter(source => source.kind === 'watchdog')) {
if (!db().prepare("SELECT 1 FROM watchdog_incidents WHERE id = ? AND server = ? AND state IN ('open','recovering')").get(source.source_id, operation.server)
if (!db().prepare("SELECT 1 FROM watchdog_incidents WHERE id = ? AND server = ? AND server_deleted_at IS NULL AND state IN ('open','recovering')").get(source.source_id, operation.server)
|| getServer(operation.server)?.desired_state === 'stopped') return 'cancelled';
const incident = db().prepare('SELECT kind FROM watchdog_incidents WHERE id = ?').get(source.source_id) as { kind: 'health' | 'drift' };
if (!maintenanceAllowed(incident.kind)) return 'held';
@@ -31,21 +31,21 @@ function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled'
let held = false;
for (const source of sources) {
if (source.kind === 'campaign') {
const row = db().prepare('SELECT restart_state, files_changed, provisioning_id FROM deployment_servers WHERE deployment_id = ? AND server = ?')
const row = db().prepare('SELECT restart_state, files_changed, provisioning_id FROM deployment_servers WHERE deployment_id = ? AND server = ? AND server_deleted_at IS NULL')
.get(source.source_id, operation.server) as { restart_state: string; files_changed: number; provisioning_id: string } | undefined;
if (row?.provisioning_id !== operation.id || !row.files_changed) continue;
if (['pending','failed'].includes(row.restart_state)) return 'ready';
if (['held','manual'].includes(row.restart_state)) held = true;
} else if (source.kind === 'beta') {
const row = db().prepare(`SELECT b.state, bs.stage, bs.provisioning_id, p.outcome FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
LEFT JOIN beta_test_plans p ON p.beta_id = b.id WHERE bs.beta_id = ? AND bs.server = ?`)
LEFT JOIN beta_test_plans p ON p.beta_id = b.id WHERE bs.beta_id = ? AND bs.server = ? AND bs.server_deleted_at IS NULL`)
.get(source.source_id, operation.server) as { state: string; stage: string; provisioning_id: string; outcome: string | null } | undefined;
if (row?.provisioning_id !== operation.id || !['running','ready-promote'].includes(row.state)) continue;
if (!['awaiting-empty','restarting','awaiting-health'].includes(row.stage)) continue;
if (!row.outcome || ['active','passed'].includes(row.outcome)) return 'ready';
if (row.outcome === 'paused') held = true;
} else if (source.kind === 'maintenance') {
const row = db().prepare('SELECT state, provisioning_id, kind, origin FROM maintenance_jobs WHERE id = ?').get(source.source_id) as { state: string; provisioning_id: string; kind: string; origin: string } | undefined;
const row = db().prepare('SELECT state, provisioning_id, kind, origin FROM maintenance_jobs WHERE id = ? AND server_deleted_at IS NULL').get(source.source_id) as { state: string; provisioning_id: string; kind: string; origin: string } | undefined;
if (row?.provisioning_id === operation.id && ['queued','restarting'].includes(row.state)) {
if (getServer(operation.server)?.desired_state === 'stopped') return 'cancelled';
if (!maintenanceAllowed() || (row.kind === 'update' && row.origin === 'automatic' && !maintenanceAllowed('updates'))) return 'held';
@@ -59,7 +59,7 @@ function sourceEligibility(operation: Operation): 'ready' | 'held' | 'cancelled'
function blocker(server: string, id: string): string | null {
const sources = sourcesFor(id);
const rank = priority(sources);
const others = db().prepare(`SELECT id, state, action, rowid FROM server_deployments WHERE server = ? AND id != ?
const others = db().prepare(`SELECT id, state, action, rowid FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL
AND state IN ('queued','waiting','running','awaiting-health')`).all(server, id) as { id: string; state: string; action: string; rowid: number }[];
const own = db().prepare('SELECT rowid FROM server_deployments WHERE id = ?').get(id) as { rowid: number };
if (others.some(other => other.state !== 'waiting' || other.action === 'destroy'
@@ -69,8 +69,8 @@ function blocker(server: string, id: string): string | null {
// Beta activation must not wait on the campaign rows that the beta itself
// holds. Campaign/maintenance callers likewise never block on their own job.
if (rank > 0 && db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
WHERE bs.server = ? AND b.state IN ('running','ready-promote') LIMIT 1`).get(server)) return 'Waiting for beta work on this server';
if (rank > 1 && db().prepare(`SELECT 1 FROM deployment_servers ds WHERE ds.server = ?
WHERE bs.server = ? AND bs.server_deleted_at IS NULL AND b.state IN ('running','ready-promote') LIMIT 1`).get(server)) return 'Waiting for beta work on this server';
if (rank > 1 && db().prepare(`SELECT 1 FROM deployment_servers ds WHERE ds.server = ? AND ds.server_deleted_at IS NULL
AND ds.restart_state IN ('pending','failed','held') AND (ds.provisioning_id IS NULL OR EXISTS (
SELECT 1 FROM server_deployments d WHERE d.id = ds.provisioning_id AND d.state IN ('queued','waiting','running','awaiting-health')))
AND NOT EXISTS (SELECT 1 FROM watchdog_actions a WHERE a.operation_id = ? AND a.campaign_id = ds.deployment_id)
@@ -93,7 +93,7 @@ async function run(): Promise<void> {
await processDestructions();
await refreshRestartRollout();
const operations = db().prepare(`SELECT id, server, action, state, stage FROM server_deployments
WHERE action IN ('start','restart','stop','rebuild') AND (state = 'waiting' OR (action = 'stop' AND state = 'awaiting-health'))
WHERE server_deleted_at IS NULL AND action IN ('start','restart','stop','rebuild') AND (state = 'waiting' OR (action = 'stop' AND state = 'awaiting-health'))
ORDER BY started_at, rowid`).all() as Operation[];
for (const operation of operations) {
try { await processOperation(operation); }
@@ -140,9 +140,9 @@ async function processOperation(operation: Operation): Promise<void> {
}
// Requests can arrive during the probes. Recheck before claiming the action.
if (blocker(server, id) || sourceEligibility(operation) !== 'ready') return;
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND state = 'waiting'").get(id)) return;
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return;
if (action === 'rebuild') { await rebuildServer(id); return; }
const claim = db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND state = 'waiting'").run(id);
const claim = db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").run(id);
if (!claim.changes) return;
if (action === 'start' || action === 'restart') {
const result = await restartInRollout(server, { operationId: id, action });
src/lib/ui/components/BetaTestProgress.svelte
+13 -12
View File
@@ -1,7 +1,7 @@
<script lang="ts">
import { timeFormat } from '$ui/stores/time-zone.js';
import { tick } from 'svelte';
import ServerIdentity from './ServerIdentity.svelte';
import HistoricalServerIdentity from './HistoricalServerIdentity.svelte';
import StatusBadge from './primitives/StatusBadge.svelte';
import ListFooter from './primitives/ListFooter.svelte';
import Button from './primitives/Button.svelte';
@@ -9,12 +9,13 @@
import { coverageSummary, requirementSummary, needsGameTelemetry } from '$lib/beta-plan-summary.js';
import { betaWaitingCondition, betaEvidenceSummary } from '$lib/beta-evidence.js';
import type { BetaRequirement, BetaTestView } from '$lib/beta-plan.js';
let { test, betaId, closed, deadline, retryBlocked, onUpdate }: { test:BetaTestView; betaId:number; closed:boolean; deadline?:string; retryBlocked?:string; onUpdate:()=>Promise<void> } = $props();
let { test, betaId, closed, deadline, retryBlocked, deletedParticipants = false, onUpdate }: { test:BetaTestView; betaId:number; closed:boolean; deadline?:string; retryBlocked?:string; deletedParticipants?:boolean; onUpdate:()=>Promise<void> } = $props();
let notes = $state<Record<string,string>>({});
let evidenceOpen = $state(false), evidenceLimit = $state(25), busy = $state(false), error = $state(''), message = $state('');
let evidenceServer = $state(''), evidenceKind = $state(''), evidenceQuery = $state('');
let evidenceSection = $state<HTMLDetailsElement>();
const canSign = $derived(!closed && ['active','passed'].includes(test.outcome));
const hasDeletedServers = $derived(deletedParticipants || test.servers.some(server => server.server_deleted_at));
const canSign = $derived(!closed && !hasDeletedServers && ['active','passed'].includes(test.outcome));
const expired = $derived(test.outcome === 'inconclusive' || (!!deadline && test.outcome !== 'passed' && Date.parse(deadline) <= Date.now()));
const ranked = $derived([...test.servers].sort((a,b) => (a.progress.some(r => r.failed) ? 0 : a.passed ? 2 : 1) - (b.progress.some(r => r.failed) ? 0 : b.passed ? 2 : 1) || a.server.localeCompare(b.server)));
const filteredEvidence = $derived(test.evidence.filter(event => (!evidenceServer || event.server === evidenceServer)
@@ -43,29 +44,29 @@
<div class="heading"><h2>Test results</h2><StatusBadge state={test.outcome}/></div>
<p>{coverageSummary(test.plan)} · Every requirement must pass on each qualifying server.</p>
{#if !closed}
<div class="actions">
{#if !hasDeletedServers}<div class="actions">
{#if test.outcome === 'paused' && !expired}<Button size="sm" disabled={busy} onclick={()=>action('resume')}>Resume testing</Button>{/if}
{#if ['active','passed'].includes(test.outcome)}<Button size="sm" disabled={busy} onclick={()=>action('pause')}>Pause testing</Button>{/if}
{#if ['paused','failed'].includes(test.outcome) && !expired}<Button size="sm" disabled={busy || !!retryBlocked} onclick={()=>action('retry')}>Start fresh attempt</Button>{/if}
</div>
</div>{/if}
{#if retryBlocked && ['paused','failed'].includes(test.outcome)}<p>{retryBlocked}</p>{/if}
{#if expired}<p class="guidance">The testing window ended. Roll back this beta first, then choose “Reuse files and test plan” to open a new beta with a longer window.</p>
{:else if test.outcome === 'paused'}<p class="guidance">Resume retains completed milestones; continuous health and in-progress timing intervals start fresh. Pausing does not extend the deadline.</p>{/if}
{:else if test.outcome === 'paused' && !hasDeletedServers}<p class="guidance">Resume retains completed milestones; continuous health and in-progress timing intervals start fresh. Pausing does not extend the deadline.</p>{/if}
{/if}
{#if error}<p role="alert" class="failed">{error}</p>{/if}
{#if message}<p role="status">{message}</p>{/if}
{#if !test.servers.length}<p>Waiting for the first server restart. Evidence collection begins with that run. The testing window is already running.</p>{/if}
{#if !test.servers.length}<p>{hasDeletedServers ? 'No test run was recorded before a participating server was deleted.' : 'Waiting for the first server restart. Evidence collection begins with that run. The testing window is already running.'}</p>{/if}
{#each ranked as server (server.server)}
<details class="server" open={!server.passed}>
<summary><ServerIdentity name={server.server}/><span>{server.progress.filter(p=>p.passed).length} / {server.progress.length} checks passed</span>{#if test.plan.requirements.some(r=>r.kind==='recovery')}<span>Recovery cycles: {server.phase}</span>{/if}</summary>
{#if test.plan.requirements.some(needsGameTelemetry)}<p class="observation">{#if server.observation?.lastGameplayAt}Latest gameplay: {server.observation.lastEvent?.replaceAll('_',' ')} · <time datetime={server.observation.lastGameplayAt} title={$timeFormat.dateTime(server.observation.lastGameplayAt)}>{reportAge(server.observation.lastGameplayAt)}</time> · {server.observation.players ?? 'Unknown'} players{#if server.observation.round !== null} · Round {server.observation.round}{/if}{:else}No gameplay reports in this attempt yet.{/if} <a href={`/servers/${encodeURIComponent(server.server)}?tab=diagnostics#telemetry`}>Telemetry diagnostics</a></p>{/if}
<summary><HistoricalServerIdentity name={server.server} deletedAt={server.server_deleted_at}/><span>{server.progress.filter(p=>p.passed).length} / {server.progress.length} checks passed</span>{#if test.plan.requirements.some(r=>r.kind==='recovery')}<span>Recovery cycles: {server.phase}</span>{/if}</summary>
{#if test.plan.requirements.some(needsGameTelemetry)}<p class="observation">{#if server.observation?.lastGameplayAt}Latest gameplay: {server.observation.lastEvent?.replaceAll('_',' ')} · <time datetime={server.observation.lastGameplayAt} title={$timeFormat.dateTime(server.observation.lastGameplayAt)}>{reportAge(server.observation.lastGameplayAt)}</time> · {server.observation.players ?? 'Unknown'} players{#if server.observation.round !== null} · Round {server.observation.round}{/if}{:else}No gameplay reports in this attempt yet.{/if} {#if !server.server_deleted_at}<a href={`/servers/${encodeURIComponent(server.server)}?tab=diagnostics#telemetry`}>Telemetry diagnostics</a>{/if}</p>{/if}
{#each server.progress as progress (progress.id)}
{@const requirement = test.plan.requirements.find(r=>r.id===progress.id)}
{#if requirement}<div class="requirement" class:failed={progress.failed}>
<div class="heading"><strong><span class="result">{progress.passed ? 'Passed' : progress.failed ? 'Failed' : 'Waiting'}</span> {requirementSummary(requirement)}</strong><span>{progress.detail}</span></div>
{#if !progress.passed && !closed}<p>{betaWaitingCondition(requirement,server.observation)}</p>{/if}
{#if !progress.passed && !closed && !hasDeletedServers}<p>{betaWaitingCondition(requirement,server.observation)}</p>{/if}
{#if !['manual','failure-budget'].includes(requirement.kind)}<progress aria-label={requirementSummary(requirement)} value={Math.min(progress.current,progress.target)} max={progress.target}></progress>{/if}
{#if requirement.kind === 'manual' && !progress.passed && !closed}
{#if requirement.kind === 'manual' && !progress.passed && !closed && !hasDeletedServers}
{#if canSign && server.observingAt}<div class="signoff"><label>Tester name and observation<textarea aria-label={`Tester and observation for ${progress.label} on ${server.server}`} rows="2" maxlength="1000" bind:value={notes[`${server.server}:${progress.id}`]} placeholder="Name · what you checked and observed"></textarea></label><Button size="sm" disabled={busy || !notes[`${server.server}:${progress.id}`]?.trim()} onclick={()=>action('sign-off',{server:server.server,requirementId:progress.id,note:notes[`${server.server}:${progress.id}`]})}>Sign off</Button></div>
{:else if canSign}<p>Waiting for this server's restart and health verification to finish before tester sign-off.</p>
{:else}<p>Sign-off is unavailable while this test is {test.outcome}. {test.outcome === 'paused' ? 'Resume testing first.' : 'Use the recovery instructions above.'}</p>{/if}
@@ -81,7 +82,7 @@
{#if evidenceOpen}
<p>Newest first. Includes retained evidence from earlier attempts and recovery cycles; current progress uses only the current attempt.</p>
<div class="filters"><label>Server<select bind:value={evidenceServer}><option value="">All servers</option>{#each [...new Set(test.evidence.map(event=>event.server))] as server}<option value={server}>{server}</option>{/each}</select></label><label>Event type<select bind:value={evidenceKind}><option value="">All events</option><option value="gameplay">Gameplay reports</option><option value="health">Health observations</option><option value="manual-sign-off">Tester sign-offs</option></select></label><label>Search evidence<input type="search" bind:value={evidenceQuery} placeholder="Round, map, tester, observation…"/></label></div>
{#each filteredEvidence.slice(0,evidenceLimit) as event(event.id)}<article class="evidence"><div class="heading"><ServerIdentity name={event.server}/><time datetime={event.received_at}>{$timeFormat.dateTime(event.received_at)}</time></div><p>{betaEvidenceSummary(event)}</p><details><summary>Raw payload · {event.kind}</summary><pre>{event.payload}</pre></details></article>{/each}
{#each filteredEvidence.slice(0,evidenceLimit) as event(event.id)}<article class="evidence"><div class="heading"><HistoricalServerIdentity name={event.server} deletedAt={event.server_deleted_at}/><time datetime={event.received_at}>{$timeFormat.dateTime(event.received_at)}</time></div><p>{betaEvidenceSummary(event)}</p><details><summary>Raw payload · {event.kind}</summary><pre>{event.payload}</pre></details></article>{/each}
<ListFooter total={filteredEvidence.length} bind:limit={evidenceLimit} label="events"/>
{#if !filteredEvidence.length}<p>{test.evidence.length ? 'No evidence matches these filters.' : 'No evidence recorded yet. Check deployment progress and telemetry diagnostics.'}</p>{/if}
{/if}
src/lib/ui/components/HistoricalServerIdentity.svelte
+17
View File
@@ -0,0 +1,17 @@
<script lang="ts">
import ServerIdentity from './ServerIdentity.svelte';
import Badge from './primitives/Badge.svelte';
let { name, deletedAt, href }: { name: string; deletedAt?: string | null; href?: string } = $props();
</script>
<span class="historical-server">
{#if href && !deletedAt}<a {href}><ServerIdentity {name} /></a>
{:else}<ServerIdentity {name} />{/if}
{#if deletedAt}<Badge>Deleted</Badge>{/if}
</span>
<style>
.historical-server { display: inline-flex; align-items: baseline; flex-wrap: wrap; gap: 6px 8px; min-width: 0; }
a { color: var(--accent-primary-hover); }
</style>
src/lib/ui/components/MatchHistory.svelte
+2 -3
View File
@@ -3,7 +3,7 @@
import type { ServerMatch } from '$lib/match-history.js';
import { pollVisible } from '$ui/poll.js';
import { formatDuration } from '$ui/duration.js';
import { serverTitle } from '$ui/server-labels.js';
import HistoricalServerIdentity from './HistoricalServerIdentity.svelte';
import Card from './primitives/Card.svelte';
import Button from './primitives/Button.svelte';
import EmptyState from './primitives/EmptyState.svelte';
@@ -60,7 +60,7 @@
<tr>
<td><time datetime={match.startedAt}>{$timeFormat.dateTime(match.startedAt)}</time></td>
<td>{#if match.endedAt}<time datetime={match.endedAt}>{$timeFormat.dateTime(match.endedAt)}</time>{:else}—{/if}</td>
{#if server === undefined}<td><a href={`/servers/${encodeURIComponent(match.server)}?tab=matches`}>{serverTitle(match.server)}</a></td>{/if}
{#if server === undefined}<td><HistoricalServerIdentity name={match.server} deletedAt={match.server_deleted_at} href={`/servers/${encodeURIComponent(match.server)}?tab=matches`} /></td>{/if}
<td>{match.map}</td>
<td>{match.status === 'active' ? 'In progress' : match.status === 'completed' ? 'Completed' : 'Interrupted'}</td>
<td>{match.startPlayers ?? '—'}</td><td>{match.uniquePlayers}</td><td>{match.endPlayers ?? '—'}</td>
@@ -84,5 +84,4 @@
th, td { padding: var(--space-3); text-align: left; border-bottom: 1px solid var(--border-subtle); white-space: nowrap; }
th { color: var(--text-secondary); font-size: var(--text-xs); }
td { font-variant-numeric: tabular-nums; }
a { color: var(--accent-primary-hover); }
</style>
src/lib/watchdog.ts
+1
View File
@@ -4,6 +4,7 @@ export type WatchdogState = 'open' | 'recovering' | 'attention' | 'resolved' | '
export interface WatchdogIncident {
id: string;
server: string;
server_deleted_at?: string | null;
kind: 'health' | 'drift';
state: WatchdogState;
summary: string;
src/routes/api/betas/[id]/+server.ts
+2 -2
View File
@@ -33,7 +33,7 @@ export async function GET({ params }: { params: { id: string } }) {
const events = db().prepare('SELECT id, created_at, message FROM beta_events WHERE beta_id = ? ORDER BY id').all(id);
const campaign = db().prepare("SELECT id FROM deployments WHERE beta_id = ? AND action = 'promote' ORDER BY started_at DESC LIMIT 1").get(id) as { id: string } | undefined;
const promotion = campaign ? { ...campaign,
pending: (db().prepare("SELECT COUNT(*) n FROM deployment_servers WHERE deployment_id = ? AND restart_state NOT IN ('done','failed','cancelled')").get(campaign.id) as { n: number }).n,
failed: (db().prepare("SELECT COUNT(*) n FROM deployment_servers WHERE deployment_id = ? AND restart_state = 'failed'").get(campaign.id) as { n: number }).n } : null;
pending: (db().prepare("SELECT COUNT(*) n FROM deployment_servers WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state NOT IN ('done','failed','cancelled')").get(campaign.id) as { n: number }).n,
failed: (db().prepare("SELECT COUNT(*) n FROM deployment_servers WHERE deployment_id = ? AND server_deleted_at IS NULL AND restart_state = 'failed'").get(campaign.id) as { n: number }).n } : null;
return json({ beta, scripts, servers, events, test: testView(id), promotion });
}
src/routes/api/betas/[id]/retry/+server.ts
+3 -2
View File
@@ -19,11 +19,12 @@ export async function POST({ params, request }: { params: { id: string }; reques
if (!body.server) return json({ ok: false, error: 'server is required' }, { status: 400 });
const row = db()
.prepare('SELECT stage, restarted_at, healthy_at FROM beta_servers WHERE beta_id = ? AND server = ?')
.prepare('SELECT stage, restarted_at, healthy_at, server_deleted_at FROM beta_servers WHERE beta_id = ? AND server = ?')
.get(betaId, body.server) as
| { stage: string; restarted_at: string | null; healthy_at: string | null }
| { stage: string; restarted_at: string | null; healthy_at: string | null; server_deleted_at: string | null }
| undefined;
if (!row) return json({ ok: false, error: 'no such beta_server' }, { status: 404 });
if (row.server_deleted_at) return json({ ok: false, error: 'This beta server was deleted; open a new beta for its replacement.' }, { status: 409 });
if (row.stage !== 'failed') return json({ ok: false, error: `stage is ${row.stage}, not failed` }, { status: 409 });
// Best-effort resume point: if we already got past restart, go back to
src/routes/api/betas/[id]/test/+server.ts
+3
View File
@@ -17,6 +17,9 @@ export async function POST({ params, request }: { params: { id: string }; reques
if (!['running', 'ready-promote'].includes(beta.state)) return json({ error: 'Beta is closed' }, { status: 409 });
try {
const body = await request.json();
if (db().prepare('SELECT 1 FROM beta_servers WHERE beta_id = ? AND server_deleted_at IS NOT NULL').get(id)) {
return json({ ok: false, error: 'A beta server was deleted. Roll back this beta and open a new one to collect fresh evidence.' }, { status: 409 });
}
const plan = parseBetaPlan(JSON.parse(stored.plan));
if (body.action === 'sign-off') {
if (!['active', 'passed'].includes(stored.outcome)) throw new Error('Resume or retry the test before signing off');
src/routes/api/campaigns/+server.ts
+3 -2
View File
@@ -40,12 +40,13 @@ export async function GET({ url }: { url: URL }) {
const placeholders = ids.map(() => '?').join(',');
const serversByDep = db()
.prepare(
`SELECT deployment_id, server, files_changed, restart_state, players, checked_at, restarted_at, attempts, error
`SELECT deployment_id, server, server_deleted_at, files_changed, restart_state, players, checked_at, restarted_at, attempts, error
FROM deployment_servers WHERE deployment_id IN (${placeholders})`
)
.all(...ids) as Array<{
deployment_id: string;
server: string;
server_deleted_at: string | null;
files_changed: number;
restart_state: string;
players: number | null;
@@ -92,7 +93,7 @@ export async function GET({ url }: { url: URL }) {
servers: grouped.get(d.id) ?? [],
scripts: scriptsByDep.get(d.id) ?? []
}))
.filter((d) => (activeOnly ? d.servers.some((s) => s.restart_state === 'pending' || s.restart_state === 'failed') : true));
.filter((d) => (activeOnly ? d.servers.some((s) => !s.server_deleted_at && (s.restart_state === 'pending' || s.restart_state === 'failed')) : true));
return json({ deployments: rows });
}
src/routes/betas/[id]/+page.svelte
+9 -6
View File
@@ -11,6 +11,7 @@
import { confirmAction } from '$ui/confirm.js';
import BetaTestProgress from '$ui/components/BetaTestProgress.svelte';
import ServerIdentity from '$ui/components/ServerIdentity.svelte';
import HistoricalServerIdentity from '$ui/components/HistoricalServerIdentity.svelte';
import OperationHistory from '$ui/components/OperationHistory.svelte';
import StatusBadge from '$ui/components/primitives/StatusBadge.svelte';
import Card from '$ui/components/primitives/Card.svelte';
@@ -21,7 +22,7 @@
import EmptyState from '$ui/components/primitives/EmptyState.svelte';
interface BetaRow { id:number; state:string; opened_at:string; closed_at:string|null; wait_for_health:number; wait_for_match:number; notes?:string|null; }
interface BetaServer { server:string; stage:string; error:string|null; deployed_at:string|null; restarted_at:string|null; healthy_at:string|null; match_seen_at:string|null; provisioning_id:string|null; }
interface BetaServer { server:string; server_deleted_at?:string|null; stage:string; error:string|null; deployed_at:string|null; restarted_at:string|null; healthy_at:string|null; match_seen_at:string|null; provisioning_id:string|null; }
interface BetaFile { id:number; name:string; kind:string; target_rel_path:string; version:{id:number; version:number; release_version:string|null; sha256:string}|null; }
interface Promotion { id:string; pending:number; failed:number; }
let test = $state<BetaTestView|null>(null);
@@ -36,6 +37,7 @@
let now = $state(Date.now());
const id = $derived(Number($page.params.id));
const closed = $derived(beta?.state === 'promoted' || beta?.state === 'rolled-back');
const hasDeletedServers = $derived(servers.some(server => server.server_deleted_at));
const deadline = $derived(beta && test ? new Date(Date.parse(beta.opened_at) + test.plan.timeoutHours * 3600000) : null);
const remaining = $derived(deadline ? Math.max(0, Math.ceil((deadline.getTime() - now) / 60000)) : 0);
const deadlineText = $derived(remaining > 0 ? `${Math.floor(remaining / 60)}h ${remaining % 60}m remaining` : 'Testing window ended');
@@ -43,6 +45,7 @@
const ranked = $derived([...servers].sort((a,b) => (a.stage === 'failed' ? 0 : a.stage === 'ready' ? 2 : 1) - (b.stage === 'failed' ? 0 : b.stage === 'ready' ? 2 : 1) || a.server.localeCompare(b.server)));
const waiting = $derived(ranked.filter(server => server.stage !== 'ready'));
const nextAction = $derived(closed ? beta?.state === 'promoted' ? 'Follow the campaign to check activation on additional servers.' : 'Reuse these files and this plan when you are ready to test again.'
: hasDeletedServers ? 'A participating server was deleted. Its results remain below. Roll back, then reuse this plan to test the current servers.'
: test?.outcome === 'inconclusive' ? 'The deadline passed without enough evidence. Roll back, then reuse this plan with a longer testing window.'
: test?.outcome === 'paused' ? 'Resume to continue collecting evidence, or start a fresh attempt. The original deadline still applies.'
: test?.outcome === 'failed' ? 'Investigate the failed check before a fresh attempt, or roll back.'
@@ -91,7 +94,7 @@
async function rollback() {
const reject = beta?.state === 'ready-promote';
const restarts = test?.plan.rollbackRestart === 'manual' ? 'Restart servers manually to load the restored files.' : 'Restarts wait for empty servers.';
if (!await confirmAction({title:reject?'Reject beta?':'Roll back beta?',message:`Restore the original files on ${servers.length} beta servers${reject ? ' and mark these versions as test-failed' : ''}. ${restarts}`,confirmLabel:reject?'Reject beta':'Roll back',danger:true})) return;
if (!await confirmAction({title:reject?'Reject beta?':'Roll back beta?',message:`Restore the original files on ${servers.filter(server => !server.server_deleted_at).length} beta servers${reject ? ' and mark these versions as test-failed' : ''}. ${restarts}`,confirmLabel:reject?'Reject beta':'Roll back',danger:true})) return;
acting = 'rollback'; flash = ''; success = '';
try {
const response = await fetch(`/api/betas/${id}/rollback`,{method:'POST'}), body = await response.json();
@@ -136,7 +139,7 @@
</div>
{#if !closed && test}<p class="muted">Waiting for empty servers uses this window. Pausing or retrying does not extend the deadline.</p>{/if}
<div class="actions">
{#if !closed}<Button variant="primary" onclick={reviewPromotion} disabled={beta.state !== 'ready-promote' || acting !== null}>Review promotion</Button><Button variant="danger" onclick={rollback} disabled={acting !== null} loading={acting === 'rollback'}>{beta.state === 'ready-promote' ? 'Reject beta' : 'Roll back'}</Button>{/if}
{#if !closed}<Button variant="primary" onclick={reviewPromotion} disabled={hasDeletedServers || beta.state !== 'ready-promote' || acting !== null}>Review promotion</Button><Button variant="danger" onclick={rollback} disabled={acting !== null} loading={acting === 'rollback'}>{beta.state === 'ready-promote' ? 'Reject beta' : 'Roll back'}</Button>{/if}
{#if closed}<Button href={`/betas/new?reuse=${id}`}>Reuse files and test plan</Button>{/if}
{#if promotion}<Button href={`/campaigns/${encodeURIComponent(promotion.id)}`}>View campaign</Button>{/if}
</div>
@@ -146,13 +149,13 @@
</Card>
{#if !closed && waiting.length}
<section aria-label="Outstanding servers"><h2>Outstanding conditions</h2><div class="server-list">
{#each waiting as server}<div class="server-row"><a href={`/servers/${encodeURIComponent(server.server)}`}><ServerIdentity name={server.server} /></a><StatusBadge state={server.stage} /><p>{server.error ?? outstanding(server.server)}</p><a href={server.error ? `/servers/${encodeURIComponent(server.server)}?tab=diagnostics` : '#test-results'}>{server.error ? 'Diagnose' : 'View checks'}</a></div>{/each}
{#each waiting as server}<div class="server-row"><HistoricalServerIdentity name={server.server} deletedAt={server.server_deleted_at} href={`/servers/${encodeURIComponent(server.server)}`} /><StatusBadge state={server.stage} /><p>{server.error ?? (server.server_deleted_at ? 'This server was deleted. Recorded progress is retained.' : outstanding(server.server))}</p>{#if server.error && !server.server_deleted_at}<a href={`/servers/${encodeURIComponent(server.server)}?tab=diagnostics`}>Diagnose</a>{:else if test}<a href="#test-results">View checks</a>{/if}</div>{/each}
</div></section>
{/if}
{#if test}<div id="test-results"><Card padding="lg"><BetaTestProgress {test} betaId={id} {closed} deadline={deadline?.toISOString()} retryBlocked={servers.some(server=>server.error?.startsWith('deploy:')) ? 'Deployment was incomplete. Roll back before opening a fresh beta.' : undefined} onUpdate={load} /></Card></div>{/if}
{#if test}<div id="test-results"><Card padding="lg"><BetaTestProgress {test} betaId={id} {closed} deletedParticipants={hasDeletedServers} deadline={deadline?.toISOString()} retryBlocked={servers.some(server=>server.error?.startsWith('deploy:')) ? 'Deployment was incomplete. Roll back before opening a fresh beta.' : undefined} onUpdate={load} /></Card></div>{/if}
<details class="ui-panel"><summary>Files and versions ({scripts.length})</summary><div class="file-list">{#each scripts as file}<div><a href={`/${file.kind === 'baseline' ? 'configs' : file.kind + 's'}/${file.id}`}>{file.name}</a><strong>{artifactVersionLabel(file.version)}</strong><code>{file.target_rel_path}</code><span>{file.version?.sha256.slice(0,12)}</span></div>{/each}</div></details>
<details class="ui-panel"><summary>Deployment progress ({servers.length} servers)</summary><div class="server-list">
{#each ranked as server}<div class="server-row"><a href={`/servers/${encodeURIComponent(server.server)}`}><ServerIdentity name={server.server}/></a><StatusBadge state={server.stage}/><p>Files {server.deployed_at ? 'installed' : 'pending'} · Restart {server.restarted_at ? 'submitted' : 'pending'}{#if beta.wait_for_health} · Health {server.healthy_at ? 'verified' : 'pending'}{/if}{#if beta.wait_for_match} · Match {server.match_seen_at ? 'observed' : 'pending'}{/if}</p>{#if server.provisioning_id}<a href={`/provisioning/${encodeURIComponent(server.provisioning_id)}`}>View provisioning</a>{/if}{#if server.error}<p class="ui-error">{server.error}</p>{/if}{#if server.stage === 'failed' && !test && !closed}<Button size="sm" onclick={() => retry(server.server)} disabled={acting !== null} loading={acting === `retry:${server.server}`}>Retry server</Button>{/if}</div>{/each}
{#each ranked as server}<div class="server-row"><HistoricalServerIdentity name={server.server} deletedAt={server.server_deleted_at} href={`/servers/${encodeURIComponent(server.server)}`}/><StatusBadge state={server.stage}/><p>Files {server.deployed_at ? 'installed' : 'pending'} · Restart {server.restarted_at ? 'submitted' : 'pending'}{#if beta.wait_for_health} · Health {server.healthy_at ? 'verified' : 'pending'}{/if}{#if beta.wait_for_match} · Match {server.match_seen_at ? 'observed' : 'pending'}{/if}</p>{#if server.provisioning_id}<a href={`/provisioning/${encodeURIComponent(server.provisioning_id)}`}>View provisioning</a>{/if}{#if server.error}<p class="ui-error">{server.error}</p>{/if}{#if server.stage === 'failed' && !server.server_deleted_at && !test && !closed}<Button size="sm" onclick={() => retry(server.server)} disabled={acting !== null} loading={acting === `retry:${server.server}`}>Retry server</Button>{/if}</div>{/each}
</div></details>
<OperationHistory {events} active={!closed} showOverview={false}/>
</div>
src/routes/campaigns/+page.svelte
+6 -4
View File
@@ -6,6 +6,7 @@
import { pollVisible } from '$ui/poll.js';
import { confirmAction } from '$ui/confirm.js';
import { serverTitle } from '$ui/server-labels.js';
import HistoricalServerIdentity from '$ui/components/HistoricalServerIdentity.svelte';
import StatusBadge from '$ui/components/primitives/StatusBadge.svelte';
import ListFooter from '$ui/components/primitives/ListFooter.svelte';
import Badge from '$ui/components/primitives/Badge.svelte';
@@ -15,6 +16,7 @@
interface ServerRow {
server: string;
server_deleted_at?: string | null;
restart_state: string;
files_changed: number;
players: number | null;
@@ -76,7 +78,7 @@
onMount(() => pollVisible(loadCampaigns, 5000));
function isActive(d: Deployment): boolean {
return d.servers.some((s) => s.restart_state === 'pending' || s.restart_state === 'failed');
return d.servers.some((s) => !s.server_deleted_at && (s.restart_state === 'pending' || s.restart_state === 'failed'));
}
function ago(iso: string): string {
const d = Date.now() - new Date(iso).getTime();
@@ -108,7 +110,7 @@
ev.preventDefault();
ev.stopPropagation();
const summary = scriptSummary(d) || 'this promote';
if (!await confirmAction({ title:'Roll back promotion?', message:`Restore the previous version of ${summary} on ${d.servers.length} servers.`, confirmLabel:'Roll back', danger:true })) return;
if (!await confirmAction({ title:'Roll back promotion?', message:`Restore the previous version of ${summary} on ${d.servers.filter(server => !server.server_deleted_at).length} servers.`, confirmLabel:'Roll back', danger:true })) return;
rollingBack = d.id;
flash = null; flashId = d.id; flashError = false;
try {
@@ -211,7 +213,7 @@
{/if}
</div>
<div class="row-head-r">
{#if d.action === 'promote' && !d.rolled_back_by}
{#if d.action === 'promote' && !d.rolled_back_by && d.servers.some(server => !server.server_deleted_at)}
<Button
variant="danger"
size="sm"
@@ -233,7 +235,7 @@
title="{s.server}: {s.restart_state}{s.error ? ' — ' + s.error : ''}"
>
<span class="dot"></span>
<a class="node-label" href="/servers/{encodeURIComponent(s.server)}?tab=diagnostics">{serverTitle(s.server)}{s.players !== null && s.players > 0 ? ` (${s.players} players at last check)` : ''}</a><StatusBadge state={s.restart_state} />
<span class="node-label"><HistoricalServerIdentity name={s.server} deletedAt={s.server_deleted_at} href={`/servers/${encodeURIComponent(s.server)}?tab=diagnostics`} />{s.players !== null && s.players > 0 ? ` (${s.players} players at last check)` : ''}</span><StatusBadge state={s.restart_state} />
</span>
{/each}
</div>
src/routes/campaigns/[id]/+page.svelte
+9 -8
View File
@@ -3,7 +3,7 @@
import { onMount } from 'svelte';
import { pollVisible } from '$ui/poll.js';
import Dialog from '$ui/components/primitives/Dialog.svelte';
import ServerIdentity from '$ui/components/ServerIdentity.svelte';
import HistoricalServerIdentity from '$ui/components/HistoricalServerIdentity.svelte';
import StatusBadge from '$ui/components/primitives/StatusBadge.svelte';
import { page } from '$app/stores';
import DiffViewer from '$ui/components/DiffViewer.svelte';
@@ -22,7 +22,7 @@
size_before: number | null; size_after: number | null;
}
interface ServerRow {
deployment_id: string; server: string; files_changed: number;
deployment_id: string; server: string; server_deleted_at?: string | null; files_changed: number;
restart_state: string; players: number | null;
checked_at: string | null; restarted_at: string | null; provisioning_id: string | null;
attempts: number; error: string | null;
@@ -48,7 +48,8 @@
let openFile = $state<{ id: number; before: string | null; after: string | null; verb: string; relPath: string } | null>(null);
let retryFile = $state<FileRow | null>(null);
const completed = $derived(detail?.servers.filter(server => server.restart_state === 'done').length ?? 0);
const unresolved = $derived(detail?.servers.filter(server => server.restart_state !== 'done').length ?? 0);
const unresolved = $derived(detail?.servers.filter(server => !server.server_deleted_at && server.restart_state !== 'done').length ?? 0);
const deletedIncomplete = $derived(detail?.servers.filter(server => server.server_deleted_at && server.restart_state !== 'done').length ?? 0);
const rankedServers = $derived([...(detail?.servers ?? [])].sort((a, b) => {
const rank: Record<string, number> = { failed: 0, manual: 1, held: 2, pending: 3, done: 4 };
return (rank[a.restart_state] ?? 3) - (rank[b.restart_state] ?? 3);
@@ -122,7 +123,7 @@
{/snippet}
</SectionHeader>
<section class="outcome" aria-label="Campaign outcome"><h2>{completed} / {detail.servers.length} servers activated{unresolved ? ` · ${unresolved} unresolved` : ''}</h2><p>{detail.watchdogIncidentId ? 'Maintenance repairs drift by deleting the existing instance files and rebuilding with the latest promoted setup after verifying a safe server state.' : 'Files are staged before activation. Servers restart when empty, then their health is verified.'}</p>{#if unresolved}<p>Review failed or waiting servers below. Diagnose connectivity or inspect startup logs from the affected row.</p>{:else}<p>Every recorded server completed activation.</p>{/if}</section>
<section class="outcome" aria-label="Campaign outcome"><h2>{completed} / {detail.servers.length} servers activated{unresolved ? ` · ${unresolved} unresolved` : ''}{deletedIncomplete ? ` · ${deletedIncomplete} deleted before activation` : ''}</h2><p>{detail.watchdogIncidentId ? 'Maintenance repairs drift by deleting the existing instance files and rebuilding with the latest promoted setup after verifying a safe server state.' : 'Files are staged before activation. Servers restart when empty, then their health is verified.'}</p>{#if unresolved || deletedIncomplete}<p>Review the recorded results below.{#if deletedIncomplete} Deleted servers retain their last recorded progress.{/if}{#if unresolved} Diagnose connectivity or inspect startup logs from the affected row.{/if}</p>{:else}<p>Every recorded server completed activation.</p>{/if}</section>
{#if detail.watchdogIncidentId}<p class="watchdog-source">Started by Maintenance · <a href={`/maintenance/${encodeURIComponent(detail.watchdogIncidentId)}`}>View incident and recovery history</a></p>{/if}
@@ -144,7 +145,7 @@
{#each rankedServers as s (s.server)}
<div class="srv-row">
<div class="srv-l">
<a class="srv-name" href="/servers/{s.server}"><ServerIdentity name={s.server} /></a>
<span class="srv-name"><HistoricalServerIdentity name={s.server} deletedAt={s.server_deleted_at} href={`/servers/${encodeURIComponent(s.server)}`} /></span>
{#if s.error}
<span class="err">{s.error}</span>
{/if}
@@ -159,7 +160,7 @@
{#if s.restarted_at}
<span class="when">restarted {fmt(s.restarted_at)}</span>
{/if}
{#if s.restart_state !== 'done'}<a href="/servers/{encodeURIComponent(s.server)}?tab=diagnostics">Diagnose</a><a href="/servers/{encodeURIComponent(s.server)}?tab=logs">Open logs</a>{/if}
{#if s.restart_state !== 'done' && !s.server_deleted_at}<a href="/servers/{encodeURIComponent(s.server)}?tab=diagnostics">Diagnose</a><a href="/servers/{encodeURIComponent(s.server)}?tab=logs">Open logs</a>{/if}
</div>
</div>
{/each}
@@ -176,7 +177,7 @@
{:else}
{#each [...byServer.entries()] as [srv, files] (srv)}
<div class="file-group">
<h3><ServerIdentity name={srv} /></h3>
<h3><HistoricalServerIdentity name={srv} deletedAt={detail.servers.find(server => server.server === srv)?.server_deleted_at} /></h3>
<div class="file-rows">
{#each files as f (f.id)}
<button class="file-row" onclick={() => loadDiff(f)}>
@@ -195,7 +196,7 @@
{/if}
</section>
<OperationHistory steps={campaignFlow(detail.servers, Boolean(detail.deployment.finished_at))} events={detail.events} active={detail.servers.some(s => s.restart_state === 'pending' || s.restart_state === 'failed')} />
<OperationHistory steps={campaignFlow(detail.servers, Boolean(detail.deployment.finished_at))} events={detail.events} active={detail.servers.some(s => !s.server_deleted_at && (s.restart_state === 'pending' || s.restart_state === 'failed'))} />
<Dialog bind:open={diffOpen} title={openFile?.relPath ?? 'File comparison'} wide>
{#if diffError}<p class="ui-error" role="alert">{diffError}</p><Button onclick={() => { if (retryFile) void loadDiff(retryFile); }}>Retry comparison</Button>
{:else if diffLoading}<EmptyState title="Loading comparison…" />
src/routes/leaderboard/players/[game]/[guid]/+page.svelte
+2 -1
View File
@@ -5,6 +5,7 @@
import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
import Button from '$ui/components/primitives/Button.svelte';
import CopyValue from '$ui/components/CopyValue.svelte';
import Badge from '$ui/components/primitives/Badge.svelte';
let { data }: { data: { profile: PlayerProfile } } = $props();
const profile = $derived(data.profile);
@@ -76,7 +77,7 @@
<table>
<thead><tr><th scope="col">Map / server</th><th scope="col">Match started</th><th scope="col">Match status</th>{#each historyColumns as key}<th scope="col">{leaderboardMetrics.find(metric => metric.key === key)!.label}</th>{/each}</tr></thead>
<tbody>{#each profile.matches as match (match.id)}
<tr><th scope="row">{match.mapName}<small>{match.server}</small></th><td>{date(match.startedAt)}</td>
<tr><th scope="row">{match.mapName}<small>{match.server} {#if match.server_deleted_at}<Badge>Deleted</Badge>{/if}</small></th><td>{date(match.startedAt)}</td>
<td class="status">{match.status}<small>{match.completed ? 'Completion credited' : 'No completion credit'}</small></td>
{#each historyColumns as key}<td class:best={key === 'highestRound'}>{number.format(match[key])}</td>{/each}
</tr>
src/routes/maintenance/+page.svelte
+10 -9
View File
@@ -9,6 +9,7 @@
import { timeFormat, timeZone } from '$ui/stores/time-zone.js';
import { serverTitle } from '$ui/server-labels.js';
import ServerIdentity from '$ui/components/ServerIdentity.svelte';
import HistoricalServerIdentity from '$ui/components/HistoricalServerIdentity.svelte';
import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
import Tabs from '$ui/components/primitives/Tabs.svelte';
import Badge from '$ui/components/primitives/Badge.svelte';
@@ -40,16 +41,16 @@
const versions = $derived(data?.runtime.servers.filter(v => !server || `${v.server} ${serverTitle(v.server)}`.toLowerCase().includes(server.toLowerCase())) ?? []);
const outdated = $derived(versions.filter(v => v.update_status === 'outdated'));
const unknown = $derived(versions.filter(v => v.update_status === 'unknown'));
const activeJobs = $derived(data?.runtime.jobs.filter(j => ['queued', 'restarting'].includes(j.state)) ?? []);
const activeJobs = $derived(data?.runtime.jobs.filter(j => !j.server_deleted_at && ['queued', 'restarting'].includes(j.state)) ?? []);
const activities = $derived([
...(data?.recovery.incidents ?? []).map(i => ({
key: i.id, kind: i.kind, server: i.server, at: i.first_seen_at,
key: i.id, kind: i.kind, server: i.server, server_deleted_at: i.server_deleted_at, at: i.first_seen_at,
title: i.kind === 'health' ? 'Health recovery' : 'Configuration drift',
state: ['open', 'recovering'].includes(i.state) ? 'active' : i.state === 'attention' ? 'attention' : 'done',
label: watchdogStateLabels[i.state], detail: i.state === 'recovering' ? i.operation_stage ?? i.summary : i.summary, href: `/maintenance/${encodeURIComponent(i.id)}`, job: null
})),
...(data?.runtime.jobs ?? []).map(j => ({
key: `job-${j.id}`, kind: j.kind === 'schedule' ? 'schedule' : 'updates', server: j.server, at: j.requested_at,
key: `job-${j.id}`, kind: j.kind === 'schedule' ? 'schedule' : 'updates', server: j.server, server_deleted_at: j.server_deleted_at, at: j.requested_at,
title: j.kind === 'schedule' ? 'Scheduled restart' : j.kind === 'update' ? 'Plutonium update' : 'Legacy maintenance',
state: ['queued', 'restarting'].includes(j.state) ? 'active' : j.state === 'failed' ? 'attention' : 'done',
label: j.state === 'failed' ? 'Needs attention' : j.state === 'done' ? 'Completed' : j.state === 'cancelled' ? 'Cancelled' : j.state === 'restarting' ? 'Recovering' : 'Queued',
@@ -58,12 +59,12 @@
}))
].sort((a, b) => b.at.localeCompare(a.at)));
const isLatest = (item: (typeof activities)[number]) => !item.job || !data?.runtime.jobs.some(j => j.server === item.server && j.kind === item.job!.kind && j.id > item.job!.id);
const attention = $derived(activities.filter(a => a.state === 'attention' && isLatest(a)).length);
const waiting = $derived(activities.filter(a => a.state === 'active').length);
const attention = $derived(activities.filter(a => !a.server_deleted_at && a.state === 'attention' && isLatest(a)).length);
const waiting = $derived(activities.filter(a => !a.server_deleted_at && a.state === 'active').length);
const matching = $derived(activities.filter(a =>
(kind === 'all' || a.kind === kind) &&
(!server || `${a.server} ${serverTitle(a.server)}`.toLowerCase().includes(server.toLowerCase())) &&
(view === 'overview' ? a.state !== 'done' && isLatest(a) : status === 'all' || a.state === status)
(view === 'overview' ? !a.server_deleted_at && a.state !== 'done' && isLatest(a) : status === 'all' || a.state === status)
));
$effect(() => {
const params = $page.url.searchParams;
@@ -186,12 +187,12 @@
<h2>{view === 'overview' ? 'Active work & attention' : 'Maintenance activity'}</h2>
<div class="activity">{#each matching.slice(0, limit) as item (item.key)}
<article class="card" id={item.key}>
<div class="row-heading"><h3>{item.title} · <ServerIdentity name={item.server} /></h3><Badge variant={item.state === 'attention' ? 'danger' : item.state === 'active' ? 'info' : 'neutral'}>{item.label}</Badge></div>
<div class="row-heading"><h3>{item.title} · <HistoricalServerIdentity name={item.server} deletedAt={item.server_deleted_at} /></h3><Badge variant={item.state === 'attention' ? 'danger' : item.state === 'active' ? 'info' : 'neutral'}>{item.label}</Badge></div>
{#if item.detail}<p>{item.detail}</p>{/if}
<p class="muted">{$timeFormat.dateTime(item.at)}{item.job?.target_revision ? ` · Expected r${item.job.target_revision} or newer` : ''}</p>
<div class="links">{#if item.href}<a href={item.href}>{item.job ? 'View restart operation' : 'View evidence and recovery'}</a>{/if}
{#if item.job?.state === 'queued'}<Button size="sm" disabled={busy} onclick={() => act({ action: 'cancel', id: item.job!.id })}>Cancel restart</Button>{/if}
{#if item.job?.state === 'failed' && item.job.operation_state === 'done'}<Button size="sm" disabled={busy || data.policies.paused} onclick={() => act({ action: 'retry', id: item.job!.id })}>Retry after inspection</Button>{/if}
{#if item.job?.state === 'queued' && !item.server_deleted_at}<Button size="sm" disabled={busy} onclick={() => act({ action: 'cancel', id: item.job!.id })}>Cancel restart</Button>{/if}
{#if item.job?.state === 'failed' && !item.server_deleted_at && item.job.operation_state === 'done'}<Button size="sm" disabled={busy || data.policies.paused} onclick={() => act({ action: 'retry', id: item.job!.id })}>Retry after inspection</Button>{/if}
</div>
</article>
{:else}<EmptyState title={view === 'overview' ? 'No active maintenance in this view' : 'No matching activity'} description="Findings, scheduled restarts, and update verification appear here." />{/each}</div>
src/routes/maintenance/[id]/+page.svelte
+6 -5
View File
@@ -33,10 +33,11 @@
const actions = $derived(detail?.actions.slice().reverse() ?? []);
const events = $derived(detail?.events.slice().reverse() ?? []);
const activeOperation = $derived(actions.some(action => ['queued', 'waiting', 'running', 'awaiting-health'].includes(action.operation_state ?? '')));
const canRetry = $derived(incident?.state === 'attention' || incident?.state === 'dismissed');
const canDismiss = $derived(incident?.state === 'open' || incident?.state === 'attention');
const canRetry = $derived(!incident?.server_deleted_at && (incident?.state === 'attention' || incident?.state === 'dismissed'));
const canDismiss = $derived(!incident?.server_deleted_at && (incident?.state === 'open' || incident?.state === 'attention'));
const explanation = $derived.by(() => {
if (!incident) return '';
if (incident.server_deleted_at) return 'This server was deleted. Its recorded finding, evidence, and recovery results are preserved below.';
if (incident.state === 'resolved') return incident.kind === 'drift' ? 'The observed file drift has cleared.' : 'The server passed the checks for this incident.';
if (incident.state === 'dismissed') return 'This incident was dismissed. Retry to allow automatic recovery again.';
if (incident.state === 'attention') return 'Automatic recovery needs attention. Review the failed operation and server diagnostics before retrying.';
@@ -107,11 +108,11 @@
{#if incident && detail}
<section class="summary" aria-label="Incident status">
<div class="row-heading"><WatchdogStatus state={incident.state} /><a href={`/servers/${encodeURIComponent(incident.server)}`}>View server</a></div>
<div class="row-heading"><WatchdogStatus state={incident.state} />{#if incident.server_deleted_at}<Badge>Deleted</Badge>{:else}<a href={`/servers/${encodeURIComponent(incident.server)}`}>View server</a>{/if}</div>
<h2>{incident.summary}</h2>
<p>{explanation}</p>
{#if activeOperation && (canRetry || canDismiss)}<p>Complete or cancel the active provisioning operation before retrying or dismissing this incident.</p>{/if}
{#if incident.kind === 'drift'}<p><strong>Drift repair deletes the existing instance and all of its files, then rebuilds it with the latest promoted setup.</strong> The repair is recorded as a campaign.</p>{:else if incident.state !== 'resolved'}<p>Automatic health recovery first tries a restart, then rebuilds if needed. A rebuild deletes the existing instance and all of its files and installs the latest promoted setup.</p>{/if}
{#if !incident.server_deleted_at}{#if incident.kind === 'drift'}<p><strong>Drift repair deletes the existing instance and all of its files, then rebuilds it with the latest promoted setup.</strong> The repair is recorded as a campaign.</p>{:else if incident.state !== 'resolved'}<p>Automatic health recovery first tries a restart, then rebuilds if needed. A rebuild deletes the existing instance and all of its files and installs the latest promoted setup.</p>{/if}{/if}
<dl>
<div><dt>First detected</dt><dd>{$timeFormat.dateTime(incident.first_seen_at)}</dd></div>
<div><dt>Last observed</dt><dd>{$timeFormat.dateTime(incident.last_seen_at)}</dd></div>
@@ -119,7 +120,7 @@
{#if evidence.containerState}<div><dt>Observed Komodo state</dt><dd>{evidence.containerState}</dd></div>{/if}
</dl>
{#if evidence.reason && evidence.reason !== incident.summary}<p class="observed-reason">Observed reason: {evidence.reason}</p>{/if}
<div class="links"><a href={`/servers/${encodeURIComponent(incident.server)}?tab=diagnostics`}>Diagnose connection</a><a href={`/servers/${encodeURIComponent(incident.server)}?tab=logs`}>Open server logs</a></div>
{#if !incident.server_deleted_at}<div class="links"><a href={`/servers/${encodeURIComponent(incident.server)}?tab=diagnostics`}>Diagnose connection</a><a href={`/servers/${encodeURIComponent(incident.server)}?tab=logs`}>Open server logs</a></div>{/if}
</section>
<section aria-labelledby="recovery-title">
src/routes/provisioning/+page.svelte
+6 -6
View File
@@ -7,7 +7,7 @@
import Tabs from '$ui/components/primitives/Tabs.svelte';
import StatusBadge from '$ui/components/primitives/StatusBadge.svelte';
import ListFooter from '$ui/components/primitives/ListFooter.svelte';
import ServerIdentity from '$ui/components/ServerIdentity.svelte';
import HistoricalServerIdentity from '$ui/components/HistoricalServerIdentity.svelte';
import { serverTitle } from '$ui/server-labels.js';
import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
import Badge from '$ui/components/primitives/Badge.svelte';
@@ -22,7 +22,7 @@
let lifecycleOpen = $state(false);
interface Deployment {
id: string; server: string; action: ProvisioningAction;
id: string; server: string; server_deleted_at?: string | null; action: ProvisioningAction;
state: 'queued' | 'awaiting-health' | 'waiting' | 'running' | 'done' | 'failed' | 'warning' | 'interrupted' | 'cancelled';
stage: string; started_at: string; finished_at: string | null;
triggered_by: string; error: string | null; warnings: string; server_exists: number;
@@ -72,10 +72,10 @@
let action = $state('all');
let search = $state('');
let refreshing = false;
let active = $derived(deployments.filter(d => ['queued', 'awaiting-health', 'running', 'waiting'].includes(d.state)));
let active = $derived(deployments.filter(d => !d.server_deleted_at && ['queued', 'awaiting-health', 'running', 'waiting'].includes(d.state)));
const needsAttention = (deployment: Deployment) => ['failed', 'interrupted', 'warning'].includes(deployment.state);
const attention = $derived(deployments.filter(needsAttention));
let history = $derived(deployments.filter(d => !['queued', 'awaiting-health', 'running', 'waiting'].includes(d.state) &&
let history = $derived(deployments.filter(d => (d.server_deleted_at || !['queued', 'awaiting-health', 'running', 'waiting'].includes(d.state)) &&
(historyState === 'all' || needsAttention(d)) &&
(action === 'all' || d.action === action) && (d.server + ' ' + serverTitle(d.server)).toLowerCase().includes(search.toLowerCase())));
async function load() {
@@ -151,7 +151,7 @@
<article class="deployment-row"><a class="row-link" href="/provisioning/{d.id}">
<div class="row-head">
<div><Badge variant={d.action === 'create' ? 'info' : 'neutral'}>{actionLabels[d.action]}</Badge>
<strong class="server"><ServerIdentity name={d.server} /></strong>
<strong class="server"><HistoricalServerIdentity name={d.server} deletedAt={d.server_deleted_at} /></strong>
</div>
<StatusBadge state={d.state} />
</div>
@@ -159,7 +159,7 @@
<div class="meta"><span>Started {fmt(d.started_at)}</span>{#if d.finished_at}<span>Finished {fmt(d.finished_at)}</span>{/if}<span>By {d.triggered_by}</span></div>
{#if d.error}<p class="error">{d.error}</p>{/if}
{#each warnings(d) as warning}<p class="error">{warning}</p>{/each}
</a>{#if d.canCancelRemoval}<div class="row-actions"><Button size="sm" disabled={cancelling} onclick={() => cancelRequests([d.id])}>Cancel queued removal</Button></div>{/if}</article>
</a>{#if d.canCancelRemoval && !d.server_deleted_at}<div class="row-actions"><Button size="sm" disabled={cancelling} onclick={() => cancelRequests([d.id])}>Cancel queued removal</Button></div>{/if}</article>
{/snippet}
<style>
src/routes/provisioning/[id]/+page.svelte
+9 -8
View File
@@ -18,7 +18,7 @@
import { actionLabels, actionCompleted, type ProvisioningAction } from '$lib/provisioning.js';
interface Deployment {
id: string; expected_map?: string; server: string; action: ProvisioningAction; state: string;
id: string; expected_map?: string; server: string; server_deleted_at?: string | null; action: ProvisioningAction; state: string;
stage: string; started_at: string; finished_at: string | null;
triggered_by: string; error: string | null; warnings: string; server_exists: number;
}
@@ -30,7 +30,7 @@
let open = $state(false);
let resuming = $state(false);
let deployment = $derived(detail?.deployment);
let active = $derived(['queued', 'awaiting-health', 'waiting', 'running'].includes(deployment?.state ?? ''));
let active = $derived(!deployment?.server_deleted_at && ['queued', 'awaiting-health', 'waiting', 'running'].includes(deployment?.state ?? ''));
let warnings = $derived.by(() => {
try { return JSON.parse(deployment?.warnings ?? '[]') as string[]; } catch { return []; }
});
@@ -58,6 +58,7 @@
});
let explanation = $derived.by(() => {
if (!deployment) return '';
if (deployment.server_deleted_at) return 'This server was deleted. Its recorded operation and results are preserved below.';
if (detail?.canResumeRebuild) return 'The old Komodo instance was deleted. Resume file cleanup, then create a fresh instance in the reserved server slot.';
if (['start', 'restart', 'stop'].includes(deployment.action)) {
if (deployment.state === 'done') return deployment.action === 'stop' ? 'Komodo confirmed the container is stopped. Start it again when ready.' : 'The container is healthy and RCON responds.';
@@ -138,16 +139,16 @@
{#if deployment}
<section class="summary">
<div class="top"><StatusBadge state={deployment.state} />
{#if deployment.server_exists}<a href="/servers/{encodeURIComponent(deployment.server)}">View server</a>{/if}
{#if deployment.server_deleted_at}<Badge>Deleted</Badge>{:else if deployment.server_exists}<a href="/servers/{encodeURIComponent(deployment.server)}">View server</a>{/if}
</div>
<h2>{deployment.state === 'done' ? actionCompleted[deployment.action] : deployment.stage}</h2>
<p class="explanation" aria-live="polite">{explanation}</p>
<div class="timing"><span>Total time <strong>{duration(deployment.started_at, deployment.finished_at ? new Date(deployment.finished_at).getTime() : now)}</strong></span>{#if active && stageStarted}<span>This step <strong>{duration(stageStarted, now)}</strong></span>{/if}</div>
{#if detail?.canResumeVerification}<div><Button disabled={resuming} loading={resuming} onclick={resume}>Resume health verification</Button></div>{/if}
{#if detail?.canResumeRebuild}<div><Button disabled={resuming} loading={resuming} onclick={resume}>Resume rebuild cleanup</Button></div>{/if}
{#if detail?.canResumeCleanup}<div><Button disabled={resuming} loading={resuming} onclick={resume}>Resume cleanup</Button></div>{/if}
{#if detail?.canCancelRemoval}<div><Button disabled={resuming} loading={resuming} onclick={cancelRemoval}>Cancel queued removal</Button><p class="muted">Available until removal begins. Cancelling keeps the server and its files.</p></div>{/if}
{#if deployment.server_exists && deployment.state !== 'done' && deployment.state !== 'cancelled'}<div class="recovery-links"><a href="/servers/{encodeURIComponent(deployment.server)}?tab=logs">Open startup logs</a><a href="/servers/{encodeURIComponent(deployment.server)}?tab=diagnostics">Diagnose connection</a>{#if deployment.stage.includes('map')}<a href="/servers/{encodeURIComponent(deployment.server)}?tab=config">Review map configuration</a>{/if}</div>{/if}
{#if detail?.canResumeVerification && !deployment.server_deleted_at}<div><Button disabled={resuming} loading={resuming} onclick={resume}>Resume health verification</Button></div>{/if}
{#if detail?.canResumeRebuild && !deployment.server_deleted_at}<div><Button disabled={resuming} loading={resuming} onclick={resume}>Resume rebuild cleanup</Button></div>{/if}
{#if detail?.canResumeCleanup && !deployment.server_deleted_at}<div><Button disabled={resuming} loading={resuming} onclick={resume}>Resume cleanup</Button></div>{/if}
{#if detail?.canCancelRemoval && !deployment.server_deleted_at}<div><Button disabled={resuming} loading={resuming} onclick={cancelRemoval}>Cancel queued removal</Button><p class="muted">Available until removal begins. Cancelling keeps the server and its files.</p></div>{/if}
{#if deployment.server_exists && !deployment.server_deleted_at && deployment.state !== 'done' && deployment.state !== 'cancelled'}<div class="recovery-links"><a href="/servers/{encodeURIComponent(deployment.server)}?tab=logs">Open startup logs</a><a href="/servers/{encodeURIComponent(deployment.server)}?tab=diagnostics">Diagnose connection</a>{#if deployment.stage.includes('map')}<a href="/servers/{encodeURIComponent(deployment.server)}?tab=config">Review map configuration</a>{/if}</div>{/if}
{#if deployment.error}<p role="alert" class="error">{deployment.error}</p>{/if}
{#each warnings as warning}<p class="warning">{warning}</p>{/each}
{#if deployment.expected_map}<p>Expected map: <strong>{deployment.expected_map}</strong></p>{/if}
src/routes/provisioning/collections/[id]/+page.svelte
+6 -5
View File
@@ -2,6 +2,7 @@
import { onMount } from 'svelte';
import { pollVisible } from '$ui/poll.js';
import ServerIdentity from '$ui/components/ServerIdentity.svelte';
import HistoricalServerIdentity from '$ui/components/HistoricalServerIdentity.svelte';
import { page } from '$app/stores';
import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
import Breadcrumbs from '$ui/components/primitives/Breadcrumbs.svelte';
@@ -13,12 +14,12 @@
let error = $state('');
let requestNumber = 0;
let collection = $derived(detail?.collection);
const launching = (server: { state: string }) => ['running', 'awaiting-health'].includes(server.state);
const launching = (server: { state: string; server_deleted_at?: string | null }) => !server.server_deleted_at && ['running', 'awaiting-health'].includes(server.state);
const needsAttention = (server: { state: string }) => ['failed', 'interrupted', 'warning', 'cancelled'].includes(server.state);
const exact = $derived(collection?.mode === 'exact');
const currentServers = $derived(detail?.servers.filter(launching) ?? []);
const otherLaunches = $derived(detail?.queue.active.filter(active => !detail?.servers.some(server => server.id === active.id)) ?? []);
const firstQueued = $derived(detail?.servers.find(server => server.state === 'queued'));
const firstQueued = $derived(detail?.servers.find(server => !server.server_deleted_at && server.state === 'queued'));
const labels: Record<string, string> = { queued: 'Queued', running: 'Launching', 'awaiting-health': 'Checking readiness', done: 'Ready', warning: 'Needs review', failed: 'Failed', interrupted: 'Interrupted', waiting: 'Waiting for empty server', cancelled: 'Cancelled' };
const waitingForPort = (server: CollectionServer) => server.state === 'queued' && (server.stage.startsWith('Waiting for removal') || server.stage.startsWith('Waiting for port'));
function stateLabel(server: CollectionServer) {
@@ -81,7 +82,7 @@
{#each currentServers as server (server.id)}
<a class="current-launch" href="/provisioning/{server.id}">
<span class="activity" aria-hidden="true"></span>
<div><strong>{server.action === 'destroy' ? 'Removing: ' : ''}<ServerIdentity name={server.server} /></strong><p>{server.state === 'awaiting-health' ? 'Checking the container, connection, and expected map.' : server.stage}</p></div>
<div><strong>{server.action === 'destroy' ? 'Removing: ' : ''}<HistoricalServerIdentity name={server.server} deletedAt={server.server_deleted_at} /></strong><p>{server.state === 'awaiting-health' ? 'Checking the container, connection, and expected map.' : server.stage}</p></div>
<span class="operation-link">Follow progress <span aria-hidden="true">→</span></span>
</a>
{/each}
@@ -100,7 +101,7 @@
{#if exact && detail.retainedServers.length}
<details><summary>{detail.retainedServers.length} retained server{detail.retainedServers.length === 1 ? '' : 's'} (unchanged)</summary>
<ul>{#each detail.retainedServers as name}<li><ServerIdentity {name} /> <span class="muted">{name}</span></li>{/each}</ul>
<ul>{#each detail.retainedServers as name}<li><HistoricalServerIdentity {name} deletedAt={detail.retainedServerDeletedAt?.[name]} /> <span class="muted">{name}</span></li>{/each}</ul>
</details>
{/if}
<section class="queue" aria-labelledby="queue-title">
@@ -111,7 +112,7 @@
<a class="server" href="/provisioning/{server.id}">
<span class="order" class:ready={server.state === 'done'} aria-hidden="true">{server.state === 'done' ? '✓' : String(index + 1).padStart(2, '0')}</span>
<div class="server-content">
<div class="server-heading"><strong><ServerIdentity name={server.server} /></strong><Badge variant={needsAttention(server) ? 'danger' : server.state === 'done' ? 'online' : launching(server) ? 'info' : 'neutral'}>{stateLabel(server)}</Badge></div>
<div class="server-heading"><strong><HistoricalServerIdentity name={server.server} deletedAt={server.server_deleted_at} /></strong><Badge variant={needsAttention(server) ? 'danger' : server.state === 'done' ? 'online' : launching(server) ? 'info' : 'neutral'}>{stateLabel(server)}</Badge></div>
<p class:error={!!server.error}>{server.error ?? (server.state === 'queued' ? queueHint(server) : server.state === 'done' ? server.action === 'destroy' ? server.stage : 'Startup checks passed.' : server.stage)}</p>
<div class="server-meta"><span>{server.action === 'destroy' ? 'Remove server' : `Create server · Port ${server.port}`}</span><span>{needsAttention(server) ? 'Review & recover' : 'View operation'} <span aria-hidden="true">→</span></span></div>
</div>