josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Preserve deleted server history

Browse Source 
Retain historical server identities, results, and evidence after deletion. Mark archived entries as Deleted and prevent old work from targeting servers that reuse the same name.

Fixes #16.
This commit is contained in:
josh
2026-09-27 21:48:55 -04:00
parent 53b29f726c
commit 981fb27c1c
61 changed files with 929 additions and 223 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/db/index.ts
+4
View File
@@ -7,6 +7,7 @@ import { requirement, type BetaPlan, type BetaRequirement } from '../../beta-pla
import { emptyMetrics } from '../fleet/beta-evaluation.js';
import { backfillLeaderboard } from '../fleet/leaderboard-ingest.js';
import { initializeFleetSettings } from './fleet-settings.js';
import { migrateServerHistory } from './server-history.js';
let _db: Database.Database | null = null;
@@ -124,6 +125,8 @@ function backfillMultiScriptJoins(handle: Database.Database): void {
FROM beta_servers bs
JOIN betas b ON b.id = bs.beta_id
WHERE b.script_id IS NOT NULL
AND bs.server_deleted_at IS NULL
AND EXISTS (SELECT 1 FROM servers WHERE name = bs.server)
AND NOT EXISTS (
SELECT 1 FROM beta_server_scripts x
WHERE x.beta_id = bs.beta_id AND x.server = bs.server AND x.script_id = b.script_id
@@ -329,6 +332,7 @@ export function db(): Database.Database {
{ name: 'mode', ddl: "ALTER TABLE server_launch_collections ADD COLUMN mode TEXT NOT NULL DEFAULT 'fill'" },
{ name: 'retained_servers', ddl: "ALTER TABLE server_launch_collections ADD COLUMN retained_servers TEXT NOT NULL DEFAULT '[]'" }
]);
migrateServerHistory(handle);
migrateTable(handle, 'server_collection_previews', [
{ name: 'mode', ddl: "ALTER TABLE server_collection_previews ADD COLUMN mode TEXT NOT NULL DEFAULT 'fill'" },
{ name: 'fleet_snapshot', ddl: 'ALTER TABLE server_collection_previews ADD COLUMN fleet_snapshot TEXT' }
src/lib/server/db/schema.sql
+11 -4
View File
@@ -11,6 +11,7 @@ PRAGMA foreign_keys = ON;
CREATE TABLE IF NOT EXISTS leaderboard_matches (
id INTEGER PRIMARY KEY AUTOINCREMENT,
server TEXT NOT NULL,
server_deleted_at TEXT,
game TEXT NOT NULL,
map TEXT NOT NULL,
map_key TEXT NOT NULL,
@@ -181,7 +182,8 @@ CREATE INDEX IF NOT EXISTS idx_betas_script ON betas(script_id);
CREATE TABLE IF NOT EXISTS beta_servers (
beta_id INTEGER NOT NULL REFERENCES betas(id) ON DELETE CASCADE,
server TEXT NOT NULL REFERENCES servers(name),
server TEXT NOT NULL,
server_deleted_at TEXT,
had_original INTEGER NOT NULL, -- deprecated; see beta_server_scripts
prior_version_id INTEGER REFERENCES script_versions(id), -- deprecated; see beta_server_scripts
stage TEXT NOT NULL, -- 'deploying'|'awaiting-empty'|'restarting'|'awaiting-health'|'awaiting-match'|'ready'|'failed'
@@ -206,7 +208,7 @@ CREATE TABLE IF NOT EXISTS beta_test_plans (
CREATE TABLE IF NOT EXISTS beta_test_runs (
id TEXT PRIMARY KEY,
beta_id INTEGER NOT NULL REFERENCES betas(id) ON DELETE CASCADE,
server TEXT NOT NULL REFERENCES servers(name),
server TEXT NOT NULL,
phase INTEGER NOT NULL DEFAULT 0,
created_at TEXT NOT NULL,
observing_at TEXT,
@@ -261,7 +263,7 @@ CREATE INDEX IF NOT EXISTS idx_beta_scripts_script ON beta_scripts(script_id);
-- lives in beta_file_snapshots.
CREATE TABLE IF NOT EXISTS beta_server_scripts (
beta_id INTEGER NOT NULL REFERENCES betas(id) ON DELETE CASCADE,
server TEXT NOT NULL REFERENCES servers(name),
server TEXT NOT NULL,
script_id INTEGER NOT NULL REFERENCES scripts(id) ON DELETE CASCADE,
had_original INTEGER NOT NULL,
prior_version_id INTEGER REFERENCES script_versions(id),
@@ -302,6 +304,7 @@ CREATE INDEX IF NOT EXISTS idx_deployments_started ON deployments(started_at DES
CREATE TABLE IF NOT EXISTS deployment_servers (
deployment_id TEXT NOT NULL REFERENCES deployments(id) ON DELETE CASCADE,
server TEXT NOT NULL, -- Campaign observation survives a rebuild's registry gap.
server_deleted_at TEXT,
files_changed INTEGER NOT NULL,
restart_state TEXT NOT NULL, -- pending | done | failed | manual | held
players INTEGER,
@@ -373,6 +376,7 @@ CREATE TABLE IF NOT EXISTS plutonium_versions (
CREATE TABLE IF NOT EXISTS maintenance_jobs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
server TEXT NOT NULL,
server_deleted_at TEXT,
state TEXT NOT NULL DEFAULT 'queued',
target_revision INTEGER,
requested_at TEXT NOT NULL,
@@ -411,6 +415,7 @@ CREATE INDEX IF NOT EXISTS idx_telemetry_requests_received ON telemetry_requests
CREATE TABLE IF NOT EXISTS server_deployments (
id TEXT PRIMARY KEY,
server TEXT NOT NULL,
server_deleted_at TEXT,
action TEXT NOT NULL CHECK (action IN ('create', 'start', 'restart', 'rebuild', 'stop', 'destroy')),
state TEXT NOT NULL,
stage TEXT NOT NULL,
@@ -486,7 +491,8 @@ CREATE TABLE IF NOT EXISTS server_launch_collections (
name TEXT NOT NULL,
created_at TEXT NOT NULL,
mode TEXT NOT NULL DEFAULT 'fill',
retained_servers TEXT NOT NULL DEFAULT '[]'
retained_servers TEXT NOT NULL DEFAULT '[]',
retained_server_deletions TEXT NOT NULL DEFAULT '{}'
);
CREATE TABLE IF NOT EXISTS server_collection_members (
@@ -629,6 +635,7 @@ CREATE TABLE IF NOT EXISTS metric_samples (
CREATE TABLE IF NOT EXISTS watchdog_incidents (
id TEXT PRIMARY KEY,
server TEXT NOT NULL,
server_deleted_at TEXT,
kind TEXT NOT NULL CHECK (kind IN ('health','drift')),
state TEXT NOT NULL CHECK (state IN ('open','recovering','attention','resolved','dismissed')),
summary TEXT NOT NULL,
src/lib/server/db/server-history.ts
+130
View File
@@ -0,0 +1,130 @@
import type Database from 'better-sqlite3';
const historyTables = ['beta_servers', 'deployment_servers', 'server_deployments', 'leaderboard_matches', 'watchdog_incidents', 'maintenance_jobs'];
const now = "strftime('%Y-%m-%dT%H:%M:%fZ','now')";
/** Detach durable beta records without dropping their evidence, indexes, or activity triggers. */
function detachBetaHistory(handle: Database.Database): void {
for (const name of ['beta_servers', 'beta_test_runs', 'beta_server_scripts']) {
const { sql } = handle.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?").get(name) as { sql: string };
const replacement = sql.replace(/server\s+TEXT NOT NULL REFERENCES servers\(name\)/, 'server TEXT NOT NULL');
if (replacement === sql) continue;
const dependents = handle.prepare("SELECT sql FROM sqlite_master WHERE tbl_name = ? AND type IN ('index','trigger') AND sql IS NOT NULL")
.all(name) as { sql: string }[];
const columns = (handle.prepare(`PRAGMA table_info(${name})`).all() as { name: string }[]).map(col => col.name).join(', ');
handle.pragma('foreign_keys = OFF');
try {
handle.transaction(() => {
handle.exec(replacement.replace(name, name + '_history_new'));
handle.exec(`INSERT INTO ${name}_history_new (rowid, ${columns}) SELECT rowid, ${columns} FROM ${name};
DROP TABLE ${name}; ALTER TABLE ${name}_history_new RENAME TO ${name};`);
for (const dependent of dependents) handle.exec(dependent.sql);
if ((handle.pragma('foreign_key_check') as unknown[]).length) throw new Error('Server history migration found broken references');
})();
} finally { handle.pragma('foreign_keys = ON'); }
}
}
// A rebuild owns the replacement instance. Only its current operation and the
// Watchdog records observing it cross the deliberate gap in the live registry.
const removingRebuild = `SELECT d.id FROM server_deployments d
JOIN server_rebuild_jobs r ON r.deployment_id = d.id
WHERE d.server = OLD.name AND d.server_deleted_at IS NULL AND d.action = 'rebuild'
AND d.state IN ('waiting','running','failed','interrupted')
AND d.stage = 'Removing server files and registry' AND r.removed_at IS NULL
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server
AND newer.rowid > d.rowid)`;
/** Installed after additive migrations so existing databases can boot safely. */
export function migrateServerHistory(handle: Database.Database): void {
detachBetaHistory(handle);
handle.transaction(() => {
for (const table of historyTables) {
const columns = handle.prepare(`PRAGMA table_info(${table})`).all() as { name: string }[];
if (!columns.some(column => column.name === 'server_deleted_at')) handle.exec(`ALTER TABLE ${table} ADD COLUMN server_deleted_at TEXT`);
}
const collectionColumns = handle.prepare('PRAGMA table_info(server_launch_collections)').all() as { name: string }[];
if (!collectionColumns.some(column => column.name === 'retained_server_deletions')) {
handle.exec("ALTER TABLE server_launch_collections ADD COLUMN retained_server_deletions TEXT NOT NULL DEFAULT '{}'");
}
// Historical unfinished work cannot claim the next server with this name.
handle.exec(`DROP INDEX IF EXISTS idx_maintenance_pending;
CREATE UNIQUE INDEX idx_maintenance_pending ON maintenance_jobs(server)
WHERE state IN ('queued', 'restarting') AND server_deleted_at IS NULL;
DROP INDEX IF EXISTS idx_watchdog_active;
CREATE UNIQUE INDEX idx_watchdog_active ON watchdog_incidents(server,kind)
WHERE state != 'resolved' AND server_deleted_at IS NULL;
DROP TRIGGER IF EXISTS preserve_deleted_server_history;
CREATE TRIGGER preserve_deleted_server_history BEFORE DELETE ON servers
BEGIN
UPDATE server_launch_collections SET retained_server_deletions = json_patch(retained_server_deletions, json_object(OLD.name, ${now}))
WHERE EXISTS (SELECT 1 FROM json_each(retained_servers) WHERE value = OLD.name)
AND NOT EXISTS (SELECT 1 FROM json_each(retained_server_deletions) WHERE key = OLD.name);
UPDATE beta_servers SET server_deleted_at = ${now} WHERE server = OLD.name AND server_deleted_at IS NULL;
UPDATE deployment_servers SET server_deleted_at = ${now}
WHERE server = OLD.name AND server_deleted_at IS NULL AND deployment_id NOT IN (
SELECT campaign_id FROM watchdog_actions WHERE campaign_id IS NOT NULL AND operation_id IN (${removingRebuild}));
UPDATE watchdog_incidents SET server_deleted_at = ${now}
WHERE server = OLD.name AND server_deleted_at IS NULL AND id NOT IN (
SELECT incident_id FROM watchdog_actions WHERE operation_id IN (${removingRebuild}));
UPDATE maintenance_jobs SET server_deleted_at = ${now},
finished_at = CASE WHEN state IN ('queued','restarting') THEN ${now} ELSE finished_at END,
detail = CASE WHEN state IN ('queued','restarting') THEN COALESCE(detail || '; ', '') || 'Server deleted' ELSE detail END,
state = CASE WHEN state IN ('queued','restarting') THEN 'cancelled' ELSE state END
WHERE server = OLD.name AND server_deleted_at IS NULL;
UPDATE leaderboard_matches SET server_deleted_at = ${now},
ended_at = CASE WHEN status = 'active' THEN COALESCE(ended_at, ${now}) ELSE ended_at END,
status = CASE WHEN status = 'active' THEN 'interrupted' ELSE status END
WHERE server = OLD.name AND server_deleted_at IS NULL;
INSERT INTO server_deployment_events (deployment_id, created_at, message)
SELECT id, ${now}, 'Cancelled: server deleted' FROM server_deployments
WHERE server = OLD.name AND server_deleted_at IS NULL AND state IN ('queued','waiting','running','awaiting-health')
AND id NOT IN (${removingRebuild})
AND NOT (action = 'destroy' AND state = 'running' AND stage = 'Removing server files and registry');
UPDATE server_deployments SET state = 'cancelled', error = 'Server deleted', finished_at = ${now}
WHERE server = OLD.name AND server_deleted_at IS NULL AND state IN ('queued','waiting','running','awaiting-health')
AND id NOT IN (${removingRebuild})
AND NOT (action = 'destroy' AND state = 'running' AND stage = 'Removing server files and registry');
UPDATE server_deployments SET server_deleted_at = ${now}
WHERE server = OLD.name AND server_deleted_at IS NULL AND id NOT IN (${removingRebuild});
END;`);
if (!handle.prepare("SELECT 1 FROM settings WHERE key = 'server_history_v1'").get()) {
// Recover the availability of history already orphaned before this
// migration. Queued launches and rebuild registry gaps are not deletions.
const liveRebuilds = `SELECT d.id FROM server_deployments d JOIN server_rebuild_jobs r ON r.deployment_id = d.id
WHERE d.server_deleted_at IS NULL AND d.action = 'rebuild' AND (d.state IN ('waiting','queued','running','awaiting-health')
OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry'))
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`;
for (const table of historyTables) {
const exception = table === 'server_deployments'
? `AND (action != 'create' OR state IN ('done','warning') OR EXISTS (
SELECT 1 FROM server_deployments removal WHERE removal.server = server_deployments.server
AND removal.rowid > server_deployments.rowid AND removal.action = 'destroy' AND removal.state IN ('done','warning')))
AND id NOT IN (${liveRebuilds})`
: table === 'deployment_servers'
? `AND deployment_id NOT IN (SELECT campaign_id FROM watchdog_actions WHERE campaign_id IS NOT NULL AND operation_id IN (${liveRebuilds}))`
: table === 'watchdog_incidents'
? `AND id NOT IN (SELECT incident_id FROM watchdog_actions WHERE operation_id IN (${liveRebuilds}))` : '';
handle.exec(`UPDATE ${table} SET server_deleted_at = ${now} WHERE server_deleted_at IS NULL
AND NOT EXISTS (SELECT 1 FROM servers WHERE name = ${table}.server AND destroyed_at IS NULL) ${exception}`);
}
handle.exec(`UPDATE server_launch_collections SET retained_server_deletions = json_patch(retained_server_deletions,
(SELECT json_group_object(value, ${now}) FROM json_each(retained_servers) retained
WHERE NOT EXISTS (SELECT 1 FROM servers WHERE name = retained.value AND destroyed_at IS NULL)
AND NOT EXISTS (SELECT 1 FROM server_deployments d WHERE d.server = retained.value
AND d.server_deleted_at IS NULL AND d.action IN ('create','rebuild')
AND d.state IN ('queued','running','awaiting-health','waiting'))
AND NOT EXISTS (SELECT 1 FROM json_each(retained_server_deletions) WHERE key = retained.value)));`);
handle.exec(`UPDATE maintenance_jobs SET state = 'cancelled', finished_at = ${now}, detail = COALESCE(detail || '; ', '') || 'Server deleted'
WHERE server_deleted_at IS NOT NULL AND state IN ('queued','restarting');
UPDATE leaderboard_matches SET status = 'interrupted', ended_at = COALESCE(ended_at, server_deleted_at)
WHERE server_deleted_at IS NOT NULL AND status = 'active';
INSERT INTO server_deployment_events (deployment_id, created_at, message)
SELECT id, ${now}, 'Cancelled: server deleted' FROM server_deployments
WHERE server_deleted_at IS NOT NULL AND state IN ('queued','waiting','running','awaiting-health');
UPDATE server_deployments SET state = 'cancelled', error = 'Server deleted', finished_at = ${now}
WHERE server_deleted_at IS NOT NULL AND state IN ('queued','waiting','running','awaiting-health');
INSERT INTO settings (key, value) VALUES ('server_history_v1', '1');`);
}
})();
}