294 lines
16 KiB
TypeScript
294 lines
16 KiB
TypeScript
import Database from 'better-sqlite3';
|
|
import { afterEach, describe, expect, it } from 'vitest';
|
|
import schema from '../../src/lib/server/db/schema.sql?raw';
|
|
import { migrateServerHistory } from '../../src/lib/server/db/server-history.js';
|
|
|
|
let handle: Database.Database;
|
|
afterEach(() => handle?.close());
|
|
|
|
function database(legacy = false) {
|
|
handle = new Database(':memory:');
|
|
let sql = schema;
|
|
if (legacy) {
|
|
sql = sql.replace(/^\s*server_deleted_at TEXT,\n/gm, '');
|
|
for (const name of ['beta_servers', 'beta_test_runs', 'beta_server_scripts']) {
|
|
sql = sql.replace(
|
|
new RegExp(
|
|
`(CREATE TABLE IF NOT EXISTS ${name} \\([\\s\\S]*?server\\s+TEXT NOT NULL)(,)`
|
|
),
|
|
'$1 REFERENCES servers(name)$2'
|
|
);
|
|
}
|
|
}
|
|
handle.exec(sql);
|
|
return handle;
|
|
}
|
|
|
|
function seedHistory(db: Database.Database) {
|
|
db.exec(`INSERT INTO servers (name, game_port) VALUES ('server', 28960);
|
|
INSERT INTO file_blobs(sha256,size,content) VALUES('hash',4,X'74657374');
|
|
INSERT INTO scripts (id, name, target_rel_path) VALUES (1, 'script', 'test.gsc');
|
|
INSERT INTO script_versions (id, script_id, version, sha256, size, content_hash) VALUES (1, 1, 1, 'hash', 4, 'hash');
|
|
INSERT INTO betas (id, script_id, script_version_id, trigger_kind, state) VALUES (1, 1, 1, 'ui', 'promoted');
|
|
INSERT INTO beta_servers (beta_id, server, had_original, stage, healthy_at) VALUES (1, 'server', 0, 'ready', 'healthy');
|
|
INSERT INTO beta_server_scripts (beta_id, server, script_id, had_original) VALUES (1, 'server', 1, 0);
|
|
INSERT INTO beta_test_runs (id, beta_id, server, created_at, metrics) VALUES ('run', 1, 'server', 'before', '{"rounds":25}');
|
|
INSERT INTO beta_test_evidence (run_id, event_id, received_at, kind, payload) VALUES ('run', 'event', 'before', 'match_end', '{"round":25}');
|
|
INSERT INTO beta_test_coverage (beta_id, server, game, map) VALUES (1, 'server', 't5', 'theater');
|
|
INSERT INTO beta_file_changes (beta_id, server, rel_path) VALUES (1, 'server', 'test.gsc');
|
|
INSERT INTO deployments (id, action, started_at, finished_at) VALUES ('campaign', 'promote', 'before', 'after');
|
|
INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state, restarted_at) VALUES ('campaign', 'server', 1, 'done', 'restarted');
|
|
INSERT INTO deployment_files (deployment_id, server, verb, rel_path, content_after_hash) VALUES ('campaign', 'server', 'updated', 'test.gsc', 'hash');
|
|
INSERT INTO server_deployments (id, server, action, state, stage, started_at, finished_at, triggered_by) VALUES ('created', 'server', 'create', 'done', 'Ready', 'before', 'after', 'ui');
|
|
INSERT INTO maintenance_jobs (server, state, requested_at, finished_at, detail) VALUES ('server', 'done', 'before', 'after', 'Updated');
|
|
INSERT INTO watchdog_incidents (id, server, kind, state, summary, first_seen_at, last_seen_at) VALUES ('incident', 'server', 'health', 'attention', 'Unhealthy', 'before', 'after');
|
|
INSERT INTO leaderboard_matches (server, game, map, map_key, map_name, started_at, last_received_at, highest_round, last_event, last_fingerprint)
|
|
VALUES ('server', 't5', 'theater', 'theater', 'Kino', 'before', 'after', 25, 'heartbeat', 'hash');`);
|
|
}
|
|
|
|
describe('durable server history', () => {
|
|
it('migrates legacy foreign keys without losing evidence, activity, or row ordering', () => {
|
|
const db = database(true);
|
|
seedHistory(db);
|
|
const events = db.prepare('SELECT * FROM beta_events ORDER BY id').all();
|
|
const runs = db.prepare('SELECT rowid, * FROM beta_test_runs').all();
|
|
const servers = db
|
|
.prepare('SELECT rowid, beta_id, server, stage, healthy_at FROM beta_servers')
|
|
.all();
|
|
migrateServerHistory(db);
|
|
migrateServerHistory(db);
|
|
expect(db.prepare('SELECT rowid, * FROM beta_test_runs').all()).toEqual(runs);
|
|
expect(
|
|
db.prepare('SELECT rowid, beta_id, server, stage, healthy_at FROM beta_servers').all()
|
|
).toEqual(servers);
|
|
expect(db.prepare('SELECT * FROM beta_events ORDER BY id').all()).toEqual(events);
|
|
db.prepare("UPDATE beta_servers SET stage = 'failed', error = 'historical failure'").run();
|
|
expect(
|
|
db.prepare('SELECT message FROM beta_events ORDER BY id DESC LIMIT 1').get()
|
|
).toEqual({ message: 'server: Failed — historical failure' });
|
|
db.prepare("DELETE FROM servers WHERE name = 'server'").run();
|
|
expect(db.prepare('SELECT metrics FROM beta_test_runs').get()).toEqual({
|
|
metrics: '{"rounds":25}'
|
|
});
|
|
expect(db.prepare('SELECT payload FROM beta_test_evidence').get()).toEqual({
|
|
payload: '{"round":25}'
|
|
});
|
|
expect(
|
|
db.prepare('SELECT had_original, prior_version_id FROM beta_server_scripts').get()
|
|
).toEqual({ had_original: 0, prior_version_id: null });
|
|
expect(db.pragma('foreign_keys', { simple: true })).toBe(1);
|
|
expect(db.pragma('foreign_key_check')).toEqual([]);
|
|
});
|
|
|
|
it('preserves names and results permanently while freeing the same name and port', () => {
|
|
const db = database();
|
|
migrateServerHistory(db);
|
|
seedHistory(db);
|
|
db.exec(
|
|
`INSERT INTO server_launch_collections (id, name, created_at, retained_servers) VALUES ('collection', 'Collection', 'before', '["server"]');`
|
|
);
|
|
db.prepare("DELETE FROM servers WHERE name = 'server'").run();
|
|
for (const table of [
|
|
'beta_servers',
|
|
'deployment_servers',
|
|
'server_deployments',
|
|
'maintenance_jobs',
|
|
'watchdog_incidents',
|
|
'leaderboard_matches'
|
|
]) {
|
|
expect(db.prepare(`SELECT server, server_deleted_at FROM ${table}`).get()).toEqual({
|
|
server: 'server',
|
|
server_deleted_at: expect.any(String)
|
|
});
|
|
}
|
|
const previous = db.prepare('SELECT server_deleted_at FROM beta_servers').get();
|
|
const retained = db
|
|
.prepare('SELECT retained_server_deletions FROM server_launch_collections')
|
|
.get() as { retained_server_deletions: string };
|
|
expect(JSON.parse(retained.retained_server_deletions)).toEqual({
|
|
server: expect.any(String)
|
|
});
|
|
expect(db.prepare('SELECT stage, healthy_at FROM beta_servers').get()).toEqual({
|
|
stage: 'ready',
|
|
healthy_at: 'healthy'
|
|
});
|
|
expect(
|
|
db.prepare('SELECT restart_state, restarted_at FROM deployment_servers').get()
|
|
).toEqual({ restart_state: 'done', restarted_at: 'restarted' });
|
|
expect(
|
|
db
|
|
.prepare(
|
|
'SELECT (SELECT content FROM file_blobs WHERE sha256=content_after_hash) AS content_after FROM deployment_files'
|
|
)
|
|
.get()
|
|
).toEqual({ content_after: Buffer.from('test') });
|
|
expect(
|
|
db.prepare('SELECT status, highest_round, ended_at FROM leaderboard_matches').get()
|
|
).toEqual({ status: 'interrupted', highest_round: 25, ended_at: expect.any(String) });
|
|
db.exec(`INSERT INTO servers (name, game_port) VALUES ('server', 28960);
|
|
INSERT INTO maintenance_jobs (server, state, requested_at) VALUES ('server', 'queued', 'new');
|
|
INSERT INTO watchdog_incidents (id, server, kind, state, summary, first_seen_at, last_seen_at) VALUES ('replacement', 'server', 'health', 'open', 'New incident', 'new', 'new');`);
|
|
migrateServerHistory(db);
|
|
expect(db.prepare('SELECT server_deleted_at FROM beta_servers').get()).toEqual(previous);
|
|
expect(
|
|
db.prepare('SELECT retained_server_deletions FROM server_launch_collections').get()
|
|
).toEqual(retained);
|
|
expect(
|
|
db
|
|
.prepare(
|
|
"SELECT server_deleted_at FROM watchdog_incidents WHERE id = 'replacement'"
|
|
)
|
|
.get()
|
|
).toEqual({ server_deleted_at: null });
|
|
});
|
|
|
|
it('cancels old pending work without rewriting completed outcomes', () => {
|
|
const db = database();
|
|
migrateServerHistory(db);
|
|
seedHistory(db);
|
|
db.exec(`INSERT INTO maintenance_jobs (server, state, requested_at) VALUES ('server', 'queued', 'pending');
|
|
INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) VALUES ('pending', 'server', 'restart', 'waiting', 'Waiting for empty', 'pending', 'ui');
|
|
INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) VALUES ('removal', 'server', 'destroy', 'running', 'Removing server files and registry', 'pending', 'ui');
|
|
DELETE FROM servers WHERE name = 'server';`);
|
|
expect(
|
|
db
|
|
.prepare("SELECT state, stage, error FROM server_deployments WHERE id = 'pending'")
|
|
.get()
|
|
).toEqual({ state: 'cancelled', stage: 'Waiting for empty', error: 'Server deleted' });
|
|
expect(
|
|
db
|
|
.prepare(
|
|
"SELECT message FROM server_deployment_events WHERE deployment_id = 'pending'"
|
|
)
|
|
.get()
|
|
).toEqual({ message: 'Cancelled: server deleted' });
|
|
expect(db.prepare('SELECT state, detail FROM maintenance_jobs ORDER BY id').all()).toEqual([
|
|
{ state: 'done', detail: 'Updated' },
|
|
{ state: 'cancelled', detail: 'Server deleted' }
|
|
]);
|
|
expect(
|
|
db
|
|
.prepare("SELECT state, finished_at FROM server_deployments WHERE id = 'created'")
|
|
.get()
|
|
).toEqual({ state: 'done', finished_at: 'after' });
|
|
expect(
|
|
db
|
|
.prepare(
|
|
"SELECT state, server_deleted_at FROM server_deployments WHERE id = 'removal'"
|
|
)
|
|
.get()
|
|
).toEqual({ state: 'running', server_deleted_at: expect.any(String) });
|
|
});
|
|
|
|
it('keeps only the current rebuild and its observing campaign and incident attached', () => {
|
|
const db = database();
|
|
migrateServerHistory(db);
|
|
seedHistory(db);
|
|
db.exec(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) VALUES ('rebuild', 'server', 'rebuild', 'running', 'Removing server files and registry', 'now', 'watchdog:incident');
|
|
INSERT INTO server_rebuild_jobs (deployment_id) VALUES ('rebuild');
|
|
INSERT INTO deployments (id, action, started_at) VALUES ('watchdog', 'rebuild', 'now');
|
|
INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES ('watchdog', 'server', 1, 'pending');
|
|
INSERT INTO watchdog_actions (id, incident_id, action, operation_id, campaign_id, created_at) VALUES ('action', 'incident', 'rebuild', 'rebuild', 'watchdog', 'now');
|
|
DELETE FROM servers WHERE name = 'server';`);
|
|
expect(
|
|
db
|
|
.prepare(
|
|
"SELECT state, server_deleted_at FROM server_deployments WHERE id = 'rebuild'"
|
|
)
|
|
.get()
|
|
).toEqual({ state: 'running', server_deleted_at: null });
|
|
expect(
|
|
db
|
|
.prepare(
|
|
"SELECT server_deleted_at FROM deployment_servers WHERE deployment_id = 'watchdog'"
|
|
)
|
|
.get()
|
|
).toEqual({ server_deleted_at: null });
|
|
expect(
|
|
db
|
|
.prepare("SELECT server_deleted_at FROM watchdog_incidents WHERE id = 'incident'")
|
|
.get()
|
|
).toEqual({ server_deleted_at: null });
|
|
expect(
|
|
db
|
|
.prepare(
|
|
"SELECT server_deleted_at FROM deployment_servers WHERE deployment_id = 'campaign'"
|
|
)
|
|
.get()
|
|
).toEqual({ server_deleted_at: expect.any(String) });
|
|
db.exec(`UPDATE server_rebuild_jobs SET removed_at = 'now'; UPDATE server_deployments SET state = 'queued' WHERE id = 'rebuild';
|
|
DELETE FROM settings WHERE key = 'server_history_v1';`);
|
|
migrateServerHistory(db);
|
|
expect(
|
|
db
|
|
.prepare("SELECT server_deleted_at FROM server_deployments WHERE id = 'rebuild'")
|
|
.get()
|
|
).toEqual({ server_deleted_at: null });
|
|
expect(
|
|
db
|
|
.prepare(
|
|
"SELECT server_deleted_at FROM deployment_servers WHERE deployment_id = 'watchdog'"
|
|
)
|
|
.get()
|
|
).toEqual({ server_deleted_at: null });
|
|
});
|
|
|
|
it('backfills missing historical servers while excluding initial launches with no deletion evidence', () => {
|
|
const db = database();
|
|
db.exec(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) VALUES
|
|
('old', 'gone', 'create', 'done', 'Ready', 'old', 'ui'),
|
|
('pending', 'new', 'create', 'queued', 'Queued', 'new', 'ui'),
|
|
('failed', 'failed', 'create', 'failed', 'Creating', 'new', 'ui'),
|
|
('interrupted-create', 'removed', 'create', 'interrupted', 'Creating', 'before', 'ui'),
|
|
('superseded', 'removed', 'rebuild', 'failed', 'Removing server files and registry', 'before', 'ui'),
|
|
('destroyed', 'removed', 'destroy', 'done', 'Removing server files and registry', 'after', 'ui');
|
|
INSERT INTO server_rebuild_jobs (deployment_id) VALUES ('superseded');
|
|
INSERT INTO servers (name, destroyed_at) VALUES ('tombstone', 'old');
|
|
INSERT INTO maintenance_jobs (server, state, requested_at) VALUES ('gone', 'queued', 'old'), ('tombstone', 'done', 'old');
|
|
INSERT INTO server_launch_collections (id, name, created_at, retained_servers) VALUES ('old', 'Old collection', 'old', '["tombstone","gone","new"]');`);
|
|
migrateServerHistory(db);
|
|
expect(
|
|
db.prepare("SELECT server_deleted_at FROM server_deployments WHERE id = 'old'").get()
|
|
).toEqual({ server_deleted_at: expect.any(String) });
|
|
for (const id of ['pending', 'failed'])
|
|
expect(
|
|
db.prepare('SELECT server_deleted_at FROM server_deployments WHERE id = ?').get(id)
|
|
).toEqual({ server_deleted_at: null });
|
|
expect(
|
|
db
|
|
.prepare(
|
|
"SELECT server_deleted_at FROM server_deployments WHERE id = 'interrupted-create'"
|
|
)
|
|
.get()
|
|
).toEqual({ server_deleted_at: expect.any(String) });
|
|
expect(
|
|
db
|
|
.prepare(
|
|
"SELECT state, server_deleted_at FROM server_deployments WHERE id = 'superseded'"
|
|
)
|
|
.get()
|
|
).toEqual({ state: 'failed', server_deleted_at: expect.any(String) });
|
|
expect(
|
|
db
|
|
.prepare(
|
|
"SELECT state, server_deleted_at FROM maintenance_jobs WHERE server = 'gone'"
|
|
)
|
|
.get()
|
|
).toEqual({ state: 'cancelled', server_deleted_at: expect.any(String) });
|
|
expect(
|
|
db
|
|
.prepare(
|
|
"SELECT state, server_deleted_at FROM maintenance_jobs WHERE server = 'tombstone'"
|
|
)
|
|
.get()
|
|
).toEqual({ state: 'done', server_deleted_at: expect.any(String) });
|
|
const collection = db
|
|
.prepare('SELECT retained_server_deletions FROM server_launch_collections')
|
|
.get() as { retained_server_deletions: string };
|
|
expect(JSON.parse(collection.retained_server_deletions)).toEqual({
|
|
tombstone: expect.any(String),
|
|
gone: expect.any(String)
|
|
});
|
|
});
|
|
});
|