Preserve deleted server history
Retain historical server identities, results, and evidence after deletion. Mark archived entries as Deleted and prevent old work from targeting servers that reuse the same name. Fixes #16.
This commit is contained in:
@@ -33,7 +33,7 @@ describe('beta evidence explanations', () => {
|
||||
expect(betaWaitingCondition(r, { ...observation, players: 3 })).toContain('next report arrives');
|
||||
});
|
||||
it('summarizes telemetry while keeping arbitrary or legacy payloads readable', () => {
|
||||
const event = { id: 1, server: 'town', received_at: '', kind: 'game_end', payload: JSON.stringify({ round: 25, players: [{ name: 'A' }], map: 'zm_transit' }) };
|
||||
const event = { id: 1, server: 'town', server_deleted_at: null, received_at: '', kind: 'game_end', payload: JSON.stringify({ round: 25, players: [{ name: 'A' }], map: 'zm_transit' }) };
|
||||
expect(betaEvidenceSummary(event)).toBe('Game ended at round 25 with 1 players on zm_transit.');
|
||||
expect(betaEvidenceSummary({ ...event, kind: 'old-event', payload: 'invalid' })).toBe('old event');
|
||||
});
|
||||
|
||||
@@ -103,7 +103,7 @@ describe('queued removal cancellation', () => {
|
||||
});
|
||||
|
||||
describe('destruction cleanup recovery', () => {
|
||||
it('removes T6 servers with beta test runs and their dependent evidence', async () => {
|
||||
it('removes T6 servers while preserving beta progress and test evidence', async () => {
|
||||
const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
@@ -119,6 +119,7 @@ describe('destruction cleanup recovery', () => {
|
||||
const { script, version } = createScript({ name: 'test script', kind: 'script', scopeType: 'game', scopeValue: 't6', targetRelPath: 'test.gsc', content: Buffer.from('main(){}') });
|
||||
const beta = db().prepare("INSERT INTO betas (script_id, script_version_id, trigger_kind, state) VALUES (?, ?, 'manual', 'running')").run(script.id, version.id).lastInsertRowid;
|
||||
for (const server of [name, serverName]) {
|
||||
db().prepare("INSERT INTO beta_servers (beta_id, server, had_original, stage, healthy_at) VALUES (?, ?, 0, 'ready', '2026-09-18')").run(beta, server);
|
||||
db().prepare("INSERT INTO beta_test_runs (id, beta_id, server, created_at) VALUES (?, ?, ?, '2026-09-18')").run(server, beta, server);
|
||||
db().prepare("INSERT INTO beta_test_evidence (run_id, event_id, received_at, kind, payload) VALUES (?, 'event', '2026-09-18', 'health', '{}')").run(server);
|
||||
db().prepare('INSERT INTO beta_test_coverage (beta_id, server) VALUES (?, ?)').run(beta, server);
|
||||
@@ -128,10 +129,12 @@ describe('destruction cleanup recovery', () => {
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'gone' });
|
||||
await processDestructions();
|
||||
expect(getDeployment(queued.deploymentId!)?.deployment).toMatchObject({ state: 'done', server_exists: 0 });
|
||||
for (const table of ['beta_test_runs', 'beta_test_coverage', 'beta_file_changes']) {
|
||||
expect(db().prepare(`SELECT server FROM ${table}`).all()).toEqual([{ server: serverName }]);
|
||||
for (const table of ['beta_servers', 'beta_test_runs', 'beta_test_coverage', 'beta_file_changes']) {
|
||||
expect(db().prepare(`SELECT server FROM ${table} ORDER BY server`).all()).toEqual([{ server: serverName }, { server: name }]);
|
||||
}
|
||||
expect(db().prepare('SELECT run_id FROM beta_test_evidence').all()).toEqual([{ run_id: serverName }]);
|
||||
expect(db().prepare('SELECT run_id FROM beta_test_evidence ORDER BY run_id').all()).toEqual([{ run_id: serverName }, { run_id: name }]);
|
||||
expect(db().prepare('SELECT stage, healthy_at, server_deleted_at FROM beta_servers WHERE server = ?').get(name))
|
||||
.toEqual({ stage: 'ready', healthy_at: '2026-09-18', server_deleted_at: expect.any(String) });
|
||||
expect(db().pragma('foreign_key_check')).toEqual([]);
|
||||
});
|
||||
|
||||
@@ -177,6 +180,43 @@ describe('destruction cleanup recovery', () => {
|
||||
expect(getDeployment(first.deploymentId!)?.deployment).toMatchObject({ state: 'waiting', stage: REMOVING_SERVER });
|
||||
});
|
||||
|
||||
it('resumes cleanup after crashing between registry deletion and the final result', async () => {
|
||||
const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { claimRemoval, deploymentStage, interruptDeployments, getDeployment, resumeCleanup, REMOVING_SERVER } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
const { komodoDestroyDeployment, komodoDeleteDeployment } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
await createServer(input);
|
||||
const { deploymentId } = destroyServer(serverName);
|
||||
claimRemoval(deploymentId!);
|
||||
deploymentStage(deploymentId!, REMOVING_SERVER);
|
||||
db().prepare('DELETE FROM servers WHERE name = ?').run(serverName);
|
||||
closeDb();
|
||||
interruptDeployments();
|
||||
expect(getDeployment(deploymentId!)?.deployment).toMatchObject({ state: 'interrupted', server_deleted_at: expect.any(String) });
|
||||
expect(resumeCleanup(deploymentId!)).toBe(true);
|
||||
await processDestructions();
|
||||
expect(getDeployment(deploymentId!)?.deployment).toMatchObject({ state: 'done', server_deleted_at: expect.any(String), server_exists: 0 });
|
||||
expect(komodoDestroyDeployment).not.toHaveBeenCalled();
|
||||
expect(komodoDeleteDeployment).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(['before', 'after'] as const)('protects a replacement registered %s an archived cleanup resume', async timing => {
|
||||
const { createServer, destroyServer, processDestructions } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { claimRemoval, deploymentStage, interruptDeployments, resumeCleanup, REMOVING_SERVER } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
await createServer(input);
|
||||
const { deploymentId } = destroyServer(serverName);
|
||||
claimRemoval(deploymentId!);
|
||||
deploymentStage(deploymentId!, REMOVING_SERVER);
|
||||
db().prepare('DELETE FROM servers WHERE name = ?').run(serverName);
|
||||
interruptDeployments();
|
||||
if (timing === 'after') expect(resumeCleanup(deploymentId!)).toBe(true);
|
||||
db().prepare('INSERT INTO servers (name, komodo_name) VALUES (?, ?)').run(serverName, 'replacement');
|
||||
if (timing === 'before') expect(resumeCleanup(deploymentId!)).toBe(false);
|
||||
await processDestructions();
|
||||
expect(db().prepare('SELECT komodo_name FROM servers WHERE name = ?').get(serverName)).toEqual({ komodo_name: 'replacement' });
|
||||
});
|
||||
|
||||
it('refuses cleanup before confirmed deletion or after a newer creation', async () => {
|
||||
const { createServer, destroyServer } = await import('../../src/lib/server/fleet/create.js');
|
||||
const { beginDeployment, deploymentStage, finishDeployment, resumeCleanup, REMOVING_SERVER } = await import('../../src/lib/server/fleet/deployments.js');
|
||||
|
||||
@@ -364,7 +364,7 @@ describe('shared creation queue', () => {
|
||||
await processCreations();
|
||||
db().prepare('DELETE FROM servers WHERE name = ?').run(jobs[0].name);
|
||||
await processCreations();
|
||||
expect(await states()).toEqual(['failed', 'awaiting-health']);
|
||||
expect(await states()).toEqual(['cancelled', 'awaiting-health']);
|
||||
});
|
||||
|
||||
it('never verifies a replacement registered under the same server name', async () => {
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { readFile, writeFile } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
import { defaultPlan, requirement } from '../../src/lib/beta-plan.js';
|
||||
import { gameEvent } from '../helpers/telemetry.js';
|
||||
|
||||
let fixture: Fixture;
|
||||
const names = ['t5_zombie_theater_1', 't5_zombie_cosmodrome_1'];
|
||||
const now = () => new Date().toISOString();
|
||||
const path = (server: string) => join(fixture.t5, server, 'runtime/plutonium/storage/t5/raw/scripts/sp/history.gsc');
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.resetModules();
|
||||
fixture = await newFixture(names);
|
||||
vi.stubEnv('KOMODO_ADDRESS', ''); vi.stubEnv('RCON_PASSWORD', '');
|
||||
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
||||
await seedServerRegistry();
|
||||
});
|
||||
afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); });
|
||||
|
||||
async function open(planned = false, servers = [names[0]]) {
|
||||
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
|
||||
const { openBeta } = await import('../../src/lib/server/fleet/beta.js');
|
||||
const { script, version } = createScript({ name: 'Retained history', targetRelPath: 'history.gsc',
|
||||
scopeType: 'game', scopeValue: 't5', content: Buffer.from('beta content') });
|
||||
const input = { scripts: [{ scriptId: script.id, scriptVersionId: version.id }], servers,
|
||||
waitForHealth: false, waitForMatch: false,
|
||||
...(planned ? { testPlan: { ...defaultPlan(), requirements: [requirement('round', 'round')] } } : {}) };
|
||||
const { betaId } = await openBeta(input);
|
||||
return { betaId, input };
|
||||
}
|
||||
|
||||
async function replace(server: string) {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
||||
db().prepare('DELETE FROM servers WHERE name = ?').run(server);
|
||||
await seedServerRegistry();
|
||||
await writeFile(path(server), 'replacement content');
|
||||
}
|
||||
|
||||
describe('deleted server operational guards', () => {
|
||||
it('does not credit replacement matches to a legacy beta or keep its old server hold', async () => {
|
||||
const { betaId, input } = await open();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
||||
const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js');
|
||||
const { openBeta, listBetaServers, getBeta } = await import('../../src/lib/server/fleet/beta.js');
|
||||
db().prepare("UPDATE beta_servers SET stage = 'awaiting-match', healthy_at = ? WHERE beta_id = ?").run(now(), betaId);
|
||||
await replace(names[0]);
|
||||
saveTelemetry({ ...gameEvent('game_end', 99), receivedAt: now() });
|
||||
await betaTick();
|
||||
expect(listBetaServers(betaId)[0]).toMatchObject({ stage: 'awaiting-match', match_seen_at: null, server_deleted_at: expect.any(String) });
|
||||
expect(getBeta(betaId)?.state).toBe('running');
|
||||
await expect(openBeta(input)).resolves.toMatchObject({ servers: [names[0]] });
|
||||
});
|
||||
|
||||
it('freezes test evidence, pauses original coverage, and releases the replacement for new beta work', async () => {
|
||||
const { betaId, input } = await open(true);
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { prepareRun, recordGameEvent, testView } = await import('../../src/lib/server/fleet/beta-tests.js');
|
||||
const { betaTick } = await import('../../src/lib/server/pipeline/beta-runner.js');
|
||||
const { deployBetaServers, rollbackBeta, openBeta, listBetaServers, getBeta } = await import('../../src/lib/server/fleet/beta.js');
|
||||
const { heldByBeta } = await import('../../src/lib/server/fleet/servers.js');
|
||||
await prepareRun(betaId, names[0]);
|
||||
db().prepare("UPDATE beta_servers SET stage = 'observing', restarted_at = ? WHERE beta_id = ?").run(now(), betaId);
|
||||
db().prepare('UPDATE beta_test_runs SET observing_at = ? WHERE beta_id = ?').run(now(), betaId);
|
||||
recordGameEvent({ ...gameEvent('round_change', 5), receivedAt: now() });
|
||||
const before = testView(betaId)!;
|
||||
expect(before.servers[0].progress[0].current).toBe(5);
|
||||
|
||||
await replace(names[0]);
|
||||
recordGameEvent({ ...gameEvent('round_change', 99), receivedAt: now() });
|
||||
await betaTick();
|
||||
const after = testView(betaId)!;
|
||||
expect(after).toMatchObject({ outcome: 'paused', reason: expect.stringContaining('Test server deleted') });
|
||||
expect(after.servers[0]).toMatchObject({ server_deleted_at: expect.any(String), progress: before.servers[0].progress });
|
||||
expect(after.evidence).toHaveLength(before.evidence.length);
|
||||
expect(after.evidence[0].server_deleted_at).toEqual(expect.any(String));
|
||||
expect(getBeta(betaId)?.state).toBe('running');
|
||||
expect(listBetaServers(betaId)[0].stage).toBe('observing');
|
||||
expect(heldByBeta(names[0]).size).toBe(0);
|
||||
expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]);
|
||||
await expect(deployBetaServers(betaId, [names[0]])).rejects.toThrow('original beta server was deleted');
|
||||
await rollbackBeta(betaId);
|
||||
expect(await readFile(path(names[0]), 'utf8')).toBe('replacement content');
|
||||
await expect(openBeta(input)).resolves.toMatchObject({ servers: [names[0]] });
|
||||
});
|
||||
|
||||
it('does not treat a same-name replacement as tested or promote from deleted coverage', async () => {
|
||||
const { betaId } = await open();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { previewBetaPromotion, promoteBeta } = await import('../../src/lib/server/fleet/beta.js');
|
||||
db().prepare("UPDATE beta_servers SET stage = 'ready' WHERE beta_id = ?").run(betaId);
|
||||
db().prepare("UPDATE betas SET state = 'ready-promote' WHERE id = ?").run(betaId);
|
||||
const reviewed = previewBetaPromotion(betaId);
|
||||
await replace(names[0]);
|
||||
const preview = previewBetaPromotion(betaId);
|
||||
expect(preview.servers.find(server => server.name === names[0])?.tested).toBe(false);
|
||||
expect(preview.blockers).toEqual([expect.stringContaining('original test server was deleted')]);
|
||||
await expect(promoteBeta(betaId, reviewed.token)).rejects.toMatchObject({ kind: 'preview-changed' });
|
||||
await expect(promoteBeta(betaId, preview.token)).rejects.toMatchObject({ kind: 'invalid-state' });
|
||||
expect(await readFile(path(names[0]), 'utf8')).toBe('replacement content');
|
||||
expect(db().prepare('SELECT id FROM deployments').all()).toEqual([]);
|
||||
});
|
||||
|
||||
it('skips deleted campaign targets during restart and rollback while restoring live targets', async () => {
|
||||
for (const name of names) await writeFile(path(name), 'original content');
|
||||
const { betaId } = await open();
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { promoteBeta } = await import('../../src/lib/server/fleet/beta.js');
|
||||
const { rollbackPromoteDeployment } = await import('../../src/lib/server/fleet/rollback.js');
|
||||
const { sweep } = await import('../../src/lib/server/pipeline/sweep.js');
|
||||
const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js');
|
||||
db().prepare("UPDATE beta_servers SET stage = 'ready', restarted_at = ? WHERE beta_id = ?").run(now(), betaId);
|
||||
db().prepare("UPDATE betas SET state = 'ready-promote' WHERE id = ?").run(betaId);
|
||||
const campaign = await promoteBeta(betaId);
|
||||
await replace(names[1]);
|
||||
await sweep();
|
||||
expect(db().prepare('SELECT id FROM server_deployments').all()).toEqual([]);
|
||||
expect(() => requestRestart(names[1], { kind: 'campaign', id: campaign.deploymentId })).toThrow('Restart source no longer exists');
|
||||
const result = await rollbackPromoteDeployment(campaign.deploymentId);
|
||||
expect(result.affectedServers).toEqual([names[0]]);
|
||||
expect(await readFile(path(names[0]), 'utf8')).toBe('original content');
|
||||
expect(await readFile(path(names[1]), 'utf8')).toBe('replacement content');
|
||||
expect(db().prepare('SELECT server FROM deployment_servers WHERE deployment_id = ?').all(result.deploymentId)).toEqual([{ server: names[0] }]);
|
||||
});
|
||||
});
|
||||
@@ -588,6 +588,8 @@ describe('matching a blueprint exactly', () => {
|
||||
db().exec('ALTER TABLE server_collection_previews DROP COLUMN mode');
|
||||
db().exec('ALTER TABLE server_collection_previews DROP COLUMN fleet_snapshot');
|
||||
db().exec('ALTER TABLE server_launch_collections DROP COLUMN mode');
|
||||
// Recreate the older schema, which predates the history preservation trigger.
|
||||
db().exec('DROP TRIGGER preserve_deleted_server_history');
|
||||
db().exec('ALTER TABLE server_launch_collections DROP COLUMN retained_servers');
|
||||
closeDb();
|
||||
const { previewCollection } = await import('../../src/lib/server/fleet/collections.js');
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
import { expect, it } from 'vitest';
|
||||
import { render } from 'svelte/server';
|
||||
import HistoricalServerIdentity from '../../src/lib/ui/components/HistoricalServerIdentity.svelte';
|
||||
|
||||
it('retains deleted identities without linking to a replacement using the same server name', () => {
|
||||
const props = { name: 't5_zombie_theater_1', href: '/servers/t5_zombie_theater_1' };
|
||||
const live = render(HistoricalServerIdentity, { props }).body;
|
||||
expect(live).toContain('href="/servers/t5_zombie_theater_1"');
|
||||
const deleted = render(HistoricalServerIdentity, { props: { ...props, deletedAt: '2026-09-27T12:00:00Z' } }).body;
|
||||
expect(deleted).toContain('t5_zombie_theater_1');
|
||||
expect(deleted).toContain('Deleted');
|
||||
expect(deleted).not.toContain('href=');
|
||||
});
|
||||
@@ -294,6 +294,30 @@ it('backfills valid saved telemetry once while skipping malformed and legacy sna
|
||||
expect(await records()).toEqual([expect.objectContaining({ kills: 150, headshots: 80, rounds_survived: 1 })]);
|
||||
});
|
||||
|
||||
it('keeps deleted match identities and starts a separate match when the server name is reused', async () => {
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
||||
const { serverMatchHistory } = await import('../../src/lib/server/fleet/match-history.js');
|
||||
const { getPlayerProfile } = await import('../../src/lib/server/fleet/player-profile.js');
|
||||
await report('heartbeat', 8, [player(undefined, { kills: 120 })]);
|
||||
const original = serverMatchHistory(server).matches[0];
|
||||
db().prepare('DELETE FROM servers WHERE name = ?').run(server);
|
||||
await seedServerRegistry();
|
||||
// Even the same report bytes must belong to the replacement's own match.
|
||||
await report('heartbeat', 8, [player(undefined, { kills: 120 })]);
|
||||
closeDb();
|
||||
const history = serverMatchHistory(server).matches;
|
||||
expect(history).toHaveLength(2);
|
||||
expect(history.find(match => match.id === original.id)).toMatchObject({
|
||||
server, server_deleted_at: expect.any(String), status: 'interrupted'
|
||||
});
|
||||
expect(history.find(match => match.id !== original.id)).toMatchObject({ server, server_deleted_at: null, status: 'active' });
|
||||
const profile = getPlayerProfile('t5', player().guid)!;
|
||||
expect(profile.stats.gamesPlayed).toBe(2);
|
||||
expect(profile.matches.find(match => match.id === original.id)?.server_deleted_at).toEqual(expect.any(String));
|
||||
expect(profile.stats.kills).toBe(240);
|
||||
});
|
||||
|
||||
it('does not fabricate players for an empty match', async () => {
|
||||
await report('game_start', 0, []);
|
||||
await report('game_end', 0, []);
|
||||
|
||||
@@ -219,7 +219,7 @@ describe('provisioning operations', () => {
|
||||
expect(destroyServer(server)).toMatchObject({ ok: true });
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'exited', reason: 'stopped' });
|
||||
await processDestructions();
|
||||
expect(await operation(id)).toMatchObject({ state: 'failed', error: 'Server destroyed before operation verification completed' });
|
||||
expect(await operation(id)).toMatchObject({ state: 'cancelled', error: 'Server deleted', server_deleted_at: expect.any(String) });
|
||||
});
|
||||
|
||||
it('validates lifecycle API input and returns the durable operation ID', async () => {
|
||||
|
||||
@@ -44,6 +44,31 @@ afterEach(async () => {
|
||||
|
||||
function nextTick() { vi.setSystemTime(Date.now() + 15_000); }
|
||||
|
||||
it('ignores a late restart reply after deletion without changing a replacement recovery slot', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
||||
const { beginDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
||||
const { restartInRollout } = await import('../../src/lib/server/provisioning/restarts.js');
|
||||
const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
let complete!: (value: { ok: true }) => void;
|
||||
vi.mocked(komodoRestart).mockImplementationOnce(() => new Promise(resolve => { complete = resolve; }));
|
||||
const oldOperation = beginDeployment(servers[0], 'restart');
|
||||
const pending = restartInRollout(servers[0], { operationId: oldOperation, action: 'restart' });
|
||||
db().prepare('DELETE FROM servers WHERE name = ?').run(servers[0]);
|
||||
await seedServerRegistry();
|
||||
const replacement = beginDeployment(servers[0], 'restart');
|
||||
const requestedAt = new Date().toISOString();
|
||||
db().prepare('INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)')
|
||||
.run(servers[0], requestedAt, replacement);
|
||||
nextTick();
|
||||
complete({ ok: true });
|
||||
expect(await pending).toMatchObject({ kind: 'failed' });
|
||||
expect(db().prepare('SELECT state, server_deleted_at FROM server_deployments WHERE id = ?').get(oldOperation))
|
||||
.toMatchObject({ state: 'cancelled', server_deleted_at: expect.any(String) });
|
||||
expect(db().prepare('SELECT operation_id, requested_at, error FROM restart_rollout WHERE server = ?').get(servers[0]))
|
||||
.toEqual({ operation_id: replacement, requested_at: requestedAt, error: null });
|
||||
});
|
||||
|
||||
async function deployment(id = 'dep-1', targets = servers, action = 'promote') {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
db().prepare('INSERT INTO deployments (id, action, started_at) VALUES (?, ?, ?)')
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
import Database from 'better-sqlite3';
|
||||
import { afterEach, describe, expect, it } from 'vitest';
|
||||
import schema from '../../src/lib/server/db/schema.sql?raw';
|
||||
import { migrateServerHistory } from '../../src/lib/server/db/server-history.js';
|
||||
|
||||
let handle: Database.Database;
|
||||
afterEach(() => handle?.close());
|
||||
|
||||
function database(legacy = false) {
|
||||
handle = new Database(':memory:');
|
||||
let sql = schema;
|
||||
if (legacy) {
|
||||
sql = sql.replace(/^\s*server_deleted_at TEXT,\n/gm, '');
|
||||
for (const name of ['beta_servers', 'beta_test_runs', 'beta_server_scripts']) {
|
||||
sql = sql.replace(new RegExp(`(CREATE TABLE IF NOT EXISTS ${name} \\([\\s\\S]*?server\\s+TEXT NOT NULL)(,)`), '$1 REFERENCES servers(name)$2');
|
||||
}
|
||||
}
|
||||
handle.exec(sql);
|
||||
return handle;
|
||||
}
|
||||
|
||||
function seedHistory(db: Database.Database) {
|
||||
db.exec(`INSERT INTO servers (name, komodo_name, game_port) VALUES ('server', 'server', 28960);
|
||||
INSERT INTO scripts (id, name, target_rel_path) VALUES (1, 'script', 'test.gsc');
|
||||
INSERT INTO script_versions (id, script_id, version, sha256, size, content) VALUES (1, 1, 1, 'hash', 4, X'74657374');
|
||||
INSERT INTO betas (id, script_id, script_version_id, trigger_kind, state) VALUES (1, 1, 1, 'ui', 'promoted');
|
||||
INSERT INTO beta_servers (beta_id, server, had_original, stage, healthy_at) VALUES (1, 'server', 0, 'ready', 'healthy');
|
||||
INSERT INTO beta_server_scripts (beta_id, server, script_id, had_original) VALUES (1, 'server', 1, 0);
|
||||
INSERT INTO beta_test_runs (id, beta_id, server, created_at, metrics) VALUES ('run', 1, 'server', 'before', '{"rounds":25}');
|
||||
INSERT INTO beta_test_evidence (run_id, event_id, received_at, kind, payload) VALUES ('run', 'event', 'before', 'match_end', '{"round":25}');
|
||||
INSERT INTO beta_test_coverage (beta_id, server, game, map) VALUES (1, 'server', 't5', 'theater');
|
||||
INSERT INTO beta_file_changes (beta_id, server, rel_path) VALUES (1, 'server', 'test.gsc');
|
||||
INSERT INTO deployments (id, action, started_at, finished_at) VALUES ('campaign', 'promote', 'before', 'after');
|
||||
INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state, restarted_at) VALUES ('campaign', 'server', 1, 'done', 'restarted');
|
||||
INSERT INTO deployment_files (deployment_id, server, verb, rel_path, content_after) VALUES ('campaign', 'server', 'updated', 'test.gsc', X'74657374');
|
||||
INSERT INTO server_deployments (id, server, action, state, stage, started_at, finished_at, triggered_by) VALUES ('created', 'server', 'create', 'done', 'Ready', 'before', 'after', 'ui');
|
||||
INSERT INTO maintenance_jobs (server, state, requested_at, finished_at, detail) VALUES ('server', 'done', 'before', 'after', 'Updated');
|
||||
INSERT INTO watchdog_incidents (id, server, kind, state, summary, first_seen_at, last_seen_at) VALUES ('incident', 'server', 'health', 'attention', 'Unhealthy', 'before', 'after');
|
||||
INSERT INTO leaderboard_matches (server, game, map, map_key, map_name, started_at, last_received_at, highest_round, last_event, last_fingerprint)
|
||||
VALUES ('server', 't5', 'theater', 'theater', 'Kino', 'before', 'after', 25, 'heartbeat', 'hash');`);
|
||||
}
|
||||
|
||||
describe('durable server history', () => {
|
||||
it('migrates legacy foreign keys without losing evidence, activity, or row ordering', () => {
|
||||
const db = database(true);
|
||||
seedHistory(db);
|
||||
const events = db.prepare('SELECT * FROM beta_events ORDER BY id').all();
|
||||
const runs = db.prepare('SELECT rowid, * FROM beta_test_runs').all();
|
||||
const servers = db.prepare('SELECT rowid, beta_id, server, stage, healthy_at FROM beta_servers').all();
|
||||
migrateServerHistory(db);
|
||||
migrateServerHistory(db);
|
||||
expect(db.prepare('SELECT rowid, * FROM beta_test_runs').all()).toEqual(runs);
|
||||
expect(db.prepare('SELECT rowid, beta_id, server, stage, healthy_at FROM beta_servers').all()).toEqual(servers);
|
||||
expect(db.prepare('SELECT * FROM beta_events ORDER BY id').all()).toEqual(events);
|
||||
db.prepare("UPDATE beta_servers SET stage = 'failed', error = 'historical failure'").run();
|
||||
expect(db.prepare('SELECT message FROM beta_events ORDER BY id DESC LIMIT 1').get()).toEqual({ message: 'server: Failed — historical failure' });
|
||||
db.prepare("DELETE FROM servers WHERE name = 'server'").run();
|
||||
expect(db.prepare('SELECT metrics FROM beta_test_runs').get()).toEqual({ metrics: '{"rounds":25}' });
|
||||
expect(db.prepare('SELECT payload FROM beta_test_evidence').get()).toEqual({ payload: '{"round":25}' });
|
||||
expect(db.prepare('SELECT had_original, prior_version_id FROM beta_server_scripts').get()).toEqual({ had_original: 0, prior_version_id: null });
|
||||
expect(db.pragma('foreign_keys', { simple: true })).toBe(1);
|
||||
expect(db.pragma('foreign_key_check')).toEqual([]);
|
||||
});
|
||||
|
||||
it('preserves names and results permanently while freeing the same name and port', () => {
|
||||
const db = database();
|
||||
migrateServerHistory(db);
|
||||
seedHistory(db);
|
||||
db.exec(`INSERT INTO server_launch_collections (id, name, created_at, retained_servers) VALUES ('collection', 'Collection', 'before', '["server"]');`);
|
||||
db.prepare("DELETE FROM servers WHERE name = 'server'").run();
|
||||
for (const table of ['beta_servers', 'deployment_servers', 'server_deployments', 'maintenance_jobs', 'watchdog_incidents', 'leaderboard_matches']) {
|
||||
expect(db.prepare(`SELECT server, server_deleted_at FROM ${table}`).get()).toEqual({ server: 'server', server_deleted_at: expect.any(String) });
|
||||
}
|
||||
const previous = db.prepare('SELECT server_deleted_at FROM beta_servers').get();
|
||||
const retained = db.prepare('SELECT retained_server_deletions FROM server_launch_collections').get() as { retained_server_deletions: string };
|
||||
expect(JSON.parse(retained.retained_server_deletions)).toEqual({ server: expect.any(String) });
|
||||
expect(db.prepare('SELECT stage, healthy_at FROM beta_servers').get()).toEqual({ stage: 'ready', healthy_at: 'healthy' });
|
||||
expect(db.prepare('SELECT restart_state, restarted_at FROM deployment_servers').get()).toEqual({ restart_state: 'done', restarted_at: 'restarted' });
|
||||
expect(db.prepare('SELECT content_after FROM deployment_files').get()).toEqual({ content_after: Buffer.from('test') });
|
||||
expect(db.prepare('SELECT status, highest_round, ended_at FROM leaderboard_matches').get()).toEqual({ status: 'interrupted', highest_round: 25, ended_at: expect.any(String) });
|
||||
db.exec(`INSERT INTO servers (name, komodo_name, game_port) VALUES ('server', 'replacement', 28960);
|
||||
INSERT INTO maintenance_jobs (server, state, requested_at) VALUES ('server', 'queued', 'new');
|
||||
INSERT INTO watchdog_incidents (id, server, kind, state, summary, first_seen_at, last_seen_at) VALUES ('replacement', 'server', 'health', 'open', 'New incident', 'new', 'new');`);
|
||||
migrateServerHistory(db);
|
||||
expect(db.prepare('SELECT server_deleted_at FROM beta_servers').get()).toEqual(previous);
|
||||
expect(db.prepare('SELECT retained_server_deletions FROM server_launch_collections').get()).toEqual(retained);
|
||||
expect(db.prepare("SELECT server_deleted_at FROM watchdog_incidents WHERE id = 'replacement'").get()).toEqual({ server_deleted_at: null });
|
||||
});
|
||||
|
||||
it('cancels old pending work without rewriting completed outcomes', () => {
|
||||
const db = database();
|
||||
migrateServerHistory(db);
|
||||
seedHistory(db);
|
||||
db.exec(`INSERT INTO maintenance_jobs (server, state, requested_at) VALUES ('server', 'queued', 'pending');
|
||||
INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) VALUES ('pending', 'server', 'restart', 'waiting', 'Waiting for empty', 'pending', 'ui');
|
||||
INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) VALUES ('removal', 'server', 'destroy', 'running', 'Removing server files and registry', 'pending', 'ui');
|
||||
DELETE FROM servers WHERE name = 'server';`);
|
||||
expect(db.prepare("SELECT state, stage, error FROM server_deployments WHERE id = 'pending'").get())
|
||||
.toEqual({ state: 'cancelled', stage: 'Waiting for empty', error: 'Server deleted' });
|
||||
expect(db.prepare("SELECT message FROM server_deployment_events WHERE deployment_id = 'pending'").get()).toEqual({ message: 'Cancelled: server deleted' });
|
||||
expect(db.prepare('SELECT state, detail FROM maintenance_jobs ORDER BY id').all()).toEqual([{ state: 'done', detail: 'Updated' }, { state: 'cancelled', detail: 'Server deleted' }]);
|
||||
expect(db.prepare("SELECT state, finished_at FROM server_deployments WHERE id = 'created'").get()).toEqual({ state: 'done', finished_at: 'after' });
|
||||
expect(db.prepare("SELECT state, server_deleted_at FROM server_deployments WHERE id = 'removal'").get()).toEqual({ state: 'running', server_deleted_at: expect.any(String) });
|
||||
});
|
||||
|
||||
it('keeps only the current rebuild and its observing campaign and incident attached', () => {
|
||||
const db = database();
|
||||
migrateServerHistory(db);
|
||||
seedHistory(db);
|
||||
db.exec(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) VALUES ('rebuild', 'server', 'rebuild', 'running', 'Removing server files and registry', 'now', 'watchdog:incident');
|
||||
INSERT INTO server_rebuild_jobs (deployment_id) VALUES ('rebuild');
|
||||
INSERT INTO deployments (id, action, started_at) VALUES ('watchdog', 'rebuild', 'now');
|
||||
INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES ('watchdog', 'server', 1, 'pending');
|
||||
INSERT INTO watchdog_actions (id, incident_id, action, operation_id, campaign_id, created_at) VALUES ('action', 'incident', 'rebuild', 'rebuild', 'watchdog', 'now');
|
||||
DELETE FROM servers WHERE name = 'server';`);
|
||||
expect(db.prepare("SELECT state, server_deleted_at FROM server_deployments WHERE id = 'rebuild'").get()).toEqual({ state: 'running', server_deleted_at: null });
|
||||
expect(db.prepare("SELECT server_deleted_at FROM deployment_servers WHERE deployment_id = 'watchdog'").get()).toEqual({ server_deleted_at: null });
|
||||
expect(db.prepare("SELECT server_deleted_at FROM watchdog_incidents WHERE id = 'incident'").get()).toEqual({ server_deleted_at: null });
|
||||
expect(db.prepare("SELECT server_deleted_at FROM deployment_servers WHERE deployment_id = 'campaign'").get()).toEqual({ server_deleted_at: expect.any(String) });
|
||||
db.exec(`UPDATE server_rebuild_jobs SET removed_at = 'now'; UPDATE server_deployments SET state = 'queued' WHERE id = 'rebuild';
|
||||
DELETE FROM settings WHERE key = 'server_history_v1';`);
|
||||
migrateServerHistory(db);
|
||||
expect(db.prepare("SELECT server_deleted_at FROM server_deployments WHERE id = 'rebuild'").get()).toEqual({ server_deleted_at: null });
|
||||
expect(db.prepare("SELECT server_deleted_at FROM deployment_servers WHERE deployment_id = 'watchdog'").get()).toEqual({ server_deleted_at: null });
|
||||
});
|
||||
|
||||
it('backfills missing historical servers while excluding initial launches with no deletion evidence', () => {
|
||||
const db = database();
|
||||
db.exec(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) VALUES
|
||||
('old', 'gone', 'create', 'done', 'Ready', 'old', 'ui'),
|
||||
('pending', 'new', 'create', 'queued', 'Queued', 'new', 'ui'),
|
||||
('failed', 'failed', 'create', 'failed', 'Creating', 'new', 'ui'),
|
||||
('interrupted-create', 'removed', 'create', 'interrupted', 'Creating', 'before', 'ui'),
|
||||
('superseded', 'removed', 'rebuild', 'failed', 'Removing server files and registry', 'before', 'ui'),
|
||||
('destroyed', 'removed', 'destroy', 'done', 'Removing server files and registry', 'after', 'ui');
|
||||
INSERT INTO server_rebuild_jobs (deployment_id) VALUES ('superseded');
|
||||
INSERT INTO servers (name, komodo_name, destroyed_at) VALUES ('tombstone', 'tombstone', 'old');
|
||||
INSERT INTO maintenance_jobs (server, state, requested_at) VALUES ('gone', 'queued', 'old'), ('tombstone', 'done', 'old');
|
||||
INSERT INTO server_launch_collections (id, name, created_at, retained_servers) VALUES ('old', 'Old collection', 'old', '["tombstone","gone","new"]');`);
|
||||
migrateServerHistory(db);
|
||||
expect(db.prepare("SELECT server_deleted_at FROM server_deployments WHERE id = 'old'").get()).toEqual({ server_deleted_at: expect.any(String) });
|
||||
for (const id of ['pending', 'failed']) expect(db.prepare('SELECT server_deleted_at FROM server_deployments WHERE id = ?').get(id)).toEqual({ server_deleted_at: null });
|
||||
expect(db.prepare("SELECT server_deleted_at FROM server_deployments WHERE id = 'interrupted-create'").get()).toEqual({ server_deleted_at: expect.any(String) });
|
||||
expect(db.prepare("SELECT state, server_deleted_at FROM server_deployments WHERE id = 'superseded'").get()).toEqual({ state: 'failed', server_deleted_at: expect.any(String) });
|
||||
expect(db.prepare("SELECT state, server_deleted_at FROM maintenance_jobs WHERE server = 'gone'").get()).toEqual({ state: 'cancelled', server_deleted_at: expect.any(String) });
|
||||
expect(db.prepare("SELECT state, server_deleted_at FROM maintenance_jobs WHERE server = 'tombstone'").get()).toEqual({ state: 'done', server_deleted_at: expect.any(String) });
|
||||
const collection = db.prepare('SELECT retained_server_deletions FROM server_launch_collections').get() as { retained_server_deletions: string };
|
||||
expect(JSON.parse(collection.retained_server_deletions)).toEqual({ tombstone: expect.any(String), gone: expect.any(String) });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,57 @@
|
||||
import { afterEach, beforeEach, expect, it, vi } from 'vitest';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
|
||||
let fixture: Fixture;
|
||||
const server = 't5_zombie_theater_1';
|
||||
const at = '2026-09-27T12:00:00.000Z';
|
||||
beforeEach(async () => {
|
||||
vi.resetModules();
|
||||
fixture = await newFixture([server]);
|
||||
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
||||
await seedServerRegistry();
|
||||
});
|
||||
afterEach(async () => fixture.cleanup());
|
||||
|
||||
it('exposes retained campaign and provisioning history without rebinding links to a reused name', async () => {
|
||||
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
|
||||
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
||||
const { beginDeployment, finishDeployment, getDeployment, listDeployments } = await import('../../src/lib/server/provisioning/history.js');
|
||||
const operation = beginDeployment(server, 'restart');
|
||||
finishDeployment(operation, { ok: true });
|
||||
db().prepare("INSERT INTO deployments (id, action, started_at, finished_at) VALUES ('history-campaign', 'promote', ?, ?)").run(at, at);
|
||||
db().prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state) VALUES ('history-campaign', ?, 1, 'done')").run(server);
|
||||
db().prepare("INSERT INTO deployment_files (deployment_id, server, verb, rel_path, content_before, content_after) VALUES ('history-campaign', ?, 'updated', 'script.gsc', ?, ?)")
|
||||
.run(server, Buffer.from('before'), Buffer.from('after'));
|
||||
db().prepare('DELETE FROM servers WHERE name = ?').run(server);
|
||||
await seedServerRegistry();
|
||||
closeDb();
|
||||
const { GET: campaignGET } = await import('../../src/routes/api/campaigns/[id]/+server.js');
|
||||
const detail = await (await campaignGET({ params: { id: 'history-campaign' } })).json();
|
||||
expect(detail.servers).toEqual([expect.objectContaining({ server, restart_state: 'done', server_deleted_at: expect.any(String) })]);
|
||||
expect(detail.files).toEqual([expect.objectContaining({ server, rel_path: 'script.gsc', size_before: 6, size_after: 5 })]);
|
||||
const { GET: listGET } = await import('../../src/routes/api/campaigns/+server.js');
|
||||
const list = await (await listGET({ url: new URL('http://localhost/api/campaigns') })).json();
|
||||
expect(list.deployments[0].servers[0].server_deleted_at).toEqual(expect.any(String));
|
||||
expect(getDeployment(operation)?.deployment).toMatchObject({ server, state: 'done', server_exists: 0, server_deleted_at: expect.any(String) });
|
||||
expect(listDeployments()).toContainEqual(expect.objectContaining({ id: operation, server_exists: 0 }));
|
||||
const replacement = beginDeployment(server, 'restart');
|
||||
expect(getDeployment(replacement)?.deployment).toMatchObject({ server_exists: 1, server_deleted_at: null });
|
||||
});
|
||||
|
||||
it('rejects historical maintenance retries and allows a new incident for a reused server name', async () => {
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { seedServerRegistry } = await import('../../src/lib/server/fleet/servers.js');
|
||||
const job = db().prepare("INSERT INTO maintenance_jobs (server, requested_at, state, detail) VALUES (?, ?, 'failed', 'Original result')").run(server, at).lastInsertRowid;
|
||||
db().prepare("INSERT INTO watchdog_incidents (id, server, kind, state, summary, first_seen_at, last_seen_at) VALUES ('old-incident', ?, 'health', 'attention', 'Original finding', ?, ?)")
|
||||
.run(server, at, at);
|
||||
db().prepare('DELETE FROM servers WHERE name = ?').run(server);
|
||||
await seedServerRegistry();
|
||||
const { retryMaintenance, plutoniumStatus } = await import('../../src/lib/server/maintenance/runtime.js');
|
||||
const { updateWatchdogIncident, watchdogDetail } = await import('../../src/lib/server/maintenance/recovery.js');
|
||||
expect(() => retryMaintenance(Number(job))).toThrow('deleted');
|
||||
expect(plutoniumStatus().jobs).toContainEqual(expect.objectContaining({ id: Number(job), state: 'failed', detail: 'Original result', server_deleted_at: expect.any(String) }));
|
||||
expect(updateWatchdogIncident('old-incident', 'retry')).toMatchObject({ ok: false, status: 409 });
|
||||
expect(watchdogDetail('old-incident')?.incident).toMatchObject({ state: 'attention', summary: 'Original finding', server_deleted_at: expect.any(String) });
|
||||
expect(() => db().prepare("INSERT INTO watchdog_incidents (id, server, kind, state, summary, first_seen_at, last_seen_at) VALUES ('new-incident', ?, 'health', 'open', 'New finding', ?, ?)")
|
||||
.run(server, at, at)).not.toThrow();
|
||||
});
|
||||
@@ -195,7 +195,7 @@ describe('Watchdog detection and ownership', () => {
|
||||
});
|
||||
|
||||
describe('Watchdog recovery persistence', () => {
|
||||
it('resolves an attention incident after an intentional successful destroy', async () => {
|
||||
it('preserves the original finding as deleted history after an intentional successful destroy', async () => {
|
||||
const { removeServerKey } = await import('../../src/lib/server/fleet/server-keys.js');
|
||||
const { destroyServer, processDestructions } = await import('../../src/lib/server/provisioning/instances.js');
|
||||
removeServerKey(server);
|
||||
@@ -209,8 +209,8 @@ describe('Watchdog recovery persistence', () => {
|
||||
expect(operations()).toEqual([expect.objectContaining({ action: 'destroy', state: 'done' })]);
|
||||
expect(database.db().prepare('SELECT name FROM servers WHERE name = ?').get(server)).toBeUndefined();
|
||||
await watchdog.watchdogTick(true);
|
||||
expect(watchdog.watchdogDetail(id)?.incident).toMatchObject({ state: 'resolved', finished_at: expect.any(String) });
|
||||
expect(watchdog.watchdogDetail(id)?.events.at(-1)?.message).toMatch(/destroyed|registered|retired/i);
|
||||
expect(watchdog.watchdogDetail(id)?.incident).toMatchObject({ state: 'attention', server_deleted_at: expect.any(String) });
|
||||
expect(watchdog.updateWatchdogIncident(id, 'retry')).toMatchObject({ ok: false, status: 409 });
|
||||
});
|
||||
|
||||
it('requests one restart, escalates once to rebuild, and retains exhausted attention across reopening', async () => {
|
||||
|
||||
Reference in New Issue
Block a user