josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Scope server snapshots and keep completed history off fleet refreshes

Browse Source
This commit is contained in:
josh
2026-10-05 15:37:56 -04:00
parent 030794635e
commit d24c52b5cc
12 changed files with 376 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/fleet-monitor.test.ts
+40
View File
@@ -1,4 +1,6 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { get } from 'svelte/store';
import type { FleetOperation } from '../../src/lib/fleet-overview.js';
vi.mock('$app/environment', () => ({ browser: true }));
@@ -33,6 +35,44 @@ async function setup() {
}
describe('fleet event refreshes', () => {
it('refreshes server history through one scoped request without replacing the fleet snapshot', async () => {
const store = await import('../../src/lib/ui/stores/fleet.js');
const integrations = { rcon: true, komodo: true, rconTimeoutMs: 2000, defaultRconHost: '' };
const snapshot = { servers: [], operations: [], integrations, serverNow: Date.now() };
store.hydrateFleet(snapshot);
const originalFleet = get(store.fleet);
const detail = store.createFleetStore('one / two');
detail.hydrateFleet(snapshot);
stop = detail.startFleetMonitor();
const history: FleetOperation[] = [{ key: 'deployment:done', group: 'deployment:done', kind: 'deployment',
server: 'one / two', state: 'done', label: 'Complete', detail: 'Complete', href: '/provisioning/done', startedAt: new Date().toISOString() }];
vi.mocked(fetch).mockResolvedValue(Response.json({ ...snapshot, operations: history }));
changed();
await vi.advanceTimersByTimeAsync(250);
expect(fetch).toHaveBeenCalledTimes(1);
expect(fetch).toHaveBeenCalledWith('/api/servers/one%20%2F%20two', expect.anything());
expect(get(detail.fleet).operations).toEqual(history);
expect(get(store.fleet)).toBe(originalFleet);
expect(get(store.fleet).operations).toEqual([]);
});
it('isolates late responses when navigating between server detail stores', async () => {
const { createFleetStore, fleet } = await import('../../src/lib/ui/stores/fleet.js');
const first = createFleetStore('first');
const second = createFleetStore('second');
let finish!: (response: Response) => void;
vi.mocked(fetch).mockImplementationOnce(() => new Promise(resolve => { finish = resolve; }));
const oldRequest = first.refreshFleet();
const newSnapshot = { serverNow: Date.now(), servers: [], integrations: {}, operations: [] };
vi.mocked(fetch).mockResolvedValue(Response.json(newSnapshot));
await second.refreshFleet();
const current = get(second.fleet);
finish(Response.json(newSnapshot)); await oldRequest;
expect(get(second.fleet)).toBe(current);
expect(get(fleet).receivedAt).toBe(0);
expect(fetch).toHaveBeenCalledTimes(2);
});
it('coalesces bursts and refreshes at most once every two seconds with a trailing refresh', async () => {
await setup();
changed(); changed(); changed();
tests/unit/fleet-overview.test.ts
+17 -12
View File
@@ -64,8 +64,10 @@ describe('persisted Fleet operations', () => {
const { db } = await import('../../src/lib/server/db/index.js');
const handle = db();
handle.prepare("INSERT INTO maintenance_jobs(server,kind,state,target_revision,requested_at,detail) VALUES ('server','update','failed',5354,?,'Revision not verified')").run(iso());
const { fleetOperations } = await import('../../src/lib/server/fleet/overview.js');
const { fleetOperations, serverOperations } = await import('../../src/lib/server/fleet/overview.js');
expect(fleetOperations(now)).toEqual([expect.objectContaining({ kind: 'maintenance', state: 'attention', detail: 'Revision not verified' })]);
expect(serverOperations('server', now)).toEqual(fleetOperations(now));
expect(serverOperations('unrelated', now)).toEqual([]);
handle.prepare("INSERT INTO maintenance_jobs(server,kind,state,target_revision,requested_at) VALUES ('server','update','queued',5354,?)").run(iso(1000));
expect(fleetOperations(now)).toEqual([]);
});
@@ -76,7 +78,7 @@ describe('persisted Fleet operations', () => {
handle.prepare("INSERT INTO servers (name, komodo_name) VALUES ('server', 'server')").run();
handle.prepare("INSERT INTO deployments (id, action, started_at) VALUES ('one', 'promote', ?)").run(iso(-2 * 86_400_000));
handle.prepare("INSERT INTO deployment_servers (deployment_id, server, files_changed, restart_state, players) VALUES ('one', 'server', 1, 'pending', 4)").run();
const { fleetOperations } = await import('../../src/lib/server/fleet/overview.js');
const { fleetOperations, serverOperations } = await import('../../src/lib/server/fleet/overview.js');
expect(fleetOperations(now)).toEqual([expect.objectContaining({ label: 'Waiting for an empty lobby', state: 'active', href: '/campaigns/one' })]);
handle.prepare("UPDATE deployment_servers SET restarted_at = ?, players = NULL").run(iso());
handle.prepare("INSERT INTO restart_rollout (server, requested_at) VALUES ('server', ?)").run(iso());
@@ -86,18 +88,20 @@ describe('persisted Fleet operations', () => {
expect(fleetOperations(now)[0].label).toBe('Recovering');
db().prepare("UPDATE deployment_servers SET restart_state = 'done', checked_at = ?").run(iso());
db().prepare('UPDATE restart_rollout SET healthy_at = ?').run(iso());
expect(fleetOperations(now)[0].state).toBe('done');
expect(fleetOperations(now)).toEqual([]);
expect(serverOperations('server', now)[0].state).toBe('done');
});
it('keeps lifecycle failures actionable until superseded and retains successful completions', async () => {
it('keeps lifecycle failures actionable until superseded and retains completions only for that server', async () => {
const { db } = await import('../../src/lib/server/db/index.js');
const handle = db();
const insert = handle.prepare("INSERT INTO server_deployments (id, server, action, state, stage, started_at, finished_at, triggered_by) VALUES (?, ?, ?, ?, ?, ?, ?, 'test')");
insert.run('failed', 'server', 'create', 'failed', 'Creating container', iso(-86_400_001), iso(-86_400_001));
const { fleetOperations } = await import('../../src/lib/server/fleet/overview.js');
const { fleetOperations, serverOperations } = await import('../../src/lib/server/fleet/overview.js');
expect(fleetOperations(now)[0].state).toBe('attention');
insert.run('retry', 'server', 'create', 'done', 'Ready', iso(), iso());
expect(fleetOperations(now)).toEqual([expect.objectContaining({ key: 'deployment:retry', state: 'done' })]);
expect(fleetOperations(now)).toEqual([]);
expect(serverOperations('server', now)).toEqual([expect.objectContaining({ key: 'deployment:retry', state: 'done' })]);
insert.run('queued', 'new-server', 'create', 'queued', 'Waiting to launch', iso(), null);
expect(fleetOperations(now)).toContainEqual(expect.objectContaining({ server: 'new-server', label: 'Waiting to launch', state: 'active' }));
});
@@ -107,7 +111,7 @@ describe('persisted Fleet operations', () => {
const handle = db();
handle.prepare("INSERT INTO servers (name, komodo_name) VALUES ('server', 'server')").run();
handle.prepare("INSERT INTO restart_rollout (server, requested_at, error) VALUES ('server', ?, 'RCON not answering')").run(iso());
const { fleetOperations } = await import('../../src/lib/server/fleet/overview.js');
const { fleetOperations, serverOperations } = await import('../../src/lib/server/fleet/overview.js');
expect(fleetOperations(now)[0]).toMatchObject({ kind: 'recovery', state: 'attention', detail: 'RCON not answering' });
handle.prepare("INSERT INTO scripts (id, name, target_rel_path) VALUES (1, 'test', 'test.gsc')").run();
handle.prepare("INSERT INTO script_versions (id, script_id, version, sha256, size, content) VALUES (1, 1, 1, 'hash', 0, X'')").run();
@@ -115,9 +119,11 @@ describe('persisted Fleet operations', () => {
handle.prepare("INSERT INTO beta_servers (beta_id, server, had_original, stage) VALUES (1, 'server', 0, 'awaiting-match')").run();
expect(fleetOperations(now)).toContainEqual(expect.objectContaining({ kind: 'beta', label: 'Observing a match', href: '/betas/1' }));
expect(fleetOperations(now)).toContainEqual(expect.objectContaining({ kind: 'recovery', state: 'attention' }));
expect(serverOperations('server', now)).toEqual(fleetOperations(now));
expect(serverOperations('unrelated', now)).toEqual([]);
});
it('preserves lifecycle selection across cutoff, deletion, supersession, and overlapping warning branches', async () => {
it('keeps only active work and unresolved failures across deletion and supersession', async () => {
const { db } = await import('../../src/lib/server/db/index.js');
const handle = db();
const since = iso(-86_400_000);
@@ -138,13 +144,12 @@ describe('persisted Fleet operations', () => {
}
}
})();
// The former OR query is the compatibility reference for all three branches.
// Completed work, including warnings and cancellations, belongs to server history.
const expected = handle.prepare(`SELECT d.id FROM server_deployments d
WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued', 'running', 'waiting', 'awaiting-health')
OR (d.state IN ('done', 'warning', 'cancelled') AND d.finished_at >= ?)
OR (d.state IN ('failed', 'interrupted', 'warning') AND NOT EXISTS
OR (d.state IN ('failed', 'interrupted') AND NOT EXISTS
(SELECT 1 FROM server_deployments n WHERE n.server = d.server AND n.rowid > d.rowid)))`)
.all(since) as { id: string }[];
.all() as { id: string }[];
const { fleetOperations } = await import('../../src/lib/server/fleet/overview.js');
expect(fleetOperations(now).map(row => row.key).sort()).toEqual(expected.map(row => `deployment:${row.id}`).sort());
});
tests/unit/fleet-snapshot-scope.test.ts
+85
View File
@@ -0,0 +1,85 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
let fixture: Fixture;
const now = Date.parse('2026-10-05T12:00:00Z');
const iso = (offset = 0) => new Date(now + offset).toISOString();
beforeEach(async () => {
vi.resetModules();
vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(now);
fixture = await newFixture([]);
const { db } = await import('../../src/lib/server/db/index.js');
db().transaction(() => {
const server = db().prepare('INSERT INTO servers(name, komodo_name) VALUES (?, ?)');
const operation = db().prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,finished_at,triggered_by)
VALUES(?,?,'restart',?,'Complete',?,?,'test')`);
for (const name of ['one', 'two']) {
server.run(name, name);
for (let i = 0; i < 100; i++) operation.run(`${name}-${i}`, name, 'done', iso(-i), iso(-i));
operation.run(`${name}-old`, name, 'done', iso(-86_400_001), iso(-86_400_001));
operation.run(`${name}-failed`, name, 'failed', iso(), iso());
operation.run(`${name}-active`, name, 'waiting', iso(), null);
}
db().prepare("INSERT INTO deployments(id,action,started_at,finished_at) VALUES('campaign','push',?,?)").run(iso(), iso());
for (const name of ['one', 'two']) {
db().prepare(`INSERT INTO deployment_servers(deployment_id,server,files_changed,restart_state,checked_at)
VALUES('campaign',?,1,'done',?)`).run(name, iso());
}
})();
});
afterEach(async () => { await fixture.cleanup(); vi.useRealTimers(); vi.restoreAllMocks(); });
describe('fleet and server response scope', () => {
it('omits completed history from fleet REST and SSR responses', async () => {
const { GET } = await import('../../src/routes/api/fleet/+server.js');
const { load } = await import('../../src/routes/+page.server.js');
const body = await GET().json();
expect(body.operations.map((operation: { key: string }) => operation.key).sort())
.toEqual(['deployment:one-active', 'deployment:two-active']);
expect(load().fleetSnapshot.operations).toEqual(body.operations);
});
it('loads only the selected server and its recent history in both REST and SSR', async () => {
const { GET } = await import('../../src/routes/api/servers/[name]/+server.js');
const { load } = await import('../../src/routes/servers/[name]/+page.server.js');
const registry = await import('../../src/lib/server/fleet/servers.js');
const { fleetSnapshot } = await import('../../src/lib/server/fleet/snapshot.js');
const expected = fleetSnapshot().servers.find(server => server.name === 'one');
const list = vi.spyOn(registry, 'listServers');
const snapshot = load({ params: { name: 'one' } }).serverSnapshot;
expect(list).not.toHaveBeenCalled();
expect(snapshot.servers).toEqual([expected]);
expect(snapshot.operations).toHaveLength(103);
expect(snapshot.operations.every(operation => operation.server === 'one')).toBe(true);
expect(snapshot.operations.map(operation => operation.key)).not.toContain('deployment:one-old');
expect(snapshot.operations).toContainEqual(expect.objectContaining({ key: 'campaign:campaign:one', state: 'done' }));
expect(snapshot.operations).toContainEqual(expect.objectContaining({ key: 'deployment:one-failed', state: 'attention' }));
const body = await GET({ params: { name: 'one' } }).json();
expect(body).toEqual(JSON.parse(JSON.stringify(snapshot)));
expect(list).not.toHaveBeenCalled();
});
it('returns no unrelated servers or history for missing and removed servers', async () => {
const { serverSnapshot } = await import('../../src/lib/server/fleet/snapshot.js');
expect(serverSnapshot('missing')).toMatchObject({ servers: [], operations: [] });
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare("UPDATE servers SET destroyed_at = ? WHERE name = 'one'").run(iso());
expect(serverSnapshot('one').servers).toEqual([]);
db().prepare("UPDATE server_deployments SET server_deleted_at = ? WHERE server = 'one'").run(iso());
db().prepare("UPDATE deployment_servers SET server_deleted_at = ? WHERE server = 'one'").run(iso());
expect(serverSnapshot('one').operations).toEqual([]);
});
it('uses a server index for recent lifecycle history', async () => {
const { db } = await import('../../src/lib/server/db/index.js');
const { serverOperations } = await import('../../src/lib/server/fleet/overview.js');
const prepare = vi.spyOn(db(), 'prepare');
serverOperations('one', now);
const sql = prepare.mock.calls.map(([sql]) => sql).find(sql => sql.includes('FROM server_deployments'))!;
prepare.mockRestore();
const plan = db().prepare(`EXPLAIN QUERY PLAN ${sql}`).all({ server: 'one', since: iso(-86_400_000) }) as { detail: string }[];
const details = plan.map(row => row.detail).join('\n');
expect(details).toMatch(/SEARCH server_deployments USING (?:COVERING )?INDEX idx_server_deployments_server/);
expect(details).not.toMatch(/SCAN server_deployments\b/);
});
});
tests/unit/provisioning-history-performance.test.ts
+2 -2
View File
@@ -70,10 +70,10 @@ describe('provisioning reads with retained lifecycle history', () => {
expect(queries).toHaveLength(1);
const statement = handle.prepare(queries[0]);
expect(statement.columns().map(column => column.name)).toEqual(['id', 'server', 'action', 'state', 'stage', 'error', 'started_at']);
const plan = handle.prepare(`EXPLAIN QUERY PLAN ${queries[0]}`).all(iso(-86_400_000)) as { detail: string }[];
const plan = handle.prepare(`EXPLAIN QUERY PLAN ${queries[0]}`).all() as { detail: string }[];
const details = plan.map(row => row.detail).join('\n');
expect(details).toContain('SEARCH server_deployments USING INDEX idx_server_deployments_state');
expect(details).toContain('SEARCH server_deployments USING INDEX idx_server_deployments_finished_at');
expect(queries[0]).not.toContain('finished_at >=');
expect(details).toContain('SEARCH server_deployments USING INTEGER PRIMARY KEY (rowid=?)');
expect(details).toContain('SCAN server_deployments USING COVERING INDEX idx_server_deployments_server');
expect(details).not.toMatch(/SCAN server_deployments(?:\n|$)/);