diff --git a/src/lib/server/db/index.ts b/src/lib/server/db/index.ts index 2f2d925..af500da 100644 --- a/src/lib/server/db/index.ts +++ b/src/lib/server/db/index.ts @@ -380,7 +380,11 @@ export function db(): Database.Database { CREATE INDEX IF NOT EXISTS idx_server_deployments_state ON server_deployments(state, action, server_deleted_at); CREATE INDEX IF NOT EXISTS idx_server_deployments_started_at - ON server_deployments(started_at);`); + ON server_deployments(started_at); + CREATE INDEX IF NOT EXISTS idx_server_deployments_submission + ON server_deployments(server, server_deleted_at, action, submitted_at); + CREATE INDEX IF NOT EXISTS idx_server_deployments_finished_at + ON server_deployments(finished_at) WHERE server_deleted_at IS NULL;`); installReportRevisions(handle); } catch (err) { handle.close(); diff --git a/src/lib/server/fleet/lifecycle-evidence.ts b/src/lib/server/fleet/lifecycle-evidence.ts index 2be68df..2c2b6ad 100644 --- a/src/lib/server/fleet/lifecycle-evidence.ts +++ b/src/lib/server/fleet/lifecycle-evidence.ts @@ -3,6 +3,17 @@ import { db } from '../db/index.js'; /** Reports from before the latest lifecycle submission describe the old process. */ export function latestGameSubmission(server: string): string | null { return (db().prepare(`SELECT MAX(at) AS at FROM ( - SELECT submitted_at AS at FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild') + SELECT MAX(submitted_at) AS at FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild') UNION ALL SELECT requested_at AS at FROM restart_rollout WHERE server=?)`).get(server, server) as { at: string | null }).at; } + +/** Look up each registered server through the covering submission index. */ +export function latestGameSubmissions(): Map { + const rows = db().prepare(`SELECT server, MAX(at) AS at FROM ( + SELECT s.name AS server, (SELECT MAX(submitted_at) FROM server_deployments + WHERE server=s.name AND server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild')) AS at + FROM servers s + UNION ALL SELECT server, requested_at AS at FROM restart_rollout) GROUP BY server`).all() as + { server: string; at: string | null }[]; + return new Map(rows.map(row => [row.server, row.at])); +} diff --git a/src/lib/server/fleet/overview.ts b/src/lib/server/fleet/overview.ts index fe4f4f5..df3cea9 100644 --- a/src/lib/server/fleet/overview.ts +++ b/src/lib/server/fleet/overview.ts @@ -34,11 +34,18 @@ export function fleetOperations(now = Date.now()): FleetOperation[] { href: `/campaigns/${encodeURIComponent(row.id)}` }); } // Only the newest lifecycle operation can still require action on a server. - const lifecycle = handle.prepare(`SELECT d.* FROM server_deployments d - WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued', 'running', 'waiting', 'awaiting-health') - OR (d.state IN ('done', 'warning', 'cancelled') AND d.finished_at >= ?) - OR (d.state IN ('failed', 'interrupted', 'warning') AND NOT EXISTS - (SELECT 1 FROM server_deployments n WHERE n.server = d.server AND n.rowid > d.rowid)))`) + // Separate branches let SQLite use state and finish indexes; UNION deduplicates + // recent warnings that are also the newest operation. Unary + keeps terminal + // branches on finished_at and rowid instead of searching historical states. + const lifecycle = handle.prepare(`SELECT id, server, action, state, stage, error, started_at FROM server_deployments + WHERE server_deleted_at IS NULL AND state IN ('queued', 'running', 'waiting', 'awaiting-health') + UNION + SELECT id, server, action, state, stage, error, started_at FROM server_deployments + WHERE server_deleted_at IS NULL AND +state IN ('done', 'warning', 'cancelled') AND finished_at >= ? + UNION + SELECT id, server, action, state, stage, error, started_at FROM server_deployments + WHERE server_deleted_at IS NULL AND +state IN ('failed', 'interrupted', 'warning') + AND rowid IN (SELECT MAX(rowid) FROM server_deployments GROUP BY server)`) .all(since) as { id: string; server: string; action: string; state: string; stage: string; error: string | null; started_at: string }[]; for (const row of lifecycle) { diff --git a/src/lib/server/fleet/status.ts b/src/lib/server/fleet/status.ts index 50fa4fe..195d85f 100644 --- a/src/lib/server/fleet/status.ts +++ b/src/lib/server/fleet/status.ts @@ -4,6 +4,7 @@ import { isProvisioning } from './provisioning.js'; import { restartSettling } from '../provisioning/restarts.js'; import { type ServerRow } from './servers.js'; import { telemetryReport, latestGameSubmission } from './status-evidence.js'; +import { latestGameSubmissions } from './lifecycle-evidence.js'; import { db } from '../db/index.js'; import { parseGameEvent, type GameTelemetry } from '../../telemetry.js'; import { restartRequestInFlight } from '../provisioning/restarts.js'; @@ -18,10 +19,7 @@ export function observationReader() { { server: string; action: string; state: string }[]).map(row => [row.server, row])); const restarts = new Map((handle.prepare('SELECT server, requested_at, healthy_at FROM restart_rollout').all() as { server: string; requested_at: string; healthy_at: string | null }[]).map(row => [row.server, row])); - const submissions = new Map((handle.prepare(`SELECT server, MAX(at) AS at FROM ( - SELECT server, submitted_at AS at FROM server_deployments WHERE server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild') - UNION ALL SELECT server, requested_at AS at FROM restart_rollout) GROUP BY server`).all() as - { server: string; at: string | null }[]).map(row => [row.server, row.at])); + const submissions = latestGameSubmissions(); const reports = new Map(); for (const row of handle.prepare('SELECT server, received_at, payload FROM server_telemetry WHERE payload IS NOT NULL').all() as { server: string; received_at: string; payload: string }[]) { diff --git a/src/lib/server/integrations/plutonium.ts b/src/lib/server/integrations/plutonium.ts index fbb1a26..9c78380 100644 --- a/src/lib/server/integrations/plutonium.ts +++ b/src/lib/server/integrations/plutonium.ts @@ -3,6 +3,7 @@ import { config } from '../config.js'; import { getServer, listServers } from '../fleet/servers.js'; import { hasActiveProvisioning } from '../fleet/provisioning.js'; import { latestGameSubmission } from '../fleet/status-evidence.js'; +import { latestGameSubmissions } from '../fleet/lifecycle-evidence.js'; import { rconPlutoniumVersion } from './rcon.js'; import { komodoInspectHealthy } from './komodo.js'; import { restartRecoveryPending } from '../provisioning/index.js'; @@ -23,9 +24,7 @@ export function runningVersions(): PlutoniumVersion[] { const versions = db().prepare('SELECT * FROM plutonium_versions').all() as PlutoniumVersion[]; const target = freshRelease(); const byServer = new Map(versions.map(version => [version.server, version])); - const epochs = new Map((db().prepare(`SELECT server, MAX(at) AS at FROM ( - SELECT server, submitted_at AS at FROM server_deployments WHERE server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild') - UNION ALL SELECT server, requested_at AS at FROM restart_rollout) GROUP BY server`).all() as { server: string; at: string | null }[]).map(row => [row.server, row.at])); + const epochs = latestGameSubmissions(); const recovering = new Set((db().prepare('SELECT server FROM restart_rollout WHERE healthy_at IS NULL').all() as { server: string }[]).map(row => row.server)); return listServers().map(server => { const value = byServer.get(server.name) diff --git a/tests/unit/fleet-overview.test.ts b/tests/unit/fleet-overview.test.ts index d1574a3..181d274 100644 --- a/tests/unit/fleet-overview.test.ts +++ b/tests/unit/fleet-overview.test.ts @@ -116,4 +116,36 @@ describe('persisted Fleet operations', () => { expect(fleetOperations(now)).toContainEqual(expect.objectContaining({ kind: 'beta', label: 'Observing a match', href: '/betas/1' })); expect(fleetOperations(now)).toContainEqual(expect.objectContaining({ kind: 'recovery', state: 'attention' })); }); + + it('preserves lifecycle selection across cutoff, deletion, supersession, and overlapping warning branches', async () => { + const { db } = await import('../../src/lib/server/db/index.js'); + const handle = db(); + const since = iso(-86_400_000); + handle.transaction(() => { + const insert = handle.prepare(`INSERT INTO server_deployments + (id, server, action, state, stage, started_at, finished_at, server_deleted_at, triggered_by) + VALUES (?, ?, 'restart', ?, 'Stage', ?, ?, ?, 'test')`); + let i = 0; + for (const state of ['queued', 'running', 'waiting', 'awaiting-health', 'done', 'warning', 'cancelled', 'failed', 'interrupted']) { + for (const finished of [null, iso(-86_400_001), since, iso()]) { + for (const deleted of [null, iso()]) { + for (const successor of ['none', 'live', 'deleted']) { + const id = `operation-${i++}`; + insert.run(id, id, state, iso(-i), finished, deleted); + if (successor !== 'none') insert.run(`${id}-next`, id, 'done', iso(-i), null, successor === 'deleted' ? iso() : null); + } + } + } + } + })(); + // The former OR query is the compatibility reference for all three branches. + const expected = handle.prepare(`SELECT d.id FROM server_deployments d + WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued', 'running', 'waiting', 'awaiting-health') + OR (d.state IN ('done', 'warning', 'cancelled') AND d.finished_at >= ?) + OR (d.state IN ('failed', 'interrupted', 'warning') AND NOT EXISTS + (SELECT 1 FROM server_deployments n WHERE n.server = d.server AND n.rowid > d.rowid)))`) + .all(since) as { id: string }[]; + const { fleetOperations } = await import('../../src/lib/server/fleet/overview.js'); + expect(fleetOperations(now).map(row => row.key).sort()).toEqual(expected.map(row => `deployment:${row.id}`).sort()); + }); }); diff --git a/tests/unit/provisioning-history-performance.test.ts b/tests/unit/provisioning-history-performance.test.ts index 90516e9..f809cac 100644 --- a/tests/unit/provisioning-history-performance.test.ts +++ b/tests/unit/provisioning-history-performance.test.ts @@ -14,6 +14,71 @@ afterEach(async () => { }); describe('provisioning reads with retained lifecycle history', () => { + it('uses indexed per-server submissions and separate lifecycle indexes for fleet reads', async () => { + const { db } = await import('../../src/lib/server/db/index.js'); + const { latestGameSubmission, latestGameSubmissions } = await import('../../src/lib/server/fleet/lifecycle-evidence.js'); + const { observationReader } = await import('../../src/lib/server/fleet/status.js'); + const { runningVersions } = await import('../../src/lib/server/integrations/plutonium.js'); + const { fleetOperations } = await import('../../src/lib/server/fleet/overview.js'); + const handle = db(); + const now = Date.parse('2026-10-05T12:00:00Z'); + const iso = (offset: number) => new Date(now + offset).toISOString(); + handle.transaction(() => { + const server = handle.prepare('INSERT INTO servers(name, komodo_name) VALUES (?, ?)'); + for (let i = 0; i < 16; i++) server.run(`server-${i}`, `server-${i}`); + server.run('empty', 'empty'); + const insert = handle.prepare(`INSERT INTO server_deployments + (id, server, action, state, stage, started_at, finished_at, submitted_at, server_deleted_at, triggered_by) + VALUES (?, ?, ?, 'done', 'Complete', ?, ?, ?, ?, 'test')`); + for (let i = 0; i < 6_000; i++) { + const at = iso(-100_000_000 - i); + insert.run(`history-${i}`, `server-${i % 16}`, 'restart', at, at, at, null); + } + for (const [i, action] of ['create', 'start', 'restart', 'rebuild'].entries()) { + insert.run(`current-${i}`, `server-${i}`, action, iso(-1000), iso(-1000), iso(-1000), null); + } + insert.run('stop', 'server-0', 'stop', iso(0), iso(0), iso(0), null); + insert.run('destroy', 'server-1', 'destroy', iso(0), iso(0), iso(0), null); + insert.run('deleted', 'server-2', 'restart', iso(0), iso(0), iso(0), iso(0)); + insert.run('unsubmitted', 'server-3', 'rebuild', iso(0), null, null, null); + handle.prepare('INSERT INTO restart_rollout(server, requested_at, healthy_at) VALUES (?, ?, ?)') + .run('server-0', iso(-500), iso(0)); + handle.prepare('INSERT INTO restart_rollout(server, requested_at) VALUES (?, ?)').run('server-1', iso(-2000)); + })(); + const submissions = latestGameSubmissions(); + expect(submissions.get('empty')).toBeNull(); + expect(submissions.get('server-0')).toBe(iso(-500)); + for (const i of [1, 2, 3]) expect(submissions.get(`server-${i}`)).toBe(iso(-1000)); + for (const [server, at] of submissions) expect(at).toBe(latestGameSubmission(server)); + + // Capture real snapshot queries so plan checks catch regressions at call sites. + for (const read of [observationReader, runningVersions]) { + const prepare = vi.spyOn(handle, 'prepare'); + read(); + const queries = prepare.mock.calls.map(([sql]) => sql).filter(sql => sql.includes('submitted_at')); + prepare.mockRestore(); + expect(queries).toHaveLength(1); + const plan = handle.prepare(`EXPLAIN QUERY PLAN ${queries[0]}`).all() as { detail: string }[]; + const details = plan.map(row => row.detail).join('\n'); + expect(details).toContain('SEARCH server_deployments USING COVERING INDEX idx_server_deployments_submission'); + expect(details).not.toMatch(/SCAN server_deployments\b/); + } + const prepare = vi.spyOn(handle, 'prepare'); + fleetOperations(now); + const queries = prepare.mock.calls.map(([sql]) => sql).filter(sql => sql.includes('server_deployments')); + prepare.mockRestore(); + expect(queries).toHaveLength(1); + const statement = handle.prepare(queries[0]); + expect(statement.columns().map(column => column.name)).toEqual(['id', 'server', 'action', 'state', 'stage', 'error', 'started_at']); + const plan = handle.prepare(`EXPLAIN QUERY PLAN ${queries[0]}`).all(iso(-86_400_000)) as { detail: string }[]; + const details = plan.map(row => row.detail).join('\n'); + expect(details).toContain('SEARCH server_deployments USING INDEX idx_server_deployments_state'); + expect(details).toContain('SEARCH server_deployments USING INDEX idx_server_deployments_finished_at'); + expect(details).toContain('SEARCH server_deployments USING INTEGER PRIMARY KEY (rowid=?)'); + expect(details).toContain('SCAN server_deployments USING COVERING INDEX idx_server_deployments_server'); + expect(details).not.toMatch(/SCAN server_deployments(?:\n|$)/); + }); + it('indexes the server and queue reads used by collection matching and health polling', async () => { const { db } = await import('../../src/lib/server/db/index.js'); const { latestGameSubmission } = await import('../../src/lib/server/fleet/lifecycle-evidence.js'); @@ -70,7 +135,8 @@ describe('provisioning reads with retained lifecycle history', () => { expect(handle.prepare('SELECT message FROM server_deployment_events').all()).toEqual([{ message: 'Creation complete' }]); const indexes = handle.pragma('index_list(server_deployments)') as { name: string }[]; expect(indexes.map(index => index.name)).toEqual(expect.arrayContaining([ - 'idx_server_deployments_server', 'idx_server_deployments_state', 'idx_server_deployments_started_at' + 'idx_server_deployments_server', 'idx_server_deployments_state', 'idx_server_deployments_started_at', + 'idx_server_deployments_submission', 'idx_server_deployments_finished_at' ])); expect(handle.pragma('foreign_key_check')).toEqual([]); closeDb();