Avoid lifecycle history scans in fleet snapshots

src/lib/server/db/index.ts

@@ -380,7 +380,11 @@ export function db(): Database.Database {
             CREATE INDEX IF NOT EXISTS idx_server_deployments_state
             ON server_deployments(state, action, server_deleted_at);
             CREATE INDEX IF NOT EXISTS idx_server_deployments_started_at
-            ON server_deployments(started_at);`);
+            ON server_deployments(started_at);
+            CREATE INDEX IF NOT EXISTS idx_server_deployments_submission
+            ON server_deployments(server, server_deleted_at, action, submitted_at);
+            CREATE INDEX IF NOT EXISTS idx_server_deployments_finished_at
+            ON server_deployments(finished_at) WHERE server_deleted_at IS NULL;`);
         installReportRevisions(handle);
     } catch (err) {
         handle.close();

src/lib/server/fleet/lifecycle-evidence.ts

@@ -3,6 +3,17 @@ import { db } from '../db/index.js';
 /** Reports from before the latest lifecycle submission describe the old process. */
 export function latestGameSubmission(server: string): string | null {
     return (db().prepare(`SELECT MAX(at) AS at FROM (
-        SELECT submitted_at AS at FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild')
+        SELECT MAX(submitted_at) AS at FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild')
         UNION ALL SELECT requested_at AS at FROM restart_rollout WHERE server=?)`).get(server, server) as { at: string | null }).at;
 }
+
+/** Look up each registered server through the covering submission index. */
+export function latestGameSubmissions(): Map<string, string | null> {
+    const rows = db().prepare(`SELECT server, MAX(at) AS at FROM (
+        SELECT s.name AS server, (SELECT MAX(submitted_at) FROM server_deployments
+            WHERE server=s.name AND server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild')) AS at
+        FROM servers s
+        UNION ALL SELECT server, requested_at AS at FROM restart_rollout) GROUP BY server`).all() as
+        { server: string; at: string | null }[];
+    return new Map(rows.map(row => [row.server, row.at]));
+}

src/lib/server/fleet/overview.ts

@@ -34,11 +34,18 @@ export function fleetOperations(now = Date.now()): FleetOperation[] {
             href: `/campaigns/${encodeURIComponent(row.id)}` });
     }
     // Only the newest lifecycle operation can still require action on a server.
-    const lifecycle = handle.prepare(`SELECT d.* FROM server_deployments d
+    // Separate branches let SQLite use state and finish indexes; UNION deduplicates
-        WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued', 'running', 'waiting', 'awaiting-health')
+    // recent warnings that are also the newest operation. Unary + keeps terminal
-        OR (d.state IN ('done', 'warning', 'cancelled') AND d.finished_at >= ?)
+    // branches on finished_at and rowid instead of searching historical states.
-        OR (d.state IN ('failed', 'interrupted', 'warning') AND NOT EXISTS
+    const lifecycle = handle.prepare(`SELECT id, server, action, state, stage, error, started_at FROM server_deployments
-            (SELECT 1 FROM server_deployments n WHERE n.server = d.server AND n.rowid > d.rowid)))`)
+        WHERE server_deleted_at IS NULL AND state IN ('queued', 'running', 'waiting', 'awaiting-health')
+        UNION
+        SELECT id, server, action, state, stage, error, started_at FROM server_deployments
+        WHERE server_deleted_at IS NULL AND +state IN ('done', 'warning', 'cancelled') AND finished_at >= ?
+        UNION
+        SELECT id, server, action, state, stage, error, started_at FROM server_deployments
+        WHERE server_deleted_at IS NULL AND +state IN ('failed', 'interrupted', 'warning')
+            AND rowid IN (SELECT MAX(rowid) FROM server_deployments GROUP BY server)`)
         .all(since) as { id: string; server: string; action: string; state: string; stage: string;
             error: string | null; started_at: string }[];
     for (const row of lifecycle) {

src/lib/server/fleet/status.ts

@@ -4,6 +4,7 @@ import { isProvisioning } from './provisioning.js';
 import { restartSettling } from '../provisioning/restarts.js';
 import { type ServerRow } from './servers.js';
 import { telemetryReport, latestGameSubmission } from './status-evidence.js';
+import { latestGameSubmissions } from './lifecycle-evidence.js';
 import { db } from '../db/index.js';
 import { parseGameEvent, type GameTelemetry } from '../../telemetry.js';
 import { restartRequestInFlight } from '../provisioning/restarts.js';
@@ -18,10 +19,7 @@ export function observationReader() {
         { server: string; action: string; state: string }[]).map(row => [row.server, row]));
     const restarts = new Map((handle.prepare('SELECT server, requested_at, healthy_at FROM restart_rollout').all() as
         { server: string; requested_at: string; healthy_at: string | null }[]).map(row => [row.server, row]));
-    const submissions = new Map((handle.prepare(`SELECT server, MAX(at) AS at FROM (
+    const submissions = latestGameSubmissions();
-        SELECT server, submitted_at AS at FROM server_deployments WHERE server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild')
-        UNION ALL SELECT server, requested_at AS at FROM restart_rollout) GROUP BY server`).all() as
-        { server: string; at: string | null }[]).map(row => [row.server, row.at]));
     const reports = new Map<string, GameTelemetry>();
     for (const row of handle.prepare('SELECT server, received_at, payload FROM server_telemetry WHERE payload IS NOT NULL').all() as
         { server: string; received_at: string; payload: string }[]) {

src/lib/server/integrations/plutonium.ts

@@ -3,6 +3,7 @@ import { config } from '../config.js';
 import { getServer, listServers } from '../fleet/servers.js';
 import { hasActiveProvisioning } from '../fleet/provisioning.js';
 import { latestGameSubmission } from '../fleet/status-evidence.js';
+import { latestGameSubmissions } from '../fleet/lifecycle-evidence.js';
 import { rconPlutoniumVersion } from './rcon.js';
 import { komodoInspectHealthy } from './komodo.js';
 import { restartRecoveryPending } from '../provisioning/index.js';
@@ -23,9 +24,7 @@ export function runningVersions(): PlutoniumVersion[] {
     const versions = db().prepare('SELECT * FROM plutonium_versions').all() as PlutoniumVersion[];
     const target = freshRelease();
     const byServer = new Map(versions.map(version => [version.server, version]));
-    const epochs = new Map((db().prepare(`SELECT server, MAX(at) AS at FROM (
+    const epochs = latestGameSubmissions();
-        SELECT server, submitted_at AS at FROM server_deployments WHERE server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild')
-        UNION ALL SELECT server, requested_at AS at FROM restart_rollout) GROUP BY server`).all() as { server: string; at: string | null }[]).map(row => [row.server, row.at]));
     const recovering = new Set((db().prepare('SELECT server FROM restart_rollout WHERE healthy_at IS NULL').all() as { server: string }[]).map(row => row.server));
     return listServers().map(server => {
         const value = byServer.get(server.name)

tests/unit/fleet-overview.test.ts

@@ -116,4 +116,36 @@ describe('persisted Fleet operations', () => {
         expect(fleetOperations(now)).toContainEqual(expect.objectContaining({ kind: 'beta', label: 'Observing a match', href: '/betas/1' }));
         expect(fleetOperations(now)).toContainEqual(expect.objectContaining({ kind: 'recovery', state: 'attention' }));
     });
+
+    it('preserves lifecycle selection across cutoff, deletion, supersession, and overlapping warning branches', async () => {
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const handle = db();
+        const since = iso(-86_400_000);
+        handle.transaction(() => {
+            const insert = handle.prepare(`INSERT INTO server_deployments
+                (id, server, action, state, stage, started_at, finished_at, server_deleted_at, triggered_by)
+                VALUES (?, ?, 'restart', ?, 'Stage', ?, ?, ?, 'test')`);
+            let i = 0;
+            for (const state of ['queued', 'running', 'waiting', 'awaiting-health', 'done', 'warning', 'cancelled', 'failed', 'interrupted']) {
+                for (const finished of [null, iso(-86_400_001), since, iso()]) {
+                    for (const deleted of [null, iso()]) {
+                        for (const successor of ['none', 'live', 'deleted']) {
+                            const id = `operation-${i++}`;
+                            insert.run(id, id, state, iso(-i), finished, deleted);
+                            if (successor !== 'none') insert.run(`${id}-next`, id, 'done', iso(-i), null, successor === 'deleted' ? iso() : null);
+                        }
+                    }
+                }
+            }
+        })();
+        // The former OR query is the compatibility reference for all three branches.
+        const expected = handle.prepare(`SELECT d.id FROM server_deployments d
+            WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued', 'running', 'waiting', 'awaiting-health')
+            OR (d.state IN ('done', 'warning', 'cancelled') AND d.finished_at >= ?)
+            OR (d.state IN ('failed', 'interrupted', 'warning') AND NOT EXISTS
+                (SELECT 1 FROM server_deployments n WHERE n.server = d.server AND n.rowid > d.rowid)))`)
+            .all(since) as { id: string }[];
+        const { fleetOperations } = await import('../../src/lib/server/fleet/overview.js');
+        expect(fleetOperations(now).map(row => row.key).sort()).toEqual(expected.map(row => `deployment:${row.id}`).sort());
+    });
 });

tests/unit/provisioning-history-performance.test.ts

@@ -14,6 +14,71 @@ afterEach(async () => {
 });
 
 describe('provisioning reads with retained lifecycle history', () => {
+    it('uses indexed per-server submissions and separate lifecycle indexes for fleet reads', async () => {
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { latestGameSubmission, latestGameSubmissions } = await import('../../src/lib/server/fleet/lifecycle-evidence.js');
+        const { observationReader } = await import('../../src/lib/server/fleet/status.js');
+        const { runningVersions } = await import('../../src/lib/server/integrations/plutonium.js');
+        const { fleetOperations } = await import('../../src/lib/server/fleet/overview.js');
+        const handle = db();
+        const now = Date.parse('2026-10-05T12:00:00Z');
+        const iso = (offset: number) => new Date(now + offset).toISOString();
+        handle.transaction(() => {
+            const server = handle.prepare('INSERT INTO servers(name, komodo_name) VALUES (?, ?)');
+            for (let i = 0; i < 16; i++) server.run(`server-${i}`, `server-${i}`);
+            server.run('empty', 'empty');
+            const insert = handle.prepare(`INSERT INTO server_deployments
+                (id, server, action, state, stage, started_at, finished_at, submitted_at, server_deleted_at, triggered_by)
+                VALUES (?, ?, ?, 'done', 'Complete', ?, ?, ?, ?, 'test')`);
+            for (let i = 0; i < 6_000; i++) {
+                const at = iso(-100_000_000 - i);
+                insert.run(`history-${i}`, `server-${i % 16}`, 'restart', at, at, at, null);
+            }
+            for (const [i, action] of ['create', 'start', 'restart', 'rebuild'].entries()) {
+                insert.run(`current-${i}`, `server-${i}`, action, iso(-1000), iso(-1000), iso(-1000), null);
+            }
+            insert.run('stop', 'server-0', 'stop', iso(0), iso(0), iso(0), null);
+            insert.run('destroy', 'server-1', 'destroy', iso(0), iso(0), iso(0), null);
+            insert.run('deleted', 'server-2', 'restart', iso(0), iso(0), iso(0), iso(0));
+            insert.run('unsubmitted', 'server-3', 'rebuild', iso(0), null, null, null);
+            handle.prepare('INSERT INTO restart_rollout(server, requested_at, healthy_at) VALUES (?, ?, ?)')
+                .run('server-0', iso(-500), iso(0));
+            handle.prepare('INSERT INTO restart_rollout(server, requested_at) VALUES (?, ?)').run('server-1', iso(-2000));
+        })();
+        const submissions = latestGameSubmissions();
+        expect(submissions.get('empty')).toBeNull();
+        expect(submissions.get('server-0')).toBe(iso(-500));
+        for (const i of [1, 2, 3]) expect(submissions.get(`server-${i}`)).toBe(iso(-1000));
+        for (const [server, at] of submissions) expect(at).toBe(latestGameSubmission(server));
+
+        // Capture real snapshot queries so plan checks catch regressions at call sites.
+        for (const read of [observationReader, runningVersions]) {
+            const prepare = vi.spyOn(handle, 'prepare');
+            read();
+            const queries = prepare.mock.calls.map(([sql]) => sql).filter(sql => sql.includes('submitted_at'));
+            prepare.mockRestore();
+            expect(queries).toHaveLength(1);
+            const plan = handle.prepare(`EXPLAIN QUERY PLAN ${queries[0]}`).all() as { detail: string }[];
+            const details = plan.map(row => row.detail).join('\n');
+            expect(details).toContain('SEARCH server_deployments USING COVERING INDEX idx_server_deployments_submission');
+            expect(details).not.toMatch(/SCAN server_deployments\b/);
+        }
+        const prepare = vi.spyOn(handle, 'prepare');
+        fleetOperations(now);
+        const queries = prepare.mock.calls.map(([sql]) => sql).filter(sql => sql.includes('server_deployments'));
+        prepare.mockRestore();
+        expect(queries).toHaveLength(1);
+        const statement = handle.prepare(queries[0]);
+        expect(statement.columns().map(column => column.name)).toEqual(['id', 'server', 'action', 'state', 'stage', 'error', 'started_at']);
+        const plan = handle.prepare(`EXPLAIN QUERY PLAN ${queries[0]}`).all(iso(-86_400_000)) as { detail: string }[];
+        const details = plan.map(row => row.detail).join('\n');
+        expect(details).toContain('SEARCH server_deployments USING INDEX idx_server_deployments_state');
+        expect(details).toContain('SEARCH server_deployments USING INDEX idx_server_deployments_finished_at');
+        expect(details).toContain('SEARCH server_deployments USING INTEGER PRIMARY KEY (rowid=?)');
+        expect(details).toContain('SCAN server_deployments USING COVERING INDEX idx_server_deployments_server');
+        expect(details).not.toMatch(/SCAN server_deployments(?:\n|$)/);
+    });
+
     it('indexes the server and queue reads used by collection matching and health polling', async () => {
         const { db } = await import('../../src/lib/server/db/index.js');
         const { latestGameSubmission } = await import('../../src/lib/server/fleet/lifecycle-evidence.js');
@@ -70,7 +135,8 @@ describe('provisioning reads with retained lifecycle history', () => {
             expect(handle.prepare('SELECT message FROM server_deployment_events').all()).toEqual([{ message: 'Creation complete' }]);
             const indexes = handle.pragma('index_list(server_deployments)') as { name: string }[];
             expect(indexes.map(index => index.name)).toEqual(expect.arrayContaining([
-                'idx_server_deployments_server', 'idx_server_deployments_state', 'idx_server_deployments_started_at'
+                'idx_server_deployments_server', 'idx_server_deployments_state', 'idx_server_deployments_started_at',
+                'idx_server_deployments_submission', 'idx_server_deployments_finished_at'
             ]));
             expect(handle.pragma('foreign_key_check')).toEqual([]);
             closeDb();