josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Scope server snapshots and keep completed history off fleet refreshes

Browse Source
This commit is contained in:
josh
2026-10-05 15:37:56 -04:00
parent 030794635e
commit d24c52b5cc
12 changed files with 376 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/fleet/overview.ts
+30 -18
View File
@@ -2,19 +2,31 @@ import { actionLabels, type ProvisioningAction } from '../../provisioning.js';
import { db } from '../db/index.js';
import type { FleetOperation } from '../../fleet-overview.js';
/** Read persisted work, including recent completions for one-shot lobby watches. */
/** Fleet clients need only active work and unresolved failures. */
export function fleetOperations(now = Date.now()): FleetOperation[] {
return readOperations(now);
}
/** Recent completions are loaded only by the server's own page. */
export function serverOperations(server: string, now = Date.now()): FleetOperation[] {
return readOperations(now, server);
}
function readOperations(now: number, server?: string): FleetOperation[] {
const since = new Date(now - 86_400_000).toISOString();
const bindings = { server: server ?? null, since };
const scope = (column: string) => server === undefined ? '' : `AND ${column} = @server`;
const handle = db();
const operations: FleetOperation[] = [];
const slots = handle.prepare('SELECT server, requested_at, error FROM restart_rollout WHERE healthy_at IS NULL')
.all() as { server: string; requested_at: string; error: string | null }[];
const slots = handle.prepare(`SELECT server, requested_at, error FROM restart_rollout WHERE healthy_at IS NULL ${scope('server')}`)
.all(bindings) as { server: string; requested_at: string; error: string | null }[];
const recovering = new Map(slots.map(slot => [slot.server, slot]));
const representedRecovery = new Set<string>();
const campaigns = handle.prepare(`SELECT d.id, d.action, d.started_at, ds.server,
ds.restart_state, ds.error, ds.restarted_at, ds.players
FROM deployments d JOIN deployment_servers ds ON ds.deployment_id = d.id
WHERE ds.server_deleted_at IS NULL AND (ds.restart_state != 'done' OR ds.checked_at >= ? OR d.finished_at >= ? OR d.started_at >= ?)`).all(since, since, since) as {
WHERE ds.server_deleted_at IS NULL ${scope('ds.server')} AND (ds.restart_state != 'done'
${server === undefined ? '' : 'OR ds.checked_at >= @since OR d.finished_at >= @since OR d.started_at >= @since'})`).all(bindings) as {
id: string; action: string; started_at: string; server: string; restart_state: string;
error: string | null; restarted_at: string | null; players: number | null;
}[];
@@ -34,19 +46,19 @@ export function fleetOperations(now = Date.now()): FleetOperation[] {
href: `/campaigns/${encodeURIComponent(row.id)}` });
}
// Only the newest lifecycle operation can still require action on a server.
// Separate branches let SQLite use state and finish indexes; UNION deduplicates
// recent warnings that are also the newest operation. Unary + keeps terminal
// branches on finished_at and rowid instead of searching historical states.
// Separate branches use state/rowid indexes for fleet reads and the server
// index for detail history. Completed work is never loaded for the full fleet.
const lifecycle = handle.prepare(`SELECT id, server, action, state, stage, error, started_at FROM server_deployments
WHERE server_deleted_at IS NULL AND state IN ('queued', 'running', 'waiting', 'awaiting-health')
WHERE server_deleted_at IS NULL ${scope('server')} AND state IN ('queued', 'running', 'waiting', 'awaiting-health')
${server === undefined ? '' : `UNION
SELECT id, server, action, state, stage, error, started_at FROM server_deployments
WHERE server_deleted_at IS NULL AND server = @server
AND +state IN ('done', 'warning', 'cancelled', 'failed', 'interrupted') AND finished_at >= @since`}
UNION
SELECT id, server, action, state, stage, error, started_at FROM server_deployments
WHERE server_deleted_at IS NULL AND +state IN ('done', 'warning', 'cancelled') AND finished_at >= ?
UNION
SELECT id, server, action, state, stage, error, started_at FROM server_deployments
WHERE server_deleted_at IS NULL AND +state IN ('failed', 'interrupted', 'warning')
AND rowid IN (SELECT MAX(rowid) FROM server_deployments GROUP BY server)`)
.all(since) as { id: string; server: string; action: string; state: string; stage: string;
WHERE server_deleted_at IS NULL ${scope('server')} AND +state IN ('failed', 'interrupted')
AND rowid IN (SELECT MAX(rowid) FROM server_deployments WHERE 1=1 ${scope('server')} GROUP BY server)`)
.all(bindings) as { id: string; server: string; action: string; state: string; stage: string;
error: string | null; started_at: string }[];
for (const row of lifecycle) {
if (row.state === 'awaiting-health') representedRecovery.add(row.server);
@@ -59,8 +71,8 @@ export function fleetOperations(now = Date.now()): FleetOperation[] {
detail: row.error ?? row.stage, href: `/provisioning/${encodeURIComponent(row.id)}` });
}
const betas = handle.prepare(`SELECT b.id, b.opened_at, bs.server, bs.stage, bs.error
FROM betas b JOIN beta_servers bs ON bs.beta_id = b.id WHERE bs.server_deleted_at IS NULL AND b.state IN ('running', 'ready-promote')`)
.all() as { id: number; opened_at: string; server: string; stage: string; error: string | null }[];
FROM betas b JOIN beta_servers bs ON bs.beta_id = b.id WHERE bs.server_deleted_at IS NULL ${scope('bs.server')} AND b.state IN ('running', 'ready-promote')`)
.all(bindings) as { id: number; opened_at: string; server: string; stage: string; error: string | null }[];
const stages: Record<string, string> = { queued: 'Waiting for an earlier test wave', deploying: 'Deploying beta',
'awaiting-empty': 'Waiting for an empty lobby', restarting: 'Restarting', 'awaiting-health': 'Recovering',
'awaiting-match': 'Observing a match', ready: 'Ready for review', failed: 'Beta needs attention' };
@@ -71,9 +83,9 @@ export function fleetOperations(now = Date.now()): FleetOperation[] {
state: row.stage === 'failed' ? 'attention' : 'active', label: stages[row.stage] ?? row.stage,
detail: row.error ?? recovering.get(row.server)?.error ?? `Beta ${row.id}`, href: `/betas/${row.id}` });
}
const maintenance = handle.prepare(`SELECT m.* FROM maintenance_jobs m WHERE m.server_deleted_at IS NULL AND m.state = 'failed'
const maintenance = handle.prepare(`SELECT m.* FROM maintenance_jobs m WHERE m.server_deleted_at IS NULL ${scope('m.server')} AND m.state = 'failed'
AND NOT EXISTS (SELECT 1 FROM maintenance_jobs n WHERE n.server = m.server AND n.kind = m.kind AND n.id > m.id)`)
.all() as { id: number; server: string; kind: string; detail: string | null; requested_at: string }[];
.all(bindings) as { id: number; server: string; kind: string; detail: string | null; requested_at: string }[];
for (const job of maintenance) {
operations.push({ key: `maintenance:${job.id}`, group: `maintenance:${job.id}`, kind: 'maintenance',
server: job.server, state: 'attention', startedAt: job.requested_at,