import { actionLabels, type ProvisioningAction } from '../../provisioning.js'; import { db } from '../db/index.js'; import type { FleetOperation } from '../../fleet-overview.js'; /** Fleet clients need only active work and unresolved failures. */ export function fleetOperations(now = Date.now()): FleetOperation[] { return readOperations(now); } /** Recent completions are loaded only by the server's own page. */ export function serverOperations(server: string, now = Date.now()): FleetOperation[] { return readOperations(now, server); } function readOperations(now: number, server?: string): FleetOperation[] { const since = new Date(now - 86_400_000).toISOString(); const bindings = { server: server ?? null, since }; const scope = (column: string) => server === undefined ? '' : `AND ${column} = @server`; const handle = db(); const operations: FleetOperation[] = []; const slots = handle.prepare(`SELECT server, requested_at, error FROM restart_rollout WHERE healthy_at IS NULL ${scope('server')}`) .all(bindings) as { server: string; requested_at: string; error: string | null }[]; const recovering = new Map(slots.map(slot => [slot.server, slot])); const representedRecovery = new Set(); const campaigns = handle.prepare(`SELECT d.id, d.action, d.started_at, ds.server, ds.restart_state, ds.error, ds.restarted_at, ds.players FROM deployments d JOIN deployment_servers ds ON ds.deployment_id = d.id WHERE ds.server_deleted_at IS NULL ${scope('ds.server')} AND (ds.restart_state != 'done' ${server === undefined ? '' : 'OR ds.checked_at >= @since OR d.finished_at >= @since OR d.started_at >= @since'})`).all(bindings) as { id: string; action: string; started_at: string; server: string; restart_state: string; error: string | null; restarted_at: string | null; players: number | null; }[]; for (const row of campaigns) { const state = row.restart_state === 'done' ? 'done' : ['failed', 'manual'].includes(row.restart_state) ? 'attention' : 'active'; const slot = recovering.get(row.server); if (row.restarted_at && state !== 'done') representedRecovery.add(row.server); const label = state === 'done' ? 'Update complete' : state === 'attention' ? 'Update needs attention' : row.restart_state === 'held' ? 'Held by a beta' : row.restarted_at ? 'Recovering' : (row.players ?? 0) > 0 ? 'Waiting for an empty lobby' : 'Queued for restart'; operations.push({ key: `campaign:${row.id}:${row.server}`, group: `campaign:${row.id}`, kind: 'campaign', server: row.server, label, state, startedAt: row.started_at, detail: state === 'done' ? 'Update applied and restart checks completed' : row.error ?? (row.restarted_at ? slot?.error ?? 'Verifying server health' : (row.players ?? 0) > 0 ? `${row.players} players at the last check` : 'Waiting for restart eligibility and available capacity'), href: `/campaigns/${encodeURIComponent(row.id)}` }); } // Only the newest lifecycle operation can still require action on a server. // Separate branches use state/rowid indexes for fleet reads and the server // index for detail history. Completed work is never loaded for the full fleet. const lifecycle = handle.prepare(`SELECT id, server, action, state, stage, error, started_at FROM server_deployments WHERE server_deleted_at IS NULL ${scope('server')} AND state IN ('queued', 'running', 'waiting', 'awaiting-health') ${server === undefined ? '' : `UNION SELECT id, server, action, state, stage, error, started_at FROM server_deployments WHERE server_deleted_at IS NULL AND server = @server AND +state IN ('done', 'warning', 'cancelled', 'failed', 'interrupted') AND finished_at >= @since`} UNION SELECT id, server, action, state, stage, error, started_at FROM server_deployments WHERE server_deleted_at IS NULL ${scope('server')} AND +state IN ('failed', 'interrupted') AND rowid IN (SELECT MAX(rowid) FROM server_deployments WHERE 1=1 ${scope('server')} GROUP BY server)`) .all(bindings) as { id: string; server: string; action: string; state: string; stage: string; error: string | null; started_at: string }[]; for (const row of lifecycle) { if (row.state === 'awaiting-health') representedRecovery.add(row.server); const state = ['done', 'warning', 'cancelled'].includes(row.state) ? 'done' : ['failed', 'interrupted'].includes(row.state) ? 'attention' : 'active'; operations.push({ key: `deployment:${row.id}`, group: `deployment:${row.id}`, kind: 'deployment', server: row.server, state, startedAt: row.started_at, label: state === 'done' ? row.state === 'cancelled' ? 'Removal cancelled' : row.state === 'warning' ? 'Completed with warnings' : 'Complete' : state === 'attention' ? `${actionLabels[row.action as ProvisioningAction]} needs attention` : row.stage, detail: row.error ?? row.stage, href: `/provisioning/${encodeURIComponent(row.id)}` }); } const betas = handle.prepare(`SELECT b.id, b.opened_at, bs.server, bs.stage, bs.error FROM betas b JOIN beta_servers bs ON bs.beta_id = b.id WHERE bs.server_deleted_at IS NULL ${scope('bs.server')} AND b.state IN ('running', 'ready-promote')`) .all(bindings) as { id: number; opened_at: string; server: string; stage: string; error: string | null }[]; const stages: Record = { queued: 'Waiting for an earlier test wave', deploying: 'Deploying beta', 'awaiting-empty': 'Waiting for an empty lobby', restarting: 'Restarting', 'awaiting-health': 'Recovering', 'awaiting-match': 'Observing a match', ready: 'Ready for review', failed: 'Beta needs attention' }; for (const row of betas) { if (['restarting', 'awaiting-health'].includes(row.stage)) representedRecovery.add(row.server); operations.push({ key: `beta:${row.id}:${row.server}`, group: `beta:${row.id}`, kind: 'beta', server: row.server, startedAt: row.opened_at, state: row.stage === 'failed' ? 'attention' : 'active', label: stages[row.stage] ?? row.stage, detail: row.error ?? recovering.get(row.server)?.error ?? `Beta ${row.id}`, href: `/betas/${row.id}` }); } const maintenance = handle.prepare(`SELECT m.* FROM maintenance_jobs m WHERE m.server_deleted_at IS NULL ${scope('m.server')} AND m.state = 'failed' AND NOT EXISTS (SELECT 1 FROM maintenance_jobs n WHERE n.server = m.server AND n.kind = m.kind AND n.id > m.id)`) .all(bindings) as { id: number; server: string; kind: string; detail: string | null; requested_at: string }[]; for (const job of maintenance) { operations.push({ key: `maintenance:${job.id}`, group: `maintenance:${job.id}`, kind: 'maintenance', server: job.server, state: 'attention', startedAt: job.requested_at, label: job.kind === 'update' ? 'Update verification needs attention' : 'Maintenance needs attention', detail: job.detail ?? 'Inspect the maintenance result', href: `/maintenance?view=activity&server=${encodeURIComponent(job.server)}#job-${job.id}` }); } for (const slot of slots) { if (representedRecovery.has(slot.server)) continue; operations.push({ key: `recovery:${slot.server}:${slot.requested_at}`, group: `recovery:${slot.server}`, kind: 'recovery', server: slot.server, startedAt: slot.requested_at, state: slot.error ? 'attention' : 'active', label: 'Recovering from restart', detail: slot.error ?? 'Verifying server health', href: `/servers/${encodeURIComponent(slot.server)}` }); } return operations.sort((a, b) => b.startedAt.localeCompare(a.startedAt)); }