Refresh Fleet with discovery, operation progress, and lobby watches

2026-09-18 11:59:14 -04:00
parent ef8ff69d17
commit 9cf0bd87f4
14 changed files with 761 additions and 240 deletions
@@ -0,0 +1,73 @@
+import { db } from '../db/index.js';
+import type { FleetOperation } from '../../fleet-overview.js';
+
+/** Read persisted work, including recent completions for one-shot lobby watches. */
+export function fleetOperations(now = Date.now()): FleetOperation[] {
+    const since = new Date(now - 86_400_000).toISOString();
+    const handle = db();
+    const operations: FleetOperation[] = [];
+    const slots = handle.prepare('SELECT server, requested_at, error FROM restart_rollout WHERE healthy_at IS NULL')
+        .all() as { server: string; requested_at: string; error: string | null }[];
+    const recovering = new Map(slots.map(slot => [slot.server, slot]));
+    const representedRecovery = new Set<string>();
+    const campaigns = handle.prepare(`SELECT d.id, d.action, d.started_at, ds.server,
+        ds.restart_state, ds.error, ds.restarted_at, ds.players
+        FROM deployments d JOIN deployment_servers ds ON ds.deployment_id = d.id
+        WHERE ds.restart_state != 'done' OR ds.checked_at >= ? OR d.finished_at >= ? OR d.started_at >= ?`).all(since, since, since) as {
+        id: string; action: string; started_at: string; server: string; restart_state: string;
+        error: string | null; restarted_at: string | null; players: number | null;
+    }[];
+    for (const row of campaigns) {
+        const state = row.restart_state === 'done' ? 'done'
+            : ['failed', 'manual'].includes(row.restart_state) ? 'attention' : 'active';
+        const slot = recovering.get(row.server);
+        if (row.restarted_at && state !== 'done') representedRecovery.add(row.server);
+        const label = state === 'done' ? 'Update complete' : state === 'attention' ? 'Update needs attention'
+            : row.restart_state === 'held' ? 'Held by a beta'
+            : row.restarted_at ? 'Recovering' : (row.players ?? 0) > 0 ? 'Waiting for an empty lobby' : 'Queued for restart';
+        operations.push({ key: `campaign:${row.id}:${row.server}`, group: `campaign:${row.id}`,
+            kind: 'campaign', server: row.server, label, state, startedAt: row.started_at,
+            detail: state === 'done' ? 'Update applied and restart checks completed' : row.error ?? (row.restarted_at ? slot?.error ?? 'Verifying server health'
+                : (row.players ?? 0) > 0 ? `${row.players} players at the last check`
+                : 'Waiting for restart eligibility and available capacity'),
+            href: `/campaigns/${encodeURIComponent(row.id)}` });
+    }
+    // Only the newest lifecycle operation can still require action on a server.
+    const lifecycle = handle.prepare(`SELECT d.* FROM server_deployments d
+        WHERE d.state IN ('queued', 'running', 'waiting', 'awaiting-health')
+        OR (d.state IN ('done', 'warning') AND d.finished_at >= ?)
+        OR (d.state IN ('failed', 'interrupted', 'warning') AND NOT EXISTS
+            (SELECT 1 FROM server_deployments n WHERE n.server = d.server AND n.rowid > d.rowid))`)
+        .all(since) as { id: string; server: string; action: string; state: string; stage: string;
+            error: string | null; started_at: string }[];
+    for (const row of lifecycle) {
+        const state = row.state === 'done' || row.state === 'warning' ? 'done'
+            : ['failed', 'interrupted'].includes(row.state) ? 'attention' : 'active';
+        operations.push({ key: `deployment:${row.id}`, group: `deployment:${row.id}`, kind: 'deployment',
+            server: row.server, state, startedAt: row.started_at,
+            label: state === 'done' ? row.state === 'warning' ? 'Completed with warnings' : 'Complete'
+                : state === 'attention' ? `${row.action === 'create' ? 'Launch' : 'Removal'} needs attention` : row.stage,
+            detail: row.error ?? row.stage, href: `/deployments/${encodeURIComponent(row.id)}` });
+    }
+    const betas = handle.prepare(`SELECT b.id, b.opened_at, bs.server, bs.stage, bs.error
+        FROM betas b JOIN beta_servers bs ON bs.beta_id = b.id WHERE b.state IN ('running', 'ready-promote')`)
+        .all() as { id: number; opened_at: string; server: string; stage: string; error: string | null }[];
+    const stages: Record<string, string> = { queued: 'Waiting for an earlier test wave', deploying: 'Deploying beta',
+        'awaiting-empty': 'Waiting for an empty lobby', restarting: 'Restarting', 'awaiting-health': 'Recovering',
+        'awaiting-match': 'Observing a match', ready: 'Ready for review', failed: 'Beta needs attention' };
+    for (const row of betas) {
+        if (['restarting', 'awaiting-health'].includes(row.stage)) representedRecovery.add(row.server);
+        operations.push({ key: `beta:${row.id}:${row.server}`, group: `beta:${row.id}`,
+            kind: 'beta', server: row.server, startedAt: row.opened_at,
+            state: row.stage === 'failed' ? 'attention' : 'active', label: stages[row.stage] ?? row.stage,
+            detail: row.error ?? recovering.get(row.server)?.error ?? `Beta ${row.id}`, href: `/betas/${row.id}` });
+    }
+    for (const slot of slots) {
+        if (representedRecovery.has(slot.server)) continue;
+        operations.push({ key: `recovery:${slot.server}:${slot.requested_at}`, group: `recovery:${slot.server}`,
+            kind: 'recovery', server: slot.server, startedAt: slot.requested_at, state: slot.error ? 'attention' : 'active',
+            label: 'Recovering from restart', detail: slot.error ?? 'Verifying server health',
+            href: `/servers/${encodeURIComponent(slot.server)}` });
+    }
+    return operations.sort((a, b) => b.startedAt.localeCompare(a.startedAt));
+}