85 lines
6.5 KiB
TypeScript
85 lines
6.5 KiB
TypeScript
import { actionLabels, type ProvisioningAction } from '../../provisioning.js';
|
|
import { db } from '../db/index.js';
|
|
import type { FleetOperation } from '../../fleet-overview.js';
|
|
|
|
/** Read persisted work, including recent completions for one-shot lobby watches. */
|
|
export function fleetOperations(now = Date.now()): FleetOperation[] {
|
|
const since = new Date(now - 86_400_000).toISOString();
|
|
const handle = db();
|
|
const operations: FleetOperation[] = [];
|
|
const slots = handle.prepare('SELECT server, requested_at, error FROM restart_rollout WHERE healthy_at IS NULL')
|
|
.all() as { server: string; requested_at: string; error: string | null }[];
|
|
const recovering = new Map(slots.map(slot => [slot.server, slot]));
|
|
const representedRecovery = new Set<string>();
|
|
const campaigns = handle.prepare(`SELECT d.id, d.action, d.started_at, ds.server,
|
|
ds.restart_state, ds.error, ds.restarted_at, ds.players
|
|
FROM deployments d JOIN deployment_servers ds ON ds.deployment_id = d.id
|
|
WHERE ds.restart_state != 'done' OR ds.checked_at >= ? OR d.finished_at >= ? OR d.started_at >= ?`).all(since, since, since) as {
|
|
id: string; action: string; started_at: string; server: string; restart_state: string;
|
|
error: string | null; restarted_at: string | null; players: number | null;
|
|
}[];
|
|
for (const row of campaigns) {
|
|
const state = row.restart_state === 'done' ? 'done'
|
|
: ['failed', 'manual'].includes(row.restart_state) ? 'attention' : 'active';
|
|
const slot = recovering.get(row.server);
|
|
if (row.restarted_at && state !== 'done') representedRecovery.add(row.server);
|
|
const label = state === 'done' ? 'Update complete' : state === 'attention' ? 'Update needs attention'
|
|
: row.restart_state === 'held' ? 'Held by a beta'
|
|
: row.restarted_at ? 'Recovering' : (row.players ?? 0) > 0 ? 'Waiting for an empty lobby' : 'Queued for restart';
|
|
operations.push({ key: `campaign:${row.id}:${row.server}`, group: `campaign:${row.id}`,
|
|
kind: 'campaign', server: row.server, label, state, startedAt: row.started_at,
|
|
detail: state === 'done' ? 'Update applied and restart checks completed' : row.error ?? (row.restarted_at ? slot?.error ?? 'Verifying server health'
|
|
: (row.players ?? 0) > 0 ? `${row.players} players at the last check`
|
|
: 'Waiting for restart eligibility and available capacity'),
|
|
href: `/campaigns/${encodeURIComponent(row.id)}` });
|
|
}
|
|
// Only the newest lifecycle operation can still require action on a server.
|
|
const lifecycle = handle.prepare(`SELECT d.* FROM server_deployments d
|
|
WHERE d.state IN ('queued', 'running', 'waiting', 'awaiting-health')
|
|
OR (d.state IN ('done', 'warning', 'cancelled') AND d.finished_at >= ?)
|
|
OR (d.state IN ('failed', 'interrupted', 'warning') AND NOT EXISTS
|
|
(SELECT 1 FROM server_deployments n WHERE n.server = d.server AND n.rowid > d.rowid))`)
|
|
.all(since) as { id: string; server: string; action: string; state: string; stage: string;
|
|
error: string | null; started_at: string }[];
|
|
for (const row of lifecycle) {
|
|
if (row.state === 'awaiting-health') representedRecovery.add(row.server);
|
|
const state = ['done', 'warning', 'cancelled'].includes(row.state) ? 'done'
|
|
: ['failed', 'interrupted'].includes(row.state) ? 'attention' : 'active';
|
|
operations.push({ key: `deployment:${row.id}`, group: `deployment:${row.id}`, kind: 'deployment',
|
|
server: row.server, state, startedAt: row.started_at,
|
|
label: state === 'done' ? row.state === 'cancelled' ? 'Removal cancelled' : row.state === 'warning' ? 'Completed with warnings' : 'Complete'
|
|
: state === 'attention' ? `${actionLabels[row.action as ProvisioningAction]} needs attention` : row.stage,
|
|
detail: row.error ?? row.stage, href: `/provisioning/${encodeURIComponent(row.id)}` });
|
|
}
|
|
const betas = handle.prepare(`SELECT b.id, b.opened_at, bs.server, bs.stage, bs.error
|
|
FROM betas b JOIN beta_servers bs ON bs.beta_id = b.id WHERE b.state IN ('running', 'ready-promote')`)
|
|
.all() as { id: number; opened_at: string; server: string; stage: string; error: string | null }[];
|
|
const stages: Record<string, string> = { queued: 'Waiting for an earlier test wave', deploying: 'Deploying beta',
|
|
'awaiting-empty': 'Waiting for an empty lobby', restarting: 'Restarting', 'awaiting-health': 'Recovering',
|
|
'awaiting-match': 'Observing a match', ready: 'Ready for review', failed: 'Beta needs attention' };
|
|
for (const row of betas) {
|
|
if (['restarting', 'awaiting-health'].includes(row.stage)) representedRecovery.add(row.server);
|
|
operations.push({ key: `beta:${row.id}:${row.server}`, group: `beta:${row.id}`,
|
|
kind: 'beta', server: row.server, startedAt: row.opened_at,
|
|
state: row.stage === 'failed' ? 'attention' : 'active', label: stages[row.stage] ?? row.stage,
|
|
detail: row.error ?? recovering.get(row.server)?.error ?? `Beta ${row.id}`, href: `/betas/${row.id}` });
|
|
}
|
|
const maintenance = handle.prepare(`SELECT m.* FROM maintenance_jobs m WHERE m.state = 'failed'
|
|
AND NOT EXISTS (SELECT 1 FROM maintenance_jobs n WHERE n.server = m.server AND n.kind = m.kind AND n.id > m.id)`)
|
|
.all() as { id: number; server: string; kind: string; detail: string | null; requested_at: string }[];
|
|
for (const job of maintenance) {
|
|
operations.push({ key: `maintenance:${job.id}`, group: `maintenance:${job.id}`, kind: 'maintenance',
|
|
server: job.server, state: 'attention', startedAt: job.requested_at,
|
|
label: job.kind === 'update' ? 'Update verification needs attention' : 'Maintenance needs attention',
|
|
detail: job.detail ?? 'Inspect the maintenance result', href: `/maintenance?view=activity&server=${encodeURIComponent(job.server)}#job-${job.id}` });
|
|
}
|
|
for (const slot of slots) {
|
|
if (representedRecovery.has(slot.server)) continue;
|
|
operations.push({ key: `recovery:${slot.server}:${slot.requested_at}`, group: `recovery:${slot.server}`,
|
|
kind: 'recovery', server: slot.server, startedAt: slot.requested_at, state: slot.error ? 'attention' : 'active',
|
|
label: 'Recovering from restart', detail: slot.error ?? 'Verifying server health',
|
|
href: `/servers/${encodeURIComponent(slot.server)}` });
|
|
}
|
|
return operations.sort((a, b) => b.startedAt.localeCompare(a.startedAt));
|
|
}
|