227 lines
9.4 KiB
TypeScript
227 lines
9.4 KiB
TypeScript
import { actionLabels, type ProvisioningAction } from '../../provisioning.js';
|
|
import { db } from '../db/index.js';
|
|
import type { FleetOperation } from '../../fleet-overview.js';
|
|
|
|
/** Fleet clients need only active work and unresolved failures. */
|
|
export function fleetOperations(now = Date.now()): FleetOperation[] {
|
|
return readOperations(now);
|
|
}
|
|
|
|
/** Recent completions are loaded only by the server's own page. */
|
|
export function serverOperations(server: string, now = Date.now()): FleetOperation[] {
|
|
return readOperations(now, server);
|
|
}
|
|
|
|
function readOperations(now: number, server?: string): FleetOperation[] {
|
|
const since = new Date(now - 86_400_000).toISOString();
|
|
const bindings = { server: server ?? null, since };
|
|
const scope = (column: string) => (server === undefined ? '' : `AND ${column} = @server`);
|
|
const handle = db();
|
|
const operations: FleetOperation[] = [];
|
|
const slots = handle
|
|
.prepare(
|
|
`SELECT server, requested_at, error FROM restart_rollout WHERE healthy_at IS NULL ${scope('server')}`
|
|
)
|
|
.all(bindings) as { server: string; requested_at: string; error: string | null }[];
|
|
const recovering = new Map(slots.map((slot) => [slot.server, slot]));
|
|
const representedRecovery = new Set<string>();
|
|
const campaigns = handle
|
|
.prepare(
|
|
`SELECT d.id, d.action, d.started_at, ds.server,
|
|
ds.restart_state, ds.error, ds.restarted_at, ds.players
|
|
FROM deployments d JOIN deployment_servers ds ON ds.deployment_id = d.id
|
|
WHERE ds.server_deleted_at IS NULL ${scope('ds.server')} AND (ds.restart_state != 'done'
|
|
${server === undefined ? '' : 'OR ds.checked_at >= @since OR d.finished_at >= @since OR d.started_at >= @since'})`
|
|
)
|
|
.all(bindings) as {
|
|
id: string;
|
|
action: string;
|
|
started_at: string;
|
|
server: string;
|
|
restart_state: string;
|
|
error: string | null;
|
|
restarted_at: string | null;
|
|
players: number | null;
|
|
}[];
|
|
for (const row of campaigns) {
|
|
const state =
|
|
row.restart_state === 'done'
|
|
? 'done'
|
|
: ['failed', 'manual'].includes(row.restart_state)
|
|
? 'attention'
|
|
: 'active';
|
|
const slot = recovering.get(row.server);
|
|
if (row.restarted_at && state !== 'done') representedRecovery.add(row.server);
|
|
const label =
|
|
state === 'done'
|
|
? 'Update complete'
|
|
: state === 'attention'
|
|
? 'Update needs attention'
|
|
: row.restart_state === 'held'
|
|
? 'Held by a beta'
|
|
: row.restarted_at
|
|
? 'Recovering'
|
|
: (row.players ?? 0) > 0
|
|
? 'Waiting for an empty lobby'
|
|
: 'Queued for restart';
|
|
operations.push({
|
|
key: `campaign:${row.id}:${row.server}`,
|
|
group: `campaign:${row.id}`,
|
|
kind: 'campaign',
|
|
server: row.server,
|
|
label,
|
|
state,
|
|
startedAt: row.started_at,
|
|
detail:
|
|
state === 'done'
|
|
? 'Update applied and restart checks completed'
|
|
: (row.error ??
|
|
(row.restarted_at
|
|
? (slot?.error ?? 'Verifying server health')
|
|
: (row.players ?? 0) > 0
|
|
? `${row.players} players at the last check`
|
|
: 'Waiting for restart eligibility and available capacity')),
|
|
href: `/campaigns/${encodeURIComponent(row.id)}`
|
|
});
|
|
}
|
|
// Only the newest lifecycle operation can still require action on a server.
|
|
// Separate branches use state/rowid indexes for fleet reads and the server
|
|
// index for detail history. Completed work is never loaded for the full fleet.
|
|
const lifecycle = handle
|
|
.prepare(
|
|
`SELECT id, server, action, state, stage, error, started_at FROM server_deployments
|
|
WHERE server_deleted_at IS NULL ${scope('server')} AND state IN ('queued', 'running', 'waiting', 'awaiting-health')
|
|
${
|
|
server === undefined
|
|
? ''
|
|
: `UNION
|
|
SELECT id, server, action, state, stage, error, started_at FROM server_deployments
|
|
WHERE server_deleted_at IS NULL AND server = @server
|
|
AND +state IN ('done', 'warning', 'cancelled', 'failed', 'interrupted') AND finished_at >= @since`
|
|
}
|
|
UNION
|
|
SELECT id, server, action, state, stage, error, started_at FROM server_deployments
|
|
WHERE server_deleted_at IS NULL ${scope('server')} AND +state IN ('failed', 'interrupted')
|
|
AND rowid IN (SELECT MAX(rowid) FROM server_deployments WHERE 1=1 ${scope('server')} GROUP BY server)`
|
|
)
|
|
.all(bindings) as {
|
|
id: string;
|
|
server: string;
|
|
action: string;
|
|
state: string;
|
|
stage: string;
|
|
error: string | null;
|
|
started_at: string;
|
|
}[];
|
|
for (const row of lifecycle) {
|
|
if (row.state === 'awaiting-health') representedRecovery.add(row.server);
|
|
const state = ['done', 'warning', 'cancelled'].includes(row.state)
|
|
? 'done'
|
|
: ['failed', 'interrupted'].includes(row.state)
|
|
? 'attention'
|
|
: 'active';
|
|
operations.push({
|
|
key: `deployment:${row.id}`,
|
|
group: `deployment:${row.id}`,
|
|
kind: 'deployment',
|
|
server: row.server,
|
|
state,
|
|
startedAt: row.started_at,
|
|
activityLabel: `${actionLabels[row.action as ProvisioningAction] ?? row.action} instance`,
|
|
resultState: row.state,
|
|
label:
|
|
state === 'done'
|
|
? row.state === 'cancelled'
|
|
? 'Removal cancelled'
|
|
: row.state === 'warning'
|
|
? 'Completed with warnings'
|
|
: 'Complete'
|
|
: state === 'attention'
|
|
? `${actionLabels[row.action as ProvisioningAction]} operation needs attention`
|
|
: row.stage,
|
|
detail: row.error ?? row.stage,
|
|
href: `/provisioning/${encodeURIComponent(row.id)}`
|
|
});
|
|
}
|
|
const betas = handle
|
|
.prepare(
|
|
`SELECT b.id, b.opened_at, bs.server, bs.stage, bs.error
|
|
FROM betas b JOIN beta_servers bs ON bs.beta_id = b.id WHERE bs.server_deleted_at IS NULL ${scope('bs.server')} AND b.state IN ('running', 'ready-promote')`
|
|
)
|
|
.all(bindings) as {
|
|
id: number;
|
|
opened_at: string;
|
|
server: string;
|
|
stage: string;
|
|
error: string | null;
|
|
}[];
|
|
const stages: Record<string, string> = {
|
|
queued: 'Waiting for an earlier test wave',
|
|
deploying: 'Deploying beta',
|
|
'awaiting-empty': 'Waiting for an empty lobby',
|
|
restarting: 'Restarting',
|
|
'awaiting-health': 'Recovering',
|
|
'awaiting-match': 'Observing a match',
|
|
ready: 'Ready for review',
|
|
failed: 'Beta needs attention'
|
|
};
|
|
for (const row of betas) {
|
|
if (['restarting', 'awaiting-health'].includes(row.stage))
|
|
representedRecovery.add(row.server);
|
|
operations.push({
|
|
key: `beta:${row.id}:${row.server}`,
|
|
group: `beta:${row.id}`,
|
|
kind: 'beta',
|
|
server: row.server,
|
|
startedAt: row.opened_at,
|
|
state: row.stage === 'failed' ? 'attention' : 'active',
|
|
label: stages[row.stage] ?? row.stage,
|
|
detail: row.error ?? recovering.get(row.server)?.error ?? `Beta ${row.id}`,
|
|
href: `/betas/${row.id}`
|
|
});
|
|
}
|
|
const maintenance = handle
|
|
.prepare(
|
|
`SELECT m.* FROM maintenance_jobs m WHERE m.server_deleted_at IS NULL ${scope('m.server')} AND m.state = 'failed'
|
|
AND NOT EXISTS (SELECT 1 FROM maintenance_jobs n WHERE n.server = m.server AND n.kind = m.kind AND n.id > m.id)`
|
|
)
|
|
.all(bindings) as {
|
|
id: number;
|
|
server: string;
|
|
kind: string;
|
|
detail: string | null;
|
|
requested_at: string;
|
|
}[];
|
|
for (const job of maintenance) {
|
|
operations.push({
|
|
key: `maintenance:${job.id}`,
|
|
group: `maintenance:${job.id}`,
|
|
kind: 'maintenance',
|
|
server: job.server,
|
|
state: 'attention',
|
|
startedAt: job.requested_at,
|
|
label:
|
|
job.kind === 'update'
|
|
? 'Update verification needs attention'
|
|
: 'Maintenance needs attention',
|
|
detail: job.detail ?? 'Inspect the maintenance result',
|
|
href: `/maintenance?view=activity&server=${encodeURIComponent(job.server)}#job-${job.id}`
|
|
});
|
|
}
|
|
for (const slot of slots) {
|
|
if (representedRecovery.has(slot.server)) continue;
|
|
operations.push({
|
|
key: `recovery:${slot.server}:${slot.requested_at}`,
|
|
group: `recovery:${slot.server}`,
|
|
kind: 'recovery',
|
|
server: slot.server,
|
|
startedAt: slot.requested_at,
|
|
state: slot.error ? 'attention' : 'active',
|
|
label: 'Recovering from restart',
|
|
detail: slot.error ?? 'Verifying server health',
|
|
href: `/servers/${encodeURIComponent(slot.server)}`
|
|
});
|
|
}
|
|
return operations.sort((a, b) => b.startedAt.localeCompare(a.startedAt));
|
|
}
|