josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Scope server snapshots and keep completed history off fleet refreshes

Browse Source
This commit is contained in:
josh
2026-10-05 15:37:56 -04:00
parent 030794635e
commit d24c52b5cc
12 changed files with 376 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/fleet/overview.ts
+30 -18
View File
@@ -2,19 +2,31 @@ import { actionLabels, type ProvisioningAction } from '../../provisioning.js';
import { db } from '../db/index.js';
import type { FleetOperation } from '../../fleet-overview.js';
/** Read persisted work, including recent completions for one-shot lobby watches. */
/** Fleet clients need only active work and unresolved failures. */
export function fleetOperations(now = Date.now()): FleetOperation[] {
return readOperations(now);
}
/** Recent completions are loaded only by the server's own page. */
export function serverOperations(server: string, now = Date.now()): FleetOperation[] {
return readOperations(now, server);
}
function readOperations(now: number, server?: string): FleetOperation[] {
const since = new Date(now - 86_400_000).toISOString();
const bindings = { server: server ?? null, since };
const scope = (column: string) => server === undefined ? '' : `AND ${column} = @server`;
const handle = db();
const operations: FleetOperation[] = [];
const slots = handle.prepare('SELECT server, requested_at, error FROM restart_rollout WHERE healthy_at IS NULL')
.all() as { server: string; requested_at: string; error: string | null }[];
const slots = handle.prepare(`SELECT server, requested_at, error FROM restart_rollout WHERE healthy_at IS NULL ${scope('server')}`)
.all(bindings) as { server: string; requested_at: string; error: string | null }[];
const recovering = new Map(slots.map(slot => [slot.server, slot]));
const representedRecovery = new Set<string>();
const campaigns = handle.prepare(`SELECT d.id, d.action, d.started_at, ds.server,
ds.restart_state, ds.error, ds.restarted_at, ds.players
FROM deployments d JOIN deployment_servers ds ON ds.deployment_id = d.id
WHERE ds.server_deleted_at IS NULL AND (ds.restart_state != 'done' OR ds.checked_at >= ? OR d.finished_at >= ? OR d.started_at >= ?)`).all(since, since, since) as {
WHERE ds.server_deleted_at IS NULL ${scope('ds.server')} AND (ds.restart_state != 'done'
${server === undefined ? '' : 'OR ds.checked_at >= @since OR d.finished_at >= @since OR d.started_at >= @since'})`).all(bindings) as {
id: string; action: string; started_at: string; server: string; restart_state: string;
error: string | null; restarted_at: string | null; players: number | null;
}[];
@@ -34,19 +46,19 @@ export function fleetOperations(now = Date.now()): FleetOperation[] {
href: `/campaigns/${encodeURIComponent(row.id)}` });
}
// Only the newest lifecycle operation can still require action on a server.
// Separate branches let SQLite use state and finish indexes; UNION deduplicates
// recent warnings that are also the newest operation. Unary + keeps terminal
// branches on finished_at and rowid instead of searching historical states.
// Separate branches use state/rowid indexes for fleet reads and the server
// index for detail history. Completed work is never loaded for the full fleet.
const lifecycle = handle.prepare(`SELECT id, server, action, state, stage, error, started_at FROM server_deployments
WHERE server_deleted_at IS NULL AND state IN ('queued', 'running', 'waiting', 'awaiting-health')
WHERE server_deleted_at IS NULL ${scope('server')} AND state IN ('queued', 'running', 'waiting', 'awaiting-health')
${server === undefined ? '' : `UNION
SELECT id, server, action, state, stage, error, started_at FROM server_deployments
WHERE server_deleted_at IS NULL AND server = @server
AND +state IN ('done', 'warning', 'cancelled', 'failed', 'interrupted') AND finished_at >= @since`}
UNION
SELECT id, server, action, state, stage, error, started_at FROM server_deployments
WHERE server_deleted_at IS NULL AND +state IN ('done', 'warning', 'cancelled') AND finished_at >= ?
UNION
SELECT id, server, action, state, stage, error, started_at FROM server_deployments
WHERE server_deleted_at IS NULL AND +state IN ('failed', 'interrupted', 'warning')
AND rowid IN (SELECT MAX(rowid) FROM server_deployments GROUP BY server)`)
.all(since) as { id: string; server: string; action: string; state: string; stage: string;
WHERE server_deleted_at IS NULL ${scope('server')} AND +state IN ('failed', 'interrupted')
AND rowid IN (SELECT MAX(rowid) FROM server_deployments WHERE 1=1 ${scope('server')} GROUP BY server)`)
.all(bindings) as { id: string; server: string; action: string; state: string; stage: string;
error: string | null; started_at: string }[];
for (const row of lifecycle) {
if (row.state === 'awaiting-health') representedRecovery.add(row.server);
@@ -59,8 +71,8 @@ export function fleetOperations(now = Date.now()): FleetOperation[] {
detail: row.error ?? row.stage, href: `/provisioning/${encodeURIComponent(row.id)}` });
}
const betas = handle.prepare(`SELECT b.id, b.opened_at, bs.server, bs.stage, bs.error
FROM betas b JOIN beta_servers bs ON bs.beta_id = b.id WHERE bs.server_deleted_at IS NULL AND b.state IN ('running', 'ready-promote')`)
.all() as { id: number; opened_at: string; server: string; stage: string; error: string | null }[];
FROM betas b JOIN beta_servers bs ON bs.beta_id = b.id WHERE bs.server_deleted_at IS NULL ${scope('bs.server')} AND b.state IN ('running', 'ready-promote')`)
.all(bindings) as { id: number; opened_at: string; server: string; stage: string; error: string | null }[];
const stages: Record<string, string> = { queued: 'Waiting for an earlier test wave', deploying: 'Deploying beta',
'awaiting-empty': 'Waiting for an empty lobby', restarting: 'Restarting', 'awaiting-health': 'Recovering',
'awaiting-match': 'Observing a match', ready: 'Ready for review', failed: 'Beta needs attention' };
@@ -71,9 +83,9 @@ export function fleetOperations(now = Date.now()): FleetOperation[] {
state: row.stage === 'failed' ? 'attention' : 'active', label: stages[row.stage] ?? row.stage,
detail: row.error ?? recovering.get(row.server)?.error ?? `Beta ${row.id}`, href: `/betas/${row.id}` });
}
const maintenance = handle.prepare(`SELECT m.* FROM maintenance_jobs m WHERE m.server_deleted_at IS NULL AND m.state = 'failed'
const maintenance = handle.prepare(`SELECT m.* FROM maintenance_jobs m WHERE m.server_deleted_at IS NULL ${scope('m.server')} AND m.state = 'failed'
AND NOT EXISTS (SELECT 1 FROM maintenance_jobs n WHERE n.server = m.server AND n.kind = m.kind AND n.id > m.id)`)
.all() as { id: number; server: string; kind: string; detail: string | null; requested_at: string }[];
.all(bindings) as { id: number; server: string; kind: string; detail: string | null; requested_at: string }[];
for (const job of maintenance) {
operations.push({ key: `maintenance:${job.id}`, group: `maintenance:${job.id}`, kind: 'maintenance',
server: job.server, state: 'attention', startedAt: job.requested_at,
src/lib/server/fleet/snapshot.ts
+18 -9
View File
@@ -1,26 +1,32 @@
import { db } from '../db/index.js';
import { listServers } from './servers.js';
import { observationReader } from './status.js';
import { getServer, listServers } from './servers.js';
import { observationReader, serverObservation } from './status.js';
import { config, komodoConfigured, rconConfigured } from '../config.js';
import { runningVersions } from '../integrations/plutonium.js';
import { listHosts } from '../hosts/index.js';
import type { NativeInstance } from '../backends/native.js';
import { serverOperations } from './overview.js';
export function fleetSnapshot() {
export function fleetSnapshot(serverName?: string) {
const bindings = serverName === undefined ? [] : [serverName];
const rconHost = config.rcon.host;
const hosts = listHosts();
const hostsById = new Map(hosts.map(({ inventory: _inventory, ...host }) => [host.id, host]));
const versions = new Map(runningVersions().map(version => [version.server, version]));
const instances = new Map((db().prepare('SELECT * FROM native_instances WHERE retired_at IS NULL').all() as NativeInstance[]).map(instance => [instance.server, instance]));
const versions = new Map(runningVersions(serverName).map(version => [version.server, version]));
const instances = new Map((db().prepare(`SELECT * FROM native_instances WHERE retired_at IS NULL ${serverName === undefined ? '' : 'AND server = ?'}`)
.all(...bindings) as NativeInstance[]).map(instance => [instance.server, instance]));
const holds = new Map<string, { target: string; betaId: number }[]>();
for (const row of db().prepare(`SELECT bsv.server, s.target_rel_path AS target, b.id AS betaId
FROM beta_servers bsv JOIN betas b ON b.id=bsv.beta_id JOIN beta_scripts bs ON bs.beta_id=b.id
JOIN scripts s ON s.id=bs.script_id WHERE bsv.server_deleted_at IS NULL AND b.state IN ('running','ready-promote')`).all() as
JOIN scripts s ON s.id=bs.script_id WHERE bsv.server_deleted_at IS NULL AND b.state IN ('running','ready-promote')
${serverName === undefined ? '' : 'AND bsv.server = ?'}`).all(...bindings) as
{ server: string; target: string; betaId: number }[]) {
const entries = holds.get(row.server) ?? []; entries.push({ target: row.target, betaId: row.betaId }); holds.set(row.server, entries);
}
const observe = observationReader();
const servers = listServers().map((s) => {
const observe = serverName === undefined ? observationReader() : serverObservation;
const selected = serverName === undefined ? undefined : getServer(serverName);
const rows = serverName === undefined ? listServers() : selected && !selected.destroyed_at ? [selected] : [];
const servers = rows.map((s) => {
const instance = s.native_instance_id ? instances.get(s.name) : null;
return {
...observe(s),
@@ -36,7 +42,7 @@ export function fleetSnapshot() {
return {
servers,
serverNow: Date.now(),
reservedNames: (db().prepare("SELECT j.server FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')").all() as { server: string }[]).map(row => row.server),
reservedNames: serverName === undefined ? (db().prepare("SELECT j.server FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')").all() as { server: string }[]).map(row => row.server) : [],
integrations: {
native: hosts.length > 0 || !komodoConfigured(),
hostsReady: hosts.filter(host => host.readiness === 'ready').length,
@@ -49,3 +55,6 @@ export function fleetSnapshot() {
};
}
export function serverSnapshot(name: string) {
return { ...fleetSnapshot(name), operations: serverOperations(name) };
}
src/lib/server/integrations/plutonium.ts
+10 -5
View File
@@ -20,13 +20,18 @@ export function freshRelease(): number | null {
const age = latest.checkedAt ? Date.now() - Date.parse(latest.checkedAt) : NaN;
return !latest.error && age >= 0 && age < HOUR ? latest.revision : null;
}
export function runningVersions(): PlutoniumVersion[] {
const versions = db().prepare('SELECT * FROM plutonium_versions').all() as PlutoniumVersion[];
export function runningVersions(serverName?: string): PlutoniumVersion[] {
const bindings = serverName === undefined ? [] : [serverName];
const versions = db().prepare(`SELECT * FROM plutonium_versions ${serverName === undefined ? '' : 'WHERE server = ?'}`)
.all(...bindings) as PlutoniumVersion[];
const target = freshRelease();
const byServer = new Map(versions.map(version => [version.server, version]));
const epochs = latestGameSubmissions();
const recovering = new Set((db().prepare('SELECT server FROM restart_rollout WHERE healthy_at IS NULL').all() as { server: string }[]).map(row => row.server));
return listServers().map(server => {
const epochs = serverName === undefined ? latestGameSubmissions() : new Map([[serverName, latestGameSubmission(serverName)]]);
const recovering = new Set((db().prepare(`SELECT server FROM restart_rollout WHERE healthy_at IS NULL ${serverName === undefined ? '' : 'AND server = ?'}`)
.all(...bindings) as { server: string }[]).map(row => row.server));
const selected = serverName === undefined ? undefined : getServer(serverName);
const servers = serverName === undefined ? listServers() : selected && !selected.destroyed_at ? [selected] : [];
return servers.map(server => {
const value = byServer.get(server.name)
?? { server: server.name, revision: null, checked_at: null, error: null };
const age = value.checked_at ? Date.now() - Date.parse(value.checked_at) : NaN;