josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Share status monitoring snapshots across each fleet pass

Browse Source
This commit is contained in:
josh
2026-10-06 14:47:14 -04:00
parent 562b770725
commit daefb22be6
4 changed files with 195 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/pipeline/worker.ts
+45 -35
View File
@@ -6,18 +6,17 @@ import { betaTick } from './beta-runner.js';
import { emit } from './bus.js';
import type { ServerStatusEntry } from './bus.js';
import type { RconFailure } from '../integrations/rcon.js';
import { getServer, listServers, setServerFields, type ServerRow } from '../fleet/servers.js';
import { inspectForMonitoring, rconForMonitoring, prepareMonitoringRcon, monitoringStatus, monitoringStatusWriter, MonitoringChangedError, MonitoringSkippedError } from '../fleet/health-observations.js';
import { restartSettling } from '../provisioning/index.js';
import { serverObservation, observationReader } from '../fleet/status.js';
import { telemetryEvidence, latestGameSubmission } from '../fleet/status-evidence.js';
import { isProvisioning } from '../fleet/provisioning.js';
import { setServerFields, type ServerRow } from '../fleet/servers.js';
import { inspectForMonitoring, rconForMonitoring, prepareMonitoringRcon, monitoringStatus, monitoringStatusWriter, monitoringIdentity, MonitoringChangedError, MonitoringSkippedError } from '../fleet/health-observations.js';
import { serverObservation } from '../fleet/status.js';
import { STATUS_REFRESH_MS } from '../../server-freshness.js';
import { recordMetrics } from '../fleet/metrics.js';
import { selectedZombieMap } from '../../map-aliases.js';
import { runMaintenance } from '../maintenance/index.js';
import { cleanupControllerLibraries } from '../game-files/index.js';
import { trackActivity } from './activity.js';
import { MonitoringSnapshot } from '../fleet/monitoring-snapshot.js';
import { withHostRequestContext } from '../hosts/request-context.js';
import { withOperationSource } from '../hosts/operation-context.js';
const TICK_MS = 15_000;
@@ -104,7 +103,7 @@ function drainStatusQueue(host: string, queue: StatusQueue): void {
if (!queue.active && !queue.queued.length) statusQueues.delete(host);
}
function queueStatus(srv: ServerRow, probe?: () => ReturnType<typeof rconForMonitoring>): Promise<number | null> {
function queueStatus(srv: ServerRow, snapshot: MonitoringSnapshot, probe?: () => ReturnType<typeof rconForMonitoring>): Promise<number | null> {
const server = srv.name;
const pending = pendingStatus.get(server);
if (pending) return pending;
@@ -118,10 +117,10 @@ function queueStatus(srv: ServerRow, probe?: () => ReturnType<typeof rconForMoni
queue.queued.push(async () => {
let capacity: number | null = null;
try {
capacity = await withOperationSource('Status monitor', () => refreshServerStatus(server, probe));
capacity = await withOperationSource('Status monitor', () => refreshServerStatus(server, snapshot, probe));
// Publish this server's latest evidence without waiting for the fleet.
const current = getServer(server);
if (current) publishStatus(statusEntry(current));
const current = snapshot.get(server)?.server;
if (current) publishStatus(statusEntry(current, snapshot.observe));
} catch (err) { console.error(`[worker] status error for ${server}:`, err); }
finally { pendingStatus.delete(server); finish(capacity); }
});
@@ -129,19 +128,24 @@ function queueStatus(srv: ServerRow, probe?: () => ReturnType<typeof rconForMoni
return work;
}
export async function fanoutStatus(): Promise<void> {
const servers = listServers();
export function fanoutStatus(): Promise<void> {
return withHostRequestContext(statusPass);
}
async function statusPass(): Promise<void> {
const snapshot = new MonitoringSnapshot();
const servers = snapshot.servers();
if (!servers.length) return;
const probes = prepareMonitoringRcon(servers.filter(server => !pendingStatus.has(server.name)
&& !isProvisioning(server.name) && !restartSettling(server.name) && !telemetryEvidence(server.name)).map(server => server.name));
const results = await Promise.all(servers.map(server => queueStatus(server, probes.get(server.name))));
&& !snapshot.provisioning(server.name) && !snapshot.settling(server.name) && !snapshot.telemetryEvidence(server.name)).map(server => server.name), snapshot);
const results = await Promise.all(servers.map(server => queueStatus(server, snapshot, probes.get(server.name))));
const capacities = new Map(servers.map((server, i) => [server.name, results[i]]));
try {
// Metrics remain complete fleet collections, using freshness at capture
// time rather than the status saved at the beginning of a long scan.
const observe = observationReader();
recordMetrics(listServers().map(srv => {
const current = observe(srv);
snapshot.refresh();
recordMetrics(snapshot.servers().map(srv => {
const current = snapshot.observe(srv);
return { server: srv.name, game: srv.game ?? 'unknown',
map: selectedZombieMap(srv.game, current.last_map ?? srv.map, srv.map_slug)?.friendly ?? current.last_map ?? srv.map ?? 'Unknown map',
players: current.health === 'online' ? current.last_players : null,
@@ -150,33 +154,38 @@ export async function fanoutStatus(): Promise<void> {
} catch (err) { console.error('[worker] metrics error:', err); }
}
async function refreshServerStatus(server: string, probeStatus?: () => ReturnType<typeof rconForMonitoring>): Promise<number | null> {
const srv = getServer(server);
async function refreshServerStatus(server: string, snapshot: MonitoringSnapshot, probeStatus?: () => ReturnType<typeof rconForMonitoring>): Promise<number | null> {
snapshot.refresh(server);
const srv = snapshot.get(server)?.server;
if (!srv) return null;
let capacity: number | null = null;
let updates: Partial<Record<keyof ServerRow, string | number | null>> = {};
const flush = () => { setServerFields(srv.name, updates); updates = {}; };
if (!isProvisioning(srv.name)) {
const previousGame = monitoringStatus(srv.name)?.game;
const publishObservation = monitoringStatusWriter(srv.name);
const flush = () => { setServerFields(srv.name, updates); snapshot.patch(srv.name, updates as Partial<ServerRow>); updates = {}; };
if (!snapshot.provisioning(srv.name)) {
const previousGame = monitoringStatus(srv.name, snapshot)?.game;
const publishObservation = monitoringStatusWriter(srv.name, snapshot);
let gameObservation: Awaited<ReturnType<typeof rconForMonitoring>> | undefined;
const settling = restartSettling(srv.name);
const epoch = latestGameSubmission(srv.name);
const settling = snapshot.settling(srv.name);
const epoch = snapshot.get(srv.name)?.submission;
let identity: string | undefined;
try { identity = monitoringIdentity(srv.name, snapshot); } catch { /* Offline hosts still need an unavailable sample. */ }
const unchanged = () => {
const current = getServer(srv.name);
return epoch === latestGameSubmission(srv.name) && !isProvisioning(srv.name)
const current = snapshot.get(srv.name)?.server;
if (identity) try { if (identity !== monitoringIdentity(srv.name, snapshot)) return false; } catch { return false; }
return epoch === snapshot.get(srv.name)?.submission && !snapshot.provisioning(srv.name)
&& current?.created_at === srv.created_at && current.desired_state === srv.desired_state
&& current.native_instance_id === srv.native_instance_id && current.instance_generation === srv.instance_generation;
};
const inspectionStartedAt = new Date().toISOString();
const inspected = settling
? { result: { ok: false as const, state: 'restarting', reason: 'Restart requested' }, observedAt: inspectionStartedAt }
: await inspectForMonitoring(srv.name).catch(err => err instanceof MonitoringChangedError ? null
: await inspectForMonitoring(srv.name, false, snapshot).catch(err => err instanceof MonitoringChangedError ? null
: { result: { ok: false as const, state: 'unknown', reason: (err as Error).message }, observedAt: inspectionStartedAt });
snapshot.refresh(srv.name);
if (!inspected) return null;
const container = inspected.result;
// A lifecycle request may be submitted while an inspection is in flight.
if (unchanged() && (getServer(srv.name)?.container_checked_at ?? '') <= inspected.observedAt) {
if (unchanged() && (snapshot.get(srv.name)?.server?.container_checked_at ?? '') <= inspected.observedAt) {
updates.container_state = container.ok ? 'running' : container.state?.toLowerCase() ?? 'unknown';
updates.container_checked_at = inspected.observedAt;
// An unavailable inspection is not evidence that the game stopped.
@@ -187,10 +196,11 @@ async function refreshServerStatus(server: string, probeStatus?: () => ReturnTyp
updates.last_error = null;
updates.rcon_failures = 0;
updates.last_reply_snippet = null;
} else if (unavailable || !telemetryEvidence(srv.name)) {
} else if (unavailable || !snapshot.telemetryEvidence(srv.name)) {
flush();
const probeStartedAt = new Date().toISOString();
const probe = await (probeStatus ? probeStatus() : rconForMonitoring(srv.name)).catch(err => {
const probe = await (probeStatus ? probeStatus() : rconForMonitoring(srv.name, false, snapshot)).catch(err => {
snapshot.refresh(srv.name);
if (err instanceof MonitoringSkippedError) {
// The container was checked, but game evidence retains its original age.
if (unchanged()) publishObservation(inspected, previousGame);
@@ -199,9 +209,10 @@ async function refreshServerStatus(server: string, probeStatus?: () => ReturnTyp
if (err instanceof MonitoringChangedError) return null;
return { result: { ok: false as const, endpoint: null, reason: 'send-error' as RconFailure, detail: (err as Error).message, rawReply: undefined }, observedAt: probeStartedAt };
});
snapshot.refresh(srv.name);
if (!probe) return null;
const result = probe.result;
const previous = getServer(srv.name);
const previous = snapshot.get(srv.name)?.server;
if (unchanged() && previous && (previous.last_checked_at ?? '') <= probe.observedAt) {
gameObservation = probe;
const now = probe.observedAt;
@@ -234,9 +245,8 @@ async function refreshServerStatus(server: string, probeStatus?: () => ReturnTyp
updates.rcon_failures = 0;
updates.last_reply_snippet = null;
}
flush();
const current = getServer(srv.name) ?? srv;
const status = serverObservation(current);
const current = { ...(snapshot.get(srv.name)?.server ?? srv), ...updates } as ServerRow;
const status = snapshot.observe(current);
// Occupancy timers and match clearing follow the same source as the UI.
if (status.health === 'online' && status.last_players !== null) {
if (status.last_players > 0) {