josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Reuse heartbeat container state and batch host RCON status

Browse Source
This commit is contained in:
josh
2026-10-05 15:14:44 -04:00
parent 6586809b35
commit 239eceb779
12 changed files with 517 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/pipeline/worker.ts
+8 -6
View File
@@ -6,7 +6,7 @@ import { emit } from './bus.js';
import type { ServerStatusEntry } from './bus.js';
import type { RconFailure } from '../integrations/rcon.js';
import { getServer, listServers, setServerFields, type ServerRow } from '../fleet/servers.js';
import { inspectForMonitoring, rconForMonitoring, MonitoringChangedError } from '../fleet/health-observations.js';
import { inspectForMonitoring, rconForMonitoring, prepareMonitoringRcon, MonitoringChangedError } from '../fleet/health-observations.js';
import { restartSettling } from '../provisioning/index.js';
import { serverObservation, observationReader } from '../fleet/status.js';
import { telemetryEvidence, latestGameSubmission } from '../fleet/status-evidence.js';
@@ -96,7 +96,7 @@ function drainStatusQueue(host: string, queue: StatusQueue): void {
if (!queue.active && !queue.queued.length) statusQueues.delete(host);
}
function queueStatus(srv: ServerRow): Promise<number | null> {
function queueStatus(srv: ServerRow, probe?: () => ReturnType<typeof rconForMonitoring>): Promise<number | null> {
const server = srv.name;
const pending = pendingStatus.get(server);
if (pending) return pending;
@@ -110,7 +110,7 @@ function queueStatus(srv: ServerRow): Promise<number | null> {
queue.queued.push(async () => {
let capacity: number | null = null;
try {
capacity = await withOperationSource('Status monitor', () => refreshServerStatus(server));
capacity = await withOperationSource('Status monitor', () => refreshServerStatus(server, probe));
// Publish this server's latest evidence without waiting for the fleet.
const current = getServer(server);
if (current) publishStatus(statusEntry(current));
@@ -124,7 +124,9 @@ function queueStatus(srv: ServerRow): Promise<number | null> {
export async function fanoutStatus(): Promise<void> {
const servers = listServers();
if (!servers.length) return;
const results = await Promise.all(servers.map(queueStatus));
const probes = prepareMonitoringRcon(servers.filter(server => !pendingStatus.has(server.name)
&& !isProvisioning(server.name) && !restartSettling(server.name) && !telemetryEvidence(server.name)).map(server => server.name));
const results = await Promise.all(servers.map(server => queueStatus(server, probes.get(server.name))));
const capacities = new Map(servers.map((server, i) => [server.name, results[i]]));
try {
// Metrics remain complete fleet collections, using freshness at capture
@@ -140,7 +142,7 @@ export async function fanoutStatus(): Promise<void> {
} catch (err) { console.error('[worker] metrics error:', err); }
}
async function refreshServerStatus(server: string): Promise<number | null> {
async function refreshServerStatus(server: string, probeStatus?: () => ReturnType<typeof rconForMonitoring>): Promise<number | null> {
const srv = getServer(server);
if (!srv) return null;
let capacity: number | null = null;
@@ -175,7 +177,7 @@ async function refreshServerStatus(server: string): Promise<number | null> {
} else if (unavailable || !telemetryEvidence(srv.name)) {
flush();
const probeStartedAt = new Date().toISOString();
const probe = await rconForMonitoring(srv.name).catch(err => err instanceof MonitoringChangedError ? null
const probe = await (probeStatus ? probeStatus() : rconForMonitoring(srv.name)).catch(err => err instanceof MonitoringChangedError ? null
: { result: { ok: false as const, reason: 'send-error' as RconFailure, detail: (err as Error).message, rawReply: undefined }, observedAt: probeStartedAt });
if (!probe) return null;
const result = probe.result;