josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Use status monitor evidence for routine watchdog health checks

Browse Source
This commit is contained in:
josh
2026-10-05 16:13:47 -04:00
parent 91f4490d34
commit 87c5cf4b4b
8 changed files with 321 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/pipeline/worker.ts
+8 -3
View File
@@ -6,7 +6,7 @@ import { emit } from './bus.js';
import type { ServerStatusEntry } from './bus.js';
import type { RconFailure } from '../integrations/rcon.js';
import { getServer, listServers, setServerFields, type ServerRow } from '../fleet/servers.js';
import { inspectForMonitoring, rconForMonitoring, prepareMonitoringRcon, MonitoringChangedError } from '../fleet/health-observations.js';
import { inspectForMonitoring, rconForMonitoring, prepareMonitoringRcon, monitoringStatusWriter, MonitoringChangedError } from '../fleet/health-observations.js';
import { restartSettling } from '../provisioning/index.js';
import { serverObservation, observationReader } from '../fleet/status.js';
import { telemetryEvidence, latestGameSubmission } from '../fleet/status-evidence.js';
@@ -149,6 +149,9 @@ async function refreshServerStatus(server: string, probeStatus?: () => ReturnTyp
let updates: Partial<Record<keyof ServerRow, string | number | null>> = {};
const flush = () => { setServerFields(srv.name, updates); updates = {}; };
if (!isProvisioning(srv.name)) {
const publishObservation = monitoringStatusWriter(srv.name);
let gameObservation: Awaited<ReturnType<typeof rconForMonitoring>> | undefined;
const settling = restartSettling(srv.name);
const epoch = latestGameSubmission(srv.name);
const unchanged = () => {
const current = getServer(srv.name);
@@ -157,7 +160,7 @@ async function refreshServerStatus(server: string, probeStatus?: () => ReturnTyp
&& current.native_instance_id === srv.native_instance_id && current.instance_generation === srv.instance_generation;
};
const inspectionStartedAt = new Date().toISOString();
const inspected = restartSettling(srv.name)
const inspected = settling
? { result: { ok: false as const, state: 'restarting', reason: 'Restart requested' }, observedAt: inspectionStartedAt }
: await inspectForMonitoring(srv.name).catch(err => err instanceof MonitoringChangedError ? null
: { result: { ok: false as const, state: 'unknown', reason: (err as Error).message }, observedAt: inspectionStartedAt });
@@ -178,10 +181,11 @@ async function refreshServerStatus(server: string, probeStatus?: () => ReturnTyp
flush();
const probeStartedAt = new Date().toISOString();
const probe = await (probeStatus ? probeStatus() : rconForMonitoring(srv.name)).catch(err => err instanceof MonitoringChangedError ? null
: { result: { ok: false as const, reason: 'send-error' as RconFailure, detail: (err as Error).message, rawReply: undefined }, observedAt: probeStartedAt });
: { result: { ok: false as const, endpoint: null, reason: 'send-error' as RconFailure, detail: (err as Error).message, rawReply: undefined }, observedAt: probeStartedAt });
if (!probe) return null;
const result = probe.result;
if (unchanged() && (getServer(srv.name)?.last_checked_at ?? '') <= probe.observedAt) {
gameObservation = probe;
const now = probe.observedAt;
updates.last_checked_at = now;
updates.last_reply_snippet = result.rawReply ?? null;
@@ -196,6 +200,7 @@ async function refreshServerStatus(server: string, probeStatus?: () => ReturnTyp
} else updates.last_error = encodeError(result.reason, result.detail);
}
}
if (!settling && unchanged()) publishObservation(inspected, gameObservation);
}
} else {
updates.last_error = null;