josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Preserve server status when batch probes miss the deadline

Browse Source
This commit is contained in:
josh
2026-10-05 16:17:29 -04:00
parent ad63b3957d
commit ae703617af
7 changed files with 105 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/pipeline/worker.ts
+11 -3
View File
@@ -6,7 +6,7 @@ import { emit } from './bus.js';
import type { ServerStatusEntry } from './bus.js';
import type { RconFailure } from '../integrations/rcon.js';
import { getServer, listServers, setServerFields, type ServerRow } from '../fleet/servers.js';
import { inspectForMonitoring, rconForMonitoring, prepareMonitoringRcon, monitoringStatusWriter, MonitoringChangedError } from '../fleet/health-observations.js';
import { inspectForMonitoring, rconForMonitoring, prepareMonitoringRcon, monitoringStatus, monitoringStatusWriter, MonitoringChangedError, MonitoringSkippedError } from '../fleet/health-observations.js';
import { restartSettling } from '../provisioning/index.js';
import { serverObservation, observationReader } from '../fleet/status.js';
import { telemetryEvidence, latestGameSubmission } from '../fleet/status-evidence.js';
@@ -149,6 +149,7 @@ async function refreshServerStatus(server: string, probeStatus?: () => ReturnTyp
let updates: Partial<Record<keyof ServerRow, string | number | null>> = {};
const flush = () => { setServerFields(srv.name, updates); updates = {}; };
if (!isProvisioning(srv.name)) {
const previousGame = monitoringStatus(srv.name)?.game;
const publishObservation = monitoringStatusWriter(srv.name);
let gameObservation: Awaited<ReturnType<typeof rconForMonitoring>> | undefined;
const settling = restartSettling(srv.name);
@@ -180,8 +181,15 @@ async function refreshServerStatus(server: string, probeStatus?: () => ReturnTyp
} else if (unavailable || !telemetryEvidence(srv.name)) {
flush();
const probeStartedAt = new Date().toISOString();
const probe = await (probeStatus ? probeStatus() : rconForMonitoring(srv.name)).catch(err => err instanceof MonitoringChangedError ? null
: { result: { ok: false as const, endpoint: null, reason: 'send-error' as RconFailure, detail: (err as Error).message, rawReply: undefined }, observedAt: probeStartedAt });
const probe = await (probeStatus ? probeStatus() : rconForMonitoring(srv.name)).catch(err => {
if (err instanceof MonitoringSkippedError) {
// The container was checked, but game evidence retains its original age.
if (unchanged()) publishObservation(inspected, previousGame);
return null;
}
if (err instanceof MonitoringChangedError) return null;
return { result: { ok: false as const, endpoint: null, reason: 'send-error' as RconFailure, detail: (err as Error).message, rawReply: undefined }, observedAt: probeStartedAt };
});
if (!probe) return null;
const result = probe.result;
if (unchanged() && (getServer(srv.name)?.last_checked_at ?? '') <= probe.observedAt) {