josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Share recent health observations between background monitors

Browse Source
This commit is contained in:
josh
2026-10-05 13:00:53 -04:00
parent f16879036e
commit 2d069907f2
5 changed files with 452 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/pipeline/worker.ts
+23 -13
View File
@@ -4,10 +4,9 @@ import { sweep } from './sweep.js';
import { betaTick } from './beta-runner.js';
import { emit } from './bus.js';
import type { ServerStatusEntry } from './bus.js';
import { rconStatusResult } from '../integrations/rcon.js';
import type { RconFailure } from '../integrations/rcon.js';
import { getServer, listServers, setServerFields, type ServerRow } from '../fleet/servers.js';
import { komodoInspectHealthy } from '../integrations/komodo.js';
import { inspectForMonitoring, rconForMonitoring, MonitoringChangedError } from '../fleet/health-observations.js';
import { restartSettling } from '../provisioning/index.js';
import { serverObservation, observationReader } from '../fleet/status.js';
import { telemetryEvidence, latestGameSubmission } from '../fleet/status-evidence.js';
@@ -143,15 +142,23 @@ async function refreshServerStatus(server: string): Promise<number | null> {
const flush = () => { setServerFields(srv.name, updates); updates = {}; };
if (!isProvisioning(srv.name)) {
const epoch = latestGameSubmission(srv.name);
const unchanged = () => epoch === latestGameSubmission(srv.name) && !isProvisioning(srv.name)
&& getServer(srv.name)?.created_at === srv.created_at;
const container = restartSettling(srv.name)
? { ok: false as const, state: 'restarting', reason: 'Restart requested' }
: await komodoInspectHealthy(srv.name).catch(err => ({ ok: false as const, state: 'unknown', reason: (err as Error).message }));
const unchanged = () => {
const current = getServer(srv.name);
return epoch === latestGameSubmission(srv.name) && !isProvisioning(srv.name)
&& current?.created_at === srv.created_at && current.desired_state === srv.desired_state
&& current.native_instance_id === srv.native_instance_id && current.instance_generation === srv.instance_generation;
};
const inspectionStartedAt = new Date().toISOString();
const inspected = restartSettling(srv.name)
? { result: { ok: false as const, state: 'restarting', reason: 'Restart requested' }, observedAt: inspectionStartedAt }
: await inspectForMonitoring(srv.name).catch(err => err instanceof MonitoringChangedError ? null
: { result: { ok: false as const, state: 'unknown', reason: (err as Error).message }, observedAt: inspectionStartedAt });
if (!inspected) return null;
const container = inspected.result;
// A lifecycle request may be submitted while an inspection is in flight.
if (unchanged()) {
if (unchanged() && (getServer(srv.name)?.container_checked_at ?? '') <= inspected.observedAt) {
updates.container_state = container.ok ? 'running' : container.state?.toLowerCase() ?? 'unknown';
updates.container_checked_at = new Date().toISOString();
updates.container_checked_at = inspected.observedAt;
// An unavailable inspection is not evidence that the game stopped.
// Probe it just as a manual RCON check would; explicit lifecycle or
// unhealthy states still take precedence over game replies.
@@ -161,10 +168,13 @@ async function refreshServerStatus(server: string): Promise<number | null> {
updates.last_reply_snippet = null;
} else if (unavailable || !telemetryEvidence(srv.name)) {
flush();
const result = await rconStatusResult(srv.name).catch(err => ({ ok: false as const,
reason: 'send-error' as RconFailure, detail: (err as Error).message, rawReply: undefined }));
if (unchanged()) {
const now = new Date().toISOString();
const probeStartedAt = new Date().toISOString();
const probe = await rconForMonitoring(srv.name).catch(err => err instanceof MonitoringChangedError ? null
: { result: { ok: false as const, reason: 'send-error' as RconFailure, detail: (err as Error).message, rawReply: undefined }, observedAt: probeStartedAt });
if (!probe) return null;
const result = probe.result;
if (unchanged() && (getServer(srv.name)?.last_checked_at ?? '') <= probe.observedAt) {
const now = probe.observedAt;
updates.last_checked_at = now;
updates.last_reply_snippet = result.rawReply ?? null;
if (result.ok) {