josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Use status monitor evidence for routine watchdog health checks

Browse Source
This commit is contained in:
josh
2026-10-05 16:13:47 -04:00
parent 91f4490d34
commit 87c5cf4b4b
8 changed files with 321 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/maintenance/recovery.ts
+31 -16
View File
@@ -4,7 +4,8 @@ import { komodoConfigured } from '../config.js';
import { nativeInstance } from '../backends/native.js';
import { getServer, listServers } from '../fleet/servers.js';
import { hasActiveProvisioning } from '../fleet/provisioning.js';
import { inspectForMonitoring, rconForMonitoring, MonitoringChangedError } from '../fleet/health-observations.js';
import { inspectForMonitoring, rconForMonitoring, monitoringStatus, monitoringIdentity, MonitoringChangedError } from '../fleet/health-observations.js';
import { ONLINE_MAX_MS } from '../../server-freshness.js';
import { latestGameSubmission, telemetryEvidence, telemetryReport } from '../fleet/status-evidence.js';
import { prepareDriftCampaign, createDriftRebuildCampaign, previewDrift } from '../fleet/drift.js';
import { cancelProvisioning, requestWatchdogLifecycle, escalateWatchdogRestart } from '../provisioning/requests.js';
@@ -41,28 +42,34 @@ function transition(id: string, state: WatchdogState, summary: string) {
event(id, summary);
}
function detect(server: string, kind: 'health' | 'drift', summary: string, details: WatchdogEvidence): WatchdogIncident {
const sampledAt = kind === 'health' ? details.observedAt ?? iso() : iso();
const sampledTime = Date.parse(sampledAt);
const previous = active(server, kind);
if (previous) {
// Repeated UI scans must not manufacture a sustained failure.
const newSample = Date.now() - Date.parse(previous.last_seen_at) >= WATCHDOG_SCAN_MS;
const sampleGap = sampledTime - Date.parse(previous.last_seen_at);
const actionable = (details as WatchdogEvidence & { actionable?: boolean }).actionable;
const priorActionable = (JSON.parse(previous.details) as { actionable?: boolean }).actionable;
const prior = JSON.parse(previous.details) as WatchdogEvidence & { actionable?: boolean };
const newSample = sampleGap > 0
&& Date.now() - Date.parse(prior.countedAt ?? previous.last_seen_at) >= WATCHDOG_SCAN_MS;
const priorActionable = prior.actionable;
// A successful routine/manual probe between watchdog samples breaks
// the failure streak, even if our next probe happens to fail again.
const after = latestGameSubmission(server);
const successful = [getServer(server)?.last_seen_at, telemetryReport(server)?.receivedAt]
.some(at => at && Date.parse(at) > Date.parse(previous.last_seen_at)
&& Date.parse(at) <= Date.now() && (!after || Date.parse(at) > Date.parse(after)));
const reset = kind === 'health' && (successful || (actionable === true && priorActionable !== true));
&& Date.parse(at) <= sampledTime && (!after || Date.parse(at) > Date.parse(after)));
const reset = kind === 'health' && (successful || (actionable === true && priorActionable !== true)
|| prior.identity !== details.identity || sampleGap > WATCHDOG_SCAN_MS + ONLINE_MAX_MS);
db().prepare('UPDATE watchdog_incidents SET details=?, last_seen_at=?, failures=?, first_seen_at=? WHERE id=?')
.run(JSON.stringify(details), newSample || reset ? iso() : previous.last_seen_at,
reset ? 1 : previous.failures + (newSample ? 1 : 0), reset ? iso() : previous.first_seen_at, previous.id);
.run(JSON.stringify({ ...details, countedAt: newSample || reset ? iso() : prior.countedAt ?? previous.last_seen_at }), newSample || reset ? sampledAt : previous.last_seen_at,
reset ? 1 : previous.failures + (newSample ? 1 : 0), reset ? sampledAt : previous.first_seen_at, previous.id);
if (previous.state === 'open') transition(previous.id, 'open', summary);
return incident(previous.id)!;
}
const id = randomUUID();
db().prepare(`INSERT INTO watchdog_incidents(id,server,kind,state,summary,details,first_seen_at,last_seen_at)
VALUES (?,?,?,'open',?,?,?,?)`).run(id, server, kind, summary, JSON.stringify(details), iso(), iso());
VALUES (?,?,?,'open',?,?,?,?)`).run(id, server, kind, summary, JSON.stringify({ ...details, countedAt: iso() }), sampledAt, sampledAt);
event(id, summary);
return incident(id)!;
}
@@ -222,23 +229,29 @@ async function reconcileActions(): Promise<void> {
}
}
interface Observation { healthy: boolean; actionable: boolean; reason: string; containerState: string | null; players?: number; source?: 'rcon' | 'telemetry' }
interface Observation { healthy: boolean; actionable: boolean; reason: string; containerState: string | null; observedAt: string; identity: string; players?: number; source?: 'rcon' | 'telemetry' }
async function observe(server: string, operationId?: string, fresh = false): Promise<Observation | null> {
const epoch = latestGameSubmission(server);
const suppressed = () => (operationId ? hasActiveProvisioning(server, operationId) : scanSuppressed(server))
|| protectedWork(server) || getServer(server)?.desired_state !== 'running' || epoch !== latestGameSubmission(server);
const inspected = await inspectForMonitoring(server, fresh || !!operationId).catch(error => {
// Routine watchdog passes only consume the status monitor. Manual checks and
// recovery verification still obtain fresh, independently verified evidence.
const snapshot = !fresh && !operationId ? monitoringStatus(server) : null;
if (!fresh && !operationId && !snapshot) return null;
const inspected = snapshot?.container ?? await inspectForMonitoring(server, true).catch(error => {
if (error instanceof MonitoringChangedError) return null;
throw error;
});
if (!inspected) return null;
const health = inspected.result;
if (suppressed()) return null;
const identity = snapshot?.identity ?? monitoringIdentity(server);
const containerEvidence = { identity, observedAt: inspected.observedAt };
const state = health.state?.toLowerCase() ?? null;
if (!health.ok) return { healthy: false, actionable: !!state && ['stopped','exited','created','dead','unhealthy','restarting','starting'].includes(state), reason: health.reason, containerState: state };
if (!health.ok) return { ...containerEvidence, healthy: false, actionable: !!state && ['stopped','exited','created','dead','unhealthy','restarting','starting'].includes(state), reason: health.reason, containerState: state };
const telemetry = telemetryEvidence(server);
if (telemetry) return { healthy: true, actionable: false, reason: 'Komodo healthy with fresh telemetry', containerState: state, players: telemetry.players, source: 'telemetry' };
const probed = await rconForMonitoring(server, fresh || !!operationId).catch(error => {
if (telemetry) return { identity, observedAt: telemetry.receivedAt, healthy: true, actionable: false, reason: 'Container healthy with fresh telemetry', containerState: state, players: telemetry.players, source: 'telemetry' };
const probed = snapshot ? snapshot.game : await rconForMonitoring(server, true).catch(error => {
if (error instanceof MonitoringChangedError) return null;
throw error;
});
@@ -246,9 +259,10 @@ async function observe(server: string, operationId?: string, fresh = false): Pro
const rcon = probed.result;
if (suppressed()) return null;
const latest = telemetryEvidence(server);
if (rcon.ok || latest) return { healthy: true, actionable: false, reason: 'Komodo healthy with fresh game status', containerState: state,
const gameEvidence = { identity: snapshot?.identity ?? monitoringIdentity(server), observedAt: latest?.receivedAt ?? probed.observedAt };
if (rcon.ok || latest) return { ...gameEvidence, healthy: true, actionable: false, reason: 'Container healthy with fresh game status', containerState: state,
players: latest?.players ?? (rcon.ok ? rcon.status.players : 0), source: latest ? 'telemetry' : 'rcon' };
return { healthy: false, actionable: rcon.reason === 'timeout' || rcon.reason === 'send-error',
return { ...gameEvidence, healthy: false, actionable: rcon.reason === 'timeout' || rcon.reason === 'send-error',
reason: `Game status unavailable: ${rcon.reason}; waiting for player-count verification before recovery`, containerState: state };
}
async function scanServer(server: string, forceScan: boolean): Promise<void> {
@@ -258,13 +272,14 @@ async function scanServer(server: string, forceScan: boolean): Promise<void> {
const observation = await observe(server, undefined, forceScan);
// Probes can overlap an API request; recheck ownership before queuing anything.
if (!observation || scanSuppressed(server) || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return;
if (observation.identity !== monitoringIdentity(server)) return;
if (!scanSuppressed(server, 'health') && observation.healthy) {
const current = active(server, 'health');
if (current) transition(current.id, 'resolved', 'Fresh game status and Komodo health verified');
} else if (!scanSuppressed(server, 'health')) {
const problem = detect(server, 'health', observation.reason, observation);
const startup = ['starting','restarting'].includes(observation.containerState ?? '');
const oldEnough = Date.now() - Date.parse(problem.first_seen_at) >= (startup ? WATCHDOG_RECOVERY_MS : WATCHDOG_FAILURE_MS);
const oldEnough = Date.parse(observation.observedAt) - Date.parse(problem.first_seen_at) >= (startup ? WATCHDOG_RECOVERY_MS : WATCHDOG_FAILURE_MS);
if (problem.state === 'open' && observation.actionable && oldEnough && problem.failures >= 3 && maintenanceAllowed('health')) {
const result = db().transaction(() => {
const next = requestWatchdogLifecycle(server, 'restart', problem.id);