josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Fix transient health checks and avoid unnecessary recovery

Browse Source
This commit is contained in:
josh
2026-10-02 13:06:18 -04:00
parent 221c4c10cf
commit 65d3a99846
16 changed files with 842 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/maintenance/recovery.ts
+71 -9
View File
@@ -5,9 +5,11 @@ import { getServer, listServers } from '../fleet/servers.js';
import { hasActiveProvisioning } from '../fleet/provisioning.js';
import { komodoInspectHealthy } from '../integrations/komodo.js';
import { rconStatusResult } from '../integrations/rcon.js';
import { telemetryEvidence } from '../fleet/status-evidence.js';
import { latestGameSubmission, telemetryEvidence, telemetryReport } from '../fleet/status-evidence.js';
import { prepareDriftCampaign, createDriftRebuildCampaign, previewDrift } from '../fleet/drift.js';
import { requestWatchdogLifecycle, escalateWatchdogRestart } from '../provisioning/requests.js';
import { cancelProvisioning, requestWatchdogLifecycle, escalateWatchdogRestart } from '../provisioning/requests.js';
import { deploymentStage, finishDeployment } from '../provisioning/history.js';
import { recordProgress } from '../provisioning/progress.js';
import type { WatchdogAction, WatchdogDetail, WatchdogEvidence, WatchdogIncident, WatchdogOverview, WatchdogState } from '../../watchdog.js';
import { maintenanceAllowed, setMaintenancePolicy } from './policies.js';
@@ -43,7 +45,13 @@ function detect(server: string, kind: 'health' | 'drift', summary: string, detai
const newSample = Date.now() - Date.parse(previous.last_seen_at) >= WATCHDOG_SCAN_MS;
const actionable = (details as WatchdogEvidence & { actionable?: boolean }).actionable;
const priorActionable = (JSON.parse(previous.details) as { actionable?: boolean }).actionable;
const reset = kind === 'health' && actionable === true && priorActionable !== true;
// A successful routine/manual probe between watchdog samples breaks
// the failure streak, even if our next probe happens to fail again.
const after = latestGameSubmission(server);
const successful = [getServer(server)?.last_seen_at, telemetryReport(server)?.receivedAt]
.some(at => at && Date.parse(at) > Date.parse(previous.last_seen_at)
&& Date.parse(at) <= Date.now() && (!after || Date.parse(at) > Date.parse(after)));
const reset = kind === 'health' && (successful || (actionable === true && priorActionable !== true));
db().prepare('UPDATE watchdog_incidents SET details=?, last_seen_at=?, failures=?, first_seen_at=? WHERE id=?')
.run(JSON.stringify(details), newSample || reset ? iso() : previous.last_seen_at,
reset ? 1 : previous.failures + (newSample ? 1 : 0), reset ? iso() : previous.first_seen_at, previous.id);
@@ -56,6 +64,43 @@ function detect(server: string, kind: 'health' | 'drift', summary: string, detai
event(id, summary);
return incident(id)!;
}
/** Call only after verifying container readiness and current game evidence. */
export function resolveRecoveredHealthOperation(operationId: string, players: number, source: 'rcon' | 'telemetry' | null): boolean {
return db().transaction(() => {
const operation = db().prepare(`SELECT d.server, d.action, d.state, d.submitted_at, d.triggered_by, i.id AS incident_id
FROM server_deployments d JOIN provisioning_sources s ON s.operation_id = d.id AND s.kind = 'watchdog'
JOIN watchdog_incidents i ON i.id = s.source_id AND i.server = d.server
WHERE d.id = ? AND d.server_deleted_at IS NULL AND i.server_deleted_at IS NULL
AND i.kind = 'health' AND i.state IN ('open','recovering')
AND NOT EXISTS (SELECT 1 FROM provisioning_sources other WHERE other.operation_id = d.id AND other.kind != 'watchdog')`)
.get(operationId) as { server: string; action: string; state: string; submitted_at: string | null; triggered_by: string; incident_id: string } | undefined;
if (!operation || getServer(operation.server)?.desired_state !== 'running') return false;
const waiting = operation.state === 'waiting' && operation.submitted_at === null;
if (waiting && !operation.triggered_by.startsWith('watchdog:')) return false;
// Cancelling our unsubmitted request is safe when later queued work
// exists, and lets that work proceed. Submitted recovery must retain
// exclusive ownership before accepting evidence or releasing its slot.
if (!waiting && hasActiveProvisioning(operation.server, operationId)) return false;
if (!waiting && (operation.action !== 'restart' || !['awaiting-health','failed'].includes(operation.state))) return false;
const message = waiting ? 'Server recovered before automatic recovery was submitted' : 'Fresh container and game health verified after restart';
if (waiting && !cancelProvisioning(operationId, message)) return false;
const now = iso();
db().prepare(`UPDATE restart_rollout SET checked_at = ?, healthy_at = ?, players = ?, error = NULL
WHERE server = ? AND operation_id = ? AND healthy_at IS NULL`).run(now, now, players, operation.server, operationId);
db().prepare('UPDATE server_deployments SET checked_at = ?, players = ? WHERE id = ?').run(now, players, operationId);
recordProgress(operationId, { blocker: null, observedAt: now,
recovery: { state: 'released', at: now, operationId, message },
checks: [{ key: 'container', label: 'Container readiness', source: 'komodo', result: 'pass', checkedAt: now },
{ key: 'game', label: 'Game status', source, result: 'pass', actual: players, checkedAt: now }] });
if (operation.state === 'awaiting-health') {
deploymentStage(operationId, message);
finishDeployment(operationId, { ok: true });
}
transition(operation.incident_id, 'resolved', message);
return true;
})();
}
function actions(id: string): WatchdogAction[] {
return db().prepare(`SELECT a.*, d.state AS operation_state, d.stage AS operation_stage, d.error AS operation_error
FROM watchdog_actions a LEFT JOIN server_deployments d ON d.id=a.operation_id
@@ -157,6 +202,15 @@ async function reconcileActions(): Promise<void> {
if (!maintenanceAllowed(row.kind)) continue;
const current = getServer(row.server);
if (!current || current.desired_state === 'stopped' || protectedWork(row.server)) continue;
// A recovery deadline is a reason to check again, not evidence that
// rebuilding is necessary. The worker may not have polled this tick yet.
const observation = await observe(row.server, action.operation_id);
if (!observation || incident(row.id)?.cycle !== row.cycle || hasActiveProvisioning(row.server, action.operation_id)) continue;
if (observation.healthy) {
resolveRecoveredHealthOperation(action.operation_id, observation.players!, observation.source ?? null);
continue;
}
if (!observation.actionable) continue;
const result = db().transaction(() => {
const next = escalateWatchdogRestart(row.id, action.operation_id, WATCHDOG_RECOVERY_MS);
if (next.ok) addAction(row, next.deploymentId, 'rebuild');
@@ -166,16 +220,24 @@ async function reconcileActions(): Promise<void> {
}
}
interface Observation { healthy: boolean; actionable: boolean; reason: string; containerState: string | null }
async function observe(server: string): Promise<Observation | null> {
interface Observation { healthy: boolean; actionable: boolean; reason: string; containerState: string | null; players?: number; source?: 'rcon' | 'telemetry' }
async function observe(server: string, operationId?: string): Promise<Observation | null> {
const epoch = latestGameSubmission(server);
const suppressed = () => (operationId ? hasActiveProvisioning(server, operationId) : scanSuppressed(server))
|| protectedWork(server) || getServer(server)?.desired_state !== 'running' || epoch !== latestGameSubmission(server);
const health = await komodoInspectHealthy(server);
if (scanSuppressed(server) || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return null;
if (suppressed()) return null;
const state = health.state?.toLowerCase() ?? null;
if (!health.ok) return { healthy: false, actionable: !!state && ['stopped','exited','created','dead','unhealthy','restarting','starting'].includes(state), reason: health.reason, containerState: state };
if (telemetryEvidence(server)) return { healthy: true, actionable: false, reason: 'Komodo healthy with fresh telemetry', containerState: state };
const telemetry = telemetryEvidence(server);
if (telemetry) return { healthy: true, actionable: false, reason: 'Komodo healthy with fresh telemetry', containerState: state, players: telemetry.players, source: 'telemetry' };
const rcon = await rconStatusResult(server);
if (rcon.ok || telemetryEvidence(server)) return { healthy: true, actionable: false, reason: 'Komodo healthy with fresh game status', containerState: state };
return { healthy: false, actionable: true, reason: `Game status unavailable: ${rcon.reason}; waiting for player-count verification before recovery`, containerState: state };
if (suppressed()) return null;
const latest = telemetryEvidence(server);
if (rcon.ok || latest) return { healthy: true, actionable: false, reason: 'Komodo healthy with fresh game status', containerState: state,
players: latest?.players ?? (rcon.ok ? rcon.status.players : 0), source: latest ? 'telemetry' : 'rcon' };
return { healthy: false, actionable: rcon.reason === 'timeout' || rcon.reason === 'send-error',
reason: `Game status unavailable: ${rcon.reason}; waiting for player-count verification before recovery`, containerState: state };
}
async function scanServer(server: string): Promise<void> {
const row = getServer(server);