feat: unify recovery, updates, and schedules under Maintenance
This commit is contained in:
@@ -0,0 +1,242 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { db } from '../db/index.js';
|
||||
import { komodoConfigured } from '../config.js';
|
||||
import { getServer, listServers } from '../fleet/servers.js';
|
||||
import { komodoInspectHealthy } from '../integrations/komodo.js';
|
||||
import { rconStatusResult } from '../integrations/rcon.js';
|
||||
import { telemetryEvidence } from '../fleet/status-evidence.js';
|
||||
import { prepareDriftCampaign, createDriftRebuildCampaign, previewDrift } from '../fleet/drift.js';
|
||||
import { requestWatchdogLifecycle, escalateWatchdogRestart } from '../provisioning/requests.js';
|
||||
import type { WatchdogAction, WatchdogDetail, WatchdogEvidence, WatchdogIncident, WatchdogOverview, WatchdogState } from '../../watchdog.js';
|
||||
|
||||
import { maintenanceAllowed, setMaintenancePolicy } from './policies.js';
|
||||
|
||||
export const WATCHDOG_SCAN_MS = 60_000;
|
||||
export const WATCHDOG_FAILURE_MS = 120_000;
|
||||
export const WATCHDOG_RECOVERY_MS = 300_000;
|
||||
const iso = () => new Date().toISOString();
|
||||
const pendingStates = ['queued', 'waiting', 'running', 'awaiting-health'];
|
||||
const setting = (key: string) => (db().prepare('SELECT value FROM settings WHERE key = ?').get(key) as { value: string } | undefined)?.value ?? null;
|
||||
function setSetting(key: string, value: string) { db().prepare('INSERT INTO settings(key,value) VALUES (?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value').run(key, value); }
|
||||
export const watchdogEnabled = () => maintenanceAllowed('health') || maintenanceAllowed('drift');
|
||||
export function setWatchdogEnabled(enabled: boolean) {
|
||||
setSetting('watchdog_enabled', enabled ? '1' : '0');
|
||||
setMaintenancePolicy('health', enabled);
|
||||
setMaintenancePolicy('drift', enabled);
|
||||
}
|
||||
|
||||
function incident(id: string) { return db().prepare('SELECT * FROM watchdog_incidents WHERE id = ?').get(id) as WatchdogIncident | undefined; }
|
||||
function active(server: string, kind: 'health' | 'drift') { return db().prepare("SELECT * FROM watchdog_incidents WHERE server = ? AND kind = ? AND state != 'resolved'").get(server, kind) as WatchdogIncident | undefined; }
|
||||
function event(id: string, message: string) { db().prepare('INSERT INTO watchdog_events(incident_id,created_at,message) VALUES (?,?,?)').run(id, iso(), message); }
|
||||
function transition(id: string, state: WatchdogState, summary: string) {
|
||||
const previous = incident(id);
|
||||
if (!previous || (previous.state === state && previous.summary === summary)) return;
|
||||
db().prepare('UPDATE watchdog_incidents SET state=?, summary=?, finished_at=? WHERE id=?')
|
||||
.run(state, summary, state === 'resolved' || state === 'dismissed' ? iso() : null, id);
|
||||
event(id, summary);
|
||||
}
|
||||
function detect(server: string, kind: 'health' | 'drift', summary: string, details: WatchdogEvidence): WatchdogIncident {
|
||||
const previous = active(server, kind);
|
||||
if (previous) {
|
||||
// Repeated UI scans must not manufacture a sustained failure.
|
||||
const newSample = Date.now() - Date.parse(previous.last_seen_at) >= WATCHDOG_SCAN_MS;
|
||||
const actionable = (details as WatchdogEvidence & { actionable?: boolean }).actionable;
|
||||
const priorActionable = (JSON.parse(previous.details) as { actionable?: boolean }).actionable;
|
||||
const reset = kind === 'health' && actionable === true && priorActionable !== true;
|
||||
db().prepare('UPDATE watchdog_incidents SET details=?, last_seen_at=?, failures=?, first_seen_at=? WHERE id=?')
|
||||
.run(JSON.stringify(details), newSample || reset ? iso() : previous.last_seen_at,
|
||||
reset ? 1 : previous.failures + (newSample ? 1 : 0), reset ? iso() : previous.first_seen_at, previous.id);
|
||||
if (previous.state === 'open') transition(previous.id, 'open', summary);
|
||||
return incident(previous.id)!;
|
||||
}
|
||||
const id = randomUUID();
|
||||
db().prepare(`INSERT INTO watchdog_incidents(id,server,kind,state,summary,details,first_seen_at,last_seen_at)
|
||||
VALUES (?,?,?,'open',?,?,?,?)`).run(id, server, kind, summary, JSON.stringify(details), iso(), iso());
|
||||
event(id, summary);
|
||||
return incident(id)!;
|
||||
}
|
||||
function actions(id: string): WatchdogAction[] {
|
||||
return db().prepare(`SELECT a.*, d.state AS operation_state, d.stage AS operation_stage, d.error AS operation_error
|
||||
FROM watchdog_actions a LEFT JOIN server_deployments d ON d.id=a.operation_id
|
||||
WHERE a.incident_id=? ORDER BY a.rowid`).all(id) as WatchdogAction[];
|
||||
}
|
||||
export function watchdogOverview(): WatchdogOverview {
|
||||
return { enabled: watchdogEnabled(), lastScanAt: setting('watchdog_last_scan'), scanError: setting('watchdog_scan_error') || null,
|
||||
incidents: db().prepare(`SELECT i.*, (SELECT d.stage FROM watchdog_actions a
|
||||
JOIN server_deployments d ON d.id = a.operation_id WHERE a.incident_id = i.id AND a.cycle = i.cycle
|
||||
ORDER BY a.rowid DESC LIMIT 1) AS operation_stage
|
||||
FROM watchdog_incidents i WHERE state != 'resolved' OR id IN (
|
||||
SELECT id FROM watchdog_incidents WHERE state = 'resolved' ORDER BY first_seen_at DESC, rowid DESC LIMIT 100)
|
||||
ORDER BY CASE WHEN state='resolved' THEN 1 ELSE 0 END, first_seen_at DESC, rowid DESC`).all() as WatchdogIncident[] };
|
||||
}
|
||||
export function watchdogDetail(id: string): WatchdogDetail | null {
|
||||
const row = incident(id);
|
||||
return row ? { incident: row, actions: actions(id), events: db().prepare('SELECT id,created_at,message FROM watchdog_events WHERE incident_id=? ORDER BY id').all(id) as WatchdogDetail['events'] } : null;
|
||||
}
|
||||
function hasPending(server: string) {
|
||||
return !!db().prepare("SELECT 1 FROM server_deployments WHERE server=? AND state IN ('queued','waiting','running','awaiting-health') LIMIT 1").get(server);
|
||||
}
|
||||
function protectedWork(server: string): boolean {
|
||||
return !!db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id=bs.beta_id WHERE bs.server=? AND b.state IN ('running','ready-promote') LIMIT 1`).get(server)
|
||||
|| !!db().prepare(`SELECT 1 FROM deployment_servers ds WHERE ds.server=? AND ds.restart_state IN ('pending','held','failed','manual')
|
||||
AND NOT EXISTS (SELECT 1 FROM watchdog_actions a WHERE a.campaign_id=ds.deployment_id) LIMIT 1`).get(server);
|
||||
}
|
||||
function addAction(row: WatchdogIncident, operationId: string, action: 'restart' | 'rebuild', campaignId: string | null = null) {
|
||||
db().prepare('INSERT INTO watchdog_actions(id,incident_id,action,operation_id,campaign_id,created_at,cycle) VALUES (?,?,?,?,?,?,?)')
|
||||
.run(randomUUID(), row.id, action, operationId, campaignId, iso(), row.cycle);
|
||||
transition(row.id, 'recovering', action === 'restart' ? 'Restart queued; waiting for safe recovery' : 'Rebuild queued with the latest promoted setup');
|
||||
event(row.id, `${action === 'restart' ? 'Restart' : 'Rebuild'} requested: ${operationId}${campaignId ? `; campaign ${campaignId}` : ''}`);
|
||||
}
|
||||
export function updateWatchdogIncident(id: string, action: 'retry' | 'dismiss') {
|
||||
return db().transaction(() => {
|
||||
const row = incident(id);
|
||||
if (!row) return { ok: false as const, status: 404, error: 'Watchdog incident not found.' };
|
||||
if (actions(id).some(a => pendingStates.includes(a.operation_state ?? ''))) return { ok: false as const, status: 409, error: 'Provisioning is still active. Complete or cancel that operation first.' };
|
||||
if (action === 'dismiss') {
|
||||
if (!['open','attention'].includes(row.state)) return { ok: false as const, status: 409, error: 'Only detected incidents or incidents needing attention can be dismissed.' };
|
||||
transition(id, 'dismissed', 'Dismissed by operator; automatic attempts remain suppressed until the condition clears');
|
||||
} else {
|
||||
if (!['attention','dismissed'].includes(row.state)) return { ok: false as const, status: 409, error: 'Only incidents needing attention or dismissed incidents can be retried.' };
|
||||
if (!getServer(row.server)) return { ok: false as const, status: 409, error: 'The server is no longer registered; inspect its provisioning history.' };
|
||||
db().prepare('UPDATE watchdog_incidents SET cycle=cycle+1, failures=1, first_seen_at=?, last_seen_at=? WHERE id=?').run(iso(), iso(), id);
|
||||
transition(id, 'open', 'Operator requested a fresh recovery attempt; checking current conditions');
|
||||
}
|
||||
return { ok: true as const };
|
||||
})();
|
||||
}
|
||||
|
||||
async function reconcileActions(): Promise<void> {
|
||||
const rows = db().prepare("SELECT * FROM watchdog_incidents WHERE state IN ('recovering','attention')").all() as WatchdogIncident[];
|
||||
for (const row of rows) {
|
||||
const action = actions(row.id).filter(a => a.cycle === row.cycle).at(-1);
|
||||
if (!action) continue;
|
||||
const operation = db().prepare('SELECT state,submitted_at,started_at,stage,error FROM server_deployments WHERE id=?').get(action.operation_id) as
|
||||
{ state: string; submitted_at: string | null; started_at: string; stage: string; error: string | null } | undefined;
|
||||
if (!operation) { transition(row.id, 'attention', 'Provisioning history is missing; inspect the server before retrying'); continue; }
|
||||
if (['done','warning'].includes(operation.state)) {
|
||||
if (row.kind === 'drift') {
|
||||
try {
|
||||
const plan = await previewDrift([row.server]);
|
||||
if (incident(row.id)?.cycle !== row.cycle) continue;
|
||||
if (plan.files.length) { transition(row.id, 'attention', 'Rebuild completed but drift remains; inspect the campaign and server files'); continue; }
|
||||
} catch (err) { transition(row.id, 'attention', `Cannot verify rebuilt files: ${(err as Error).message}`); continue; }
|
||||
}
|
||||
transition(row.id, 'resolved', action.action === 'restart' ? 'Server recovered after restart' : 'Server recovered after rebuild');
|
||||
continue;
|
||||
}
|
||||
if (row.state === 'attention') continue; // Exhausted attempts stay latched, including across app restarts.
|
||||
if (['cancelled','interrupted'].includes(operation.state)) {
|
||||
transition(row.id, 'attention', `Provisioning ${operation.state}; inspect the recorded operation before retrying`); continue;
|
||||
}
|
||||
if (action.action === 'rebuild') {
|
||||
if (operation.state === 'failed') transition(row.id, 'attention', `Rebuild failed: ${operation.error ?? operation.stage}`);
|
||||
else if (operation.state === 'awaiting-health' && Date.now() - Date.parse(operation.submitted_at ?? operation.started_at) >= WATCHDOG_RECOVERY_MS)
|
||||
transition(row.id, 'attention', 'Rebuild has not recovered within five minutes; provisioning continues verification');
|
||||
continue;
|
||||
}
|
||||
if (!['awaiting-health','failed'].includes(operation.state)) continue;
|
||||
if (Date.now() - Date.parse(operation.submitted_at ?? operation.started_at) < WATCHDOG_RECOVERY_MS) continue;
|
||||
if (!maintenanceAllowed(row.kind)) continue;
|
||||
const current = getServer(row.server);
|
||||
if (!current || current.desired_state === 'stopped' || protectedWork(row.server)) continue;
|
||||
const result = db().transaction(() => {
|
||||
const next = escalateWatchdogRestart(row.id, action.operation_id, WATCHDOG_RECOVERY_MS);
|
||||
if (next.ok) addAction(row, next.deploymentId, 'rebuild');
|
||||
return next;
|
||||
})();
|
||||
if (!result.ok && result.status !== 409 && result.status !== 503) transition(row.id, 'attention', `Could not queue rebuild: ${result.error}`);
|
||||
}
|
||||
}
|
||||
|
||||
interface Observation { healthy: boolean; actionable: boolean; reason: string; containerState: string | null }
|
||||
async function observe(server: string): Promise<Observation> {
|
||||
const health = await komodoInspectHealthy(server);
|
||||
const state = health.state?.toLowerCase() ?? null;
|
||||
if (!health.ok) return { healthy: false, actionable: !!state && ['stopped','exited','created','dead','unhealthy','restarting','starting'].includes(state), reason: health.reason, containerState: state };
|
||||
if (telemetryEvidence(server)) return { healthy: true, actionable: false, reason: 'Komodo healthy with fresh telemetry', containerState: state };
|
||||
const rcon = await rconStatusResult(server);
|
||||
if (rcon.ok || telemetryEvidence(server)) return { healthy: true, actionable: false, reason: 'Komodo healthy with fresh game status', containerState: state };
|
||||
return { healthy: false, actionable: true, reason: `Game status unavailable: ${rcon.reason}; waiting for player-count verification before recovery`, containerState: state };
|
||||
}
|
||||
async function scanServer(server: string): Promise<void> {
|
||||
const row = getServer(server);
|
||||
if (!row || row.desired_state === 'stopped') return;
|
||||
if (hasPending(server) || protectedWork(server)) return;
|
||||
const observation = await observe(server);
|
||||
// Probes can overlap an API request; recheck ownership before queuing anything.
|
||||
if (hasPending(server) || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return;
|
||||
if (observation.healthy) {
|
||||
const current = active(server, 'health');
|
||||
if (current) transition(current.id, 'resolved', 'Fresh game status and Komodo health verified');
|
||||
} else {
|
||||
const problem = detect(server, 'health', observation.reason, observation);
|
||||
const startup = ['starting','restarting'].includes(observation.containerState ?? '');
|
||||
const oldEnough = Date.now() - Date.parse(problem.first_seen_at) >= (startup ? WATCHDOG_RECOVERY_MS : WATCHDOG_FAILURE_MS);
|
||||
if (problem.state === 'open' && observation.actionable && oldEnough && problem.failures >= 3 && maintenanceAllowed('health')) {
|
||||
const result = db().transaction(() => {
|
||||
const next = requestWatchdogLifecycle(server, 'restart', problem.id);
|
||||
if (next.ok) addAction(problem, next.deploymentId, 'restart');
|
||||
return next;
|
||||
})();
|
||||
if (!result.ok && result.status !== 409 && result.status !== 503) transition(problem.id, 'attention', result.error);
|
||||
}
|
||||
// Unknown container health is an observation failure, not proof that a rebuild is safe.
|
||||
if (!observation.actionable || hasPending(server)) return;
|
||||
}
|
||||
const prepared = await prepareDriftCampaign(server);
|
||||
if (hasPending(server) || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return;
|
||||
if (!prepared.plan.files.length) {
|
||||
const current = active(server, 'drift');
|
||||
if (current) transition(current.id, 'resolved', 'Server files match the promoted setup');
|
||||
return;
|
||||
}
|
||||
const problem = detect(server, 'drift', `${prepared.plan.files.length} file difference(s) from the promoted setup`, { files: prepared.plan.files, fingerprint: prepared.plan.fingerprint });
|
||||
if (problem.state !== 'open' || !maintenanceAllowed('drift')) return;
|
||||
const result = db().transaction(() => {
|
||||
const next = requestWatchdogLifecycle(server, 'rebuild', problem.id);
|
||||
if (next.ok) {
|
||||
const campaignId = createDriftRebuildCampaign(server, next.deploymentId, problem.id, prepared);
|
||||
addAction(problem, next.deploymentId, 'rebuild', campaignId);
|
||||
}
|
||||
return next;
|
||||
})();
|
||||
if (!result.ok && result.status !== 409 && result.status !== 503) transition(problem.id, 'attention', `Could not queue drift rebuild: ${result.error}`);
|
||||
}
|
||||
|
||||
let inFlight: Promise<void> | null = null;
|
||||
export function watchdogTick(forceScan = false): Promise<void> {
|
||||
if (inFlight) return inFlight;
|
||||
inFlight = run(forceScan).finally(() => { inFlight = null; });
|
||||
return inFlight;
|
||||
}
|
||||
async function run(forceScan: boolean): Promise<void> {
|
||||
await reconcileActions();
|
||||
const lastScan = setting('watchdog_last_scan');
|
||||
if (!forceScan && lastScan && Date.now() - Date.parse(lastScan) < WATCHDOG_SCAN_MS) return;
|
||||
setSetting('watchdog_last_scan', iso());
|
||||
if (!komodoConfigured()) {
|
||||
setSetting('watchdog_scan_error', 'Configure Komodo before automatic recovery. RCON or fresh telemetry verifies game status.'); return;
|
||||
}
|
||||
const errors: string[] = [];
|
||||
const servers = listServers();
|
||||
let cursor = 0;
|
||||
async function pump() {
|
||||
while (cursor < servers.length) {
|
||||
const server = servers[cursor++];
|
||||
try { await scanServer(server.name); }
|
||||
catch (err) { errors.push(`${server.name}: ${(err as Error).message}`); }
|
||||
}
|
||||
}
|
||||
await Promise.all(Array.from({ length: Math.min(4, servers.length) }, pump));
|
||||
// Preserve incidents for temporarily absent rebuild instances, but close retired registrations.
|
||||
const abandoned = db().prepare("SELECT * FROM watchdog_incidents WHERE state != 'resolved'").all() as WatchdogIncident[];
|
||||
for (const row of abandoned) {
|
||||
if (getServer(row.server) || hasPending(row.server)) continue;
|
||||
const last = db().prepare('SELECT action,state FROM server_deployments WHERE server=? ORDER BY rowid DESC LIMIT 1').get(row.server) as { action: string; state: string } | undefined;
|
||||
// An absent registry can also mean a failed rebuild. Only a completed
|
||||
// destruction retires that failure without requiring inspection.
|
||||
if (['open','dismissed'].includes(row.state) || (last?.action === 'destroy' && ['done','warning'].includes(last.state)))
|
||||
transition(row.id, 'resolved', 'Server is no longer registered');
|
||||
}
|
||||
setSetting('watchdog_scan_error', errors.join('\n'));
|
||||
}
|
||||
Reference in New Issue
Block a user