josh
e5c0fda1ed
Introduce an outbound Python host agent, Debian installer, durable remote operations, immutable game-file libraries, host-aware deployments and ports, and occupancy-safe campaigns and draining. Native hosts do not require Komodo. Add administrator, operator and viewer accounts, protected management routes, secure sessions, password recovery, audit history and separate telemetry tokens. Document first-admin setup, reporter migration and live-host acceptance checks. Validation: 1,343 TypeScript tests and 39 agent tests passed, along with type checking, the production build, compiled SSR/API smoke tests and administrator recovery. Real Debian/Docker T5/T6 acceptance remains unrun.
327 lines
24 KiB
TypeScript
327 lines
24 KiB
TypeScript
import { randomUUID } from 'node:crypto';
|
|
import { db } from '../db/index.js';
|
|
import { komodoConfigured } from '../config.js';
|
|
import { nativeInstance } from '../backends/native.js';
|
|
import { getServer, listServers } from '../fleet/servers.js';
|
|
import { hasActiveProvisioning } from '../fleet/provisioning.js';
|
|
import { komodoInspectHealthy } from '../integrations/komodo.js';
|
|
import { rconStatusResult } from '../integrations/rcon.js';
|
|
import { latestGameSubmission, telemetryEvidence, telemetryReport } from '../fleet/status-evidence.js';
|
|
import { prepareDriftCampaign, createDriftRebuildCampaign, previewDrift } from '../fleet/drift.js';
|
|
import { cancelProvisioning, requestWatchdogLifecycle, escalateWatchdogRestart } from '../provisioning/requests.js';
|
|
import { deploymentStage, finishDeployment } from '../provisioning/history.js';
|
|
import { recordProgress } from '../provisioning/progress.js';
|
|
import type { WatchdogAction, WatchdogDetail, WatchdogEvidence, WatchdogIncident, WatchdogOverview, WatchdogState } from '../../watchdog.js';
|
|
|
|
import { maintenanceAllowed, setMaintenancePolicy } from './policies.js';
|
|
|
|
export const WATCHDOG_SCAN_MS = 60_000;
|
|
export const WATCHDOG_FAILURE_MS = 120_000;
|
|
export const WATCHDOG_RECOVERY_MS = 300_000;
|
|
const iso = () => new Date().toISOString();
|
|
const pendingStates = ['queued', 'waiting', 'running', 'awaiting-health'];
|
|
const setting = (key: string) => (db().prepare('SELECT value FROM settings WHERE key = ?').get(key) as { value: string } | undefined)?.value ?? null;
|
|
function setSetting(key: string, value: string) { db().prepare('INSERT INTO settings(key,value) VALUES (?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value').run(key, value); }
|
|
export const watchdogEnabled = () => maintenanceAllowed('health') || maintenanceAllowed('drift');
|
|
export function setWatchdogEnabled(enabled: boolean) {
|
|
setSetting('watchdog_enabled', enabled ? '1' : '0');
|
|
setMaintenancePolicy('health', enabled);
|
|
setMaintenancePolicy('drift', enabled);
|
|
}
|
|
|
|
function incident(id: string) { return db().prepare('SELECT * FROM watchdog_incidents WHERE id = ?').get(id) as WatchdogIncident | undefined; }
|
|
function active(server: string, kind: 'health' | 'drift') { return db().prepare("SELECT * FROM watchdog_incidents WHERE server = ? AND server_deleted_at IS NULL AND kind = ? AND state != 'resolved'").get(server, kind) as WatchdogIncident | undefined; }
|
|
function event(id: string, message: string) { db().prepare('INSERT INTO watchdog_events(incident_id,created_at,message) VALUES (?,?,?)').run(id, iso(), message); }
|
|
function transition(id: string, state: WatchdogState, summary: string) {
|
|
const previous = incident(id);
|
|
if (!previous || (previous.state === state && previous.summary === summary)) return;
|
|
db().prepare('UPDATE watchdog_incidents SET state=?, summary=?, finished_at=? WHERE id=?')
|
|
.run(state, summary, state === 'resolved' || state === 'dismissed' ? iso() : null, id);
|
|
event(id, summary);
|
|
}
|
|
function detect(server: string, kind: 'health' | 'drift', summary: string, details: WatchdogEvidence): WatchdogIncident {
|
|
const previous = active(server, kind);
|
|
if (previous) {
|
|
// Repeated UI scans must not manufacture a sustained failure.
|
|
const newSample = Date.now() - Date.parse(previous.last_seen_at) >= WATCHDOG_SCAN_MS;
|
|
const actionable = (details as WatchdogEvidence & { actionable?: boolean }).actionable;
|
|
const priorActionable = (JSON.parse(previous.details) as { actionable?: boolean }).actionable;
|
|
// A successful routine/manual probe between watchdog samples breaks
|
|
// the failure streak, even if our next probe happens to fail again.
|
|
const after = latestGameSubmission(server);
|
|
const successful = [getServer(server)?.last_seen_at, telemetryReport(server)?.receivedAt]
|
|
.some(at => at && Date.parse(at) > Date.parse(previous.last_seen_at)
|
|
&& Date.parse(at) <= Date.now() && (!after || Date.parse(at) > Date.parse(after)));
|
|
const reset = kind === 'health' && (successful || (actionable === true && priorActionable !== true));
|
|
db().prepare('UPDATE watchdog_incidents SET details=?, last_seen_at=?, failures=?, first_seen_at=? WHERE id=?')
|
|
.run(JSON.stringify(details), newSample || reset ? iso() : previous.last_seen_at,
|
|
reset ? 1 : previous.failures + (newSample ? 1 : 0), reset ? iso() : previous.first_seen_at, previous.id);
|
|
if (previous.state === 'open') transition(previous.id, 'open', summary);
|
|
return incident(previous.id)!;
|
|
}
|
|
const id = randomUUID();
|
|
db().prepare(`INSERT INTO watchdog_incidents(id,server,kind,state,summary,details,first_seen_at,last_seen_at)
|
|
VALUES (?,?,?,'open',?,?,?,?)`).run(id, server, kind, summary, JSON.stringify(details), iso(), iso());
|
|
event(id, summary);
|
|
return incident(id)!;
|
|
}
|
|
|
|
/** Call only after verifying container readiness and current game evidence. */
|
|
export function resolveRecoveredHealthOperation(operationId: string, players: number, source: 'rcon' | 'telemetry' | null): boolean {
|
|
return db().transaction(() => {
|
|
const operation = db().prepare(`SELECT d.server, d.action, d.state, d.submitted_at, d.triggered_by, i.id AS incident_id
|
|
FROM server_deployments d JOIN provisioning_sources s ON s.operation_id = d.id AND s.kind = 'watchdog'
|
|
JOIN watchdog_incidents i ON i.id = s.source_id AND i.server = d.server
|
|
WHERE d.id = ? AND d.server_deleted_at IS NULL AND i.server_deleted_at IS NULL
|
|
AND i.kind = 'health' AND i.state IN ('open','recovering')
|
|
AND NOT EXISTS (SELECT 1 FROM provisioning_sources other WHERE other.operation_id = d.id AND other.kind != 'watchdog')`)
|
|
.get(operationId) as { server: string; action: string; state: string; submitted_at: string | null; triggered_by: string; incident_id: string } | undefined;
|
|
if (!operation || getServer(operation.server)?.desired_state !== 'running') return false;
|
|
const waiting = operation.state === 'waiting' && operation.submitted_at === null;
|
|
if (waiting && !operation.triggered_by.startsWith('watchdog:')) return false;
|
|
// Cancelling our unsubmitted request is safe when later queued work
|
|
// exists, and lets that work proceed. Submitted recovery must retain
|
|
// exclusive ownership before accepting evidence or releasing its slot.
|
|
if (!waiting && hasActiveProvisioning(operation.server, operationId)) return false;
|
|
if (!waiting && (operation.action !== 'restart' || !['awaiting-health','failed'].includes(operation.state))) return false;
|
|
const message = waiting ? 'Server recovered before automatic recovery was submitted' : 'Fresh container and game health verified after restart';
|
|
if (waiting && !cancelProvisioning(operationId, message)) return false;
|
|
const now = iso();
|
|
db().prepare(`UPDATE restart_rollout SET checked_at = ?, healthy_at = ?, players = ?, error = NULL
|
|
WHERE server = ? AND operation_id = ? AND healthy_at IS NULL`).run(now, now, players, operation.server, operationId);
|
|
db().prepare('UPDATE server_deployments SET checked_at = ?, players = ? WHERE id = ?').run(now, players, operationId);
|
|
recordProgress(operationId, { blocker: null, observedAt: now,
|
|
recovery: { state: 'released', at: now, operationId, message },
|
|
checks: [{ key: 'container', label: 'Container readiness', source: 'komodo', result: 'pass', checkedAt: now },
|
|
{ key: 'game', label: 'Game status', source, result: 'pass', actual: players, checkedAt: now }] });
|
|
if (operation.state === 'awaiting-health') {
|
|
deploymentStage(operationId, message);
|
|
finishDeployment(operationId, { ok: true });
|
|
}
|
|
transition(operation.incident_id, 'resolved', message);
|
|
return true;
|
|
})();
|
|
}
|
|
function actions(id: string): WatchdogAction[] {
|
|
return db().prepare(`SELECT a.*, d.state AS operation_state, d.stage AS operation_stage, d.error AS operation_error
|
|
FROM watchdog_actions a LEFT JOIN server_deployments d ON d.id=a.operation_id
|
|
WHERE a.incident_id=? ORDER BY a.rowid`).all(id) as WatchdogAction[];
|
|
}
|
|
export function watchdogOverview(): WatchdogOverview {
|
|
return { enabled: watchdogEnabled(), lastScanAt: setting('watchdog_last_scan'), scanError: setting('watchdog_scan_error') || null,
|
|
incidents: db().prepare(`SELECT i.*, (SELECT d.stage FROM watchdog_actions a
|
|
JOIN server_deployments d ON d.id = a.operation_id WHERE a.incident_id = i.id AND a.cycle = i.cycle
|
|
ORDER BY a.rowid DESC LIMIT 1) AS operation_stage
|
|
FROM watchdog_incidents i WHERE state != 'resolved' OR id IN (
|
|
SELECT id FROM watchdog_incidents WHERE state = 'resolved' ORDER BY first_seen_at DESC, rowid DESC LIMIT 100)
|
|
ORDER BY CASE WHEN state='resolved' THEN 1 ELSE 0 END, first_seen_at DESC, rowid DESC`).all() as WatchdogIncident[] };
|
|
}
|
|
export function watchdogDetail(id: string): WatchdogDetail | null {
|
|
const row = incident(id);
|
|
return row ? { incident: row, actions: actions(id), events: db().prepare('SELECT id,created_at,message FROM watchdog_events WHERE incident_id=? ORDER BY id').all(id) as WatchdogDetail['events'] } : null;
|
|
}
|
|
function scanSuppressed(server: string, kind?: 'health' | 'drift'): boolean {
|
|
// An explicit retry can reclaim its own earlier attempt's recovery slot.
|
|
// Keep other findings and every active lifecycle operation suppressed.
|
|
const retained = db().prepare(`SELECT r.operation_id FROM restart_rollout r
|
|
JOIN server_deployments d ON d.id = r.operation_id AND d.server_deleted_at IS NULL
|
|
JOIN watchdog_actions a ON a.operation_id = d.id
|
|
JOIN watchdog_incidents i ON i.id = a.incident_id
|
|
WHERE r.server = ? AND r.healthy_at IS NULL AND d.state IN ('failed','interrupted','cancelled')
|
|
AND i.server = r.server AND i.server_deleted_at IS NULL AND i.state = 'open' AND a.cycle < i.cycle
|
|
AND (? IS NULL OR i.kind = ?) LIMIT 1`).get(server, kind ?? null, kind ?? null) as { operation_id: string } | undefined;
|
|
return hasActiveProvisioning(server, retained?.operation_id);
|
|
}
|
|
function protectedWork(server: string): boolean {
|
|
return !!db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id=bs.beta_id WHERE bs.server=? AND bs.server_deleted_at IS NULL AND b.state IN ('running','ready-promote') LIMIT 1`).get(server)
|
|
|| !!db().prepare(`SELECT 1 FROM deployment_servers ds WHERE ds.server=? AND ds.server_deleted_at IS NULL AND ds.restart_state IN ('pending','held','failed','manual')
|
|
AND NOT EXISTS (SELECT 1 FROM watchdog_actions a WHERE a.campaign_id=ds.deployment_id) LIMIT 1`).get(server);
|
|
}
|
|
function addAction(row: WatchdogIncident, operationId: string, action: 'restart' | 'rebuild', campaignId: string | null = null) {
|
|
db().prepare('INSERT INTO watchdog_actions(id,incident_id,action,operation_id,campaign_id,created_at,cycle) VALUES (?,?,?,?,?,?,?)')
|
|
.run(randomUUID(), row.id, action, operationId, campaignId, iso(), row.cycle);
|
|
transition(row.id, 'recovering', action === 'restart' ? 'Restart queued; waiting for safe recovery' : 'Rebuild queued with the latest promoted setup');
|
|
event(row.id, `${action === 'restart' ? 'Restart' : 'Rebuild'} requested: ${operationId}${campaignId ? `; campaign ${campaignId}` : ''}`);
|
|
}
|
|
export function updateWatchdogIncident(id: string, action: 'retry' | 'dismiss') {
|
|
return db().transaction(() => {
|
|
const row = incident(id);
|
|
if (!row) return { ok: false as const, status: 404, error: 'Watchdog incident not found.' };
|
|
if (row.server_deleted_at) return { ok: false as const, status: 409, error: 'This server was deleted; its recovery history cannot be changed.' };
|
|
if (actions(id).some(a => pendingStates.includes(a.operation_state ?? ''))) return { ok: false as const, status: 409, error: 'Provisioning is still active. Complete or cancel that operation first.' };
|
|
if (action === 'dismiss') {
|
|
if (!['open','attention'].includes(row.state)) return { ok: false as const, status: 409, error: 'Only detected incidents or incidents needing attention can be dismissed.' };
|
|
transition(id, 'dismissed', 'Dismissed by operator; automatic attempts remain suppressed until the condition clears');
|
|
} else {
|
|
if (!['attention','dismissed'].includes(row.state)) return { ok: false as const, status: 409, error: 'Only incidents needing attention or dismissed incidents can be retried.' };
|
|
if (!getServer(row.server)) return { ok: false as const, status: 409, error: 'The server is no longer registered; inspect its provisioning history.' };
|
|
db().prepare('UPDATE watchdog_incidents SET cycle=cycle+1, failures=1, first_seen_at=?, last_seen_at=? WHERE id=?').run(iso(), iso(), id);
|
|
transition(id, 'open', 'Operator requested a fresh recovery attempt; checking current conditions');
|
|
}
|
|
return { ok: true as const };
|
|
})();
|
|
}
|
|
|
|
async function reconcileActions(): Promise<void> {
|
|
const rows = db().prepare("SELECT * FROM watchdog_incidents WHERE server_deleted_at IS NULL AND state IN ('recovering','attention')").all() as WatchdogIncident[];
|
|
for (const row of rows) {
|
|
const action = actions(row.id).filter(a => a.cycle === row.cycle).at(-1);
|
|
if (!action) continue;
|
|
// Continue observing our own recovery, but do not verify or escalate
|
|
// against files and health owned by another lifecycle operation.
|
|
if (hasActiveProvisioning(row.server, action.operation_id)) continue;
|
|
const operation = db().prepare('SELECT state,submitted_at,started_at,stage,error FROM server_deployments WHERE id=?').get(action.operation_id) as
|
|
{ state: string; submitted_at: string | null; started_at: string; stage: string; error: string | null } | undefined;
|
|
if (!operation) { transition(row.id, 'attention', 'Provisioning history is missing; inspect the server before retrying'); continue; }
|
|
if (['done','warning'].includes(operation.state)) {
|
|
if (row.kind === 'drift') {
|
|
try {
|
|
const plan = await previewDrift([row.server]);
|
|
if (incident(row.id)?.cycle !== row.cycle || hasActiveProvisioning(row.server, action.operation_id)) continue;
|
|
if (plan.files.length) { transition(row.id, 'attention', 'Rebuild completed but drift remains; inspect the campaign and server files'); continue; }
|
|
} catch (err) {
|
|
if (!hasActiveProvisioning(row.server, action.operation_id))
|
|
transition(row.id, 'attention', `Cannot verify rebuilt files: ${(err as Error).message}`);
|
|
continue;
|
|
}
|
|
}
|
|
transition(row.id, 'resolved', action.action === 'restart' ? 'Server recovered after restart' : 'Server recovered after rebuild');
|
|
continue;
|
|
}
|
|
if (row.state === 'attention') continue; // Exhausted attempts stay latched, including across app restarts.
|
|
if (['cancelled','interrupted'].includes(operation.state)) {
|
|
transition(row.id, 'attention', `Provisioning ${operation.state}; inspect the recorded operation before retrying`); continue;
|
|
}
|
|
if (action.action === 'rebuild') {
|
|
if (operation.state === 'failed') transition(row.id, 'attention', `Rebuild failed: ${operation.error ?? operation.stage}`);
|
|
else if (operation.state === 'awaiting-health' && Date.now() - Date.parse(operation.submitted_at ?? operation.started_at) >= WATCHDOG_RECOVERY_MS)
|
|
transition(row.id, 'attention', 'Rebuild has not recovered within five minutes; provisioning continues verification');
|
|
continue;
|
|
}
|
|
if (!['awaiting-health','failed'].includes(operation.state)) continue;
|
|
if (Date.now() - Date.parse(operation.submitted_at ?? operation.started_at) < WATCHDOG_RECOVERY_MS) continue;
|
|
if (!maintenanceAllowed(row.kind)) continue;
|
|
const current = getServer(row.server);
|
|
if (!current || current.desired_state === 'stopped' || protectedWork(row.server)) continue;
|
|
// A recovery deadline is a reason to check again, not evidence that
|
|
// rebuilding is necessary. The worker may not have polled this tick yet.
|
|
const observation = await observe(row.server, action.operation_id);
|
|
if (!observation || incident(row.id)?.cycle !== row.cycle || hasActiveProvisioning(row.server, action.operation_id)) continue;
|
|
if (observation.healthy) {
|
|
resolveRecoveredHealthOperation(action.operation_id, observation.players!, observation.source ?? null);
|
|
continue;
|
|
}
|
|
if (!observation.actionable) continue;
|
|
const result = db().transaction(() => {
|
|
const next = escalateWatchdogRestart(row.id, action.operation_id, WATCHDOG_RECOVERY_MS);
|
|
if (next.ok) addAction(row, next.deploymentId, 'rebuild');
|
|
return next;
|
|
})();
|
|
if (!result.ok && result.status !== 409 && result.status !== 503) transition(row.id, 'attention', `Could not queue rebuild: ${result.error}`);
|
|
}
|
|
}
|
|
|
|
interface Observation { healthy: boolean; actionable: boolean; reason: string; containerState: string | null; players?: number; source?: 'rcon' | 'telemetry' }
|
|
async function observe(server: string, operationId?: string): Promise<Observation | null> {
|
|
const epoch = latestGameSubmission(server);
|
|
const suppressed = () => (operationId ? hasActiveProvisioning(server, operationId) : scanSuppressed(server))
|
|
|| protectedWork(server) || getServer(server)?.desired_state !== 'running' || epoch !== latestGameSubmission(server);
|
|
const health = await komodoInspectHealthy(server);
|
|
if (suppressed()) return null;
|
|
const state = health.state?.toLowerCase() ?? null;
|
|
if (!health.ok) return { healthy: false, actionable: !!state && ['stopped','exited','created','dead','unhealthy','restarting','starting'].includes(state), reason: health.reason, containerState: state };
|
|
const telemetry = telemetryEvidence(server);
|
|
if (telemetry) return { healthy: true, actionable: false, reason: 'Komodo healthy with fresh telemetry', containerState: state, players: telemetry.players, source: 'telemetry' };
|
|
const rcon = await rconStatusResult(server);
|
|
if (suppressed()) return null;
|
|
const latest = telemetryEvidence(server);
|
|
if (rcon.ok || latest) return { healthy: true, actionable: false, reason: 'Komodo healthy with fresh game status', containerState: state,
|
|
players: latest?.players ?? (rcon.ok ? rcon.status.players : 0), source: latest ? 'telemetry' : 'rcon' };
|
|
return { healthy: false, actionable: rcon.reason === 'timeout' || rcon.reason === 'send-error',
|
|
reason: `Game status unavailable: ${rcon.reason}; waiting for player-count verification before recovery`, containerState: state };
|
|
}
|
|
async function scanServer(server: string): Promise<void> {
|
|
const row = getServer(server);
|
|
if (!row || row.desired_state === 'stopped') return;
|
|
if (scanSuppressed(server) || protectedWork(server)) return;
|
|
const observation = await observe(server);
|
|
// Probes can overlap an API request; recheck ownership before queuing anything.
|
|
if (!observation || scanSuppressed(server) || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return;
|
|
if (!scanSuppressed(server, 'health') && observation.healthy) {
|
|
const current = active(server, 'health');
|
|
if (current) transition(current.id, 'resolved', 'Fresh game status and Komodo health verified');
|
|
} else if (!scanSuppressed(server, 'health')) {
|
|
const problem = detect(server, 'health', observation.reason, observation);
|
|
const startup = ['starting','restarting'].includes(observation.containerState ?? '');
|
|
const oldEnough = Date.now() - Date.parse(problem.first_seen_at) >= (startup ? WATCHDOG_RECOVERY_MS : WATCHDOG_FAILURE_MS);
|
|
if (problem.state === 'open' && observation.actionable && oldEnough && problem.failures >= 3 && maintenanceAllowed('health')) {
|
|
const result = db().transaction(() => {
|
|
const next = requestWatchdogLifecycle(server, 'restart', problem.id);
|
|
if (next.ok) addAction(problem, next.deploymentId, 'restart');
|
|
return next;
|
|
})();
|
|
if (!result.ok && result.status !== 409 && result.status !== 503) transition(problem.id, 'attention', result.error);
|
|
}
|
|
}
|
|
// Unknown container health is an observation failure, not proof that a rebuild is safe.
|
|
if ((!observation.healthy && !observation.actionable) || scanSuppressed(server, 'drift')) return;
|
|
const prepared = await prepareDriftCampaign(server);
|
|
if (scanSuppressed(server, 'drift') || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return;
|
|
if (!prepared.plan.files.length) {
|
|
const current = active(server, 'drift');
|
|
if (current) transition(current.id, 'resolved', 'Server files match the promoted setup');
|
|
return;
|
|
}
|
|
const problem = detect(server, 'drift', `${prepared.plan.files.length} file difference(s) from the promoted setup`, { files: prepared.plan.files, fingerprint: prepared.plan.fingerprint });
|
|
if (problem.state !== 'open' || !maintenanceAllowed('drift')) return;
|
|
const result = db().transaction(() => {
|
|
const next = requestWatchdogLifecycle(server, 'rebuild', problem.id);
|
|
if (next.ok) {
|
|
const campaignId = createDriftRebuildCampaign(server, next.deploymentId, problem.id, prepared);
|
|
addAction(problem, next.deploymentId, 'rebuild', campaignId);
|
|
}
|
|
return next;
|
|
})();
|
|
if (!result.ok && result.status !== 409 && result.status !== 503) transition(problem.id, 'attention', `Could not queue drift rebuild: ${result.error}`);
|
|
}
|
|
|
|
let inFlight: Promise<void> | null = null;
|
|
export function watchdogTick(forceScan = false): Promise<void> {
|
|
if (inFlight) return inFlight;
|
|
inFlight = run(forceScan).finally(() => { inFlight = null; });
|
|
return inFlight;
|
|
}
|
|
async function run(forceScan: boolean): Promise<void> {
|
|
await reconcileActions();
|
|
const lastScan = setting('watchdog_last_scan');
|
|
if (!forceScan && lastScan && Date.now() - Date.parse(lastScan) < WATCHDOG_SCAN_MS) return;
|
|
setSetting('watchdog_last_scan', iso());
|
|
if (!komodoConfigured() && !listServers().some(server => nativeInstance(server.name))) {
|
|
setSetting('watchdog_scan_error', 'Configure Komodo before automatic recovery. RCON or fresh telemetry verifies game status.'); return;
|
|
}
|
|
const errors: string[] = [];
|
|
const servers = listServers();
|
|
let cursor = 0;
|
|
async function pump() {
|
|
while (cursor < servers.length) {
|
|
const server = servers[cursor++];
|
|
try { await scanServer(server.name); }
|
|
catch (err) {
|
|
if (!hasActiveProvisioning(server.name)) errors.push(`${server.name}: ${(err as Error).message}`);
|
|
}
|
|
}
|
|
}
|
|
await Promise.all(Array.from({ length: Math.min(4, servers.length) }, pump));
|
|
// Preserve incidents for temporarily absent rebuild instances, but close retired registrations.
|
|
const abandoned = db().prepare("SELECT * FROM watchdog_incidents WHERE server_deleted_at IS NULL AND state != 'resolved'").all() as WatchdogIncident[];
|
|
for (const row of abandoned) {
|
|
if (getServer(row.server) || hasActiveProvisioning(row.server)) continue;
|
|
const last = db().prepare('SELECT action,state FROM server_deployments WHERE server=? ORDER BY rowid DESC LIMIT 1').get(row.server) as { action: string; state: string } | undefined;
|
|
// An absent registry can also mean a failed rebuild. Only a completed
|
|
// destruction retires that failure without requiring inspection.
|
|
if (['open','dismissed'].includes(row.state) || (last?.action === 'destroy' && ['done','warning'].includes(last.state)))
|
|
transition(row.id, 'resolved', 'Server is no longer registered');
|
|
}
|
|
setSetting('watchdog_scan_error', errors.join('\n'));
|
|
}
|