josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Suppress maintenance watchers during provisioning

Browse Source 
Keep maintenance observations and automatic requests clear of active lifecycle operations and retained recovery capacity while preserving owned recovery and explicit retries.

Fixes #55
This commit is contained in:
josh
2026-10-01 11:54:22 -04:00
parent a3cd3ee585
commit 0cb48c435e
8 changed files with 366 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/maintenance/recovery.ts
+34 -14
View File
@@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto';
import { db } from '../db/index.js';
import { komodoConfigured } from '../config.js';
import { getServer, listServers } from '../fleet/servers.js';
import { hasActiveProvisioning } from '../fleet/provisioning.js';
import { komodoInspectHealthy } from '../integrations/komodo.js';
import { rconStatusResult } from '../integrations/rcon.js';
import { telemetryEvidence } from '../fleet/status-evidence.js';
@@ -73,8 +74,17 @@ export function watchdogDetail(id: string): WatchdogDetail | null {
const row = incident(id);
return row ? { incident: row, actions: actions(id), events: db().prepare('SELECT id,created_at,message FROM watchdog_events WHERE incident_id=? ORDER BY id').all(id) as WatchdogDetail['events'] } : null;
}
function hasPending(server: string) {
return !!db().prepare("SELECT 1 FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND state IN ('queued','waiting','running','awaiting-health') LIMIT 1").get(server);
function scanSuppressed(server: string, kind?: 'health' | 'drift'): boolean {
// An explicit retry can reclaim its own earlier attempt's recovery slot.
// Keep other findings and every active lifecycle operation suppressed.
const retained = db().prepare(`SELECT r.operation_id FROM restart_rollout r
JOIN server_deployments d ON d.id = r.operation_id AND d.server_deleted_at IS NULL
JOIN watchdog_actions a ON a.operation_id = d.id
JOIN watchdog_incidents i ON i.id = a.incident_id
WHERE r.server = ? AND r.healthy_at IS NULL AND d.state IN ('failed','interrupted','cancelled')
AND i.server = r.server AND i.server_deleted_at IS NULL AND i.state = 'open' AND a.cycle < i.cycle
AND (? IS NULL OR i.kind = ?) LIMIT 1`).get(server, kind ?? null, kind ?? null) as { operation_id: string } | undefined;
return hasActiveProvisioning(server, retained?.operation_id);
}
function protectedWork(server: string): boolean {
return !!db().prepare(`SELECT 1 FROM beta_servers bs JOIN betas b ON b.id=bs.beta_id WHERE bs.server=? AND bs.server_deleted_at IS NULL AND b.state IN ('running','ready-promote') LIMIT 1`).get(server)
@@ -111,6 +121,9 @@ async function reconcileActions(): Promise<void> {
for (const row of rows) {
const action = actions(row.id).filter(a => a.cycle === row.cycle).at(-1);
if (!action) continue;
// Continue observing our own recovery, but do not verify or escalate
// against files and health owned by another lifecycle operation.
if (hasActiveProvisioning(row.server, action.operation_id)) continue;
const operation = db().prepare('SELECT state,submitted_at,started_at,stage,error FROM server_deployments WHERE id=?').get(action.operation_id) as
{ state: string; submitted_at: string | null; started_at: string; stage: string; error: string | null } | undefined;
if (!operation) { transition(row.id, 'attention', 'Provisioning history is missing; inspect the server before retrying'); continue; }
@@ -118,9 +131,13 @@ async function reconcileActions(): Promise<void> {
if (row.kind === 'drift') {
try {
const plan = await previewDrift([row.server]);
if (incident(row.id)?.cycle !== row.cycle) continue;
if (incident(row.id)?.cycle !== row.cycle || hasActiveProvisioning(row.server, action.operation_id)) continue;
if (plan.files.length) { transition(row.id, 'attention', 'Rebuild completed but drift remains; inspect the campaign and server files'); continue; }
} catch (err) { transition(row.id, 'attention', `Cannot verify rebuilt files: ${(err as Error).message}`); continue; }
} catch (err) {
if (!hasActiveProvisioning(row.server, action.operation_id))
transition(row.id, 'attention', `Cannot verify rebuilt files: ${(err as Error).message}`);
continue;
}
}
transition(row.id, 'resolved', action.action === 'restart' ? 'Server recovered after restart' : 'Server recovered after rebuild');
continue;
@@ -150,8 +167,9 @@ async function reconcileActions(): Promise<void> {
}
interface Observation { healthy: boolean; actionable: boolean; reason: string; containerState: string | null }
async function observe(server: string): Promise<Observation> {
async function observe(server: string): Promise<Observation | null> {
const health = await komodoInspectHealthy(server);
if (scanSuppressed(server) || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return null;
const state = health.state?.toLowerCase() ?? null;
if (!health.ok) return { healthy: false, actionable: !!state && ['stopped','exited','created','dead','unhealthy','restarting','starting'].includes(state), reason: health.reason, containerState: state };
if (telemetryEvidence(server)) return { healthy: true, actionable: false, reason: 'Komodo healthy with fresh telemetry', containerState: state };
@@ -162,14 +180,14 @@ async function observe(server: string): Promise<Observation> {
async function scanServer(server: string): Promise<void> {
const row = getServer(server);
if (!row || row.desired_state === 'stopped') return;
if (hasPending(server) || protectedWork(server)) return;
if (scanSuppressed(server) || protectedWork(server)) return;
const observation = await observe(server);
// Probes can overlap an API request; recheck ownership before queuing anything.
if (hasPending(server) || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return;
if (observation.healthy) {
if (!observation || scanSuppressed(server) || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return;
if (!scanSuppressed(server, 'health') && observation.healthy) {
const current = active(server, 'health');
if (current) transition(current.id, 'resolved', 'Fresh game status and Komodo health verified');
} else {
} else if (!scanSuppressed(server, 'health')) {
const problem = detect(server, 'health', observation.reason, observation);
const startup = ['starting','restarting'].includes(observation.containerState ?? '');
const oldEnough = Date.now() - Date.parse(problem.first_seen_at) >= (startup ? WATCHDOG_RECOVERY_MS : WATCHDOG_FAILURE_MS);
@@ -181,11 +199,11 @@ async function scanServer(server: string): Promise<void> {
})();
if (!result.ok && result.status !== 409 && result.status !== 503) transition(problem.id, 'attention', result.error);
}
// Unknown container health is an observation failure, not proof that a rebuild is safe.
if (!observation.actionable || hasPending(server)) return;
}
// Unknown container health is an observation failure, not proof that a rebuild is safe.
if ((!observation.healthy && !observation.actionable) || scanSuppressed(server, 'drift')) return;
const prepared = await prepareDriftCampaign(server);
if (hasPending(server) || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return;
if (scanSuppressed(server, 'drift') || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return;
if (!prepared.plan.files.length) {
const current = active(server, 'drift');
if (current) transition(current.id, 'resolved', 'Server files match the promoted setup');
@@ -225,14 +243,16 @@ async function run(forceScan: boolean): Promise<void> {
while (cursor < servers.length) {
const server = servers[cursor++];
try { await scanServer(server.name); }
catch (err) { errors.push(`${server.name}: ${(err as Error).message}`); }
catch (err) {
if (!hasActiveProvisioning(server.name)) errors.push(`${server.name}: ${(err as Error).message}`);
}
}
}
await Promise.all(Array.from({ length: Math.min(4, servers.length) }, pump));
// Preserve incidents for temporarily absent rebuild instances, but close retired registrations.
const abandoned = db().prepare("SELECT * FROM watchdog_incidents WHERE server_deleted_at IS NULL AND state != 'resolved'").all() as WatchdogIncident[];
for (const row of abandoned) {
if (getServer(row.server) || hasPending(row.server)) continue;
if (getServer(row.server) || hasActiveProvisioning(row.server)) continue;
const last = db().prepare('SELECT action,state FROM server_deployments WHERE server=? ORDER BY rowid DESC LIMIT 1').get(row.server) as { action: string; state: string } | undefined;
// An absent registry can also mean a failed rebuild. Only a completed
// destruction retires that failure without requiring inspection.