diff --git a/README.md b/README.md index c9e6dc0..1b966eb 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,14 @@ or restarting containers hide old game data and RCON errors; stopped, unhealthy, or unverifiable containers do not trigger routine RCON polling. Containers without a Docker health check use their running state as the readiness signal. +Status polling runs every 30 seconds with at most four server checks in flight. +Each completed check publishes a current fleet snapshot; a slow check does not +hold back updates or the next poll for servers that have finished. RCON status +queries retry timeouts and send errors twice, with 250ms then 500ms backoff. +Overlapping status and occupancy probes share the in-flight query for the same +server process and endpoint. Configuration and invalid-reply errors are reported +immediately; commands that change server state are never retried by this logic. + With fresh container health, validated telemetry received in the last 45 seconds supplies the current player count, roster, map, and game status. RCON is the fallback when telemetry is unavailable or stale. Fleet, server details, metrics, collection diff --git a/docs/maintenance.md b/docs/maintenance.md index e086607..74ecfa0 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -106,10 +106,18 @@ The health observer (the watchdog) scans once per minute. Three observations ove at least two minutes are required before an automatic health restart; starting or restarting containers get five minutes. Unknown Komodo responses are observations, not proof of a stopped container. Repeated manual checks do not accelerate this -window. +window. A successful routine or manual game-status probe between scans resets +the failure streak. Missing RCON configuration, unresolved ports, and invalid +replies remain diagnostic findings and cannot authorize health recovery. A health finding gets one restart, then one rebuild if recovery fails after five -minutes. Escalation transfers only that finding's restart slot. A failed rebuild +minutes and fresh health checks still justify recovery. If current container and +game checks succeed before an automatic health restart or rebuild is submitted, +the obsolete operation is cancelled and the finding is resolved. Unknown health +holds recovery for another check. Drift repairs and requested lifecycle operations +retain their own eligibility rules. + +Escalation transfers only that finding's restart slot. A failed rebuild requires operator attention. **Rebuild deletes all instance files** and uses the latest promoted setup, the same server identity and port, and the saved server key. Missing keys block teardown. Submitted rebuild verification continues while paused. diff --git a/src/lib/server/fleet/lifecycle-evidence.ts b/src/lib/server/fleet/lifecycle-evidence.ts new file mode 100644 index 0000000..2be68df --- /dev/null +++ b/src/lib/server/fleet/lifecycle-evidence.ts @@ -0,0 +1,8 @@ +import { db } from '../db/index.js'; + +/** Reports from before the latest lifecycle submission describe the old process. */ +export function latestGameSubmission(server: string): string | null { + return (db().prepare(`SELECT MAX(at) AS at FROM ( + SELECT submitted_at AS at FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild') + UNION ALL SELECT requested_at AS at FROM restart_rollout WHERE server=?)`).get(server, server) as { at: string | null }).at; +} diff --git a/src/lib/server/fleet/status-evidence.ts b/src/lib/server/fleet/status-evidence.ts index 66ff843..4721c37 100644 --- a/src/lib/server/fleet/status-evidence.ts +++ b/src/lib/server/fleet/status-evidence.ts @@ -2,16 +2,12 @@ import { db } from '../db/index.js'; import { parseGameEvent, type GameTelemetry } from '../../telemetry.js'; import { ONLINE_MAX_MS } from '../../server-freshness.js'; import { rconGetPlayerCount, rconStatusResult } from '../integrations/rcon.js'; +import { latestGameSubmission } from './lifecycle-evidence.js'; + +export { latestGameSubmission } from './lifecycle-evidence.js'; export interface TelemetryEvidence { players: number; map: string; receivedAt: string } -/** Reports from before the latest lifecycle submission describe the old process. */ -export function latestGameSubmission(server: string): string | null { - return (db().prepare(`SELECT MAX(at) AS at FROM ( - SELECT submitted_at AS at FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND action IN ('create','start','restart','rebuild') - UNION ALL SELECT requested_at AS at FROM restart_rollout WHERE server=?)`).get(server, server) as { at: string | null }).at; -} - export function telemetryReport(server: string): GameTelemetry | null { const row = db().prepare('SELECT received_at, payload FROM server_telemetry WHERE server = ?').get(server) as { received_at: string; payload: string | null } | undefined; diff --git a/src/lib/server/integrations/komodo.ts b/src/lib/server/integrations/komodo.ts index 028f9bb..ed584b8 100644 --- a/src/lib/server/integrations/komodo.ts +++ b/src/lib/server/integrations/komodo.ts @@ -320,7 +320,7 @@ const HEALTHY_STATES = new Set(['running', 'healthy', 'ok', 'started', 'up']); async function fetchJsonWithSnippet( path: string, body: Record -): Promise<{ ok: true; json: unknown; snippet: string } | { ok: false; reason: string; snippet?: string }> { +): Promise<{ ok: true; json: unknown; snippet: string } | { ok: false; reason: string; snippet?: string; unsupportedEndpoint?: boolean }> { const url = `${config.komodo.address.replace(/\/+$/, '')}/${path}`; let res: Response; try { @@ -330,7 +330,11 @@ async function fetchJsonWithSnippet( } const text = await res.text().catch(() => ''); if (!res.ok) { - return { ok: false, reason: `HTTP ${res.status}`, snippet: truncate(text) }; + const unsupportedEndpoint = res.status === 404 || ( + [400, 405, 422, 500, 501].includes(res.status) + && /\b(?:unknown|unsupported|unrecognized)\s+(?:method|endpoint|variant)\b|\bmethod not found\b/i.test(text) + ); + return { ok: false, reason: `HTTP ${res.status}`, snippet: truncate(text), unsupportedEndpoint }; } let json: unknown; try { @@ -382,9 +386,10 @@ export async function komodoInspectHealthy(serverName: string): Promise { - const res = await endpointFor(server); - if (!res.endpoint) return null; - const send = await sendRcon(res.endpoint.host, res.endpoint.port, 'status'); - if (!send.ok) return null; - const text = stripPrefix(send.buf); - if (!text) return null; - return parsePlayerCount(text); + const result = await rconStatusResult(server); + return result.ok ? result.status.players : null; } export interface RconStatus { @@ -338,6 +334,18 @@ export type RconStatusResult = }; const RAW_REPLY_LIMIT = 512; +const statusInFlight = new Map>(); + +/** Retry only this read-only query: commands with effects must never be replayed. */ +async function sendStatus(endpoint: RconEndpoint): Promise { + for (let attempt = 0; ; attempt++) { + const result = await sendRcon(endpoint.host, endpoint.port, 'status'); + if (result.ok || (result.reason !== 'timeout' && result.reason !== 'send-error') || attempt === 2) { + return result; + } + await new Promise(resolve => setTimeout(resolve, 250 * 2 ** attempt)); + } +} /** * Render a Buffer as a short hex + printable-ASCII snippet — same layout as @@ -365,6 +373,10 @@ export function formatRawReply(buf: Buffer): string { * rather than just "unreachable". */ export async function rconStatusResult(server: string): Promise { + // Snapshot before any network work. A probe begun after a restart or a + // replacement registration must not join one observing the old process. + const createdAt = getServer(server)?.created_at; + const epoch = createdAt ? latestGameSubmission(server) : null; const res = await endpointFor(server); if (!res.endpoint) { if (res.reason === 'unknown-server') { @@ -387,13 +399,22 @@ export async function rconStatusResult(server: string): Promise { statusInFlight.delete(key); }); + statusInFlight.set(key, request); + return request; +} + +async function probeStatus(ep: RconEndpoint): Promise { // Plutonium T5 doesn't recognize `getstatus` as an rcon subcommand // ("Unknown command"), so we use `status` — a print-body response with // a header table plus one row per player. Q3-derived engines that // *do* answer statusResponse (via anonymous `getstatus` query) would // give richer info-line dvars, but every engine we care about supports // `status` under the rcon wrapper. - const send = await sendRcon(ep.host, ep.port, 'status'); + const send = await sendStatus(ep); if (!send.ok) { return { ok: false, endpoint: ep, reason: send.reason, detail: send.detail }; } diff --git a/src/lib/server/maintenance/recovery.ts b/src/lib/server/maintenance/recovery.ts index 3344ab3..50d7d58 100644 --- a/src/lib/server/maintenance/recovery.ts +++ b/src/lib/server/maintenance/recovery.ts @@ -5,9 +5,11 @@ import { getServer, listServers } from '../fleet/servers.js'; import { hasActiveProvisioning } from '../fleet/provisioning.js'; import { komodoInspectHealthy } from '../integrations/komodo.js'; import { rconStatusResult } from '../integrations/rcon.js'; -import { telemetryEvidence } from '../fleet/status-evidence.js'; +import { latestGameSubmission, telemetryEvidence, telemetryReport } from '../fleet/status-evidence.js'; import { prepareDriftCampaign, createDriftRebuildCampaign, previewDrift } from '../fleet/drift.js'; -import { requestWatchdogLifecycle, escalateWatchdogRestart } from '../provisioning/requests.js'; +import { cancelProvisioning, requestWatchdogLifecycle, escalateWatchdogRestart } from '../provisioning/requests.js'; +import { deploymentStage, finishDeployment } from '../provisioning/history.js'; +import { recordProgress } from '../provisioning/progress.js'; import type { WatchdogAction, WatchdogDetail, WatchdogEvidence, WatchdogIncident, WatchdogOverview, WatchdogState } from '../../watchdog.js'; import { maintenanceAllowed, setMaintenancePolicy } from './policies.js'; @@ -43,7 +45,13 @@ function detect(server: string, kind: 'health' | 'drift', summary: string, detai const newSample = Date.now() - Date.parse(previous.last_seen_at) >= WATCHDOG_SCAN_MS; const actionable = (details as WatchdogEvidence & { actionable?: boolean }).actionable; const priorActionable = (JSON.parse(previous.details) as { actionable?: boolean }).actionable; - const reset = kind === 'health' && actionable === true && priorActionable !== true; + // A successful routine/manual probe between watchdog samples breaks + // the failure streak, even if our next probe happens to fail again. + const after = latestGameSubmission(server); + const successful = [getServer(server)?.last_seen_at, telemetryReport(server)?.receivedAt] + .some(at => at && Date.parse(at) > Date.parse(previous.last_seen_at) + && Date.parse(at) <= Date.now() && (!after || Date.parse(at) > Date.parse(after))); + const reset = kind === 'health' && (successful || (actionable === true && priorActionable !== true)); db().prepare('UPDATE watchdog_incidents SET details=?, last_seen_at=?, failures=?, first_seen_at=? WHERE id=?') .run(JSON.stringify(details), newSample || reset ? iso() : previous.last_seen_at, reset ? 1 : previous.failures + (newSample ? 1 : 0), reset ? iso() : previous.first_seen_at, previous.id); @@ -56,6 +64,43 @@ function detect(server: string, kind: 'health' | 'drift', summary: string, detai event(id, summary); return incident(id)!; } + +/** Call only after verifying container readiness and current game evidence. */ +export function resolveRecoveredHealthOperation(operationId: string, players: number, source: 'rcon' | 'telemetry' | null): boolean { + return db().transaction(() => { + const operation = db().prepare(`SELECT d.server, d.action, d.state, d.submitted_at, d.triggered_by, i.id AS incident_id + FROM server_deployments d JOIN provisioning_sources s ON s.operation_id = d.id AND s.kind = 'watchdog' + JOIN watchdog_incidents i ON i.id = s.source_id AND i.server = d.server + WHERE d.id = ? AND d.server_deleted_at IS NULL AND i.server_deleted_at IS NULL + AND i.kind = 'health' AND i.state IN ('open','recovering') + AND NOT EXISTS (SELECT 1 FROM provisioning_sources other WHERE other.operation_id = d.id AND other.kind != 'watchdog')`) + .get(operationId) as { server: string; action: string; state: string; submitted_at: string | null; triggered_by: string; incident_id: string } | undefined; + if (!operation || getServer(operation.server)?.desired_state !== 'running') return false; + const waiting = operation.state === 'waiting' && operation.submitted_at === null; + if (waiting && !operation.triggered_by.startsWith('watchdog:')) return false; + // Cancelling our unsubmitted request is safe when later queued work + // exists, and lets that work proceed. Submitted recovery must retain + // exclusive ownership before accepting evidence or releasing its slot. + if (!waiting && hasActiveProvisioning(operation.server, operationId)) return false; + if (!waiting && (operation.action !== 'restart' || !['awaiting-health','failed'].includes(operation.state))) return false; + const message = waiting ? 'Server recovered before automatic recovery was submitted' : 'Fresh container and game health verified after restart'; + if (waiting && !cancelProvisioning(operationId, message)) return false; + const now = iso(); + db().prepare(`UPDATE restart_rollout SET checked_at = ?, healthy_at = ?, players = ?, error = NULL + WHERE server = ? AND operation_id = ? AND healthy_at IS NULL`).run(now, now, players, operation.server, operationId); + db().prepare('UPDATE server_deployments SET checked_at = ?, players = ? WHERE id = ?').run(now, players, operationId); + recordProgress(operationId, { blocker: null, observedAt: now, + recovery: { state: 'released', at: now, operationId, message }, + checks: [{ key: 'container', label: 'Container readiness', source: 'komodo', result: 'pass', checkedAt: now }, + { key: 'game', label: 'Game status', source, result: 'pass', actual: players, checkedAt: now }] }); + if (operation.state === 'awaiting-health') { + deploymentStage(operationId, message); + finishDeployment(operationId, { ok: true }); + } + transition(operation.incident_id, 'resolved', message); + return true; + })(); +} function actions(id: string): WatchdogAction[] { return db().prepare(`SELECT a.*, d.state AS operation_state, d.stage AS operation_stage, d.error AS operation_error FROM watchdog_actions a LEFT JOIN server_deployments d ON d.id=a.operation_id @@ -157,6 +202,15 @@ async function reconcileActions(): Promise { if (!maintenanceAllowed(row.kind)) continue; const current = getServer(row.server); if (!current || current.desired_state === 'stopped' || protectedWork(row.server)) continue; + // A recovery deadline is a reason to check again, not evidence that + // rebuilding is necessary. The worker may not have polled this tick yet. + const observation = await observe(row.server, action.operation_id); + if (!observation || incident(row.id)?.cycle !== row.cycle || hasActiveProvisioning(row.server, action.operation_id)) continue; + if (observation.healthy) { + resolveRecoveredHealthOperation(action.operation_id, observation.players!, observation.source ?? null); + continue; + } + if (!observation.actionable) continue; const result = db().transaction(() => { const next = escalateWatchdogRestart(row.id, action.operation_id, WATCHDOG_RECOVERY_MS); if (next.ok) addAction(row, next.deploymentId, 'rebuild'); @@ -166,16 +220,24 @@ async function reconcileActions(): Promise { } } -interface Observation { healthy: boolean; actionable: boolean; reason: string; containerState: string | null } -async function observe(server: string): Promise { +interface Observation { healthy: boolean; actionable: boolean; reason: string; containerState: string | null; players?: number; source?: 'rcon' | 'telemetry' } +async function observe(server: string, operationId?: string): Promise { + const epoch = latestGameSubmission(server); + const suppressed = () => (operationId ? hasActiveProvisioning(server, operationId) : scanSuppressed(server)) + || protectedWork(server) || getServer(server)?.desired_state !== 'running' || epoch !== latestGameSubmission(server); const health = await komodoInspectHealthy(server); - if (scanSuppressed(server) || protectedWork(server) || getServer(server)?.desired_state === 'stopped') return null; + if (suppressed()) return null; const state = health.state?.toLowerCase() ?? null; if (!health.ok) return { healthy: false, actionable: !!state && ['stopped','exited','created','dead','unhealthy','restarting','starting'].includes(state), reason: health.reason, containerState: state }; - if (telemetryEvidence(server)) return { healthy: true, actionable: false, reason: 'Komodo healthy with fresh telemetry', containerState: state }; + const telemetry = telemetryEvidence(server); + if (telemetry) return { healthy: true, actionable: false, reason: 'Komodo healthy with fresh telemetry', containerState: state, players: telemetry.players, source: 'telemetry' }; const rcon = await rconStatusResult(server); - if (rcon.ok || telemetryEvidence(server)) return { healthy: true, actionable: false, reason: 'Komodo healthy with fresh game status', containerState: state }; - return { healthy: false, actionable: true, reason: `Game status unavailable: ${rcon.reason}; waiting for player-count verification before recovery`, containerState: state }; + if (suppressed()) return null; + const latest = telemetryEvidence(server); + if (rcon.ok || latest) return { healthy: true, actionable: false, reason: 'Komodo healthy with fresh game status', containerState: state, + players: latest?.players ?? (rcon.ok ? rcon.status.players : 0), source: latest ? 'telemetry' : 'rcon' }; + return { healthy: false, actionable: rcon.reason === 'timeout' || rcon.reason === 'send-error', + reason: `Game status unavailable: ${rcon.reason}; waiting for player-count verification before recovery`, containerState: state }; } async function scanServer(server: string): Promise { const row = getServer(server); diff --git a/src/lib/server/pipeline/worker.ts b/src/lib/server/pipeline/worker.ts index ca9adfe..79d8927 100644 --- a/src/lib/server/pipeline/worker.ts +++ b/src/lib/server/pipeline/worker.ts @@ -6,7 +6,7 @@ import { emit } from './bus.js'; import type { ServerStatusEntry } from './bus.js'; import { rconStatusResult } from '../integrations/rcon.js'; import type { RconFailure } from '../integrations/rcon.js'; -import { getServer, listServers, setServerField } from '../fleet/servers.js'; +import { getServer, listServers, setServerField, type ServerRow } from '../fleet/servers.js'; import { komodoInspectHealthy } from '../integrations/komodo.js'; import { restartSettling } from '../provisioning/index.js'; import { serverObservation } from '../fleet/status.js'; @@ -58,92 +58,126 @@ function encodeError(reason: RconFailure, detail: string): string { return `${reason}:${detail}`; } -let statusInFlight: Promise | null = null; +// Deduplicate queued/running checks per server. A slow server must not prevent +// completed servers from being polled again on the next 30-second tick. +const pendingStatus = new Map>(); +const statusQueue: Array<() => Promise> = []; +let activeStatus = 0; -export function fanoutStatus(): Promise { - if (statusInFlight) return statusInFlight; - statusInFlight = refreshStatus().finally(() => { statusInFlight = null; }); - return statusInFlight; +function drainStatusQueue(): void { + while (activeStatus < STATUS_CONCURRENCY && statusQueue.length) { + const next = statusQueue.shift()!; + activeStatus++; + void next().finally(() => { activeStatus--; drainStatusQueue(); }); + } } -async function refreshStatus(): Promise { +function queueStatus(server: string): Promise { + const pending = pendingStatus.get(server); + if (pending) return pending; + let finish!: (capacity: number | null) => void; + const work = new Promise(resolve => { finish = resolve; }); + pendingStatus.set(server, work); + statusQueue.push(async () => { + let capacity: number | null = null; + try { + capacity = await refreshServerStatus(server); + // Consumers expect a full snapshot. Re-read it now so neither game + // data nor container evidence waits for the slowest fleet member. + emit({ type: 'server-status', entries: listServers().map(statusEntry) }); + } catch (err) { console.error(`[worker] status error for ${server}:`, err); } + finally { pendingStatus.delete(server); finish(capacity); } + }); + drainStatusQueue(); + return work; +} + +export async function fanoutStatus(): Promise { const servers = listServers(); if (!servers.length) return; - const entries: ServerStatusEntry[] = new Array(servers.length); - const capacities = new Map(); - let cursor = 0; - async function pump() { - while (cursor < servers.length) { - const i = cursor++; - const srv = servers[i]; - if (!isProvisioning(srv.name)) { - const epoch = latestGameSubmission(srv.name); - const container = restartSettling(srv.name) - ? { ok: false as const, state: 'restarting', reason: 'Restart requested' } - : await komodoInspectHealthy(srv.name).catch(err => ({ ok: false as const, state: 'unknown', reason: (err as Error).message })); - // A lifecycle request may be submitted while an inspection is in flight. - if (epoch === latestGameSubmission(srv.name) && !isProvisioning(srv.name)) { - setServerField(srv.name, 'container_state', container.ok ? 'running' : container.state?.toLowerCase() ?? 'unknown'); - setServerField(srv.name, 'container_checked_at', new Date().toISOString()); - if (!container.ok) { - setServerField(srv.name, 'last_error', null); - setServerField(srv.name, 'last_reply_snippet', null); - } else if (!telemetryEvidence(srv.name)) { - const result = await rconStatusResult(srv.name).catch(err => ({ ok: false as const, - reason: 'send-error' as RconFailure, detail: (err as Error).message, rawReply: undefined })); - if (epoch === latestGameSubmission(srv.name) && !isProvisioning(srv.name)) { - const now = new Date().toISOString(); - setServerField(srv.name, 'last_checked_at', now); - setServerField(srv.name, 'last_reply_snippet', result.rawReply ?? null); - if (result.ok) { - capacities.set(srv.name, result.status.capacity ?? null); - setServerField(srv.name, 'last_players', result.status.players); - setServerField(srv.name, 'last_map', result.status.map); - setServerField(srv.name, 'last_gametype', result.status.gametype); - setServerField(srv.name, 'last_roster', JSON.stringify(result.status.roster)); - setServerField(srv.name, 'last_seen_at', now); - setServerField(srv.name, 'last_error', null); - } else setServerField(srv.name, 'last_error', encodeError(result.reason, result.detail)); - } - } - } - } else { + const results = await Promise.all(servers.map(server => queueStatus(server.name))); + const capacities = new Map(servers.map((server, i) => [server.name, results[i]])); + try { + // Metrics remain complete fleet collections, using freshness at capture + // time rather than the status saved at the beginning of a long scan. + recordMetrics(listServers().map(srv => { + const current = serverObservation(srv); + return { server: srv.name, game: srv.game ?? 'unknown', + map: selectedZombieMap(srv.game, current.last_map ?? srv.map, srv.map_slug)?.friendly ?? current.last_map ?? srv.map ?? 'Unknown map', + players: current.health === 'online' ? current.last_players : null, + capacity: capacities.get(srv.name) ?? null }; + })); + } catch (err) { console.error('[worker] metrics error:', err); } +} + +async function refreshServerStatus(server: string): Promise { + const srv = getServer(server); + if (!srv) return null; + let capacity: number | null = null; + if (!isProvisioning(srv.name)) { + const epoch = latestGameSubmission(srv.name); + const unchanged = () => epoch === latestGameSubmission(srv.name) && !isProvisioning(srv.name) + && getServer(srv.name)?.created_at === srv.created_at; + const container = restartSettling(srv.name) + ? { ok: false as const, state: 'restarting', reason: 'Restart requested' } + : await komodoInspectHealthy(srv.name).catch(err => ({ ok: false as const, state: 'unknown', reason: (err as Error).message })); + // A lifecycle request may be submitted while an inspection is in flight. + if (unchanged()) { + setServerField(srv.name, 'container_state', container.ok ? 'running' : container.state?.toLowerCase() ?? 'unknown'); + setServerField(srv.name, 'container_checked_at', new Date().toISOString()); + if (!container.ok) { setServerField(srv.name, 'last_error', null); setServerField(srv.name, 'last_reply_snippet', null); - } - const current = getServer(srv.name) ?? srv; - const status = serverObservation(current); - // Occupancy timers and match clearing follow the same source as the UI. - if (status.health === 'online' && status.last_players !== null) { - if (status.last_players > 0) { - setServerField(srv.name, 'active_since', current.active_since ?? status.last_seen_at); - setServerField(srv.name, 'inactive_since', null); - } else { - setServerField(srv.name, 'active_since', null); - setServerField(srv.name, 'inactive_since', current.inactive_since ?? status.last_seen_at); - setServerField(srv.name, 'game_started_at', null); - setServerField(srv.name, 'game_ended_at', null); + } else if (!telemetryEvidence(srv.name)) { + const result = await rconStatusResult(srv.name).catch(err => ({ ok: false as const, + reason: 'send-error' as RconFailure, detail: (err as Error).message, rawReply: undefined })); + if (unchanged()) { + const now = new Date().toISOString(); + setServerField(srv.name, 'last_checked_at', now); + setServerField(srv.name, 'last_reply_snippet', result.rawReply ?? null); + if (result.ok) { + capacity = result.status.capacity ?? null; + setServerField(srv.name, 'last_players', result.status.players); + setServerField(srv.name, 'last_map', result.status.map); + setServerField(srv.name, 'last_gametype', result.status.gametype); + setServerField(srv.name, 'last_roster', JSON.stringify(result.status.roster)); + setServerField(srv.name, 'last_seen_at', now); + setServerField(srv.name, 'last_error', null); + } else setServerField(srv.name, 'last_error', encodeError(result.reason, result.detail)); } } - const final = serverObservation(getServer(srv.name) ?? current); - entries[i] = { server: srv.name, players: final.last_players, map: final.last_map ?? srv.map, - gametype: final.last_gametype, roster: final.roster, health: final.health, lastSeenAt: final.last_seen_at, - lastError: final.last_error, replySnippet: final.replySnippet, - endpoint: current.game_port ? { host: current.rcon_host ?? config.rcon.host, port: current.game_port } : null, reason: final.last_error ? final.last_error.split(':')[0] as RconFailure : null, - activeSince: final.active_since, inactiveSince: final.inactive_since, - gameStartedAt: final.game_started_at, gameEndedAt: final.game_ended_at, - containerState: final.container_state, containerCheckedAt: final.container_checked_at, - telemetryAfter: final.telemetry_after, rconStatus: final.rcon_status, telemetry: final.telemetry }; + } + } else { + setServerField(srv.name, 'last_error', null); + setServerField(srv.name, 'last_reply_snippet', null); + } + const current = getServer(srv.name) ?? srv; + const status = serverObservation(current); + // Occupancy timers and match clearing follow the same source as the UI. + if (status.health === 'online' && status.last_players !== null) { + if (status.last_players > 0) { + setServerField(srv.name, 'active_since', current.active_since ?? status.last_seen_at); + setServerField(srv.name, 'inactive_since', null); + } else { + setServerField(srv.name, 'active_since', null); + setServerField(srv.name, 'inactive_since', current.inactive_since ?? status.last_seen_at); + setServerField(srv.name, 'game_started_at', null); + setServerField(srv.name, 'game_ended_at', null); } } - await Promise.all(Array.from({ length: Math.min(STATUS_CONCURRENCY, servers.length) }, pump)); - try { - recordMetrics(servers.map((srv, i) => ({ server: srv.name, game: srv.game ?? 'unknown', - map: selectedZombieMap(srv.game, entries[i].map ?? srv.map, srv.map_slug)?.friendly ?? entries[i].map ?? srv.map ?? 'Unknown map', - players: entries[i].health === 'online' ? entries[i].players : null, - capacity: capacities.get(srv.name) ?? null }))); - } catch (err) { console.error('[worker] metrics error:', err); } - emit({ type: 'server-status', entries }); + return capacity; +} + +function statusEntry(srv: ServerRow): ServerStatusEntry { + const final = serverObservation(srv); + return { server: srv.name, players: final.last_players, map: final.last_map ?? srv.map, + gametype: final.last_gametype, roster: final.roster, health: final.health, lastSeenAt: final.last_seen_at, + lastError: final.last_error, replySnippet: final.replySnippet, + endpoint: srv.game_port ? { host: srv.rcon_host ?? config.rcon.host, port: srv.game_port } : null, reason: final.last_error ? final.last_error.split(':')[0] as RconFailure : null, + activeSince: final.active_since, inactiveSince: final.inactive_since, + gameStartedAt: final.game_started_at, gameEndedAt: final.game_ended_at, + containerState: final.container_state, containerCheckedAt: final.container_checked_at, + telemetryAfter: final.telemetry_after, rconStatus: final.rcon_status, telemetry: final.telemetry }; } export function startWorker(): void { diff --git a/src/lib/server/provisioning/requests.ts b/src/lib/server/provisioning/requests.ts index 76a9569..079de3d 100644 --- a/src/lib/server/provisioning/requests.ts +++ b/src/lib/server/provisioning/requests.ts @@ -139,12 +139,21 @@ export function requestLifecycle(server: string, action: RuntimeAction) { return db().transaction(() => { const blocker = action === 'rebuild' ? activeBetaRemovalBlocker(server) : null; if (blocker) return { ok: false as const, status: 409, error: blocker }; - const existing = db().prepare(`SELECT id, action FROM server_deployments WHERE server = ? + const existing = db().prepare(`SELECT id, action, state, submitted_at, triggered_by FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL - AND state IN ('queued','waiting','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(server) as { id: string; action: string } | undefined; + AND state IN ('queued','waiting','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(server) as + { id: string; action: string; state: string; submitted_at: string | null; triggered_by: string } | undefined; if (existing) { if (existing.action !== action) return { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' }; db().prepare('UPDATE servers SET desired_state = ? WHERE name = ?').run(action === 'stop' ? 'stopped' : 'running', server); + if (existing.state === 'waiting' && !existing.submitted_at && existing.triggered_by.startsWith('watchdog:') + && db().prepare(`SELECT 1 FROM provisioning_sources s JOIN watchdog_incidents i ON i.id = s.source_id + WHERE s.operation_id = ? AND s.kind = 'watchdog' AND i.kind = 'health'`).get(existing.id)) { + // An operator explicitly requesting this queued action keeps + // that intent even if the original health condition clears. + db().prepare("UPDATE server_deployments SET triggered_by = 'ui' WHERE id = ?").run(existing.id); + recordDeploymentEvent(existing.id, 'Operator explicitly requested the queued health recovery action'); + } return { ok: true as const, deploymentId: existing.id }; } const row = getServer(server); diff --git a/src/lib/server/provisioning/worker.ts b/src/lib/server/provisioning/worker.ts index 33e720c..8bd57d7 100644 --- a/src/lib/server/provisioning/worker.ts +++ b/src/lib/server/provisioning/worker.ts @@ -1,11 +1,12 @@ import { maintenanceAllowed } from '../maintenance/policies.js'; +import { resolveRecoveredHealthOperation } from '../maintenance/recovery.js'; import { db } from '../db/index.js'; import { komodoConfigured, rconConfigured } from '../config.js'; import { getServer } from '../fleet/servers.js'; import { deploymentStage, finishDeployment, REMOVING_SERVER } from './history.js'; import { processCreations, processDestructions, rebuildServer } from './instances.js'; import { komodoInspectHealthy, komodoRuntimeAction } from '../integrations/komodo.js'; -import { probePlayerEvidence } from '../fleet/status-evidence.js'; +import { latestGameSubmission, probePlayerEvidence } from '../fleet/status-evidence.js'; import { refreshRestartRollout, restartInRollout, rolloutBlocker } from './restarts.js'; import { cancelProvisioning } from './requests.js'; import type { RuntimeAction } from '../../provisioning.js'; @@ -158,11 +159,28 @@ async function processOperation(operation: Operation): Promise { } const reason = blocker(server, id, action); if (reason) { deploymentStage(id, reason.message, { phase: 'eligibility', checks: [], blocker: reason }); return; } + const healthRecovery = watchdog && !!db().prepare(`SELECT 1 FROM provisioning_sources s + JOIN watchdog_incidents i ON i.id = s.source_id JOIN server_deployments d ON d.id = s.operation_id + WHERE s.operation_id = ? AND s.kind = 'watchdog' AND i.kind = 'health' AND d.triggered_by LIKE 'watchdog:%'`).get(id); + const epoch = latestGameSubmission(server); const health = await komodoInspectHealthy(server); + if (healthRecovery && epoch !== latestGameSubmission(server)) return; + if (healthRecovery && !health.ok && !['stopped','exited','created','dead','unhealthy','starting','restarting'].includes(health.state?.toLowerCase() ?? '')) { + deploymentStage(id, 'Waiting for current container health before automatic recovery', { phase: 'eligibility', checks: [], + blocker: { code: 'observation-unavailable', message: health.reason, automatic: true } }); + return; + } if (action !== 'start' && !isStopped(health.state)) { const evidence = await probePlayerEvidence(server); if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return; const players = evidence.players; + if (healthRecovery && epoch !== latestGameSubmission(server)) return; + if (healthRecovery && health.ok && players !== null) { + // The fresh occupancy check also proves that the game recovered. + // Empty is not a reason to execute an obsolete health restart. + resolveRecoveredHealthOperation(id, players, evidence.source); + return; + } recordProgress(id, { phase: 'eligibility', checks: [{ key: 'occupancy', label: 'Player occupancy', result: players === 0 ? 'pass' : players === null ? 'unknown' : 'waiting', source: evidence.source, checkedAt: evidence.observedAt, receivedAt: evidence.receivedAt, expected: 0, actual: players }], blocker: players === 0 ? null : { code: players === null ? 'players-unknown' : 'players-present', diff --git a/src/routes/api/servers/[name]/probe-rcon/+server.ts b/src/routes/api/servers/[name]/probe-rcon/+server.ts index 3c959f8..505b011 100644 --- a/src/routes/api/servers/[name]/probe-rcon/+server.ts +++ b/src/routes/api/servers/[name]/probe-rcon/+server.ts @@ -2,6 +2,8 @@ import { json } from '@sveltejs/kit'; import type { RequestHandler } from './$types'; import { rconStatusResult } from '$lib/server/integrations/rcon.js'; import { getServer, setServerField } from '$lib/server/fleet/servers.js'; +import { latestGameSubmission } from '$lib/server/fleet/status-evidence.js'; +import { isProvisioning } from '$lib/server/fleet/provisioning.js'; /** * On-demand RCON probe for a specific server. Same code path as the periodic @@ -18,7 +20,14 @@ export const POST: RequestHandler = async ({ params }) => { const row = getServer(name); if (!row) return json({ ok: false, error: `unknown server: ${name}` }, { status: 404 }); + const epoch = latestGameSubmission(name); const result = await rconStatusResult(name); + // Keep the diagnostic response, but never persist old-process evidence + // across a lifecycle request or replacement registration. + const current = getServer(name); + if (!current || current.created_at !== row.created_at || epoch !== latestGameSubmission(name) || isProvisioning(name)) { + return json(result); + } const now = new Date().toISOString(); if (result.ok) { diff --git a/tests/unit/komodo-health.test.ts b/tests/unit/komodo-health.test.ts new file mode 100644 index 0000000..f8d661b --- /dev/null +++ b/tests/unit/komodo-health.test.ts @@ -0,0 +1,106 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { komodoInspectHealthy } from '../../src/lib/server/integrations/komodo.js'; + +vi.mock('../../src/lib/server/config.js', () => ({ + config: { + komodo: { address: 'http://komodo.test', apiKey: 'test', apiSecret: 'test' }, + httpTimeoutMs: 1000 + }, + komodoConfigured: () => true +})); +vi.mock('../../src/lib/server/fleet/servers.js', () => ({ + getServer: () => ({ komodo_type: 'deployment', komodo_name: 'saved-resource' }) +})); + +const fetchMock = vi.fn(); + +beforeEach(() => { + fetchMock.mockReset(); + vi.stubGlobal('fetch', fetchMock); +}); +afterEach(() => { + vi.unstubAllGlobals(); + vi.useRealTimers(); +}); + +describe('Komodo health endpoint fallback', () => { + it('limits an unreachable probe to one timeout and preserves its diagnosis', async () => { + vi.useFakeTimers(); + fetchMock.mockImplementation((_url, options) => new Promise((_resolve, reject) => { + options?.signal?.addEventListener('abort', () => reject(new DOMException('aborted', 'AbortError'))); + })); + + const result = komodoInspectHealthy('server'); + await vi.advanceTimersByTimeAsync(1000); + + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(await result).toMatchObject({ + ok: false, + reason: 'timed out after 1000ms dialing http://komodo.test/read/GetDeploymentContainer' + }); + }); + + it('preserves the network cause without probing alternate endpoints', async () => { + fetchMock.mockRejectedValue(new TypeError('fetch failed', { + cause: Object.assign(new Error('connection refused'), { code: 'ECONNREFUSED' }) + })); + + expect(await komodoInspectHealthy('server')).toMatchObject({ + ok: false, + reason: expect.stringContaining('ECONNREFUSED: connection refused') + }); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it.each([400, 401, 403, 429, 500, 502, 503])('stops on HTTP %i without hiding the response', async status => { + fetchMock.mockResolvedValue(new Response('request failed', { status })); + + expect(await komodoInspectHealthy('server')).toEqual({ + ok: false, reason: `HTTP ${status}`, snippet: 'request failed' + }); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it.each([ + [404, 'Not Found'], + [400, 'unknown variant `GetDeploymentContainer`'], + [405, 'unsupported method'], + [500, 'unknown method'], + [501, 'method not found'] + ])('tries a compatible endpoint after HTTP %i: %s', async (status, message) => { + fetchMock + .mockResolvedValueOnce(new Response(message as string, { status: status as number })) + .mockResolvedValueOnce(Response.json({ State: { Status: 'running' } })); + + expect(await komodoInspectHealthy('server')).toEqual({ ok: true, state: 'running' }); + expect(fetchMock.mock.calls.map(([url]) => url)).toEqual([ + 'http://komodo.test/read/GetDeploymentContainer', + 'http://komodo.test/read/InspectDeploymentContainer' + ]); + }); + + it('continues past successful responses with an unknown shape', async () => { + fetchMock + .mockResolvedValueOnce(Response.json({ config: { image: 'server' } })) + .mockResolvedValueOnce(Response.json({ State: { Status: 'running' } })); + + expect(await komodoInspectHealthy('server')).toEqual({ ok: true, state: 'running' }); + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + + it('does not overwrite an unhealthy container state with a later response', async () => { + fetchMock.mockResolvedValueOnce(Response.json({ State: { Status: 'exited' } })); + + expect(await komodoInspectHealthy('server')).toMatchObject({ ok: false, state: 'exited', reason: 'container state: exited' }); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it('preserves malformed response diagnostics', async () => { + fetchMock.mockResolvedValueOnce(new Response('not json')); + + expect(await komodoInspectHealthy('server')).toMatchObject({ + ok: false, reason: expect.stringContaining('unparseable json:'), snippet: 'not json' + }); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); +}); diff --git a/tests/unit/provisioning.test.ts b/tests/unit/provisioning.test.ts index b53cd3f..24d18fc 100644 --- a/tests/unit/provisioning.test.ts +++ b/tests/unit/provisioning.test.ts @@ -256,6 +256,10 @@ describe('provisioning operations', () => { async function incident(id = 'incident') { const { db } = await import('../../src/lib/server/db/index.js'); + const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); + // These lifecycle tests exercise an actual container failure. Healthy + // containers with fresh game evidence now cancel obsolete health work. + vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: false, state: 'unhealthy', reason: 'Container unhealthy' }); const now = new Date().toISOString(); db().prepare(`INSERT INTO watchdog_incidents (id,server,kind,state,summary,first_seen_at,last_seen_at) VALUES (?,?,'health','open','Server down',?,?)`).run(id, server, now, now); @@ -323,7 +327,7 @@ describe('Watchdog provisioning and telemetry evidence', () => { const { requestWatchdogLifecycle } = await import('../../src/lib/server/provisioning/index.js'); const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); const { rconGetPlayerCount, rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); - const { komodoRestart } = await import('../../src/lib/server/integrations/komodo.js'); + const { komodoRestart, komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); const { config } = await import('../../src/lib/server/config.js'); vi.spyOn(config.rcon, 'password', 'get').mockReturnValue(''); vi.mocked(rconGetPlayerCount).mockResolvedValue(null); @@ -333,6 +337,7 @@ describe('Watchdog provisioning and telemetry evidence', () => { if (!result.ok) throw new Error(result.error); await processProvisioning(); expect(komodoRestart).toHaveBeenCalledTimes(1); + vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); vi.setSystemTime(Date.now() + 15_000); await processProvisioning(); expect(await operation(result.deploymentId)).toMatchObject({ state: 'awaiting-health' }); diff --git a/tests/unit/rcon-status.test.ts b/tests/unit/rcon-status.test.ts new file mode 100644 index 0000000..8733c2b --- /dev/null +++ b/tests/unit/rcon-status.test.ts @@ -0,0 +1,167 @@ +import { EventEmitter } from 'node:events'; +import { createSocket, type Socket } from 'node:dgram'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { komodoConfigured, rconConfigured } from '../../src/lib/server/config.js'; +import { getServer, type ServerRow } from '../../src/lib/server/fleet/servers.js'; +import { latestGameSubmission } from '../../src/lib/server/fleet/lifecycle-evidence.js'; +import { rconGetPlayerCount, rconPlutoniumVersion, rconStatusResult } from '../../src/lib/server/integrations/rcon.js'; + +vi.mock('node:dgram', () => ({ createSocket: vi.fn() })); +vi.mock('../../src/lib/server/config.js', () => ({ + config: { rcon: { host: 'localhost', password: 'test', timeoutMs: 1_000 } }, + komodoConfigured: vi.fn(), rconConfigured: vi.fn() +})); +vi.mock('../../src/lib/server/fleet/servers.js', () => ({ getServer: vi.fn(), setServerField: vi.fn() })); +vi.mock('../../src/lib/server/fleet/lifecycle-evidence.js', () => ({ latestGameSubmission: vi.fn() })); +vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoResolveGamePort: vi.fn() })); + +class FakeSocket extends EventEmitter { + send = vi.fn(); + close = vi.fn(); +} + +let sockets: FakeSocket[]; +let server: ServerRow; +const packet = (body: string) => Buffer.concat([Buffer.from([255, 255, 255, 255]), Buffer.from(body)]); +const reply = (index = 0) => sockets[index].emit('message', packet('statusResponse\n\\mapname\\mp_kino\n10 45 "joe"')); + +beforeEach(() => { + vi.resetAllMocks(); + vi.useFakeTimers(); + sockets = []; + server = { name: 'test-server', created_at: '2026-10-01T00:00:00Z', game_port: 28960, rcon_host: null } as ServerRow; + vi.mocked(getServer).mockImplementation(() => ({ ...server })); + vi.mocked(latestGameSubmission).mockReturnValue(null); + vi.mocked(komodoConfigured).mockReturnValue(false); + vi.mocked(rconConfigured).mockReturnValue(true); + vi.mocked(createSocket).mockImplementation(() => { + const socket = new FakeSocket(); + sockets.push(socket); + return socket as unknown as Socket; + }); +}); + +afterEach(() => { + expect(vi.getTimerCount()).toBe(0); + vi.useRealTimers(); +}); + +describe('RCON status transport', () => { + it('shares overlapping health and occupancy probes and probes again after completion', async () => { + const health = rconStatusResult(server.name); + const occupancy = rconGetPlayerCount(server.name); + const duplicate = rconStatusResult(server.name); + await vi.advanceTimersByTimeAsync(0); + expect(sockets).toHaveLength(1); + expect(sockets[0].send.mock.calls[0][0].toString()).toContain('rcon test status\n'); + reply(); + expect(await health).toMatchObject({ ok: true, status: { players: 1, map: 'mp_kino' } }); + expect(await occupancy).toBe(1); + expect(await duplicate).toEqual(await health); + expect(sockets[0].close).toHaveBeenCalledOnce(); + + const fresh = rconStatusResult(server.name); + await vi.advanceTimersByTimeAsync(0); + expect(sockets).toHaveLength(2); + reply(1); + expect(await fresh).toMatchObject({ ok: true }); + }); + + it('recovers a dropped UDP reply after the first backoff', async () => { + const request = rconStatusResult(server.name); + await vi.advanceTimersByTimeAsync(1_000); + expect(sockets).toHaveLength(1); + expect(sockets[0].close).toHaveBeenCalledOnce(); + await vi.advanceTimersByTimeAsync(249); + expect(sockets).toHaveLength(1); + await vi.advanceTimersByTimeAsync(1); + expect(sockets).toHaveLength(2); + reply(1); + expect(await request).toMatchObject({ ok: true }); + }); + + it('bounds retries at three attempts with 250ms then 500ms backoffs', async () => { + const request = rconStatusResult(server.name); + await vi.advanceTimersByTimeAsync(1_250); + expect(sockets).toHaveLength(2); + await vi.advanceTimersByTimeAsync(1_499); + expect(sockets).toHaveLength(2); + await vi.advanceTimersByTimeAsync(1); + expect(sockets).toHaveLength(3); + await vi.advanceTimersByTimeAsync(1_000); + expect(await request).toMatchObject({ ok: false, reason: 'timeout' }); + expect(sockets.every(socket => socket.close.mock.calls.length === 1)).toBe(true); + + const fresh = rconStatusResult(server.name); + await vi.advanceTimersByTimeAsync(0); + expect(sockets).toHaveLength(4); + reply(3); + expect(await fresh).toMatchObject({ ok: true }); + }); + + it.each(['event', 'callback'])('retries transient send errors reported through the %s', async source => { + const request = rconStatusResult(server.name); + await vi.advanceTimersByTimeAsync(0); + if (source === 'event') sockets[0].emit('error', new Error('UDP unavailable')); + else sockets[0].send.mock.calls[0][5](new Error('UDP unavailable')); + await vi.advanceTimersByTimeAsync(250); + expect(sockets).toHaveLength(2); + reply(1); + expect(await request).toMatchObject({ ok: true }); + }); + + it.each([ + ['missing prefix', Buffer.from('wrong listener')], + ['wrong password', packet('print\nBad rcon_password')], + ['unknown dialect', packet('unexpected reply')] + ])('does not retry a %s reply', async (_name, response) => { + const request = rconStatusResult(server.name); + await vi.advanceTimersByTimeAsync(0); + sockets[0].emit('message', response); + expect(await request).toMatchObject({ ok: false, reason: 'bad-reply' }); + expect(sockets).toHaveLength(1); + }); + + it('does not retry missing configuration or endpoint failures', async () => { + vi.mocked(rconConfigured).mockReturnValue(false); + expect(await rconStatusResult(server.name)).toMatchObject({ ok: false, reason: 'no-config' }); + server.game_port = null; + expect(await rconStatusResult(server.name)).toMatchObject({ ok: false, reason: 'no-port' }); + vi.mocked(getServer).mockReturnValue(undefined); + expect(await rconStatusResult(server.name)).toMatchObject({ ok: false, reason: 'no-port' }); + expect(sockets).toHaveLength(0); + }); + + it('cleans up a rejected in-flight probe so the next one can proceed', async () => { + vi.mocked(createSocket).mockImplementationOnce(() => { throw new Error('Socket initialization failed'); }); + await expect(rconStatusResult(server.name)).rejects.toThrow('Socket initialization failed'); + const request = rconStatusResult(server.name); + await vi.advanceTimersByTimeAsync(0); + reply(); + expect(await request).toMatchObject({ ok: true }); + }); + + it.each(['restart', 'registration', 'host', 'port', 'server'])('does not share probes across a changed %s', async change => { + const original = rconStatusResult(server.name); + await vi.advanceTimersByTimeAsync(0); + if (change === 'restart') vi.mocked(latestGameSubmission).mockReturnValue('2026-10-02T00:00:00Z'); + if (change === 'registration') server.created_at = '2026-10-02T00:00:00Z'; + if (change === 'host') server.rcon_host = 'new-host'; + if (change === 'port') server.game_port = 28961; + if (change === 'server') server.name = 'other-server'; + const current = rconStatusResult(server.name); + await vi.advanceTimersByTimeAsync(0); + expect(sockets).toHaveLength(2); + reply(); + sockets[1].emit('message', packet('statusResponse\n\\mapname\\mp_ascension\n')); + expect(await original).toMatchObject({ ok: true, status: { players: 1 } }); + expect(await current).toMatchObject({ ok: true, status: { players: 0 } }); + }); + + it('leaves other RCON commands on their existing single-attempt behavior', async () => { + const request = rconPlutoniumVersion(server.name); + await vi.advanceTimersByTimeAsync(1_000); + expect(await request).toMatchObject({ revision: null, error: 'no reply within 1000ms' }); + expect(sockets).toHaveLength(1); + }); +}); diff --git a/tests/unit/server-status.test.ts b/tests/unit/server-status.test.ts index 37208d1..0374f8f 100644 --- a/tests/unit/server-status.test.ts +++ b/tests/unit/server-status.test.ts @@ -30,6 +30,110 @@ async function setup() { } describe('database fleet and container readiness', () => { + it('refreshes a ten-server fleet while one check is slow, without duplicating or exceeding four checks', async () => { + const { db, inspect, rcon, fanoutStatus, getServer } = await setup(); + const { subscribe } = await import('../../src/lib/server/pipeline/bus.js'); + vi.useFakeTimers({ toFake: ['Date'] }); + const now = Date.parse('2026-10-02T12:00:00Z'); + vi.setSystemTime(now); + for (let i = 0; i < 9; i++) db.prepare('INSERT INTO servers (name, komodo_name) VALUES (?, ?)').run(`fast-${i}`, `fast-${i}`); + let release!: (result: { ok: true; state: string }) => void; + let active = 0; + let peak = 0; + inspect.mockImplementation(async name => { + peak = Math.max(peak, ++active); + try { + await Promise.resolve(); + return name === 'managed' + ? await new Promise<{ ok: true; state: string }>(resolve => { release = resolve; }) + : { ok: true as const, state: 'running' }; + } finally { active--; } + }); + rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status', + status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } }); + const events: unknown[] = []; + const unsubscribe = subscribe(event => events.push(event)); + const first = fanoutStatus(); + try { + await vi.waitFor(() => expect(rcon).toHaveBeenCalledTimes(9)); + expect(events).toContainEqual(expect.objectContaining({ type: 'server-status', entries: expect.arrayContaining([ + expect.objectContaining({ server: 'fast-8', health: 'online', players: 2 }) + ]) })); + vi.setSystemTime(now + 30_000); + const second = fanoutStatus(); + await vi.waitFor(() => expect(Date.parse(getServer('fast-8')?.last_seen_at ?? '')).toBeGreaterThanOrEqual(now + 30_000)); + expect(rcon).toHaveBeenCalledTimes(18); + expect(inspect.mock.calls.filter(([name]) => name === 'managed')).toHaveLength(1); + expect(peak).toBe(4); + release({ ok: true, state: 'running' }); + await Promise.all([first, second]); + expect(rcon).toHaveBeenCalledTimes(19); + } finally { + release({ ok: true, state: 'running' }); + await first; + unsubscribe(); + } + }); + + it('expires early results before recording a fleet collection delayed by a slow check', async () => { + const { db, inspect, rcon, fanoutStatus } = await setup(); + vi.useFakeTimers({ toFake: ['Date'] }); + const now = Date.parse('2026-10-02T12:00:00Z'); + vi.setSystemTime(now); + db.prepare("INSERT INTO servers (name, komodo_name) VALUES ('slow', 'slow')").run(); + let release!: (result: { ok: true; state: string }) => void; + inspect.mockImplementation(async name => name === 'slow' + ? await new Promise<{ ok: true; state: string }>(resolve => { release = resolve; }) + : { ok: true as const, state: 'running' }); + rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status', + status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } }); + const pending = fanoutStatus(); + try { + await vi.waitFor(() => expect(rcon).toHaveBeenCalledTimes(1)); + vi.setSystemTime(now + 46_000); + } finally { + release({ ok: true, state: 'running' }); + await pending; + } + expect(db.prepare('SELECT server, players FROM metric_samples ORDER BY server').all()).toEqual([ + { server: 'managed', players: null }, { server: 'slow', players: 2 } + ]); + }); + + it('keeps a manual probe diagnostic without persisting it across a restart', async () => { + const { db, rcon, getServer } = await setup(); + const { POST } = await import('../../src/routes/api/servers/[name]/probe-rcon/+server.js'); + let release!: (result: Awaited>) => void; + rcon.mockImplementation(() => new Promise(resolve => { release = resolve; })); + const pending = POST({ params: { name: 'managed' } } as never); + db.prepare("INSERT INTO restart_rollout (server, requested_at) VALUES ('managed', ?)").run(new Date().toISOString()); + release({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'old status', + status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } }); + expect(await (await pending).json()).toMatchObject({ ok: true }); + expect(getServer('managed')).toMatchObject({ last_seen_at: null, last_error: 'timeout:old process' }); + }); + + it.each(['container', 'rcon'])('ignores an old registration whose %s check finishes after replacement', async phase => { + const { db, inspect, rcon, fanoutStatus, getServer } = await setup(); + let release!: () => void; + const blocked = new Promise(resolve => { release = resolve; }); + inspect.mockImplementation(async () => { + if (phase === 'container') await blocked; + return { ok: true, state: 'running' }; + }); + rcon.mockImplementation(async () => { + if (phase === 'rcon') await blocked; + return { ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'old status', + status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } }; + }); + const pending = fanoutStatus(); + try { + if (phase === 'rcon') await vi.waitFor(() => expect(rcon).toHaveBeenCalled()); + db.prepare("UPDATE servers SET created_at = 'replacement', container_state = NULL, last_error = NULL WHERE name = 'managed'").run(); + } finally { release(); await pending; } + expect(getServer('managed')).toMatchObject({ container_state: null, last_seen_at: null, last_error: null }); + }); + it.each(['queued', 'running', 'awaiting-health'])('shows provisioning during %s creation and resumes health reporting afterward', async (state) => { const { db, inspect, rcon, fanoutStatus } = await setup(); const { GET } = await import('../../src/routes/api/servers/+server.js'); diff --git a/tests/unit/watchdog.test.ts b/tests/unit/watchdog.test.ts index bf1f1be..580af17 100644 --- a/tests/unit/watchdog.test.ts +++ b/tests/unit/watchdog.test.ts @@ -153,6 +153,105 @@ describe('Watchdog detection and ownership', () => { expect(operations()).toEqual([]); }); + it.each(['no-config', 'no-port', 'bad-reply', 'no-players'] as const)('keeps %s RCON failures observational', async reason => { + vi.mocked(rcon.rconStatusResult).mockResolvedValue({ ...missingRcon, reason }); + for (let n = 0; n < 4; n++) { await watchdog.watchdogTick(true); advance(watchdog.WATCHDOG_SCAN_MS); } + expect(onlyIncident()).toMatchObject({ state: 'open' }); + expect(operations()).toEqual([]); + }); + + it('breaks the sustained failure window on a successful probe between watchdog samples', async () => { + vi.mocked(rcon.rconStatusResult).mockResolvedValue(missingRcon); + await watchdog.watchdogTick(true); + advance(watchdog.WATCHDOG_SCAN_MS); + await watchdog.watchdogTick(true); + advance(20_000); + const { POST } = await import('../../src/routes/api/servers/[name]/probe-rcon/+server.js'); + vi.mocked(rcon.rconStatusResult).mockResolvedValueOnce(healthyRcon); + await POST({ params: { name: server } } as Parameters[0]); + advance(40_000); + await watchdog.watchdogTick(true); + expect(onlyIncident()).toMatchObject({ failures: 1, state: 'open', first_seen_at: new Date().toISOString() }); + expect(operations()).toEqual([]); + advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true); + expect(operations()).toEqual([]); + advance(watchdog.WATCHDOG_SCAN_MS); await watchdog.watchdogTick(true); + expect(operations()).toEqual([expect.objectContaining({ action: 'restart' })]); + }); + + it.each([0, 2])('cancels queued health recovery when fresh RCON responds with %i players', async players => { + const row = await sustainedFailure(); + vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); + vi.mocked(rcon.rconGetPlayerCount).mockResolvedValue(players); + const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); + await processProvisioning(); + expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'cancelled' })]); + expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved'); + expect(komodo.komodoRestart).not.toHaveBeenCalled(); + }); + + it('holds queued health recovery when container readiness becomes unknown', async () => { + await sustainedFailure(); + vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'Komodo timed out' }); + const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); + await processProvisioning(); + expect(operations()[0].state).toBe('waiting'); + expect(komodo.komodoRestart).not.toHaveBeenCalled(); + }); + + it('does not apply a recovered occupancy probe after its operation is replaced', async () => { + const row = await sustainedFailure(); + const restart = operations()[0]; + const { cancelProvisioning, requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); + vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); + vi.mocked(rcon.rconGetPlayerCount).mockImplementationOnce(async () => { + expect(cancelProvisioning(restart.id)).toBe(true); + expect(requestLifecycle(server, 'stop').ok).toBe(true); + return 0; + }); + const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); + await processProvisioning(); + expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('recovering'); + expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'cancelled' }), + expect.objectContaining({ action: 'stop', state: 'waiting' })]); + expect(komodo.komodoRestart).not.toHaveBeenCalled(); + }); + + it('cancels obsolete health recovery so a later queued maintenance restart can proceed', async () => { + const row = await sustainedFailure(); + const { requestRestart } = await import('../../src/lib/server/provisioning/requests.js'); + const job = database.db().prepare('INSERT INTO maintenance_jobs (server, requested_at) VALUES (?, ?)') + .run(server, new Date().toISOString()).lastInsertRowid; + const maintenance = requestRestart(server, { kind: 'maintenance', id: String(job) }); + vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); + const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); + await processProvisioning(); + expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved'); + expect(operations()).toEqual([expect.objectContaining({ state: 'cancelled' }), + expect.objectContaining({ id: maintenance.id, state: 'awaiting-health' })]); + expect(komodo.komodoRestart).toHaveBeenCalledTimes(1); + }); + + it.each(['restart', 'rebuild'] as const)('preserves an operator request adopting a queued health %s', async action => { + await sustainedFailure(); + const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); + if (action === 'rebuild') { + await processProvisioning(); + advance(watchdog.WATCHDOG_RECOVERY_MS); + await watchdog.watchdogTick(true); + } + const queued = operations().at(-1)!; + const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); + expect(requestLifecycle(server, action)).toEqual({ ok: true, deploymentId: queued.id }); + expect(database.db().prepare('SELECT triggered_by FROM server_deployments WHERE id = ?').get(queued.id)).toEqual({ triggered_by: 'ui' }); + vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); + if (action === 'rebuild') vi.mocked(komodo.komodoDestroyDeployment).mockImplementationOnce(async () => { down(); return { ok: true }; }); + await processProvisioning(); + expect(operations().at(-1)).toMatchObject({ id: queued.id, action, state: action === 'restart' ? 'awaiting-health' : 'queued' }); + if (action === 'restart') expect(komodo.komodoRestart).toHaveBeenCalledTimes(1); + else expect(komodo.komodoDestroyDeployment).toHaveBeenCalledTimes(1); + }); + it('preserves intentional stop intent after completion and database reopening', async () => { const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js'); const stop = requestLifecycle(server, 'stop'); @@ -241,6 +340,79 @@ describe('Watchdog detection and ownership', () => { }); describe('Watchdog recovery persistence', () => { + it('verifies recovery at the deadline before deciding whether to rebuild', async () => { + const row = await sustainedFailure(); + const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); + await processProvisioning(); + advance(watchdog.WATCHDOG_RECOVERY_MS); + vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); + await watchdog.watchdogTick(true); + expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'done' })]); + expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved'); + expect(database.db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server)) + .toEqual({ healthy_at: new Date().toISOString() }); + }); + + it('defers escalation while current container health is unknown', async () => { + await sustainedFailure(); + const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); + await processProvisioning(); + advance(watchdog.WATCHDOG_RECOVERY_MS); + vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: false, reason: 'Komodo timed out' }); + await watchdog.watchdogTick(true); + expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'awaiting-health' })]); + }); + + it('keeps failed submission history when fresh evidence later releases recovery', async () => { + const row = await sustainedFailure(); + const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); + vi.mocked(komodo.komodoRestart).mockResolvedValueOnce({ ok: false, kind: 'unreachable', message: 'Reply lost' }); + await processProvisioning(); + advance(watchdog.WATCHDOG_RECOVERY_MS); + vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); + await watchdog.watchdogTick(true); + expect(operations()).toEqual([expect.objectContaining({ action: 'restart', state: 'failed' })]); + expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved'); + expect(database.db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server)) + .toEqual({ healthy_at: new Date().toISOString() }); + expect(watchdog.watchdogDetail(row.id)?.actions[0].operation_error).toContain('Reply lost'); + }); + + it('discards escalation evidence if another lifecycle submission appears during its probe', async () => { + const row = await sustainedFailure(); + const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); + await processProvisioning(); + advance(watchdog.WATCHDOG_RECOVERY_MS); + vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); + vi.mocked(rcon.rconStatusResult).mockImplementationOnce(async () => { + database.db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, submitted_at, triggered_by) + VALUES ('newer', ?, 'restart', 'done', 'Complete', ?, ?, 'manual')`).run(server, new Date().toISOString(), new Date().toISOString()); + return healthyRcon; + }); + await watchdog.watchdogTick(true); + expect(operations()[0]).toMatchObject({ action: 'restart', state: 'awaiting-health' }); + expect(operations().some(op => op.action === 'rebuild')).toBe(false); + expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('recovering'); + expect(database.db().prepare('SELECT healthy_at FROM restart_rollout WHERE server = ?').get(server)).toEqual({ healthy_at: null }); + }); + + it('cancels a queued health rebuild and releases its transferred slot when the game recovers', async () => { + const row = await sustainedFailure(); + const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); + await processProvisioning(); + advance(watchdog.WATCHDOG_RECOVERY_MS); + await watchdog.watchdogTick(true); + expect(operations().map(op => op.action)).toEqual(['restart', 'rebuild']); + const rebuild = operations()[1]; + vi.mocked(komodo.komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' }); + await processProvisioning(); + expect(operations()).toEqual([expect.objectContaining({ state: 'failed' }), expect.objectContaining({ state: 'cancelled' })]); + expect(watchdog.watchdogDetail(row.id)?.incident.state).toBe('resolved'); + expect(database.db().prepare('SELECT operation_id, healthy_at FROM restart_rollout WHERE server = ?').get(server)) + .toEqual({ operation_id: rebuild.id, healthy_at: new Date().toISOString() }); + expect(komodo.komodoDestroyDeployment).not.toHaveBeenCalled(); + }); + it('preserves the original finding as deleted history after an intentional successful destroy', async () => { const { removeServerKey } = await import('../../src/lib/server/fleet/server-keys.js'); const { destroyServer, processDestructions } = await import('../../src/lib/server/provisioning/instances.js');