import { config } from '../config.js'; import { processProvisioning } from '../provisioning/worker.js'; import { sweep } from './sweep.js'; import { betaTick } from './beta-runner.js'; import { emit } from './bus.js'; import type { ServerStatusEntry } from './bus.js'; import { rconStatusResult } from '../integrations/rcon.js'; import type { RconFailure } from '../integrations/rcon.js'; import { getServer, listServers, setServerField, type ServerRow } from '../fleet/servers.js'; import { komodoInspectHealthy } from '../integrations/komodo.js'; import { restartSettling } from '../provisioning/index.js'; import { serverObservation } from '../fleet/status.js'; import { telemetryEvidence, latestGameSubmission } from '../fleet/status-evidence.js'; import { isProvisioning } from '../fleet/provisioning.js'; import { STATUS_REFRESH_MS } from '../../server-freshness.js'; import { recordMetrics } from '../fleet/metrics.js'; import { selectedZombieMap } from '../../map-aliases.js'; import { runMaintenance } from '../maintenance/index.js'; const TICK_MS = 15_000; // Cap concurrent UDP RCON calls so a 15-server fleet doesn't blast the // host all at once. const STATUS_CONCURRENCY = 4; let started = false; let tickTimer: NodeJS.Timeout | null = null; let statusTimer: NodeJS.Timeout | null = null; let tickInFlight: Promise | null = null; function tick(): Promise { if (tickInFlight) return tickInFlight; tickInFlight = runTick().finally(() => { tickInFlight = null; }); return tickInFlight; } async function reconcileRequesters(): Promise { try { await sweep(); } catch (err) { console.error('[worker] campaign error:', err); } try { await betaTick(); } catch (err) { console.error('[worker] beta error:', err); } try { await runMaintenance(); } catch (err) { console.error('[worker] maintenance error:', err); } } async function runTick(): Promise { // Callers submit durable requests first. Exactly one owner admits and runs // lifecycle work, then callers observe the resulting jobs by ID. await reconcileRequesters(); try { await processProvisioning(); } catch (err) { console.error('[worker] provisioning error:', err); } await reconcileRequesters(); } /** * Encode a structured RCON failure as `"reason:detail"` so `last_error` * stays a single column but the UI can split it apart when rendering the * diagnostic panel. Reasons are stable; details are human-facing. */ function encodeError(reason: RconFailure, detail: string): string { return `${reason}:${detail}`; } // Deduplicate queued/running checks per server. A slow server must not prevent // completed servers from being polled again on the next 30-second tick. const pendingStatus = new Map>(); const statusQueue: Array<() => Promise> = []; let activeStatus = 0; function drainStatusQueue(): void { while (activeStatus < STATUS_CONCURRENCY && statusQueue.length) { const next = statusQueue.shift()!; activeStatus++; void next().finally(() => { activeStatus--; drainStatusQueue(); }); } } function queueStatus(server: string): Promise { const pending = pendingStatus.get(server); if (pending) return pending; let finish!: (capacity: number | null) => void; const work = new Promise(resolve => { finish = resolve; }); pendingStatus.set(server, work); statusQueue.push(async () => { let capacity: number | null = null; try { capacity = await refreshServerStatus(server); // Consumers expect a full snapshot. Re-read it now so neither game // data nor container evidence waits for the slowest fleet member. emit({ type: 'server-status', entries: listServers().map(statusEntry) }); } catch (err) { console.error(`[worker] status error for ${server}:`, err); } finally { pendingStatus.delete(server); finish(capacity); } }); drainStatusQueue(); return work; } export async function fanoutStatus(): Promise { const servers = listServers(); if (!servers.length) return; const results = await Promise.all(servers.map(server => queueStatus(server.name))); const capacities = new Map(servers.map((server, i) => [server.name, results[i]])); try { // Metrics remain complete fleet collections, using freshness at capture // time rather than the status saved at the beginning of a long scan. recordMetrics(listServers().map(srv => { const current = serverObservation(srv); return { server: srv.name, game: srv.game ?? 'unknown', map: selectedZombieMap(srv.game, current.last_map ?? srv.map, srv.map_slug)?.friendly ?? current.last_map ?? srv.map ?? 'Unknown map', players: current.health === 'online' ? current.last_players : null, capacity: capacities.get(srv.name) ?? null }; })); } catch (err) { console.error('[worker] metrics error:', err); } } async function refreshServerStatus(server: string): Promise { const srv = getServer(server); if (!srv) return null; let capacity: number | null = null; if (!isProvisioning(srv.name)) { const epoch = latestGameSubmission(srv.name); const unchanged = () => epoch === latestGameSubmission(srv.name) && !isProvisioning(srv.name) && getServer(srv.name)?.created_at === srv.created_at; const container = restartSettling(srv.name) ? { ok: false as const, state: 'restarting', reason: 'Restart requested' } : await komodoInspectHealthy(srv.name).catch(err => ({ ok: false as const, state: 'unknown', reason: (err as Error).message })); // A lifecycle request may be submitted while an inspection is in flight. if (unchanged()) { setServerField(srv.name, 'container_state', container.ok ? 'running' : container.state?.toLowerCase() ?? 'unknown'); setServerField(srv.name, 'container_checked_at', new Date().toISOString()); if (!container.ok) { setServerField(srv.name, 'last_error', null); setServerField(srv.name, 'last_reply_snippet', null); } else if (!telemetryEvidence(srv.name)) { const result = await rconStatusResult(srv.name).catch(err => ({ ok: false as const, reason: 'send-error' as RconFailure, detail: (err as Error).message, rawReply: undefined })); if (unchanged()) { const now = new Date().toISOString(); setServerField(srv.name, 'last_checked_at', now); setServerField(srv.name, 'last_reply_snippet', result.rawReply ?? null); if (result.ok) { capacity = result.status.capacity ?? null; setServerField(srv.name, 'last_players', result.status.players); setServerField(srv.name, 'last_map', result.status.map); setServerField(srv.name, 'last_gametype', result.status.gametype); setServerField(srv.name, 'last_roster', JSON.stringify(result.status.roster)); setServerField(srv.name, 'last_seen_at', now); setServerField(srv.name, 'last_error', null); } else setServerField(srv.name, 'last_error', encodeError(result.reason, result.detail)); } } } } else { setServerField(srv.name, 'last_error', null); setServerField(srv.name, 'last_reply_snippet', null); } const current = getServer(srv.name) ?? srv; const status = serverObservation(current); // Occupancy timers and match clearing follow the same source as the UI. if (status.health === 'online' && status.last_players !== null) { if (status.last_players > 0) { setServerField(srv.name, 'active_since', current.active_since ?? status.last_seen_at); setServerField(srv.name, 'inactive_since', null); } else { setServerField(srv.name, 'active_since', null); setServerField(srv.name, 'inactive_since', current.inactive_since ?? status.last_seen_at); setServerField(srv.name, 'game_started_at', null); setServerField(srv.name, 'game_ended_at', null); } } return capacity; } function statusEntry(srv: ServerRow): ServerStatusEntry { const final = serverObservation(srv); return { server: srv.name, players: final.last_players, map: final.last_map ?? srv.map, gametype: final.last_gametype, roster: final.roster, health: final.health, lastSeenAt: final.last_seen_at, lastError: final.last_error, replySnippet: final.replySnippet, endpoint: srv.game_port ? { host: srv.rcon_host ?? config.rcon.host, port: srv.game_port } : null, reason: final.last_error ? final.last_error.split(':')[0] as RconFailure : null, activeSince: final.active_since, inactiveSince: final.inactive_since, gameStartedAt: final.game_started_at, gameEndedAt: final.game_ended_at, containerState: final.container_state, containerCheckedAt: final.container_checked_at, telemetryAfter: final.telemetry_after, rconStatus: final.rcon_status, telemetry: final.telemetry }; } export function startWorker(): void { if (started) return; started = true; void tick(); void fanoutStatus(); tickTimer = setInterval(tick, TICK_MS); statusTimer = setInterval(fanoutStatus, STATUS_REFRESH_MS); console.log('[worker] started — sweep every', TICK_MS, 'ms, status every', STATUS_REFRESH_MS, 'ms'); } export function stopWorker(): void { if (tickTimer) clearInterval(tickTimer); if (statusTimer) clearInterval(statusTimer); tickTimer = null; statusTimer = null; started = false; } /** * External trigger — called from `/api/push` and beta endpoints so a sweep * fires immediately after a state change instead of waiting for the next tick. */ export async function kick(): Promise { await tick(); }