import { db } from '../db/index.js'; import { config } from '../config.js'; import { getServer, listServers } from '../fleet/servers.js'; import { hasActiveProvisioning } from '../fleet/provisioning.js'; import { latestGameSubmission } from '../fleet/status-evidence.js'; import { latestGameSubmissions } from '../fleet/lifecycle-evidence.js'; import { rconPlutoniumVersion } from './rcon.js'; import { nativeHealth } from '../backends/native.js'; import { restartRecoveryPending } from '../provisioning/index.js'; import { parsePlutoniumManifest, PLUTONIUM_MANIFEST_URL } from '../../plutonium.js'; import type { PlutoniumRelease, PlutoniumVersion } from '../../plutonium.js'; import { readSetting, writeSetting } from '../maintenance/settings.js'; export const HOUR = 60 * 60 * 1000; const nowIso = () => new Date().toISOString(); export function release(): PlutoniumRelease { return readSetting('plutonium_release', { revision: null, checkedAt: null, error: null }); } export function freshRelease(): number | null { const latest = release(); const age = latest.checkedAt ? Date.now() - Date.parse(latest.checkedAt) : NaN; return !latest.error && age >= 0 && age < HOUR ? latest.revision : null; } export function runningVersions(serverName?: string): PlutoniumVersion[] { const bindings = serverName === undefined ? [] : [serverName]; const versions = db().prepare(`SELECT * FROM plutonium_versions ${serverName === undefined ? '' : 'WHERE server = ?'}`) .all(...bindings) as PlutoniumVersion[]; const target = freshRelease(); const byServer = new Map(versions.map(version => [version.server, version])); const epochs = serverName === undefined ? latestGameSubmissions() : new Map([[serverName, latestGameSubmission(serverName)]]); const recovering = new Set((db().prepare(`SELECT server FROM restart_rollout WHERE healthy_at IS NULL ${serverName === undefined ? '' : 'AND server = ?'}`) .all(...bindings) as { server: string }[]).map(row => row.server)); const selected = serverName === undefined ? undefined : getServer(serverName); const servers = serverName === undefined ? listServers() : selected && !selected.destroyed_at ? [selected] : []; return servers.map(server => { const value = byServer.get(server.name) ?? { server: server.name, revision: null, checked_at: null, error: null }; const age = value.checked_at ? Date.now() - Date.parse(value.checked_at) : NaN; const epoch = epochs.get(server.name); const stopped = server.desired_state === 'stopped'; const valid = !value.error && age >= 0 && age < HOUR && (!epoch || value.checked_at! > epoch) && !recovering.has(server.name); const revision = valid && !stopped ? value.revision : null; return { ...value, revision, stopped, update_status: stopped ? 'stopped' : revision === null || target === null ? 'unknown' : revision < target ? 'outdated' : revision > target ? 'ahead' : 'current' }; }); } export async function probeVersion(server: string, options: { maintenance?: boolean; operationId?: string } = {}): Promise { // Startup verification calls this directly; only maintenance observers yield // ownership, without replacing provisioning's version evidence with an error. const suppressed = () => (options.maintenance && hasActiveProvisioning(server)) || (options.operationId !== undefined && !db().prepare(`SELECT 1 FROM server_deployments WHERE id=? AND server=? AND server_deleted_at IS NULL AND action IN ('create','rebuild') AND state='awaiting-health' AND submitted_at IS NOT NULL`).get(options.operationId, server)); // The verifying operation keeps its recovery slot until every readiness // check passes. Waiting on its own slot would deadlock native rebuilds. const recoveryBlocked = () => restartRecoveryPending(server) && !(options.operationId && db().prepare('SELECT 1 FROM restart_rollout WHERE server=? AND operation_id=? AND healthy_at IS NULL').get(server, options.operationId)); if (suppressed()) return null; const original = getServer(server); if (!original) return null; const epoch = latestGameSubmission(server); const recovering = recoveryBlocked(); const health = !recovering && original.desired_state !== 'stopped' ? await nativeHealth(server).catch(() => ({ ok: false })) : { ok: false }; if (suppressed()) return null; let result = !health.ok ? { revision: null, error: 'Waiting for a healthy running server.' } : await rconPlutoniumVersion(server).catch(() => ({ revision: null, error: 'Version check failed.' })); if (suppressed()) return null; const current = getServer(server); // An older probe must not clear evidence already recorded for a new process // or replacement registration, including replacements with no submission yet. if (!current || current.created_at !== original.created_at || epoch !== latestGameSubmission(server)) return null; const checkedAt = nowIso(); if (recoveryBlocked()) { result = { revision: null, error: 'Waiting for restart recovery.' }; } else if (current.desired_state === 'stopped') { result = { revision: null, error: 'Server is intentionally stopped.' }; } else if (epoch && checkedAt <= epoch) { result = { revision: null, error: 'Waiting for a version observation after startup.' }; } // Clear the old revision on failure: it is no longer evidence of what is running. db().prepare(`INSERT INTO plutonium_versions (server, revision, checked_at, error) VALUES (?, ?, ?, ?) ON CONFLICT(server) DO UPDATE SET revision = excluded.revision, checked_at = excluded.checked_at, error = excluded.error`) .run(server, result.revision, checkedAt, result.error); return result.revision; } let checking: Promise | null = null; export function checkPlutonium(): Promise { if (checking) return checking; checking = check().finally(() => { checking = null; }); return checking; } async function check(): Promise { await checkRelease(); const servers = listServers(); for (let i = 0; i < servers.length; i += 4) { await Promise.all(servers.slice(i, i + 4).map(s => probeVersion(s.name, { maintenance: true }))); } writeSetting('plutonium_versions_checked_at', nowIso()); } let checkingRelease: Promise | null = null; /** Refresh the published target without probing unrelated fleet servers. */ export function checkRelease(): Promise { if (checkingRelease) return checkingRelease; checkingRelease = fetchRelease().finally(() => { checkingRelease = null; }); return checkingRelease; } async function fetchRelease(): Promise { const previous = release(); try { const response = await fetch(PLUTONIUM_MANIFEST_URL, { signal: AbortSignal.timeout(config.httpTimeoutMs), redirect: 'error' }); if (!response.ok) throw new Error(`Updater manifest request failed (HTTP ${response.status}).`); const body = await response.text(); if (body.length > 2_000_000) throw new Error('Updater manifest response was too large.'); writeSetting('plutonium_release', { revision: parsePlutoniumManifest(JSON.parse(body)), checkedAt: nowIso(), error: null }); } catch (err) { writeSetting('plutonium_release', { ...previous, checkedAt: nowIso(), error: (err as Error).message }); } }