122 lines
7.3 KiB
TypeScript
122 lines
7.3 KiB
TypeScript
import { db } from '../db/index.js';
|
|
import { config } from '../config.js';
|
|
import { getServer, listServers } from '../fleet/servers.js';
|
|
import { hasActiveProvisioning } from '../fleet/provisioning.js';
|
|
import { latestGameSubmission } from '../fleet/status-evidence.js';
|
|
import { latestGameSubmissions } from '../fleet/lifecycle-evidence.js';
|
|
import { rconPlutoniumVersion } from './rcon.js';
|
|
import { nativeHealth } from '../backends/native.js';
|
|
import { restartRecoveryPending } from '../provisioning/index.js';
|
|
import { parsePlutoniumManifest, PLUTONIUM_MANIFEST_URL } from '../../plutonium.js';
|
|
import type { PlutoniumRelease, PlutoniumVersion } from '../../plutonium.js';
|
|
import { readSetting, writeSetting } from '../maintenance/settings.js';
|
|
export const HOUR = 60 * 60 * 1000;
|
|
const nowIso = () => new Date().toISOString();
|
|
export function release(): PlutoniumRelease {
|
|
return readSetting('plutonium_release', { revision: null, checkedAt: null, error: null });
|
|
}
|
|
export function freshRelease(): number | null {
|
|
const latest = release();
|
|
const age = latest.checkedAt ? Date.now() - Date.parse(latest.checkedAt) : NaN;
|
|
return !latest.error && age >= 0 && age < HOUR ? latest.revision : null;
|
|
}
|
|
export function runningVersions(serverName?: string): PlutoniumVersion[] {
|
|
const bindings = serverName === undefined ? [] : [serverName];
|
|
const versions = db().prepare(`SELECT * FROM plutonium_versions ${serverName === undefined ? '' : 'WHERE server = ?'}`)
|
|
.all(...bindings) as PlutoniumVersion[];
|
|
const target = freshRelease();
|
|
const byServer = new Map(versions.map(version => [version.server, version]));
|
|
const epochs = serverName === undefined ? latestGameSubmissions() : new Map([[serverName, latestGameSubmission(serverName)]]);
|
|
const recovering = new Set((db().prepare(`SELECT server FROM restart_rollout WHERE healthy_at IS NULL ${serverName === undefined ? '' : 'AND server = ?'}`)
|
|
.all(...bindings) as { server: string }[]).map(row => row.server));
|
|
const selected = serverName === undefined ? undefined : getServer(serverName);
|
|
const servers = serverName === undefined ? listServers() : selected && !selected.destroyed_at ? [selected] : [];
|
|
return servers.map(server => {
|
|
const value = byServer.get(server.name)
|
|
?? { server: server.name, revision: null, checked_at: null, error: null };
|
|
const age = value.checked_at ? Date.now() - Date.parse(value.checked_at) : NaN;
|
|
const epoch = epochs.get(server.name);
|
|
const stopped = server.desired_state === 'stopped';
|
|
const valid = !value.error && age >= 0 && age < HOUR && (!epoch || value.checked_at! > epoch) && !recovering.has(server.name);
|
|
const revision = valid && !stopped ? value.revision : null;
|
|
return { ...value, revision, stopped,
|
|
update_status: stopped ? 'stopped' : revision === null || target === null ? 'unknown'
|
|
: revision < target ? 'outdated' : revision > target ? 'ahead' : 'current' };
|
|
});
|
|
}
|
|
export async function probeVersion(server: string, options: { maintenance?: boolean; operationId?: string } = {}): Promise<number | null> {
|
|
// Startup verification calls this directly; only maintenance observers yield
|
|
// ownership, without replacing provisioning's version evidence with an error.
|
|
const suppressed = () => (options.maintenance && hasActiveProvisioning(server)) || (options.operationId !== undefined
|
|
&& !db().prepare(`SELECT 1 FROM server_deployments WHERE id=? AND server=? AND server_deleted_at IS NULL
|
|
AND action IN ('create','rebuild') AND state='awaiting-health' AND submitted_at IS NOT NULL`).get(options.operationId, server));
|
|
// The verifying operation keeps its recovery slot until every readiness
|
|
// check passes. Waiting on its own slot would deadlock native rebuilds.
|
|
const recoveryBlocked = () => restartRecoveryPending(server) && !(options.operationId
|
|
&& db().prepare('SELECT 1 FROM restart_rollout WHERE server=? AND operation_id=? AND healthy_at IS NULL').get(server, options.operationId));
|
|
if (suppressed()) return null;
|
|
const original = getServer(server);
|
|
if (!original) return null;
|
|
const epoch = latestGameSubmission(server);
|
|
const recovering = recoveryBlocked();
|
|
const health = !recovering && original.desired_state !== 'stopped'
|
|
? await nativeHealth(server).catch(() => ({ ok: false })) : { ok: false };
|
|
if (suppressed()) return null;
|
|
let result = !health.ok ? { revision: null, error: 'Waiting for a healthy running server.' }
|
|
: await rconPlutoniumVersion(server).catch(() => ({ revision: null, error: 'Version check failed.' }));
|
|
if (suppressed()) return null;
|
|
const current = getServer(server);
|
|
// An older probe must not clear evidence already recorded for a new process
|
|
// or replacement registration, including replacements with no submission yet.
|
|
if (!current || current.created_at !== original.created_at || epoch !== latestGameSubmission(server)) return null;
|
|
const checkedAt = nowIso();
|
|
if (recoveryBlocked()) {
|
|
result = { revision: null, error: 'Waiting for restart recovery.' };
|
|
} else if (current.desired_state === 'stopped') {
|
|
result = { revision: null, error: 'Server is intentionally stopped.' };
|
|
} else if (epoch && checkedAt <= epoch) {
|
|
result = { revision: null, error: 'Waiting for a version observation after startup.' };
|
|
}
|
|
// Clear the old revision on failure: it is no longer evidence of what is running.
|
|
db().prepare(`INSERT INTO plutonium_versions (server, revision, checked_at, error) VALUES (?, ?, ?, ?)
|
|
ON CONFLICT(server) DO UPDATE SET revision = excluded.revision, checked_at = excluded.checked_at, error = excluded.error`)
|
|
.run(server, result.revision, checkedAt, result.error);
|
|
return result.revision;
|
|
}
|
|
|
|
let checking: Promise<void> | null = null;
|
|
export function checkPlutonium(): Promise<void> {
|
|
if (checking) return checking;
|
|
checking = check().finally(() => { checking = null; });
|
|
return checking;
|
|
}
|
|
async function check(): Promise<void> {
|
|
await checkRelease();
|
|
const servers = listServers();
|
|
for (let i = 0; i < servers.length; i += 4) {
|
|
await Promise.all(servers.slice(i, i + 4).map(s => probeVersion(s.name, { maintenance: true })));
|
|
}
|
|
writeSetting('plutonium_versions_checked_at', nowIso());
|
|
}
|
|
|
|
let checkingRelease: Promise<void> | null = null;
|
|
/** Refresh the published target without probing unrelated fleet servers. */
|
|
export function checkRelease(): Promise<void> {
|
|
if (checkingRelease) return checkingRelease;
|
|
checkingRelease = fetchRelease().finally(() => { checkingRelease = null; });
|
|
return checkingRelease;
|
|
}
|
|
|
|
async function fetchRelease(): Promise<void> {
|
|
const previous = release();
|
|
try {
|
|
const response = await fetch(PLUTONIUM_MANIFEST_URL, { signal: AbortSignal.timeout(config.httpTimeoutMs), redirect: 'error' });
|
|
if (!response.ok) throw new Error(`Updater manifest request failed (HTTP ${response.status}).`);
|
|
const body = await response.text();
|
|
if (body.length > 2_000_000) throw new Error('Updater manifest response was too large.');
|
|
writeSetting('plutonium_release', { revision: parsePlutoniumManifest(JSON.parse(body)), checkedAt: nowIso(), error: null });
|
|
} catch (err) {
|
|
writeSetting('plutonium_release', { ...previous, checkedAt: nowIso(), error: (err as Error).message });
|
|
}
|
|
}
|