feat: unify recovery, updates, and schedules under Maintenance

src/lib/server/integrations/plutonium.ts

@@ -0,0 +1,76 @@
+import { db } from '../db/index.js';
+import { config } from '../config.js';
+import { getServer, listServers } from '../fleet/servers.js';
+import { latestGameSubmission } from '../fleet/status-evidence.js';
+import { rconPlutoniumVersion } from './rcon.js';
+import { komodoInspectHealthy } from './komodo.js';
+import { restartRecoveryPending } from '../provisioning/index.js';
+import { parsePlutoniumManifest, PLUTONIUM_MANIFEST_URL } from '../../plutonium.js';
+import type { PlutoniumRelease, PlutoniumVersion } from '../../plutonium.js';
+import { readSetting, writeSetting } from '../maintenance/settings.js';
+export const HOUR = 60 * 60 * 1000;
+const nowIso = () => new Date().toISOString();
+export function release(): PlutoniumRelease {
+    return readSetting('plutonium_release', { revision: null, checkedAt: null, error: null });
+}
+export function freshRelease(): number | null {
+    const latest = release();
+    const age = latest.checkedAt ? Date.now() - Date.parse(latest.checkedAt) : NaN;
+    return !latest.error && age >= 0 && age < HOUR ? latest.revision : null;
+}
+export function runningVersions(): PlutoniumVersion[] {
+    const versions = db().prepare('SELECT * FROM plutonium_versions').all() as PlutoniumVersion[];
+    const target = freshRelease();
+    return listServers().map(server => {
+        const value = versions.find(v => v.server === server.name)
+            ?? { server: server.name, revision: null, checked_at: null, error: null };
+        const age = value.checked_at ? Date.now() - Date.parse(value.checked_at) : NaN;
+        const epoch = latestGameSubmission(server.name);
+        const stopped = server.desired_state === 'stopped';
+        const valid = !value.error && age >= 0 && age < HOUR && (!epoch || value.checked_at! > epoch) && !restartRecoveryPending(server.name);
+        const revision = valid && !stopped ? value.revision : null;
+        return { ...value, revision, stopped,
+            update_status: stopped ? 'stopped' : revision === null || target === null ? 'unknown'
+                : revision < target ? 'outdated' : revision > target ? 'ahead' : 'current' };
+    });
+}
+export async function probeVersion(server: string): Promise<number | null> {
+    const epoch = latestGameSubmission(server);
+    const recovering = restartRecoveryPending(server);
+    const health = !recovering && getServer(server)?.desired_state !== 'stopped'
+        ? await komodoInspectHealthy(server).catch(() => ({ ok: false })) : { ok: false };
+    let result = !health.ok ? { revision: null, error: 'Waiting for a healthy running server.' }
+        : await rconPlutoniumVersion(server).catch(() => ({ revision: null, error: 'Version check failed.' }));
+    if (!getServer(server)) return null;
+    if (epoch !== latestGameSubmission(server) || restartRecoveryPending(server)) {
+        result = { revision: null, error: 'Waiting for restart recovery.' };
+    }
+    // Clear the old revision on failure: it is no longer evidence of what is running.
+    db().prepare(`INSERT INTO plutonium_versions (server, revision, checked_at, error) VALUES (?, ?, ?, ?)
+        ON CONFLICT(server) DO UPDATE SET revision = excluded.revision, checked_at = excluded.checked_at, error = excluded.error`)
+        .run(server, result.revision, nowIso(), result.error);
+    return result.revision;
+}
+
+let checking: Promise<void> | null = null;
+export function checkPlutonium(): Promise<void> {
+    if (checking) return checking;
+    checking = check().finally(() => { checking = null; });
+    return checking;
+}
+async function check(): Promise<void> {
+    const previous = release();
+    try {
+        const response = await fetch(PLUTONIUM_MANIFEST_URL, { signal: AbortSignal.timeout(config.httpTimeoutMs), redirect: 'error' });
+        if (!response.ok) throw new Error(`Updater manifest request failed (HTTP ${response.status}).`);
+        const body = await response.text();
+        if (body.length > 2_000_000) throw new Error('Updater manifest response was too large.');
+        writeSetting('plutonium_release', { revision: parsePlutoniumManifest(JSON.parse(body)), checkedAt: nowIso(), error: null });
+    } catch (err) {
+        writeSetting('plutonium_release', { ...previous, checkedAt: nowIso(), error: (err as Error).message });
+    }
+    const servers = listServers();
+    for (let i = 0; i < servers.length; i += 4) {
+        await Promise.all(servers.slice(i, i + 4).map(s => probeVersion(s.name)));
+    }
+}