import { db } from '../db/index.js'; import { config, komodoConfigured, rconConfigured } from '../config.js'; import { getServer, listServers } from './servers.js'; import { rconGetPlayerCount, rconPlutoniumVersion } from '../integrations/rcon.js'; import { refreshRestartRollout, restartInRollout, rolloutHealth, rolloutWaitReason } from '../pipeline/restart-rollout.js'; import { nextMaintenanceAt, parsePlutoniumManifest, PLUTONIUM_MANIFEST_URL } from '../../plutonium.js'; import type { MaintenanceJob, MaintenanceSchedule, PlutoniumRelease, PlutoniumStatus, PlutoniumVersion } from '../../plutonium.js'; const HOUR = 60 * 60 * 1000; const nowIso = () => new Date().toISOString(); function readSetting(key: string, fallback: T): T { const row = db().prepare('SELECT value FROM settings WHERE key = ?').get(key) as { value: string } | undefined; return row ? JSON.parse(row.value) : fallback; } function writeSetting(key: string, value: unknown): void { db().prepare('INSERT INTO settings (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value') .run(key, JSON.stringify(value)); } function release(): PlutoniumRelease { return readSetting('plutonium_release', { revision: null, checkedAt: null, error: null }); } function schedule(): MaintenanceSchedule { // Legacy schedules were explicitly UTC. Preserve their timing until saved. return { timeZone: 'UTC', ...readSetting('plutonium_schedule', { enabled: false, time: '06:00', nextAt: null }) }; } export function saveMaintenanceSchedule(enabled: boolean, time: string, timeZone: string): void { const nextAt = nextMaintenanceAt(time, new Date(), timeZone); writeSetting('plutonium_schedule', { enabled, time, timeZone, nextAt: enabled ? nextAt : null }); } export function plutoniumStatus(): PlutoniumStatus { const versions = db().prepare('SELECT * FROM plutonium_versions').all() as PlutoniumVersion[]; return { release: release(), schedule: schedule(), servers: listServers().map(s => versions.find(v => v.server === s.name) ?? { server: s.name, revision: null, checked_at: null, error: null }), jobs: db().prepare(`SELECT * FROM maintenance_jobs WHERE state IN ('queued','restarting') OR id IN (SELECT id FROM maintenance_jobs ORDER BY id DESC LIMIT 100) ORDER BY id DESC`).all() as MaintenanceJob[] }; } async function probeVersion(server: string): Promise { const recovering = db().prepare('SELECT 1 FROM restart_rollout WHERE server = ? AND healthy_at IS NULL').get(server); const result = recovering ? { revision: null, error: 'Waiting for restart recovery.' } : await rconPlutoniumVersion(server).catch(() => ({ revision: null, error: 'Version check failed.' })); // Clear the old revision on failure: it is no longer evidence of what is running. db().prepare(`INSERT INTO plutonium_versions (server, revision, checked_at, error) VALUES (?, ?, ?, ?) ON CONFLICT(server) DO UPDATE SET revision = excluded.revision, checked_at = excluded.checked_at, error = excluded.error`) .run(server, result.revision, nowIso(), result.error); return result.revision; } let checking: Promise | null = null; export function checkPlutonium(): Promise { if (checking) return checking; checking = check().finally(() => { checking = null; }); return checking; } async function check(): Promise { const previous = release(); try { const response = await fetch(PLUTONIUM_MANIFEST_URL, { signal: AbortSignal.timeout(config.httpTimeoutMs), redirect: 'error' }); if (!response.ok) throw new Error(`Updater manifest request failed (HTTP ${response.status}).`); const body = await response.text(); if (body.length > 2_000_000) throw new Error('Updater manifest response was too large.'); writeSetting('plutonium_release', { revision: parsePlutoniumManifest(JSON.parse(body)), checkedAt: nowIso(), error: null }); } catch (err) { writeSetting('plutonium_release', { ...previous, checkedAt: nowIso(), error: (err as Error).message }); } const servers = listServers(); for (let i = 0; i < servers.length; i += 4) { await Promise.all(servers.slice(i, i + 4).map(s => probeVersion(s.name))); } } export function queueMaintenance(servers: string[]): number { const names = new Set(listServers().map(s => s.name)); if (!servers.length || servers.some(s => !names.has(s))) throw new Error('Choose existing fleet servers.'); return db().transaction(() => { const latest = release(); // An old cached release is still displayed, but never used as a fresh target. const target = !latest.error && latest.checkedAt && Date.now() - Date.parse(latest.checkedAt) < HOUR ? latest.revision : null; let added = 0; for (const server of new Set(servers)) { added += db().prepare(`INSERT OR IGNORE INTO maintenance_jobs (server, target_revision, requested_at) VALUES (?, ?, ?)`).run(server, target, nowIso()).changes; } return added; })(); } export function cancelMaintenance(id: number): boolean { return db().prepare(`UPDATE maintenance_jobs SET state = 'cancelled', finished_at = ?, detail = 'Cancelled by operator' WHERE id = ? AND state = 'queued'`).run(nowIso(), id).changes > 0; } function conflictingWork(server: string): boolean { return !!db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND state IN ('queued','running','awaiting-health') UNION ALL SELECT 1 FROM deployment_servers WHERE server = ? AND restart_state IN ('pending','failed') UNION ALL SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id WHERE bs.server = ? AND b.state IN ('running','ready-promote') LIMIT 1`).get(server, server, server); } function finish(job: MaintenanceJob, state: 'done' | 'failed' | 'cancelled', detail: string) { db().prepare('UPDATE maintenance_jobs SET state = ?, detail = ?, finished_at = ? WHERE id = ?') .run(state, detail, nowIso(), job.id); } function hold(job: MaintenanceJob, detail: string) { db().prepare('UPDATE maintenance_jobs SET detail = ? WHERE id = ?').run(detail, job.id); } let ticking: Promise | null = null; export function maintenanceTick(): Promise { if (ticking) return ticking; ticking = runTick().finally(() => { ticking = null; }); return ticking; } async function runTick(): Promise { const latest = release(); if (!latest.checkedAt || Date.now() - Date.parse(latest.checkedAt) >= HOUR) await checkPlutonium(); db().transaction(() => { const plan = schedule(); if (plan.enabled && plan.nextAt && plan.nextAt <= nowIso()) { const names = listServers().map(s => s.name); if (names.length) queueMaintenance(names); // Coalesce missed days after downtime into one run; no restart backlog. writeSetting('plutonium_schedule', { ...plan, nextAt: nextMaintenanceAt(plan.time, new Date(), plan.timeZone) }); } })(); await refreshRestartRollout(); const jobs = db().prepare(`SELECT * FROM maintenance_jobs WHERE state IN ('queued','restarting') ORDER BY id`).all() as MaintenanceJob[]; for (const job of jobs) { try { await processJob(job); } catch { hold(job, 'Maintenance check failed; waiting for the next check.'); } } db().prepare(`DELETE FROM maintenance_jobs WHERE finished_at < ? AND state IN ('done','failed','cancelled')`) .run(new Date(Date.now() - 30 * 24 * HOUR).toISOString()); } async function processJob(job: MaintenanceJob): Promise { const server = getServer(job.server); if (!server || server.destroyed_at) return finish(job, 'cancelled', 'Server removed.'); if (job.state === 'restarting') { const slot = db().prepare('SELECT requested_at FROM restart_rollout WHERE server = ?').get(job.server) as { requested_at: string } | undefined; if (!slot || !job.restarted_at || slot.requested_at < job.restarted_at) { return finish(job, 'failed', 'Restart submission was interrupted. Inspect the server before queuing another restart.'); } const health = rolloutHealth(job.server); if (!health.ok) return hold(job, health.reason); const revision = await probeVersion(job.server); if (job.target_revision && (revision === null || revision < job.target_revision)) { return finish(job, 'failed', `Server recovered, but r${job.target_revision} was not verified (${revision === null ? 'version unknown' : `running r${revision}`}). Check Plutainer auto-update settings.`); } return finish(job, 'done', `Server recovered; ${revision === null ? 'running revision unknown' : `running r${revision}`}.`); } if (!komodoConfigured() || !rconConfigured()) return hold(job, 'Configure Komodo and RCON before restarting.'); if (conflictingWork(job.server)) return hold(job, 'Waiting for deployment or beta work on this server.'); const reason = rolloutWaitReason(job.server); if (reason) return hold(job, reason); const players = await rconGetPlayerCount(job.server); if (players === null || players > 0) return hold(job, players === null ? 'Waiting for a fresh RCON player count.' : `Waiting for ${players} player(s) to leave.`); // Recheck after async I/O; cancellation and lifecycle operations may have arrived. const current = getServer(job.server); if (!current || current.destroyed_at) return finish(job, 'cancelled', 'Server removed.'); if (conflictingWork(job.server)) return hold(job, 'Waiting for deployment or beta work on this server.'); const claimed = db().prepare(`UPDATE maintenance_jobs SET state = 'restarting', restarted_at = ?, detail = 'Restart requested' WHERE id = ? AND state = 'queued'`).run(nowIso(), job.id); if (!claimed.changes) return; db().prepare('UPDATE plutonium_versions SET revision = NULL, error = ? WHERE server = ?') .run('Waiting for restart recovery.', job.server); // Persist intent before remote I/O. An ambiguous timeout must never cause an automatic second restart. const result = await restartInRollout(job.server); if (result.kind === 'queued') { db().prepare(`UPDATE maintenance_jobs SET state = 'queued', restarted_at = NULL, detail = ? WHERE id = ?`).run(result.message, job.id); } else if (result.kind === 'failed') finish(job, 'failed', result.message); }