josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: unify recovery, updates, and schedules under Maintenance

Browse Source
This commit is contained in:
josh
2026-09-27 16:00:18 -04:00
parent 8ea696417f
commit fc46cb9da4
42 changed files with 1256 additions and 833 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/fleet/plutonium.ts
+2 -156
View File
@@ -1,156 +1,2 @@
import { db } from '../db/index.js';
import { config } from '../config.js';
import { getServer, listServers } from './servers.js';
import { rconPlutoniumVersion } from '../integrations/rcon.js';
import { requestRestart, cancelProvisioning, restartRecoveryPending } from '../provisioning/index.js';
import { nextMaintenanceAt, parsePlutoniumManifest, PLUTONIUM_MANIFEST_URL } from '../../plutonium.js';
import type { MaintenanceJob, MaintenanceSchedule, PlutoniumRelease, PlutoniumStatus, PlutoniumVersion } from '../../plutonium.js';
const HOUR = 60 * 60 * 1000;
const nowIso = () => new Date().toISOString();
function readSetting<T>(key: string, fallback: T): T {
const row = db().prepare('SELECT value FROM settings WHERE key = ?').get(key) as { value: string } | undefined;
return row ? JSON.parse(row.value) : fallback;
}
function writeSetting(key: string, value: unknown): void {
db().prepare('INSERT INTO settings (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value')
.run(key, JSON.stringify(value));
}
function release(): PlutoniumRelease {
return readSetting('plutonium_release', { revision: null, checkedAt: null, error: null });
}
function schedule(): MaintenanceSchedule {
// Legacy schedules were explicitly UTC. Preserve their timing until saved.
return { timeZone: 'UTC', ...readSetting('plutonium_schedule', { enabled: false, time: '06:00', nextAt: null }) };
}
export function saveMaintenanceSchedule(enabled: boolean, time: string, timeZone: string): void {
const nextAt = nextMaintenanceAt(time, new Date(), timeZone);
writeSetting('plutonium_schedule', { enabled, time, timeZone, nextAt: enabled ? nextAt : null });
}
export function plutoniumStatus(): PlutoniumStatus {
const versions = db().prepare('SELECT * FROM plutonium_versions').all() as PlutoniumVersion[];
return {
release: release(), schedule: schedule(),
servers: listServers().map(s => versions.find(v => v.server === s.name)
?? { server: s.name, revision: null, checked_at: null, error: null }),
jobs: db().prepare(`SELECT * FROM maintenance_jobs WHERE state IN ('queued','restarting')
OR id IN (SELECT id FROM maintenance_jobs ORDER BY id DESC LIMIT 100) ORDER BY id DESC`).all() as MaintenanceJob[]
};
}
async function probeVersion(server: string): Promise<number | null> {
const recovering = restartRecoveryPending(server);
const result = recovering ? { revision: null, error: 'Waiting for restart recovery.' }
: await rconPlutoniumVersion(server).catch(() => ({ revision: null, error: 'Version check failed.' }));
// Clear the old revision on failure: it is no longer evidence of what is running.
db().prepare(`INSERT INTO plutonium_versions (server, revision, checked_at, error) VALUES (?, ?, ?, ?)
ON CONFLICT(server) DO UPDATE SET revision = excluded.revision, checked_at = excluded.checked_at, error = excluded.error`)
.run(server, result.revision, nowIso(), result.error);
return result.revision;
}
let checking: Promise<void> | null = null;
export function checkPlutonium(): Promise<void> {
if (checking) return checking;
checking = check().finally(() => { checking = null; });
return checking;
}
async function check(): Promise<void> {
const previous = release();
try {
const response = await fetch(PLUTONIUM_MANIFEST_URL, { signal: AbortSignal.timeout(config.httpTimeoutMs), redirect: 'error' });
if (!response.ok) throw new Error(`Updater manifest request failed (HTTP ${response.status}).`);
const body = await response.text();
if (body.length > 2_000_000) throw new Error('Updater manifest response was too large.');
writeSetting('plutonium_release', { revision: parsePlutoniumManifest(JSON.parse(body)), checkedAt: nowIso(), error: null });
} catch (err) {
writeSetting('plutonium_release', { ...previous, checkedAt: nowIso(), error: (err as Error).message });
}
const servers = listServers();
for (let i = 0; i < servers.length; i += 4) {
await Promise.all(servers.slice(i, i + 4).map(s => probeVersion(s.name)));
}
}
export function queueMaintenance(servers: string[]): number {
const names = new Set(listServers().map(s => s.name));
if (!servers.length || servers.some(s => !names.has(s))) throw new Error('Choose existing fleet servers.');
return db().transaction(() => {
const latest = release();
// An old cached release is still displayed, but never used as a fresh target.
const target = !latest.error && latest.checkedAt && Date.now() - Date.parse(latest.checkedAt) < HOUR
? latest.revision : null;
let added = 0;
for (const server of new Set(servers)) {
added += db().prepare(`INSERT OR IGNORE INTO maintenance_jobs (server, target_revision, requested_at)
VALUES (?, ?, ?)`).run(server, target, nowIso()).changes;
const job = db().prepare("SELECT id FROM maintenance_jobs WHERE server = ? AND state IN ('queued','restarting')").get(server) as { id: number };
requestRestart(server, { kind: 'maintenance', id: String(job.id) });
}
return added;
})();
}
export function cancelMaintenance(id: number): boolean {
return db().transaction(() => {
const job = db().prepare("SELECT provisioning_id FROM maintenance_jobs WHERE id = ? AND state = 'queued'").get(id) as { provisioning_id: string | null } | undefined;
if (!job) return false;
if (job.provisioning_id && !cancelProvisioning(job.provisioning_id)) return false;
return db().prepare(`UPDATE maintenance_jobs SET state = 'cancelled', finished_at = ?, detail = 'Cancelled by operator'
WHERE id = ? AND state = 'queued'`).run(nowIso(), id).changes > 0;
})();
}
function finish(job: MaintenanceJob, state: 'done' | 'failed' | 'cancelled', detail: string) {
db().prepare('UPDATE maintenance_jobs SET state = ?, detail = ?, finished_at = ? WHERE id = ?')
.run(state, detail, nowIso(), job.id);
}
function hold(job: MaintenanceJob, detail: string) {
db().prepare('UPDATE maintenance_jobs SET detail = ? WHERE id = ?').run(detail, job.id);
}
let ticking: Promise<void> | null = null;
export function maintenanceTick(): Promise<void> {
if (ticking) return ticking;
ticking = runTick().finally(() => { ticking = null; });
return ticking;
}
async function runTick(): Promise<void> {
const latest = release();
if (!latest.checkedAt || Date.now() - Date.parse(latest.checkedAt) >= HOUR) await checkPlutonium();
db().transaction(() => {
const plan = schedule();
if (plan.enabled && plan.nextAt && plan.nextAt <= nowIso()) {
const names = listServers().map(s => s.name);
if (names.length) queueMaintenance(names);
// Coalesce missed days after downtime into one run; no restart backlog.
writeSetting('plutonium_schedule', { ...plan, nextAt: nextMaintenanceAt(plan.time, new Date(), plan.timeZone) });
}
})();
const jobs = db().prepare(`SELECT * FROM maintenance_jobs WHERE state IN ('queued','restarting') ORDER BY id`).all() as MaintenanceJob[];
for (const job of jobs) {
try { await processJob(job); }
catch { hold(job, 'Maintenance check failed; waiting for the next check.'); }
}
db().prepare(`DELETE FROM maintenance_jobs WHERE finished_at < ? AND state IN ('done','failed','cancelled')`)
.run(new Date(Date.now() - 30 * 24 * HOUR).toISOString());
}
async function processJob(job: MaintenanceJob): Promise<void> {
const server = getServer(job.server);
if (!server || server.destroyed_at) return finish(job, 'cancelled', 'Server removed.');
const operation = requestRestart(job.server, { kind: 'maintenance', id: String(job.id) });
if (['failed','interrupted','cancelled'].includes(operation.state)) {
return finish(job, operation.state === 'cancelled' ? 'cancelled' : 'failed', operation.error ?? operation.stage);
}
if (operation.state !== 'done') {
if (operation.submitted_at) {
db().prepare("UPDATE maintenance_jobs SET state = 'restarting', restarted_at = ? WHERE id = ?").run(operation.submitted_at, job.id);
db().prepare('UPDATE plutonium_versions SET revision = NULL, error = ? WHERE server = ?').run('Waiting for restart recovery.', job.server);
}
return hold(job, operation.stage);
}
const revision = await probeVersion(job.server);
if (job.target_revision && (revision === null || revision < job.target_revision)) {
return finish(job, 'failed', `Server recovered, but r${job.target_revision} was not verified (${revision === null ? 'version unknown' : `running r${revision}`}). Check Plutainer auto-update settings.`);
}
return finish(job, 'done', `Server recovered; ${revision === null ? 'running revision unknown' : `running r${revision}`}.`);
}
// Compatibility for existing jobs and API clients.
export * from "../maintenance/runtime.js";