feat: unify recovery, updates, and schedules under Maintenance
This commit is contained in:
@@ -0,0 +1,139 @@
|
||||
import { db } from '../db/index.js';
|
||||
import { getServer, listServers } from '../fleet/servers.js';
|
||||
import { requestRestart, cancelProvisioning } from '../provisioning/index.js';
|
||||
import { nextMaintenanceAt } from '../../plutonium.js';
|
||||
import type { MaintenanceJob, MaintenanceSchedule, PlutoniumStatus } from '../../plutonium.js';
|
||||
|
||||
import { readSetting, writeSetting } from './settings.js';
|
||||
import { checkPlutonium, probeVersion, release, freshRelease, runningVersions, HOUR } from '../integrations/plutonium.js';
|
||||
import { maintenanceAllowed } from './policies.js';
|
||||
export { checkPlutonium } from '../integrations/plutonium.js';
|
||||
const nowIso = () => new Date().toISOString();
|
||||
function schedule(): MaintenanceSchedule {
|
||||
// Legacy schedules were explicitly UTC. Preserve their timing until saved.
|
||||
return { timeZone: 'UTC', ...readSetting('plutonium_schedule', { enabled: false, time: '06:00', nextAt: null }) };
|
||||
}
|
||||
export function saveMaintenanceSchedule(enabled: boolean, time: string, timeZone: string): void {
|
||||
const nextAt = nextMaintenanceAt(time, new Date(), timeZone);
|
||||
writeSetting('plutonium_schedule', { enabled, time, timeZone, nextAt: enabled ? nextAt : null });
|
||||
}
|
||||
export function plutoniumStatus(): PlutoniumStatus {
|
||||
return {
|
||||
release: release(), schedule: schedule(), servers: runningVersions(),
|
||||
jobs: db().prepare(`SELECT m.*, d.stage AS operation_stage, d.state AS operation_state
|
||||
FROM maintenance_jobs m LEFT JOIN server_deployments d ON d.id = m.provisioning_id
|
||||
WHERE m.state IN ('queued','restarting')
|
||||
OR (m.state = 'failed' AND NOT EXISTS (SELECT 1 FROM maintenance_jobs n WHERE n.server = m.server AND n.kind = m.kind AND n.id > m.id))
|
||||
OR m.id IN (SELECT id FROM maintenance_jobs ORDER BY id DESC LIMIT 100) ORDER BY m.id DESC`).all() as MaintenanceJob[]
|
||||
};
|
||||
}
|
||||
|
||||
export function queueMaintenance(servers: string[], kind: MaintenanceJob['kind'] = 'legacy', origin: MaintenanceJob['origin'] = 'manual'): number {
|
||||
const names = new Set(listServers().map(s => s.name));
|
||||
if (!servers.length || servers.some(s => !names.has(s))) throw new Error('Choose existing fleet servers.');
|
||||
return db().transaction(() => {
|
||||
const target = kind === 'schedule' ? null : freshRelease();
|
||||
if (kind === 'update' && target === null) throw new Error('Check the published revision before requesting an update.');
|
||||
let added = 0;
|
||||
for (const server of new Set(servers)) {
|
||||
if (getServer(server)?.desired_state === 'stopped') continue;
|
||||
added += db().prepare(`INSERT OR IGNORE INTO maintenance_jobs (server, target_revision, requested_at, kind, origin)
|
||||
VALUES (?, ?, ?, ?, ?)`).run(server, target, nowIso(), kind, origin).changes;
|
||||
const job = db().prepare("SELECT id FROM maintenance_jobs WHERE server = ? AND state IN ('queued','restarting')").get(server) as { id: number };
|
||||
requestRestart(server, { kind: 'maintenance', id: String(job.id) });
|
||||
}
|
||||
return added;
|
||||
})();
|
||||
}
|
||||
|
||||
/** One automatic attempt per server/revision. Explicit retries preserve the old outcome. */
|
||||
export function queueUpdates(servers?: string[], automatic = false): number {
|
||||
if (!maintenanceAllowed()) throw new Error('Maintenance is paused. Resume it before requesting updates.');
|
||||
if (automatic && !maintenanceAllowed('updates')) return 0;
|
||||
if (freshRelease() === null) throw new Error('Check the published revision before requesting an update.');
|
||||
const versions = runningVersions();
|
||||
if (servers && (!servers.length || servers.some(server => !versions.some(v => v.server === server)))) throw new Error('Choose existing fleet servers.');
|
||||
const candidates = versions.filter(v => v.update_status === 'outdated' && (!servers || servers.includes(v.server)));
|
||||
const names = candidates.filter(v => !automatic || !db().prepare(`SELECT 1 FROM maintenance_jobs
|
||||
WHERE server = ? AND target_revision = ? AND kind = 'update'`).get(v.server, freshRelease())).map(v => v.server);
|
||||
return names.length ? queueMaintenance(names, 'update', automatic ? 'automatic' : 'manual') : 0;
|
||||
}
|
||||
|
||||
export function retryMaintenance(id: number): number {
|
||||
const job = db().prepare("SELECT * FROM maintenance_jobs WHERE id = ? AND state = 'failed'").get(id) as MaintenanceJob | undefined;
|
||||
if (!job) throw new Error('Only failed maintenance can be retried.');
|
||||
if (!maintenanceAllowed()) throw new Error('Maintenance is paused.');
|
||||
if (getServer(job.server)?.desired_state === 'stopped') throw new Error('This server is intentionally stopped.');
|
||||
// Ambiguous lifecycle failures retain their slot and require diagnostics, not another request.
|
||||
if (job.provisioning_id) {
|
||||
const operation = db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(job.provisioning_id) as { state: string } | undefined;
|
||||
if (operation?.state !== 'done') throw new Error('Inspect and recover the existing Provisioning operation before retrying.');
|
||||
}
|
||||
return queueMaintenance([job.server], job.kind, 'manual');
|
||||
}
|
||||
export function cancelMaintenance(id: number): boolean {
|
||||
return db().transaction(() => {
|
||||
const job = db().prepare("SELECT provisioning_id FROM maintenance_jobs WHERE id = ? AND state = 'queued'").get(id) as { provisioning_id: string | null } | undefined;
|
||||
if (!job) return false;
|
||||
if (job.provisioning_id && !cancelProvisioning(job.provisioning_id)) return false;
|
||||
return db().prepare(`UPDATE maintenance_jobs SET state = 'cancelled', finished_at = ?, detail = 'Cancelled by operator'
|
||||
WHERE id = ? AND state = 'queued'`).run(nowIso(), id).changes > 0;
|
||||
})();
|
||||
}
|
||||
|
||||
function finish(job: MaintenanceJob, state: 'done' | 'failed' | 'cancelled', detail: string) {
|
||||
db().prepare('UPDATE maintenance_jobs SET state = ?, detail = ?, finished_at = ? WHERE id = ?')
|
||||
.run(state, detail, nowIso(), job.id);
|
||||
}
|
||||
function hold(job: MaintenanceJob, detail: string) {
|
||||
db().prepare('UPDATE maintenance_jobs SET detail = ? WHERE id = ?').run(detail, job.id);
|
||||
}
|
||||
|
||||
let ticking: Promise<void> | null = null;
|
||||
export function maintenanceTick(): Promise<void> {
|
||||
if (ticking) return ticking;
|
||||
ticking = runTick().finally(() => { ticking = null; });
|
||||
return ticking;
|
||||
}
|
||||
async function runTick(): Promise<void> {
|
||||
const latest = release();
|
||||
if (!latest.checkedAt || Date.now() - Date.parse(latest.checkedAt) >= HOUR) await checkPlutonium();
|
||||
if (maintenanceAllowed('updates') && freshRelease() !== null) queueUpdates(undefined, true);
|
||||
db().transaction(() => {
|
||||
const plan = schedule();
|
||||
if (maintenanceAllowed() && plan.enabled && plan.nextAt && plan.nextAt <= nowIso()) {
|
||||
const names = listServers().filter(s => s.desired_state !== 'stopped').map(s => s.name);
|
||||
if (names.length) queueMaintenance(names, 'schedule', 'schedule');
|
||||
// Coalesce missed days after downtime into one run; no restart backlog.
|
||||
writeSetting('plutonium_schedule', { ...plan, nextAt: nextMaintenanceAt(plan.time, new Date(), plan.timeZone) });
|
||||
}
|
||||
})();
|
||||
const jobs = db().prepare(`SELECT * FROM maintenance_jobs WHERE state IN ('queued','restarting') ORDER BY id`).all() as MaintenanceJob[];
|
||||
for (const job of jobs) {
|
||||
try { await processJob(job); }
|
||||
catch { hold(job, 'Maintenance check failed; waiting for the next check.'); }
|
||||
}
|
||||
// Update attempts are also durable suppression evidence; never age them into an automatic retry.
|
||||
db().prepare(`DELETE FROM maintenance_jobs WHERE kind != 'update' AND finished_at < ? AND state IN ('done','failed','cancelled')`)
|
||||
.run(new Date(Date.now() - 30 * 24 * HOUR).toISOString());
|
||||
}
|
||||
async function processJob(job: MaintenanceJob): Promise<void> {
|
||||
const server = getServer(job.server);
|
||||
if (!server || server.destroyed_at) return finish(job, 'cancelled', 'Server removed.');
|
||||
const operation = requestRestart(job.server, { kind: 'maintenance', id: String(job.id) });
|
||||
if (['failed','interrupted','cancelled'].includes(operation.state)) {
|
||||
return finish(job, operation.state === 'cancelled' ? 'cancelled' : 'failed', operation.error ?? operation.stage);
|
||||
}
|
||||
if (operation.state !== 'done') {
|
||||
if (operation.submitted_at) {
|
||||
db().prepare("UPDATE maintenance_jobs SET state = 'restarting', restarted_at = ? WHERE id = ?").run(operation.submitted_at, job.id);
|
||||
db().prepare('UPDATE plutonium_versions SET revision = NULL, error = ? WHERE server = ?').run('Waiting for restart recovery.', job.server);
|
||||
}
|
||||
return hold(job, operation.stage);
|
||||
}
|
||||
const revision = await probeVersion(job.server);
|
||||
if (job.target_revision && (revision === null || revision < job.target_revision)) {
|
||||
return finish(job, 'failed', `Server recovered, but r${job.target_revision} was not verified (${revision === null ? 'version unknown' : `running r${revision}`}). Check Plutainer auto-update settings.`);
|
||||
}
|
||||
return finish(job, 'done', `Server recovered; ${revision === null ? 'running revision unknown' : `running r${revision}`}.`);
|
||||
}
|
||||
Reference in New Issue
Block a user