import { db } from '../db/index.js'; import { getServer, listServers } from '../fleet/servers.js'; import { hasActiveProvisioning } from '../fleet/provisioning.js'; import { requestRestart, cancelProvisioning } from '../provisioning/index.js'; import { nextMaintenanceAt } from '../../plutonium.js'; import type { MaintenanceJob, MaintenanceSchedule, PlutoniumStatus } from '../../plutonium.js'; import { readSetting, writeSetting } from './settings.js'; import { checkPlutonium, probeVersion, release, freshRelease, runningVersions, HOUR } from '../integrations/plutonium.js'; import { maintenanceAllowed } from './policies.js'; export { checkPlutonium } from '../integrations/plutonium.js'; const nowIso = () => new Date().toISOString(); function schedule(): MaintenanceSchedule { // Legacy schedules were explicitly UTC. Preserve their timing until saved. return { timeZone: 'UTC', ...readSetting('plutonium_schedule', { enabled: false, time: '06:00', nextAt: null }) }; } export function saveMaintenanceSchedule(enabled: boolean, time: string, timeZone: string): void { const nextAt = nextMaintenanceAt(time, new Date(), timeZone); writeSetting('plutonium_schedule', { enabled, time, timeZone, nextAt: enabled ? nextAt : null }); } export function plutoniumStatus(selectedJob: number | null = null): PlutoniumStatus { return { release: release(), schedule: schedule(), servers: runningVersions(), jobs: db() .prepare( `SELECT m.*, d.stage AS operation_stage, d.state AS operation_state FROM maintenance_jobs m LEFT JOIN server_deployments d ON d.id = m.provisioning_id WHERE m.state IN ('queued','restarting') OR (m.state = 'failed' AND NOT EXISTS (SELECT 1 FROM maintenance_jobs n WHERE n.server = m.server AND n.kind = m.kind AND n.id > m.id)) OR m.id IN (SELECT id FROM maintenance_jobs ORDER BY id DESC LIMIT 100) OR m.id = ? ORDER BY m.id DESC` ) .all(selectedJob) as MaintenanceJob[] }; } export function queueMaintenance( servers: string[], kind: MaintenanceJob['kind'] = 'legacy', origin: MaintenanceJob['origin'] = 'manual' ): number { const names = new Set(listServers().map((s) => s.name)); if (!servers.length || servers.some((s) => !names.has(s))) throw new Error('Choose existing fleet servers.'); return db().transaction(() => { const target = kind === 'schedule' ? null : freshRelease(); if (kind === 'update' && target === null) throw new Error('Check the published revision before requesting an update.'); let added = 0; for (const server of new Set(servers)) { if (getServer(server)?.desired_state === 'stopped') continue; if (origin !== 'manual' && hasActiveProvisioning(server)) continue; added += db() .prepare( `INSERT OR IGNORE INTO maintenance_jobs (server, target_revision, requested_at, kind, origin) VALUES (?, ?, ?, ?, ?)` ) .run(server, target, nowIso(), kind, origin).changes; const job = db() .prepare( "SELECT id FROM maintenance_jobs WHERE server = ? AND server_deleted_at IS NULL AND state IN ('queued','restarting')" ) .get(server) as { id: number }; requestRestart(server, { kind: 'maintenance', id: String(job.id) }); } return added; })(); } /** One automatic attempt per server/revision. Explicit retries preserve the old outcome. */ export function queueUpdates(servers?: string[], automatic = false): number { if (!maintenanceAllowed()) throw new Error('Maintenance is paused. Resume it before requesting updates.'); if (automatic && !maintenanceAllowed('updates')) return 0; if (freshRelease() === null) throw new Error('Check the published revision before requesting an update.'); const versions = runningVersions(); if ( servers && (!servers.length || servers.some((server) => !versions.some((v) => v.server === server))) ) throw new Error('Choose existing fleet servers.'); const candidates = versions.filter( (v) => v.update_status === 'outdated' && (!servers || servers.includes(v.server)) ); const names = candidates .filter( (v) => !automatic || !db() .prepare( `SELECT 1 FROM maintenance_jobs WHERE server = ? AND server_deleted_at IS NULL AND target_revision = ? AND kind = 'update'` ) .get(v.server, freshRelease()) ) .map((v) => v.server); return names.length ? queueMaintenance(names, 'update', automatic ? 'automatic' : 'manual') : 0; } export function retryMaintenance(id: number): number { const job = db() .prepare("SELECT * FROM maintenance_jobs WHERE id = ? AND state = 'failed'") .get(id) as MaintenanceJob | undefined; if (!job) throw new Error('Only failed maintenance can be retried.'); if (job.server_deleted_at) throw new Error('This server was deleted; its maintenance history cannot be retried.'); if (!maintenanceAllowed()) throw new Error('Maintenance is paused.'); if (getServer(job.server)?.desired_state === 'stopped') throw new Error('This server is intentionally stopped.'); // Ambiguous lifecycle failures retain their slot and require diagnostics, not another request. if (job.provisioning_id) { const operation = db() .prepare('SELECT state FROM server_deployments WHERE id = ?') .get(job.provisioning_id) as { state: string } | undefined; if (operation?.state !== 'done') throw new Error( 'Inspect and recover the existing Provisioning operation before retrying.' ); } return queueMaintenance([job.server], job.kind, 'manual'); } export function cancelMaintenance(id: number): boolean { return db().transaction(() => { const job = db() .prepare( "SELECT provisioning_id FROM maintenance_jobs WHERE id = ? AND state = 'queued'" ) .get(id) as { provisioning_id: string | null } | undefined; if (!job) return false; if (job.provisioning_id && !cancelProvisioning(job.provisioning_id)) return false; return ( db() .prepare( `UPDATE maintenance_jobs SET state = 'cancelled', finished_at = ?, detail = 'Cancelled by operator' WHERE id = ? AND state = 'queued'` ) .run(nowIso(), id).changes > 0 ); })(); } function finish(job: MaintenanceJob, state: 'done' | 'failed' | 'cancelled', detail: string) { db() .prepare( 'UPDATE maintenance_jobs SET state = ?, detail = ?, finished_at = ? WHERE id = ? AND server_deleted_at IS NULL' ) .run(state, detail, nowIso(), job.id); } function hold(job: MaintenanceJob, detail: string) { db() .prepare( 'UPDATE maintenance_jobs SET detail = ? WHERE id = ? AND server_deleted_at IS NULL' ) .run(detail, job.id); } let ticking: Promise | null = null; export function maintenanceTick(): Promise { if (ticking) return ticking; ticking = runTick().finally(() => { ticking = null; }); return ticking; } async function runTick(): Promise { const latest = release(); // Provisioning refreshes the manifest independently; that must not postpone // the hourly version checks for the rest of the fleet. const versionsCheckedAt = readSetting('plutonium_versions_checked_at', null); if ( !latest.checkedAt || Date.now() - Date.parse(latest.checkedAt) >= HOUR || !versionsCheckedAt || Date.now() - Date.parse(versionsCheckedAt) >= HOUR ) await checkPlutonium(); if (maintenanceAllowed('updates') && freshRelease() !== null) queueUpdates(undefined, true); db().transaction(() => { const plan = schedule(); if (maintenanceAllowed() && plan.enabled && plan.nextAt && plan.nextAt <= nowIso()) { const names = listServers() .filter((s) => s.desired_state !== 'stopped') .map((s) => s.name); if (names.length) queueMaintenance(names, 'schedule', 'schedule'); // Coalesce missed days after downtime into one run; no restart backlog. writeSetting('plutonium_schedule', { ...plan, nextAt: nextMaintenanceAt(plan.time, new Date(), plan.timeZone) }); } })(); const jobs = db() .prepare( `SELECT * FROM maintenance_jobs WHERE server_deleted_at IS NULL AND state IN ('queued','restarting') ORDER BY id` ) .all() as MaintenanceJob[]; for (const job of jobs) { try { await processJob(job); } catch { hold(job, 'Maintenance check failed; waiting for the next check.'); } } } async function processJob(job: MaintenanceJob): Promise { const server = getServer(job.server); if (job.server_deleted_at || !server || server.destroyed_at) return finish(job, 'cancelled', 'Server removed.'); if (!job.provisioning_id && !job.restarted_at && hasActiveProvisioning(job.server)) { return hold(job, 'Waiting for another provisioning operation.'); } const operation = requestRestart(job.server, { kind: 'maintenance', id: String(job.id) }); if (['failed', 'interrupted', 'cancelled'].includes(operation.state)) { return finish( job, operation.state === 'cancelled' ? 'cancelled' : 'failed', operation.error ?? operation.stage ); } if (operation.state !== 'done') { if (operation.submitted_at) { db() .prepare( "UPDATE maintenance_jobs SET state = 'restarting', restarted_at = ? WHERE id = ?" ) .run(operation.submitted_at, job.id); db() .prepare( 'UPDATE plutonium_versions SET revision = NULL, error = ? WHERE server = ?' ) .run('Waiting for restart recovery.', job.server); } return hold(job, operation.stage); } if (hasActiveProvisioning(job.server, operation.id)) { return hold( job, 'Waiting for another provisioning operation before verifying the running revision.' ); } const revision = await probeVersion(job.server, { maintenance: true }); if (hasActiveProvisioning(job.server, operation.id)) { return hold( job, 'Waiting for another provisioning operation before verifying the running revision.' ); } if (job.target_revision && (revision === null || revision < job.target_revision)) { return finish( job, 'failed', `Server recovered, but r${job.target_revision} was not verified (${revision === null ? 'version unknown' : `running r${revision}`}). Check Plutainer auto-update settings.` ); } return finish( job, 'done', `Server recovered; ${revision === null ? 'running revision unknown' : `running r${revision}`}.` ); }