278 lines
11 KiB
TypeScript
278 lines
11 KiB
TypeScript
import { db } from '../db/index.js';
|
|
import { getServer, listServers } from '../fleet/servers.js';
|
|
import { hasActiveProvisioning } from '../fleet/provisioning.js';
|
|
import { requestRestart, cancelProvisioning } from '../provisioning/index.js';
|
|
import { nextMaintenanceAt } from '../../plutonium.js';
|
|
import type { MaintenanceJob, MaintenanceSchedule, PlutoniumStatus } from '../../plutonium.js';
|
|
|
|
import { readSetting, writeSetting } from './settings.js';
|
|
import {
|
|
checkPlutonium,
|
|
probeVersion,
|
|
release,
|
|
freshRelease,
|
|
runningVersions,
|
|
HOUR
|
|
} from '../integrations/plutonium.js';
|
|
import { maintenanceAllowed } from './policies.js';
|
|
export { checkPlutonium } from '../integrations/plutonium.js';
|
|
const nowIso = () => new Date().toISOString();
|
|
function schedule(): MaintenanceSchedule {
|
|
// Legacy schedules were explicitly UTC. Preserve their timing until saved.
|
|
return {
|
|
timeZone: 'UTC',
|
|
...readSetting('plutonium_schedule', { enabled: false, time: '06:00', nextAt: null })
|
|
};
|
|
}
|
|
export function saveMaintenanceSchedule(enabled: boolean, time: string, timeZone: string): void {
|
|
const nextAt = nextMaintenanceAt(time, new Date(), timeZone);
|
|
writeSetting('plutonium_schedule', {
|
|
enabled,
|
|
time,
|
|
timeZone,
|
|
nextAt: enabled ? nextAt : null
|
|
});
|
|
}
|
|
export function plutoniumStatus(selectedJob: number | null = null): PlutoniumStatus {
|
|
return {
|
|
release: release(),
|
|
schedule: schedule(),
|
|
servers: runningVersions(),
|
|
jobs: db()
|
|
.prepare(
|
|
`SELECT m.*, d.stage AS operation_stage, d.state AS operation_state
|
|
FROM maintenance_jobs m LEFT JOIN server_deployments d ON d.id = m.provisioning_id
|
|
WHERE m.state IN ('queued','restarting')
|
|
OR (m.state = 'failed' AND NOT EXISTS (SELECT 1 FROM maintenance_jobs n WHERE n.server = m.server AND n.kind = m.kind AND n.id > m.id))
|
|
OR m.id IN (SELECT id FROM maintenance_jobs ORDER BY id DESC LIMIT 100) OR m.id = ? ORDER BY m.id DESC`
|
|
)
|
|
.all(selectedJob) as MaintenanceJob[]
|
|
};
|
|
}
|
|
|
|
export function queueMaintenance(
|
|
servers: string[],
|
|
kind: MaintenanceJob['kind'] = 'legacy',
|
|
origin: MaintenanceJob['origin'] = 'manual'
|
|
): number {
|
|
const names = new Set(listServers().map((s) => s.name));
|
|
if (!servers.length || servers.some((s) => !names.has(s)))
|
|
throw new Error('Choose existing fleet servers.');
|
|
return db().transaction(() => {
|
|
const target = kind === 'schedule' ? null : freshRelease();
|
|
if (kind === 'update' && target === null)
|
|
throw new Error('Check the published revision before requesting an update.');
|
|
let added = 0;
|
|
for (const server of new Set(servers)) {
|
|
if (getServer(server)?.desired_state === 'stopped') continue;
|
|
if (origin !== 'manual' && hasActiveProvisioning(server)) continue;
|
|
added += db()
|
|
.prepare(
|
|
`INSERT OR IGNORE INTO maintenance_jobs (server, target_revision, requested_at, kind, origin)
|
|
VALUES (?, ?, ?, ?, ?)`
|
|
)
|
|
.run(server, target, nowIso(), kind, origin).changes;
|
|
const job = db()
|
|
.prepare(
|
|
"SELECT id FROM maintenance_jobs WHERE server = ? AND server_deleted_at IS NULL AND state IN ('queued','restarting')"
|
|
)
|
|
.get(server) as { id: number };
|
|
requestRestart(server, { kind: 'maintenance', id: String(job.id) });
|
|
}
|
|
return added;
|
|
})();
|
|
}
|
|
|
|
/** One automatic attempt per server/revision. Explicit retries preserve the old outcome. */
|
|
export function queueUpdates(servers?: string[], automatic = false): number {
|
|
if (!maintenanceAllowed())
|
|
throw new Error('Maintenance is paused. Resume it before requesting updates.');
|
|
if (automatic && !maintenanceAllowed('updates')) return 0;
|
|
if (freshRelease() === null)
|
|
throw new Error('Check the published revision before requesting an update.');
|
|
const versions = runningVersions();
|
|
if (
|
|
servers &&
|
|
(!servers.length || servers.some((server) => !versions.some((v) => v.server === server)))
|
|
)
|
|
throw new Error('Choose existing fleet servers.');
|
|
const candidates = versions.filter(
|
|
(v) => v.update_status === 'outdated' && (!servers || servers.includes(v.server))
|
|
);
|
|
const names = candidates
|
|
.filter(
|
|
(v) =>
|
|
!automatic ||
|
|
!db()
|
|
.prepare(
|
|
`SELECT 1 FROM maintenance_jobs
|
|
WHERE server = ? AND server_deleted_at IS NULL AND target_revision = ? AND kind = 'update'`
|
|
)
|
|
.get(v.server, freshRelease())
|
|
)
|
|
.map((v) => v.server);
|
|
return names.length ? queueMaintenance(names, 'update', automatic ? 'automatic' : 'manual') : 0;
|
|
}
|
|
|
|
export function retryMaintenance(id: number): number {
|
|
const job = db()
|
|
.prepare("SELECT * FROM maintenance_jobs WHERE id = ? AND state = 'failed'")
|
|
.get(id) as MaintenanceJob | undefined;
|
|
if (!job) throw new Error('Only failed maintenance can be retried.');
|
|
if (job.server_deleted_at)
|
|
throw new Error('This server was deleted; its maintenance history cannot be retried.');
|
|
if (!maintenanceAllowed()) throw new Error('Maintenance is paused.');
|
|
if (getServer(job.server)?.desired_state === 'stopped')
|
|
throw new Error('This server is intentionally stopped.');
|
|
// Ambiguous lifecycle failures retain their slot and require diagnostics, not another request.
|
|
if (job.provisioning_id) {
|
|
const operation = db()
|
|
.prepare('SELECT state FROM server_deployments WHERE id = ?')
|
|
.get(job.provisioning_id) as { state: string } | undefined;
|
|
if (operation?.state !== 'done')
|
|
throw new Error(
|
|
'Inspect and recover the existing Provisioning operation before retrying.'
|
|
);
|
|
}
|
|
return queueMaintenance([job.server], job.kind, 'manual');
|
|
}
|
|
export function cancelMaintenance(id: number): boolean {
|
|
return db().transaction(() => {
|
|
const job = db()
|
|
.prepare(
|
|
"SELECT provisioning_id FROM maintenance_jobs WHERE id = ? AND state = 'queued'"
|
|
)
|
|
.get(id) as { provisioning_id: string | null } | undefined;
|
|
if (!job) return false;
|
|
if (job.provisioning_id && !cancelProvisioning(job.provisioning_id)) return false;
|
|
return (
|
|
db()
|
|
.prepare(
|
|
`UPDATE maintenance_jobs SET state = 'cancelled', finished_at = ?, detail = 'Cancelled by operator'
|
|
WHERE id = ? AND state = 'queued'`
|
|
)
|
|
.run(nowIso(), id).changes > 0
|
|
);
|
|
})();
|
|
}
|
|
|
|
function finish(job: MaintenanceJob, state: 'done' | 'failed' | 'cancelled', detail: string) {
|
|
db()
|
|
.prepare(
|
|
'UPDATE maintenance_jobs SET state = ?, detail = ?, finished_at = ? WHERE id = ? AND server_deleted_at IS NULL'
|
|
)
|
|
.run(state, detail, nowIso(), job.id);
|
|
}
|
|
function hold(job: MaintenanceJob, detail: string) {
|
|
db()
|
|
.prepare(
|
|
'UPDATE maintenance_jobs SET detail = ? WHERE id = ? AND server_deleted_at IS NULL'
|
|
)
|
|
.run(detail, job.id);
|
|
}
|
|
|
|
let ticking: Promise<void> | null = null;
|
|
export function maintenanceTick(): Promise<void> {
|
|
if (ticking) return ticking;
|
|
ticking = runTick().finally(() => {
|
|
ticking = null;
|
|
});
|
|
return ticking;
|
|
}
|
|
async function runTick(): Promise<void> {
|
|
const latest = release();
|
|
// Provisioning refreshes the manifest independently; that must not postpone
|
|
// the hourly version checks for the rest of the fleet.
|
|
const versionsCheckedAt = readSetting<string | null>('plutonium_versions_checked_at', null);
|
|
if (
|
|
!latest.checkedAt ||
|
|
Date.now() - Date.parse(latest.checkedAt) >= HOUR ||
|
|
!versionsCheckedAt ||
|
|
Date.now() - Date.parse(versionsCheckedAt) >= HOUR
|
|
)
|
|
await checkPlutonium();
|
|
if (maintenanceAllowed('updates') && freshRelease() !== null) queueUpdates(undefined, true);
|
|
db().transaction(() => {
|
|
const plan = schedule();
|
|
if (maintenanceAllowed() && plan.enabled && plan.nextAt && plan.nextAt <= nowIso()) {
|
|
const names = listServers()
|
|
.filter((s) => s.desired_state !== 'stopped')
|
|
.map((s) => s.name);
|
|
if (names.length) queueMaintenance(names, 'schedule', 'schedule');
|
|
// Coalesce missed days after downtime into one run; no restart backlog.
|
|
writeSetting('plutonium_schedule', {
|
|
...plan,
|
|
nextAt: nextMaintenanceAt(plan.time, new Date(), plan.timeZone)
|
|
});
|
|
}
|
|
})();
|
|
const jobs = db()
|
|
.prepare(
|
|
`SELECT * FROM maintenance_jobs WHERE server_deleted_at IS NULL AND state IN ('queued','restarting') ORDER BY id`
|
|
)
|
|
.all() as MaintenanceJob[];
|
|
for (const job of jobs) {
|
|
try {
|
|
await processJob(job);
|
|
} catch {
|
|
hold(job, 'Maintenance check failed; waiting for the next check.');
|
|
}
|
|
}
|
|
}
|
|
async function processJob(job: MaintenanceJob): Promise<void> {
|
|
const server = getServer(job.server);
|
|
if (job.server_deleted_at || !server || server.destroyed_at)
|
|
return finish(job, 'cancelled', 'Server removed.');
|
|
if (!job.provisioning_id && !job.restarted_at && hasActiveProvisioning(job.server)) {
|
|
return hold(job, 'Waiting for another provisioning operation.');
|
|
}
|
|
const operation = requestRestart(job.server, { kind: 'maintenance', id: String(job.id) });
|
|
if (['failed', 'interrupted', 'cancelled'].includes(operation.state)) {
|
|
return finish(
|
|
job,
|
|
operation.state === 'cancelled' ? 'cancelled' : 'failed',
|
|
operation.error ?? operation.stage
|
|
);
|
|
}
|
|
if (operation.state !== 'done') {
|
|
if (operation.submitted_at) {
|
|
db()
|
|
.prepare(
|
|
"UPDATE maintenance_jobs SET state = 'restarting', restarted_at = ? WHERE id = ?"
|
|
)
|
|
.run(operation.submitted_at, job.id);
|
|
db()
|
|
.prepare(
|
|
'UPDATE plutonium_versions SET revision = NULL, error = ? WHERE server = ?'
|
|
)
|
|
.run('Waiting for restart recovery.', job.server);
|
|
}
|
|
return hold(job, operation.stage);
|
|
}
|
|
if (hasActiveProvisioning(job.server, operation.id)) {
|
|
return hold(
|
|
job,
|
|
'Waiting for another provisioning operation before verifying the running revision.'
|
|
);
|
|
}
|
|
const revision = await probeVersion(job.server, { maintenance: true });
|
|
if (hasActiveProvisioning(job.server, operation.id)) {
|
|
return hold(
|
|
job,
|
|
'Waiting for another provisioning operation before verifying the running revision.'
|
|
);
|
|
}
|
|
if (job.target_revision && (revision === null || revision < job.target_revision)) {
|
|
return finish(
|
|
job,
|
|
'failed',
|
|
`Server recovered, but r${job.target_revision} was not verified (${revision === null ? 'version unknown' : `running r${revision}`}). Check Plutainer auto-update settings.`
|
|
);
|
|
}
|
|
return finish(
|
|
job,
|
|
'done',
|
|
`Server recovered; ${revision === null ? 'running revision unknown' : `running r${revision}`}.`
|
|
);
|
|
}
|