josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Add Prettier and ESLint tooling (Refs #77)

Browse Source
This commit is contained in:
josh
2026-10-09 15:56:13 -04:00
parent 8ed059b8d3
commit 16469e2d43
711 changed files with 99737 additions and 28286 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/maintenance/runtime.ts
+171 -47
View File
@@ -6,42 +6,77 @@ import { nextMaintenanceAt } from '../../plutonium.js';
import type { MaintenanceJob, MaintenanceSchedule, PlutoniumStatus } from '../../plutonium.js';
import { readSetting, writeSetting } from './settings.js';
import { checkPlutonium, probeVersion, release, freshRelease, runningVersions, HOUR } from '../integrations/plutonium.js';
import {
checkPlutonium,
probeVersion,
release,
freshRelease,
runningVersions,
HOUR
} from '../integrations/plutonium.js';
import { maintenanceAllowed } from './policies.js';
export { checkPlutonium } from '../integrations/plutonium.js';
const nowIso = () => new Date().toISOString();
function schedule(): MaintenanceSchedule {
// Legacy schedules were explicitly UTC. Preserve their timing until saved.
return { timeZone: 'UTC', ...readSetting('plutonium_schedule', { enabled: false, time: '06:00', nextAt: null }) };
return {
timeZone: 'UTC',
...readSetting('plutonium_schedule', { enabled: false, time: '06:00', nextAt: null })
};
}
export function saveMaintenanceSchedule(enabled: boolean, time: string, timeZone: string): void {
const nextAt = nextMaintenanceAt(time, new Date(), timeZone);
writeSetting('plutonium_schedule', { enabled, time, timeZone, nextAt: enabled ? nextAt : null });
writeSetting('plutonium_schedule', {
enabled,
time,
timeZone,
nextAt: enabled ? nextAt : null
});
}
export function plutoniumStatus(selectedJob: number | null = null): PlutoniumStatus {
return {
release: release(), schedule: schedule(), servers: runningVersions(),
jobs: db().prepare(`SELECT m.*, d.stage AS operation_stage, d.state AS operation_state
release: release(),
schedule: schedule(),
servers: runningVersions(),
jobs: db()
.prepare(
`SELECT m.*, d.stage AS operation_stage, d.state AS operation_state
FROM maintenance_jobs m LEFT JOIN server_deployments d ON d.id = m.provisioning_id
WHERE m.state IN ('queued','restarting')
OR (m.state = 'failed' AND NOT EXISTS (SELECT 1 FROM maintenance_jobs n WHERE n.server = m.server AND n.kind = m.kind AND n.id > m.id))
OR m.id IN (SELECT id FROM maintenance_jobs ORDER BY id DESC LIMIT 100) OR m.id = ? ORDER BY m.id DESC`).all(selectedJob) as MaintenanceJob[]
OR m.id IN (SELECT id FROM maintenance_jobs ORDER BY id DESC LIMIT 100) OR m.id = ? ORDER BY m.id DESC`
)
.all(selectedJob) as MaintenanceJob[]
};
}
export function queueMaintenance(servers: string[], kind: MaintenanceJob['kind'] = 'legacy', origin: MaintenanceJob['origin'] = 'manual'): number {
const names = new Set(listServers().map(s => s.name));
if (!servers.length || servers.some(s => !names.has(s))) throw new Error('Choose existing fleet servers.');
export function queueMaintenance(
servers: string[],
kind: MaintenanceJob['kind'] = 'legacy',
origin: MaintenanceJob['origin'] = 'manual'
): number {
const names = new Set(listServers().map((s) => s.name));
if (!servers.length || servers.some((s) => !names.has(s)))
throw new Error('Choose existing fleet servers.');
return db().transaction(() => {
const target = kind === 'schedule' ? null : freshRelease();
if (kind === 'update' && target === null) throw new Error('Check the published revision before requesting an update.');
if (kind === 'update' && target === null)
throw new Error('Check the published revision before requesting an update.');
let added = 0;
for (const server of new Set(servers)) {
if (getServer(server)?.desired_state === 'stopped') continue;
if (origin !== 'manual' && hasActiveProvisioning(server)) continue;
added += db().prepare(`INSERT OR IGNORE INTO maintenance_jobs (server, target_revision, requested_at, kind, origin)
VALUES (?, ?, ?, ?, ?)`).run(server, target, nowIso(), kind, origin).changes;
const job = db().prepare("SELECT id FROM maintenance_jobs WHERE server = ? AND server_deleted_at IS NULL AND state IN ('queued','restarting')").get(server) as { id: number };
added += db()
.prepare(
`INSERT OR IGNORE INTO maintenance_jobs (server, target_revision, requested_at, kind, origin)
VALUES (?, ?, ?, ?, ?)`
)
.run(server, target, nowIso(), kind, origin).changes;
const job = db()
.prepare(
"SELECT id FROM maintenance_jobs WHERE server = ? AND server_deleted_at IS NULL AND state IN ('queued','restarting')"
)
.get(server) as { id: number };
requestRestart(server, { kind: 'maintenance', id: String(job.id) });
}
return added;
@@ -50,52 +85,98 @@ export function queueMaintenance(servers: string[], kind: MaintenanceJob['kind']
/** One automatic attempt per server/revision. Explicit retries preserve the old outcome. */
export function queueUpdates(servers?: string[], automatic = false): number {
if (!maintenanceAllowed()) throw new Error('Maintenance is paused. Resume it before requesting updates.');
if (!maintenanceAllowed())
throw new Error('Maintenance is paused. Resume it before requesting updates.');
if (automatic && !maintenanceAllowed('updates')) return 0;
if (freshRelease() === null) throw new Error('Check the published revision before requesting an update.');
if (freshRelease() === null)
throw new Error('Check the published revision before requesting an update.');
const versions = runningVersions();
if (servers && (!servers.length || servers.some(server => !versions.some(v => v.server === server)))) throw new Error('Choose existing fleet servers.');
const candidates = versions.filter(v => v.update_status === 'outdated' && (!servers || servers.includes(v.server)));
const names = candidates.filter(v => !automatic || !db().prepare(`SELECT 1 FROM maintenance_jobs
WHERE server = ? AND server_deleted_at IS NULL AND target_revision = ? AND kind = 'update'`).get(v.server, freshRelease())).map(v => v.server);
if (
servers &&
(!servers.length || servers.some((server) => !versions.some((v) => v.server === server)))
)
throw new Error('Choose existing fleet servers.');
const candidates = versions.filter(
(v) => v.update_status === 'outdated' && (!servers || servers.includes(v.server))
);
const names = candidates
.filter(
(v) =>
!automatic ||
!db()
.prepare(
`SELECT 1 FROM maintenance_jobs
WHERE server = ? AND server_deleted_at IS NULL AND target_revision = ? AND kind = 'update'`
)
.get(v.server, freshRelease())
)
.map((v) => v.server);
return names.length ? queueMaintenance(names, 'update', automatic ? 'automatic' : 'manual') : 0;
}
export function retryMaintenance(id: number): number {
const job = db().prepare("SELECT * FROM maintenance_jobs WHERE id = ? AND state = 'failed'").get(id) as MaintenanceJob | undefined;
const job = db()
.prepare("SELECT * FROM maintenance_jobs WHERE id = ? AND state = 'failed'")
.get(id) as MaintenanceJob | undefined;
if (!job) throw new Error('Only failed maintenance can be retried.');
if (job.server_deleted_at) throw new Error('This server was deleted; its maintenance history cannot be retried.');
if (job.server_deleted_at)
throw new Error('This server was deleted; its maintenance history cannot be retried.');
if (!maintenanceAllowed()) throw new Error('Maintenance is paused.');
if (getServer(job.server)?.desired_state === 'stopped') throw new Error('This server is intentionally stopped.');
if (getServer(job.server)?.desired_state === 'stopped')
throw new Error('This server is intentionally stopped.');
// Ambiguous lifecycle failures retain their slot and require diagnostics, not another request.
if (job.provisioning_id) {
const operation = db().prepare('SELECT state FROM server_deployments WHERE id = ?').get(job.provisioning_id) as { state: string } | undefined;
if (operation?.state !== 'done') throw new Error('Inspect and recover the existing Provisioning operation before retrying.');
const operation = db()
.prepare('SELECT state FROM server_deployments WHERE id = ?')
.get(job.provisioning_id) as { state: string } | undefined;
if (operation?.state !== 'done')
throw new Error(
'Inspect and recover the existing Provisioning operation before retrying.'
);
}
return queueMaintenance([job.server], job.kind, 'manual');
}
export function cancelMaintenance(id: number): boolean {
return db().transaction(() => {
const job = db().prepare("SELECT provisioning_id FROM maintenance_jobs WHERE id = ? AND state = 'queued'").get(id) as { provisioning_id: string | null } | undefined;
const job = db()
.prepare(
"SELECT provisioning_id FROM maintenance_jobs WHERE id = ? AND state = 'queued'"
)
.get(id) as { provisioning_id: string | null } | undefined;
if (!job) return false;
if (job.provisioning_id && !cancelProvisioning(job.provisioning_id)) return false;
return db().prepare(`UPDATE maintenance_jobs SET state = 'cancelled', finished_at = ?, detail = 'Cancelled by operator'
WHERE id = ? AND state = 'queued'`).run(nowIso(), id).changes > 0;
return (
db()
.prepare(
`UPDATE maintenance_jobs SET state = 'cancelled', finished_at = ?, detail = 'Cancelled by operator'
WHERE id = ? AND state = 'queued'`
)
.run(nowIso(), id).changes > 0
);
})();
}
function finish(job: MaintenanceJob, state: 'done' | 'failed' | 'cancelled', detail: string) {
db().prepare('UPDATE maintenance_jobs SET state = ?, detail = ?, finished_at = ? WHERE id = ? AND server_deleted_at IS NULL')
db()
.prepare(
'UPDATE maintenance_jobs SET state = ?, detail = ?, finished_at = ? WHERE id = ? AND server_deleted_at IS NULL'
)
.run(state, detail, nowIso(), job.id);
}
function hold(job: MaintenanceJob, detail: string) {
db().prepare('UPDATE maintenance_jobs SET detail = ? WHERE id = ? AND server_deleted_at IS NULL').run(detail, job.id);
db()
.prepare(
'UPDATE maintenance_jobs SET detail = ? WHERE id = ? AND server_deleted_at IS NULL'
)
.run(detail, job.id);
}
let ticking: Promise<void> | null = null;
export function maintenanceTick(): Promise<void> {
if (ticking) return ticking;
ticking = runTick().finally(() => { ticking = null; });
ticking = runTick().finally(() => {
ticking = null;
});
return ticking;
}
async function runTick(): Promise<void> {
@@ -103,51 +184,94 @@ async function runTick(): Promise<void> {
// Provisioning refreshes the manifest independently; that must not postpone
// the hourly version checks for the rest of the fleet.
const versionsCheckedAt = readSetting<string | null>('plutonium_versions_checked_at', null);
if (!latest.checkedAt || Date.now() - Date.parse(latest.checkedAt) >= HOUR
|| !versionsCheckedAt || Date.now() - Date.parse(versionsCheckedAt) >= HOUR) await checkPlutonium();
if (
!latest.checkedAt ||
Date.now() - Date.parse(latest.checkedAt) >= HOUR ||
!versionsCheckedAt ||
Date.now() - Date.parse(versionsCheckedAt) >= HOUR
)
await checkPlutonium();
if (maintenanceAllowed('updates') && freshRelease() !== null) queueUpdates(undefined, true);
db().transaction(() => {
const plan = schedule();
if (maintenanceAllowed() && plan.enabled && plan.nextAt && plan.nextAt <= nowIso()) {
const names = listServers().filter(s => s.desired_state !== 'stopped').map(s => s.name);
const names = listServers()
.filter((s) => s.desired_state !== 'stopped')
.map((s) => s.name);
if (names.length) queueMaintenance(names, 'schedule', 'schedule');
// Coalesce missed days after downtime into one run; no restart backlog.
writeSetting('plutonium_schedule', { ...plan, nextAt: nextMaintenanceAt(plan.time, new Date(), plan.timeZone) });
writeSetting('plutonium_schedule', {
...plan,
nextAt: nextMaintenanceAt(plan.time, new Date(), plan.timeZone)
});
}
})();
const jobs = db().prepare(`SELECT * FROM maintenance_jobs WHERE server_deleted_at IS NULL AND state IN ('queued','restarting') ORDER BY id`).all() as MaintenanceJob[];
const jobs = db()
.prepare(
`SELECT * FROM maintenance_jobs WHERE server_deleted_at IS NULL AND state IN ('queued','restarting') ORDER BY id`
)
.all() as MaintenanceJob[];
for (const job of jobs) {
try { await processJob(job); }
catch { hold(job, 'Maintenance check failed; waiting for the next check.'); }
try {
await processJob(job);
} catch {
hold(job, 'Maintenance check failed; waiting for the next check.');
}
}
}
async function processJob(job: MaintenanceJob): Promise<void> {
const server = getServer(job.server);
if (job.server_deleted_at || !server || server.destroyed_at) return finish(job, 'cancelled', 'Server removed.');
if (job.server_deleted_at || !server || server.destroyed_at)
return finish(job, 'cancelled', 'Server removed.');
if (!job.provisioning_id && !job.restarted_at && hasActiveProvisioning(job.server)) {
return hold(job, 'Waiting for another provisioning operation.');
}
const operation = requestRestart(job.server, { kind: 'maintenance', id: String(job.id) });
if (['failed','interrupted','cancelled'].includes(operation.state)) {
return finish(job, operation.state === 'cancelled' ? 'cancelled' : 'failed', operation.error ?? operation.stage);
if (['failed', 'interrupted', 'cancelled'].includes(operation.state)) {
return finish(
job,
operation.state === 'cancelled' ? 'cancelled' : 'failed',
operation.error ?? operation.stage
);
}
if (operation.state !== 'done') {
if (operation.submitted_at) {
db().prepare("UPDATE maintenance_jobs SET state = 'restarting', restarted_at = ? WHERE id = ?").run(operation.submitted_at, job.id);
db().prepare('UPDATE plutonium_versions SET revision = NULL, error = ? WHERE server = ?').run('Waiting for restart recovery.', job.server);
db()
.prepare(
"UPDATE maintenance_jobs SET state = 'restarting', restarted_at = ? WHERE id = ?"
)
.run(operation.submitted_at, job.id);
db()
.prepare(
'UPDATE plutonium_versions SET revision = NULL, error = ? WHERE server = ?'
)
.run('Waiting for restart recovery.', job.server);
}
return hold(job, operation.stage);
}
if (hasActiveProvisioning(job.server, operation.id)) {
return hold(job, 'Waiting for another provisioning operation before verifying the running revision.');
return hold(
job,
'Waiting for another provisioning operation before verifying the running revision.'
);
}
const revision = await probeVersion(job.server, { maintenance: true });
if (hasActiveProvisioning(job.server, operation.id)) {
return hold(job, 'Waiting for another provisioning operation before verifying the running revision.');
return hold(
job,
'Waiting for another provisioning operation before verifying the running revision.'
);
}
if (job.target_revision && (revision === null || revision < job.target_revision)) {
return finish(job, 'failed', `Server recovered, but r${job.target_revision} was not verified (${revision === null ? 'version unknown' : `running r${revision}`}). Check Plutainer auto-update settings.`);
return finish(
job,
'failed',
`Server recovered, but r${job.target_revision} was not verified (${revision === null ? 'version unknown' : `running r${revision}`}). Check Plutainer auto-update settings.`
);
}
return finish(job, 'done', `Server recovered; ${revision === null ? 'running revision unknown' : `running r${revision}`}.`);
return finish(
job,
'done',
`Server recovered; ${revision === null ? 'running revision unknown' : `running r${revision}`}.`
);
}