josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: make provisioning own instance lifecycle operations

Browse Source
This commit is contained in:
josh
2026-09-27 14:45:00 -04:00
parent cfed584614
commit df265d1900
44 changed files with 1791 additions and 1609 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/fleet/plutonium.ts
+28 -45
View File
@@ -1,8 +1,8 @@
import { db } from '../db/index.js';
import { config, komodoConfigured, rconConfigured } from '../config.js';
import { config } from '../config.js';
import { getServer, listServers } from './servers.js';
import { rconGetPlayerCount, rconPlutoniumVersion } from '../integrations/rcon.js';
import { refreshRestartRollout, restartInRollout, rolloutHealth, rolloutWaitReason } from '../pipeline/restart-rollout.js';
import { rconPlutoniumVersion } from '../integrations/rcon.js';
import { requestRestart, cancelProvisioning, restartRecoveryPending } from '../provisioning/index.js';
import { nextMaintenanceAt, parsePlutoniumManifest, PLUTONIUM_MANIFEST_URL } from '../../plutonium.js';
import type { MaintenanceJob, MaintenanceSchedule, PlutoniumRelease, PlutoniumStatus, PlutoniumVersion } from '../../plutonium.js';
@@ -39,7 +39,7 @@ export function plutoniumStatus(): PlutoniumStatus {
}
async function probeVersion(server: string): Promise<number | null> {
const recovering = db().prepare('SELECT 1 FROM restart_rollout WHERE server = ? AND healthy_at IS NULL').get(server);
const recovering = restartRecoveryPending(server);
const result = recovering ? { revision: null, error: 'Waiting for restart recovery.' }
: await rconPlutoniumVersion(server).catch(() => ({ revision: null, error: 'Version check failed.' }));
// Clear the old revision on failure: it is no longer evidence of what is running.
@@ -84,21 +84,22 @@ export function queueMaintenance(servers: string[]): number {
for (const server of new Set(servers)) {
added += db().prepare(`INSERT OR IGNORE INTO maintenance_jobs (server, target_revision, requested_at)
VALUES (?, ?, ?)`).run(server, target, nowIso()).changes;
const job = db().prepare("SELECT id FROM maintenance_jobs WHERE server = ? AND state IN ('queued','restarting')").get(server) as { id: number };
requestRestart(server, { kind: 'maintenance', id: String(job.id) });
}
return added;
})();
}
export function cancelMaintenance(id: number): boolean {
return db().prepare(`UPDATE maintenance_jobs SET state = 'cancelled', finished_at = ?, detail = 'Cancelled by operator'
WHERE id = ? AND state = 'queued'`).run(nowIso(), id).changes > 0;
return db().transaction(() => {
const job = db().prepare("SELECT provisioning_id FROM maintenance_jobs WHERE id = ? AND state = 'queued'").get(id) as { provisioning_id: string | null } | undefined;
if (!job) return false;
if (job.provisioning_id && !cancelProvisioning(job.provisioning_id)) return false;
return db().prepare(`UPDATE maintenance_jobs SET state = 'cancelled', finished_at = ?, detail = 'Cancelled by operator'
WHERE id = ? AND state = 'queued'`).run(nowIso(), id).changes > 0;
})();
}
function conflictingWork(server: string): boolean {
return !!db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND state IN ('queued','running','awaiting-health')
UNION ALL SELECT 1 FROM deployment_servers WHERE server = ? AND restart_state IN ('pending','failed')
UNION ALL SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
WHERE bs.server = ? AND b.state IN ('running','ready-promote') LIMIT 1`).get(server, server, server);
}
function finish(job: MaintenanceJob, state: 'done' | 'failed' | 'cancelled', detail: string) {
db().prepare('UPDATE maintenance_jobs SET state = ?, detail = ?, finished_at = ? WHERE id = ?')
.run(state, detail, nowIso(), job.id);
@@ -125,7 +126,6 @@ async function runTick(): Promise<void> {
writeSetting('plutonium_schedule', { ...plan, nextAt: nextMaintenanceAt(plan.time, new Date(), plan.timeZone) });
}
})();
await refreshRestartRollout();
const jobs = db().prepare(`SELECT * FROM maintenance_jobs WHERE state IN ('queued','restarting') ORDER BY id`).all() as MaintenanceJob[];
for (const job of jobs) {
try { await processJob(job); }
@@ -137,37 +137,20 @@ async function runTick(): Promise<void> {
async function processJob(job: MaintenanceJob): Promise<void> {
const server = getServer(job.server);
if (!server || server.destroyed_at) return finish(job, 'cancelled', 'Server removed.');
if (job.state === 'restarting') {
const slot = db().prepare('SELECT requested_at FROM restart_rollout WHERE server = ?').get(job.server) as { requested_at: string } | undefined;
if (!slot || !job.restarted_at || slot.requested_at < job.restarted_at) {
return finish(job, 'failed', 'Restart submission was interrupted. Inspect the server before queuing another restart.');
}
const health = rolloutHealth(job.server);
if (!health.ok) return hold(job, health.reason);
const revision = await probeVersion(job.server);
if (job.target_revision && (revision === null || revision < job.target_revision)) {
return finish(job, 'failed', `Server recovered, but r${job.target_revision} was not verified (${revision === null ? 'version unknown' : `running r${revision}`}). Check Plutainer auto-update settings.`);
}
return finish(job, 'done', `Server recovered; ${revision === null ? 'running revision unknown' : `running r${revision}`}.`);
const operation = requestRestart(job.server, { kind: 'maintenance', id: String(job.id) });
if (['failed','interrupted','cancelled'].includes(operation.state)) {
return finish(job, operation.state === 'cancelled' ? 'cancelled' : 'failed', operation.error ?? operation.stage);
}
if (!komodoConfigured() || !rconConfigured()) return hold(job, 'Configure Komodo and RCON before restarting.');
if (conflictingWork(job.server)) return hold(job, 'Waiting for deployment or beta work on this server.');
const reason = rolloutWaitReason(job.server);
if (reason) return hold(job, reason);
const players = await rconGetPlayerCount(job.server);
if (players === null || players > 0) return hold(job, players === null ? 'Waiting for a fresh RCON player count.' : `Waiting for ${players} player(s) to leave.`);
// Recheck after async I/O; cancellation and lifecycle operations may have arrived.
const current = getServer(job.server);
if (!current || current.destroyed_at) return finish(job, 'cancelled', 'Server removed.');
if (conflictingWork(job.server)) return hold(job, 'Waiting for deployment or beta work on this server.');
const claimed = db().prepare(`UPDATE maintenance_jobs SET state = 'restarting', restarted_at = ?, detail = 'Restart requested'
WHERE id = ? AND state = 'queued'`).run(nowIso(), job.id);
if (!claimed.changes) return;
db().prepare('UPDATE plutonium_versions SET revision = NULL, error = ? WHERE server = ?')
.run('Waiting for restart recovery.', job.server);
// Persist intent before remote I/O. An ambiguous timeout must never cause an automatic second restart.
const result = await restartInRollout(job.server);
if (result.kind === 'queued') {
db().prepare(`UPDATE maintenance_jobs SET state = 'queued', restarted_at = NULL, detail = ? WHERE id = ?`).run(result.message, job.id);
} else if (result.kind === 'failed') finish(job, 'failed', result.message);
if (operation.state !== 'done') {
if (operation.submitted_at) {
db().prepare("UPDATE maintenance_jobs SET state = 'restarting', restarted_at = ? WHERE id = ?").run(operation.submitted_at, job.id);
db().prepare('UPDATE plutonium_versions SET revision = NULL, error = ? WHERE server = ?').run('Waiting for restart recovery.', job.server);
}
return hold(job, operation.stage);
}
const revision = await probeVersion(job.server);
if (job.target_revision && (revision === null || revision < job.target_revision)) {
return finish(job, 'failed', `Server recovered, but r${job.target_revision} was not verified (${revision === null ? 'version unknown' : `running r${revision}`}). Check Plutainer auto-update settings.`);
}
return finish(job, 'done', `Server recovered; ${revision === null ? 'running revision unknown' : `running r${revision}`}.`);
}