josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: make provisioning own instance lifecycle operations

Browse Source
This commit is contained in:
josh
2026-09-27 14:45:00 -04:00
parent cfed584614
commit df265d1900
44 changed files with 1791 additions and 1609 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/fleet/lifecycle.ts
+2 -117
View File
@@ -1,117 +1,2 @@
import { db } from '../db/index.js';
import { komodoConfigured, rconConfigured } from '../config.js';
import { getServer } from './servers.js';
import { beginDeployment, deploymentStage, finishDeployment, REMOVING_SERVER } from './deployments.js';
import { queueRebuildServer, rebuildServer } from './create.js';
import { komodoInspectHealthy, komodoRuntimeAction } from '../integrations/komodo.js';
import { rconGetPlayerCount } from '../integrations/rcon.js';
import { refreshRestartRollout, restartInRollout, rolloutWaitReason } from '../pipeline/restart-rollout.js';
import { runtimeActions, type RuntimeAction } from '../../provisioning.js';
const stoppedStates = new Set(['stopped', 'exited', 'created', 'dead']);
export const isStopped = (state?: string) => stoppedStates.has(state?.toLowerCase() ?? '');
function blocker(server: string, id = ''): string | null {
if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND id != ?
AND state IN ('queued','waiting','running','awaiting-health')`).get(server, id)) return 'Waiting for another provisioning operation on this server';
if (db().prepare(`SELECT 1 FROM deployment_servers WHERE server = ? AND restart_state IN ('pending','failed','held')
UNION ALL SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
WHERE bs.server = ? AND b.state IN ('running','ready-promote')
UNION ALL SELECT 1 FROM maintenance_jobs WHERE server = ? AND state IN ('queued','restarting') LIMIT 1`).get(server, server, server)) {
return 'Waiting for campaign, beta, or maintenance work on this server';
}
return rolloutWaitReason(server, id);
}
export function requestLifecycle(server: string, action: RuntimeAction) {
if (!runtimeActions.includes(action)) return { ok: false as const, status: 400, error: 'Choose start, restart, rebuild, or stop.' };
if (!komodoConfigured() || !rconConfigured()) return { ok: false as const, status: 503, error: 'Configure Komodo and RCON before provisioning.' };
return db().transaction(() => {
const existing = db().prepare(`SELECT id, action FROM server_deployments WHERE server = ?
AND state IN ('queued','waiting','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(server) as { id: string; action: string } | undefined;
if (existing) return existing.action === action
? { ok: true as const, deploymentId: existing.id }
: { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' };
const row = getServer(server);
if (!row || row.destroyed_at) return { ok: false as const, status: 404, error: 'Server not found.' };
if (action === 'rebuild') {
const result = queueRebuildServer(server);
return result.ok ? { ok: true as const, deploymentId: result.deploymentId }
: { ok: false as const, status: 400, error: result.message };
}
const id = beginDeployment(server, action);
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
deploymentStage(id, action === 'start' ? 'Waiting to start the container' : 'Waiting to verify the server is empty');
return { ok: true as const, deploymentId: id };
})();
}
interface Operation { id: string; server: string; action: RuntimeAction; state: string; stage: string }
let ticking: Promise<void> | null = null;
export function processLifecycle(): Promise<void> {
if (ticking) return ticking;
ticking = run().finally(() => { ticking = null; });
return ticking;
}
async function run(): Promise<void> {
await refreshRestartRollout();
const operations = db().prepare(`SELECT id, server, action, state, stage FROM server_deployments
WHERE action IN ('start','restart','stop','rebuild') AND (state = 'waiting' OR (action = 'stop' AND state = 'awaiting-health'))
ORDER BY started_at, rowid`).all() as Operation[];
for (const operation of operations) {
try { await processOperation(operation); }
catch (err) {
// Observation failures are retried; remote mutations are never replayed.
deploymentStage(operation.id, 'Waiting for verification: ' + (err as Error).message);
}
}
}
async function processOperation(operation: Operation): Promise<void> {
const { id, server, action } = operation;
if (action === 'rebuild' && operation.stage === REMOVING_SERVER) { await rebuildServer(id); return; }
if (!getServer(server)) { finishDeployment(id, { ok: false, message: 'Server is no longer registered.' }); return; }
if (!komodoConfigured() || !rconConfigured()) { deploymentStage(id, 'Waiting for Komodo and RCON configuration'); return; }
if (operation.state === 'awaiting-health') {
const health = await komodoInspectHealthy(server);
if (isStopped(health.state)) {
db().prepare("UPDATE servers SET container_state = ?, last_players = NULL, last_roster = NULL, game_started_at = NULL, game_ended_at = NULL WHERE name = ?").run(health.state, server);
deploymentStage(id, 'Verified container stopped');
finishDeployment(id, { ok: true });
} else deploymentStage(id, 'Waiting for container to stop' + (!health.ok ? ': ' + health.reason : ''));
return;
}
const reason = blocker(server, id);
if (reason) { deploymentStage(id, reason); return; }
const health = await komodoInspectHealthy(server);
if (action !== 'start' && !isStopped(health.state)) {
const players = await rconGetPlayerCount(server);
if (players !== 0) {
deploymentStage(id, players === null ? 'Waiting for a fresh player count' : `Waiting for ${players} player(s) to leave`);
return;
}
}
// Requests can arrive during the probes. Recheck before claiming the action.
if (blocker(server, id)) return;
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND state = 'waiting'").get(id)) return;
if (action === 'rebuild') { await rebuildServer(id); return; }
const claim = db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND state = 'waiting'").run(id);
if (!claim.changes) return;
if (action === 'start' || action === 'restart') {
const result = await restartInRollout(server, { operationId: id, action });
if (result.kind === 'queued') {
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
deploymentStage(id, result.message);
}
return;
}
deploymentStage(id, 'Stopping container');
// Persist intent first; a lost reply must not issue another stop later.
try {
const result = await komodoRuntimeAction(server, 'stop');
if (!result.ok) { finishDeployment(id, { ok: false, message: result.message }); return; }
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
deploymentStage(id, 'Waiting for container to stop');
} catch (err) { finishDeployment(id, { ok: false, message: (err as Error).message }); }
}
export { requestLifecycle } from '../provisioning/requests.js';
export { processLifecycle } from '../provisioning/worker.js';