refactor: make provisioning own instance lifecycle operations
This commit is contained in:
@@ -0,0 +1,152 @@
|
||||
import { config } from '../config.js';
|
||||
import { db } from '../db/index.js';
|
||||
import { komodoInspectHealthy, komodoRestart, komodoRuntimeAction } from '../integrations/komodo.js';
|
||||
import { rconStatusResult } from '../integrations/rcon.js';
|
||||
import { deploymentStage, finishDeployment } from './history.js';
|
||||
|
||||
// Komodo queues restart work. Don't mistake the still-running old container
|
||||
// for a completed restart when an API kick immediately follows the request.
|
||||
const HEALTH_POLL_MS = 15_000;
|
||||
const requestsInFlight = new Set<string>();
|
||||
|
||||
interface RestartSlot {
|
||||
server: string;
|
||||
requested_at: string;
|
||||
checked_at: string | null;
|
||||
healthy_at: string | null;
|
||||
players: number | null;
|
||||
error: string | null;
|
||||
operation_id: string | null;
|
||||
}
|
||||
|
||||
function slotFor(server: string): RestartSlot | undefined {
|
||||
return db().prepare('SELECT * FROM restart_rollout WHERE server = ?').get(server) as RestartSlot | undefined;
|
||||
}
|
||||
|
||||
export function restartRecoveryPending(server: string): boolean {
|
||||
return slotFor(server)?.healthy_at === null;
|
||||
}
|
||||
|
||||
/** The queued restart may still expose the previous healthy container. */
|
||||
export function restartSettling(server: string): boolean {
|
||||
const slot = slotFor(server);
|
||||
return requestsInFlight.has(server) || !!(slot && !slot.healthy_at
|
||||
&& Date.now() - Date.parse(slot.requested_at) < HEALTH_POLL_MS);
|
||||
}
|
||||
|
||||
export function rolloutWaitReason(server: string, operationId = ''): string | null {
|
||||
if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND id != ?
|
||||
AND ((action IN ('start','stop','rebuild') AND state IN ('running','awaiting-health'))
|
||||
OR (action = 'destroy' AND state IN ('waiting','running')))`).get(server, operationId)) {
|
||||
return 'Waiting for the current provisioning operation';
|
||||
}
|
||||
const slot = slotFor(server);
|
||||
if (slot?.healthy_at === null) {
|
||||
return `Waiting for the previous restart of this server to recover${slot.error ? `: ${slot.error}` : ''}`;
|
||||
}
|
||||
const { n } = db().prepare('SELECT COUNT(*) AS n FROM restart_rollout WHERE healthy_at IS NULL').get() as { n: number };
|
||||
return n >= config.restartConcurrency ? 'Queued for a rolling restart; waiting for another server to recover' : null;
|
||||
}
|
||||
|
||||
export type RolloutRestartResult =
|
||||
| { kind: 'started' }
|
||||
| { kind: 'queued'; message: string }
|
||||
| { kind: 'failed'; message: string };
|
||||
|
||||
/** Reserve capacity before any async I/O; callers still check for players first. */
|
||||
export async function restartInRollout(server: string, options: { operationId: string; action: 'start' | 'restart' }): Promise<RolloutRestartResult> {
|
||||
const operationId = options.operationId;
|
||||
const waitReason = db().transaction(() => {
|
||||
const reason = rolloutWaitReason(server, operationId);
|
||||
if (reason) return reason;
|
||||
db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), operationId);
|
||||
db().prepare(`INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)
|
||||
ON CONFLICT(server) DO UPDATE SET requested_at = excluded.requested_at,
|
||||
checked_at = NULL, healthy_at = NULL, players = NULL, error = NULL, operation_id = excluded.operation_id`)
|
||||
.run(server, new Date().toISOString(), operationId);
|
||||
deploymentStage(operationId, options.action === 'start' ? 'Starting container' : 'Restarting container');
|
||||
return null;
|
||||
}).immediate();
|
||||
if (waitReason) return { kind: 'queued', message: waitReason };
|
||||
|
||||
requestsInFlight.add(server);
|
||||
db().prepare(`UPDATE servers SET container_state = 'restarting', last_error = NULL, last_reply_snippet = NULL,
|
||||
game_started_at = NULL, game_ended_at = NULL WHERE name = ?`).run(server);
|
||||
try {
|
||||
const result = options.action === 'start' ? await komodoRuntimeAction(server, 'start') : await komodoRestart(server);
|
||||
if (result.ok) {
|
||||
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(operationId);
|
||||
deploymentStage(operationId!, 'Waiting for container and RCON health');
|
||||
return { kind: 'started' };
|
||||
}
|
||||
// These failures cannot have queued a remote restart. Other failures
|
||||
// (especially a timeout) may have: retain capacity until recovery.
|
||||
if (result.kind === 'no-name' || result.kind === 'unconfigured') {
|
||||
db().prepare('DELETE FROM restart_rollout WHERE server = ?').run(server);
|
||||
} else {
|
||||
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ?').run(`komodo: ${result.message}`, server);
|
||||
}
|
||||
finishDeployment(operationId!, { ok: false, message: `komodo: ${result.message}` });
|
||||
return { kind: 'failed', message: `komodo: ${result.message}` };
|
||||
} catch (err) {
|
||||
const message = `komodo: ${(err as Error).message}`;
|
||||
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ?').run(message, server);
|
||||
finishDeployment(operationId!, { ok: false, message });
|
||||
return { kind: 'failed', message };
|
||||
} finally {
|
||||
// Start the grace period at completion, including slow/failed requests.
|
||||
db().prepare('UPDATE restart_rollout SET requested_at = ? WHERE server = ?')
|
||||
.run(new Date().toISOString(), server);
|
||||
requestsInFlight.delete(server);
|
||||
}
|
||||
}
|
||||
|
||||
let refreshInFlight: Promise<void> | null = null;
|
||||
|
||||
/** Poll all occupied slots, even if their originating beta was already closed. */
|
||||
export function refreshRestartRollout(): Promise<void> {
|
||||
if (refreshInFlight) return refreshInFlight;
|
||||
refreshInFlight = refresh().finally(() => { refreshInFlight = null; });
|
||||
return refreshInFlight;
|
||||
}
|
||||
|
||||
async function refresh(): Promise<void> {
|
||||
const slots = db().prepare('SELECT * FROM restart_rollout WHERE healthy_at IS NULL ORDER BY requested_at, server')
|
||||
.all() as RestartSlot[];
|
||||
for (const slot of slots) {
|
||||
if (requestsInFlight.has(slot.server)) continue;
|
||||
if (Date.now() - Date.parse(slot.checked_at ?? slot.requested_at) < HEALTH_POLL_MS) continue;
|
||||
let error: string | null = null;
|
||||
let players: number | null = null;
|
||||
try {
|
||||
const health = await komodoInspectHealthy(slot.server);
|
||||
if (!health.ok) {
|
||||
error = `unhealthy: ${health.reason}${health.snippet ? ` snippet=${health.snippet.slice(0, 400)}` : ''}`;
|
||||
} else {
|
||||
const rcon = await rconStatusResult(slot.server);
|
||||
if (!rcon.ok) error = `rcon not answering: ${rcon.reason}`;
|
||||
else players = rcon.status.players;
|
||||
}
|
||||
} catch (err) {
|
||||
error = `health check: ${(err as Error).message}`;
|
||||
}
|
||||
const now = new Date().toISOString();
|
||||
db().prepare(`UPDATE restart_rollout SET checked_at = ?, healthy_at = ?, players = ?, error = ?
|
||||
WHERE server = ? AND requested_at = ? AND healthy_at IS NULL`)
|
||||
.run(now, error ? null : now, players, error, slot.server, slot.requested_at);
|
||||
if (slot.operation_id && db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND state = 'awaiting-health'").get(slot.operation_id)) {
|
||||
db().prepare('UPDATE server_deployments SET checked_at = ?, players = ? WHERE id = ?').run(now, players, slot.operation_id);
|
||||
if (error) deploymentStage(slot.operation_id, 'Waiting for health: ' + error);
|
||||
else {
|
||||
deploymentStage(slot.operation_id, 'Verified container and RCON health');
|
||||
finishDeployment(slot.operation_id, { ok: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function rolloutHealth(server: string): { ok: true; players: number } | { ok: false; reason: string } {
|
||||
const slot = slotFor(server);
|
||||
if (slot?.healthy_at) return { ok: true, players: slot.players ?? 0 };
|
||||
return { ok: false, reason: slot?.error ?? 'Waiting for restart recovery and health checks' };
|
||||
}
|
||||
Reference in New Issue
Block a user