248 lines
17 KiB
TypeScript
248 lines
17 KiB
TypeScript
import { db } from '../db/index.js';
|
|
import { komodoConfigured, rconConfigured } from '../config.js';
|
|
import { getServer } from '../fleet/servers.js';
|
|
import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent } from './history.js';
|
|
import { queueRebuildServer } from './instances.js';
|
|
import { restartRequestInFlight } from './restarts.js';
|
|
import { runtimeActions, type RuntimeAction } from '../../provisioning.js';
|
|
|
|
export interface LifecycleOperation {
|
|
id: string; server: string; action: string; state: string; stage: string;
|
|
submitted_at: string | null; finished_at: string | null; checked_at: string | null;
|
|
players: number | null; error: string | null;
|
|
started_at: string;
|
|
}
|
|
export interface RestartSource { kind: 'campaign' | 'beta' | 'maintenance'; id: string }
|
|
export function provisioningOperation(id: string): LifecycleOperation | undefined {
|
|
return db().prepare('SELECT * FROM server_deployments WHERE id = ?').get(id) as LifecycleOperation | undefined;
|
|
}
|
|
|
|
const sourceTables = {
|
|
campaign: { table: 'deployment_servers', key: 'deployment_id' },
|
|
beta: { table: 'beta_servers', key: 'beta_id' },
|
|
maintenance: { table: 'maintenance_jobs', key: 'id' }
|
|
} as const;
|
|
|
|
/** Submit once per caller attempt; a lost response or terminal failure never replays a restart. */
|
|
export function requestRestart(server: string, source: RestartSource): LifecycleOperation {
|
|
return db().transaction(() => {
|
|
const { table, key } = sourceTables[source.kind];
|
|
const row = db().prepare(`SELECT provisioning_id, restarted_at FROM ${table} WHERE ${key} = ? AND server = ?`)
|
|
.get(source.id, server) as { provisioning_id: string | null; restarted_at: string | null } | undefined;
|
|
if (!row) throw new Error('Restart source no longer exists');
|
|
if (row.provisioning_id) {
|
|
const operation = provisioningOperation(row.provisioning_id);
|
|
if (!operation) throw new Error('Provisioning job is missing; inspect the server before retrying');
|
|
if (row.restarted_at && !operation.submitted_at && operation.state === 'waiting') {
|
|
adoptSubmittedRestart(operation.id, server, row.restarted_at);
|
|
return provisioningOperation(operation.id)!;
|
|
}
|
|
return operation;
|
|
}
|
|
let id: string | undefined;
|
|
if (row.restarted_at) {
|
|
// The previous release already recorded some submissions in history.
|
|
// Bind to that exact interval, never to a newer restart of the server.
|
|
id = (db().prepare(`SELECT d.id FROM restart_rollout r JOIN server_deployments d ON d.id = r.operation_id
|
|
WHERE r.server = ? AND d.action = 'restart' AND d.started_at <= ? AND r.requested_at >= ?`)
|
|
.get(server, row.restarted_at, row.restarted_at) as { id: string } | undefined)?.id;
|
|
if (id) db().prepare('UPDATE server_deployments SET submitted_at = COALESCE(submitted_at, ?) WHERE id = ?').run(row.restarted_at, id);
|
|
}
|
|
// Only not-yet-submitted campaign changes can share an activation. A later
|
|
// file deployment must never be credited to an already running restart.
|
|
if (source.kind === 'campaign' && !row.restarted_at) {
|
|
id = (db().prepare(`SELECT d.id FROM server_deployments d WHERE d.server = ? AND d.action = 'restart'
|
|
AND d.state = 'waiting' AND d.submitted_at IS NULL
|
|
AND EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'campaign')
|
|
AND NOT EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind != 'campaign')
|
|
ORDER BY d.rowid LIMIT 1`).get(server) as { id: string } | undefined)?.id;
|
|
}
|
|
if (!id) {
|
|
id = beginDeployment(server, 'restart', `${source.kind}:${source.id}`);
|
|
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
|
|
deploymentStage(id, 'Waiting to verify the server is empty');
|
|
if (row.restarted_at) adoptSubmittedRestart(id, server, row.restarted_at);
|
|
}
|
|
db().prepare('INSERT OR IGNORE INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, ?, ?, ?)')
|
|
.run(id, source.kind, source.id, server);
|
|
db().prepare(`UPDATE ${table} SET provisioning_id = ? WHERE ${key} = ? AND server = ?`).run(id, source.id, server);
|
|
return provisioningOperation(id)!;
|
|
})();
|
|
}
|
|
|
|
/** Upgrade observation only: existing restart intent is never resubmitted. */
|
|
function adoptSubmittedRestart(id: string, server: string, submittedAt: string): void {
|
|
const slot = db().prepare('SELECT operation_id, requested_at, healthy_at, players FROM restart_rollout WHERE server = ?').get(server) as
|
|
{ operation_id: string | null; requested_at: string; healthy_at: string | null; players: number | null } | undefined;
|
|
db().prepare("UPDATE server_deployments SET state = 'awaiting-health', submitted_at = ? WHERE id = ?").run(submittedAt, id);
|
|
if (slot && ((slot.operation_id && slot.requested_at >= submittedAt) || (slot.requested_at < submittedAt && !slot.healthy_at))) {
|
|
db().prepare("UPDATE server_deployments SET state = 'interrupted', finished_at = ?, error = ? WHERE id = ?")
|
|
.run(new Date().toISOString(), 'Existing restart cannot be correlated safely. Inspect the server before retrying.', id);
|
|
return;
|
|
}
|
|
if (slot && slot.requested_at >= submittedAt) {
|
|
db().prepare('UPDATE restart_rollout SET operation_id = ? WHERE server = ?').run(id, server);
|
|
} else {
|
|
db().prepare(`INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)
|
|
ON CONFLICT(server) DO UPDATE SET requested_at = excluded.requested_at, operation_id = excluded.operation_id,
|
|
checked_at = NULL, healthy_at = NULL, players = NULL, error = NULL`).run(server, submittedAt, id);
|
|
}
|
|
if (slot?.healthy_at && slot.requested_at >= submittedAt) {
|
|
db().prepare("UPDATE server_deployments SET state = 'done', finished_at = ?, players = ? WHERE id = ?").run(slot.healthy_at, slot.players, id);
|
|
}
|
|
deploymentStage(id, 'Adopted existing restart; verifying recovery without restarting again');
|
|
}
|
|
|
|
export function cancelProvisioning(id: string, reason = 'Cancelled by requester'): boolean {
|
|
return db().transaction(() => {
|
|
const result = db().prepare(`UPDATE server_deployments SET state = 'cancelled', finished_at = ?, stage = ?, error = NULL
|
|
WHERE id = ? AND action IN ('start','restart','stop','rebuild') AND state = 'waiting' AND submitted_at IS NULL`)
|
|
.run(new Date().toISOString(), reason, id);
|
|
if (result.changes) recordDeploymentEvent(id, reason);
|
|
return result.changes > 0;
|
|
})();
|
|
}
|
|
|
|
/** Explicit caller retry starts a new job and keeps the old job as history. */
|
|
export function releaseRestartRequest(server: string, source: RestartSource): void {
|
|
const { table, key } = sourceTables[source.kind];
|
|
const row = db().prepare(`SELECT provisioning_id FROM ${table} WHERE ${key} = ? AND server = ?`).get(source.id, server) as { provisioning_id: string | null } | undefined;
|
|
if (row?.provisioning_id) cancelProvisioning(row.provisioning_id, 'Superseded by a fresh caller attempt');
|
|
db().prepare(`UPDATE ${table} SET provisioning_id = NULL WHERE ${key} = ? AND server = ?`).run(source.id, server);
|
|
}
|
|
|
|
export function requestLifecycle(server: string, action: RuntimeAction) {
|
|
if (!runtimeActions.includes(action)) return { ok: false as const, status: 400, error: 'Choose start, restart, rebuild, or stop.' };
|
|
if (!komodoConfigured() || !rconConfigured()) return { ok: false as const, status: 503, error: 'Configure Komodo and RCON before provisioning.' };
|
|
return db().transaction(() => {
|
|
const existing = db().prepare(`SELECT id, action FROM server_deployments WHERE server = ?
|
|
AND state IN ('queued','waiting','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(server) as { id: string; action: string } | undefined;
|
|
if (existing) {
|
|
if (existing.action !== action) return { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' };
|
|
db().prepare('UPDATE servers SET desired_state = ? WHERE name = ?').run(action === 'stop' ? 'stopped' : 'running', server);
|
|
return { ok: true as const, deploymentId: existing.id };
|
|
}
|
|
const row = getServer(server);
|
|
if (!row || row.destroyed_at) return { ok: false as const, status: 404, error: 'Server not found.' };
|
|
if (action === 'rebuild') {
|
|
const result = queueRebuildServer(server);
|
|
if (result.ok) db().prepare("UPDATE servers SET desired_state = 'running' WHERE name = ?").run(server);
|
|
return result.ok ? { ok: true as const, deploymentId: result.deploymentId }
|
|
: { ok: false as const, status: 400, error: result.message };
|
|
}
|
|
const id = beginDeployment(server, action);
|
|
db().prepare('UPDATE servers SET desired_state = ? WHERE name = ?').run(action === 'stop' ? 'stopped' : 'running', server);
|
|
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
|
|
deploymentStage(id, action === 'start' ? 'Waiting to start the container' : 'Waiting to verify the server is empty');
|
|
return { ok: true as const, deploymentId: id };
|
|
})();
|
|
}
|
|
|
|
type LifecycleRequestResult = { ok: true; deploymentId: string } | { ok: false; status: number; error: string };
|
|
|
|
function watchdogRequestGuard(server: string, incidentId: string): Extract<LifecycleRequestResult, { ok: false }> | null {
|
|
const incident = db().prepare("SELECT 1 FROM watchdog_incidents WHERE id = ? AND server = ? AND state IN ('open','recovering')").get(incidentId, server);
|
|
if (!incident) return { ok: false, status: 409, error: 'The Watchdog incident is no longer active.' };
|
|
if (db().prepare("SELECT 1 FROM settings WHERE key = 'watchdog_enabled' AND value = '0'").get()) {
|
|
return { ok: false, status: 409, error: 'Watchdog is paused.' };
|
|
}
|
|
const row = getServer(server);
|
|
if (!row || row.destroyed_at) return { ok: false, status: 404, error: 'Server not found.' };
|
|
if (row.desired_state === 'stopped') return { ok: false, status: 409, error: 'This server is intentionally stopped.' };
|
|
if (!komodoConfigured()) return { ok: false, status: 503, error: 'Configure Komodo before provisioning.' };
|
|
return null;
|
|
}
|
|
|
|
function existingWatchdogAction(incidentId: string, action: 'restart' | 'rebuild'): string | undefined {
|
|
return (db().prepare(`SELECT a.operation_id FROM watchdog_actions a JOIN watchdog_incidents i ON i.id = a.incident_id
|
|
WHERE a.incident_id = ? AND a.action = ? AND a.cycle = i.cycle`).get(incidentId, action) as { operation_id: string } | undefined)?.operation_id;
|
|
}
|
|
|
|
function queueWatchdogAction(server: string, action: 'restart' | 'rebuild', incidentId: string): LifecycleRequestResult {
|
|
let id: string;
|
|
if (action === 'rebuild') {
|
|
const result = queueRebuildServer(server, `watchdog:${incidentId}`);
|
|
if (!result.ok) return { ok: false, status: 400, error: result.message };
|
|
id = result.deploymentId;
|
|
} else {
|
|
id = beginDeployment(server, action, `watchdog:${incidentId}`);
|
|
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
|
|
deploymentStage(id, 'Waiting to verify the server is empty');
|
|
}
|
|
db().prepare("INSERT INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, 'watchdog', ?, ?)").run(id, incidentId, server);
|
|
return { ok: true, deploymentId: id };
|
|
}
|
|
|
|
/** Watchdog submits durable work; Provisioning remains the only execution owner. */
|
|
export function requestWatchdogLifecycle(server: string, action: 'restart' | 'rebuild', incidentId: string): LifecycleRequestResult {
|
|
return db().transaction(() => {
|
|
const previous = existingWatchdogAction(incidentId, action);
|
|
if (previous) return { ok: true as const, deploymentId: previous };
|
|
const error = watchdogRequestGuard(server, incidentId);
|
|
if (error) return error;
|
|
const pending = db().prepare(`SELECT d.id, d.action, s.source_id FROM server_deployments d
|
|
LEFT JOIN provisioning_sources s ON s.operation_id = d.id AND s.kind = 'watchdog'
|
|
WHERE d.server = ? AND d.state IN ('queued','waiting','running','awaiting-health') ORDER BY d.rowid DESC LIMIT 1`)
|
|
.get(server) as { id: string; action: string; source_id: string | null } | undefined;
|
|
if (pending) return pending.source_id === incidentId && pending.action === action
|
|
? { ok: true as const, deploymentId: pending.id }
|
|
: { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' };
|
|
const next = queueWatchdogAction(server, action, incidentId);
|
|
if (!next.ok) return next;
|
|
// An operator retry starts a new incident cycle. Keep the previous
|
|
// failed attempt's capacity, but let that new attempt use it instead
|
|
// of waiting forever behind its own incident's terminal operation.
|
|
const retained = db().prepare(`SELECT r.operation_id FROM restart_rollout r
|
|
JOIN server_deployments d ON d.id = r.operation_id
|
|
JOIN watchdog_actions a ON a.operation_id = d.id AND a.incident_id = ?
|
|
JOIN watchdog_incidents i ON i.id = a.incident_id
|
|
WHERE r.server = ? AND r.healthy_at IS NULL AND d.state IN ('failed','interrupted','cancelled')
|
|
AND a.cycle < i.cycle`).get(incidentId, server) as { operation_id: string } | undefined;
|
|
if (retained && !restartRequestInFlight(server)) {
|
|
db().prepare('UPDATE restart_rollout SET operation_id = ? WHERE server = ? AND operation_id = ? AND healthy_at IS NULL')
|
|
.run(next.deploymentId, server, retained.operation_id);
|
|
recordDeploymentEvent(retained.operation_id, 'Recovery capacity handed to explicit Watchdog retry ' + next.deploymentId);
|
|
recordDeploymentEvent(next.deploymentId, 'Retained recovery capacity from prior incident attempt ' + retained.operation_id);
|
|
}
|
|
return next;
|
|
})();
|
|
}
|
|
|
|
/** Atomically reserve a replacement before handing it the failed restart's capacity. */
|
|
export function escalateWatchdogRestart(incidentId: string, operationId: string, recoveryTimeoutMs = 300_000): LifecycleRequestResult {
|
|
return db().transaction(() => {
|
|
const previous = existingWatchdogAction(incidentId, 'rebuild');
|
|
if (previous) return { ok: true as const, deploymentId: previous };
|
|
const operation = provisioningOperation(operationId);
|
|
if (!operation || operation.action !== 'restart' || !['awaiting-health','failed'].includes(operation.state)
|
|
|| !db().prepare("SELECT 1 FROM provisioning_sources WHERE operation_id = ? AND kind = 'watchdog' AND source_id = ?").get(operationId, incidentId)) {
|
|
return { ok: false as const, status: 409, error: 'The restart is not eligible for Watchdog escalation.' };
|
|
}
|
|
const error = watchdogRequestGuard(operation.server, incidentId);
|
|
if (error) return error;
|
|
const pending = db().prepare(`SELECT d.id, d.action, s.source_id FROM server_deployments d
|
|
LEFT JOIN provisioning_sources s ON s.operation_id = d.id AND s.kind = 'watchdog'
|
|
WHERE d.server = ? AND d.id != ? AND d.state IN ('queued','waiting','running','awaiting-health') LIMIT 1`)
|
|
.get(operation.server, operationId) as { id: string; action: string; source_id: string | null } | undefined;
|
|
if (pending) return pending.action === 'rebuild' && pending.source_id === incidentId
|
|
? { ok: true as const, deploymentId: pending.id }
|
|
: { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' };
|
|
const slot = db().prepare('SELECT operation_id, requested_at, healthy_at FROM restart_rollout WHERE server = ?').get(operation.server) as
|
|
{ operation_id: string | null; requested_at: string; healthy_at: string | null } | undefined;
|
|
const anchor = Date.parse(slot?.requested_at ?? operation.submitted_at ?? operation.started_at);
|
|
if (restartRequestInFlight(operation.server) || (slot && (slot.operation_id !== operationId || slot.healthy_at))
|
|
|| !Number.isFinite(anchor) || Date.now() - anchor < Math.max(300_000, recoveryTimeoutMs)) {
|
|
return { ok: false as const, status: 409, error: 'Waiting for the submitted restart to finish its recovery window.' };
|
|
}
|
|
const replacement = queueWatchdogAction(operation.server, 'rebuild', incidentId);
|
|
if (!replacement.ok) return replacement;
|
|
const reason = 'Watchdog restart recovery timed out; handed recovery to rebuild ' + replacement.deploymentId;
|
|
if (operation.state === 'awaiting-health') finishDeployment(operationId, { ok: false, message: reason });
|
|
else recordDeploymentEvent(operationId, reason);
|
|
if (slot) db().prepare('UPDATE restart_rollout SET operation_id = ?, error = ? WHERE server = ? AND operation_id = ? AND healthy_at IS NULL')
|
|
.run(replacement.deploymentId, reason, operation.server, operationId);
|
|
recordDeploymentEvent(replacement.deploymentId, 'Recovery handed off from restart ' + operationId + '; waiting for a fresh empty or stopped server');
|
|
return replacement;
|
|
})();
|
|
}
|