301 lines
23 KiB
TypeScript
301 lines
23 KiB
TypeScript
import { maintenanceAllowed } from '../maintenance/policies.js';
|
|
import { db } from '../db/index.js';
|
|
import { komodoConfigured, rconConfigured } from '../config.js';
|
|
import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
|
|
import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent } from './history.js';
|
|
import { queueRebuildServer } from './instances.js';
|
|
import { restartRequestInFlight } from './restarts.js';
|
|
import { runtimeActions, type RuntimeAction } from '../../provisioning.js';
|
|
import { recordProgress } from './progress.js';
|
|
import { nativeInstance, unplacedNativeCreation } from '../backends/native.js';
|
|
|
|
export interface LifecycleOperation {
|
|
id: string; server: string; action: string; state: string; stage: string;
|
|
submitted_at: string | null; finished_at: string | null; checked_at: string | null;
|
|
players: number | null; error: string | null;
|
|
started_at: string;
|
|
}
|
|
export interface RestartSource { kind: 'campaign' | 'beta' | 'maintenance'; id: string }
|
|
export function provisioningOperation(id: string): LifecycleOperation | undefined {
|
|
return db().prepare('SELECT * FROM server_deployments WHERE id = ?').get(id) as LifecycleOperation | undefined;
|
|
}
|
|
|
|
const sourceTables = {
|
|
campaign: { table: 'deployment_servers', key: 'deployment_id' },
|
|
beta: { table: 'beta_servers', key: 'beta_id' },
|
|
maintenance: { table: 'maintenance_jobs', key: 'id' }
|
|
} as const;
|
|
|
|
/** Submit once per caller attempt; a lost response or terminal failure never replays a restart. */
|
|
export function requestRestart(server: string, source: RestartSource): LifecycleOperation {
|
|
return db().transaction(() => {
|
|
const { table, key } = sourceTables[source.kind];
|
|
const row = db().prepare(`SELECT provisioning_id, restarted_at FROM ${table} WHERE ${key} = ? AND server = ? AND server_deleted_at IS NULL`)
|
|
.get(source.id, server) as { provisioning_id: string | null; restarted_at: string | null } | undefined;
|
|
if (!row) throw new Error('Restart source no longer exists');
|
|
if (row.provisioning_id) {
|
|
const operation = provisioningOperation(row.provisioning_id);
|
|
if (!operation) throw new Error('Provisioning job is missing; inspect the server before retrying');
|
|
if (row.restarted_at && !operation.submitted_at && operation.state === 'waiting') {
|
|
adoptSubmittedRestart(operation.id, server, row.restarted_at);
|
|
return provisioningOperation(operation.id)!;
|
|
}
|
|
return operation;
|
|
}
|
|
let id: string | undefined;
|
|
if (row.restarted_at) {
|
|
// The previous release already recorded some submissions in history.
|
|
// Bind to that exact interval, never to a newer restart of the server.
|
|
id = (db().prepare(`SELECT d.id FROM restart_rollout r JOIN server_deployments d ON d.id = r.operation_id
|
|
WHERE r.server = ? AND d.server_deleted_at IS NULL AND d.action = 'restart' AND d.started_at <= ? AND r.requested_at >= ?`)
|
|
.get(server, row.restarted_at, row.restarted_at) as { id: string } | undefined)?.id;
|
|
if (id) db().prepare('UPDATE server_deployments SET submitted_at = COALESCE(submitted_at, ?) WHERE id = ?').run(row.restarted_at, id);
|
|
}
|
|
// Only not-yet-submitted campaign changes can share an activation. A later
|
|
// file deployment must never be credited to an already running restart.
|
|
// Restorations need their own activation. An older campaign can have
|
|
// stale file preconditions after a beta; its failure must not strand
|
|
// the rollback that would restore the server.
|
|
const rollback = source.kind === 'campaign' && !!db().prepare("SELECT 1 FROM deployments WHERE id = ? AND action = 'rollback'").get(source.id);
|
|
if (source.kind === 'campaign' && !row.restarted_at && !rollback && !nativeInstance(server)) {
|
|
id = (db().prepare(`SELECT d.id FROM server_deployments d WHERE d.server = ? AND d.server_deleted_at IS NULL AND d.action = 'restart'
|
|
AND d.state = 'waiting' AND d.submitted_at IS NULL
|
|
AND NOT EXISTS (SELECT 1 FROM native_provisioning n WHERE n.deployment_id=d.id AND n.stage_operation_id IS NOT NULL)
|
|
AND EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'campaign')
|
|
AND NOT EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind != 'campaign')
|
|
AND NOT EXISTS (SELECT 1 FROM provisioning_sources s JOIN deployments c ON c.id = s.source_id
|
|
WHERE s.operation_id = d.id AND s.kind = 'campaign' AND c.action = 'rollback')
|
|
ORDER BY d.rowid LIMIT 1`).get(server) as { id: string } | undefined)?.id;
|
|
}
|
|
if (!id) {
|
|
id = beginDeployment(server, 'restart', `${source.kind}:${source.id}`);
|
|
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
|
|
deploymentStage(id, 'Waiting to verify the server is empty', { phase: 'eligibility', observedAt: null,
|
|
blocker: { code: 'eligibility', message: 'Waiting for eligibility and a fresh empty or stopped server.', automatic: true } });
|
|
if (row.restarted_at) adoptSubmittedRestart(id, server, row.restarted_at);
|
|
}
|
|
db().prepare('INSERT OR IGNORE INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, ?, ?, ?)')
|
|
.run(id, source.kind, source.id, server);
|
|
db().prepare(`UPDATE ${table} SET provisioning_id = ? WHERE ${key} = ? AND server = ?`).run(id, source.id, server);
|
|
return provisioningOperation(id)!;
|
|
})();
|
|
}
|
|
|
|
/** Upgrade observation only: existing restart intent is never resubmitted. */
|
|
function adoptSubmittedRestart(id: string, server: string, submittedAt: string): void {
|
|
const slot = db().prepare('SELECT operation_id, requested_at, checked_at, healthy_at, players FROM restart_rollout WHERE server = ?').get(server) as
|
|
{ operation_id: string | null; requested_at: string; checked_at: string | null; healthy_at: string | null; players: number | null } | undefined;
|
|
db().prepare("UPDATE server_deployments SET state = 'awaiting-health', submitted_at = ? WHERE id = ?").run(submittedAt, id);
|
|
recordProgress(id, { command: { state: 'attempted', at: submittedAt, message: 'Adopted a previously recorded submission; no command is replayed.' } });
|
|
if (slot && ((slot.operation_id && slot.requested_at >= submittedAt) || (slot.requested_at < submittedAt && !slot.healthy_at))) {
|
|
const finishedAt = new Date().toISOString();
|
|
const error = 'Existing restart cannot be correlated safely. Inspect the server before retrying.';
|
|
db().prepare("UPDATE server_deployments SET state = 'interrupted', finished_at = ?, error = ? WHERE id = ?")
|
|
.run(finishedAt, error, id);
|
|
recordProgress(id, { recovery: { state: 'unknown', message: 'Existing restart recovery could not be correlated with this operation.', operationId: id },
|
|
observedAt: null, event: { kind: 'outcome', message: error, details: { state: 'interrupted', finishedAt, error } } });
|
|
return;
|
|
}
|
|
if (slot && slot.requested_at >= submittedAt) {
|
|
db().prepare('UPDATE restart_rollout SET operation_id = ? WHERE server = ?').run(id, server);
|
|
} else {
|
|
db().prepare(`INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)
|
|
ON CONFLICT(server) DO UPDATE SET requested_at = excluded.requested_at, operation_id = excluded.operation_id,
|
|
checked_at = NULL, healthy_at = NULL, players = NULL, error = NULL`).run(server, submittedAt, id);
|
|
}
|
|
if (slot?.healthy_at && slot.requested_at >= submittedAt) {
|
|
db().prepare("UPDATE server_deployments SET state = 'done', finished_at = ?, players = ? WHERE id = ?").run(slot.healthy_at, slot.players, id);
|
|
recordProgress(id, { observedAt: slot.healthy_at, event: { kind: 'outcome', message: 'Adopted recorded recovery completion.',
|
|
details: { state: 'done', finishedAt: slot.healthy_at, error: null, warnings: [] } } });
|
|
}
|
|
deploymentStage(id, 'Adopted existing restart; verifying recovery without restarting again', {
|
|
phase: 'verify-container', checks: [], blocker: null, observedAt: slot?.checked_at ?? null,
|
|
recovery: { state: slot?.healthy_at && slot.requested_at >= submittedAt ? 'released' : 'held',
|
|
at: slot?.healthy_at && slot.requested_at >= submittedAt ? slot.healthy_at : undefined,
|
|
message: slot?.healthy_at && slot.requested_at >= submittedAt ? 'Adopted recorded recovery completion.' : 'Observing the previously submitted restart without replaying it.', operationId: id }
|
|
});
|
|
}
|
|
|
|
export function cancelProvisioning(id: string, reason = 'Cancelled by requester'): boolean {
|
|
return db().transaction(() => {
|
|
const finishedAt = new Date().toISOString();
|
|
const result = db().prepare(`UPDATE server_deployments SET state = 'cancelled', finished_at = ?, stage = ?, error = NULL
|
|
WHERE id = ? AND action IN ('start','restart','stop','rebuild') AND state = 'waiting' AND submitted_at IS NULL
|
|
AND NOT EXISTS (SELECT 1 FROM ec_host_migrations m WHERE m.deployment_id=server_deployments.id AND m.remove_operation_id IS NOT NULL)`)
|
|
.run(finishedAt, reason, id);
|
|
if (result.changes) recordProgress(id, { blocker: null, checks: [], event: { kind: 'outcome', message: reason, details: { state: 'cancelled', finishedAt, error: null, warnings: [] } } });
|
|
return result.changes > 0;
|
|
})();
|
|
}
|
|
|
|
/** Explicit caller retry starts a new job and keeps the old job as history. */
|
|
export function releaseRestartRequest(server: string, source: RestartSource): void {
|
|
const { table, key } = sourceTables[source.kind];
|
|
const row = db().prepare(`SELECT provisioning_id FROM ${table} WHERE ${key} = ? AND server = ? AND server_deleted_at IS NULL`).get(source.id, server) as { provisioning_id: string | null } | undefined;
|
|
if (row?.provisioning_id) cancelProvisioning(row.provisioning_id, 'Superseded by a fresh caller attempt');
|
|
db().prepare(`UPDATE ${table} SET provisioning_id = NULL WHERE ${key} = ? AND server = ? AND server_deleted_at IS NULL`).run(source.id, server);
|
|
}
|
|
|
|
export function requestLifecycle(server: string, action: RuntimeAction) {
|
|
if (!runtimeActions.includes(action)) return { ok: false as const, status: 400, error: 'Choose start, restart, rebuild, or stop.' };
|
|
if (!nativeInstance(server) && !unplacedNativeCreation(server) && (!komodoConfigured() || !rconConfigured())) return { ok: false as const, status: 503, error: 'Configure a host and RCON before provisioning.' };
|
|
return db().transaction(() => {
|
|
const blocker = action === 'rebuild' ? activeBetaRemovalBlocker(server) : null;
|
|
if (blocker) return { ok: false as const, status: 409, error: blocker };
|
|
const existing = db().prepare(`SELECT id, action, state, submitted_at, triggered_by FROM server_deployments WHERE server = ?
|
|
AND server_deleted_at IS NULL
|
|
AND state IN ('queued','waiting','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(server) as
|
|
{ id: string; action: string; state: string; submitted_at: string | null; triggered_by: string } | undefined;
|
|
if (existing) {
|
|
if (existing.action !== action) return { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' };
|
|
db().prepare('UPDATE servers SET desired_state = ? WHERE name = ?').run(action === 'stop' ? 'stopped' : 'running', server);
|
|
if (existing.state === 'waiting' && !existing.submitted_at && existing.triggered_by.startsWith('watchdog:')
|
|
&& db().prepare(`SELECT 1 FROM provisioning_sources s JOIN watchdog_incidents i ON i.id = s.source_id
|
|
WHERE s.operation_id = ? AND s.kind = 'watchdog' AND i.kind = 'health'`).get(existing.id)) {
|
|
// An operator explicitly requesting this queued action keeps
|
|
// that intent even if the original health condition clears.
|
|
db().prepare("UPDATE server_deployments SET triggered_by = 'ui' WHERE id = ?").run(existing.id);
|
|
recordDeploymentEvent(existing.id, 'Operator explicitly requested the queued health recovery action');
|
|
}
|
|
return { ok: true as const, deploymentId: existing.id };
|
|
}
|
|
const row = getServer(server);
|
|
if (!row || row.destroyed_at) return { ok: false as const, status: 404, error: 'Server not found.' };
|
|
if (action === 'rebuild') {
|
|
const result = queueRebuildServer(server);
|
|
if (result.ok) db().prepare("UPDATE servers SET desired_state = 'running' WHERE name = ?").run(server);
|
|
return result.ok ? { ok: true as const, deploymentId: result.deploymentId }
|
|
: { ok: false as const, status: result.kind === 'conflict' ? 409 : 400, error: result.message };
|
|
}
|
|
const id = beginDeployment(server, action);
|
|
db().prepare('UPDATE servers SET desired_state = ? WHERE name = ?').run(action === 'stop' ? 'stopped' : 'running', server);
|
|
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
|
|
deploymentStage(id, action === 'start' ? 'Waiting to start the container' : 'Waiting to verify the server is empty', { phase: 'eligibility', observedAt: null,
|
|
blocker: { code: 'eligibility', message: action === 'start' ? 'Waiting for start eligibility and recovery capacity.' : 'Waiting for eligibility and a fresh empty or stopped server.', automatic: true } });
|
|
return { ok: true as const, deploymentId: id };
|
|
})();
|
|
}
|
|
|
|
type LifecycleRequestResult = { ok: true; deploymentId: string } | { ok: false; status: number; error: string };
|
|
|
|
function watchdogRequestGuard(server: string, incidentId: string): Extract<LifecycleRequestResult, { ok: false }> | null {
|
|
const incident = db().prepare("SELECT kind FROM watchdog_incidents WHERE id = ? AND server = ? AND server_deleted_at IS NULL AND state IN ('open','recovering')").get(incidentId, server) as { kind: 'health' | 'drift' } | undefined;
|
|
if (!incident) return { ok: false, status: 409, error: 'The Watchdog incident is no longer active.' };
|
|
if (!maintenanceAllowed(incident.kind)) {
|
|
return { ok: false, status: 409, error: 'This maintenance policy is paused.' };
|
|
}
|
|
const row = getServer(server);
|
|
if (!row || row.destroyed_at) return { ok: false, status: 404, error: 'Server not found.' };
|
|
if (row.desired_state === 'stopped') return { ok: false, status: 409, error: 'This server is intentionally stopped.' };
|
|
if (!nativeInstance(server) && !komodoConfigured()) return { ok: false, status: 503, error: 'Configure a host before provisioning.' };
|
|
return null;
|
|
}
|
|
|
|
function existingWatchdogAction(incidentId: string, action: 'restart' | 'rebuild'): string | undefined {
|
|
return (db().prepare(`SELECT a.operation_id FROM watchdog_actions a JOIN watchdog_incidents i ON i.id = a.incident_id
|
|
WHERE a.incident_id = ? AND a.action = ? AND a.cycle = i.cycle`).get(incidentId, action) as { operation_id: string } | undefined)?.operation_id;
|
|
}
|
|
|
|
function queueWatchdogAction(server: string, action: 'restart' | 'rebuild', incidentId: string): LifecycleRequestResult {
|
|
let id: string;
|
|
if (action === 'rebuild') {
|
|
const result = queueRebuildServer(server, `watchdog:${incidentId}`);
|
|
if (!result.ok) return { ok: false, status: result.kind === 'conflict' ? 409 : 400, error: result.message };
|
|
id = result.deploymentId;
|
|
} else {
|
|
id = beginDeployment(server, action, `watchdog:${incidentId}`);
|
|
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
|
|
deploymentStage(id, 'Waiting to verify the server is empty', { phase: 'eligibility', observedAt: null,
|
|
blocker: { code: 'eligibility', message: 'Waiting for eligibility and a fresh empty or stopped server.', automatic: true } });
|
|
}
|
|
db().prepare("INSERT INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, 'watchdog', ?, ?)").run(id, incidentId, server);
|
|
return { ok: true, deploymentId: id };
|
|
}
|
|
|
|
/** Watchdog submits durable work; Provisioning remains the only execution owner. */
|
|
export function requestWatchdogLifecycle(server: string, action: 'restart' | 'rebuild', incidentId: string): LifecycleRequestResult {
|
|
return db().transaction(() => {
|
|
const previous = existingWatchdogAction(incidentId, action);
|
|
if (previous) return { ok: true as const, deploymentId: previous };
|
|
const error = watchdogRequestGuard(server, incidentId);
|
|
if (error) return error;
|
|
const pending = db().prepare(`SELECT d.id, d.action, s.source_id FROM server_deployments d
|
|
LEFT JOIN provisioning_sources s ON s.operation_id = d.id AND s.kind = 'watchdog'
|
|
WHERE d.server = ? AND d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health') ORDER BY d.rowid DESC LIMIT 1`)
|
|
.get(server) as { id: string; action: string; source_id: string | null } | undefined;
|
|
if (pending) return pending.source_id === incidentId && pending.action === action
|
|
? { ok: true as const, deploymentId: pending.id }
|
|
: { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' };
|
|
const next = queueWatchdogAction(server, action, incidentId);
|
|
if (!next.ok) return next;
|
|
// An operator retry starts a new incident cycle. Keep the previous
|
|
// failed attempt's capacity, but let that new attempt use it instead
|
|
// of waiting forever behind its own incident's terminal operation.
|
|
const retained = db().prepare(`SELECT r.operation_id FROM restart_rollout r
|
|
JOIN server_deployments d ON d.id = r.operation_id
|
|
JOIN watchdog_actions a ON a.operation_id = d.id AND a.incident_id = ?
|
|
JOIN watchdog_incidents i ON i.id = a.incident_id
|
|
WHERE r.server = ? AND r.healthy_at IS NULL AND d.state IN ('failed','interrupted','cancelled')
|
|
AND a.cycle < i.cycle`).get(incidentId, server) as { operation_id: string } | undefined;
|
|
if (retained && !restartRequestInFlight(server)) {
|
|
const transferred = db().prepare('UPDATE restart_rollout SET operation_id = ? WHERE server = ? AND operation_id = ? AND healthy_at IS NULL')
|
|
.run(next.deploymentId, server, retained.operation_id);
|
|
if (transferred.changes) {
|
|
recordProgress(retained.operation_id, { recovery: { state: 'transferred', operationId: next.deploymentId,
|
|
message: 'Recovery capacity transferred to the explicit new incident attempt.' } });
|
|
recordProgress(next.deploymentId, { recovery: { state: 'held', operationId: next.deploymentId,
|
|
message: 'Recovery capacity retained from the prior incident attempt.' } });
|
|
}
|
|
recordDeploymentEvent(retained.operation_id, 'Recovery capacity handed to explicit Watchdog retry ' + next.deploymentId);
|
|
recordDeploymentEvent(next.deploymentId, 'Retained recovery capacity from prior incident attempt ' + retained.operation_id);
|
|
}
|
|
return next;
|
|
})();
|
|
}
|
|
|
|
/** Atomically reserve a replacement before handing it the failed restart's capacity. */
|
|
export function escalateWatchdogRestart(incidentId: string, operationId: string, recoveryTimeoutMs = 300_000): LifecycleRequestResult {
|
|
return db().transaction(() => {
|
|
const previous = existingWatchdogAction(incidentId, 'rebuild');
|
|
if (previous) return { ok: true as const, deploymentId: previous };
|
|
const operation = provisioningOperation(operationId);
|
|
if (!operation || operation.action !== 'restart' || !['awaiting-health','failed'].includes(operation.state)
|
|
|| !db().prepare("SELECT 1 FROM provisioning_sources WHERE operation_id = ? AND kind = 'watchdog' AND source_id = ?").get(operationId, incidentId)) {
|
|
return { ok: false as const, status: 409, error: 'The restart is not eligible for Watchdog escalation.' };
|
|
}
|
|
const error = watchdogRequestGuard(operation.server, incidentId);
|
|
if (error) return error;
|
|
const pending = db().prepare(`SELECT d.id, d.action, s.source_id FROM server_deployments d
|
|
LEFT JOIN provisioning_sources s ON s.operation_id = d.id AND s.kind = 'watchdog'
|
|
WHERE d.server = ? AND d.server_deleted_at IS NULL AND d.id != ? AND d.state IN ('queued','waiting','running','awaiting-health') LIMIT 1`)
|
|
.get(operation.server, operationId) as { id: string; action: string; source_id: string | null } | undefined;
|
|
if (pending) return pending.action === 'rebuild' && pending.source_id === incidentId
|
|
? { ok: true as const, deploymentId: pending.id }
|
|
: { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' };
|
|
const slot = db().prepare('SELECT operation_id, requested_at, healthy_at FROM restart_rollout WHERE server = ?').get(operation.server) as
|
|
{ operation_id: string | null; requested_at: string; healthy_at: string | null } | undefined;
|
|
const anchor = Date.parse(slot?.requested_at ?? operation.submitted_at ?? operation.started_at);
|
|
if (restartRequestInFlight(operation.server) || (slot && (slot.operation_id !== operationId || slot.healthy_at))
|
|
|| !Number.isFinite(anchor) || Date.now() - anchor < Math.max(300_000, recoveryTimeoutMs)) {
|
|
return { ok: false as const, status: 409, error: 'Waiting for the submitted restart to finish its recovery window.' };
|
|
}
|
|
const replacement = queueWatchdogAction(operation.server, 'rebuild', incidentId);
|
|
if (!replacement.ok) return replacement;
|
|
const reason = 'Watchdog restart recovery timed out; handed recovery to rebuild ' + replacement.deploymentId;
|
|
if (operation.state === 'awaiting-health') finishDeployment(operationId, { ok: false, message: reason });
|
|
else recordDeploymentEvent(operationId, reason);
|
|
if (slot) {
|
|
const transferred = db().prepare('UPDATE restart_rollout SET operation_id = ?, error = ? WHERE server = ? AND operation_id = ? AND healthy_at IS NULL')
|
|
.run(replacement.deploymentId, reason, operation.server, operationId);
|
|
if (transferred.changes) {
|
|
recordProgress(operationId, { recovery: { state: 'transferred', operationId: replacement.deploymentId,
|
|
message: 'Recovery capacity transferred to the rebuild operation.' } });
|
|
recordProgress(replacement.deploymentId, { recovery: { state: 'held', operationId: replacement.deploymentId,
|
|
message: 'Recovery capacity retained from the previous restart.' } });
|
|
}
|
|
}
|
|
recordDeploymentEvent(replacement.deploymentId, 'Recovery handed off from restart ' + operationId + '; waiting for a fresh empty or stopped server');
|
|
return replacement;
|
|
})();
|
|
}
|