134 lines
8.9 KiB
TypeScript
134 lines
8.9 KiB
TypeScript
import { db } from '../db/index.js';
|
|
import { komodoConfigured, rconConfigured } from '../config.js';
|
|
import { getServer } from '../fleet/servers.js';
|
|
import { beginDeployment, deploymentStage, recordDeploymentEvent } from './history.js';
|
|
import { queueRebuildServer } from './instances.js';
|
|
import { runtimeActions, type RuntimeAction } from '../../provisioning.js';
|
|
|
|
export interface LifecycleOperation {
|
|
id: string; server: string; action: string; state: string; stage: string;
|
|
submitted_at: string | null; finished_at: string | null; checked_at: string | null;
|
|
players: number | null; error: string | null;
|
|
}
|
|
export interface RestartSource { kind: 'campaign' | 'beta' | 'maintenance'; id: string }
|
|
export function provisioningOperation(id: string): LifecycleOperation | undefined {
|
|
return db().prepare('SELECT * FROM server_deployments WHERE id = ?').get(id) as LifecycleOperation | undefined;
|
|
}
|
|
|
|
const sourceTables = {
|
|
campaign: { table: 'deployment_servers', key: 'deployment_id' },
|
|
beta: { table: 'beta_servers', key: 'beta_id' },
|
|
maintenance: { table: 'maintenance_jobs', key: 'id' }
|
|
} as const;
|
|
|
|
/** Submit once per caller attempt; a lost response or terminal failure never replays a restart. */
|
|
export function requestRestart(server: string, source: RestartSource): LifecycleOperation {
|
|
return db().transaction(() => {
|
|
const { table, key } = sourceTables[source.kind];
|
|
const row = db().prepare(`SELECT provisioning_id, restarted_at FROM ${table} WHERE ${key} = ? AND server = ?`)
|
|
.get(source.id, server) as { provisioning_id: string | null; restarted_at: string | null } | undefined;
|
|
if (!row) throw new Error('Restart source no longer exists');
|
|
if (row.provisioning_id) {
|
|
const operation = provisioningOperation(row.provisioning_id);
|
|
if (!operation) throw new Error('Provisioning job is missing; inspect the server before retrying');
|
|
if (row.restarted_at && !operation.submitted_at && operation.state === 'waiting') {
|
|
adoptSubmittedRestart(operation.id, server, row.restarted_at);
|
|
return provisioningOperation(operation.id)!;
|
|
}
|
|
return operation;
|
|
}
|
|
let id: string | undefined;
|
|
if (row.restarted_at) {
|
|
// The previous release already recorded some submissions in history.
|
|
// Bind to that exact interval, never to a newer restart of the server.
|
|
id = (db().prepare(`SELECT d.id FROM restart_rollout r JOIN server_deployments d ON d.id = r.operation_id
|
|
WHERE r.server = ? AND d.action = 'restart' AND d.started_at <= ? AND r.requested_at >= ?`)
|
|
.get(server, row.restarted_at, row.restarted_at) as { id: string } | undefined)?.id;
|
|
if (id) db().prepare('UPDATE server_deployments SET submitted_at = COALESCE(submitted_at, ?) WHERE id = ?').run(row.restarted_at, id);
|
|
}
|
|
// Only not-yet-submitted campaign changes can share an activation. A later
|
|
// file deployment must never be credited to an already running restart.
|
|
if (source.kind === 'campaign' && !row.restarted_at) {
|
|
id = (db().prepare(`SELECT d.id FROM server_deployments d WHERE d.server = ? AND d.action = 'restart'
|
|
AND d.state = 'waiting' AND d.submitted_at IS NULL
|
|
AND EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'campaign')
|
|
AND NOT EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind != 'campaign')
|
|
ORDER BY d.rowid LIMIT 1`).get(server) as { id: string } | undefined)?.id;
|
|
}
|
|
if (!id) {
|
|
id = beginDeployment(server, 'restart', `${source.kind}:${source.id}`);
|
|
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
|
|
deploymentStage(id, 'Waiting to verify the server is empty');
|
|
if (row.restarted_at) adoptSubmittedRestart(id, server, row.restarted_at);
|
|
}
|
|
db().prepare('INSERT OR IGNORE INTO provisioning_sources (operation_id, kind, source_id, server) VALUES (?, ?, ?, ?)')
|
|
.run(id, source.kind, source.id, server);
|
|
db().prepare(`UPDATE ${table} SET provisioning_id = ? WHERE ${key} = ? AND server = ?`).run(id, source.id, server);
|
|
return provisioningOperation(id)!;
|
|
})();
|
|
}
|
|
|
|
/** Upgrade observation only: existing restart intent is never resubmitted. */
|
|
function adoptSubmittedRestart(id: string, server: string, submittedAt: string): void {
|
|
const slot = db().prepare('SELECT operation_id, requested_at, healthy_at, players FROM restart_rollout WHERE server = ?').get(server) as
|
|
{ operation_id: string | null; requested_at: string; healthy_at: string | null; players: number | null } | undefined;
|
|
db().prepare("UPDATE server_deployments SET state = 'awaiting-health', submitted_at = ? WHERE id = ?").run(submittedAt, id);
|
|
if (slot && ((slot.operation_id && slot.requested_at >= submittedAt) || (slot.requested_at < submittedAt && !slot.healthy_at))) {
|
|
db().prepare("UPDATE server_deployments SET state = 'interrupted', finished_at = ?, error = ? WHERE id = ?")
|
|
.run(new Date().toISOString(), 'Existing restart cannot be correlated safely. Inspect the server before retrying.', id);
|
|
return;
|
|
}
|
|
if (slot && slot.requested_at >= submittedAt) {
|
|
db().prepare('UPDATE restart_rollout SET operation_id = ? WHERE server = ?').run(id, server);
|
|
} else {
|
|
db().prepare(`INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)
|
|
ON CONFLICT(server) DO UPDATE SET requested_at = excluded.requested_at, operation_id = excluded.operation_id,
|
|
checked_at = NULL, healthy_at = NULL, players = NULL, error = NULL`).run(server, submittedAt, id);
|
|
}
|
|
if (slot?.healthy_at && slot.requested_at >= submittedAt) {
|
|
db().prepare("UPDATE server_deployments SET state = 'done', finished_at = ?, players = ? WHERE id = ?").run(slot.healthy_at, slot.players, id);
|
|
}
|
|
deploymentStage(id, 'Adopted existing restart; verifying recovery without restarting again');
|
|
}
|
|
|
|
export function cancelProvisioning(id: string, reason = 'Cancelled by requester'): boolean {
|
|
return db().transaction(() => {
|
|
const result = db().prepare(`UPDATE server_deployments SET state = 'cancelled', finished_at = ?, stage = ?, error = NULL
|
|
WHERE id = ? AND action IN ('start','restart','stop','rebuild') AND state = 'waiting' AND submitted_at IS NULL`)
|
|
.run(new Date().toISOString(), reason, id);
|
|
if (result.changes) recordDeploymentEvent(id, reason);
|
|
return result.changes > 0;
|
|
})();
|
|
}
|
|
|
|
/** Explicit caller retry starts a new job and keeps the old job as history. */
|
|
export function releaseRestartRequest(server: string, source: RestartSource): void {
|
|
const { table, key } = sourceTables[source.kind];
|
|
const row = db().prepare(`SELECT provisioning_id FROM ${table} WHERE ${key} = ? AND server = ?`).get(source.id, server) as { provisioning_id: string | null } | undefined;
|
|
if (row?.provisioning_id) cancelProvisioning(row.provisioning_id, 'Superseded by a fresh caller attempt');
|
|
db().prepare(`UPDATE ${table} SET provisioning_id = NULL WHERE ${key} = ? AND server = ?`).run(source.id, server);
|
|
}
|
|
|
|
export function requestLifecycle(server: string, action: RuntimeAction) {
|
|
if (!runtimeActions.includes(action)) return { ok: false as const, status: 400, error: 'Choose start, restart, rebuild, or stop.' };
|
|
if (!komodoConfigured() || !rconConfigured()) return { ok: false as const, status: 503, error: 'Configure Komodo and RCON before provisioning.' };
|
|
return db().transaction(() => {
|
|
const existing = db().prepare(`SELECT id, action FROM server_deployments WHERE server = ?
|
|
AND state IN ('queued','waiting','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(server) as { id: string; action: string } | undefined;
|
|
if (existing) return existing.action === action
|
|
? { ok: true as const, deploymentId: existing.id }
|
|
: { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' };
|
|
const row = getServer(server);
|
|
if (!row || row.destroyed_at) return { ok: false as const, status: 404, error: 'Server not found.' };
|
|
if (action === 'rebuild') {
|
|
const result = queueRebuildServer(server);
|
|
return result.ok ? { ok: true as const, deploymentId: result.deploymentId }
|
|
: { ok: false as const, status: 400, error: result.message };
|
|
}
|
|
const id = beginDeployment(server, action);
|
|
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
|
|
deploymentStage(id, action === 'start' ? 'Waiting to start the container' : 'Waiting to verify the server is empty');
|
|
return { ok: true as const, deploymentId: id };
|
|
})();
|
|
}
|