refactor: make provisioning own instance lifecycle operations

2026-09-27 14:45:00 -04:00
parent cfed584614
commit df265d1900
44 changed files with 1791 additions and 1609 deletions
@@ -0,0 +1,134 @@
+import { randomUUID } from 'node:crypto';
+import { db } from '../db/index.js';
+import { recipeForDeployment, summarizeRecipe } from '../fleet/launch-recipes.js';
+import { actionLabels, type ProvisioningAction } from '../../provisioning.js';
+
+export function beginDeployment(server: string, action: ProvisioningAction, triggeredBy = 'ui'): string {
+    const id = randomUUID();
+    db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
+        VALUES (?, ?, ?, 'running', 'Starting', ?, ?)`).run(id, server, action, new Date().toISOString(), triggeredBy);
+    recordDeploymentEvent(id, action === 'create' ? 'Server creation requested'
+        : action === 'destroy' ? 'Server destruction requested' : `${actionLabels[action]} server requested`);
+    return id;
+}
+
+export function deploymentStage(id: string, stage: string): void {
+    const previous = db().prepare('SELECT stage FROM server_deployments WHERE id = ?').get(id) as { stage: string } | undefined;
+    db().prepare('UPDATE server_deployments SET stage = ? WHERE id = ?').run(stage, id);
+    if (previous && previous.stage !== stage) recordDeploymentEvent(id, stage);
+}
+
+export function recordDeploymentEvent(id: string, message: string): void {
+    db().prepare('INSERT INTO server_deployment_events (deployment_id, created_at, message) VALUES (?, ?, ?)')
+        .run(id, new Date().toISOString(), message);
+}
+
+export function getDeployment(id: string) {
+    const deployment = db().prepare(`SELECT d.*, j.expected_map, EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server) AS server_exists
+        FROM server_deployments d LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE d.id = ?`).get(id);
+    if (!deployment) return null;
+    const events = db().prepare('SELECT id, created_at, message FROM server_deployment_events WHERE deployment_id = ? ORDER BY id').all(id);
+    const sources = db().prepare('SELECT kind, source_id, server FROM provisioning_sources WHERE operation_id = ? ORDER BY kind, source_id').all(id);
+    const launch = recipeForDeployment(id);
+    const collection = db().prepare(`SELECT c.id, c.name FROM (
+        SELECT collection_id, deployment_id FROM server_collection_members
+        UNION ALL SELECT collection_id, deployment_id FROM server_collection_removals) m
+        JOIN server_launch_collections c ON c.id = m.collection_id WHERE m.deployment_id = ?`).get(id) ?? null;
+    return { deployment, events, sources, recipe: launch ? summarizeRecipe(launch.recipe) : null, blueprintName: launch?.blueprintName ?? null,
+        canResumeVerification: canResumeVerification(id), canResumeCleanup: canResumeCleanup(id), canResumeRebuild: canResumeRebuild(id), canCancelRemoval: canCancelRemoval(id), collection };
+}
+
+// This stage is written only after Komodo has confirmed deployment deletion.
+export const REMOVING_SERVER = 'Removing server files and registry';
+
+export function canCancelRemoval(id: string): boolean {
+    return !!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND action = 'destroy' AND state = 'waiting' AND stage != ?").get(id, REMOVING_SERVER);
+}
+
+/** Competes atomically with the worker's claim, before any external removal. */
+export function cancelRemoval(id: string): boolean {
+    return db().transaction(() => {
+        const result = db().prepare(`UPDATE server_deployments SET state = 'cancelled', stage = 'Removal cancelled', finished_at = ?, error = NULL
+            WHERE id = ? AND action = 'destroy' AND state = 'waiting' AND stage != ?`).run(new Date().toISOString(), id, REMOVING_SERVER);
+        if (!result.changes) return false;
+        recordDeploymentEvent(id, 'Queued removal cancelled. The server and its files were kept.');
+        return true;
+    })();
+}
+
+export function claimRemoval(id: string): boolean {
+    return db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND action = 'destroy' AND state = 'waiting'").run(id).changes > 0;
+}
+
+export function canResumeCleanup(id: string): boolean {
+    return !!db().prepare(`SELECT d.id FROM server_deployments d
+        WHERE d.id = ? AND d.action = 'destroy' AND d.state IN ('failed', 'interrupted') AND d.stage = ?
+        AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
+        .get(id, REMOVING_SERVER);
+}
+
+export function resumeCleanup(id: string): boolean {
+    return db().transaction(() => {
+        if (!canResumeCleanup(id)) return false;
+        db().prepare("UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?").run(id);
+        recordDeploymentEvent(id, 'Resumed local cleanup after confirmed Komodo deletion');
+        return true;
+    })();
+}
+
+export function canResumeVerification(id: string): boolean {
+    return !!db().prepare(`SELECT d.id FROM server_deployments d
+        JOIN server_creation_jobs j ON j.deployment_id = d.id
+        JOIN servers s ON s.name = j.server AND s.game_port = j.port
+        WHERE d.id = ? AND (d.action = 'create' OR (d.action = 'rebuild' AND EXISTS (
+            SELECT 1 FROM server_rebuild_jobs r WHERE r.deployment_id = d.id AND r.removed_at IS NOT NULL))) AND d.state IN ('failed', 'interrupted')
+        AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
+        .get(id);
+}
+
+export function canResumeRebuild(id: string): boolean {
+    return !!db().prepare(`SELECT d.id FROM server_deployments d JOIN server_rebuild_jobs r ON r.deployment_id = d.id
+        WHERE d.id = ? AND d.state IN ('failed','interrupted') AND d.stage = ? AND r.removed_at IS NULL
+        AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
+        .get(id, REMOVING_SERVER);
+}
+
+export function resumeRebuild(id: string): boolean {
+    if (!canResumeRebuild(id)) return false;
+    db().prepare("UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?").run(id);
+    recordDeploymentEvent(id, 'Resumed rebuild cleanup after confirmed Komodo deletion');
+    return true;
+}
+
+/** Resume observation only. Never repeat a container creation or restart. */
+export function resumeVerification(id: string): boolean {
+    return db().transaction(() => {
+        if (!canResumeVerification(id)) return false;
+        db().prepare("UPDATE server_deployments SET state = 'awaiting-health', finished_at = NULL, error = NULL WHERE id = ?").run(id);
+        recordDeploymentEvent(id, 'Resumed verification of the existing server');
+        deploymentStage(id, 'Waiting for Komodo health');
+        return true;
+    })();
+}
+
+export function finishDeployment(id: string, result: { ok: boolean; message?: string; warnings?: string[] }): void {
+    const state = !result.ok ? 'failed' : result.warnings?.length ? 'warning' : 'done';
+    db().prepare(`UPDATE server_deployments SET state = ?, finished_at = ?, error = ?, warnings = ? WHERE id = ?`)
+        .run(state, new Date().toISOString(), result.message ?? null, JSON.stringify(result.warnings ?? []), id);
+    for (const warning of result.warnings ?? []) recordDeploymentEvent(id, warning);
+    recordDeploymentEvent(id, result.ok ? (state === 'warning' ? 'Completed with warnings' : 'Completed') : `Failed: ${result.message ?? 'Unknown error'}`);
+}
+
+export function listDeployments() {
+    return db().prepare(`SELECT d.*, EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server) AS server_exists,
+        (d.action = 'destroy' AND d.state = 'waiting' AND d.stage != ?) AS canCancelRemoval
+        FROM server_deployments d ORDER BY started_at DESC, rowid DESC`).all(REMOVING_SERVER);
+}
+
+export function interruptDeployments(): void {
+    const running = db().prepare("SELECT id FROM server_deployments WHERE state = 'running'").all() as { id: string }[];
+    db().prepare(`UPDATE server_deployments SET state = 'interrupted', finished_at = ?,
+        error = 'The application restarted before this operation finished. Check the server before retrying.'
+        WHERE state = 'running'`).run(new Date().toISOString());
+    for (const { id } of running) recordDeploymentEvent(id, 'Interrupted by application restart. Check the server before retrying.');
+}