feat: add provisioning lifecycle actions

Rename Deployments to Provisioning with compatible redirects and APIs. Add durable Start, Restart, Stop, and Rebuild operations alongside Create and Destroy; rebuild wipes the instance and recreates it in the same slot and port.

Add lifecycle, migration, recovery, and Komodo command tests. Type checks and production build pass; the full suite retains a telemetry assertion failure reproduced on the unchanged branch.

Refs #39

src/lib/server/db/index.ts

@@ -227,6 +227,25 @@ function migrateGameTelemetry(handle: Database.Database): void {
     })();
 }
 
+function migrateProvisioningActions(handle: Database.Database): void {
+    const table = handle.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'server_deployments'").get() as { sql: string };
+    if (table.sql.includes("'rebuild'")) return;
+    // Rebuild the CHECK constraint without cascading into events, recipes, or
+    // collection members. Preserve rowids: they order same-server operations.
+    handle.pragma('foreign_keys = OFF');
+    try {
+        handle.transaction(() => {
+            handle.exec(table.sql.replace('server_deployments', 'server_deployments_new')
+                .replace("'create', 'destroy'", "'create', 'start', 'restart', 'rebuild', 'stop', 'destroy'"));
+            handle.exec(`INSERT INTO server_deployments_new (rowid, id, server, action, state, stage, started_at, finished_at, triggered_by, error, warnings)
+                SELECT rowid, id, server, action, state, stage, started_at, finished_at, triggered_by, error, warnings FROM server_deployments;
+                DROP TABLE server_deployments;
+                ALTER TABLE server_deployments_new RENAME TO server_deployments;`);
+            if ((handle.pragma('foreign_key_check') as unknown[]).length) throw new Error('Provisioning migration found broken references');
+        })();
+    } finally { handle.pragma('foreign_keys = ON'); }
+}
+
 export function db(): Database.Database {
     if (_db) return _db;
 
@@ -240,6 +259,8 @@ export function db(): Database.Database {
         handle.pragma('foreign_keys = ON');
         dropLegacyBetaTables(handle);
         handle.exec(schemaSql);
+        migrateProvisioningActions(handle);
+        migrateTable(handle, 'restart_rollout', [{ name: 'operation_id', ddl: 'ALTER TABLE restart_rollout ADD COLUMN operation_id TEXT' }]);
         migrateServers(handle);
         // Column additions must run BEFORE any statement that references the
         // new columns. schema.sql cannot include those references — see the

src/lib/server/db/schema.sql

@@ -409,7 +409,7 @@ CREATE INDEX IF NOT EXISTS idx_telemetry_requests_received ON telemetry_requests
 CREATE TABLE IF NOT EXISTS server_deployments (
     id TEXT PRIMARY KEY,
     server TEXT NOT NULL,
-    action TEXT NOT NULL CHECK (action IN ('create', 'destroy')),
+    action TEXT NOT NULL CHECK (action IN ('create', 'start', 'restart', 'rebuild', 'stop', 'destroy')),
     state TEXT NOT NULL,
     stage TEXT NOT NULL,
     started_at TEXT NOT NULL,
@@ -436,6 +436,11 @@ CREATE TABLE IF NOT EXISTS server_creation_jobs (
     expected_map TEXT NOT NULL
 );
 
+CREATE TABLE IF NOT EXISTS server_rebuild_jobs (
+    deployment_id TEXT PRIMARY KEY REFERENCES server_deployments(id) ON DELETE CASCADE,
+    removed_at TEXT
+);
+
 -- Blueprints retain map/count selections only. The legacy column name remains
 -- compatible with older snapshot-based definitions, read as selections at runtime.
 CREATE TABLE IF NOT EXISTS server_blueprints (

src/lib/server/fleet/collections.ts

@@ -34,12 +34,12 @@ function coverage(entries: BlueprintEntry[]) {
             return matches.length === 1 && matches[0].slug === entry.recipe.mapSlug;
         }).map(server => server.name).sort();
         const coverage = existingServers.map(name => {
-            const operation = db().prepare("SELECT id, state FROM server_deployments WHERE server = ? AND action = 'create' AND state IN ('queued', 'running', 'awaiting-health') ORDER BY rowid DESC LIMIT 1").get(name) as { id: string; state: string } | undefined;
+            const operation = db().prepare("SELECT id, state FROM server_deployments WHERE server = ? AND action IN ('create','rebuild') AND state IN ('queued', 'running', 'awaiting-health') ORDER BY rowid DESC LIMIT 1").get(name) as { id: string; state: string } | undefined;
             const server = getServer(name);
             return { name, health: operation ? operation.state === 'queued' ? 'queued' : 'starting'
                 : server?.container_state && !['running', 'starting', 'restarting'].includes(server.container_state) ? 'stopped'
                 : classifyHealth(server?.last_seen_at ?? null, server?.last_error ?? null, server?.container_state),
-                href: operation ? `/deployments/${encodeURIComponent(operation.id)}` : `/servers/${encodeURIComponent(name)}` };
+                href: operation ? `/provisioning/${encodeURIComponent(operation.id)}` : `/servers/${encodeURIComponent(name)}` };
         });
         return { existingServers, coverage, existing: existingServers.length, missing: Math.max(0, entry.count - existingServers.length) };
     });
@@ -48,7 +48,7 @@ function coverage(entries: BlueprintEntry[]) {
 function fleetSnapshot() {
     return housePartyTargets().map(name => ({ name,
         server: db().prepare('SELECT game, map_slug, map, game_port FROM servers WHERE name = ?').get(name) ?? null,
-        creation: db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'create' ORDER BY rowid DESC LIMIT 1").get(name) ?? null,
+        creation: db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action IN ('create','rebuild') ORDER BY rowid DESC LIMIT 1").get(name) ?? null,
         removal: db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) ?? null
     }));
 }

src/lib/server/fleet/create.ts

@@ -42,7 +42,7 @@ export type CreateServerResult =
 
 export type DestroyServerResult =
     | { ok: true; warnings: string[] }
-    | { ok: false; kind: 'not-found' | 'komodo' | 'fs' | 'db'; message: string; warnings: string[] };
+    | { ok: false; kind: 'not-found' | 'conflict' | 'komodo' | 'fs' | 'db'; message: string; warnings: string[] };
 
 export interface CreateServerInput {
     game?: ZombieGame;    // omitted by older clients: T5
@@ -63,7 +63,7 @@ export function nextLobbyNumber(baseName: string, additionalNames: string[] = []
     const rows = db()
         .prepare(`SELECT name FROM servers WHERE name LIKE ? UNION
             SELECT j.server AS name FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
-            WHERE d.state IN ('queued', 'running', 'awaiting-health')`)
+            WHERE d.state IN ('queued', 'running', 'awaiting-health') OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry')))`)
         .all(`${baseName}_%`) as Array<{ name: string }>;
     const taken = new Set<number>();
     const prefix = `${baseName}_`;
@@ -87,7 +87,7 @@ export function nextAvailablePort(additionalPorts: number[] = [], releasingServe
     const rows = db()
         .prepare(`SELECT name, game_port FROM servers WHERE game_port IS NOT NULL UNION
             SELECT j.server AS name, j.port AS game_port FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
-            WHERE d.state IN ('queued', 'running', 'awaiting-health')`)
+            WHERE d.state IN ('queued', 'running', 'awaiting-health') OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry')))`)
         .all() as Array<{ name: string; game_port: number }>;
     const taken = new Set([...rows.filter(row => !releasingServers.includes(row.name)).map((r) => r.game_port), ...additionalPorts]);
     for (let p = config.portRange.min; p <= config.portRange.max; p++) {
@@ -229,6 +229,63 @@ export function queueCreateServer(input: CreateServerInput, releasingServers: st
     })();
 }
 
+/** Reserve the same identity and port before removing any part of the instance. */
+export function queueRebuildServer(name: string): CreateServerResult {
+    const server = getServer(name);
+    const game = gameOfServer(name);
+    const map = server?.map_slug && zombieMapBySlug(game, server.map_slug);
+    if (!server || !map || !server.game_port || server.komodo_type !== 'deployment' || server.komodo_name !== name
+        || (server.host_dir && server.host_dir !== serverDir(name))) {
+        return { ok: false, kind: 'bad-request', message: 'Rebuild requires a managed server with its original map, directory, and port.' };
+    }
+    if (!config.komodo.targetServer || !config.rcon.password) return { ok: false, kind: 'unconfigured', message: 'Configure the Komodo target host and RCON before rebuilding.' };
+    if (!getServerKey(name)) return { ok: false, kind: 'bad-request', message: `Save a server key for ${name} in Keys before rebuilding.` };
+    // Validate that the replacement setup can be composed before destruction.
+    const recipe = captureRecipe(game, map.slug);
+    return db().transaction(() => {
+        const id = beginDeployment(name, 'rebuild');
+        db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
+        db().prepare('INSERT INTO server_creation_jobs (deployment_id, server, port, game_mode, map_slug, expected_map) VALUES (?, ?, ?, ?, ?, ?)')
+            .run(id, name, server.game_port, 'zombie', map.slug, map.canonical);
+        db().prepare('INSERT INTO server_rebuild_jobs (deployment_id) VALUES (?)').run(id);
+        db().prepare('INSERT INTO server_launch_recipes (deployment_id, recipe) VALUES (?, ?)').run(id, JSON.stringify(recipe));
+        deploymentStage(id, 'Waiting to rebuild the empty server');
+        return { ok: true as const, name, port: server.game_port!, deploymentId: id };
+    })();
+}
+
+/** Caller has just verified emptiness and excluded conflicting work. */
+export async function rebuildServer(id: string): Promise<void> {
+    const job = db().prepare(`SELECT j.*, d.stage FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
+        WHERE d.id = ? AND d.action = 'rebuild' AND d.state = 'waiting'`).get(id) as (CreationJob & { stage: string }) | undefined;
+    if (!job) return;
+    if (creating || destroying.size || creationQueue().active.length >= config.createConcurrency) {
+        // The cleanup checkpoint must survive capacity waits and app restarts.
+        if (job.stage !== REMOVING_SERVER) deploymentStage(id, 'Waiting for a server creation slot before rebuilding');
+        return;
+    }
+    // Recheck prerequisites while the original instance still exists.
+    if (!getServerKey(job.server)) { finishDeployment(id, { ok: false, message: 'Saved server key was removed; rebuild did not begin.' }); return; }
+    try { captureRecipe(gameOfServer(job.server), job.map_slug); }
+    catch (err) { finishDeployment(id, { ok: false, message: (err as Error).message }); return; }
+    db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND state = 'waiting'").run(id);
+    destroying.add(job.server);
+    try {
+        const result = job.stage === REMOVING_SERVER
+            ? await cleanupDestroyedServer(job.server, id) : await destroyServerUnlocked(job.server, id);
+        if (!result.ok || result.warnings.length) {
+            finishDeployment(id, { ok: false, message: result.ok ? 'Old server files could not be completely removed.' : result.message, warnings: result.warnings });
+            return;
+        }
+        db().transaction(() => {
+            db().prepare('UPDATE server_rebuild_jobs SET removed_at = ? WHERE deployment_id = ?').run(new Date().toISOString(), id);
+            db().prepare("UPDATE server_deployments SET state = 'queued' WHERE id = ?").run(id);
+            deploymentStage(id, 'Queued for server creation');
+        })();
+    } catch (err) { finishDeployment(id, { ok: false, message: (err as Error).message }); }
+    finally { destroying.delete(job.server); }
+}
+
 function nextName(input: CreateServerInput): string | null {
     const base = `${input.game ?? 't5'}_${input.gameMode}_${input.mapSlug}`;
     const lobby = nextLobbyNumber(base);
@@ -437,6 +494,10 @@ function hasActiveCreation(name: string): boolean {
 }
 
 export function destroyServer(name: string, triggeredBy = 'ui'): DestroyServerResult & { deploymentId?: string } {
+    if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND action IN ('start','restart','stop','rebuild')
+        AND state IN ('waiting','queued','running')`).get(name)) {
+        return { ok: false, kind: 'conflict', message: 'Wait for the current provisioning operation before destroying this server.', warnings: [] };
+    }
     const existing = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) as { id: string } | undefined;
     if (existing) return { ok: true, warnings: [], deploymentId: existing.id };
     const previous = db().prepare("SELECT id FROM server_deployments WHERE server = ? ORDER BY rowid DESC LIMIT 1").get(name) as { id: string } | undefined;
@@ -485,7 +546,11 @@ async function runDestructions(): Promise<void> {
                 continue;
             }
             // A fresh successful zero is required; cached occupancy and timeouts are not evidence of emptiness.
-            const players = await rconGetPlayerCount(operation.server);
+            let players = await rconGetPlayerCount(operation.server);
+            if (players === null) {
+                const health = await komodoInspectHealthy(operation.server);
+                if (['stopped','exited','created','dead'].includes(health.state?.toLowerCase() ?? '')) players = 0;
+            }
             if (!db().prepare("SELECT id FROM server_deployments WHERE id = ? AND state = 'waiting'").get(operation.id)) continue;
             if (players !== 0) {
                 deploymentStage(operation.id, players === null
@@ -580,10 +645,12 @@ async function cleanupDestroyedServer(name: string, operation: string): Promise<
         };
     }
 
-    const unfinished = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'create' AND state = 'awaiting-health'").all(name) as { id: string }[];
-    for (const { id } of unfinished) finishDeployment(id, { ok: false, message: 'Server destroyed before creation health checks completed' });
+    const unfinished = db().prepare("SELECT id, action FROM server_deployments WHERE server = ? AND state = 'awaiting-health'").all(name) as { id: string; action: string }[];
+    for (const { id, action } of unfinished) finishDeployment(id, { ok: false, message: action === 'create'
+        ? 'Server destroyed before creation health checks completed' : 'Server destroyed before operation verification completed' });
     return { ok: true, warnings };
-}␍interface CreationJob {
+}
+interface CreationJob {
     deployment_id: string; server: string; port: number; game_mode: GameMode; map_slug: string; expected_map: string; triggered_by?: string;
 }
 
@@ -636,7 +703,7 @@ export async function processCreations(): Promise<void> {
 export function creationQueue() {
     type Entry = { id: string; server: string; state: string; stage: string };
     const jobs = db().prepare(`SELECT id, server, state, stage FROM server_deployments
-        WHERE action = 'create' AND state IN ('queued', 'running', 'awaiting-health')
+        WHERE action IN ('create', 'rebuild') AND state IN ('queued', 'running', 'awaiting-health')
         ORDER BY started_at, rowid`).all() as Entry[];
     return { active: jobs.filter(job => job.state !== 'queued'), queued: jobs.filter(job => job.state === 'queued') };
 }
@@ -656,7 +723,7 @@ async function runCreations(): Promise<CreationOutcomes> {
     if (creating || destroying.size > 0) return outcomes;
     // Restart recovery is observation-only: never repeat external creation or
     // deployment. A known interrupted startup must retain its readiness gate.
-    const interrupted = db().prepare("SELECT id FROM server_deployments WHERE action = 'create' AND state = 'interrupted' ORDER BY started_at, rowid").all() as { id: string }[];
+    const interrupted = db().prepare("SELECT id FROM server_deployments WHERE action IN ('create', 'rebuild') AND state = 'interrupted' ORDER BY started_at, rowid").all() as { id: string }[];
     for (const { id } of interrupted) {
         if (resumeVerification(id)) continue;
         // A queued removal does not make the interrupted container disappear.
@@ -717,7 +784,7 @@ async function runCreations(): Promise<CreationOutcomes> {
         // API calls can cancel a selected job or resume an older startup while
         // a previous provision awaits Komodo. Recheck without expanding this pass.
         if (creationQueue().active.length >= config.createConcurrency) break;
-        if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND action = 'create' AND state = 'queued'").get(id)) continue;
+        if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND action IN ('create', 'rebuild') AND state = 'queued'").get(id)) continue;
         const blocker = creationBlocker(id);
         if (blocker) {
             deploymentStage(id, blocker);

src/lib/server/fleet/deployments.ts

@@ -1,12 +1,14 @@
 import { randomUUID } from 'node:crypto';
 import { db } from '../db/index.js';
 import { recipeForDeployment, summarizeRecipe } from './launch-recipes.js';
+import { actionLabels, type ProvisioningAction } from '../../provisioning.js';
 
-export function beginDeployment(server: string, action: 'create' | 'destroy', triggeredBy = 'ui'): string {
+export function beginDeployment(server: string, action: ProvisioningAction, triggeredBy = 'ui'): string {
     const id = randomUUID();
     db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
         VALUES (?, ?, ?, 'running', 'Starting', ?, ?)`).run(id, server, action, new Date().toISOString(), triggeredBy);
-    recordDeploymentEvent(id, action === 'create' ? 'Server creation requested' : 'Server destruction requested');
+    recordDeploymentEvent(id, action === 'create' ? 'Server creation requested'
+        : action === 'destroy' ? 'Server destruction requested' : `${actionLabels[action]} server requested`);
     return id;
 }
 
@@ -32,7 +34,7 @@ export function getDeployment(id: string) {
         UNION ALL SELECT collection_id, deployment_id FROM server_collection_removals) m
         JOIN server_launch_collections c ON c.id = m.collection_id WHERE m.deployment_id = ?`).get(id) ?? null;
     return { deployment, events, recipe: launch ? summarizeRecipe(launch.recipe) : null, blueprintName: launch?.blueprintName ?? null,
-        canResumeVerification: canResumeVerification(id), canResumeCleanup: canResumeCleanup(id), canCancelRemoval: canCancelRemoval(id), collection };
+        canResumeVerification: canResumeVerification(id), canResumeCleanup: canResumeCleanup(id), canResumeRebuild: canResumeRebuild(id), canCancelRemoval: canCancelRemoval(id), collection };
 }
 
 // This stage is written only after Komodo has confirmed deployment deletion.
@@ -77,11 +79,26 @@ export function canResumeVerification(id: string): boolean {
     return !!db().prepare(`SELECT d.id FROM server_deployments d
         JOIN server_creation_jobs j ON j.deployment_id = d.id
         JOIN servers s ON s.name = j.server AND s.game_port = j.port
-        WHERE d.id = ? AND d.action = 'create' AND d.state IN ('failed', 'interrupted')
+        WHERE d.id = ? AND (d.action = 'create' OR (d.action = 'rebuild' AND EXISTS (
+            SELECT 1 FROM server_rebuild_jobs r WHERE r.deployment_id = d.id AND r.removed_at IS NOT NULL))) AND d.state IN ('failed', 'interrupted')
         AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
         .get(id);
 }
 
+export function canResumeRebuild(id: string): boolean {
+    return !!db().prepare(`SELECT d.id FROM server_deployments d JOIN server_rebuild_jobs r ON r.deployment_id = d.id
+        WHERE d.id = ? AND d.state IN ('failed','interrupted') AND d.stage = ? AND r.removed_at IS NULL
+        AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
+        .get(id, REMOVING_SERVER);
+}
+
+export function resumeRebuild(id: string): boolean {
+    if (!canResumeRebuild(id)) return false;
+    db().prepare("UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?").run(id);
+    recordDeploymentEvent(id, 'Resumed rebuild cleanup after confirmed Komodo deletion');
+    return true;
+}
+
 /** Resume observation only. Never repeat a container creation or restart. */
 export function resumeVerification(id: string): boolean {
     return db().transaction(() => {

src/lib/server/fleet/house-party.ts

@@ -6,7 +6,7 @@ export function housePartyTargets(): string[] {
         SELECT name FROM servers WHERE destroyed_at IS NULL
         UNION
         SELECT server AS name FROM server_deployments
-        WHERE action = 'create' AND state IN ('queued', 'running', 'awaiting-health')
+        WHERE action IN ('create', 'rebuild') AND state IN ('queued', 'running', 'awaiting-health')
         ORDER BY name
     `).all() as { name: string }[]).map(row => row.name);
 }

src/lib/server/fleet/lifecycle.ts

@@ -0,0 +1,117 @@
+import { db } from '../db/index.js';
+import { komodoConfigured, rconConfigured } from '../config.js';
+import { getServer } from './servers.js';
+import { beginDeployment, deploymentStage, finishDeployment, REMOVING_SERVER } from './deployments.js';
+import { queueRebuildServer, rebuildServer } from './create.js';
+import { komodoInspectHealthy, komodoRuntimeAction } from '../integrations/komodo.js';
+import { rconGetPlayerCount } from '../integrations/rcon.js';
+import { refreshRestartRollout, restartInRollout, rolloutWaitReason } from '../pipeline/restart-rollout.js';
+import { runtimeActions, type RuntimeAction } from '../../provisioning.js';
+
+const stoppedStates = new Set(['stopped', 'exited', 'created', 'dead']);
+export const isStopped = (state?: string) => stoppedStates.has(state?.toLowerCase() ?? '');
+
+function blocker(server: string, id = ''): string | null {
+    if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND id != ?
+        AND state IN ('queued','waiting','running','awaiting-health')`).get(server, id)) return 'Waiting for another provisioning operation on this server';
+    if (db().prepare(`SELECT 1 FROM deployment_servers WHERE server = ? AND restart_state IN ('pending','failed','held')
+        UNION ALL SELECT 1 FROM beta_servers bs JOIN betas b ON b.id = bs.beta_id
+            WHERE bs.server = ? AND b.state IN ('running','ready-promote')
+        UNION ALL SELECT 1 FROM maintenance_jobs WHERE server = ? AND state IN ('queued','restarting') LIMIT 1`).get(server, server, server)) {
+        return 'Waiting for campaign, beta, or maintenance work on this server';
+    }
+    return rolloutWaitReason(server, id);
+}
+
+export function requestLifecycle(server: string, action: RuntimeAction) {
+    if (!runtimeActions.includes(action)) return { ok: false as const, status: 400, error: 'Choose start, restart, rebuild, or stop.' };
+    if (!komodoConfigured() || !rconConfigured()) return { ok: false as const, status: 503, error: 'Configure Komodo and RCON before provisioning.' };
+    return db().transaction(() => {
+        const existing = db().prepare(`SELECT id, action FROM server_deployments WHERE server = ?
+            AND state IN ('queued','waiting','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(server) as { id: string; action: string } | undefined;
+        if (existing) return existing.action === action
+            ? { ok: true as const, deploymentId: existing.id }
+            : { ok: false as const, status: 409, error: 'Another provisioning operation is already pending for this server.' };
+        const row = getServer(server);
+        if (!row || row.destroyed_at) return { ok: false as const, status: 404, error: 'Server not found.' };
+        if (action === 'rebuild') {
+            const result = queueRebuildServer(server);
+            return result.ok ? { ok: true as const, deploymentId: result.deploymentId }
+                : { ok: false as const, status: 400, error: result.message };
+        }
+        const id = beginDeployment(server, action);
+        db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
+        deploymentStage(id, action === 'start' ? 'Waiting to start the container' : 'Waiting to verify the server is empty');
+        return { ok: true as const, deploymentId: id };
+    })();
+}
+
+interface Operation { id: string; server: string; action: RuntimeAction; state: string; stage: string }
+let ticking: Promise<void> | null = null;
+export function processLifecycle(): Promise<void> {
+    if (ticking) return ticking;
+    ticking = run().finally(() => { ticking = null; });
+    return ticking;
+}
+
+async function run(): Promise<void> {
+    await refreshRestartRollout();
+    const operations = db().prepare(`SELECT id, server, action, state, stage FROM server_deployments
+        WHERE action IN ('start','restart','stop','rebuild') AND (state = 'waiting' OR (action = 'stop' AND state = 'awaiting-health'))
+        ORDER BY started_at, rowid`).all() as Operation[];
+    for (const operation of operations) {
+        try { await processOperation(operation); }
+        catch (err) {
+            // Observation failures are retried; remote mutations are never replayed.
+            deploymentStage(operation.id, 'Waiting for verification: ' + (err as Error).message);
+        }
+    }
+}
+
+async function processOperation(operation: Operation): Promise<void> {
+    const { id, server, action } = operation;
+    if (action === 'rebuild' && operation.stage === REMOVING_SERVER) { await rebuildServer(id); return; }
+    if (!getServer(server)) { finishDeployment(id, { ok: false, message: 'Server is no longer registered.' }); return; }
+    if (!komodoConfigured() || !rconConfigured()) { deploymentStage(id, 'Waiting for Komodo and RCON configuration'); return; }
+    if (operation.state === 'awaiting-health') {
+        const health = await komodoInspectHealthy(server);
+        if (isStopped(health.state)) {
+            db().prepare("UPDATE servers SET container_state = ?, last_players = NULL, last_roster = NULL, game_started_at = NULL, game_ended_at = NULL WHERE name = ?").run(health.state, server);
+            deploymentStage(id, 'Verified container stopped');
+            finishDeployment(id, { ok: true });
+        } else deploymentStage(id, 'Waiting for container to stop' + (!health.ok ? ': ' + health.reason : ''));
+        return;
+    }
+    const reason = blocker(server, id);
+    if (reason) { deploymentStage(id, reason); return; }
+    const health = await komodoInspectHealthy(server);
+    if (action !== 'start' && !isStopped(health.state)) {
+        const players = await rconGetPlayerCount(server);
+        if (players !== 0) {
+            deploymentStage(id, players === null ? 'Waiting for a fresh player count' : `Waiting for ${players} player(s) to leave`);
+            return;
+        }
+    }
+    // Requests can arrive during the probes. Recheck before claiming the action.
+    if (blocker(server, id)) return;
+    if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND state = 'waiting'").get(id)) return;
+    if (action === 'rebuild') { await rebuildServer(id); return; }
+    const claim = db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND state = 'waiting'").run(id);
+    if (!claim.changes) return;
+    if (action === 'start' || action === 'restart') {
+        const result = await restartInRollout(server, { operationId: id, action });
+        if (result.kind === 'queued') {
+            db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
+            deploymentStage(id, result.message);
+        }
+        return;
+    }
+    deploymentStage(id, 'Stopping container');
+    // Persist intent first; a lost reply must not issue another stop later.
+    try {
+        const result = await komodoRuntimeAction(server, 'stop');
+        if (!result.ok) { finishDeployment(id, { ok: false, message: result.message }); return; }
+        db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
+        deploymentStage(id, 'Waiting for container to stop');
+    } catch (err) { finishDeployment(id, { ok: false, message: (err as Error).message }); }
+}

src/lib/server/fleet/overview.ts

@@ -1,3 +1,4 @@
+import { actionLabels, type ProvisioningAction } from '../../provisioning.js';
 import { db } from '../db/index.js';
 import type { FleetOperation } from '../../fleet-overview.js';
 
@@ -41,13 +42,14 @@ export function fleetOperations(now = Date.now()): FleetOperation[] {
         .all(since) as { id: string; server: string; action: string; state: string; stage: string;
             error: string | null; started_at: string }[];
     for (const row of lifecycle) {
+        if (row.state === 'awaiting-health') representedRecovery.add(row.server);
         const state = ['done', 'warning', 'cancelled'].includes(row.state) ? 'done'
             : ['failed', 'interrupted'].includes(row.state) ? 'attention' : 'active';
         operations.push({ key: `deployment:${row.id}`, group: `deployment:${row.id}`, kind: 'deployment',
             server: row.server, state, startedAt: row.started_at,
             label: state === 'done' ? row.state === 'cancelled' ? 'Removal cancelled' : row.state === 'warning' ? 'Completed with warnings' : 'Complete'
-                : state === 'attention' ? `${row.action === 'create' ? 'Launch' : 'Removal'} needs attention` : row.stage,
-            detail: row.error ?? row.stage, href: `/deployments/${encodeURIComponent(row.id)}` });
+                : state === 'attention' ? `${actionLabels[row.action as ProvisioningAction]} needs attention` : row.stage,
+            detail: row.error ?? row.stage, href: `/provisioning/${encodeURIComponent(row.id)}` });
     }
     const betas = handle.prepare(`SELECT b.id, b.opened_at, bs.server, bs.stage, bs.error
         FROM betas b JOIN beta_servers bs ON bs.beta_id = b.id WHERE b.state IN ('running', 'ready-promote')`)

src/lib/server/fleet/provisioning.ts

@@ -5,5 +5,5 @@ export function isProvisioning(server: string): boolean {
     const operation = db().prepare(`SELECT action, state FROM server_deployments
         WHERE server = ? ORDER BY rowid DESC LIMIT 1`).get(server) as
         { action: string; state: string } | undefined;
-    return operation?.action === 'create' && ['queued', 'running', 'awaiting-health'].includes(operation.state);
+    return !!operation && ['create', 'rebuild', 'start'].includes(operation.action) && ['queued', 'running', 'awaiting-health'].includes(operation.state);
 }

src/lib/server/integrations/komodo.ts

@@ -520,6 +520,21 @@ export async function komodoDeploy(deployment: string): Promise<KomodoWriteResul
     return postAndClassify('execute/Deploy', { deployment });
 }
 
+/** Runtime controls use the saved Komodo resource, never the EC display name. */
+export async function komodoRuntimeAction(server: string, action: 'start' | 'stop'): Promise<KomodoRestartResult> {
+    if (!komodoConfigured()) return { ok: false, kind: 'unconfigured', message: 'komodo not configured' };
+    const row = getServer(server);
+    if (!row?.komodo_name) return { ok: false, kind: 'no-name', message: 'no komodo_name set for server' };
+    const type = row.komodo_type ?? 'deployment';
+    const verb = action === 'start' ? 'Start' : 'Stop';
+    if (type === 'container') {
+        if (!row.komodo_server) return { ok: false, kind: 'no-name', message: 'container type needs komodo_server' };
+        return postAndClassify(`execute/${verb}Container`, { server: row.komodo_server, container: row.komodo_name });
+    }
+    if (type === 'stack') return postAndClassify(`execute/${verb}Stack`, { stack: row.komodo_name });
+    return postAndClassify(`execute/${verb}Deployment`, { deployment: row.komodo_name });
+}
+
 /** Stop + remove the container backing a Komodo deployment. Entity remains. */
 export async function komodoDestroyDeployment(deployment: string): Promise<KomodoWriteResult> {
     if (!komodoConfigured()) {

src/lib/server/pipeline/restart-rollout.ts

@@ -1,7 +1,8 @@
 import { config } from '../config.js';
 import { db } from '../db/index.js';
-import { komodoInspectHealthy, komodoRestart } from '../integrations/komodo.js';
+import { komodoInspectHealthy, komodoRestart, komodoRuntimeAction } from '../integrations/komodo.js';
 import { rconStatusResult } from '../integrations/rcon.js';
+import { beginDeployment, deploymentStage, finishDeployment } from '../fleet/deployments.js';
 
 // Komodo queues restart work. Don't mistake the still-running old container
 // for a completed restart when an API kick immediately follows the request.
@@ -15,6 +16,7 @@ interface RestartSlot {
     healthy_at: string | null;
     players: number | null;
     error: string | null;
+    operation_id: string | null;
 }
 
 function slotFor(server: string): RestartSlot | undefined {
@@ -28,7 +30,12 @@ export function restartSettling(server: string): boolean {
         && Date.now() - Date.parse(slot.requested_at) < HEALTH_POLL_MS);
 }
 
-export function rolloutWaitReason(server: string): string | null {
+export function rolloutWaitReason(server: string, operationId = ''): string | null {
+    if (db().prepare(`SELECT 1 FROM server_deployments WHERE server = ? AND id != ?
+        AND ((action IN ('start','stop','rebuild') AND state IN ('running','awaiting-health'))
+            OR (action = 'destroy' AND state IN ('waiting','running')))`).get(server, operationId)) {
+        return 'Waiting for the current provisioning operation';
+    }
     const slot = slotFor(server);
     if (slot?.healthy_at === null) {
         return `Waiting for the previous restart of this server to recover${slot.error ? `: ${slot.error}` : ''}`;
@@ -43,14 +50,17 @@ export type RolloutRestartResult =
     | { kind: 'failed'; message: string };
 
 /** Reserve capacity before any async I/O; callers still check for players first. */
-export async function restartInRollout(server: string): Promise<RolloutRestartResult> {
+export async function restartInRollout(server: string, options: { operationId?: string; action?: 'start' | 'restart'; triggeredBy?: string } = {}): Promise<RolloutRestartResult> {
+    let operationId = options.operationId;
     const waitReason = db().transaction(() => {
-        const reason = rolloutWaitReason(server);
+        const reason = rolloutWaitReason(server, operationId);
         if (reason) return reason;
-        db().prepare(`INSERT INTO restart_rollout (server, requested_at) VALUES (?, ?)
+        operationId ??= beginDeployment(server, options.action ?? 'restart', options.triggeredBy ?? 'automation');
+        db().prepare(`INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)
             ON CONFLICT(server) DO UPDATE SET requested_at = excluded.requested_at,
-                checked_at = NULL, healthy_at = NULL, players = NULL, error = NULL`)
-            .run(server, new Date().toISOString());
+                checked_at = NULL, healthy_at = NULL, players = NULL, error = NULL, operation_id = excluded.operation_id`)
+            .run(server, new Date().toISOString(), operationId);
+        deploymentStage(operationId, options.action === 'start' ? 'Starting container' : 'Restarting container');
         return null;
     }).immediate();
     if (waitReason) return { kind: 'queued', message: waitReason };
@@ -59,8 +69,12 @@ export async function restartInRollout(server: string): Promise<RolloutRestartRe
     db().prepare(`UPDATE servers SET container_state = 'restarting', last_error = NULL, last_reply_snippet = NULL,
         game_started_at = NULL, game_ended_at = NULL WHERE name = ?`).run(server);
     try {
-        const result = await komodoRestart(server);
-        if (result.ok) return { kind: 'started' };
+        const result = options.action === 'start' ? await komodoRuntimeAction(server, 'start') : await komodoRestart(server);
+        if (result.ok) {
+            db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(operationId);
+            deploymentStage(operationId!, 'Waiting for container and RCON health');
+            return { kind: 'started' };
+        }
         // These failures cannot have queued a remote restart. Other failures
         // (especially a timeout) may have: retain capacity until recovery.
         if (result.kind === 'no-name' || result.kind === 'unconfigured') {
@@ -68,10 +82,12 @@ export async function restartInRollout(server: string): Promise<RolloutRestartRe
         } else {
             db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ?').run(`komodo: ${result.message}`, server);
         }
+        finishDeployment(operationId!, { ok: false, message: `komodo: ${result.message}` });
         return { kind: 'failed', message: `komodo: ${result.message}` };
     } catch (err) {
         const message = `komodo: ${(err as Error).message}`;
         db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ?').run(message, server);
+        finishDeployment(operationId!, { ok: false, message });
         return { kind: 'failed', message };
     } finally {
         // Start the grace period at completion, including slow/failed requests.
@@ -124,6 +140,13 @@ async function refresh(): Promise<void> {
         db().prepare(`UPDATE restart_rollout SET checked_at = ?, healthy_at = ?, players = ?, error = ?
             WHERE server = ? AND requested_at = ? AND healthy_at IS NULL`)
             .run(now, error ? null : now, players, error, slot.server, slot.requested_at);
+        if (slot.operation_id && db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND state = 'awaiting-health'").get(slot.operation_id)) {
+            if (error) deploymentStage(slot.operation_id, 'Waiting for health: ' + error);
+            else {
+                deploymentStage(slot.operation_id, 'Verified container and RCON health');
+                finishDeployment(slot.operation_id, { ok: true });
+            }
+        }
     }
 }
 

src/lib/server/pipeline/worker.ts

@@ -1,4 +1,5 @@
 import { processDestructions, processCreations } from '../fleet/create.js';
+import { processLifecycle } from '../fleet/lifecycle.js';
 import { sweep } from './sweep.js';
 import { betaTick } from './beta-runner.js';
 import { emit } from './bus.js';
@@ -37,6 +38,8 @@ async function runTick(): Promise<void> {
     catch (err) { console.error('[worker] creation error:', err); }
     try { await processDestructions(); }
     catch (err) { console.error('[worker] destruction error:', err); }
+    try { await processLifecycle(); }
+    catch (err) { console.error('[worker] provisioning error:', err); }
     try {
         await sweep();
     } catch (err) {