Cancel active provisioning jobs when rolling back betas

src/lib/server/provisioning/cancel-creation.ts

@@ -13,7 +13,7 @@ export function canCancelCreation(id: string): boolean {
             AND EXISTS (SELECT 1 FROM native_creation_intents intent WHERE intent.deployment_id=d.id)
             AND NOT EXISTS (SELECT 1 FROM native_instances replacement WHERE replacement.server=d.server AND replacement.retired_at IS NULL)
             AND NOT EXISTS (SELECT 1 FROM servers replacement WHERE replacement.name=d.server AND replacement.destroyed_at IS NULL)))
-        AND d.state IN ('queued','waiting','running','awaiting-health','interrupted','failed')
+        AND d.state IN ('queued','waiting','running','awaiting-health','interrupted','failed','cancelled')
         AND NOT EXISTS (SELECT 1 FROM server_deployments other WHERE other.server=d.server
             AND other.server_deleted_at IS NULL AND other.rowid>d.rowid AND other.state!='cancelled'
             AND NOT (other.action='destroy' AND other.state='waiting'

src/lib/server/provisioning/cancellation.ts

@@ -0,0 +1,59 @@
+import { db } from '../db/index.js';
+import { ensureHostSchema } from '../hosts/schema.js';
+import { recordProgress } from './progress.js';
+
+export function provisioningCancelled(id: string): boolean {
+    return !!db().prepare("SELECT 1 FROM server_deployments WHERE id=? AND state='cancelled'").get(id);
+}
+
+export function assertProvisioningNotCancelled(id: string): void {
+    if (provisioningCancelled(id)) throw new Error('Provisioning operation cancelled');
+}
+
+export function canCancelProvisioningJob(id: string): boolean {
+    return !!db().prepare(`SELECT 1 FROM server_deployments WHERE id=? AND server_deleted_at IS NULL
+        AND state IN ('queued','waiting','running','awaiting-health','interrupted')`).get(id);
+}
+
+/** Cancel remaining work, not effects already performed by the host. */
+export function cancelProvisioningJob(id: string, reason = 'Cancelled by operator'): boolean {
+    ensureHostSchema(db());
+    return db().transaction(() => {
+        if (provisioningCancelled(id)) return true;
+        if (!canCancelProvisioningJob(id)) return false;
+        const at = new Date().toISOString();
+        // Only commands never delivered can be withdrawn. Delivered commands retain
+        // their immutable IDs and receipts; replacements wait for their resolution.
+        const commands = db().prepare(`SELECT o.id FROM ec_host_operations o
+            WHERE o.state='pending' AND o.last_delivered_at IS NULL AND o.id IN (
+                SELECT operation_id FROM native_provisioning WHERE deployment_id=?
+                UNION SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?
+                UNION SELECT remove_operation_id FROM ec_host_migrations WHERE deployment_id=?)`).all(id, id, id) as { id: string }[];
+        for (const command of commands) {
+            db().prepare("UPDATE ec_host_operations SET state='failed',error=?,updated_at=? WHERE id=?").run(reason, at, command.id);
+            db().prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,'failed',?,?)").run(command.id, at, reason);
+        }
+        db().prepare(`UPDATE staged_file_changes SET state='cancelled' WHERE state='pending' AND EXISTS (
+            SELECT 1 FROM provisioning_sources s WHERE s.operation_id=? AND s.kind=staged_file_changes.source_kind
+                AND s.source_id=staged_file_changes.source_id AND s.server=staged_file_changes.server)`).run(id);
+        db().prepare('DELETE FROM restart_rollout WHERE operation_id=?').run(id);
+        db().prepare("UPDATE server_deployments SET state='cancelled',finished_at=?,stage=?,error=NULL WHERE id=?").run(at, reason, id);
+        recordProgress(id, { blocker: null, checks: [],
+            recovery: { state: 'released', at, operationId: id, message: 'Recovery capacity released by cancellation; this does not certify server health.' },
+            event: { kind: 'outcome', message: reason, details: { state: 'cancelled', finishedAt: at, error: null, warnings: [] } } });
+        return true;
+    })();
+}
+
+/** A cancelled controller job must not let a replacement overtake a delivered mutation. */
+export function unsettledCancelledCommand(server: string, replacementId: string): string | null {
+    const row = db().prepare(`SELECT d.id FROM server_deployments d
+        JOIN native_provisioning n ON n.deployment_id=d.id
+        LEFT JOIN ec_host_migrations m ON m.deployment_id=d.id
+        JOIN ec_host_operations o ON o.id=n.operation_id OR o.id=m.remove_operation_id
+        WHERE d.server=? AND d.server_deleted_at IS NULL AND d.state='cancelled'
+        AND (o.state NOT IN ('completed','failed')
+            OR (o.state='completed' AND o.id=n.operation_id AND n.files_json IS NOT NULL AND n.applied_at IS NULL))
+        AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations c WHERE c.creation_id=d.id AND c.cleanup_id=?) LIMIT 1`).get(server, replacementId) as { id: string } | undefined;
+    return row?.id ?? null;
+}

src/lib/server/provisioning/history.ts

@@ -9,6 +9,7 @@ import { readProgress, recordProgress, type ProgressUpdate } from './progress.js
 import { bindNativeOperation } from './native.js';
 import { workerActivity } from '../pipeline/activity.js';
 import { canCancelCreation } from './cancel-creation.js';
+import { canCancelProvisioningJob, provisioningCancelled } from './cancellation.js';
 import { nativeDeployment } from '../backends/native.js';
 import { serverTitle } from '../../ui/server-labels.js';
 import { provisioningPageSize, provisioningPageOffset, type ProvisioningListOptions, type ProvisioningListPage, type ProvisioningListRow } from '../../provisioning-list.js';
@@ -121,7 +122,7 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
             UNION ALL SELECT collection_id, deployment_id FROM server_collection_removals) m
             JOIN server_launch_collections c ON c.id = m.collection_id WHERE m.deployment_id = ? ORDER BY c.created_at, c.id LIMIT 1`).get(id) as { id: string; name: string } | undefined;
         return { deployment, backend: native ? 'native' as const : 'komodo' as const, agentOperation: agentOperation ?? null,
-            canCancelCreation: canCancelCreation(id), cancellation: cancellation ?? null,
+            canCancel: canCancelProvisioningJob(id), canCancelCreation: canCancelCreation(id), cancellation: cancellation ?? null,
             workerActivity: workerActivity(), events: deploymentEvents(id, options.events === 'recent'),
             activity: { ...activitySummary(id), failureExcerpt: latestFailureEvent(id) }, sources,
             recipe: launch?.recipe ?? null, blueprintName: launch?.blueprintName ?? null,
@@ -224,6 +225,9 @@ export function resumeVerification(id: string): boolean {
 }
 
 export function finishDeployment(id: string, result: { ok: boolean; message?: string; warnings?: string[] }): void {
+    // Creation failure cleanup deletes its own registry row, whose history trigger
+    // marks it cancelled. Preserve that real failure; explicit cancellation stays terminal.
+    if (provisioningCancelled(id) && !db().prepare("SELECT 1 FROM server_deployments WHERE id=? AND server_deleted_at IS NOT NULL AND error='Server deleted'").get(id)) return;
     const state = !result.ok ? 'failed' : result.warnings?.length ? 'warning' : 'done';
     db().transaction(() => {
         const finishedAt = new Date().toISOString();
@@ -257,7 +261,8 @@ const reviewDeployment = "d.state IN ('failed', 'interrupted', 'warning')";
 const deploymentListColumns = `d.id, d.server, d.server_deleted_at, d.action, d.state, d.stage,
         d.started_at, d.finished_at, d.triggered_by, d.error, d.warnings,
         (d.server_deleted_at IS NULL AND EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server AND s.destroyed_at IS NULL)) AS server_exists,
-        (d.server_deleted_at IS NULL AND d.action = 'destroy' AND d.state = 'waiting' AND d.stage != ?) AS canCancelRemoval
+        (d.server_deleted_at IS NULL AND d.action = 'destroy' AND d.state = 'waiting' AND d.stage != ?) AS canCancelRemoval,
+        (d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted')) AS canCancel
     `;
 
 function deploymentListQuery(options: ProvisioningListOptions) {

src/lib/server/provisioning/host-drain.ts

@@ -1,4 +1,5 @@
 import { db } from '../db/index.js';
+import { provisioningCancelled } from './cancellation.js';
 import { nativeInstance, nativeRequest, type NativeInstance } from '../backends/native.js';
 import { getHost, getHostOperationStatus, listHosts, queueHostOperation, reserveHostPort, updateHost } from '../hosts/index.js';
 import { choosePlacement } from '../hosts/capacity.js';
@@ -43,6 +44,10 @@ export function reconcileHostDrains(): void {
             for (const instance of instances) {
                 const server = getServer(instance.server);
                 if (!server?.map_slug || activeBetaRemovalBlocker(instance.server)) continue;
+                // An explicit cancellation withdraws this instance's current drain attempt.
+                const latest = db().prepare(`SELECT d.state FROM ec_host_migrations m JOIN server_deployments d ON d.id=m.deployment_id
+                    WHERE m.instance_id=? AND d.server_deleted_at IS NULL ORDER BY d.rowid DESC LIMIT 1`).get(instance.id) as { state: string } | undefined;
+                if (latest?.state === 'cancelled') continue;
                 if (db().prepare(`SELECT 1 FROM server_deployments WHERE server=? AND server_deleted_at IS NULL
                     AND state IN ('queued','waiting','running','awaiting-health','interrupted')`).get(instance.server)) continue;
                 const recipe = { ...captureRecipePlan(server.game as 't5' | 't6', server.map_slug), image: instance.image,
@@ -67,7 +72,8 @@ export function reconcileHostDrains(): void {
 /** Reserve a destination, remove the source once, then hand creation to the normal health-verified rebuild queue. */
 export async function processMigrationRemoval(job: Job, instance: NativeInstance, migration: Migration,
     eligibility: () => string | null): Promise<void> {
-    const hold = (message: string) => deploymentStage(job.id, message, { blocker: { code: 'native-host', message, automatic: true } });
+    if (provisioningCancelled(job.id)) return;
+    const hold = (message: string) => { if (!provisioningCancelled(job.id)) deploymentStage(job.id, message, { blocker: { code: 'native-host', message, automatic: true } }); };
     if (migration.remove_operation_id) {
         const operation = getHostOperationStatus(migration.remove_operation_id);
         if (operation?.state === 'failed' && Date.now() - Date.parse(operation.updatedAt) >= 60_000) {
@@ -121,8 +127,10 @@ export async function processMigrationRemoval(job: Job, instance: NativeInstance
     // No source mutation is submitted unless another host can take this instance.
     choosePlacement(options);
     const health = await nativeRequest<{ state: string }>(instance, 'instance.inspect');
+    if (provisioningCancelled(job.id)) return;
     if (!['stopped','exited','created','dead','missing'].includes(health.state)) {
         const occupancy = await probePlayerEvidence(job.server);
+        if (provisioningCancelled(job.id)) return;
         if (occupancy.players !== 0) { hold(occupancy.players === null ? 'Waiting for a fresh local player count.' : `Waiting for ${occupancy.players} player(s) to leave.`); return; }
     }
     db().transaction(() => {

src/lib/server/provisioning/instances.ts

@@ -1,6 +1,7 @@
 import { listHosts } from '../hosts/index.js';
 import { fleetSettings } from '../settings.js';
 import { probePlayerEvidence, probeServerStatus } from '../fleet/status-evidence.js';
+import { assertProvisioningNotCancelled, provisioningCancelled } from './cancellation.js';
 import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent, REMOVING_SERVER, resumeCleanup, resumeVerification, claimRemoval } from './history.js';
 import { randomUUID } from 'node:crypto';
 import type { LaunchPreview } from '$lib/deployment-plan.js';
@@ -76,7 +77,7 @@ export function nextLobbyNumber(baseName: string, additionalNames: string[] = []
     const rows = db()
         .prepare(`SELECT name FROM servers WHERE name LIKE ? UNION
             SELECT j.server AS name FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
-            WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued', 'running', 'awaiting-health') OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry'))))`)
+            WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued', 'running', 'awaiting-health') OR (d.state = 'cancelled' AND d.submitted_at IS NOT NULL) OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry'))))`)
         .all(`${baseName}_%`) as Array<{ name: string }>;
     const taken = new Set<number>();
     const prefix = `${baseName}_`;
@@ -100,7 +101,7 @@ export function nextAvailablePort(additionalPorts: number[] = [], releasingServe
     const rows = db()
         .prepare(`SELECT name, game_port FROM servers WHERE game_port IS NOT NULL UNION
             SELECT j.server AS name, j.port AS game_port FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
-            WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued', 'running', 'awaiting-health') OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry'))))`)
+            WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued', 'running', 'awaiting-health') OR (d.state = 'cancelled' AND d.submitted_at IS NOT NULL) OR (d.action = 'rebuild' AND (d.state = 'waiting' OR (d.state IN ('failed','interrupted') AND d.stage = 'Removing server files and registry'))))`)
         .all() as Array<{ name: string; game_port: number }>;
     const taken = new Set([...rows.filter(row => !releasingServers.includes(row.name)).map((r) => r.game_port), ...additionalPorts]);
     for (let p = config.portRange.min; p <= config.portRange.max; p++) {
@@ -159,9 +160,14 @@ function buildSpec(args: {
 }
 
 let creating = false;
-async function observeCommand<T>(operation: string, command: () => Promise<T>): Promise<T> {
-    try { return await command(); }
-    catch (err) {
+async function observeCommand<T>(operation: string, command: () => Promise<T>, retainReceipt = false): Promise<T> {
+    assertProvisioningNotCancelled(operation);
+    try {
+        const result = await command();
+        if (!retainReceipt) assertProvisioningNotCancelled(operation);
+        return result;
+    } catch (err) {
+        if (provisioningCancelled(operation)) throw err;
         recordProgress(operation, { command: { state: 'uncertain', message: 'No confirmed command result was received: ' + (err as Error).message } });
         throw err;
     }
@@ -314,6 +320,7 @@ export async function rebuildServer(id: string): Promise<void> {
     try {
         const result = job.stage === REMOVING_SERVER
             ? await cleanupDestroyedServer(job.server, id) : await destroyServerUnlocked(job.server, id);
+        if (provisioningCancelled(id)) return;
         if (!result.ok || result.warnings.length) {
             finishDeployment(id, { ok: false, message: result.ok ? 'Old server files could not be completely removed.' : result.message, warnings: result.warnings });
             return;
@@ -435,12 +442,16 @@ async function createServerUnlocked(input: CreateServerInput, prepared: Prepared
         return { ok: false, kind: 'fs', message: `failed to create ${hostDir}: ${(err as Error).message}` };
     }
 
+    assertProvisioningNotCancelled(operation);
     // Step 2: Komodo CreateDeployment.
     deploymentStage(operation, 'Creating Komodo deployment', { phase: 'create-deployment',
         checkpoint: { key: 'prepare', label: 'Server files prepared' },
         command: { state: 'attempted', message: 'Komodo deployment creation attempted.' } });
     const spec = buildSpec({ game, name, port, hostDir, plutoKey, recipe });
-    const createRes = await observeCommand(operation, () => komodoCreateDeployment(spec));
+    db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(new Date().toISOString(), operation);
+    // A confirmed creation must retain its identity for inspection/removal even if cancelled in flight.
+    const createRes = await observeCommand(operation, () => komodoCreateDeployment(spec), true);
+    if (!createRes.ok && provisioningCancelled(operation)) assertProvisioningNotCancelled(operation);
     if (!createRes.ok) {
         recordProgress(operation, { command: { state: createRes.kind === 'unreachable' || (createRes.httpStatus !== undefined && createRes.httpStatus >= 500) ? 'uncertain' : 'rejected', message: createRes.message } });
         await rm(hostDir, { recursive: true, force: true }).catch(() => {});
@@ -482,6 +493,7 @@ async function createServerUnlocked(input: CreateServerInput, prepared: Prepared
         return { ok: false, kind: 'db', message: `failed to insert server row: ${(err as Error).message}` };
     }
 
+    assertProvisioningNotCancelled(operation);
     // Step 4: apply Komodo tags so operators can filter the deployment list
     // by game/map. Best-effort — a tag failure doesn't roll back the server;
     // the deployment is fully functional without cosmetic tags. Fires in
@@ -718,6 +730,7 @@ async function destroyServerUnlocked(name: string, operation: string): Promise<D
     const pollDeadline = Date.now() + 30_000;
     while (Date.now() < pollDeadline) {
         const health = await komodoInspectHealthy(name);
+        assertProvisioningNotCancelled(operation);
         if (!health.ok) break;
         await new Promise((resolve) => setTimeout(resolve, 1500));
     }
@@ -740,6 +753,7 @@ async function destroyServerUnlocked(name: string, operation: string): Promise<D
 }
 
 async function cleanupDestroyedServer(name: string, operation: string): Promise<DestroyServerResult> {
+    assertProvisioningNotCancelled(operation);
     const blocker = activeBetaRemovalBlocker(name);
     if (blocker) return { ok: false, kind: 'conflict', message: blocker, warnings: [] };
     const row = getServer(name);
@@ -753,11 +767,13 @@ async function cleanupDestroyedServer(name: string, operation: string): Promise<
     const hostDir = row?.host_dir ?? serverDir(name);
     try {
         await rm(hostDir, { recursive: true, force: true });
+        assertProvisioningNotCancelled(operation);
         recordProgress(operation, { checkpoint: { key: 'files-removed', label: 'Instance files removed', phase: 'cleanup' } });
     } catch (err) {
         warnings.push(`rm ${hostDir} failed: ${(err as Error).message}`);
     }
 
+    assertProvisioningNotCancelled(operation);
     // The registry trigger preserves history and marks its old server identity.
     // Live inventory, rollback snapshots, and restart slots still cascade away.
     try {
@@ -802,6 +818,7 @@ async function provisionCreation(id: string): Promise<CreateServerResult> {
         }
         const result = await createServerUnlocked({ gameMode: job.game_mode, mapSlug: job.map_slug, createdBy: job.triggered_by },
             { ok: true, name: job.server, port: job.port, mapEntry, plutoKey }, id);
+        if (provisioningCancelled(id) && result.ok) return result;
         if (!result.ok) finishDeployment(id, result);
         else {
             db().transaction(() => {
@@ -812,6 +829,7 @@ async function provisionCreation(id: string): Promise<CreateServerResult> {
         }
         return result;
     } catch (err) {
+        if (provisioningCancelled(id)) throw err;
         const server = getServer(job.server);
         if (server?.game_port === job.port) {
             // A lost response may have happened after Komodo started the server.

src/lib/server/provisioning/native.ts

@@ -4,6 +4,8 @@ import { choosePlacement, availableHostPort, type PlacementOptions } from '../ho
 import { randomBytes, randomUUID } from 'node:crypto';
 import { posix } from 'node:path';
 import { db } from '../db/index.js';
+import { ensureHostSchema } from '../hosts/schema.js';
+import { unsettledCancelledCommand } from './cancellation.js';
 import { config, relDestForKind, serverDir } from '../config.js';
 import { nativeInstance, nativeRequest, unplacedNativeCreation, type NativeInstance } from '../backends/native.js';
 import { getHost, getHostOperationStatus, queueHostOperation, queueHostOperationSerialized, releaseHostPorts, reserveHostPort } from '../hosts/index.js';
@@ -244,7 +246,64 @@ function applyReceipt(job: Job, result: Record<string, unknown>) {
     })();
 }
 
-function hold(job: Job, message: string) { deploymentStage(job.id, message, { blocker: { code: 'native-host', message, automatic: true } }); }
+/** Retain confirmed file snapshots and instance identity even after observation is cancelled. */
+export function reconcileNativeProvisioningReceipt(id: string): void {
+    const job = db().prepare(`SELECT d.*,n.instance_id,n.generation,n.operation_id,n.files_json,n.applied_at
+        FROM server_deployments d JOIN native_provisioning n ON n.deployment_id=d.id
+        WHERE d.id=? AND d.server_deleted_at IS NULL`).get(id) as Job | undefined;
+    if (!job) return;
+    const migration = hostMigration(id);
+    if (job.state === 'cancelled' && migration?.remove_operation_id && !migration.removed_at) {
+        const removal = getHostOperationStatus(migration.remove_operation_id);
+        if (removal?.state === 'failed') {
+            db().prepare('DELETE FROM ec_host_ports WHERE host_id=? AND instance_id=?').run(migration.target_host_id, migration.instance_id);
+        } else if (removal?.state === 'completed') {
+            const original = nativeInstance(job.server);
+            if (original?.id === migration.instance_id && original.host_id === migration.source_host_id && original.generation === job.generation) {
+                db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(removal.updatedAt, original.id);
+                releaseHostPorts(original.id);
+                db().prepare('DELETE FROM servers WHERE name=?').run(job.server);
+                db().prepare('UPDATE ec_host_migrations SET removed_at=? WHERE deployment_id=?').run(removal.updatedAt, id);
+                recordProgress(id, { event: { kind: 'cleanup', message: 'Cancelled migration: source removal completed; replacement creation was not started.' } });
+            }
+        }
+    }
+    if (!job.operation_id) return;
+    const operation = getHostOperationStatus(job.operation_id);
+    if (operation?.state === 'failed') {
+        for (const file of JSON.parse(job.files_json ?? '[]') as FilePlan[]) {
+            db().prepare("UPDATE staged_file_changes SET state=? WHERE id=? AND state='applying'")
+                .run(job.state === 'cancelled' ? 'cancelled' : 'pending', file.id);
+        }
+        return;
+    }
+    if (operation?.state !== 'completed') return;
+    const instance = nativeInstance(job.server);
+    if (!instance || instance.id !== job.instance_id) return;
+    if (job.action === 'destroy') {
+        if (instance.generation !== job.generation) return;
+        db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(operation.updatedAt, instance.id);
+        releaseHostPorts(instance.id);
+        db().prepare('DELETE FROM servers WHERE name=?').run(job.server);
+        finishCreationCancellation(job.id);
+        return;
+    }
+    if (job.action === 'rebuild' && !hostMigration(id) && instance.generation === job.generation) {
+        db().prepare('UPDATE native_instances SET generation=generation+1 WHERE id=?').run(instance.id);
+        // Replacements queued while the delivered rebuild was settling bound the
+        // previous generation. Only untouched, newer jobs may follow this receipt.
+        if (job.state === 'cancelled') db().prepare(`UPDATE native_provisioning SET generation=?
+            WHERE instance_id=? AND generation=? AND operation_id IS NULL AND stage_operation_id IS NULL
+            AND deployment_id IN (SELECT newer.id FROM server_deployments newer
+                WHERE newer.rowid>(SELECT rowid FROM server_deployments WHERE id=?)
+                AND newer.server_deleted_at IS NULL AND newer.state IN ('queued','waiting'))`)
+            .run(job.generation + 1, instance.id, job.generation, id);
+    } else if (instance.generation !== job.generation && !(job.action === 'rebuild' && instance.generation === job.generation + 1)) return;
+    applyReceipt(job, (operation.result ?? {}) as Record<string, unknown>);
+    db().prepare('UPDATE native_provisioning SET applied_at=COALESCE(applied_at,?) WHERE deployment_id=?').run(operation.updatedAt, id);
+}
+
+function hold(job: Job, message: string) { if (!stillActive(job)) return; deploymentStage(job.id, message, { blocker: { code: 'native-host', message, automatic: true } }); }
 function stillActive(job: Job): boolean {
     return !!db().prepare(`SELECT 1 FROM server_deployments WHERE id=? AND server_deleted_at IS NULL
         AND state IN ('queued','waiting','running','awaiting-health','interrupted')`).get(job.id);
@@ -259,6 +318,26 @@ function complete(job: Job) {
 
 export async function processNativeProvisioning(eligibility: (operation: { id: string; server: string; action: string; state: string; stage: string }) => string | null,
     options: { wait?: boolean } = {}): Promise<void> {
+    ensureHostSchema(db());
+    const cancelled = db().prepare(`SELECT d.id FROM server_deployments d
+        JOIN native_provisioning n ON n.deployment_id=d.id
+        LEFT JOIN ec_host_operations o ON o.id=n.operation_id
+        LEFT JOIN ec_host_migrations m ON m.deployment_id=d.id
+        LEFT JOIN ec_host_operations removal ON removal.id=m.remove_operation_id
+        WHERE d.state='cancelled' AND d.server_deleted_at IS NULL AND (
+            (o.state='completed' AND (n.applied_at IS NULL OR d.action='destroy'))
+            OR (o.state='failed' AND EXISTS (SELECT 1 FROM staged_file_changes f JOIN provisioning_sources s
+                ON s.kind=f.source_kind AND s.source_id=f.source_id AND s.server=f.server
+                WHERE s.operation_id=d.id AND f.state='applying'))
+            OR (m.removed_at IS NULL AND (removal.state='completed' OR (removal.state='failed'
+                AND EXISTS (SELECT 1 FROM ec_host_ports p WHERE p.host_id=m.target_host_id AND p.instance_id=m.instance_id))))
+        )`).all() as { id: string }[];
+    for (const row of cancelled) {
+        try { reconcileNativeProvisioningReceipt(row.id); }
+        catch (error) {
+            deploymentStage(row.id, 'Cancelled; waiting to reconcile the delivered command: ' + (error as Error).message);
+        }
+    }
     reconcileHostDrains();
     const jobs = db().prepare(`SELECT d.*,n.instance_id,n.generation,n.stage_operation_id,n.operation_id,n.files_json,n.applied_at,q.selection
         FROM server_deployments d LEFT JOIN native_provisioning n ON n.deployment_id=d.id
@@ -346,6 +425,8 @@ async function placeQueuedCreation(job: Job, instance: NativeInstance | null): P
 
 async function processJob(job: Job, eligibility: (job: Job) => string | null) {
     if (!stillActive(job)) return;
+    const unsettled = unsettledCancelledCommand(job.server, job.id);
+    if (unsettled) { hold(job, 'Waiting for the already-delivered command from cancelled operation ' + unsettled + ' to settle.'); return; }
     let instance = nativeInstance(job.server);
     if (job.action === 'create' && !job.operation_id && !job.stage_operation_id && !job.submitted_at) {
         // A resumed bound job owns its original identity. An unplaced intent
@@ -514,6 +595,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
         bindAddress: instance.bind_address, excludeInstanceId: instance.id, thorough: false, dispatch: true });
     if (job.action === 'create' || job.action === 'rebuild') assertReadyLibrary(instance.host_id, instance.library_id);
     const checkOccupancy = async (): Promise<boolean> => {
+        if (!stillActive(job)) return false;
         // Fast controller check keeps occupied lobbies out of the operation slots.
         // The agent repeats this immediately before its local mutation.
         deploymentStage(job.id, 'Checking container and player occupancy before submission', { phase: 'eligibility', blocker: null });

src/lib/server/provisioning/restarts.ts

@@ -1,5 +1,6 @@
 import { fleetSettings } from '../settings.js';
 import { db } from '../db/index.js';
+import { provisioningCancelled } from './cancellation.js';
 import { komodoInspectHealthy, komodoRestart, komodoRuntimeAction } from '../integrations/komodo.js';
 import { probeServerStatus } from '../fleet/status-evidence.js';
 import { deploymentStage, finishDeployment } from './history.js';
@@ -45,6 +46,7 @@ export function rolloutWaitReason(server: string, operationId = '', action: Runt
 }
 
 export function rolloutBlocker(server: string, operationId = '', action: RuntimeAction = 'restart'): ProvisioningBlocker | null {
+    if (requestsInFlight.has(server)) return { code: 'operation-conflict', message: 'Waiting for the previous command submission to finish', automatic: true };
     const conflict = db().prepare(`SELECT id FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL
         AND ((action IN ('start','stop','rebuild') AND state IN ('running','awaiting-health'))
             OR (action = 'destroy' AND state IN ('waiting','running')))`).get(server, operationId) as { id: string } | undefined;
@@ -74,6 +76,7 @@ export type RolloutRestartResult =
 export async function restartInRollout(server: string, options: { operationId: string; action: 'start' | 'restart' }): Promise<RolloutRestartResult> {
     const operationId = options.operationId;
     const waitReason = db().transaction(() => {
+        if (provisioningCancelled(operationId)) return 'Operation was cancelled';
         if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return 'Original server was deleted';
         const reason = rolloutWaitReason(server, operationId, options.action);
         if (reason) return reason;
@@ -111,6 +114,7 @@ export async function restartInRollout(server: string, options: { operationId: s
         game_started_at = NULL, game_ended_at = NULL WHERE name = ?`).run(server);
     try {
         const result = options.action === 'start' ? await komodoRuntimeAction(server, 'start') : await komodoRestart(server);
+        if (provisioningCancelled(operationId)) return { kind: 'failed', message: 'Operation was cancelled' };
         if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return { kind: 'failed', message: 'Original server was deleted' };
         if (result.ok) {
             db().transaction(() => {
@@ -138,6 +142,7 @@ export async function restartInRollout(server: string, options: { operationId: s
         })();
         return { kind: 'failed', message: `komodo: ${result.message}` };
     } catch (err) {
+        if (provisioningCancelled(operationId)) return { kind: 'failed', message: 'Operation was cancelled' };
         if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return { kind: 'failed', message: 'Original server was deleted' };
         const message = `komodo: ${(err as Error).message}`;
         db().transaction(() => {

src/lib/server/provisioning/worker.ts

@@ -1,6 +1,7 @@
 import { maintenanceAllowed } from '../maintenance/policies.js';
 import { resolveRecoveredHealthOperation } from '../maintenance/recovery.js';
 import { db } from '../db/index.js';
+import { provisioningCancelled } from './cancellation.js';
 import { komodoConfigured, rconConfigured } from '../config.js';
 import { getServer } from '../fleet/servers.js';
 import { deploymentStage, finishDeployment, REMOVING_SERVER } from './history.js';
@@ -137,6 +138,7 @@ async function run(): Promise<void> {
         if (nativeDeployment(operation.id)) continue;
         try { await processOperation(operation); }
         catch (err) {
+            if (provisioningCancelled(operation.id)) continue;
             // Observation failures are retried; remote mutations are never replayed.
             deploymentStage(operation.id, 'Waiting for verification: ' + (err as Error).message, {
                 checks: [], blocker: { code: 'observation-unavailable', message: (err as Error).message, automatic: true }
@@ -147,6 +149,7 @@ async function run(): Promise<void> {
 
 async function processOperation(operation: Operation): Promise<void> {
     const { id, server, action } = operation;
+    if (provisioningCancelled(id)) return;
     // An explicit cleanup resume follows confirmed remote deletion. Complete
     // that durable checkpoint even if the incident now needs attention or the
     // Watchdog is paused; never strand the reserved replacement identity.
@@ -194,6 +197,7 @@ async function processOperation(operation: Operation): Promise<void> {
         WHERE s.operation_id = ? AND s.kind = 'watchdog' AND i.kind = 'health' AND d.triggered_by LIKE 'watchdog:%'`).get(id);
     const epoch = latestGameSubmission(server);
     const health = await komodoInspectHealthy(server);
+    if (provisioningCancelled(id)) return;
     if (healthRecovery && epoch !== latestGameSubmission(server)) return;
     if (healthRecovery && !health.ok && !['stopped','exited','created','dead','unhealthy','starting','restarting'].includes(health.state?.toLowerCase() ?? '')) {
         deploymentStage(id, 'Waiting for current container health before automatic recovery', { phase: 'eligibility', checks: [],
@@ -235,6 +239,7 @@ async function processOperation(operation: Operation): Promise<void> {
     if (!claim.changes) return;
     if (action === 'start' || action === 'restart') {
         const result = await restartInRollout(server, { operationId: id, action });
+        if (provisioningCancelled(id)) return;
         if (result.kind === 'queued') {
             db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
             deploymentStage(id, result.message);
@@ -249,6 +254,7 @@ async function processOperation(operation: Operation): Promise<void> {
     // Persist intent first; a lost reply must not issue another stop later.
     try {
         const result = await komodoRuntimeAction(server, 'stop');
+        if (provisioningCancelled(id)) return;
         if (!result.ok) {
             recordProgress(id, { command: { state: result.kind === 'no-name' || result.kind === 'unconfigured' ? 'rejected' : 'uncertain', message: result.message } });
             finishDeployment(id, { ok: false, message: result.message }); return;
@@ -260,6 +266,7 @@ async function processOperation(operation: Operation): Promise<void> {
                 command: { state: 'accepted', message: 'Komodo accepted the stop request; stopped state still requires verification.' } });
         })();
     } catch (err) {
+        if (provisioningCancelled(id)) return;
         recordProgress(id, { command: { state: 'uncertain', message: (err as Error).message } });
         finishDeployment(id, { ok: false, message: (err as Error).message });
     }