Add cancellation and cleanup for native server creation

agent/ec_agent.py

@@ -30,7 +30,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.2.2"
+VERSION = "1.3.0"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 LIBRARY_DOWNLOAD_WORKERS = 4
@@ -672,6 +672,7 @@ class Agent:
         self.docker = docker or Docker()
         self.stopping = threading.Event()
         self.current_operation = None
+        self.cancelled_creations = frozenset()
         self.library_progress = {}
         self.library_transfers = {}
         self.config_path = config.get("configPath", "/etc/electric-cherry/agent.json")
@@ -952,7 +953,7 @@ class Agent:
                 if line.startswith("PRETTY_NAME="):
                     os_name = line.split("=", 1)[1].strip('"')
         update = self.update_inventory()
-        capabilities = (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
+        capabilities = ["instance.cancel-create"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
         return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
 
     def safe_to_disrupt(self, instance_id, generation):
@@ -1040,6 +1041,7 @@ class Agent:
         return game, library, image, port, bind, env
 
     def create(self, instance_id, generation, payload):
+        self.check_creation_cancellation()
         directory = self.instance_dir(instance_id, generation)
         current_marker = directory.parent / "current.json"
         if current_marker.exists():
@@ -1068,12 +1070,19 @@ class Agent:
         for protocol in ("udp", "tcp"):
             args.extend(["--publish", f"{bind_arg}:{port}:{port}/{protocol}"])
         args.append(image)
+        self.check_creation_cancellation()
         self.checkpoint({"phase": "creating", "generation": generation, "specSha256": spec_sha})
         self.docker.call(*args, timeout=120)
+        self.check_creation_cancellation()
         self.docker.call("container", "start", name)
+        self.check_creation_cancellation()
         atomic_json(current_marker, {"generation": generation})
         return self.inspect(instance_id, generation)
 
+    def check_creation_cancellation(self):
+        if self.executing_id and self.executing_id in self.cancelled_creations:
+            raise AgentError("Creation cancelled; waiting for instance cleanup")
+
     def write_initial_files(self, root, files):
         if not isinstance(files, list) or len(files) > 10000:
             raise AgentError("Invalid file list")
@@ -1093,6 +1102,8 @@ class Agent:
         return content
 
     def lifecycle(self, kind, instance_id, generation, payload):
+        if kind == "instance.cancel-create":
+            return self.cancel_creation(instance_id, generation, payload)
         if kind == "instance.create":
             return self.create(instance_id, generation, payload)
         if kind == "instance.inspect":
@@ -1146,6 +1157,38 @@ class Agent:
             raise AgentError("Unsupported instance operation")
         return {**self.inspect(instance_id, generation), "before": before}
 
+    def cancel_creation(self, instance_id, generation, payload):
+        """Explicit launch cancellation, fenced to its immutable creation identity.
+
+        The operation loop serializes this with create. Once delivered, removal
+        needs no RCON response and a durable tombstone prevents a delayed create.
+        Repeating this cleanup after an ambiguous Docker result is safe.
+        """
+        create_id = identifier(payload.get("createOperationId"))
+        marker = self.instance_dir(instance_id, generation).parent / "current.json"
+        if marker.exists() and json.loads(marker.read_text())["generation"] > generation:
+            raise Rejected("Cancelled creation was superseded by a newer generation")
+        saved = self.journal.db.execute("SELECT request FROM operations WHERE id=?", (create_id,)).fetchone()
+        if saved:
+            original = json.loads(saved["request"])
+            if original["kind"] != "instance.create" or original["instanceId"] != instance_id or original["generation"] != generation:
+                raise Rejected("Cancellation does not match the original creation")
+        health = self.inspect(instance_id, generation)
+        if health["exists"] and health.get("creationOperation") != create_id:
+            raise Rejected("Container belongs to a different creation operation")
+        self.checkpoint({"phase": "cancelling-creation", "createOperationId": create_id})
+        # Fence before deleting: a crash must never permit the old creation to run again.
+        self.mark_removed(instance_id, generation)
+        if health["exists"]:
+            self.docker.call("container", "rm", "--force", "--volumes", self.container_name(instance_id, generation))
+        after = self.inspect(instance_id, generation)
+        if after["exists"]:
+            raise AgentError("Cancelled creation container still exists")
+        directory = self.instance_dir(instance_id, generation)
+        if directory.exists():
+            shutil.rmtree(directory)
+        return {**after, "cancelledCreation": create_id}
+
     def stage(self, instance_id, generation, payload, operation_id):
         directory = self.instance_dir(instance_id, generation)
         self.spec(instance_id, generation)
@@ -1609,7 +1652,7 @@ class Agent:
         current_marker = self.instance_dir(instance_id, generation).parent / "current.json"
         if current_marker.exists():
             current = json.loads(current_marker.read_text())
-            if current["generation"] > generation or current["generation"] == generation and current.get("removed") and kind not in ("instance.inspect", "instance.remove"):
+            if current["generation"] > generation or current["generation"] == generation and current.get("removed") and kind not in ("instance.inspect", "instance.remove", "instance.cancel-create"):
                 raise AgentError("Instance generation has been superseded or removed")
         if kind.startswith("instance."):
             return self.lifecycle(kind, instance_id, generation, payload)
@@ -1673,6 +1716,14 @@ class Agent:
         instance_id, generation = operation.get("instanceId"), operation.get("generation")
         checkpoint = json.loads(row["checkpoint"]) if row.get("checkpoint") else {}
         try:
+            if kind == "instance.cancel-create":
+                return self.journal.update(operation["id"], "completed", self.cancel_creation(instance_id, generation, payload))
+            if kind == "instance.create":
+                marker = self.instance_dir(instance_id, generation).parent / "current.json"
+                if marker.exists():
+                    current = json.loads(marker.read_text())
+                    if current["generation"] > generation or current["generation"] == generation and current.get("removed"):
+                        return self.journal.update(operation["id"], "failed", error="Creation was cancelled or superseded")
             if kind == "host.decommission":
                 self.executing_id = operation["id"]
                 try:
@@ -1792,6 +1843,9 @@ class Agent:
 
     def heartbeat(self):
         response = self.request("POST", "/api/agent/heartbeat", {"inventory": self.inventory(), "operationId": self.current_operation})
+        # Only the heartbeat thread replaces this immutable snapshot; it never
+        # mutates the operation journal or races Docker cleanup against create.
+        self.cancelled_creations = frozenset(value for value in response.get("cancelledCreations", []) if isinstance(value, str))
         host = response.get("host", {})
         if isinstance(host.get("portMin"), int) and isinstance(host.get("portMax"), int):
             self.config["portRange"] = {"min": host["portMin"], "max": host["portMax"]}

agent/tests/test_agent.py

@@ -209,6 +209,97 @@ class AgentTests(unittest.TestCase):
         self.assertTrue((self.library / "content/game.ff").exists())
         self.assertFalse(self.agent.instance_dir(INSTANCE, 1).exists())
 
+    def test_cancel_creation_bypasses_occupancy_and_preserves_library(self):
+        self.create()
+        self.agent.rcon = lambda *_: self.fail("Cancellation must not wait for player evidence")
+        operation = self.operation("instance.cancel-create", {"createOperationId": "create"}, "cancel")
+        self.assertEqual(self.agent.execute(operation)["state"], "completed")
+        self.assertFalse(self.agent.inspect(INSTANCE, 1)["exists"])
+        self.assertFalse(self.agent.instance_dir(INSTANCE, 1).exists())
+        self.assertTrue((self.library / "content/game.ff").exists())
+        self.assertEqual(self.agent.execute(operation)["state"], "completed")
+        self.assertEqual(sum(call[:2] == ("container", "rm") for call in self.docker.calls), 1)
+
+    def test_heartbeat_cancellation_stops_create_before_starting_container(self):
+        original_call = self.docker.call
+        def cancel_during_create(*args, **kwargs):
+            result = original_call(*args, **kwargs)
+            if args[:2] == ("container", "create"):
+                self.agent.cancelled_creations = frozenset(["create"])
+            return result
+        self.docker.call = cancel_during_create
+        payload = {"name": "test", "game": "t5", "libraryId": self.library_id, "image": "ghcr.io/ayymoss/plutainer:latest", "port": 28960, "env": {}, "rconPassword": "secret", "files": []}
+        self.assertEqual(self.agent.execute(self.operation("instance.create", payload, "create"))["state"], "unknown")
+        self.assertFalse(any(call[:2] == ("container", "start") for call in self.docker.calls))
+        result = self.agent.execute(self.operation("instance.cancel-create", {"createOperationId": "create"}))
+        self.assertEqual(result["state"], "completed")
+        self.assertFalse(self.agent.inspect(INSTANCE, 1)["exists"])
+
+    def test_cancellation_cleanup_resumes_after_agent_restart(self):
+        self.create()
+        self.docker.lose_reply_after = "rm"
+        cancel = self.operation("instance.cancel-create", {"createOperationId": "create"}, "cancel")
+        self.assertEqual(self.agent.execute(cancel)["state"], "unknown")
+        self.agent.journal.db.close()
+        self.agent = Agent(self.config, self.docker)
+        self.assertEqual(self.agent.execute(cancel)["state"], "completed")
+        self.assertFalse(self.agent.instance_dir(INSTANCE, 1).exists())
+
+    def test_cancel_before_creation_delivery_fences_delayed_create(self):
+        result = self.agent.execute(self.operation("instance.cancel-create", {"createOperationId": "late-create"}))
+        self.assertEqual(result["state"], "completed", result)
+        result = self.agent.execute(self.operation("instance.create", {}, "late-create"))
+        self.assertEqual(result["state"], "failed")
+        self.assertFalse(any(call[:2] == ("container", "create") for call in self.docker.calls))
+
+    def test_cancel_removes_partial_files_without_a_container(self):
+        directory = self.agent.instance_dir(INSTANCE, 1)
+        (directory / "app").mkdir(parents=True)
+        (directory / "app/partial.cfg").write_text("partial")
+        self.agent.journal.accept(self.operation("instance.create", {}, "create"))
+        result = self.agent.execute(self.operation("instance.cancel-create", {"createOperationId": "create"}))
+        self.assertEqual(result["state"], "completed", result)
+        self.assertFalse(directory.exists())
+        self.assertTrue((self.library / "content/game.ff").exists())
+
+    def test_cancel_rejects_a_creation_id_from_another_instance(self):
+        other = self.operation("instance.create", {}, "other-create")
+        other["instanceId"] = "another-instance"
+        self.agent.journal.accept(other)
+        result = self.agent.execute(self.operation("instance.cancel-create", {"createOperationId": "other-create"}))
+        self.assertEqual(result["state"], "failed", result)
+        self.assertFalse((self.agent.instance_dir(INSTANCE, 1).parent / "current.json").exists())
+
+    def test_cancel_creation_rejects_other_creation_and_newer_generation(self):
+        self.create()
+        result = self.agent.execute(self.operation("instance.cancel-create", {"createOperationId": "wrong"}))
+        self.assertEqual(result["state"], "failed")
+        module.atomic_json(self.agent.instance_dir(INSTANCE, 1).parent / "current.json", {"generation": 2})
+        result = self.agent.execute(self.operation("instance.cancel-create", {"createOperationId": "create"}, "old-generation"))
+        self.assertEqual(result["state"], "failed")
+        self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"])
+        self.assertFalse(any(call[:2] == ("container", "rm") for call in self.docker.calls))
+
+    def test_cancel_creation_reconciles_lost_removal_reply_and_fences_unknown_create(self):
+        self.docker.lose_reply_after = "start"
+        payload = {"name": "test", "game": "t5", "libraryId": self.library_id, "image": "ghcr.io/ayymoss/plutainer:latest", "port": 28960, "env": {}, "rconPassword": "secret", "files": []}
+        create = self.operation("instance.create", payload, "create")
+        self.assertEqual(self.agent.execute(create)["state"], "unknown")
+        self.docker.lose_reply_after = "rm"
+        cancel = self.operation("instance.cancel-create", {"createOperationId": "create"}, "cancel")
+        self.assertEqual(self.agent.execute(cancel)["state"], "unknown")
+        self.assertEqual(self.agent.execute(cancel)["state"], "completed")
+        self.assertEqual(self.agent.execute(create)["state"], "failed")
+        self.assertFalse(self.agent.instance_dir(INSTANCE, 1).exists())
+        self.assertTrue(json.loads((self.agent.instance_dir(INSTANCE, 1).parent / "current.json").read_text())["removed"])
+
+    def test_cancel_creation_rejects_unowned_container(self):
+        self.create()
+        self.docker.containers[self.agent.container_name(INSTANCE, 1)]["Config"]["Labels"] = {}
+        result = self.agent.execute(self.operation("instance.cancel-create", {"createOperationId": "create"}))
+        self.assertEqual(result["state"], "failed")
+        self.assertFalse(any(call[:2] == ("container", "rm") for call in self.docker.calls))
+
     def test_unowned_container_never_mutated(self):
         self.create()
         self.docker.containers[self.agent.container_name(INSTANCE, 1)]["Config"]["Labels"] = {}

docs/native-hosts.md

@@ -400,3 +400,29 @@ remain enforced at command admission.
 Older completed operations whose progress snapshots were already discarded
 cannot recover missing check results from this change. Their surviving events
 and host command records remain available without inventing verification times.
+
+### Cancel an unfinished creation
+
+On the native creation operation page, **Cancel creation and remove server**
+stops the launch and queues removal of its exact instance and generation. It
+also works when readiness checks are stuck or the creation failed. This explicit
+cancellation skips player-count checks on both the controller and agent, even
+if players have joined. Completed creations use the normal safe removal flow.
+
+The cancelled operation links to a durable cleanup operation with agent receipts
+and progress. An unsubmitted launch releases its reservations immediately. Once
+an agent command was queued, ports and the instance identity remain reserved
+until the agent confirms container and instance-file removal. Shared game
+libraries, runtime images, saved server keys, and operation history are retained.
+Repeated cancellation requests return the same cleanup operation. A queued
+normal removal waiting for occupancy is superseded by this cleanup.
+
+Remote cancellation requires agent 1.3.0 or later (`instance.cancel-create`
+capability). An older agent is never sent the unsupported cleanup command;
+the cleanup page explains that it is waiting for an update. Offline or disabled
+hosts retain the cleanup request until they reconnect or are enabled. Heartbeats
+notify an executing create to stop between Docker commands; an already executing
+Docker command must return or time out before cleanup can run. Cancellation
+does not require the original creation to acknowledge completion. The agent
+records a removal tombstone before cleanup so a delayed creation command cannot
+recreate the instance, and resumes interrupted cleanup under the same operation ID.

src/lib/provisioning-detail.ts

@@ -95,6 +95,8 @@ export interface ProvisioningCapacity {
     }[];
 }
 export interface ProvisioningDetail {
+    canCancelCreation?: boolean;
+    cancellation?: { cleanupId: string; state: string; stage: string; error: string | null } | null;
     backend?: 'native' | 'komodo';
     workerActivity?: WorkerActivity[];
     agentOperation?: { id: string; hostId: string; state: string; updatedAt: string; createdAt: string; error: string | null } | null;

src/lib/server/backends/schema.ts

@@ -16,5 +16,9 @@ export function migrateNativeInstances(handle: Database.Database): void {
             stage_operation_id TEXT, operation_id TEXT, files_json TEXT,
             applied_at TEXT, verified_at TEXT
         );
+        CREATE TABLE IF NOT EXISTS native_creation_cancellations (
+            creation_id TEXT PRIMARY KEY REFERENCES server_deployments(id),
+            cleanup_id TEXT NOT NULL UNIQUE REFERENCES server_deployments(id)
+        );
     `);
 }

src/lib/server/hosts/index.ts

@@ -288,6 +288,11 @@ type OperationRow = { id: string; host_id: string; kind: string; instance_id: st
 function operationFromRow(row: OperationRow): HostOperation { return { id: row.id, hostId: row.host_id, kind: row.kind, instanceId: row.instance_id, generation: row.generation, payload: JSON.parse(row.payload), state: row.state, result: row.result === null ? null : JSON.parse(row.result), error: row.error, createdAt: row.created_at, updatedAt: row.updated_at }; }
 export function getHostOperation(id: string): HostOperation | null { const row = database().prepare('SELECT * FROM ec_host_operations WHERE id=?').get(id) as OperationRow | undefined; return row ? operationFromRow(row) : null; }
 export function listHostOperations(hostId: string, limit = 100): HostOperation[] { return (database().prepare('SELECT * FROM ec_host_operations WHERE host_id=? ORDER BY created_at DESC,rowid DESC LIMIT ?').all(hostId, Math.min(500, Math.max(1, limit))) as OperationRow[]).map(operationFromRow); }
+/** Heartbeats can tell an executing create to stop at its next safe command boundary. */
+export function pendingCreationCancellations(hostId: string): string[] {
+    return (database().prepare(`SELECT json_extract(payload,'$.createOperationId') AS id FROM ec_host_operations
+        WHERE host_id=? AND kind='instance.cancel-create' AND state IN ('pending','accepted','running','unknown')`).all(hostId) as { id: string }[]).map(row => row.id);
+}
 export function queueHostOperation(hostId: string, kind: string, payload: Record<string, unknown>, options: { instanceId?: string; generation?: number; idempotencyKey?: string; id?: string } = {}): HostOperation {
     const host = requireHost(hostId);
     if (host.decommission) throw new Error('Host cleanup is in progress; new operations are blocked.');
@@ -304,7 +309,7 @@ export function queueHostOperation(hostId: string, kind: string, payload: Record
             }
         }
         if (['instance.create','create'].includes(kind)) requireReadyHost(hostId);
-        if (host.mode === 'disabled') throw new Error('Host is disabled. Enable it before requesting operations.');
+        if (host.mode === 'disabled' && kind !== 'instance.cancel-create') throw new Error('Host is disabled. Enable it before requesting operations.');
         const id = options.id ?? randomUUID(); const timestamp = now();
         database().prepare('INSERT INTO ec_host_operations(id,host_id,kind,instance_id,generation,idempotency_key,payload,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?)').run(id, hostId, kind, options.instanceId ?? null, options.generation ?? null, options.idempotencyKey ?? null, JSON.stringify(payload), timestamp, timestamp);
         database().prepare('INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,?,?)').run(id, 'pending', timestamp);
@@ -317,9 +322,13 @@ export function pendingHostOperations(hostId: string): HostOperation[] {
     // Accepted/running/unknown commands keep the same immutable ID. The agent
     // consults its durable journal before acknowledging or executing a replay.
     return database().transaction(() => {
-        const rows = database().prepare(`SELECT * FROM ec_host_operations WHERE host_id=? AND state IN ('pending','accepted','running','unknown')
+        const rows = database().prepare(`SELECT * FROM ec_host_operations o WHERE host_id=? AND state IN ('pending','accepted','running','unknown')
             AND (? OR state != 'pending' OR kind NOT IN ('instance.create','create'))
-            ORDER BY COALESCE(last_delivered_at,''),created_at,rowid LIMIT 25`).all(hostId, Number(host.readiness === 'ready')) as OperationRow[];
+            AND (kind!='instance.cancel-create' OR ?)
+            AND NOT EXISTS (SELECT 1 FROM ec_host_operations cancellation WHERE cancellation.host_id=o.host_id
+                AND cancellation.kind='instance.cancel-create' AND json_extract(cancellation.payload,'$.createOperationId')=o.id)
+            ORDER BY CASE WHEN kind='instance.cancel-create' THEN 0 ELSE 1 END,COALESCE(last_delivered_at,''),created_at,rowid LIMIT 25`)
+            .all(hostId, Number(host.readiness === 'ready'), Number(host.inventory.capabilities?.includes('instance.cancel-create') ?? false)) as OperationRow[];
         const deliveredAt = now();
         for (const operation of rows) database().prepare('UPDATE ec_host_operations SET last_delivered_at=? WHERE id=?').run(deliveredAt, operation.id);
         return rows.map(operationFromRow);

src/lib/server/provisioning/cancel-creation.ts

@@ -0,0 +1,74 @@
+import { db } from '../db/index.js';
+import { nativeInstance } from '../backends/native.js';
+import { queueHostOperation, releaseHostPorts } from '../hosts/index.js';
+import { beginDeployment, cancelRemoval, deploymentStage, finishDeployment } from './history.js';
+import { recordProgress } from './progress.js';
+
+export function canCancelCreation(id: string): boolean {
+    return !!db().prepare(`SELECT 1 FROM server_deployments d
+        JOIN native_provisioning n ON n.deployment_id=d.id
+        JOIN native_instances i ON i.id=n.instance_id AND i.generation=n.generation AND i.retired_at IS NULL
+        WHERE d.id=? AND d.action='create' AND d.server_deleted_at IS NULL
+        AND d.state IN ('queued','waiting','running','awaiting-health','interrupted','failed')
+        AND NOT EXISTS (SELECT 1 FROM server_deployments other WHERE other.server=d.server
+            AND other.server_deleted_at IS NULL AND other.rowid>d.rowid AND other.state!='cancelled'
+            AND NOT (other.action='destroy' AND other.state='waiting'
+                AND NOT EXISTS (SELECT 1 FROM native_provisioning p WHERE p.deployment_id=other.id AND p.operation_id IS NOT NULL)))`).get(id);
+}
+
+/** Stops observation immediately; remote cleanup remains a durable, separately visible job. */
+export function cancelCreation(id: string, triggeredBy = 'ui'): string | null {
+    return db().transaction(() => {
+        const prior = db().prepare('SELECT cleanup_id FROM native_creation_cancellations WHERE creation_id=?').get(id) as { cleanup_id: string } | undefined;
+        if (prior) return prior.cleanup_id;
+        if (!canCancelCreation(id)) return null;
+        const job = db().prepare(`SELECT d.server,n.operation_id,n.stage_operation_id FROM server_deployments d
+            JOIN native_provisioning n ON n.deployment_id=d.id WHERE d.id=?`).get(id) as
+            { server: string; operation_id: string | null; stage_operation_id: string | null };
+        const instance = nativeInstance(job.server)!;
+        // Replace a removal still waiting on occupancy, including one whose asynchronous probe is outstanding.
+        for (const removal of db().prepare("SELECT id FROM server_deployments WHERE server=? AND action='destroy' AND state='waiting' AND server_deleted_at IS NULL").all(job.server) as { id: string }[]) {
+            cancelRemoval(removal.id);
+        }
+        const cleanupId = beginDeployment(job.server, 'destroy', triggeredBy);
+        db().prepare('INSERT INTO native_creation_cancellations(creation_id,cleanup_id) VALUES(?,?)').run(id, cleanupId);
+        db().prepare("UPDATE server_deployments SET state='cancelled',finished_at=?,error=NULL WHERE id=?").run(new Date().toISOString(), id);
+        deploymentStage(id, 'Creation cancelled; cleanup requested', { blocker: null,
+            event: { kind: 'cancelled', message: 'Creation cancelled. Its partial instance will be removed without a player-count check.', details: { cleanupId } } });
+        if (!job.operation_id && !job.stage_operation_id) {
+            // No remote mutation was ever queued. The reservation is the only resource to remove.
+            db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(new Date().toISOString(), instance.id);
+            releaseHostPorts(instance.id);
+            deploymentStage(cleanupId, 'Unsubmitted creation cancelled; reservations released', { phase: 'cleanup',
+                checkpoint: { key: 'cleanup', label: 'No host command submitted; local reservations released' } });
+            finishDeployment(cleanupId, { ok: true });
+            finishCreationCancellation(cleanupId);
+        } else {
+            const operation = queueHostOperation(instance.host_id, 'instance.cancel-create', { createOperationId: job.operation_id ?? id },
+                { instanceId: instance.id, generation: instance.generation, idempotencyKey: cleanupId });
+            db().prepare('UPDATE native_provisioning SET operation_id=? WHERE deployment_id=?').run(operation.id, cleanupId);
+            db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(new Date().toISOString(), cleanupId);
+            deploymentStage(cleanupId, 'Waiting for host agent to cancel creation and remove its partial instance', { phase: 'container-action', blocker: null,
+                command: { state: 'attempted', message: `Creation cancellation queued as agent operation ${operation.id}. Ports remain reserved until cleanup is confirmed.` } });
+        }
+        return cleanupId;
+    })();
+}
+
+export function finishCreationCancellation(cleanupId: string): void {
+    const cancellation = db().prepare(`SELECT c.creation_id,n.operation_id FROM native_creation_cancellations c
+        JOIN native_provisioning n ON n.deployment_id=c.creation_id WHERE c.cleanup_id=?`).get(cleanupId) as
+        { creation_id: string; operation_id: string | null } | undefined;
+    if (!cancellation) return;
+    const now = new Date().toISOString();
+    db().prepare('UPDATE server_deployments SET server_deleted_at=? WHERE id IN (?,?)').run(now, cancellation.creation_id, cleanupId);
+    deploymentStage(cancellation.creation_id, 'Creation cancelled; partial instance removed', { blocker: null,
+        checkpoint: { key: 'cleanup', phase: 'cleanup', label: 'Cancellation cleanup confirmed; instance and reservations removed' } });
+    // The original command may have lost its receipt. Cleanup proves that its effects are now gone.
+    if (cancellation.operation_id) {
+        const result = db().prepare("UPDATE ec_host_operations SET state='failed',error='Creation cancelled; cleanup confirmed',updated_at=? WHERE id=? AND state NOT IN ('completed','failed')")
+            .run(now, cancellation.operation_id);
+        if (result.changes) db().prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,'failed',?,'Creation cancelled; cleanup confirmed')").run(cancellation.operation_id, now);
+    }
+    recordProgress(cleanupId, { event: { kind: 'cleanup', message: 'Cancelled creation cleanup completed.', details: { creationId: cancellation.creation_id } } });
+}

src/lib/server/provisioning/history.ts

@@ -8,6 +8,7 @@ import { activitySummary, deploymentEvents, latestFailureEvent } from './activit
 import { readProgress, recordProgress, type ProgressUpdate } from './progress.js';
 import { bindNativeOperation } from './native.js';
 import { workerActivity } from '../pipeline/activity.js';
+import { canCancelCreation } from './cancel-creation.js';
 
 export function beginDeployment(server: string, action: ProvisioningAction, triggeredBy = 'ui'): string {
     const id = randomUUID();
@@ -105,6 +106,8 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
         const native = db().prepare('SELECT instance_id FROM native_provisioning WHERE deployment_id=?').get(id);
         const agentOperation = native ? db().prepare(`SELECT o.id, o.host_id AS hostId, o.state, o.updated_at AS updatedAt, o.created_at AS createdAt, o.error
             FROM native_provisioning n JOIN ec_host_operations o ON o.id=n.operation_id WHERE n.deployment_id=?`).get(id) as ProvisioningDetail['agentOperation'] : null;
+        const cancellation = db().prepare(`SELECT c.cleanup_id AS cleanupId,d.state,d.stage,d.error FROM native_creation_cancellations c
+            JOIN server_deployments d ON d.id=c.cleanup_id WHERE c.creation_id=?`).get(id) as ProvisioningDetail['cancellation'];
         const { observation_json: _snapshot, phase_key: _phase, phase_started_at: _phaseStarted, observed_at: _observed, ...deployment } = row;
         const sources = (db().prepare('SELECT kind, source_id, server FROM provisioning_sources WHERE operation_id = ? ORDER BY kind, source_id')
             .all(id) as Omit<ProvisioningSource, 'label' | 'href'>[]).map(sourceDetails);
@@ -114,6 +117,7 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
             UNION ALL SELECT collection_id, deployment_id FROM server_collection_removals) m
             JOIN server_launch_collections c ON c.id = m.collection_id WHERE m.deployment_id = ? ORDER BY c.created_at, c.id LIMIT 1`).get(id) as { id: string; name: string } | undefined;
         return { deployment, backend: native ? 'native' as const : 'komodo' as const, agentOperation: agentOperation ?? null,
+            canCancelCreation: canCancelCreation(id), cancellation: cancellation ?? null,
             workerActivity: workerActivity(), events: deploymentEvents(id, options.events === 'recent'),
             activity: { ...activitySummary(id), failureExcerpt: latestFailureEvent(id) }, sources,
             recipe: launch ? summarizeRecipe(launch.recipe) : null, blueprintName: launch?.blueprintName ?? null,

src/lib/server/provisioning/native.ts

@@ -21,6 +21,7 @@ import { sha256Buffer } from '../fleet/files.js';
 import type { StagedFileChange } from '../fleet/staged-files.js';
 import type { LaunchPreview } from '../../deployment-plan.js';
 import type { ProvisioningAction } from '../../provisioning.js';
+import { cancelCreation, finishCreationCancellation } from './cancel-creation.js';
 
 export interface NativeSelection { hostId: string; libraryId: string; image?: string; bindAddress?: string; publicAddress?: string; publicPort?: number }
 type NativeRecipe = LaunchRecipe & { native?: NativeSelection };
@@ -146,15 +147,7 @@ export function cancelQueuedNativeCreation(server: string, triggeredBy: string):
         const creation = db().prepare(`SELECT d.id FROM server_deployments d JOIN native_provisioning n ON n.deployment_id=d.id
             WHERE n.instance_id=? AND d.action='create' AND d.state='queued' AND n.operation_id IS NULL`).get(instance.id) as { id: string } | undefined;
         if (!creation) return null;
-        const id = beginDeployment(server, 'destroy', triggeredBy);
+        return cancelCreation(creation.id, triggeredBy);
-        const now = new Date().toISOString();
-        db().prepare("UPDATE server_deployments SET state='cancelled',finished_at=?,stage='Queued native creation cancelled',server_deleted_at=? WHERE id=?")
-            .run(now, now, creation.id);
-        db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(now, instance.id);
-        releaseHostPorts(instance.id);
-        deploymentStage(id, 'Cancelled queued creation; no agent mutation was submitted');
-        finishDeployment(id, { ok: true });
-        return id;
     })();
 }
 
@@ -277,6 +270,7 @@ export async function processNativeProvisioning(eligibility: (operation: { id: s
 const activeJobs = new Map<string, Promise<void>>();
 
 async function processJob(job: Job, eligibility: (job: Job) => string | null) {
+    if (!stillActive(job)) return;
     let instance = nativeInstance(job.server);
     if (!instance || instance.id !== job.instance_id || (instance.generation !== job.generation && !(job.action === 'rebuild' && instance.generation === job.generation + 1))) {
         finishDeployment(job.id, { ok: false, message: 'Original instance was retired or replaced; no command was sent.' }); return;
@@ -288,6 +282,9 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
     }
     if (job.operation_id) {
         const operation = getHostOperation(job.operation_id);
+        if (operation?.kind === 'instance.cancel-create' && operation.state === 'pending' && !host.inventory.capabilities?.includes('instance.cancel-create')) {
+            hold(job, 'Update the host agent to support creation cancellation. Creation is cancelled; cleanup and port release await the updated agent.'); return;
+        }
         if (!operation || operation.state === 'unknown') { hold(job, 'Agent result is unknown. Reconciling the original operation; no replacement or repeated command will be sent.'); return; }
         if (operation.state === 'failed') {
             // The agent reports failed only when no mutation began. Ambiguous
@@ -312,6 +309,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
                 db().prepare('DELETE FROM servers WHERE name=?').run(job.server);
                 recordProgress(job.id, { checkpoint: { key: 'cleanup', label: 'Instance retired and port reservations released' } });
                 complete(job);
+                finishCreationCancellation(job.id);
             })();
             return;
         }

src/lib/ui/provisioning-presentation.ts

@@ -86,6 +86,9 @@ export function provisioningOutcome(detail: ProvisioningDetail) {
     if (detail.canResumeCleanup) description = 'The remote deployment was deleted. Resume the remaining local cleanup without repeating container removal.';
     else if (detail.canResumeRebuild) description = 'The remote deployment was deleted. Resume local cleanup, then create a fresh instance in the reserved server slot.';
     else if (detail.canResumeVerification) description = 'Resume observation of the existing instance. This does not create, restart, or rebuild a container.';
+    else if (detail.cancellation) description = detail.cancellation.state === 'done'
+        ? 'Creation was cancelled. Its instance files, container, and port reservations have been removed.'
+        : 'Creation was cancelled. Cleanup is tracked separately below; ports remain reserved until the host confirms removal.';
     else if (operation.state === 'cancelled') description = 'This request is cancelled. Recorded prior work is preserved below; remaining work will not run.';
     else if (failure) description = progress?.command?.state === 'uncertain'
         ? 'The remote response was uncertain. Inspect the recorded result and server before requesting another operation.'
@@ -160,6 +163,10 @@ export function isProvisioningEventPage(value: unknown): value is ProvisioningEv
 /** Validate every nested shape consumed by the page before replacing its last good snapshot. */
 export function isProvisioningDetail(value: unknown, id: string): value is ProvisioningDetail {
     if (!record(value) || !record(value.deployment)) return false;
+    if (value.canCancelCreation !== undefined && typeof value.canCancelCreation !== 'boolean') return false;
+    if (value.cancellation !== undefined && value.cancellation !== null && !(record(value.cancellation)
+        && text(value.cancellation.cleanupId) && text(value.cancellation.state) && text(value.cancellation.stage)
+        && nullableText(value.cancellation.error))) return false;
     if (value.backend !== undefined && !['native', 'komodo'].includes(value.backend as string)) return false;
     if (value.workerActivity !== undefined && (!Array.isArray(value.workerActivity) || !value.workerActivity.every(item =>
         record(item) && text(item.key) && text(item.label) && timestamp(item.startedAt)))) return false;

src/routes/api/agent/heartbeat/+server.ts

@@ -1,5 +1,5 @@
 import { json } from '@sveltejs/kit';
-import { heartbeat } from '$lib/server/hosts/index.js';
+import { heartbeat, pendingCreationCancellations } from '$lib/server/hosts/index.js';
 import { authenticateAgent, hostApiError, objectBody } from '$lib/server/hosts/http.js';
 import { reconcileHostPreparation } from '$lib/server/hosts/preparation.js';
 export async function POST({ request }: { request: Request }) {
@@ -13,6 +13,6 @@ export async function POST({ request }: { request: Request }) {
             updateHostLibraryCaches(host.id, inventory.libraryCaches, { snapshot: true });
         }
         reconcileHostPreparation(host.id);
-        return json({ host, heartbeatSeconds: 15, serverTime: new Date().toISOString() });
+        return json({ host, cancelledCreations: pendingCreationCancellations(host.id), heartbeatSeconds: 15, serverTime: new Date().toISOString() });
     } catch (cause) { return hostApiError(cause); }
 }

src/routes/api/deployments/[id]/cancel/+server.ts

@@ -1,7 +1,14 @@
 import { json } from '@sveltejs/kit';
-import { cancelRemoval } from '$lib/server/provisioning/index.js';
+import { cancelRemoval } from '$lib/server/provisioning/history.js';
+import { cancelCreation } from '$lib/server/provisioning/cancel-creation.js';
+import { kickProvisioning } from '$lib/server/pipeline/worker.js';
 
 export function POST({ params }: { params: { id: string } }) {
-    if (!cancelRemoval(params.id)) return json({ error: 'Removal has already begun or this request is no longer waiting. Refresh to see its current state.' }, { status: 409 });
+    const cleanupId = cancelCreation(params.id);
+    if (cleanupId) {
+        void kickProvisioning();
+        return json({ ok: true, cleanupId });
+    }
+    if (!cancelRemoval(params.id)) return json({ error: 'This operation can no longer be cancelled. Refresh to see its current state.' }, { status: 409 });
     return json({ ok: true });
 }

src/routes/provisioning/[id]/+page.svelte

@@ -87,14 +87,16 @@
             {#if detail.activity.failureExcerpt && detail.activity.failureExcerpt.message !== deployment.error && ['failed', 'interrupted'].includes(deployment.state)}<p class="failure-excerpt"><span>Last failure event · {fmt(detail.activity.failureExcerpt.created_at)}</span>{detail.activity.failureExcerpt.message}</p>{/if}
             {#if detail.warningMessages.length}<div class="warnings"><h3>Recorded warnings</h3><ul>{#each detail.warningMessages as warning}<li>{warning}</li>{/each}</ul></div>{/if}
 
-            {#if detail.canResumeVerification || detail.canResumeCleanup || detail.canResumeRebuild || detail.canCancelRemoval}
+            {#if detail.canResumeVerification || detail.canResumeCleanup || detail.canResumeRebuild || detail.canCancelRemoval || detail.canCancelCreation}
                 <div class="recovery-actions">
                     {#if detail.canResumeVerification}<Button variant="primary" loading={view.busy === 'resume'} disabled={!!view.busy} onclick={() => controller.act('resume', 'Health verification requested. The existing instance will be observed without another creation or restart.')}>Resume health verification</Button>{/if}
                     {#if detail.canResumeRebuild}<Button variant="primary" loading={view.busy === 'resume'} disabled={!!view.busy} onclick={() => controller.act('resume', 'Rebuild cleanup requested. Fresh creation waits for local cleanup to finish.')}>Resume rebuild cleanup</Button>{/if}
                     {#if detail.canResumeCleanup}<Button variant="primary" loading={view.busy === 'resume'} disabled={!!view.busy} onclick={() => controller.act('resume', 'Local cleanup requested. Remote container deletion will not be repeated.')}>Resume cleanup</Button>{/if}
                     {#if detail.canCancelRemoval}<div><Button disabled={!!view.busy} loading={view.busy === 'cancel'} onclick={() => controller.act('cancel', 'Queued removal cancelled. The server and its files were kept.')}>Cancel queued removal</Button><p class="action-help">Available until removal begins. Cancelling keeps the server and its files.</p></div>{/if}
+                    {#if detail.canCancelCreation}<div><Button variant="danger" disabled={!!view.busy} loading={view.busy === 'cancel'} onclick={() => controller.act('cancel', 'Creation cancelled. Follow cleanup below until removal is confirmed.')}>Cancel creation and remove server</Button><p class="action-help">Stops this launch and deletes its container and instance files, even if player status is unknown or players have joined. Shared game files and the saved server key are kept.</p></div>{/if}
                 </div>
             {/if}
+            {#if detail.cancellation}<div class="recovery-note"><strong>{detail.cancellation.state === 'done' ? 'Cancellation cleanup completed' : 'Cancellation cleanup'}</strong><p>{detail.cancellation.error ?? detail.cancellation.stage}</p><a href={`/provisioning/${encodeURIComponent(detail.cancellation.cleanupId)}`}>View cleanup progress →</a></div>{/if}
             {#if view.actionError}<div class="action-error" role="alert"><strong>Request not confirmed</strong><p>{view.actionError}</p></div>{/if}
             {#if view.notice}<p class="notice" role="status">{view.notice}</p>{/if}
             {#if detail.progress?.blocker?.href}<a class="dependency" href={detail.progress.blocker.href}>{detail.progress.blocker.label ?? 'View blocking work'} →</a>{/if}

static/agent/ec-agent.py

@@ -30,7 +30,7 @@ import urllib.parse
 import urllib.request
 import uuid
 
-VERSION = "1.2.2"
+VERSION = "1.3.0"
 LABEL = "io.electric-cherry"
 MAX_FILE = 32 * 1024 * 1024
 LIBRARY_DOWNLOAD_WORKERS = 4
@@ -672,6 +672,7 @@ class Agent:
         self.docker = docker or Docker()
         self.stopping = threading.Event()
         self.current_operation = None
+        self.cancelled_creations = frozenset()
         self.library_progress = {}
         self.library_transfers = {}
         self.config_path = config.get("configPath", "/etc/electric-cherry/agent.json")
@@ -952,7 +953,7 @@ class Agent:
                 if line.startswith("PRETTY_NAME="):
                     os_name = line.split("=", 1)[1].strip('"')
         update = self.update_inventory()
-        capabilities = (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
+        capabilities = ["instance.cancel-create"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else [])
         return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})}
 
     def safe_to_disrupt(self, instance_id, generation):
@@ -1040,6 +1041,7 @@ class Agent:
         return game, library, image, port, bind, env
 
     def create(self, instance_id, generation, payload):
+        self.check_creation_cancellation()
         directory = self.instance_dir(instance_id, generation)
         current_marker = directory.parent / "current.json"
         if current_marker.exists():
@@ -1068,12 +1070,19 @@ class Agent:
         for protocol in ("udp", "tcp"):
             args.extend(["--publish", f"{bind_arg}:{port}:{port}/{protocol}"])
         args.append(image)
+        self.check_creation_cancellation()
         self.checkpoint({"phase": "creating", "generation": generation, "specSha256": spec_sha})
         self.docker.call(*args, timeout=120)
+        self.check_creation_cancellation()
         self.docker.call("container", "start", name)
+        self.check_creation_cancellation()
         atomic_json(current_marker, {"generation": generation})
         return self.inspect(instance_id, generation)
 
+    def check_creation_cancellation(self):
+        if self.executing_id and self.executing_id in self.cancelled_creations:
+            raise AgentError("Creation cancelled; waiting for instance cleanup")
+
     def write_initial_files(self, root, files):
         if not isinstance(files, list) or len(files) > 10000:
             raise AgentError("Invalid file list")
@@ -1093,6 +1102,8 @@ class Agent:
         return content
 
     def lifecycle(self, kind, instance_id, generation, payload):
+        if kind == "instance.cancel-create":
+            return self.cancel_creation(instance_id, generation, payload)
         if kind == "instance.create":
             return self.create(instance_id, generation, payload)
         if kind == "instance.inspect":
@@ -1146,6 +1157,38 @@ class Agent:
             raise AgentError("Unsupported instance operation")
         return {**self.inspect(instance_id, generation), "before": before}
 
+    def cancel_creation(self, instance_id, generation, payload):
+        """Explicit launch cancellation, fenced to its immutable creation identity.
+
+        The operation loop serializes this with create. Once delivered, removal
+        needs no RCON response and a durable tombstone prevents a delayed create.
+        Repeating this cleanup after an ambiguous Docker result is safe.
+        """
+        create_id = identifier(payload.get("createOperationId"))
+        marker = self.instance_dir(instance_id, generation).parent / "current.json"
+        if marker.exists() and json.loads(marker.read_text())["generation"] > generation:
+            raise Rejected("Cancelled creation was superseded by a newer generation")
+        saved = self.journal.db.execute("SELECT request FROM operations WHERE id=?", (create_id,)).fetchone()
+        if saved:
+            original = json.loads(saved["request"])
+            if original["kind"] != "instance.create" or original["instanceId"] != instance_id or original["generation"] != generation:
+                raise Rejected("Cancellation does not match the original creation")
+        health = self.inspect(instance_id, generation)
+        if health["exists"] and health.get("creationOperation") != create_id:
+            raise Rejected("Container belongs to a different creation operation")
+        self.checkpoint({"phase": "cancelling-creation", "createOperationId": create_id})
+        # Fence before deleting: a crash must never permit the old creation to run again.
+        self.mark_removed(instance_id, generation)
+        if health["exists"]:
+            self.docker.call("container", "rm", "--force", "--volumes", self.container_name(instance_id, generation))
+        after = self.inspect(instance_id, generation)
+        if after["exists"]:
+            raise AgentError("Cancelled creation container still exists")
+        directory = self.instance_dir(instance_id, generation)
+        if directory.exists():
+            shutil.rmtree(directory)
+        return {**after, "cancelledCreation": create_id}
+
     def stage(self, instance_id, generation, payload, operation_id):
         directory = self.instance_dir(instance_id, generation)
         self.spec(instance_id, generation)
@@ -1609,7 +1652,7 @@ class Agent:
         current_marker = self.instance_dir(instance_id, generation).parent / "current.json"
         if current_marker.exists():
             current = json.loads(current_marker.read_text())
-            if current["generation"] > generation or current["generation"] == generation and current.get("removed") and kind not in ("instance.inspect", "instance.remove"):
+            if current["generation"] > generation or current["generation"] == generation and current.get("removed") and kind not in ("instance.inspect", "instance.remove", "instance.cancel-create"):
                 raise AgentError("Instance generation has been superseded or removed")
         if kind.startswith("instance."):
             return self.lifecycle(kind, instance_id, generation, payload)
@@ -1673,6 +1716,14 @@ class Agent:
         instance_id, generation = operation.get("instanceId"), operation.get("generation")
         checkpoint = json.loads(row["checkpoint"]) if row.get("checkpoint") else {}
         try:
+            if kind == "instance.cancel-create":
+                return self.journal.update(operation["id"], "completed", self.cancel_creation(instance_id, generation, payload))
+            if kind == "instance.create":
+                marker = self.instance_dir(instance_id, generation).parent / "current.json"
+                if marker.exists():
+                    current = json.loads(marker.read_text())
+                    if current["generation"] > generation or current["generation"] == generation and current.get("removed"):
+                        return self.journal.update(operation["id"], "failed", error="Creation was cancelled or superseded")
             if kind == "host.decommission":
                 self.executing_id = operation["id"]
                 try:
@@ -1792,6 +1843,9 @@ class Agent:
 
     def heartbeat(self):
         response = self.request("POST", "/api/agent/heartbeat", {"inventory": self.inventory(), "operationId": self.current_operation})
+        # Only the heartbeat thread replaces this immutable snapshot; it never
+        # mutates the operation journal or races Docker cleanup against create.
+        self.cancelled_creations = frozenset(value for value in response.get("cancelledCreations", []) if isinstance(value, str))
         host = response.get("host", {})
         if isinstance(host.get("portMin"), int) and isinstance(host.get("portMax"), int):
             self.config["portRange"] = {"min": host["portMin"], "max": host["portMax"]}

static/agent/manifest.json

@@ -1,8 +1,8 @@
 {
-  "version": "1.2.2",
+  "version": "1.3.0",
   "protocolVersion": 1,
   "platform": "debian-amd64",
-  "sha256": "c266a8099130051ee540cdc3b9cdc7e42144fa67e21cce76eea6cf0a426bf40d",
+  "sha256": "a698bdbba3716726a5a63fbd18392b53bd4c1e0c4cbaca94eabd66d8ca328dc2",
   "updaterProtocolVersion": 1,
   "updaterSha256": "066bb896726600ece9ce72697ce523c9a707cc9a621505931ba0892dec59d3c8"
 }

tests/unit/native-provisioning.test.ts

@@ -9,7 +9,7 @@ vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
     rconGetPlayerCount: vi.fn(async () => remote.players), rconPlutoniumVersion: vi.fn(async () => ({ revision: 5354, error: null }))
 }));
 vi.mock('../../src/lib/server/integrations/plutonium.js', () => ({ checkRelease: vi.fn(async () => {}), freshRelease: () => 5354, probeVersion: vi.fn(async () => 5354) }));
-vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
+vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) }));
 
 let fixture: Fixture; let hostId: string; let libraryId: string;
 const image = 'example.test/plutainer:test';
@@ -73,6 +73,119 @@ async function detail(id: string) {
 }
 
 describe('native instance provisioning', () => {
+    it('accepts cancellation through the provisioning API and rejects an already completed creation', async () => {
+        const job = await queue();
+        const { POST } = await import('../../src/routes/api/provisioning/[id]/cancel/+server.js');
+        const response = POST({ params: { id: job.deploymentId } });
+        expect(response.status).toBe(200);
+        expect(await response.json()).toMatchObject({ ok: true, cleanupId: expect.any(String) });
+        const next = await queue(); await advance(); await acknowledge(next.deploymentId);
+        expect(POST({ params: { id: next.deploymentId } }).status).toBe(409);
+        expect(POST({ params: { id: job.deploymentId } }).status).toBe(200);
+    });
+    it('cancels an unsubmitted creation locally and returns the same cleanup on repeated requests', async () => {
+        const job = await queue();
+        const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
+        const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
+        const { db } = await import('../../src/lib/server/db/index.js');
+        expect(getDeployment(job.deploymentId)!.canCancelCreation).toBe(true);
+        const cleanup = cancelCreation(job.deploymentId)!;
+        expect(cancelCreation(job.deploymentId)).toBe(cleanup);
+        expect((await detail(job.deploymentId)).state).toBe('cancelled');
+        expect((await detail(cleanup)).state).toBe('done');
+        expect(getDeployment(job.deploymentId)!.cancellation?.state).toBe('done');
+        expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
+        await advance();
+        expect(await operationFor(job.deploymentId)).toBeNull();
+        expect(await operationFor(cleanup)).toBeNull();
+    });
+
+    it('cancels a submitted creation without occupancy checks, suppresses replay, and releases resources only after cleanup', async () => {
+        const job = await queue(); await advance(); remote.players = 4;
+        const original = (await operationFor(job.deploymentId))!;
+        const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
+        const { heartbeat, pendingHostOperations, reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
+        heartbeat(hostId, { ...inventory(), capabilities: ['instance.cancel-create'] });
+        // A previously delivered command might still be executing when cancellation arrives.
+        expect(pendingHostOperations(hostId).map(op => op.id)).toContain(original.id);
+        const cleanup = cancelCreation(job.deploymentId)!;
+        const removal = (await operationFor(cleanup))!;
+        expect(removal.kind).toBe('instance.cancel-create');
+        expect(removal.payload).toEqual({ createOperationId: original.id });
+        expect(cancelCreation(job.deploymentId)).toBe(cleanup);
+        expect(pendingHostOperations(hostId).map(op => op.id)).toContain(removal.id);
+        expect(pendingHostOperations(hostId).map(op => op.id)).not.toContain(original.id);
+        await advance();
+        expect(nativeRequest).not.toHaveBeenCalled();
+        expect(db().prepare('SELECT * FROM ec_host_ports').all()).toHaveLength(2);
+        reportHostOperation(hostId, original.id, { state: 'completed', result: { healthy: true } });
+        await advance();
+        expect((await detail(job.deploymentId)).state).toBe('cancelled');
+        expect(nativeRequest).not.toHaveBeenCalled();
+        reportHostOperation(hostId, removal.id, { state: 'completed', result: { exists: false } });
+        await advance();
+        expect((await detail(cleanup)).state).toBe('done');
+        expect((await detail(job.deploymentId)).server_deleted_at).toBeTruthy();
+        expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]);
+        expect(db().prepare('SELECT * FROM servers WHERE name=?').get(job.name)).toBeUndefined();
+        expect(db().prepare('SELECT retired_at FROM native_instances WHERE id=?').get(original.instanceId)).toMatchObject({ retired_at: expect.any(String) });
+    });
+
+    it('holds cancellation for an old or offline host without delivering an unsupported command or losing reservations', async () => {
+        const job = await queue(); await advance();
+        const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
+        const { heartbeat, pendingHostOperations } = await import('../../src/lib/server/hosts/index.js');
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const cleanup = cancelCreation(job.deploymentId)!;
+        expect(pendingHostOperations(hostId).filter(op => op.kind.startsWith('instance.'))).toEqual([]);
+        await advance();
+        expect((await detail(cleanup)).stage).toContain('Update the host agent');
+        db().prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?').run('2000-01-01T00:00:00.000Z', hostId);
+        await advance();
+        expect((await detail(cleanup)).stage).toContain('unavailable');
+        expect(db().prepare('SELECT * FROM ec_host_ports').all()).toHaveLength(2);
+        heartbeat(hostId, { ...inventory(), capabilities: ['instance.cancel-create'] });
+        expect(pendingHostOperations(hostId).filter(op => op.kind === 'instance.cancel-create')).toHaveLength(1);
+        await acknowledge(cleanup);
+        expect((await detail(cleanup)).state).toBe('done');
+    });
+
+    it('replaces an occupancy-blocked removal and ignores an outstanding creation health reply', async () => {
+        const job = await queue(); await advance();
+        const operation = (await operationFor(job.deploymentId))!;
+        const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        reportHostOperation(hostId, operation.id, { state: 'completed', result: { healthy: true } });
+        const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
+        let release!: (value: never) => void;
+        vi.mocked(nativeRequest).mockImplementationOnce(() => new Promise(resolve => { release = resolve; }));
+        const pending = advance();
+        try {
+            const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js');
+            const waitingRemoval = destroyServer(job.name).deploymentId!;
+            const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
+            const cleanup = cancelCreation(job.deploymentId)!;
+            expect(cleanup).toBeTruthy();
+            expect((await detail(waitingRemoval)).state).toBe('cancelled');
+            release({ healthy: true, state: 'running' } as never);
+            await pending;
+            expect((await detail(job.deploymentId)).state).toBe('cancelled');
+            expect((await detail(cleanup)).state).toBe('running');
+        } finally { release({ healthy: true, state: 'running' } as never); await pending; }
+    });
+
+    it('does not offer creation cancellation after success or against a replacement instance', async () => {
+        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
+        const { canCancelCreation, cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
+        expect(canCancelCreation(job.deploymentId)).toBe(false);
+        expect(cancelCreation(job.deploymentId)).toBeNull();
+        const { db } = await import('../../src/lib/server/db/index.js');
+        db().prepare("UPDATE server_deployments SET state='awaiting-health' WHERE id=?").run(job.deploymentId);
+        db().prepare('UPDATE native_instances SET generation=generation+1 WHERE server=?').run(job.name);
+        expect(cancelCreation(job.deploymentId)).toBeNull();
+    });
+
     it('retains every native launch checkpoint and readiness check through completion and UI validation', async () => {
         const job = await queue();
         const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');

tests/unit/provisioning-presentation.test.ts

@@ -19,6 +19,17 @@ function fixture(): ProvisioningDetail {
 }
 
 describe('provisioning outcome and evidence presentation', () => {
+    it('distinguishes cancellation from confirmed cleanup and validates its link data', () => {
+        const detail = fixture();
+        detail.deployment.state = 'cancelled';
+        detail.canCancelCreation = false;
+        detail.cancellation = { cleanupId: 'cleanup-a', state: 'running', stage: 'Waiting for host', error: null };
+        expect(isProvisioningDetail(detail, detail.deployment.id)).toBe(true);
+        expect(provisioningOutcome(detail).description).toContain('ports remain reserved');
+        detail.cancellation.state = 'done';
+        expect(provisioningOutcome(detail).description).toContain('have been removed');
+        expect(isProvisioningDetail({ ...detail, cancellation: { state: 'done' } }, detail.deployment.id)).toBe(false);
+    });
     it.each(provisioningActions)('describes cancelled %s as terminal, with no automatic execution claim', action => {
         const detail = fixture();
         detail.deployment.action = action;