diff --git a/agent/ec_agent.py b/agent/ec_agent.py index 22f605e..e8b9018 100644 --- a/agent/ec_agent.py +++ b/agent/ec_agent.py @@ -30,7 +30,7 @@ import urllib.parse import urllib.request import uuid -VERSION = "1.2.2" +VERSION = "1.3.0" LABEL = "io.electric-cherry" MAX_FILE = 32 * 1024 * 1024 LIBRARY_DOWNLOAD_WORKERS = 4 @@ -672,6 +672,7 @@ class Agent: self.docker = docker or Docker() self.stopping = threading.Event() self.current_operation = None + self.cancelled_creations = frozenset() self.library_progress = {} self.library_transfers = {} self.config_path = config.get("configPath", "/etc/electric-cherry/agent.json") @@ -952,7 +953,7 @@ class Agent: if line.startswith("PRETTY_NAME="): os_name = line.split("=", 1)[1].strip('"') update = self.update_inventory() - capabilities = (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else []) + capabilities = ["instance.cancel-create"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else []) return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})} def safe_to_disrupt(self, instance_id, generation): @@ -1040,6 +1041,7 @@ class Agent: return game, library, image, port, bind, env def create(self, instance_id, generation, payload): + self.check_creation_cancellation() directory = self.instance_dir(instance_id, generation) current_marker = directory.parent / "current.json" if current_marker.exists(): @@ -1068,12 +1070,19 @@ class Agent: for protocol in ("udp", "tcp"): args.extend(["--publish", f"{bind_arg}:{port}:{port}/{protocol}"]) args.append(image) + self.check_creation_cancellation() self.checkpoint({"phase": "creating", "generation": generation, "specSha256": spec_sha}) self.docker.call(*args, timeout=120) + self.check_creation_cancellation() self.docker.call("container", "start", name) + self.check_creation_cancellation() atomic_json(current_marker, {"generation": generation}) return self.inspect(instance_id, generation) + def check_creation_cancellation(self): + if self.executing_id and self.executing_id in self.cancelled_creations: + raise AgentError("Creation cancelled; waiting for instance cleanup") + def write_initial_files(self, root, files): if not isinstance(files, list) or len(files) > 10000: raise AgentError("Invalid file list") @@ -1093,6 +1102,8 @@ class Agent: return content def lifecycle(self, kind, instance_id, generation, payload): + if kind == "instance.cancel-create": + return self.cancel_creation(instance_id, generation, payload) if kind == "instance.create": return self.create(instance_id, generation, payload) if kind == "instance.inspect": @@ -1146,6 +1157,38 @@ class Agent: raise AgentError("Unsupported instance operation") return {**self.inspect(instance_id, generation), "before": before} + def cancel_creation(self, instance_id, generation, payload): + """Explicit launch cancellation, fenced to its immutable creation identity. + + The operation loop serializes this with create. Once delivered, removal + needs no RCON response and a durable tombstone prevents a delayed create. + Repeating this cleanup after an ambiguous Docker result is safe. + """ + create_id = identifier(payload.get("createOperationId")) + marker = self.instance_dir(instance_id, generation).parent / "current.json" + if marker.exists() and json.loads(marker.read_text())["generation"] > generation: + raise Rejected("Cancelled creation was superseded by a newer generation") + saved = self.journal.db.execute("SELECT request FROM operations WHERE id=?", (create_id,)).fetchone() + if saved: + original = json.loads(saved["request"]) + if original["kind"] != "instance.create" or original["instanceId"] != instance_id or original["generation"] != generation: + raise Rejected("Cancellation does not match the original creation") + health = self.inspect(instance_id, generation) + if health["exists"] and health.get("creationOperation") != create_id: + raise Rejected("Container belongs to a different creation operation") + self.checkpoint({"phase": "cancelling-creation", "createOperationId": create_id}) + # Fence before deleting: a crash must never permit the old creation to run again. + self.mark_removed(instance_id, generation) + if health["exists"]: + self.docker.call("container", "rm", "--force", "--volumes", self.container_name(instance_id, generation)) + after = self.inspect(instance_id, generation) + if after["exists"]: + raise AgentError("Cancelled creation container still exists") + directory = self.instance_dir(instance_id, generation) + if directory.exists(): + shutil.rmtree(directory) + return {**after, "cancelledCreation": create_id} + def stage(self, instance_id, generation, payload, operation_id): directory = self.instance_dir(instance_id, generation) self.spec(instance_id, generation) @@ -1609,7 +1652,7 @@ class Agent: current_marker = self.instance_dir(instance_id, generation).parent / "current.json" if current_marker.exists(): current = json.loads(current_marker.read_text()) - if current["generation"] > generation or current["generation"] == generation and current.get("removed") and kind not in ("instance.inspect", "instance.remove"): + if current["generation"] > generation or current["generation"] == generation and current.get("removed") and kind not in ("instance.inspect", "instance.remove", "instance.cancel-create"): raise AgentError("Instance generation has been superseded or removed") if kind.startswith("instance."): return self.lifecycle(kind, instance_id, generation, payload) @@ -1673,6 +1716,14 @@ class Agent: instance_id, generation = operation.get("instanceId"), operation.get("generation") checkpoint = json.loads(row["checkpoint"]) if row.get("checkpoint") else {} try: + if kind == "instance.cancel-create": + return self.journal.update(operation["id"], "completed", self.cancel_creation(instance_id, generation, payload)) + if kind == "instance.create": + marker = self.instance_dir(instance_id, generation).parent / "current.json" + if marker.exists(): + current = json.loads(marker.read_text()) + if current["generation"] > generation or current["generation"] == generation and current.get("removed"): + return self.journal.update(operation["id"], "failed", error="Creation was cancelled or superseded") if kind == "host.decommission": self.executing_id = operation["id"] try: @@ -1792,6 +1843,9 @@ class Agent: def heartbeat(self): response = self.request("POST", "/api/agent/heartbeat", {"inventory": self.inventory(), "operationId": self.current_operation}) + # Only the heartbeat thread replaces this immutable snapshot; it never + # mutates the operation journal or races Docker cleanup against create. + self.cancelled_creations = frozenset(value for value in response.get("cancelledCreations", []) if isinstance(value, str)) host = response.get("host", {}) if isinstance(host.get("portMin"), int) and isinstance(host.get("portMax"), int): self.config["portRange"] = {"min": host["portMin"], "max": host["portMax"]} diff --git a/agent/tests/test_agent.py b/agent/tests/test_agent.py index 92eb8c4..900f44c 100644 --- a/agent/tests/test_agent.py +++ b/agent/tests/test_agent.py @@ -209,6 +209,97 @@ class AgentTests(unittest.TestCase): self.assertTrue((self.library / "content/game.ff").exists()) self.assertFalse(self.agent.instance_dir(INSTANCE, 1).exists()) + def test_cancel_creation_bypasses_occupancy_and_preserves_library(self): + self.create() + self.agent.rcon = lambda *_: self.fail("Cancellation must not wait for player evidence") + operation = self.operation("instance.cancel-create", {"createOperationId": "create"}, "cancel") + self.assertEqual(self.agent.execute(operation)["state"], "completed") + self.assertFalse(self.agent.inspect(INSTANCE, 1)["exists"]) + self.assertFalse(self.agent.instance_dir(INSTANCE, 1).exists()) + self.assertTrue((self.library / "content/game.ff").exists()) + self.assertEqual(self.agent.execute(operation)["state"], "completed") + self.assertEqual(sum(call[:2] == ("container", "rm") for call in self.docker.calls), 1) + + def test_heartbeat_cancellation_stops_create_before_starting_container(self): + original_call = self.docker.call + def cancel_during_create(*args, **kwargs): + result = original_call(*args, **kwargs) + if args[:2] == ("container", "create"): + self.agent.cancelled_creations = frozenset(["create"]) + return result + self.docker.call = cancel_during_create + payload = {"name": "test", "game": "t5", "libraryId": self.library_id, "image": "ghcr.io/ayymoss/plutainer:latest", "port": 28960, "env": {}, "rconPassword": "secret", "files": []} + self.assertEqual(self.agent.execute(self.operation("instance.create", payload, "create"))["state"], "unknown") + self.assertFalse(any(call[:2] == ("container", "start") for call in self.docker.calls)) + result = self.agent.execute(self.operation("instance.cancel-create", {"createOperationId": "create"})) + self.assertEqual(result["state"], "completed") + self.assertFalse(self.agent.inspect(INSTANCE, 1)["exists"]) + + def test_cancellation_cleanup_resumes_after_agent_restart(self): + self.create() + self.docker.lose_reply_after = "rm" + cancel = self.operation("instance.cancel-create", {"createOperationId": "create"}, "cancel") + self.assertEqual(self.agent.execute(cancel)["state"], "unknown") + self.agent.journal.db.close() + self.agent = Agent(self.config, self.docker) + self.assertEqual(self.agent.execute(cancel)["state"], "completed") + self.assertFalse(self.agent.instance_dir(INSTANCE, 1).exists()) + + def test_cancel_before_creation_delivery_fences_delayed_create(self): + result = self.agent.execute(self.operation("instance.cancel-create", {"createOperationId": "late-create"})) + self.assertEqual(result["state"], "completed", result) + result = self.agent.execute(self.operation("instance.create", {}, "late-create")) + self.assertEqual(result["state"], "failed") + self.assertFalse(any(call[:2] == ("container", "create") for call in self.docker.calls)) + + def test_cancel_removes_partial_files_without_a_container(self): + directory = self.agent.instance_dir(INSTANCE, 1) + (directory / "app").mkdir(parents=True) + (directory / "app/partial.cfg").write_text("partial") + self.agent.journal.accept(self.operation("instance.create", {}, "create")) + result = self.agent.execute(self.operation("instance.cancel-create", {"createOperationId": "create"})) + self.assertEqual(result["state"], "completed", result) + self.assertFalse(directory.exists()) + self.assertTrue((self.library / "content/game.ff").exists()) + + def test_cancel_rejects_a_creation_id_from_another_instance(self): + other = self.operation("instance.create", {}, "other-create") + other["instanceId"] = "another-instance" + self.agent.journal.accept(other) + result = self.agent.execute(self.operation("instance.cancel-create", {"createOperationId": "other-create"})) + self.assertEqual(result["state"], "failed", result) + self.assertFalse((self.agent.instance_dir(INSTANCE, 1).parent / "current.json").exists()) + + def test_cancel_creation_rejects_other_creation_and_newer_generation(self): + self.create() + result = self.agent.execute(self.operation("instance.cancel-create", {"createOperationId": "wrong"})) + self.assertEqual(result["state"], "failed") + module.atomic_json(self.agent.instance_dir(INSTANCE, 1).parent / "current.json", {"generation": 2}) + result = self.agent.execute(self.operation("instance.cancel-create", {"createOperationId": "create"}, "old-generation")) + self.assertEqual(result["state"], "failed") + self.assertTrue(self.agent.inspect(INSTANCE, 1)["running"]) + self.assertFalse(any(call[:2] == ("container", "rm") for call in self.docker.calls)) + + def test_cancel_creation_reconciles_lost_removal_reply_and_fences_unknown_create(self): + self.docker.lose_reply_after = "start" + payload = {"name": "test", "game": "t5", "libraryId": self.library_id, "image": "ghcr.io/ayymoss/plutainer:latest", "port": 28960, "env": {}, "rconPassword": "secret", "files": []} + create = self.operation("instance.create", payload, "create") + self.assertEqual(self.agent.execute(create)["state"], "unknown") + self.docker.lose_reply_after = "rm" + cancel = self.operation("instance.cancel-create", {"createOperationId": "create"}, "cancel") + self.assertEqual(self.agent.execute(cancel)["state"], "unknown") + self.assertEqual(self.agent.execute(cancel)["state"], "completed") + self.assertEqual(self.agent.execute(create)["state"], "failed") + self.assertFalse(self.agent.instance_dir(INSTANCE, 1).exists()) + self.assertTrue(json.loads((self.agent.instance_dir(INSTANCE, 1).parent / "current.json").read_text())["removed"]) + + def test_cancel_creation_rejects_unowned_container(self): + self.create() + self.docker.containers[self.agent.container_name(INSTANCE, 1)]["Config"]["Labels"] = {} + result = self.agent.execute(self.operation("instance.cancel-create", {"createOperationId": "create"})) + self.assertEqual(result["state"], "failed") + self.assertFalse(any(call[:2] == ("container", "rm") for call in self.docker.calls)) + def test_unowned_container_never_mutated(self): self.create() self.docker.containers[self.agent.container_name(INSTANCE, 1)]["Config"]["Labels"] = {} diff --git a/docs/native-hosts.md b/docs/native-hosts.md index a09b196..ed990a6 100644 --- a/docs/native-hosts.md +++ b/docs/native-hosts.md @@ -400,3 +400,29 @@ remain enforced at command admission. Older completed operations whose progress snapshots were already discarded cannot recover missing check results from this change. Their surviving events and host command records remain available without inventing verification times. + +### Cancel an unfinished creation + +On the native creation operation page, **Cancel creation and remove server** +stops the launch and queues removal of its exact instance and generation. It +also works when readiness checks are stuck or the creation failed. This explicit +cancellation skips player-count checks on both the controller and agent, even +if players have joined. Completed creations use the normal safe removal flow. + +The cancelled operation links to a durable cleanup operation with agent receipts +and progress. An unsubmitted launch releases its reservations immediately. Once +an agent command was queued, ports and the instance identity remain reserved +until the agent confirms container and instance-file removal. Shared game +libraries, runtime images, saved server keys, and operation history are retained. +Repeated cancellation requests return the same cleanup operation. A queued +normal removal waiting for occupancy is superseded by this cleanup. + +Remote cancellation requires agent 1.3.0 or later (`instance.cancel-create` +capability). An older agent is never sent the unsupported cleanup command; +the cleanup page explains that it is waiting for an update. Offline or disabled +hosts retain the cleanup request until they reconnect or are enabled. Heartbeats +notify an executing create to stop between Docker commands; an already executing +Docker command must return or time out before cleanup can run. Cancellation +does not require the original creation to acknowledge completion. The agent +records a removal tombstone before cleanup so a delayed creation command cannot +recreate the instance, and resumes interrupted cleanup under the same operation ID. diff --git a/src/lib/provisioning-detail.ts b/src/lib/provisioning-detail.ts index 4d392fd..61a63be 100644 --- a/src/lib/provisioning-detail.ts +++ b/src/lib/provisioning-detail.ts @@ -95,6 +95,8 @@ export interface ProvisioningCapacity { }[]; } export interface ProvisioningDetail { + canCancelCreation?: boolean; + cancellation?: { cleanupId: string; state: string; stage: string; error: string | null } | null; backend?: 'native' | 'komodo'; workerActivity?: WorkerActivity[]; agentOperation?: { id: string; hostId: string; state: string; updatedAt: string; createdAt: string; error: string | null } | null; diff --git a/src/lib/server/backends/schema.ts b/src/lib/server/backends/schema.ts index 5d35823..1926131 100644 --- a/src/lib/server/backends/schema.ts +++ b/src/lib/server/backends/schema.ts @@ -16,5 +16,9 @@ export function migrateNativeInstances(handle: Database.Database): void { stage_operation_id TEXT, operation_id TEXT, files_json TEXT, applied_at TEXT, verified_at TEXT ); + CREATE TABLE IF NOT EXISTS native_creation_cancellations ( + creation_id TEXT PRIMARY KEY REFERENCES server_deployments(id), + cleanup_id TEXT NOT NULL UNIQUE REFERENCES server_deployments(id) + ); `); } diff --git a/src/lib/server/hosts/index.ts b/src/lib/server/hosts/index.ts index 9a5ab33..fbb02c9 100644 --- a/src/lib/server/hosts/index.ts +++ b/src/lib/server/hosts/index.ts @@ -288,6 +288,11 @@ type OperationRow = { id: string; host_id: string; kind: string; instance_id: st function operationFromRow(row: OperationRow): HostOperation { return { id: row.id, hostId: row.host_id, kind: row.kind, instanceId: row.instance_id, generation: row.generation, payload: JSON.parse(row.payload), state: row.state, result: row.result === null ? null : JSON.parse(row.result), error: row.error, createdAt: row.created_at, updatedAt: row.updated_at }; } export function getHostOperation(id: string): HostOperation | null { const row = database().prepare('SELECT * FROM ec_host_operations WHERE id=?').get(id) as OperationRow | undefined; return row ? operationFromRow(row) : null; } export function listHostOperations(hostId: string, limit = 100): HostOperation[] { return (database().prepare('SELECT * FROM ec_host_operations WHERE host_id=? ORDER BY created_at DESC,rowid DESC LIMIT ?').all(hostId, Math.min(500, Math.max(1, limit))) as OperationRow[]).map(operationFromRow); } +/** Heartbeats can tell an executing create to stop at its next safe command boundary. */ +export function pendingCreationCancellations(hostId: string): string[] { + return (database().prepare(`SELECT json_extract(payload,'$.createOperationId') AS id FROM ec_host_operations + WHERE host_id=? AND kind='instance.cancel-create' AND state IN ('pending','accepted','running','unknown')`).all(hostId) as { id: string }[]).map(row => row.id); +} export function queueHostOperation(hostId: string, kind: string, payload: Record, options: { instanceId?: string; generation?: number; idempotencyKey?: string; id?: string } = {}): HostOperation { const host = requireHost(hostId); if (host.decommission) throw new Error('Host cleanup is in progress; new operations are blocked.'); @@ -304,7 +309,7 @@ export function queueHostOperation(hostId: string, kind: string, payload: Record } } if (['instance.create','create'].includes(kind)) requireReadyHost(hostId); - if (host.mode === 'disabled') throw new Error('Host is disabled. Enable it before requesting operations.'); + if (host.mode === 'disabled' && kind !== 'instance.cancel-create') throw new Error('Host is disabled. Enable it before requesting operations.'); const id = options.id ?? randomUUID(); const timestamp = now(); database().prepare('INSERT INTO ec_host_operations(id,host_id,kind,instance_id,generation,idempotency_key,payload,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?)').run(id, hostId, kind, options.instanceId ?? null, options.generation ?? null, options.idempotencyKey ?? null, JSON.stringify(payload), timestamp, timestamp); database().prepare('INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,?,?)').run(id, 'pending', timestamp); @@ -317,9 +322,13 @@ export function pendingHostOperations(hostId: string): HostOperation[] { // Accepted/running/unknown commands keep the same immutable ID. The agent // consults its durable journal before acknowledging or executing a replay. return database().transaction(() => { - const rows = database().prepare(`SELECT * FROM ec_host_operations WHERE host_id=? AND state IN ('pending','accepted','running','unknown') + const rows = database().prepare(`SELECT * FROM ec_host_operations o WHERE host_id=? AND state IN ('pending','accepted','running','unknown') AND (? OR state != 'pending' OR kind NOT IN ('instance.create','create')) - ORDER BY COALESCE(last_delivered_at,''),created_at,rowid LIMIT 25`).all(hostId, Number(host.readiness === 'ready')) as OperationRow[]; + AND (kind!='instance.cancel-create' OR ?) + AND NOT EXISTS (SELECT 1 FROM ec_host_operations cancellation WHERE cancellation.host_id=o.host_id + AND cancellation.kind='instance.cancel-create' AND json_extract(cancellation.payload,'$.createOperationId')=o.id) + ORDER BY CASE WHEN kind='instance.cancel-create' THEN 0 ELSE 1 END,COALESCE(last_delivered_at,''),created_at,rowid LIMIT 25`) + .all(hostId, Number(host.readiness === 'ready'), Number(host.inventory.capabilities?.includes('instance.cancel-create') ?? false)) as OperationRow[]; const deliveredAt = now(); for (const operation of rows) database().prepare('UPDATE ec_host_operations SET last_delivered_at=? WHERE id=?').run(deliveredAt, operation.id); return rows.map(operationFromRow); diff --git a/src/lib/server/provisioning/cancel-creation.ts b/src/lib/server/provisioning/cancel-creation.ts new file mode 100644 index 0000000..95c0af2 --- /dev/null +++ b/src/lib/server/provisioning/cancel-creation.ts @@ -0,0 +1,74 @@ +import { db } from '../db/index.js'; +import { nativeInstance } from '../backends/native.js'; +import { queueHostOperation, releaseHostPorts } from '../hosts/index.js'; +import { beginDeployment, cancelRemoval, deploymentStage, finishDeployment } from './history.js'; +import { recordProgress } from './progress.js'; + +export function canCancelCreation(id: string): boolean { + return !!db().prepare(`SELECT 1 FROM server_deployments d + JOIN native_provisioning n ON n.deployment_id=d.id + JOIN native_instances i ON i.id=n.instance_id AND i.generation=n.generation AND i.retired_at IS NULL + WHERE d.id=? AND d.action='create' AND d.server_deleted_at IS NULL + AND d.state IN ('queued','waiting','running','awaiting-health','interrupted','failed') + AND NOT EXISTS (SELECT 1 FROM server_deployments other WHERE other.server=d.server + AND other.server_deleted_at IS NULL AND other.rowid>d.rowid AND other.state!='cancelled' + AND NOT (other.action='destroy' AND other.state='waiting' + AND NOT EXISTS (SELECT 1 FROM native_provisioning p WHERE p.deployment_id=other.id AND p.operation_id IS NOT NULL)))`).get(id); +} + +/** Stops observation immediately; remote cleanup remains a durable, separately visible job. */ +export function cancelCreation(id: string, triggeredBy = 'ui'): string | null { + return db().transaction(() => { + const prior = db().prepare('SELECT cleanup_id FROM native_creation_cancellations WHERE creation_id=?').get(id) as { cleanup_id: string } | undefined; + if (prior) return prior.cleanup_id; + if (!canCancelCreation(id)) return null; + const job = db().prepare(`SELECT d.server,n.operation_id,n.stage_operation_id FROM server_deployments d + JOIN native_provisioning n ON n.deployment_id=d.id WHERE d.id=?`).get(id) as + { server: string; operation_id: string | null; stage_operation_id: string | null }; + const instance = nativeInstance(job.server)!; + // Replace a removal still waiting on occupancy, including one whose asynchronous probe is outstanding. + for (const removal of db().prepare("SELECT id FROM server_deployments WHERE server=? AND action='destroy' AND state='waiting' AND server_deleted_at IS NULL").all(job.server) as { id: string }[]) { + cancelRemoval(removal.id); + } + const cleanupId = beginDeployment(job.server, 'destroy', triggeredBy); + db().prepare('INSERT INTO native_creation_cancellations(creation_id,cleanup_id) VALUES(?,?)').run(id, cleanupId); + db().prepare("UPDATE server_deployments SET state='cancelled',finished_at=?,error=NULL WHERE id=?").run(new Date().toISOString(), id); + deploymentStage(id, 'Creation cancelled; cleanup requested', { blocker: null, + event: { kind: 'cancelled', message: 'Creation cancelled. Its partial instance will be removed without a player-count check.', details: { cleanupId } } }); + if (!job.operation_id && !job.stage_operation_id) { + // No remote mutation was ever queued. The reservation is the only resource to remove. + db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(new Date().toISOString(), instance.id); + releaseHostPorts(instance.id); + deploymentStage(cleanupId, 'Unsubmitted creation cancelled; reservations released', { phase: 'cleanup', + checkpoint: { key: 'cleanup', label: 'No host command submitted; local reservations released' } }); + finishDeployment(cleanupId, { ok: true }); + finishCreationCancellation(cleanupId); + } else { + const operation = queueHostOperation(instance.host_id, 'instance.cancel-create', { createOperationId: job.operation_id ?? id }, + { instanceId: instance.id, generation: instance.generation, idempotencyKey: cleanupId }); + db().prepare('UPDATE native_provisioning SET operation_id=? WHERE deployment_id=?').run(operation.id, cleanupId); + db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(new Date().toISOString(), cleanupId); + deploymentStage(cleanupId, 'Waiting for host agent to cancel creation and remove its partial instance', { phase: 'container-action', blocker: null, + command: { state: 'attempted', message: `Creation cancellation queued as agent operation ${operation.id}. Ports remain reserved until cleanup is confirmed.` } }); + } + return cleanupId; + })(); +} + +export function finishCreationCancellation(cleanupId: string): void { + const cancellation = db().prepare(`SELECT c.creation_id,n.operation_id FROM native_creation_cancellations c + JOIN native_provisioning n ON n.deployment_id=c.creation_id WHERE c.cleanup_id=?`).get(cleanupId) as + { creation_id: string; operation_id: string | null } | undefined; + if (!cancellation) return; + const now = new Date().toISOString(); + db().prepare('UPDATE server_deployments SET server_deleted_at=? WHERE id IN (?,?)').run(now, cancellation.creation_id, cleanupId); + deploymentStage(cancellation.creation_id, 'Creation cancelled; partial instance removed', { blocker: null, + checkpoint: { key: 'cleanup', phase: 'cleanup', label: 'Cancellation cleanup confirmed; instance and reservations removed' } }); + // The original command may have lost its receipt. Cleanup proves that its effects are now gone. + if (cancellation.operation_id) { + const result = db().prepare("UPDATE ec_host_operations SET state='failed',error='Creation cancelled; cleanup confirmed',updated_at=? WHERE id=? AND state NOT IN ('completed','failed')") + .run(now, cancellation.operation_id); + if (result.changes) db().prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,'failed',?,'Creation cancelled; cleanup confirmed')").run(cancellation.operation_id, now); + } + recordProgress(cleanupId, { event: { kind: 'cleanup', message: 'Cancelled creation cleanup completed.', details: { creationId: cancellation.creation_id } } }); +} diff --git a/src/lib/server/provisioning/history.ts b/src/lib/server/provisioning/history.ts index 045ea94..65a4f80 100644 --- a/src/lib/server/provisioning/history.ts +++ b/src/lib/server/provisioning/history.ts @@ -8,6 +8,7 @@ import { activitySummary, deploymentEvents, latestFailureEvent } from './activit import { readProgress, recordProgress, type ProgressUpdate } from './progress.js'; import { bindNativeOperation } from './native.js'; import { workerActivity } from '../pipeline/activity.js'; +import { canCancelCreation } from './cancel-creation.js'; export function beginDeployment(server: string, action: ProvisioningAction, triggeredBy = 'ui'): string { const id = randomUUID(); @@ -105,6 +106,8 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' } const native = db().prepare('SELECT instance_id FROM native_provisioning WHERE deployment_id=?').get(id); const agentOperation = native ? db().prepare(`SELECT o.id, o.host_id AS hostId, o.state, o.updated_at AS updatedAt, o.created_at AS createdAt, o.error FROM native_provisioning n JOIN ec_host_operations o ON o.id=n.operation_id WHERE n.deployment_id=?`).get(id) as ProvisioningDetail['agentOperation'] : null; + const cancellation = db().prepare(`SELECT c.cleanup_id AS cleanupId,d.state,d.stage,d.error FROM native_creation_cancellations c + JOIN server_deployments d ON d.id=c.cleanup_id WHERE c.creation_id=?`).get(id) as ProvisioningDetail['cancellation']; const { observation_json: _snapshot, phase_key: _phase, phase_started_at: _phaseStarted, observed_at: _observed, ...deployment } = row; const sources = (db().prepare('SELECT kind, source_id, server FROM provisioning_sources WHERE operation_id = ? ORDER BY kind, source_id') .all(id) as Omit[]).map(sourceDetails); @@ -114,6 +117,7 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' } UNION ALL SELECT collection_id, deployment_id FROM server_collection_removals) m JOIN server_launch_collections c ON c.id = m.collection_id WHERE m.deployment_id = ? ORDER BY c.created_at, c.id LIMIT 1`).get(id) as { id: string; name: string } | undefined; return { deployment, backend: native ? 'native' as const : 'komodo' as const, agentOperation: agentOperation ?? null, + canCancelCreation: canCancelCreation(id), cancellation: cancellation ?? null, workerActivity: workerActivity(), events: deploymentEvents(id, options.events === 'recent'), activity: { ...activitySummary(id), failureExcerpt: latestFailureEvent(id) }, sources, recipe: launch ? summarizeRecipe(launch.recipe) : null, blueprintName: launch?.blueprintName ?? null, diff --git a/src/lib/server/provisioning/native.ts b/src/lib/server/provisioning/native.ts index 793dde2..8d7c038 100644 --- a/src/lib/server/provisioning/native.ts +++ b/src/lib/server/provisioning/native.ts @@ -21,6 +21,7 @@ import { sha256Buffer } from '../fleet/files.js'; import type { StagedFileChange } from '../fleet/staged-files.js'; import type { LaunchPreview } from '../../deployment-plan.js'; import type { ProvisioningAction } from '../../provisioning.js'; +import { cancelCreation, finishCreationCancellation } from './cancel-creation.js'; export interface NativeSelection { hostId: string; libraryId: string; image?: string; bindAddress?: string; publicAddress?: string; publicPort?: number } type NativeRecipe = LaunchRecipe & { native?: NativeSelection }; @@ -146,15 +147,7 @@ export function cancelQueuedNativeCreation(server: string, triggeredBy: string): const creation = db().prepare(`SELECT d.id FROM server_deployments d JOIN native_provisioning n ON n.deployment_id=d.id WHERE n.instance_id=? AND d.action='create' AND d.state='queued' AND n.operation_id IS NULL`).get(instance.id) as { id: string } | undefined; if (!creation) return null; - const id = beginDeployment(server, 'destroy', triggeredBy); - const now = new Date().toISOString(); - db().prepare("UPDATE server_deployments SET state='cancelled',finished_at=?,stage='Queued native creation cancelled',server_deleted_at=? WHERE id=?") - .run(now, now, creation.id); - db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(now, instance.id); - releaseHostPorts(instance.id); - deploymentStage(id, 'Cancelled queued creation; no agent mutation was submitted'); - finishDeployment(id, { ok: true }); - return id; + return cancelCreation(creation.id, triggeredBy); })(); } @@ -277,6 +270,7 @@ export async function processNativeProvisioning(eligibility: (operation: { id: s const activeJobs = new Map>(); async function processJob(job: Job, eligibility: (job: Job) => string | null) { + if (!stillActive(job)) return; let instance = nativeInstance(job.server); if (!instance || instance.id !== job.instance_id || (instance.generation !== job.generation && !(job.action === 'rebuild' && instance.generation === job.generation + 1))) { finishDeployment(job.id, { ok: false, message: 'Original instance was retired or replaced; no command was sent.' }); return; @@ -288,6 +282,9 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) { } if (job.operation_id) { const operation = getHostOperation(job.operation_id); + if (operation?.kind === 'instance.cancel-create' && operation.state === 'pending' && !host.inventory.capabilities?.includes('instance.cancel-create')) { + hold(job, 'Update the host agent to support creation cancellation. Creation is cancelled; cleanup and port release await the updated agent.'); return; + } if (!operation || operation.state === 'unknown') { hold(job, 'Agent result is unknown. Reconciling the original operation; no replacement or repeated command will be sent.'); return; } if (operation.state === 'failed') { // The agent reports failed only when no mutation began. Ambiguous @@ -312,6 +309,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) { db().prepare('DELETE FROM servers WHERE name=?').run(job.server); recordProgress(job.id, { checkpoint: { key: 'cleanup', label: 'Instance retired and port reservations released' } }); complete(job); + finishCreationCancellation(job.id); })(); return; } diff --git a/src/lib/ui/provisioning-presentation.ts b/src/lib/ui/provisioning-presentation.ts index 838a677..0e4b939 100644 --- a/src/lib/ui/provisioning-presentation.ts +++ b/src/lib/ui/provisioning-presentation.ts @@ -86,6 +86,9 @@ export function provisioningOutcome(detail: ProvisioningDetail) { if (detail.canResumeCleanup) description = 'The remote deployment was deleted. Resume the remaining local cleanup without repeating container removal.'; else if (detail.canResumeRebuild) description = 'The remote deployment was deleted. Resume local cleanup, then create a fresh instance in the reserved server slot.'; else if (detail.canResumeVerification) description = 'Resume observation of the existing instance. This does not create, restart, or rebuild a container.'; + else if (detail.cancellation) description = detail.cancellation.state === 'done' + ? 'Creation was cancelled. Its instance files, container, and port reservations have been removed.' + : 'Creation was cancelled. Cleanup is tracked separately below; ports remain reserved until the host confirms removal.'; else if (operation.state === 'cancelled') description = 'This request is cancelled. Recorded prior work is preserved below; remaining work will not run.'; else if (failure) description = progress?.command?.state === 'uncertain' ? 'The remote response was uncertain. Inspect the recorded result and server before requesting another operation.' @@ -160,6 +163,10 @@ export function isProvisioningEventPage(value: unknown): value is ProvisioningEv /** Validate every nested shape consumed by the page before replacing its last good snapshot. */ export function isProvisioningDetail(value: unknown, id: string): value is ProvisioningDetail { if (!record(value) || !record(value.deployment)) return false; + if (value.canCancelCreation !== undefined && typeof value.canCancelCreation !== 'boolean') return false; + if (value.cancellation !== undefined && value.cancellation !== null && !(record(value.cancellation) + && text(value.cancellation.cleanupId) && text(value.cancellation.state) && text(value.cancellation.stage) + && nullableText(value.cancellation.error))) return false; if (value.backend !== undefined && !['native', 'komodo'].includes(value.backend as string)) return false; if (value.workerActivity !== undefined && (!Array.isArray(value.workerActivity) || !value.workerActivity.every(item => record(item) && text(item.key) && text(item.label) && timestamp(item.startedAt)))) return false; diff --git a/src/routes/api/agent/heartbeat/+server.ts b/src/routes/api/agent/heartbeat/+server.ts index ffb312a..d2a85c0 100644 --- a/src/routes/api/agent/heartbeat/+server.ts +++ b/src/routes/api/agent/heartbeat/+server.ts @@ -1,5 +1,5 @@ import { json } from '@sveltejs/kit'; -import { heartbeat } from '$lib/server/hosts/index.js'; +import { heartbeat, pendingCreationCancellations } from '$lib/server/hosts/index.js'; import { authenticateAgent, hostApiError, objectBody } from '$lib/server/hosts/http.js'; import { reconcileHostPreparation } from '$lib/server/hosts/preparation.js'; export async function POST({ request }: { request: Request }) { @@ -13,6 +13,6 @@ export async function POST({ request }: { request: Request }) { updateHostLibraryCaches(host.id, inventory.libraryCaches, { snapshot: true }); } reconcileHostPreparation(host.id); - return json({ host, heartbeatSeconds: 15, serverTime: new Date().toISOString() }); + return json({ host, cancelledCreations: pendingCreationCancellations(host.id), heartbeatSeconds: 15, serverTime: new Date().toISOString() }); } catch (cause) { return hostApiError(cause); } } diff --git a/src/routes/api/deployments/[id]/cancel/+server.ts b/src/routes/api/deployments/[id]/cancel/+server.ts index 8acc1dd..a76d8e7 100644 --- a/src/routes/api/deployments/[id]/cancel/+server.ts +++ b/src/routes/api/deployments/[id]/cancel/+server.ts @@ -1,7 +1,14 @@ import { json } from '@sveltejs/kit'; -import { cancelRemoval } from '$lib/server/provisioning/index.js'; +import { cancelRemoval } from '$lib/server/provisioning/history.js'; +import { cancelCreation } from '$lib/server/provisioning/cancel-creation.js'; +import { kickProvisioning } from '$lib/server/pipeline/worker.js'; export function POST({ params }: { params: { id: string } }) { - if (!cancelRemoval(params.id)) return json({ error: 'Removal has already begun or this request is no longer waiting. Refresh to see its current state.' }, { status: 409 }); + const cleanupId = cancelCreation(params.id); + if (cleanupId) { + void kickProvisioning(); + return json({ ok: true, cleanupId }); + } + if (!cancelRemoval(params.id)) return json({ error: 'This operation can no longer be cancelled. Refresh to see its current state.' }, { status: 409 }); return json({ ok: true }); } diff --git a/src/routes/provisioning/[id]/+page.svelte b/src/routes/provisioning/[id]/+page.svelte index dcc43b9..bc8853c 100644 --- a/src/routes/provisioning/[id]/+page.svelte +++ b/src/routes/provisioning/[id]/+page.svelte @@ -87,14 +87,16 @@ {#if detail.activity.failureExcerpt && detail.activity.failureExcerpt.message !== deployment.error && ['failed', 'interrupted'].includes(deployment.state)}

Last failure event · {fmt(detail.activity.failureExcerpt.created_at)}{detail.activity.failureExcerpt.message}

{/if} {#if detail.warningMessages.length}

Recorded warnings

    {#each detail.warningMessages as warning}
  • {warning}
    • {/each}
{/if} - {#if detail.canResumeVerification || detail.canResumeCleanup || detail.canResumeRebuild || detail.canCancelRemoval} + {#if detail.canResumeVerification || detail.canResumeCleanup || detail.canResumeRebuild || detail.canCancelRemoval || detail.canCancelCreation}
{#if detail.canResumeVerification}{/if} {#if detail.canResumeRebuild}{/if} {#if detail.canResumeCleanup}{/if} {#if detail.canCancelRemoval}

Available until removal begins. Cancelling keeps the server and its files.

{/if} + {#if detail.canCancelCreation}

Stops this launch and deletes its container and instance files, even if player status is unknown or players have joined. Shared game files and the saved server key are kept.

{/if}
{/if} + {#if detail.cancellation}
{detail.cancellation.state === 'done' ? 'Cancellation cleanup completed' : 'Cancellation cleanup'}

{detail.cancellation.error ?? detail.cancellation.stage}

View cleanup progress →
{/if} {#if view.actionError}{/if} {#if view.notice}

{view.notice}

{/if} {#if detail.progress?.blocker?.href}{detail.progress.blocker.label ?? 'View blocking work'} →{/if} diff --git a/static/agent/ec-agent.py b/static/agent/ec-agent.py index 22f605e..e8b9018 100644 --- a/static/agent/ec-agent.py +++ b/static/agent/ec-agent.py @@ -30,7 +30,7 @@ import urllib.parse import urllib.request import uuid -VERSION = "1.2.2" +VERSION = "1.3.0" LABEL = "io.electric-cherry" MAX_FILE = 32 * 1024 * 1024 LIBRARY_DOWNLOAD_WORKERS = 4 @@ -672,6 +672,7 @@ class Agent: self.docker = docker or Docker() self.stopping = threading.Event() self.current_operation = None + self.cancelled_creations = frozenset() self.library_progress = {} self.library_transfers = {} self.config_path = config.get("configPath", "/etc/electric-cherry/agent.json") @@ -952,7 +953,7 @@ class Agent: if line.startswith("PRETTY_NAME="): os_name = line.split("=", 1)[1].strip('"') update = self.update_inventory() - capabilities = (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else []) + capabilities = ["instance.cancel-create"] + (["host.decommission"] if self.can_uninstall() else []) + (["agent.update"] if update["enabled"] else []) return {"hostname": socket.gethostname(), "agentVersion": VERSION, "protocolVersion": 1, "capabilities": capabilities, "agentUpdate": update, "os": os_name, "osVersion": platform.release(), "architecture": platform.machine(), "cpu": platform.processor() or platform.machine(), "cpuCount": os.cpu_count() or 0, "memoryTotal": memory.get("MemTotal", 0), "memoryAvailable": memory.get("MemAvailable", 0), "storageTotal": usage.total, "storageAvailable": usage.free, "dockerVersion": docker_version, "dockerRunning": docker_running, "directoriesWritable": os.access(self.root, os.W_OK), "runtimeImages": sorted(set(images)), "libraries": libraries, "libraryCaches": caches, "currentOperation": self.current_operation, "portRange": self.config.get("portRange", {"min": 28960, "max": 29060})} def safe_to_disrupt(self, instance_id, generation): @@ -1040,6 +1041,7 @@ class Agent: return game, library, image, port, bind, env def create(self, instance_id, generation, payload): + self.check_creation_cancellation() directory = self.instance_dir(instance_id, generation) current_marker = directory.parent / "current.json" if current_marker.exists(): @@ -1068,12 +1070,19 @@ class Agent: for protocol in ("udp", "tcp"): args.extend(["--publish", f"{bind_arg}:{port}:{port}/{protocol}"]) args.append(image) + self.check_creation_cancellation() self.checkpoint({"phase": "creating", "generation": generation, "specSha256": spec_sha}) self.docker.call(*args, timeout=120) + self.check_creation_cancellation() self.docker.call("container", "start", name) + self.check_creation_cancellation() atomic_json(current_marker, {"generation": generation}) return self.inspect(instance_id, generation) + def check_creation_cancellation(self): + if self.executing_id and self.executing_id in self.cancelled_creations: + raise AgentError("Creation cancelled; waiting for instance cleanup") + def write_initial_files(self, root, files): if not isinstance(files, list) or len(files) > 10000: raise AgentError("Invalid file list") @@ -1093,6 +1102,8 @@ class Agent: return content def lifecycle(self, kind, instance_id, generation, payload): + if kind == "instance.cancel-create": + return self.cancel_creation(instance_id, generation, payload) if kind == "instance.create": return self.create(instance_id, generation, payload) if kind == "instance.inspect": @@ -1146,6 +1157,38 @@ class Agent: raise AgentError("Unsupported instance operation") return {**self.inspect(instance_id, generation), "before": before} + def cancel_creation(self, instance_id, generation, payload): + """Explicit launch cancellation, fenced to its immutable creation identity. + + The operation loop serializes this with create. Once delivered, removal + needs no RCON response and a durable tombstone prevents a delayed create. + Repeating this cleanup after an ambiguous Docker result is safe. + """ + create_id = identifier(payload.get("createOperationId")) + marker = self.instance_dir(instance_id, generation).parent / "current.json" + if marker.exists() and json.loads(marker.read_text())["generation"] > generation: + raise Rejected("Cancelled creation was superseded by a newer generation") + saved = self.journal.db.execute("SELECT request FROM operations WHERE id=?", (create_id,)).fetchone() + if saved: + original = json.loads(saved["request"]) + if original["kind"] != "instance.create" or original["instanceId"] != instance_id or original["generation"] != generation: + raise Rejected("Cancellation does not match the original creation") + health = self.inspect(instance_id, generation) + if health["exists"] and health.get("creationOperation") != create_id: + raise Rejected("Container belongs to a different creation operation") + self.checkpoint({"phase": "cancelling-creation", "createOperationId": create_id}) + # Fence before deleting: a crash must never permit the old creation to run again. + self.mark_removed(instance_id, generation) + if health["exists"]: + self.docker.call("container", "rm", "--force", "--volumes", self.container_name(instance_id, generation)) + after = self.inspect(instance_id, generation) + if after["exists"]: + raise AgentError("Cancelled creation container still exists") + directory = self.instance_dir(instance_id, generation) + if directory.exists(): + shutil.rmtree(directory) + return {**after, "cancelledCreation": create_id} + def stage(self, instance_id, generation, payload, operation_id): directory = self.instance_dir(instance_id, generation) self.spec(instance_id, generation) @@ -1609,7 +1652,7 @@ class Agent: current_marker = self.instance_dir(instance_id, generation).parent / "current.json" if current_marker.exists(): current = json.loads(current_marker.read_text()) - if current["generation"] > generation or current["generation"] == generation and current.get("removed") and kind not in ("instance.inspect", "instance.remove"): + if current["generation"] > generation or current["generation"] == generation and current.get("removed") and kind not in ("instance.inspect", "instance.remove", "instance.cancel-create"): raise AgentError("Instance generation has been superseded or removed") if kind.startswith("instance."): return self.lifecycle(kind, instance_id, generation, payload) @@ -1673,6 +1716,14 @@ class Agent: instance_id, generation = operation.get("instanceId"), operation.get("generation") checkpoint = json.loads(row["checkpoint"]) if row.get("checkpoint") else {} try: + if kind == "instance.cancel-create": + return self.journal.update(operation["id"], "completed", self.cancel_creation(instance_id, generation, payload)) + if kind == "instance.create": + marker = self.instance_dir(instance_id, generation).parent / "current.json" + if marker.exists(): + current = json.loads(marker.read_text()) + if current["generation"] > generation or current["generation"] == generation and current.get("removed"): + return self.journal.update(operation["id"], "failed", error="Creation was cancelled or superseded") if kind == "host.decommission": self.executing_id = operation["id"] try: @@ -1792,6 +1843,9 @@ class Agent: def heartbeat(self): response = self.request("POST", "/api/agent/heartbeat", {"inventory": self.inventory(), "operationId": self.current_operation}) + # Only the heartbeat thread replaces this immutable snapshot; it never + # mutates the operation journal or races Docker cleanup against create. + self.cancelled_creations = frozenset(value for value in response.get("cancelledCreations", []) if isinstance(value, str)) host = response.get("host", {}) if isinstance(host.get("portMin"), int) and isinstance(host.get("portMax"), int): self.config["portRange"] = {"min": host["portMin"], "max": host["portMax"]} diff --git a/static/agent/manifest.json b/static/agent/manifest.json index 60a70cb..54d8575 100644 --- a/static/agent/manifest.json +++ b/static/agent/manifest.json @@ -1,8 +1,8 @@ { - "version": "1.2.2", + "version": "1.3.0", "protocolVersion": 1, "platform": "debian-amd64", - "sha256": "c266a8099130051ee540cdc3b9cdc7e42144fa67e21cce76eea6cf0a426bf40d", + "sha256": "a698bdbba3716726a5a63fbd18392b53bd4c1e0c4cbaca94eabd66d8ca328dc2", "updaterProtocolVersion": 1, "updaterSha256": "066bb896726600ece9ce72697ce523c9a707cc9a621505931ba0892dec59d3c8" } diff --git a/tests/unit/native-provisioning.test.ts b/tests/unit/native-provisioning.test.ts index a5af79f..6d55456 100644 --- a/tests/unit/native-provisioning.test.ts +++ b/tests/unit/native-provisioning.test.ts @@ -9,7 +9,7 @@ vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconGetPlayerCount: vi.fn(async () => remote.players), rconPlutoniumVersion: vi.fn(async () => ({ revision: 5354, error: null })) })); vi.mock('../../src/lib/server/integrations/plutonium.js', () => ({ checkRelease: vi.fn(async () => {}), freshRelease: () => 5354, probeVersion: vi.fn(async () => 5354) })); -vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) })); +vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) })); let fixture: Fixture; let hostId: string; let libraryId: string; const image = 'example.test/plutainer:test'; @@ -73,6 +73,119 @@ async function detail(id: string) { } describe('native instance provisioning', () => { + it('accepts cancellation through the provisioning API and rejects an already completed creation', async () => { + const job = await queue(); + const { POST } = await import('../../src/routes/api/provisioning/[id]/cancel/+server.js'); + const response = POST({ params: { id: job.deploymentId } }); + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ ok: true, cleanupId: expect.any(String) }); + const next = await queue(); await advance(); await acknowledge(next.deploymentId); + expect(POST({ params: { id: next.deploymentId } }).status).toBe(409); + expect(POST({ params: { id: job.deploymentId } }).status).toBe(200); + }); + it('cancels an unsubmitted creation locally and returns the same cleanup on repeated requests', async () => { + const job = await queue(); + const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js'); + const { getDeployment } = await import('../../src/lib/server/provisioning/history.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + expect(getDeployment(job.deploymentId)!.canCancelCreation).toBe(true); + const cleanup = cancelCreation(job.deploymentId)!; + expect(cancelCreation(job.deploymentId)).toBe(cleanup); + expect((await detail(job.deploymentId)).state).toBe('cancelled'); + expect((await detail(cleanup)).state).toBe('done'); + expect(getDeployment(job.deploymentId)!.cancellation?.state).toBe('done'); + expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]); + await advance(); + expect(await operationFor(job.deploymentId)).toBeNull(); + expect(await operationFor(cleanup)).toBeNull(); + }); + + it('cancels a submitted creation without occupancy checks, suppresses replay, and releases resources only after cleanup', async () => { + const job = await queue(); await advance(); remote.players = 4; + const original = (await operationFor(job.deploymentId))!; + const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js'); + const { heartbeat, pendingHostOperations, reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + const { nativeRequest } = await import('../../src/lib/server/backends/native.js'); + heartbeat(hostId, { ...inventory(), capabilities: ['instance.cancel-create'] }); + // A previously delivered command might still be executing when cancellation arrives. + expect(pendingHostOperations(hostId).map(op => op.id)).toContain(original.id); + const cleanup = cancelCreation(job.deploymentId)!; + const removal = (await operationFor(cleanup))!; + expect(removal.kind).toBe('instance.cancel-create'); + expect(removal.payload).toEqual({ createOperationId: original.id }); + expect(cancelCreation(job.deploymentId)).toBe(cleanup); + expect(pendingHostOperations(hostId).map(op => op.id)).toContain(removal.id); + expect(pendingHostOperations(hostId).map(op => op.id)).not.toContain(original.id); + await advance(); + expect(nativeRequest).not.toHaveBeenCalled(); + expect(db().prepare('SELECT * FROM ec_host_ports').all()).toHaveLength(2); + reportHostOperation(hostId, original.id, { state: 'completed', result: { healthy: true } }); + await advance(); + expect((await detail(job.deploymentId)).state).toBe('cancelled'); + expect(nativeRequest).not.toHaveBeenCalled(); + reportHostOperation(hostId, removal.id, { state: 'completed', result: { exists: false } }); + await advance(); + expect((await detail(cleanup)).state).toBe('done'); + expect((await detail(job.deploymentId)).server_deleted_at).toBeTruthy(); + expect(db().prepare('SELECT * FROM ec_host_ports').all()).toEqual([]); + expect(db().prepare('SELECT * FROM servers WHERE name=?').get(job.name)).toBeUndefined(); + expect(db().prepare('SELECT retired_at FROM native_instances WHERE id=?').get(original.instanceId)).toMatchObject({ retired_at: expect.any(String) }); + }); + + it('holds cancellation for an old or offline host without delivering an unsupported command or losing reservations', async () => { + const job = await queue(); await advance(); + const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js'); + const { heartbeat, pendingHostOperations } = await import('../../src/lib/server/hosts/index.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + const cleanup = cancelCreation(job.deploymentId)!; + expect(pendingHostOperations(hostId).filter(op => op.kind.startsWith('instance.'))).toEqual([]); + await advance(); + expect((await detail(cleanup)).stage).toContain('Update the host agent'); + db().prepare('UPDATE ec_hosts SET last_heartbeat=? WHERE id=?').run('2000-01-01T00:00:00.000Z', hostId); + await advance(); + expect((await detail(cleanup)).stage).toContain('unavailable'); + expect(db().prepare('SELECT * FROM ec_host_ports').all()).toHaveLength(2); + heartbeat(hostId, { ...inventory(), capabilities: ['instance.cancel-create'] }); + expect(pendingHostOperations(hostId).filter(op => op.kind === 'instance.cancel-create')).toHaveLength(1); + await acknowledge(cleanup); + expect((await detail(cleanup)).state).toBe('done'); + }); + + it('replaces an occupancy-blocked removal and ignores an outstanding creation health reply', async () => { + const job = await queue(); await advance(); + const operation = (await operationFor(job.deploymentId))!; + const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js'); + reportHostOperation(hostId, operation.id, { state: 'completed', result: { healthy: true } }); + const { nativeRequest } = await import('../../src/lib/server/backends/native.js'); + let release!: (value: never) => void; + vi.mocked(nativeRequest).mockImplementationOnce(() => new Promise(resolve => { release = resolve; })); + const pending = advance(); + try { + const { destroyServer } = await import('../../src/lib/server/provisioning/instances.js'); + const waitingRemoval = destroyServer(job.name).deploymentId!; + const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js'); + const cleanup = cancelCreation(job.deploymentId)!; + expect(cleanup).toBeTruthy(); + expect((await detail(waitingRemoval)).state).toBe('cancelled'); + release({ healthy: true, state: 'running' } as never); + await pending; + expect((await detail(job.deploymentId)).state).toBe('cancelled'); + expect((await detail(cleanup)).state).toBe('running'); + } finally { release({ healthy: true, state: 'running' } as never); await pending; } + }); + + it('does not offer creation cancellation after success or against a replacement instance', async () => { + const job = await queue(); await advance(); await acknowledge(job.deploymentId); + const { canCancelCreation, cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js'); + expect(canCancelCreation(job.deploymentId)).toBe(false); + expect(cancelCreation(job.deploymentId)).toBeNull(); + const { db } = await import('../../src/lib/server/db/index.js'); + db().prepare("UPDATE server_deployments SET state='awaiting-health' WHERE id=?").run(job.deploymentId); + db().prepare('UPDATE native_instances SET generation=generation+1 WHERE server=?').run(job.name); + expect(cancelCreation(job.deploymentId)).toBeNull(); + }); + it('retains every native launch checkpoint and readiness check through completion and UI validation', async () => { const job = await queue(); const { getDeployment } = await import('../../src/lib/server/provisioning/history.js'); diff --git a/tests/unit/provisioning-presentation.test.ts b/tests/unit/provisioning-presentation.test.ts index d7904fd..5aace95 100644 --- a/tests/unit/provisioning-presentation.test.ts +++ b/tests/unit/provisioning-presentation.test.ts @@ -19,6 +19,17 @@ function fixture(): ProvisioningDetail { } describe('provisioning outcome and evidence presentation', () => { + it('distinguishes cancellation from confirmed cleanup and validates its link data', () => { + const detail = fixture(); + detail.deployment.state = 'cancelled'; + detail.canCancelCreation = false; + detail.cancellation = { cleanupId: 'cleanup-a', state: 'running', stage: 'Waiting for host', error: null }; + expect(isProvisioningDetail(detail, detail.deployment.id)).toBe(true); + expect(provisioningOutcome(detail).description).toContain('ports remain reserved'); + detail.cancellation.state = 'done'; + expect(provisioningOutcome(detail).description).toContain('have been removed'); + expect(isProvisioningDetail({ ...detail, cancellation: { state: 'done' } }, detail.deployment.id)).toBe(false); + }); it.each(provisioningActions)('describes cancelled %s as terminal, with no automatic execution claim', action => { const detail = fixture(); detail.deployment.action = action;