Add cancellation and cleanup for native server creation

src/lib/server/backends/schema.ts

@@ -16,5 +16,9 @@ export function migrateNativeInstances(handle: Database.Database): void {
             stage_operation_id TEXT, operation_id TEXT, files_json TEXT,
             applied_at TEXT, verified_at TEXT
         );
+        CREATE TABLE IF NOT EXISTS native_creation_cancellations (
+            creation_id TEXT PRIMARY KEY REFERENCES server_deployments(id),
+            cleanup_id TEXT NOT NULL UNIQUE REFERENCES server_deployments(id)
+        );
     `);
 }

src/lib/server/hosts/index.ts

@@ -288,6 +288,11 @@ type OperationRow = { id: string; host_id: string; kind: string; instance_id: st
 function operationFromRow(row: OperationRow): HostOperation { return { id: row.id, hostId: row.host_id, kind: row.kind, instanceId: row.instance_id, generation: row.generation, payload: JSON.parse(row.payload), state: row.state, result: row.result === null ? null : JSON.parse(row.result), error: row.error, createdAt: row.created_at, updatedAt: row.updated_at }; }
 export function getHostOperation(id: string): HostOperation | null { const row = database().prepare('SELECT * FROM ec_host_operations WHERE id=?').get(id) as OperationRow | undefined; return row ? operationFromRow(row) : null; }
 export function listHostOperations(hostId: string, limit = 100): HostOperation[] { return (database().prepare('SELECT * FROM ec_host_operations WHERE host_id=? ORDER BY created_at DESC,rowid DESC LIMIT ?').all(hostId, Math.min(500, Math.max(1, limit))) as OperationRow[]).map(operationFromRow); }
+/** Heartbeats can tell an executing create to stop at its next safe command boundary. */
+export function pendingCreationCancellations(hostId: string): string[] {
+    return (database().prepare(`SELECT json_extract(payload,'$.createOperationId') AS id FROM ec_host_operations
+        WHERE host_id=? AND kind='instance.cancel-create' AND state IN ('pending','accepted','running','unknown')`).all(hostId) as { id: string }[]).map(row => row.id);
+}
 export function queueHostOperation(hostId: string, kind: string, payload: Record<string, unknown>, options: { instanceId?: string; generation?: number; idempotencyKey?: string; id?: string } = {}): HostOperation {
     const host = requireHost(hostId);
     if (host.decommission) throw new Error('Host cleanup is in progress; new operations are blocked.');
@@ -304,7 +309,7 @@ export function queueHostOperation(hostId: string, kind: string, payload: Record
             }
         }
         if (['instance.create','create'].includes(kind)) requireReadyHost(hostId);
-        if (host.mode === 'disabled') throw new Error('Host is disabled. Enable it before requesting operations.');
+        if (host.mode === 'disabled' && kind !== 'instance.cancel-create') throw new Error('Host is disabled. Enable it before requesting operations.');
         const id = options.id ?? randomUUID(); const timestamp = now();
         database().prepare('INSERT INTO ec_host_operations(id,host_id,kind,instance_id,generation,idempotency_key,payload,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?)').run(id, hostId, kind, options.instanceId ?? null, options.generation ?? null, options.idempotencyKey ?? null, JSON.stringify(payload), timestamp, timestamp);
         database().prepare('INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,?,?)').run(id, 'pending', timestamp);
@@ -317,9 +322,13 @@ export function pendingHostOperations(hostId: string): HostOperation[] {
     // Accepted/running/unknown commands keep the same immutable ID. The agent
     // consults its durable journal before acknowledging or executing a replay.
     return database().transaction(() => {
-        const rows = database().prepare(`SELECT * FROM ec_host_operations WHERE host_id=? AND state IN ('pending','accepted','running','unknown')
+        const rows = database().prepare(`SELECT * FROM ec_host_operations o WHERE host_id=? AND state IN ('pending','accepted','running','unknown')
             AND (? OR state != 'pending' OR kind NOT IN ('instance.create','create'))
-            ORDER BY COALESCE(last_delivered_at,''),created_at,rowid LIMIT 25`).all(hostId, Number(host.readiness === 'ready')) as OperationRow[];
+            AND (kind!='instance.cancel-create' OR ?)
+            AND NOT EXISTS (SELECT 1 FROM ec_host_operations cancellation WHERE cancellation.host_id=o.host_id
+                AND cancellation.kind='instance.cancel-create' AND json_extract(cancellation.payload,'$.createOperationId')=o.id)
+            ORDER BY CASE WHEN kind='instance.cancel-create' THEN 0 ELSE 1 END,COALESCE(last_delivered_at,''),created_at,rowid LIMIT 25`)
+            .all(hostId, Number(host.readiness === 'ready'), Number(host.inventory.capabilities?.includes('instance.cancel-create') ?? false)) as OperationRow[];
         const deliveredAt = now();
         for (const operation of rows) database().prepare('UPDATE ec_host_operations SET last_delivered_at=? WHERE id=?').run(deliveredAt, operation.id);
         return rows.map(operationFromRow);

src/lib/server/provisioning/cancel-creation.ts

@@ -0,0 +1,74 @@
+import { db } from '../db/index.js';
+import { nativeInstance } from '../backends/native.js';
+import { queueHostOperation, releaseHostPorts } from '../hosts/index.js';
+import { beginDeployment, cancelRemoval, deploymentStage, finishDeployment } from './history.js';
+import { recordProgress } from './progress.js';
+
+export function canCancelCreation(id: string): boolean {
+    return !!db().prepare(`SELECT 1 FROM server_deployments d
+        JOIN native_provisioning n ON n.deployment_id=d.id
+        JOIN native_instances i ON i.id=n.instance_id AND i.generation=n.generation AND i.retired_at IS NULL
+        WHERE d.id=? AND d.action='create' AND d.server_deleted_at IS NULL
+        AND d.state IN ('queued','waiting','running','awaiting-health','interrupted','failed')
+        AND NOT EXISTS (SELECT 1 FROM server_deployments other WHERE other.server=d.server
+            AND other.server_deleted_at IS NULL AND other.rowid>d.rowid AND other.state!='cancelled'
+            AND NOT (other.action='destroy' AND other.state='waiting'
+                AND NOT EXISTS (SELECT 1 FROM native_provisioning p WHERE p.deployment_id=other.id AND p.operation_id IS NOT NULL)))`).get(id);
+}
+
+/** Stops observation immediately; remote cleanup remains a durable, separately visible job. */
+export function cancelCreation(id: string, triggeredBy = 'ui'): string | null {
+    return db().transaction(() => {
+        const prior = db().prepare('SELECT cleanup_id FROM native_creation_cancellations WHERE creation_id=?').get(id) as { cleanup_id: string } | undefined;
+        if (prior) return prior.cleanup_id;
+        if (!canCancelCreation(id)) return null;
+        const job = db().prepare(`SELECT d.server,n.operation_id,n.stage_operation_id FROM server_deployments d
+            JOIN native_provisioning n ON n.deployment_id=d.id WHERE d.id=?`).get(id) as
+            { server: string; operation_id: string | null; stage_operation_id: string | null };
+        const instance = nativeInstance(job.server)!;
+        // Replace a removal still waiting on occupancy, including one whose asynchronous probe is outstanding.
+        for (const removal of db().prepare("SELECT id FROM server_deployments WHERE server=? AND action='destroy' AND state='waiting' AND server_deleted_at IS NULL").all(job.server) as { id: string }[]) {
+            cancelRemoval(removal.id);
+        }
+        const cleanupId = beginDeployment(job.server, 'destroy', triggeredBy);
+        db().prepare('INSERT INTO native_creation_cancellations(creation_id,cleanup_id) VALUES(?,?)').run(id, cleanupId);
+        db().prepare("UPDATE server_deployments SET state='cancelled',finished_at=?,error=NULL WHERE id=?").run(new Date().toISOString(), id);
+        deploymentStage(id, 'Creation cancelled; cleanup requested', { blocker: null,
+            event: { kind: 'cancelled', message: 'Creation cancelled. Its partial instance will be removed without a player-count check.', details: { cleanupId } } });
+        if (!job.operation_id && !job.stage_operation_id) {
+            // No remote mutation was ever queued. The reservation is the only resource to remove.
+            db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(new Date().toISOString(), instance.id);
+            releaseHostPorts(instance.id);
+            deploymentStage(cleanupId, 'Unsubmitted creation cancelled; reservations released', { phase: 'cleanup',
+                checkpoint: { key: 'cleanup', label: 'No host command submitted; local reservations released' } });
+            finishDeployment(cleanupId, { ok: true });
+            finishCreationCancellation(cleanupId);
+        } else {
+            const operation = queueHostOperation(instance.host_id, 'instance.cancel-create', { createOperationId: job.operation_id ?? id },
+                { instanceId: instance.id, generation: instance.generation, idempotencyKey: cleanupId });
+            db().prepare('UPDATE native_provisioning SET operation_id=? WHERE deployment_id=?').run(operation.id, cleanupId);
+            db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(new Date().toISOString(), cleanupId);
+            deploymentStage(cleanupId, 'Waiting for host agent to cancel creation and remove its partial instance', { phase: 'container-action', blocker: null,
+                command: { state: 'attempted', message: `Creation cancellation queued as agent operation ${operation.id}. Ports remain reserved until cleanup is confirmed.` } });
+        }
+        return cleanupId;
+    })();
+}
+
+export function finishCreationCancellation(cleanupId: string): void {
+    const cancellation = db().prepare(`SELECT c.creation_id,n.operation_id FROM native_creation_cancellations c
+        JOIN native_provisioning n ON n.deployment_id=c.creation_id WHERE c.cleanup_id=?`).get(cleanupId) as
+        { creation_id: string; operation_id: string | null } | undefined;
+    if (!cancellation) return;
+    const now = new Date().toISOString();
+    db().prepare('UPDATE server_deployments SET server_deleted_at=? WHERE id IN (?,?)').run(now, cancellation.creation_id, cleanupId);
+    deploymentStage(cancellation.creation_id, 'Creation cancelled; partial instance removed', { blocker: null,
+        checkpoint: { key: 'cleanup', phase: 'cleanup', label: 'Cancellation cleanup confirmed; instance and reservations removed' } });
+    // The original command may have lost its receipt. Cleanup proves that its effects are now gone.
+    if (cancellation.operation_id) {
+        const result = db().prepare("UPDATE ec_host_operations SET state='failed',error='Creation cancelled; cleanup confirmed',updated_at=? WHERE id=? AND state NOT IN ('completed','failed')")
+            .run(now, cancellation.operation_id);
+        if (result.changes) db().prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,'failed',?,'Creation cancelled; cleanup confirmed')").run(cancellation.operation_id, now);
+    }
+    recordProgress(cleanupId, { event: { kind: 'cleanup', message: 'Cancelled creation cleanup completed.', details: { creationId: cancellation.creation_id } } });
+}

src/lib/server/provisioning/history.ts

@@ -8,6 +8,7 @@ import { activitySummary, deploymentEvents, latestFailureEvent } from './activit
 import { readProgress, recordProgress, type ProgressUpdate } from './progress.js';
 import { bindNativeOperation } from './native.js';
 import { workerActivity } from '../pipeline/activity.js';
+import { canCancelCreation } from './cancel-creation.js';
 
 export function beginDeployment(server: string, action: ProvisioningAction, triggeredBy = 'ui'): string {
     const id = randomUUID();
@@ -105,6 +106,8 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
         const native = db().prepare('SELECT instance_id FROM native_provisioning WHERE deployment_id=?').get(id);
         const agentOperation = native ? db().prepare(`SELECT o.id, o.host_id AS hostId, o.state, o.updated_at AS updatedAt, o.created_at AS createdAt, o.error
             FROM native_provisioning n JOIN ec_host_operations o ON o.id=n.operation_id WHERE n.deployment_id=?`).get(id) as ProvisioningDetail['agentOperation'] : null;
+        const cancellation = db().prepare(`SELECT c.cleanup_id AS cleanupId,d.state,d.stage,d.error FROM native_creation_cancellations c
+            JOIN server_deployments d ON d.id=c.cleanup_id WHERE c.creation_id=?`).get(id) as ProvisioningDetail['cancellation'];
         const { observation_json: _snapshot, phase_key: _phase, phase_started_at: _phaseStarted, observed_at: _observed, ...deployment } = row;
         const sources = (db().prepare('SELECT kind, source_id, server FROM provisioning_sources WHERE operation_id = ? ORDER BY kind, source_id')
             .all(id) as Omit<ProvisioningSource, 'label' | 'href'>[]).map(sourceDetails);
@@ -114,6 +117,7 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
             UNION ALL SELECT collection_id, deployment_id FROM server_collection_removals) m
             JOIN server_launch_collections c ON c.id = m.collection_id WHERE m.deployment_id = ? ORDER BY c.created_at, c.id LIMIT 1`).get(id) as { id: string; name: string } | undefined;
         return { deployment, backend: native ? 'native' as const : 'komodo' as const, agentOperation: agentOperation ?? null,
+            canCancelCreation: canCancelCreation(id), cancellation: cancellation ?? null,
             workerActivity: workerActivity(), events: deploymentEvents(id, options.events === 'recent'),
             activity: { ...activitySummary(id), failureExcerpt: latestFailureEvent(id) }, sources,
             recipe: launch ? summarizeRecipe(launch.recipe) : null, blueprintName: launch?.blueprintName ?? null,

src/lib/server/provisioning/native.ts

@@ -21,6 +21,7 @@ import { sha256Buffer } from '../fleet/files.js';
 import type { StagedFileChange } from '../fleet/staged-files.js';
 import type { LaunchPreview } from '../../deployment-plan.js';
 import type { ProvisioningAction } from '../../provisioning.js';
+import { cancelCreation, finishCreationCancellation } from './cancel-creation.js';
 
 export interface NativeSelection { hostId: string; libraryId: string; image?: string; bindAddress?: string; publicAddress?: string; publicPort?: number }
 type NativeRecipe = LaunchRecipe & { native?: NativeSelection };
@@ -146,15 +147,7 @@ export function cancelQueuedNativeCreation(server: string, triggeredBy: string):
         const creation = db().prepare(`SELECT d.id FROM server_deployments d JOIN native_provisioning n ON n.deployment_id=d.id
             WHERE n.instance_id=? AND d.action='create' AND d.state='queued' AND n.operation_id IS NULL`).get(instance.id) as { id: string } | undefined;
         if (!creation) return null;
-        const id = beginDeployment(server, 'destroy', triggeredBy);
-        const now = new Date().toISOString();
-        db().prepare("UPDATE server_deployments SET state='cancelled',finished_at=?,stage='Queued native creation cancelled',server_deleted_at=? WHERE id=?")
-            .run(now, now, creation.id);
-        db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(now, instance.id);
-        releaseHostPorts(instance.id);
-        deploymentStage(id, 'Cancelled queued creation; no agent mutation was submitted');
-        finishDeployment(id, { ok: true });
-        return id;
+        return cancelCreation(creation.id, triggeredBy);
     })();
 }
 
@@ -277,6 +270,7 @@ export async function processNativeProvisioning(eligibility: (operation: { id: s
 const activeJobs = new Map<string, Promise<void>>();
 
 async function processJob(job: Job, eligibility: (job: Job) => string | null) {
+    if (!stillActive(job)) return;
     let instance = nativeInstance(job.server);
     if (!instance || instance.id !== job.instance_id || (instance.generation !== job.generation && !(job.action === 'rebuild' && instance.generation === job.generation + 1))) {
         finishDeployment(job.id, { ok: false, message: 'Original instance was retired or replaced; no command was sent.' }); return;
@@ -288,6 +282,9 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
     }
     if (job.operation_id) {
         const operation = getHostOperation(job.operation_id);
+        if (operation?.kind === 'instance.cancel-create' && operation.state === 'pending' && !host.inventory.capabilities?.includes('instance.cancel-create')) {
+            hold(job, 'Update the host agent to support creation cancellation. Creation is cancelled; cleanup and port release await the updated agent.'); return;
+        }
         if (!operation || operation.state === 'unknown') { hold(job, 'Agent result is unknown. Reconciling the original operation; no replacement or repeated command will be sent.'); return; }
         if (operation.state === 'failed') {
             // The agent reports failed only when no mutation began. Ambiguous
@@ -312,6 +309,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
                 db().prepare('DELETE FROM servers WHERE name=?').run(job.server);
                 recordProgress(job.id, { checkpoint: { key: 'cleanup', label: 'Instance retired and port reservations released' } });
                 complete(job);
+                finishCreationCancellation(job.id);
             })();
             return;
         }