Add cancellation and cleanup for native server creation

src/lib/server/provisioning/cancel-creation.ts

@@ -0,0 +1,74 @@
+import { db } from '../db/index.js';
+import { nativeInstance } from '../backends/native.js';
+import { queueHostOperation, releaseHostPorts } from '../hosts/index.js';
+import { beginDeployment, cancelRemoval, deploymentStage, finishDeployment } from './history.js';
+import { recordProgress } from './progress.js';
+
+export function canCancelCreation(id: string): boolean {
+    return !!db().prepare(`SELECT 1 FROM server_deployments d
+        JOIN native_provisioning n ON n.deployment_id=d.id
+        JOIN native_instances i ON i.id=n.instance_id AND i.generation=n.generation AND i.retired_at IS NULL
+        WHERE d.id=? AND d.action='create' AND d.server_deleted_at IS NULL
+        AND d.state IN ('queued','waiting','running','awaiting-health','interrupted','failed')
+        AND NOT EXISTS (SELECT 1 FROM server_deployments other WHERE other.server=d.server
+            AND other.server_deleted_at IS NULL AND other.rowid>d.rowid AND other.state!='cancelled'
+            AND NOT (other.action='destroy' AND other.state='waiting'
+                AND NOT EXISTS (SELECT 1 FROM native_provisioning p WHERE p.deployment_id=other.id AND p.operation_id IS NOT NULL)))`).get(id);
+}
+
+/** Stops observation immediately; remote cleanup remains a durable, separately visible job. */
+export function cancelCreation(id: string, triggeredBy = 'ui'): string | null {
+    return db().transaction(() => {
+        const prior = db().prepare('SELECT cleanup_id FROM native_creation_cancellations WHERE creation_id=?').get(id) as { cleanup_id: string } | undefined;
+        if (prior) return prior.cleanup_id;
+        if (!canCancelCreation(id)) return null;
+        const job = db().prepare(`SELECT d.server,n.operation_id,n.stage_operation_id FROM server_deployments d
+            JOIN native_provisioning n ON n.deployment_id=d.id WHERE d.id=?`).get(id) as
+            { server: string; operation_id: string | null; stage_operation_id: string | null };
+        const instance = nativeInstance(job.server)!;
+        // Replace a removal still waiting on occupancy, including one whose asynchronous probe is outstanding.
+        for (const removal of db().prepare("SELECT id FROM server_deployments WHERE server=? AND action='destroy' AND state='waiting' AND server_deleted_at IS NULL").all(job.server) as { id: string }[]) {
+            cancelRemoval(removal.id);
+        }
+        const cleanupId = beginDeployment(job.server, 'destroy', triggeredBy);
+        db().prepare('INSERT INTO native_creation_cancellations(creation_id,cleanup_id) VALUES(?,?)').run(id, cleanupId);
+        db().prepare("UPDATE server_deployments SET state='cancelled',finished_at=?,error=NULL WHERE id=?").run(new Date().toISOString(), id);
+        deploymentStage(id, 'Creation cancelled; cleanup requested', { blocker: null,
+            event: { kind: 'cancelled', message: 'Creation cancelled. Its partial instance will be removed without a player-count check.', details: { cleanupId } } });
+        if (!job.operation_id && !job.stage_operation_id) {
+            // No remote mutation was ever queued. The reservation is the only resource to remove.
+            db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(new Date().toISOString(), instance.id);
+            releaseHostPorts(instance.id);
+            deploymentStage(cleanupId, 'Unsubmitted creation cancelled; reservations released', { phase: 'cleanup',
+                checkpoint: { key: 'cleanup', label: 'No host command submitted; local reservations released' } });
+            finishDeployment(cleanupId, { ok: true });
+            finishCreationCancellation(cleanupId);
+        } else {
+            const operation = queueHostOperation(instance.host_id, 'instance.cancel-create', { createOperationId: job.operation_id ?? id },
+                { instanceId: instance.id, generation: instance.generation, idempotencyKey: cleanupId });
+            db().prepare('UPDATE native_provisioning SET operation_id=? WHERE deployment_id=?').run(operation.id, cleanupId);
+            db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(new Date().toISOString(), cleanupId);
+            deploymentStage(cleanupId, 'Waiting for host agent to cancel creation and remove its partial instance', { phase: 'container-action', blocker: null,
+                command: { state: 'attempted', message: `Creation cancellation queued as agent operation ${operation.id}. Ports remain reserved until cleanup is confirmed.` } });
+        }
+        return cleanupId;
+    })();
+}
+
+export function finishCreationCancellation(cleanupId: string): void {
+    const cancellation = db().prepare(`SELECT c.creation_id,n.operation_id FROM native_creation_cancellations c
+        JOIN native_provisioning n ON n.deployment_id=c.creation_id WHERE c.cleanup_id=?`).get(cleanupId) as
+        { creation_id: string; operation_id: string | null } | undefined;
+    if (!cancellation) return;
+    const now = new Date().toISOString();
+    db().prepare('UPDATE server_deployments SET server_deleted_at=? WHERE id IN (?,?)').run(now, cancellation.creation_id, cleanupId);
+    deploymentStage(cancellation.creation_id, 'Creation cancelled; partial instance removed', { blocker: null,
+        checkpoint: { key: 'cleanup', phase: 'cleanup', label: 'Cancellation cleanup confirmed; instance and reservations removed' } });
+    // The original command may have lost its receipt. Cleanup proves that its effects are now gone.
+    if (cancellation.operation_id) {
+        const result = db().prepare("UPDATE ec_host_operations SET state='failed',error='Creation cancelled; cleanup confirmed',updated_at=? WHERE id=? AND state NOT IN ('completed','failed')")
+            .run(now, cancellation.operation_id);
+        if (result.changes) db().prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,'failed',?,'Creation cancelled; cleanup confirmed')").run(cancellation.operation_id, now);
+    }
+    recordProgress(cleanupId, { event: { kind: 'cleanup', message: 'Cancelled creation cleanup completed.', details: { creationId: cancellation.creation_id } } });
+}

src/lib/server/provisioning/history.ts

@@ -8,6 +8,7 @@ import { activitySummary, deploymentEvents, latestFailureEvent } from './activit
 import { readProgress, recordProgress, type ProgressUpdate } from './progress.js';
 import { bindNativeOperation } from './native.js';
 import { workerActivity } from '../pipeline/activity.js';
+import { canCancelCreation } from './cancel-creation.js';
 
 export function beginDeployment(server: string, action: ProvisioningAction, triggeredBy = 'ui'): string {
     const id = randomUUID();
@@ -105,6 +106,8 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
         const native = db().prepare('SELECT instance_id FROM native_provisioning WHERE deployment_id=?').get(id);
         const agentOperation = native ? db().prepare(`SELECT o.id, o.host_id AS hostId, o.state, o.updated_at AS updatedAt, o.created_at AS createdAt, o.error
             FROM native_provisioning n JOIN ec_host_operations o ON o.id=n.operation_id WHERE n.deployment_id=?`).get(id) as ProvisioningDetail['agentOperation'] : null;
+        const cancellation = db().prepare(`SELECT c.cleanup_id AS cleanupId,d.state,d.stage,d.error FROM native_creation_cancellations c
+            JOIN server_deployments d ON d.id=c.cleanup_id WHERE c.creation_id=?`).get(id) as ProvisioningDetail['cancellation'];
         const { observation_json: _snapshot, phase_key: _phase, phase_started_at: _phaseStarted, observed_at: _observed, ...deployment } = row;
         const sources = (db().prepare('SELECT kind, source_id, server FROM provisioning_sources WHERE operation_id = ? ORDER BY kind, source_id')
             .all(id) as Omit<ProvisioningSource, 'label' | 'href'>[]).map(sourceDetails);
@@ -114,6 +117,7 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
             UNION ALL SELECT collection_id, deployment_id FROM server_collection_removals) m
             JOIN server_launch_collections c ON c.id = m.collection_id WHERE m.deployment_id = ? ORDER BY c.created_at, c.id LIMIT 1`).get(id) as { id: string; name: string } | undefined;
         return { deployment, backend: native ? 'native' as const : 'komodo' as const, agentOperation: agentOperation ?? null,
+            canCancelCreation: canCancelCreation(id), cancellation: cancellation ?? null,
             workerActivity: workerActivity(), events: deploymentEvents(id, options.events === 'recent'),
             activity: { ...activitySummary(id), failureExcerpt: latestFailureEvent(id) }, sources,
             recipe: launch ? summarizeRecipe(launch.recipe) : null, blueprintName: launch?.blueprintName ?? null,

src/lib/server/provisioning/native.ts

@@ -21,6 +21,7 @@ import { sha256Buffer } from '../fleet/files.js';
 import type { StagedFileChange } from '../fleet/staged-files.js';
 import type { LaunchPreview } from '../../deployment-plan.js';
 import type { ProvisioningAction } from '../../provisioning.js';
+import { cancelCreation, finishCreationCancellation } from './cancel-creation.js';
 
 export interface NativeSelection { hostId: string; libraryId: string; image?: string; bindAddress?: string; publicAddress?: string; publicPort?: number }
 type NativeRecipe = LaunchRecipe & { native?: NativeSelection };
@@ -146,15 +147,7 @@ export function cancelQueuedNativeCreation(server: string, triggeredBy: string):
         const creation = db().prepare(`SELECT d.id FROM server_deployments d JOIN native_provisioning n ON n.deployment_id=d.id
             WHERE n.instance_id=? AND d.action='create' AND d.state='queued' AND n.operation_id IS NULL`).get(instance.id) as { id: string } | undefined;
         if (!creation) return null;
-        const id = beginDeployment(server, 'destroy', triggeredBy);
-        const now = new Date().toISOString();
-        db().prepare("UPDATE server_deployments SET state='cancelled',finished_at=?,stage='Queued native creation cancelled',server_deleted_at=? WHERE id=?")
-            .run(now, now, creation.id);
-        db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(now, instance.id);
-        releaseHostPorts(instance.id);
-        deploymentStage(id, 'Cancelled queued creation; no agent mutation was submitted');
-        finishDeployment(id, { ok: true });
-        return id;
+        return cancelCreation(creation.id, triggeredBy);
     })();
 }
 
@@ -277,6 +270,7 @@ export async function processNativeProvisioning(eligibility: (operation: { id: s
 const activeJobs = new Map<string, Promise<void>>();
 
 async function processJob(job: Job, eligibility: (job: Job) => string | null) {
+    if (!stillActive(job)) return;
     let instance = nativeInstance(job.server);
     if (!instance || instance.id !== job.instance_id || (instance.generation !== job.generation && !(job.action === 'rebuild' && instance.generation === job.generation + 1))) {
         finishDeployment(job.id, { ok: false, message: 'Original instance was retired or replaced; no command was sent.' }); return;
@@ -288,6 +282,9 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
     }
     if (job.operation_id) {
         const operation = getHostOperation(job.operation_id);
+        if (operation?.kind === 'instance.cancel-create' && operation.state === 'pending' && !host.inventory.capabilities?.includes('instance.cancel-create')) {
+            hold(job, 'Update the host agent to support creation cancellation. Creation is cancelled; cleanup and port release await the updated agent.'); return;
+        }
         if (!operation || operation.state === 'unknown') { hold(job, 'Agent result is unknown. Reconciling the original operation; no replacement or repeated command will be sent.'); return; }
         if (operation.state === 'failed') {
             // The agent reports failed only when no mutation began. Ambiguous
@@ -312,6 +309,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
                 db().prepare('DELETE FROM servers WHERE name=?').run(job.server);
                 recordProgress(job.id, { checkpoint: { key: 'cleanup', label: 'Instance retired and port reservations released' } });
                 complete(job);
+                finishCreationCancellation(job.id);
             })();
             return;
         }