Add cancellation and cleanup for native server creation

src/lib/server/provisioning/cancel-creation.ts

@@ -0,0 +1,74 @@
+import { db } from '../db/index.js';
+import { nativeInstance } from '../backends/native.js';
+import { queueHostOperation, releaseHostPorts } from '../hosts/index.js';
+import { beginDeployment, cancelRemoval, deploymentStage, finishDeployment } from './history.js';
+import { recordProgress } from './progress.js';
+
+export function canCancelCreation(id: string): boolean {
+    return !!db().prepare(`SELECT 1 FROM server_deployments d
+        JOIN native_provisioning n ON n.deployment_id=d.id
+        JOIN native_instances i ON i.id=n.instance_id AND i.generation=n.generation AND i.retired_at IS NULL
+        WHERE d.id=? AND d.action='create' AND d.server_deleted_at IS NULL
+        AND d.state IN ('queued','waiting','running','awaiting-health','interrupted','failed')
+        AND NOT EXISTS (SELECT 1 FROM server_deployments other WHERE other.server=d.server
+            AND other.server_deleted_at IS NULL AND other.rowid>d.rowid AND other.state!='cancelled'
+            AND NOT (other.action='destroy' AND other.state='waiting'
+                AND NOT EXISTS (SELECT 1 FROM native_provisioning p WHERE p.deployment_id=other.id AND p.operation_id IS NOT NULL)))`).get(id);
+}
+
+/** Stops observation immediately; remote cleanup remains a durable, separately visible job. */
+export function cancelCreation(id: string, triggeredBy = 'ui'): string | null {
+    return db().transaction(() => {
+        const prior = db().prepare('SELECT cleanup_id FROM native_creation_cancellations WHERE creation_id=?').get(id) as { cleanup_id: string } | undefined;
+        if (prior) return prior.cleanup_id;
+        if (!canCancelCreation(id)) return null;
+        const job = db().prepare(`SELECT d.server,n.operation_id,n.stage_operation_id FROM server_deployments d
+            JOIN native_provisioning n ON n.deployment_id=d.id WHERE d.id=?`).get(id) as
+            { server: string; operation_id: string | null; stage_operation_id: string | null };
+        const instance = nativeInstance(job.server)!;
+        // Replace a removal still waiting on occupancy, including one whose asynchronous probe is outstanding.
+        for (const removal of db().prepare("SELECT id FROM server_deployments WHERE server=? AND action='destroy' AND state='waiting' AND server_deleted_at IS NULL").all(job.server) as { id: string }[]) {
+            cancelRemoval(removal.id);
+        }
+        const cleanupId = beginDeployment(job.server, 'destroy', triggeredBy);
+        db().prepare('INSERT INTO native_creation_cancellations(creation_id,cleanup_id) VALUES(?,?)').run(id, cleanupId);
+        db().prepare("UPDATE server_deployments SET state='cancelled',finished_at=?,error=NULL WHERE id=?").run(new Date().toISOString(), id);
+        deploymentStage(id, 'Creation cancelled; cleanup requested', { blocker: null,
+            event: { kind: 'cancelled', message: 'Creation cancelled. Its partial instance will be removed without a player-count check.', details: { cleanupId } } });
+        if (!job.operation_id && !job.stage_operation_id) {
+            // No remote mutation was ever queued. The reservation is the only resource to remove.
+            db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(new Date().toISOString(), instance.id);
+            releaseHostPorts(instance.id);
+            deploymentStage(cleanupId, 'Unsubmitted creation cancelled; reservations released', { phase: 'cleanup',
+                checkpoint: { key: 'cleanup', label: 'No host command submitted; local reservations released' } });
+            finishDeployment(cleanupId, { ok: true });
+            finishCreationCancellation(cleanupId);
+        } else {
+            const operation = queueHostOperation(instance.host_id, 'instance.cancel-create', { createOperationId: job.operation_id ?? id },
+                { instanceId: instance.id, generation: instance.generation, idempotencyKey: cleanupId });
+            db().prepare('UPDATE native_provisioning SET operation_id=? WHERE deployment_id=?').run(operation.id, cleanupId);
+            db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(new Date().toISOString(), cleanupId);
+            deploymentStage(cleanupId, 'Waiting for host agent to cancel creation and remove its partial instance', { phase: 'container-action', blocker: null,
+                command: { state: 'attempted', message: `Creation cancellation queued as agent operation ${operation.id}. Ports remain reserved until cleanup is confirmed.` } });
+        }
+        return cleanupId;
+    })();
+}
+
+export function finishCreationCancellation(cleanupId: string): void {
+    const cancellation = db().prepare(`SELECT c.creation_id,n.operation_id FROM native_creation_cancellations c
+        JOIN native_provisioning n ON n.deployment_id=c.creation_id WHERE c.cleanup_id=?`).get(cleanupId) as
+        { creation_id: string; operation_id: string | null } | undefined;
+    if (!cancellation) return;
+    const now = new Date().toISOString();
+    db().prepare('UPDATE server_deployments SET server_deleted_at=? WHERE id IN (?,?)').run(now, cancellation.creation_id, cleanupId);
+    deploymentStage(cancellation.creation_id, 'Creation cancelled; partial instance removed', { blocker: null,
+        checkpoint: { key: 'cleanup', phase: 'cleanup', label: 'Cancellation cleanup confirmed; instance and reservations removed' } });
+    // The original command may have lost its receipt. Cleanup proves that its effects are now gone.
+    if (cancellation.operation_id) {
+        const result = db().prepare("UPDATE ec_host_operations SET state='failed',error='Creation cancelled; cleanup confirmed',updated_at=? WHERE id=? AND state NOT IN ('completed','failed')")
+            .run(now, cancellation.operation_id);
+        if (result.changes) db().prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,'failed',?,'Creation cancelled; cleanup confirmed')").run(cancellation.operation_id, now);
+    }
+    recordProgress(cleanupId, { event: { kind: 'cleanup', message: 'Cancelled creation cleanup completed.', details: { creationId: cancellation.creation_id } } });
+}