Cancel active provisioning jobs when rolling back betas

src/lib/server/provisioning/cancellation.ts

@@ -0,0 +1,59 @@
+import { db } from '../db/index.js';
+import { ensureHostSchema } from '../hosts/schema.js';
+import { recordProgress } from './progress.js';
+
+export function provisioningCancelled(id: string): boolean {
+    return !!db().prepare("SELECT 1 FROM server_deployments WHERE id=? AND state='cancelled'").get(id);
+}
+
+export function assertProvisioningNotCancelled(id: string): void {
+    if (provisioningCancelled(id)) throw new Error('Provisioning operation cancelled');
+}
+
+export function canCancelProvisioningJob(id: string): boolean {
+    return !!db().prepare(`SELECT 1 FROM server_deployments WHERE id=? AND server_deleted_at IS NULL
+        AND state IN ('queued','waiting','running','awaiting-health','interrupted')`).get(id);
+}
+
+/** Cancel remaining work, not effects already performed by the host. */
+export function cancelProvisioningJob(id: string, reason = 'Cancelled by operator'): boolean {
+    ensureHostSchema(db());
+    return db().transaction(() => {
+        if (provisioningCancelled(id)) return true;
+        if (!canCancelProvisioningJob(id)) return false;
+        const at = new Date().toISOString();
+        // Only commands never delivered can be withdrawn. Delivered commands retain
+        // their immutable IDs and receipts; replacements wait for their resolution.
+        const commands = db().prepare(`SELECT o.id FROM ec_host_operations o
+            WHERE o.state='pending' AND o.last_delivered_at IS NULL AND o.id IN (
+                SELECT operation_id FROM native_provisioning WHERE deployment_id=?
+                UNION SELECT stage_operation_id FROM native_provisioning WHERE deployment_id=?
+                UNION SELECT remove_operation_id FROM ec_host_migrations WHERE deployment_id=?)`).all(id, id, id) as { id: string }[];
+        for (const command of commands) {
+            db().prepare("UPDATE ec_host_operations SET state='failed',error=?,updated_at=? WHERE id=?").run(reason, at, command.id);
+            db().prepare("INSERT INTO ec_host_operation_events(operation_id,state,created_at,error) VALUES(?,'failed',?,?)").run(command.id, at, reason);
+        }
+        db().prepare(`UPDATE staged_file_changes SET state='cancelled' WHERE state='pending' AND EXISTS (
+            SELECT 1 FROM provisioning_sources s WHERE s.operation_id=? AND s.kind=staged_file_changes.source_kind
+                AND s.source_id=staged_file_changes.source_id AND s.server=staged_file_changes.server)`).run(id);
+        db().prepare('DELETE FROM restart_rollout WHERE operation_id=?').run(id);
+        db().prepare("UPDATE server_deployments SET state='cancelled',finished_at=?,stage=?,error=NULL WHERE id=?").run(at, reason, id);
+        recordProgress(id, { blocker: null, checks: [],
+            recovery: { state: 'released', at, operationId: id, message: 'Recovery capacity released by cancellation; this does not certify server health.' },
+            event: { kind: 'outcome', message: reason, details: { state: 'cancelled', finishedAt: at, error: null, warnings: [] } } });
+        return true;
+    })();
+}
+
+/** A cancelled controller job must not let a replacement overtake a delivered mutation. */
+export function unsettledCancelledCommand(server: string, replacementId: string): string | null {
+    const row = db().prepare(`SELECT d.id FROM server_deployments d
+        JOIN native_provisioning n ON n.deployment_id=d.id
+        LEFT JOIN ec_host_migrations m ON m.deployment_id=d.id
+        JOIN ec_host_operations o ON o.id=n.operation_id OR o.id=m.remove_operation_id
+        WHERE d.server=? AND d.server_deleted_at IS NULL AND d.state='cancelled'
+        AND (o.state NOT IN ('completed','failed')
+            OR (o.state='completed' AND o.id=n.operation_id AND n.files_json IS NOT NULL AND n.applied_at IS NULL))
+        AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations c WHERE c.creation_id=d.id AND c.cleanup_id=?) LIMIT 1`).get(server, replacementId) as { id: string } | undefined;
+    return row?.id ?? null;
+}