Cancel active provisioning jobs when rolling back betas

src/lib/server/fleet/beta.ts

@@ -13,6 +13,7 @@ async function exclusive<T>(id: number, action: () => Promise<T>): Promise<T> {
 import { mkdir, rm, writeFile } from 'node:fs/promises';
 import { readFile, exists } from '../backends/files.js';
 import { getHostOperation } from '../hosts/index.js';
+import { ensureHostSchema } from '../hosts/schema.js';
 
 import { join, posix } from 'node:path';
 import { betaArchiveOf, configFileOf, destOfKind, type ArtifactKind } from '../config.js';
@@ -35,6 +36,9 @@ import { composeServerCfg } from './config-compose.js';
 import { getServerScript, removeScript } from './inventory.js';
 import { cancelStagedFiles, isFileApplied, queueFileChange } from './staged-files.js';
 import { emit } from '../pipeline/bus.js';
+import { cancelProvisioningJob } from '../provisioning/cancellation.js';
+import { reconcileNativeProvisioningReceipt } from '../provisioning/native.js';
+import { requestRestart } from '../provisioning/requests.js';
 import type { BetaStage } from '../pipeline/bus.js';
 
 // Whether an artifact writes a dedicated on-disk file (script/plugin) or
@@ -573,7 +577,6 @@ export async function rollbackBeta(betaId: number, options: { restart?: 'manual'
     return exclusive(betaId, () => rollbackBetaInternal(betaId, options));
 }
 async function rollbackBetaInternal(betaId: number, options: { restart?: 'manual' | 'when-empty' }): Promise<RollbackResult> {
-    assertNativeActivationSettled(betaId);
     const beta = getBeta(betaId);
     if (!beta) throw new BetaError(`no such beta (id=${betaId})`, 'no-such-beta');
     if (beta.state !== 'running' && beta.state !== 'ready-promote') {
@@ -588,6 +591,20 @@ async function rollbackBetaInternal(betaId: number, options: { restart?: 'manual
         versionId: br.script_version_id
     }));
 
+    ensureHostSchema(db());
+    db().transaction(() => {
+        const jobs = db().prepare(`SELECT s.operation_id FROM provisioning_sources s WHERE s.kind='beta' AND s.source_id=?`)
+            .all(String(betaId)) as { operation_id: string }[];
+        for (const job of jobs) {
+            const undelivered = db().prepare(`SELECT 1 FROM native_provisioning n JOIN ec_host_operations o ON o.id=n.operation_id
+                WHERE n.deployment_id=? AND o.state='pending' AND o.last_delivered_at IS NULL`).get(job.operation_id);
+            if (undelivered) cancelProvisioningJob(job.operation_id, `Superseded by rollback of beta #${betaId}`);
+            reconcileNativeProvisioningReceipt(job.operation_id);
+        }
+        // Before images remain mandatory for a mutation already delivered to a host.
+        // Completed commands can be reconciled here without waiting for game health.
+        assertNativeActivationSettled(betaId);
+    })();
     const servers = listBetaServers(betaId);
     const restored = new Set<string>();
     const dropped = new Set<string>();
@@ -640,6 +657,12 @@ async function rollbackBetaInternal(betaId: number, options: { restart?: 'manual
 
     const deploymentId = restorations.length ? `dep-beta-rollback-${betaId}-${Date.now()}` : null;
     db().transaction(() => {
+        for (const bs of servers) {
+            if (!isActiveBetaServer(betaId, bs.server)) continue;
+            const pending = db().prepare(`SELECT id FROM server_deployments WHERE server=? AND server_deleted_at IS NULL
+                AND state IN ('queued','waiting','running','awaiting-health','interrupted')`).all(bs.server) as { id: string }[];
+            for (const job of pending) cancelProvisioningJob(job.id, `Superseded by rollback of beta #${betaId}`);
+        }
         cancelStagedFiles('beta', String(betaId));
         if (deploymentId) {
             db().prepare("INSERT INTO deployments (id, action, started_at, triggered_by, beta_id) VALUES (?, 'rollback', ?, ?, ?)")
@@ -661,6 +684,11 @@ async function rollbackBetaInternal(betaId: number, options: { restart?: 'manual
                     deploymentFileId: Number(file.lastInsertRowid) });
             }
         }
+        if (deploymentId) {
+            for (const server of new Set(restorations.map(change => change.server))) {
+                requestRestart(server, { kind: 'campaign', id: deploymentId });
+            }
+        }
         // The queued restoration owns a durable copy before snapshots are released.
         db().prepare('DELETE FROM beta_file_snapshots WHERE beta_id = ?').run(betaId);
         db().prepare("UPDATE betas SET state = 'rolled-back', closed_at = ? WHERE id = ?").run(nowIso(), betaId);