Cancel active provisioning jobs when rolling back betas

src/lib/server/provisioning/host-drain.ts

@@ -1,4 +1,5 @@
 import { db } from '../db/index.js';
+import { provisioningCancelled } from './cancellation.js';
 import { nativeInstance, nativeRequest, type NativeInstance } from '../backends/native.js';
 import { getHost, getHostOperationStatus, listHosts, queueHostOperation, reserveHostPort, updateHost } from '../hosts/index.js';
 import { choosePlacement } from '../hosts/capacity.js';
@@ -43,6 +44,10 @@ export function reconcileHostDrains(): void {
             for (const instance of instances) {
                 const server = getServer(instance.server);
                 if (!server?.map_slug || activeBetaRemovalBlocker(instance.server)) continue;
+                // An explicit cancellation withdraws this instance's current drain attempt.
+                const latest = db().prepare(`SELECT d.state FROM ec_host_migrations m JOIN server_deployments d ON d.id=m.deployment_id
+                    WHERE m.instance_id=? AND d.server_deleted_at IS NULL ORDER BY d.rowid DESC LIMIT 1`).get(instance.id) as { state: string } | undefined;
+                if (latest?.state === 'cancelled') continue;
                 if (db().prepare(`SELECT 1 FROM server_deployments WHERE server=? AND server_deleted_at IS NULL
                     AND state IN ('queued','waiting','running','awaiting-health','interrupted')`).get(instance.server)) continue;
                 const recipe = { ...captureRecipePlan(server.game as 't5' | 't6', server.map_slug), image: instance.image,
@@ -67,7 +72,8 @@ export function reconcileHostDrains(): void {
 /** Reserve a destination, remove the source once, then hand creation to the normal health-verified rebuild queue. */
 export async function processMigrationRemoval(job: Job, instance: NativeInstance, migration: Migration,
     eligibility: () => string | null): Promise<void> {
-    const hold = (message: string) => deploymentStage(job.id, message, { blocker: { code: 'native-host', message, automatic: true } });
+    if (provisioningCancelled(job.id)) return;
+    const hold = (message: string) => { if (!provisioningCancelled(job.id)) deploymentStage(job.id, message, { blocker: { code: 'native-host', message, automatic: true } }); };
     if (migration.remove_operation_id) {
         const operation = getHostOperationStatus(migration.remove_operation_id);
         if (operation?.state === 'failed' && Date.now() - Date.parse(operation.updatedAt) >= 60_000) {
@@ -121,8 +127,10 @@ export async function processMigrationRemoval(job: Job, instance: NativeInstance
     // No source mutation is submitted unless another host can take this instance.
     choosePlacement(options);
     const health = await nativeRequest<{ state: string }>(instance, 'instance.inspect');
+    if (provisioningCancelled(job.id)) return;
     if (!['stopped','exited','created','dead','missing'].includes(health.state)) {
         const occupancy = await probePlayerEvidence(job.server);
+        if (provisioningCancelled(job.id)) return;
         if (occupancy.players !== 0) { hold(occupancy.players === null ? 'Waiting for a fresh local player count.' : `Waiting for ${occupancy.players} player(s) to leave.`); return; }
     }
     db().transaction(() => {