Cancel active provisioning jobs when rolling back betas

src/lib/server/provisioning/worker.ts

@@ -1,6 +1,7 @@
 import { maintenanceAllowed } from '../maintenance/policies.js';
 import { resolveRecoveredHealthOperation } from '../maintenance/recovery.js';
 import { db } from '../db/index.js';
+import { provisioningCancelled } from './cancellation.js';
 import { komodoConfigured, rconConfigured } from '../config.js';
 import { getServer } from '../fleet/servers.js';
 import { deploymentStage, finishDeployment, REMOVING_SERVER } from './history.js';
@@ -137,6 +138,7 @@ async function run(): Promise<void> {
         if (nativeDeployment(operation.id)) continue;
         try { await processOperation(operation); }
         catch (err) {
+            if (provisioningCancelled(operation.id)) continue;
             // Observation failures are retried; remote mutations are never replayed.
             deploymentStage(operation.id, 'Waiting for verification: ' + (err as Error).message, {
                 checks: [], blocker: { code: 'observation-unavailable', message: (err as Error).message, automatic: true }
@@ -147,6 +149,7 @@ async function run(): Promise<void> {
 
 async function processOperation(operation: Operation): Promise<void> {
     const { id, server, action } = operation;
+    if (provisioningCancelled(id)) return;
     // An explicit cleanup resume follows confirmed remote deletion. Complete
     // that durable checkpoint even if the incident now needs attention or the
     // Watchdog is paused; never strand the reserved replacement identity.
@@ -194,6 +197,7 @@ async function processOperation(operation: Operation): Promise<void> {
         WHERE s.operation_id = ? AND s.kind = 'watchdog' AND i.kind = 'health' AND d.triggered_by LIKE 'watchdog:%'`).get(id);
     const epoch = latestGameSubmission(server);
     const health = await komodoInspectHealthy(server);
+    if (provisioningCancelled(id)) return;
     if (healthRecovery && epoch !== latestGameSubmission(server)) return;
     if (healthRecovery && !health.ok && !['stopped','exited','created','dead','unhealthy','starting','restarting'].includes(health.state?.toLowerCase() ?? '')) {
         deploymentStage(id, 'Waiting for current container health before automatic recovery', { phase: 'eligibility', checks: [],
@@ -235,6 +239,7 @@ async function processOperation(operation: Operation): Promise<void> {
     if (!claim.changes) return;
     if (action === 'start' || action === 'restart') {
         const result = await restartInRollout(server, { operationId: id, action });
+        if (provisioningCancelled(id)) return;
         if (result.kind === 'queued') {
             db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
             deploymentStage(id, result.message);
@@ -249,6 +254,7 @@ async function processOperation(operation: Operation): Promise<void> {
     // Persist intent first; a lost reply must not issue another stop later.
     try {
         const result = await komodoRuntimeAction(server, 'stop');
+        if (provisioningCancelled(id)) return;
         if (!result.ok) {
             recordProgress(id, { command: { state: result.kind === 'no-name' || result.kind === 'unconfigured' ? 'rejected' : 'uncertain', message: result.message } });
             finishDeployment(id, { ok: false, message: result.message }); return;
@@ -260,6 +266,7 @@ async function processOperation(operation: Operation): Promise<void> {
                 command: { state: 'accepted', message: 'Komodo accepted the stop request; stopped state still requires verification.' } });
         })();
     } catch (err) {
+        if (provisioningCancelled(id)) return;
         recordProgress(id, { command: { state: 'uncertain', message: (err as Error).message } });
         finishDeployment(id, { ok: false, message: (err as Error).message });
     }