Add safe manual instance rebalancing

src/lib/server/provisioning/cancellation.ts

@@ -2,6 +2,15 @@ import { pauseScalingForOperation } from '../autoscaling/ownership.js';
 import { db } from '../db/index.js';
 import { ensureHostSchema } from '../hosts/schema.js';
 import { recordProgress } from './progress.js';
+import { requestRebalanceStopForOperation } from '../rebalancing/admission.js';
+
+export const rebalanceCancellationGuard = `NOT (d.triggered_by LIKE 'rebalance:%' AND EXISTS (
+    SELECT 1 FROM ec_host_migrations m WHERE m.deployment_id=d.id AND m.remove_operation_id IS NOT NULL))`;
+
+export function rebalanceCancellationBlocker(id: string): string | null {
+    return db().prepare(`SELECT 1 FROM server_deployments d WHERE d.id=? AND NOT (${rebalanceCancellationGuard})`).get(id)
+        ? 'This rebalance move has committed source removal and must finish recreating the lobby. Stop the rebalance run to withdraw its remaining moves.' : null;
+}
 
 export function provisioningCancelled(id: string): boolean {
     return !!db().prepare("SELECT 1 FROM server_deployments WHERE id=? AND state='cancelled'").get(id);
@@ -12,17 +21,20 @@ export function assertProvisioningNotCancelled(id: string): void {
 }
 
 export function canCancelProvisioningJob(id: string): boolean {
-    return !!db().prepare(`SELECT 1 FROM server_deployments WHERE id=? AND server_deleted_at IS NULL
-        AND state IN ('queued','waiting','running','awaiting-health','interrupted')`).get(id);
+    return !!db().prepare(`SELECT 1 FROM server_deployments d WHERE d.id=? AND d.server_deleted_at IS NULL
+        AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') AND ${rebalanceCancellationGuard}`).get(id);
 }
 
 /** Cancel remaining work, not effects already performed by the host. */
-export function cancelProvisioningJob(id: string, reason = 'Cancelled by operator'): boolean {
+export function cancelProvisioningJob(id: string, reason = 'Cancelled by operator', options: { preserveRebalanceRun?: boolean } = {}): boolean {
     ensureHostSchema(db());
     return db().transaction(() => {
         if (provisioningCancelled(id)) return true;
         if (!canCancelProvisioningJob(id)) return false;
-        pauseScalingForOperation(id);
+        const rebalance = !!db().prepare("SELECT 1 FROM server_deployments WHERE id=? AND triggered_by LIKE 'rebalance:%'").get(id);
+        if (rebalance) {
+            if (!options.preserveRebalanceRun) requestRebalanceStopForOperation(id);
+        } else pauseScalingForOperation(id);
         const at = new Date().toISOString();
         // Only commands never delivered can be withdrawn. Delivered commands retain
         // their immutable IDs and receipts; replacements wait for their resolution.