Add safe manual instance rebalancing

src/lib/server/backends/schema.ts

@@ -32,7 +32,7 @@ export function migrateNativeInstances(handle: Database.Database): void {
             deployment_id TEXT PRIMARY KEY REFERENCES server_deployments(id),
             source_host_id TEXT NOT NULL, instance_id TEXT NOT NULL REFERENCES native_instances(id),
             target_host_id TEXT, target_port INTEGER, target_bind TEXT,
-            remove_operation_id TEXT, removed_at TEXT
+            remove_operation_id TEXT, removed_at TEXT, runtime_password TEXT
         );
         CREATE INDEX IF NOT EXISTS ec_host_migrations_source ON ec_host_migrations(source_host_id);
         CREATE INDEX IF NOT EXISTS ec_host_migrations_target ON ec_host_migrations(target_host_id);
@@ -43,6 +43,9 @@ export function migrateNativeInstances(handle: Database.Database): void {
         );
         CREATE INDEX IF NOT EXISTS idx_ec_host_migrations_instance ON ec_host_migrations(instance_id);
     `);
+    if (!(handle.pragma('table_info(ec_host_migrations)') as { name: string }[]).some(column => column.name === 'runtime_password')) {
+        handle.exec('ALTER TABLE ec_host_migrations ADD COLUMN runtime_password TEXT');
+    }
     if (!handle.prepare("SELECT 1 FROM settings WHERE key='native_file_plans_v1'").get()) {
         handle.transaction(() => {
             handle.exec(`UPDATE native_provisioning SET files_json=(SELECT json_group_array(json_remove(value,'$.contentBase64'))

src/lib/server/db/index.ts

@@ -12,6 +12,7 @@ import { migrateServerHistory } from './server-history.js';
 import { migrateNativeInstances } from '../backends/schema.js';
 import { migrateServerKeys } from './server-keys.js';
 import { ensureAutoscalingSchema } from '../autoscaling/schema.js';
+import { ensureRebalancingSchema } from '../rebalancing/schema.js';
 import { installPerformanceIndexes } from './performance-indexes.js';
 import { migrateFileBlobs } from './file-blobs.js';
 import { enableIncrementalVacuum, retentionClocks } from './retention-clocks.js';
@@ -402,6 +403,7 @@ export function db(): Database.Database {
         migrationStep('native instance schema and file plans', () => migrateNativeInstances(handle));
         migrationStep('per-game server key pools', () => migrateServerKeys(handle));
         migrationStep('autoscaling schema', () => ensureAutoscalingSchema(handle));
+        migrationStep('rebalancing schema', () => ensureRebalancingSchema(handle));
         // File-blob migration removes the legacy content column altogether.
         // Reordering it first would unnecessarily copy every artifact twice.
         migrateFileBlobs(handle);

src/lib/server/hosts/availability.ts

@@ -3,6 +3,7 @@ import { fleetSettings } from '../settings.js';
 import { subscribe } from '../pipeline/bus.js';
 import { capacityCalibration } from './calibration.js';
 import { HEARTBEAT_MAX_AGE_MS, listHosts } from './index.js';
+import { committedRebalanceStartup } from '../provisioning/concurrency.js';
 import type { CapacityCalibration, Host, NativePlacement } from '../../hosts.js';
 import type { LaunchAvailability } from '../../deployment-plan.js';
 
@@ -37,9 +38,9 @@ function snapshotSchemaReady(handle: ReturnType<typeof db>): boolean {
 
 function queueSnapshot(handle: ReturnType<typeof db>, hosts: Map<string, HostAvailability>): Snapshot {
     const counts = handle.prepare(`SELECT
-        COALESCE(SUM(state IN ('running','awaiting-health')),0) AS active,
+        COALESCE(SUM(state IN ('running','awaiting-health') OR ${committedRebalanceStartup}),0) AS active,
         COALESCE(SUM(state='queued'),0) AS queued FROM server_deployments
-        WHERE action IN ('create','rebuild') AND state IN ('queued','running','awaiting-health')
+        WHERE action IN ('create','rebuild') AND (state IN ('queued','running','awaiting-health') OR ${committedRebalanceStartup})
         AND server_deleted_at IS NULL`).get() as { active: number; queued: number };
     return { hosts, ...counts, startupLimit: fleetSettings().createConcurrency, updatedAt: new Date().toISOString() };
 }

src/lib/server/hosts/capacity.ts

@@ -11,6 +11,7 @@ const CPU_USABLE_FRACTION = 0.95;
 export class CapacityError extends Error {}
 export interface PlacementOptions {
     hostId?: string; libraryId: string; image: string; bindAddress?: string;
+    excludeHostIds?: string[];
     thorough?: boolean; excludeInstanceId?: string;
     // Execution only: not-yet-submitted launches wait their turn instead of
     // requiring their entire backlog to fit before any one can start.
@@ -100,7 +101,7 @@ function projectedLoad(host: Host, options: PlacementOptions): number {
 export function choosePlacement(options: PlacementOptions): { host: Host; port: number; load: number } {
     const candidates: { host: Host; port: number; load: number }[] = [];
     const reasons: string[] = [];
-    for (const host of listHosts().filter(host => !options.hostId || host.id === options.hostId)) {
+    for (const host of listHosts().filter(host => (!options.hostId || host.id === options.hostId) && !options.excludeHostIds?.includes(host.id))) {
         try {
             requireReadyHost(host.id, { image: options.image, requiredBytes: INSTANCE_STORAGE_BYTES });
             assertReadyLibrary(host.id, options.libraryId);

src/lib/server/pipeline/worker.ts

@@ -1,6 +1,7 @@
 import { pruneHistory, pruneSampleHistory } from '../retention.js';
 import { collectFleetChat } from '../fleet/chat.js';
 import { runAutoscaling, pruneAutoscaling } from '../autoscaling/index.js';
+import { runRebalancing, pruneRebalancing } from '../rebalancing/index.js';
 import { config } from '../config.js';
 import { processProvisioning } from '../provisioning/worker.js';
 import { sweep } from './sweep.js';
@@ -59,9 +60,10 @@ function reconcileRequesters(): void {
         ['beta', 'Reconciling beta tests', betaTick],
         ['maintenance', 'Checking maintenance and recovery', runMaintenance],
         ['autoscaling', 'Evaluating lobby autoscaling', runAutoscaling],
+        ['rebalancing', 'Reconciling rebalance moves', runRebalancing],
         ['game-files', 'Removing unused game files', cleanupControllerLibraries],
         ['chat', 'Saving match chat', collectFleetChat],
-        ['retention', 'Cleaning expired history', async () => { pruneHistory(); pruneSampleHistory(); pruneAutoscaling(); }]
+        ['retention', 'Cleaning expired history', async () => { pruneHistory(); pruneSampleHistory(); pruneAutoscaling(); pruneRebalancing(); }]
     ];
     for (const [key, label, work] of requesters) {
         if (requesterFlights.has(key)) continue;
@@ -71,7 +73,7 @@ function reconcileRequesters(): void {
         }
         const flight = trackActivity(key, label, work)
             .catch(err => { console.error(`[worker] ${key} error:`, err); })
-            .then(async result => { if (key !== 'game-files' && key !== 'retention' && key !== 'chat' && (key !== 'autoscaling' || result === true)) await tick(true); })
+            .then(async result => { if (key !== 'game-files' && key !== 'retention' && key !== 'chat' && (!['autoscaling', 'rebalancing'].includes(key) || result === true)) await tick(true); })
             .finally(() => { requesterFlights.delete(key); });
         requesterFlights.set(key, flight);
     }

src/lib/server/provisioning/cancellation.ts

@@ -2,6 +2,15 @@ import { pauseScalingForOperation } from '../autoscaling/ownership.js';
 import { db } from '../db/index.js';
 import { ensureHostSchema } from '../hosts/schema.js';
 import { recordProgress } from './progress.js';
+import { requestRebalanceStopForOperation } from '../rebalancing/admission.js';
+
+export const rebalanceCancellationGuard = `NOT (d.triggered_by LIKE 'rebalance:%' AND EXISTS (
+    SELECT 1 FROM ec_host_migrations m WHERE m.deployment_id=d.id AND m.remove_operation_id IS NOT NULL))`;
+
+export function rebalanceCancellationBlocker(id: string): string | null {
+    return db().prepare(`SELECT 1 FROM server_deployments d WHERE d.id=? AND NOT (${rebalanceCancellationGuard})`).get(id)
+        ? 'This rebalance move has committed source removal and must finish recreating the lobby. Stop the rebalance run to withdraw its remaining moves.' : null;
+}
 
 export function provisioningCancelled(id: string): boolean {
     return !!db().prepare("SELECT 1 FROM server_deployments WHERE id=? AND state='cancelled'").get(id);
@@ -12,17 +21,20 @@ export function assertProvisioningNotCancelled(id: string): void {
 }
 
 export function canCancelProvisioningJob(id: string): boolean {
-    return !!db().prepare(`SELECT 1 FROM server_deployments WHERE id=? AND server_deleted_at IS NULL
-        AND state IN ('queued','waiting','running','awaiting-health','interrupted')`).get(id);
+    return !!db().prepare(`SELECT 1 FROM server_deployments d WHERE d.id=? AND d.server_deleted_at IS NULL
+        AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') AND ${rebalanceCancellationGuard}`).get(id);
 }
 
 /** Cancel remaining work, not effects already performed by the host. */
-export function cancelProvisioningJob(id: string, reason = 'Cancelled by operator'): boolean {
+export function cancelProvisioningJob(id: string, reason = 'Cancelled by operator', options: { preserveRebalanceRun?: boolean } = {}): boolean {
     ensureHostSchema(db());
     return db().transaction(() => {
         if (provisioningCancelled(id)) return true;
         if (!canCancelProvisioningJob(id)) return false;
-        pauseScalingForOperation(id);
+        const rebalance = !!db().prepare("SELECT 1 FROM server_deployments WHERE id=? AND triggered_by LIKE 'rebalance:%'").get(id);
+        if (rebalance) {
+            if (!options.preserveRebalanceRun) requestRebalanceStopForOperation(id);
+        } else pauseScalingForOperation(id);
         const at = new Date().toISOString();
         // Only commands never delivered can be withdrawn. Delivered commands retain
         // their immutable IDs and receipts; replacements wait for their resolution.

src/lib/server/provisioning/concurrency.ts

@@ -4,6 +4,9 @@ import { fleetSettings } from '../settings.js';
 
 // A preparation owns capacity across asynchronous launch-file generation.
 export const preparingJobs = new Set<string>();
+/** A committed rebalance keeps its startup reservation while between source removal and creation. */
+export const committedRebalanceStartup = `(triggered_by LIKE 'rebalance:%' AND state IN ('waiting','interrupted')
+    AND EXISTS (SELECT 1 FROM ec_host_migrations m WHERE m.deployment_id=server_deployments.id AND m.remove_operation_id IS NOT NULL))`;
 interface Holder { id: string; action: string; host_id: string | null; target_host_id: string | null; removing: number }
 
 export function lifecycleHolders(): Holder[] {

src/lib/server/provisioning/history.ts

@@ -11,7 +11,8 @@ import { readProgress, recordProgress, type ProgressUpdate } from './progress.js
 import { bindNativeOperation } from './native.js';
 import { workerActivity } from '../pipeline/activity.js';
 import { canCancelCreation } from './cancel-creation.js';
-import { canCancelProvisioningJob, provisioningCancelled } from './cancellation.js';
+import { canCancelProvisioningJob, provisioningCancelled, rebalanceCancellationGuard } from './cancellation.js';
+import { committedRebalanceStartup } from './concurrency.js';
 import { nativeDeployment } from '../backends/native.js';
 import { safetyOverride, safetyOverrideAvailability } from './safety-override.js';
 import { serverTitle } from '../../ui/server-labels.js';
@@ -23,7 +24,7 @@ export function beginDeployment(server: string, action: ProvisioningAction, trig
     db().transaction(() => {
         db().prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by, requested_by)
             VALUES (?, ?, ?, 'running', 'Starting', ?, ?, ?)`).run(id, server, action, new Date().toISOString(), triggeredBy,
-                triggeredBy === 'ui' ? currentRequestUsername() : null);
+                triggeredBy === 'ui' || triggeredBy.startsWith('rebalance:') ? currentRequestUsername() : null);
         bindNativeOperation(id, server);
         recordProgress(id, { phase: 'requested', observedAt: null, checkpoint: { key: 'requested', phase: 'requested',
             label: action === 'create' ? 'Server creation requested'
@@ -63,6 +64,11 @@ function warningMessages(raw: string): string[] {
 
 function sourceDetails(source: Omit<ProvisioningSource, 'label' | 'href'>): ProvisioningSource {
     const id = source.source_id;
+    if (source.kind === 'rebalance') {
+        const exists = !!db().prepare('SELECT 1 FROM rebalancing_runs WHERE id=?').get(id);
+        return { ...source, label: exists ? 'Rebalance run' : 'Rebalance history expired',
+            href: exists ? `/capacity?rebalance=1&rebalanceRun=${encodeURIComponent(id)}` : null };
+    }
     if (source.kind === 'autoscaling') {
         const exists = !!db().prepare('SELECT 1 FROM autoscaling_policies WHERE id=?').get(id);
         return { ...source, label: exists ? 'Autoscaling policy' : 'Deleted autoscaling policy', href: exists ? `/autoscaling#${encodeURIComponent(id)}` : null };
@@ -94,7 +100,7 @@ function capacityFor(deployment: ProvisioningDeployment, evaluatedAt: string): P
     const pools: ProvisioningCapacity['pools'] = [];
     if (deployment.action === 'create' || deployment.action === 'rebuild') {
         const jobs = db().prepare(`SELECT id, server, state FROM server_deployments
-            WHERE server_deleted_at IS NULL AND action IN ('create','rebuild') AND state IN ('queued','running','awaiting-health')
+            WHERE server_deleted_at IS NULL AND action IN ('create','rebuild') AND (state IN ('queued','running','awaiting-health') OR ${committedRebalanceStartup})
             ORDER BY started_at, rowid`).all() as { id: string; server: string; state: string }[];
         const holders = jobs.filter(job => job.state !== 'queued').map(({ id, server }) => ({ id, server }));
         const order = jobs.filter(job => job.state === 'queued').findIndex(job => job.id === deployment.id);
@@ -127,6 +133,7 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
         const { observation_json: _snapshot, phase_key: _phase, phase_started_at: _phaseStarted, observed_at: _observed, ...deployment } = row;
         const sources = (db().prepare('SELECT kind, source_id, server FROM provisioning_sources WHERE operation_id = ? ORDER BY kind, source_id')
             .all(id) as Omit<ProvisioningSource, 'label' | 'href'>[]).map(sourceDetails);
+        if (row.triggered_by.startsWith('rebalance:')) sources.push(sourceDetails({ kind: 'rebalance', source_id: row.triggered_by.slice('rebalance:'.length) }));
         const launch = recipeSummaryForDeployment(id);
         const collection = db().prepare(`SELECT c.id, c.name FROM (
             SELECT collection_id, deployment_id FROM server_collection_members
@@ -271,7 +278,8 @@ const deploymentListColumns = `d.id, d.server, d.server_deleted_at, d.action, d.
         CASE WHEN json_valid(d.observation_json) THEN json_extract(d.observation_json,'$.blocker.code') END AS waitCode,
         (d.server_deleted_at IS NULL AND EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server AND s.destroyed_at IS NULL)) AS server_exists,
         (d.server_deleted_at IS NULL AND d.action = 'destroy' AND d.state = 'waiting' AND d.stage != ?) AS canCancelRemoval,
-        (d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted')) AS canCancel
+        (d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted')
+            AND ${rebalanceCancellationGuard}) AS canCancel
     `;
 
 function deploymentListQuery(options: ProvisioningListOptions) {
@@ -286,8 +294,9 @@ function deploymentListQuery(options: ProvisioningListOptions) {
         OR ${kinds.map(kind => `d.triggered_by LIKE '${kind}:%'`).join(' OR ')})`;
     if (options.source === 'campaign' || options.source === 'beta') conditions.push(linked([options.source]));
     else if (options.source === 'maintenance') conditions.push(linked(['maintenance', 'watchdog']));
+    else if (options.source === 'rebalance') conditions.push("d.triggered_by LIKE 'rebalance:%'");
     else if (options.source === 'manual' || options.source === 'system') {
-        conditions.push(`NOT ${linked(['campaign', 'beta', 'maintenance', 'watchdog'])}`);
+        conditions.push(`NOT ${linked(['campaign', 'beta', 'maintenance', 'watchdog', 'rebalance'])}`);
         conditions.push(options.source === 'manual' ? "d.triggered_by='ui'" : "d.triggered_by!='ui'");
     }
     const search = (options.server ?? '').slice(0, 100).toLowerCase();
@@ -332,7 +341,7 @@ function deploymentListRows(query: ReturnType<typeof deploymentListQuery>, activ
     const cache = new Map<string, ProvisioningSource>();
     for (const row of rows) {
         const sources = byOperation.get(row.id) ?? [];
-        const fallback = row.triggered_by.match(/^(campaign|beta|maintenance|watchdog|autoscaling):(.+)$/);
+        const fallback = row.triggered_by.match(/^(campaign|beta|maintenance|watchdog|autoscaling|rebalance):(.+)$/);
         if (fallback && !sources.some(source => source.kind === fallback[1] && source.source_id === fallback[2])) sources.unshift({ kind: fallback[1] as ProvisioningSource['kind'], source_id: fallback[2] });
         row.sources = sources.map(source => {
             const key = `${source.kind}:${source.source_id}`;

src/lib/server/provisioning/host-drain.ts

@@ -1,29 +1,13 @@
-import { concurrencyBlocker } from './concurrency.js';
 import { db } from '../db/index.js';
-import { provisioningCancelled } from './cancellation.js';
-import { nativeInstance, nativeRequest, type NativeInstance } from '../backends/native.js';
-import { getHost, getHostOperationStatus, listHosts, queueHostOperation, reserveHostPort, updateHost } from '../hosts/index.js';
-import { choosePlacement } from '../hosts/capacity.js';
-import { scheduleAvailabilityRefresh } from '../hosts/availability.js';
+import type { NativeInstance } from '../backends/native.js';
+import { listHosts, updateHost } from '../hosts/index.js';
 import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
-import { getServerKey } from '../fleet/server-keys.js';
-import { captureRecipePlan, recipeForDeployment, recipeStorage, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
-import { probePlayerEvidence } from '../fleet/status-evidence.js';
+import { captureRecipePlan, recipeStorage } from '../fleet/launch-recipes.js';
 import { beginDeployment, deploymentStage } from './history.js';
-import { fleetSettings } from '../settings.js';
 import type { NativePlacement } from '../../hosts.js';
 
-interface Migration {
-    deployment_id: string; source_host_id: string; instance_id: string;
-    target_host_id: string | null; target_port: number | null; target_bind: string | null;
-    remove_operation_id: string | null; removed_at: string | null;
-}
-interface Job { id: string; server: string; action: string; state: string; stage: string }
-type Recipe = LaunchRecipePlan & { native: NativePlacement };
-
-export function hostMigration(id: string): Migration | undefined {
-    return db().prepare('SELECT * FROM ec_host_migrations WHERE deployment_id=?').get(id) as Migration | undefined;
-}
+export { captureMoveRecipe } from './move-recipes.js';
+export { hostMigration, isRebalanceMigration, processMigrationRemoval, requestInstanceMove } from './migrations.js';
 
 /** Drain requests survive controller restarts and never make a source slot available early. */
 export function reconcileHostDrains(): void {
@@ -47,7 +31,8 @@ export function reconcileHostDrains(): void {
                 if (!server?.map_slug || activeBetaRemovalBlocker(instance.server)) continue;
                 // An explicit cancellation withdraws this instance's current drain attempt.
                 const latest = db().prepare(`SELECT d.state FROM ec_host_migrations m JOIN server_deployments d ON d.id=m.deployment_id
-                    WHERE m.instance_id=? AND d.server_deleted_at IS NULL ORDER BY d.rowid DESC LIMIT 1`).get(instance.id) as { state: string } | undefined;
+                    WHERE m.instance_id=? AND m.source_host_id=? AND d.triggered_by='host-drain'
+                    AND d.server_deleted_at IS NULL ORDER BY d.rowid DESC LIMIT 1`).get(instance.id, host.id) as { state: string } | undefined;
                 if (latest?.state === 'cancelled') continue;
                 if (db().prepare(`SELECT 1 FROM server_deployments WHERE server=? AND server_deleted_at IS NULL
                     AND state IN ('queued','waiting','running','awaiting-health','interrupted')`).get(instance.server)) continue;
@@ -69,94 +54,3 @@ export function reconcileHostDrains(): void {
         })();
     }
 }
-
-/** Reserve a destination, remove the source once, then hand creation to the normal health-verified rebuild queue. */
-export async function processMigrationRemoval(job: Job, instance: NativeInstance, migration: Migration,
-    eligibility: () => string | null): Promise<void> {
-    if (provisioningCancelled(job.id)) return;
-    const hold = (message: string) => { if (!provisioningCancelled(job.id)) deploymentStage(job.id, message, { blocker: { code: 'native-host', message, automatic: true } }); };
-    if (migration.remove_operation_id) {
-        const operation = getHostOperationStatus(migration.remove_operation_id);
-        if (operation?.state === 'failed' && Date.now() - Date.parse(operation.updatedAt) >= 60_000) {
-            // A failed receipt certifies no mutation began. Unknown results keep
-            // their reservation and original operation ID until reconciled.
-            db().transaction(() => {
-                db().prepare('DELETE FROM ec_host_ports WHERE host_id=? AND instance_id=?').run(migration.target_host_id, instance.id);
-                db().prepare(`UPDATE ec_host_migrations SET target_host_id=NULL,target_port=NULL,target_bind=NULL,
-                    remove_operation_id=NULL WHERE deployment_id=?`).run(job.id);
-                db().prepare("UPDATE server_deployments SET state='waiting',submitted_at=NULL WHERE id=?").run(job.id);
-                scheduleAvailabilityRefresh();
-            })();
-            hold('Source removal failed before mutation; checking destination capacity again before retrying.');
-            return;
-        }
-        if (operation?.state !== 'completed') {
-            hold(operation?.error ?? 'Waiting for confirmed source removal. The original agent operation will reconcile.');
-            return;
-        }
-        db().transaction(() => {
-            if (hostMigration(job.id)?.removed_at) return;
-            const current = nativeInstance(job.server);
-            if (current?.id !== instance.id || current.generation !== instance.generation || current.host_id !== migration.source_host_id) throw new Error('Migration source changed.');
-            const target = getHost(migration.target_host_id!);
-            if (!target) throw new Error('Migration destination is unavailable.');
-            const recipe = recipeForDeployment(job.id)!.recipe as Recipe;
-            recipe.native = { libraryId: instance.library_id, image: instance.image, hostId: target.id,
-                bindAddress: migration.target_bind!, publicAddress: target.publicAddress };
-            const stored = recipeStorage(recipe);
-            db().prepare('UPDATE server_launch_recipes SET recipe=?,recipe_summary=? WHERE deployment_id=?').run(stored.recipe, stored.recipeSummary, job.id);
-            db().prepare('DELETE FROM ec_host_ports WHERE host_id=? AND instance_id=?').run(instance.host_id, instance.id);
-            db().prepare(`UPDATE native_instances SET host_id=?,generation=generation+1,bind_address=?,port=?,public_address=?,public_port=NULL WHERE id=?`)
-                .run(target.id, migration.target_bind, migration.target_port, target.publicAddress, instance.id);
-            db().prepare('UPDATE native_provisioning SET generation=generation+1 WHERE deployment_id=?').run(job.id);
-            db().prepare('UPDATE server_creation_jobs SET port=? WHERE deployment_id=?').run(migration.target_port, job.id);
-            db().prepare(`UPDATE servers SET game_port=?,container_state='starting',container_checked_at=NULL,
-                last_players=NULL,last_roster=NULL,last_seen_at=NULL,last_error=NULL,rcon_failures=0,active_since=NULL,inactive_since=NULL,
-                game_started_at=NULL,game_ended_at=NULL WHERE name=?`).run(migration.target_port, job.server);
-            db().prepare('UPDATE ec_host_migrations SET removed_at=? WHERE deployment_id=?').run(operation.updatedAt, job.id);
-            db().prepare("UPDATE server_deployments SET state='waiting',submitted_at=NULL WHERE id=?").run(job.id);
-            deploymentStage(job.id, `Source removed; rebuilding on ${target.name}`, { phase: 'startup-queue', blocker: null });
-            scheduleAvailabilityRefresh();
-        })();
-        return;
-    }
-    if (!getHost(instance.host_id)?.connected) { hold('Waiting for the source host to reconnect.'); return; }
-    const reason = eligibility() ?? activeBetaRemovalBlocker(job.server);
-    if (reason) { hold(reason); return; }
-    if (!getServerKey(job.server)) { hold('Save a server key before migrating this instance.'); return; }
-    const options = { libraryId: instance.library_id, image: instance.image, thorough: true, dispatch: true };
-    // No source mutation is submitted unless another host can take this instance.
-    choosePlacement(options);
-    const health = await nativeRequest<{ state: string }>(instance, 'instance.inspect');
-    if (provisioningCancelled(job.id)) return;
-    if (!['stopped','exited','created','dead','missing'].includes(health.state)) {
-        const occupancy = await probePlayerEvidence(job.server);
-        if (provisioningCancelled(job.id)) return;
-        if (occupancy.players !== 0) { hold(occupancy.players === null ? 'Waiting for a fresh local player count.' : `Waiting for ${occupancy.players} player(s) to leave.`); return; }
-    }
-    db().transaction(() => {
-        const state = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined;
-        if (!state || !['waiting','queued','interrupted'].includes(state.state) || hostMigration(job.id)?.remove_operation_id) return;
-        const current = nativeInstance(job.server);
-        if (current?.id !== instance.id || current.generation !== instance.generation || getHost(instance.host_id)?.mode !== 'draining') return;
-        const blocker = eligibility() ?? activeBetaRemovalBlocker(job.server);
-        if (blocker) { hold(blocker); return; }
-        const removing = db().prepare("SELECT COUNT(*) AS n FROM ec_host_operations WHERE kind='instance.remove' AND state IN ('pending','accepted','running','unknown')").get() as { n: number };
-        if (removing.n >= fleetSettings().destroyConcurrency) { hold('Waiting for destruction capacity.'); return; }
-        const { host, port } = choosePlacement(options);
-        const capacityBlocker = concurrencyBlocker(job.id, 'rebuild', instance.host_id, true) ?? concurrencyBlocker(job.id, 'rebuild', host.id);
-        if (capacityBlocker) { hold(capacityBlocker); return; }
-        reserveHostPort(host.id, instance.id, { port, bindAddress: host.bindAddress, protocol: 'udp' });
-        reserveHostPort(host.id, instance.id, { port, bindAddress: host.bindAddress, protocol: 'tcp' });
-        const attempt = (db().prepare(`SELECT COUNT(*) AS n FROM ec_host_operations
-            WHERE host_id=? AND instance_id=? AND kind='instance.remove' AND idempotency_key LIKE ?`)
-            .get(instance.host_id, instance.id, `${job.id}:migrate-remove:%`) as { n: number }).n;
-        const operation = queueHostOperation(instance.host_id, 'instance.remove', {}, {
-            instanceId: instance.id, generation: instance.generation, idempotencyKey: `${job.id}:migrate-remove:${attempt}`
-        });
-        db().prepare('UPDATE ec_host_migrations SET target_host_id=?,target_port=?,target_bind=?,remove_operation_id=? WHERE deployment_id=?')
-            .run(host.id, port, host.bindAddress, operation.id, job.id);
-        db().prepare("UPDATE server_deployments SET state='running',submitted_at=? WHERE id=?").run(new Date().toISOString(), job.id);
-        deploymentStage(job.id, `Capacity reserved on ${host.name}; waiting for source removal`, { phase: 'container-action', blocker: null });
-    })();
-}

src/lib/server/provisioning/launch-preparation.ts

@@ -15,6 +15,8 @@ export interface NativeLaunchInput {
     rconPassword: string;
     rebuild: boolean;
     safetyOverride?: boolean;
+    /** Rebalance replays recorded deployed bytes instead of current promoted files. */
+    frozenRecipeId?: string;
 }
 
 export interface PreparedNativeLaunch {

src/lib/server/provisioning/launch-worker.ts

@@ -2,7 +2,7 @@ import { parentPort } from 'node:worker_threads';
 import { posix } from 'node:path';
 import { db } from '../db/index.js';
 import { relDestForKind } from '../config.js';
-import { captureRecipe, summarizeRecipe } from '../fleet/launch-recipes.js';
+import { captureRecipe, recipeForDeployment, summarizeRecipe, type LaunchRecipe } from '../fleet/launch-recipes.js';
 import { sha256Buffer } from '../fleet/files.js';
 import type { NativeLaunchInput } from './launch-preparation.js';
 import { choosePlacement, type PlacementOptions } from '../hosts/capacity.js';
@@ -17,12 +17,26 @@ parentPort!.on('message', (message: { id: number; kind: 'launch'; input: NativeL
         }
         const input = message.input;
         // All file versions and composed configuration belong to one snapshot.
-        const recipe = db().transaction(() => ({ ...captureRecipe(input.game, input.mapSlug, input.rconPassword),
-            native: input.native, image: input.instance.image }))();
+        const { recipe, password } = db().transaction(() => {
+            if (!input.frozenRecipeId) return { recipe: { ...captureRecipe(input.game, input.mapSlug, input.rconPassword),
+                native: input.native, image: input.instance.image }, password: input.rconPassword };
+            const stored = recipeForDeployment(input.frozenRecipeId)?.recipe;
+            const migration = db().prepare('SELECT runtime_password FROM ec_host_migrations WHERE deployment_id=?')
+                .get(input.frozenRecipeId) as { runtime_password: string | null } | undefined;
+            if (!stored || !migration?.runtime_password || stored.files.some(file => file.content === undefined)) {
+                throw new Error('The preserved deployed recipe or runtime credentials are unavailable.');
+            }
+            for (const file of stored.files) {
+                const bytes = Buffer.from(file.content!, 'base64');
+                if (bytes.length !== file.size || sha256Buffer(bytes) !== file.sha256) throw new Error('Preserved deployed file verification failed.');
+            }
+            return { recipe: { ...stored, native: input.native, image: input.instance.image } as LaunchRecipe,
+                password: migration.runtime_password };
+        })();
         const instance = input.instance;
         const payload = {
             name: instance.server, game: recipe.game, libraryId: instance.library_id, image: instance.image,
-            port: instance.port, bindAddress: instance.bind_address, rconPassword: input.rconPassword,
+            port: instance.port, bindAddress: instance.bind_address, rconPassword: password,
             env: { PLUTAINER_SERVER_NAME: instance.server, PLUTAINER_PORT: String(instance.port),
                 PLUTAINER_GAME: recipe.game === 't6' ? 't6zm' : 't5sp', PLUTAINER_CONFIG_FILE: recipe.configFile,
                 PLUTAINER_MAP_ROTATE: 'false',

src/lib/server/provisioning/migrations.ts

@@ -0,0 +1,183 @@
+import { concurrencyBlocker, preparingJobs, committedRebalanceStartup } from './concurrency.js';
+import { db } from '../db/index.js';
+import { provisioningCancelled } from './cancellation.js';
+import { nativeInstance, nativeRequest, runtimePasswordForServer, type NativeInstance } from '../backends/native.js';
+import { getHost, getHostOperationStatus, queueHostOperation, reserveHostPort } from '../hosts/index.js';
+import { choosePlacement } from '../hosts/capacity.js';
+import { scheduleAvailabilityRefresh } from '../hosts/availability.js';
+import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
+import { getServerKey } from '../fleet/server-keys.js';
+import { recipeForDeployment, recipeStorage, recipeSummaryForDeployment, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
+import { captureMoveRecipe, moveRecipeMatches, moveRecipeStorage } from './move-recipes.js';
+import { rebalanceAdmission } from '../rebalancing/admission.js';
+import { probePlayerEvidence } from '../fleet/status-evidence.js';
+import { beginDeployment, deploymentStage } from './history.js';
+import { fleetSettings } from '../settings.js';
+import type { NativePlacement } from '../../hosts.js';
+
+interface Migration {
+    deployment_id: string; source_host_id: string; instance_id: string;
+    target_host_id: string | null; target_port: number | null; target_bind: string | null;
+    remove_operation_id: string | null; removed_at: string | null;
+}
+interface Job { id: string; server: string; action: string; state: string; stage: string }
+type Recipe = LaunchRecipePlan & { native: NativePlacement };
+export { captureMoveRecipe } from './move-recipes.js';
+
+export function isRebalanceMigration(id: string): boolean {
+    return !!db().prepare("SELECT 1 FROM server_deployments WHERE id=? AND triggered_by LIKE 'rebalance:%'").get(id);
+}
+
+/** Queue a move to one explicit host; its source remains active throughout. */
+export function requestInstanceMove(serverName: string, targetHostId: string,
+    options: { triggeredBy: string; recipe?: LaunchRecipePlan }): string {
+    return db().transaction(() => {
+        const instance = nativeInstance(serverName);
+        const server = getServer(serverName);
+        if (!instance || !server?.map_slug) throw new Error('The native instance is unavailable.');
+        if (instance.host_id === targetHostId) throw new Error('Choose a different destination host.');
+        if (getHost(instance.host_id)?.mode !== 'active') throw new Error('The source host must be active.');
+        if (!options.triggeredBy.startsWith('rebalance:')) throw new Error('An instance move requires a rebalance run.');
+        const blocker = activeBetaRemovalBlocker(serverName);
+        if (blocker) throw new Error(blocker);
+        if (!getServerKey(serverName)) throw new Error('Save a server key before moving this instance.');
+        if (db().prepare(`SELECT 1 FROM server_deployments WHERE server=? AND server_deleted_at IS NULL
+            AND state IN ('queued','waiting','running','awaiting-health','interrupted')`).get(serverName)) {
+            throw new Error('Another provisioning operation is active for this instance.');
+        }
+        const recipe = options.recipe ?? captureMoveRecipe(serverName);
+        if (recipe.game !== server.game || recipe.mapSlug !== server.map_slug || recipe.image !== instance.image
+            || !moveRecipeMatches(serverName, recipe)) throw new Error('The deployed files changed; generate a fresh rebalance preview.');
+        const password = runtimePasswordForServer(serverName);
+        if (!password) throw new Error('The deployed runtime credentials are unavailable.');
+        choosePlacement({ hostId: targetHostId, excludeHostIds: [instance.host_id], libraryId: instance.library_id,
+            image: instance.image, thorough: true, dispatch: true });
+        const id = beginDeployment(serverName, 'rebuild', options.triggeredBy);
+        db().prepare("UPDATE server_deployments SET state='waiting' WHERE id=?").run(id);
+        db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,NULL,?,?,?)')
+            .run(id, serverName, 'zombie', recipe.mapSlug, recipe.expectedMap);
+        const stored = moveRecipeStorage({ ...recipe, native: { libraryId: instance.library_id, image: instance.image, hostId: targetHostId } });
+        db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary) VALUES(?,?,?)').run(id, stored.recipe, stored.recipeSummary);
+        db().prepare('INSERT INTO ec_host_migrations(deployment_id,source_host_id,instance_id,runtime_password) VALUES(?,?,?,?)')
+            .run(id, instance.host_id, instance.id, password);
+        deploymentStage(id, 'Rebalance queued; waiting to confirm an empty lobby and destination capacity', { phase: 'eligibility', recipeStatus: 'queued' });
+        return id;
+    })();
+}
+
+export function hostMigration(id: string): Migration | undefined {
+    return db().prepare('SELECT * FROM ec_host_migrations WHERE deployment_id=?').get(id) as Migration | undefined;
+}
+
+/** Reserve a destination, remove the source once, then hand creation to the normal health-verified rebuild queue. */
+export async function processMigrationRemoval(job: Job, instance: NativeInstance, migration: Migration,
+    eligibility: () => string | null): Promise<void> {
+    if (provisioningCancelled(job.id)) return;
+    const hold = (message: string) => { if (!provisioningCancelled(job.id)) deploymentStage(job.id, message, { blocker: { code: 'native-host', message, automatic: true } }); };
+    if (migration.remove_operation_id) {
+        const operation = getHostOperationStatus(migration.remove_operation_id);
+        if (operation?.state === 'failed' && Date.now() - Date.parse(operation.updatedAt) >= 60_000) {
+            // A failed receipt certifies no mutation began. Unknown results keep
+            // their reservation and original operation ID until reconciled.
+            db().transaction(() => {
+                db().prepare('DELETE FROM ec_host_ports WHERE host_id=? AND instance_id=?').run(migration.target_host_id, instance.id);
+                db().prepare(`UPDATE ec_host_migrations SET target_host_id=NULL,target_port=NULL,target_bind=NULL,
+                    remove_operation_id=NULL WHERE deployment_id=?`).run(job.id);
+                db().prepare("UPDATE server_deployments SET state='waiting',submitted_at=NULL WHERE id=?").run(job.id);
+                scheduleAvailabilityRefresh();
+            })();
+            hold('Source removal failed before mutation; checking destination capacity again before retrying.');
+            return;
+        }
+        if (operation?.state !== 'completed') {
+            hold(operation?.error ?? 'Waiting for confirmed source removal. The original agent operation will reconcile.');
+            return;
+        }
+        db().transaction(() => {
+            if (hostMigration(job.id)?.removed_at) return;
+            const current = nativeInstance(job.server);
+            if (current?.id !== instance.id || current.generation !== instance.generation || current.host_id !== migration.source_host_id) throw new Error('Migration source changed.');
+            const target = getHost(migration.target_host_id!);
+            if (!target) throw new Error('Migration destination is unavailable.');
+            const rebalance = isRebalanceMigration(job.id);
+            const recipe = (rebalance ? { format: 1, ...recipeSummaryForDeployment(job.id)!.recipe }
+                : recipeForDeployment(job.id)!.recipe) as Recipe;
+            recipe.native = { libraryId: instance.library_id, image: instance.image, hostId: target.id,
+                bindAddress: migration.target_bind!, publicAddress: target.publicAddress };
+            const stored = rebalance ? moveRecipeStorage(recipe) : recipeStorage(recipe);
+            db().prepare('UPDATE server_launch_recipes SET recipe=?,recipe_summary=? WHERE deployment_id=?').run(stored.recipe, stored.recipeSummary, job.id);
+            db().prepare('DELETE FROM ec_host_ports WHERE host_id=? AND instance_id=?').run(instance.host_id, instance.id);
+            db().prepare(`UPDATE native_instances SET host_id=?,generation=generation+1,bind_address=?,port=?,public_address=?,public_port=NULL WHERE id=?`)
+                .run(target.id, migration.target_bind, migration.target_port, target.publicAddress, instance.id);
+            db().prepare('UPDATE native_provisioning SET generation=generation+1 WHERE deployment_id=?').run(job.id);
+            db().prepare('UPDATE server_creation_jobs SET port=? WHERE deployment_id=?').run(migration.target_port, job.id);
+            db().prepare(`UPDATE servers SET game_port=?,container_state='starting',container_checked_at=NULL,
+                last_players=NULL,last_roster=NULL,last_seen_at=NULL,last_error=NULL,rcon_failures=0,active_since=NULL,inactive_since=NULL,
+                game_started_at=NULL,game_ended_at=NULL WHERE name=?`).run(migration.target_port, job.server);
+            db().prepare('UPDATE ec_host_migrations SET removed_at=? WHERE deployment_id=?').run(operation.updatedAt, job.id);
+            db().prepare("UPDATE server_deployments SET state='waiting',submitted_at=NULL WHERE id=?").run(job.id);
+            deploymentStage(job.id, `Source removed; rebuilding on ${target.name}`, { phase: 'startup-queue', blocker: null });
+            scheduleAvailabilityRefresh();
+        })();
+        return;
+    }
+    if (!getHost(instance.host_id)?.connected) { hold('Waiting for the source host to reconnect.'); return; }
+    const reason = rebalanceAdmission(job.id) ?? eligibility() ?? activeBetaRemovalBlocker(job.server);
+    if (reason) { hold(reason); return; }
+    if (!getServerKey(job.server)) { hold('Save a server key before migrating this instance.'); return; }
+    const rebalance = isRebalanceMigration(job.id);
+    const recipe = rebalance ? recipeSummaryForDeployment(job.id)!.recipe : null;
+    if (rebalance && !moveRecipeMatches(job.server, { format: 1, ...recipe! })) {
+        hold('The deployed files changed; stop this run and generate a fresh rebalance preview.'); return;
+    }
+    const options = { libraryId: instance.library_id, image: instance.image, thorough: true, dispatch: true,
+        excludeHostIds: [instance.host_id], ...(rebalance ? { hostId: recipe!.native!.hostId } : {}) };
+    // No source mutation is submitted unless another host can take this instance.
+    choosePlacement(options);
+    const health = await nativeRequest<{ state: string; healthy?: boolean }>(instance, 'instance.inspect');
+    if (provisioningCancelled(job.id)) return;
+    if (rebalance && (health.state !== 'running' || health.healthy !== true)) {
+        hold('The source lobby is no longer healthy and running. Waiting for current health evidence.'); return;
+    }
+    if (!['stopped','exited','created','dead','missing'].includes(health.state)) {
+        const occupancy = await probePlayerEvidence(job.server);
+        if (provisioningCancelled(job.id)) return;
+        if (occupancy.players !== 0) { hold(occupancy.players === null ? 'Waiting for a fresh local player count.' : `Waiting for ${occupancy.players} player(s) to leave.`); return; }
+    }
+    db().transaction(() => {
+        const state = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined;
+        if (!state || !['waiting','queued','interrupted'].includes(state.state) || hostMigration(job.id)?.remove_operation_id) return;
+        const current = nativeInstance(job.server);
+        if (current?.id !== instance.id || current.generation !== instance.generation
+            || getHost(instance.host_id)?.mode !== (rebalance ? 'active' : 'draining')) return;
+        const blocker = rebalanceAdmission(job.id) ?? eligibility() ?? activeBetaRemovalBlocker(job.server);
+        if (blocker) { hold(blocker); return; }
+        if (rebalance && !moveRecipeMatches(job.server, { format: 1, ...recipe! })) {
+            hold('The deployed files changed; stop this run and generate a fresh rebalance preview.'); return;
+        }
+        if (rebalance) {
+            const startups = db().prepare(`SELECT COUNT(*) AS n FROM (
+                SELECT id FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL
+                    AND action IN ('create','rebuild') AND (state IN ('running','awaiting-health') OR ${committedRebalanceStartup})
+                UNION SELECT value FROM json_each(?) WHERE value!=?)`).get(job.id, JSON.stringify([...preparingJobs]), job.id) as { n: number };
+            if (startups.n >= fleetSettings().createConcurrency) { hold('Waiting for startup capacity before moving this lobby.'); return; }
+        }
+        const removing = db().prepare("SELECT COUNT(*) AS n FROM ec_host_operations WHERE kind='instance.remove' AND state IN ('pending','accepted','running','unknown')").get() as { n: number };
+        if (removing.n >= fleetSettings().destroyConcurrency) { hold('Waiting for destruction capacity.'); return; }
+        const { host, port } = choosePlacement(options);
+        const capacityBlocker = concurrencyBlocker(job.id, 'rebuild', instance.host_id, true) ?? concurrencyBlocker(job.id, 'rebuild', host.id);
+        if (capacityBlocker) { hold(capacityBlocker); return; }
+        reserveHostPort(host.id, instance.id, { port, bindAddress: host.bindAddress, protocol: 'udp' });
+        reserveHostPort(host.id, instance.id, { port, bindAddress: host.bindAddress, protocol: 'tcp' });
+        const attempt = (db().prepare(`SELECT COUNT(*) AS n FROM ec_host_operations
+            WHERE host_id=? AND instance_id=? AND kind='instance.remove' AND idempotency_key LIKE ?`)
+            .get(instance.host_id, instance.id, `${job.id}:migrate-remove:%`) as { n: number }).n;
+        const operation = queueHostOperation(instance.host_id, 'instance.remove', {}, {
+            instanceId: instance.id, generation: instance.generation, idempotencyKey: `${job.id}:migrate-remove:${attempt}`
+        });
+        db().prepare('UPDATE ec_host_migrations SET target_host_id=?,target_port=?,target_bind=?,remove_operation_id=? WHERE deployment_id=?')
+            .run(host.id, port, host.bindAddress, operation.id, job.id);
+        db().prepare("UPDATE server_deployments SET state='running',submitted_at=? WHERE id=?").run(new Date().toISOString(), job.id);
+        deploymentStage(job.id, `Capacity reserved on ${host.name}; waiting for source removal`, { phase: 'container-action', blocker: null });
+    })();
+}

src/lib/server/provisioning/move-recipes.ts

@@ -0,0 +1,59 @@
+import { db } from '../db/index.js';
+import { nativeInstance, runtimePasswordForServer } from '../backends/native.js';
+import { getServer } from '../fleet/servers.js';
+import { listServerScripts } from '../fleet/inventory.js';
+import { recipeStorage, recipeSummaryForDeployment, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
+
+/** Preserve the recorded deployed inventory, including changes made after launch. */
+export function captureMoveRecipe(server: string): LaunchRecipePlan {
+    const instance = nativeInstance(server);
+    const row = getServer(server);
+    if (!instance || !row?.map_slug) throw new Error('A live native instance and saved map are required before moving.');
+    if (!runtimePasswordForServer(server)) throw new Error('The deployed runtime credentials are unavailable.');
+    const launch = db().prepare(`SELECT d.id FROM server_deployments d
+        JOIN native_provisioning n ON n.deployment_id=d.id
+        JOIN server_launch_recipes r ON r.deployment_id=d.id
+        WHERE n.instance_id=? AND d.state IN ('done','warning') AND d.server_deleted_at IS NULL
+        ORDER BY d.rowid DESC LIMIT 1`).get(instance.id) as { id: string } | undefined;
+    const previous = launch && recipeSummaryForDeployment(launch.id)?.recipe;
+    if (!previous || previous.game !== row.game || previous.mapSlug !== row.map_slug) {
+        throw new Error('The deployed launch recipe is unavailable or differs from the saved map.');
+    }
+    const inventory = listServerScripts(server);
+    if (!inventory.length) throw new Error('The deployed file inventory is unavailable.');
+    const files = inventory.map(file => {
+        const before = previous.files.find(item => item.path === file.rel_path);
+        const version = file.script_version_id === null ? undefined : db().prepare(`SELECT v.version,v.kind,s.name
+            FROM script_versions v JOIN scripts s ON s.id=v.script_id WHERE v.id=?`).get(file.script_version_id) as
+            { version: number; kind: string; name: string } | undefined;
+        const kind = version?.kind === 'baseline' ? 'config' : version?.kind ?? before?.kind;
+        if (!['script', 'plugin', 'config'].includes(kind ?? '')) throw new Error(`The deployed file type is unavailable for ${file.rel_path}.`);
+        if (!db().prepare('SELECT 1 FROM file_blobs WHERE sha256=? AND size=?').get(file.sha256, file.size)) {
+            throw new Error(`The deployed bytes are unavailable for ${file.rel_path}.`);
+        }
+        return { name: version?.name ?? before?.name ?? file.rel_path, kind: kind as 'script' | 'plugin' | 'config',
+            path: file.rel_path, versionId: file.script_version_id, version: version?.version ?? null,
+            sha256: file.sha256, size: file.size };
+    });
+    if (!files.some(file => file.path === 'electric_cherry_map.cfg' && file.kind === 'config')) {
+        throw new Error('The deployed map configuration is unavailable.');
+    }
+    return { format: 1, game: previous.game, mapSlug: previous.mapSlug, mapName: previous.mapName,
+        expectedMap: previous.expectedMap, configFile: previous.configFile,
+        configSource: 'Preserved deployed files', capturedAt: new Date().toISOString(), image: instance.image,
+        native: { libraryId: instance.library_id, image: instance.image }, files };
+}
+
+/** Immutable hash references keep deployed bytes alive without decoding plugins on the controller. */
+export function moveRecipeStorage(recipe: LaunchRecipePlan): ReturnType<typeof recipeStorage> {
+    const metadata = { ...recipe, files: recipe.files.map(({ content: _content, ...file }) => file) };
+    const stored = recipeStorage(metadata);
+    return { recipeSummary: stored.recipeSummary,
+        recipe: JSON.stringify({ ...metadata, files: metadata.files.map(file => ({ ...file, contentHash: file.sha256 })) }) };
+}
+
+export function moveRecipeMatches(server: string, recipe: LaunchRecipePlan): boolean {
+    const inventory = listServerScripts(server);
+    return inventory.length === recipe.files.length && inventory.every(file => recipe.files.some(saved =>
+        saved.path === file.rel_path && saved.sha256 === file.sha256 && saved.size === file.size));
+}

src/lib/server/provisioning/native.ts

@@ -2,7 +2,7 @@ import { assertScalingOwnership } from '../autoscaling/ownership.js';
 import { removalAuthorization } from '../autoscaling/authorization.js';
 import { scalingIntent } from '../autoscaling/admission.js';
 import { putFileBlob, getFileBlob, blobSize } from '../db/file-blobs.js';
-import { concurrencyBlocker, preparingJobs } from './concurrency.js';
+import { concurrencyBlocker, preparingJobs, committedRebalanceStartup } from './concurrency.js';
 import { isIP } from 'node:net';
 import { setImmediate as yieldToController } from 'node:timers/promises';
 import { choosePlacement, availableHostPort, type PlacementOptions } from '../hosts/capacity.js';
@@ -15,7 +15,7 @@ import { config, relDestForKind, serverDir } from '../config.js';
 import { nativeInstance, nativeRequest, unplacedNativeCreation, type NativeInstance } from '../backends/native.js';
 import { getHost, listHosts, getHostOperationStatus, queueHostOperation, queueHostOperationSerialized, releaseHostPorts, reserveHostPort } from '../hosts/index.js';
 import { newestReadyLibraryId, assertReadyLibrary } from '../game-files/index.js';
-import { blueprintRecipe, captureRecipePlan, recipeForDeployment, recipeStorage, summarizeRecipe, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
+import { blueprintRecipe, captureRecipePlan, recipeForDeployment, recipeSummaryForDeployment, recipeStorage, summarizeRecipe, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
 import { prepareNativeLaunch, evaluateNativePlacement, type PreparedNativeLaunch } from './launch-preparation.js';
 import { getServerKey, reserveServerKey, releaseServerKey, availableServerKeyCount } from '../fleet/server-keys.js';
 import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
@@ -36,7 +36,8 @@ import type { LaunchPreview } from '../../deployment-plan.js';
 import type { ProvisioningAction } from '../../provisioning.js';
 import { hostAgentUpdatesEnabled } from '../../hosts.js';
 import { cancelCreation, finishCreationCancellation } from './cancel-creation.js';
-import { hostMigration, processMigrationRemoval, reconcileHostDrains } from './host-drain.js';
+import { reconcileHostDrains } from './host-drain.js';
+import { hostMigration, isRebalanceMigration, processMigrationRemoval } from './migrations.js';
 import { safetyOverride, safetyOverrideCapability } from './safety-override.js';
 
 export type NativeSelection = import('../../hosts.js').NativePlacement;
@@ -390,7 +391,12 @@ export async function processNativeProvisioning(eligibility: (operation: { id: s
 const activeJobs = new Map<string, Promise<void>>();
 // Preparation owns startup capacity until its command is committed or discarded.
 
-function otherPreparations(id: string): number { return preparingJobs.size - Number(preparingJobs.has(id)); }
+function otherPreparations(id: string): number {
+    if (!preparingJobs.size) return 0;
+    const reserved = new Set((db().prepare(`SELECT id FROM server_deployments WHERE id IN (SELECT value FROM json_each(?))
+        AND ${committedRebalanceStartup}`).all(JSON.stringify([...preparingJobs])) as { id: string }[]).map(row => row.id));
+    return [...preparingJobs].filter(other => other !== id && !reserved.has(other)).length;
+}
 
 let launchCommit: Promise<void> = Promise.resolve();
 function commitPreparedLaunch(commit: () => void): Promise<void> {
@@ -469,7 +475,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
         }
         const settings = fleetSettings();
         const active = db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL
-            AND action IN ('create','rebuild') AND state IN ('running','awaiting-health')`).get(job.id) as { n: number };
+            AND action IN ('create','rebuild') AND (state IN ('running','awaiting-health') OR ${committedRebalanceStartup})`).get(job.id) as { n: number };
         if (active.n + otherPreparations(job.id) >= settings.createConcurrency) { hold(job, 'Waiting for startup capacity.'); return; }
         const capacityBlocker = concurrencyBlocker(job.id, job.action);
         if (capacityBlocker) { hold(job, capacityBlocker); return; }
@@ -645,7 +651,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
     if (capacityBlocker) { hold(job, capacityBlocker); return; }
     const settings = fleetSettings();
     const count = (actions: string[]) => (db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id != ? AND server_deleted_at IS NULL
-        AND action IN (${actions.map(() => '?').join(',')}) AND state IN ('running','awaiting-health')`).get(job.id, ...actions) as { n: number }).n;
+        AND action IN (${actions.map(() => '?').join(',')}) AND (state IN ('running','awaiting-health') OR ${committedRebalanceStartup})`).get(job.id, ...actions) as { n: number }).n;
     if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) + otherPreparations(job.id) >= settings.createConcurrency) {
         deploymentStage(job.id, 'Waiting for startup capacity.', { phase: 'startup-queue',
             blocker: { code: 'startup-capacity', message: 'Waiting for startup capacity.', automatic: true } }); return;
@@ -694,13 +700,15 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             preparingJobs.add(job.id);
             preparedKey = getServerKey(job.server) ?? null;
             if (!preparedKey) throw new Error('The saved server key was removed.');
-            const prior = recipeForDeployment(job.id)!.recipe as NativeRecipe;
+            const frozenRecipeId = migration && isRebalanceMigration(job.id) ? job.id : undefined;
+            const prior = (frozenRecipeId ? { format: 1, ...recipeSummaryForDeployment(job.id)!.recipe }
+                : recipeForDeployment(job.id)!.recipe) as NativeRecipe;
             deploymentStage(job.id, 'Preparing launch files for the host agent', { phase: 'prepare',
                 checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } });
             preparedPassword = config.rcon.password || randomBytes(32).toString('hex');
             prepared = await prepareNativeLaunch({ game: prior.game, mapSlug: prior.mapSlug, native: prior.native,
                 instance, serverKey: preparedKey, rconPassword: preparedPassword,
-                rebuild: job.action === 'rebuild' && !migration, safetyOverride: overrideSafety });
+                rebuild: job.action === 'rebuild' && !migration, safetyOverride: overrideSafety, frozenRecipeId });
             if (job.action === 'rebuild' && !migration && !await checkOccupancy()) return;
         }
         const commit = db().transaction(() => {
@@ -731,8 +739,10 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             if (job.action === 'destroy' && count(['destroy']) >= fleetSettings().destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; }
             if (prepared) {
                 // Compare the same runtime config bytes while rechecking current promoted files.
-                if (getServerKey(job.server) !== preparedKey || !sameLaunchFiles(prepared.recipe,
-                    captureRecipePlan(prepared.recipe.game, prepared.recipe.mapSlug, preparedPassword))) {
+                const expected = migration && isRebalanceMigration(job.id)
+                    ? { format: 1 as const, ...recipeSummaryForDeployment(job.id)!.recipe }
+                    : captureRecipePlan(prepared.recipe.game, prepared.recipe.mapSlug, preparedPassword);
+                if (getServerKey(job.server) !== preparedKey || !sameLaunchFiles(prepared.recipe, expected)) {
                     hold(job, 'Launch files or credentials changed during preparation; preparing the current selection next pass.'); return;
                 }
                 assertReadyLibrary(instance!.host_id, instance!.library_id);

src/lib/server/rebalancing/admission.ts

@@ -0,0 +1,26 @@
+import { db } from '../db/index.js';
+import type { RebalanceCandidate, RebalanceSettings } from '../../rebalancing.js';
+import { rebalanceMoveBlocker } from './planner.js';
+import { rebalanceSnapshot } from './snapshot.js';
+
+/** A delivered migration must finish even after the operator stops its parent run. */
+export function rebalanceAdmission(operationId: string): string | null {
+    const operation = db().prepare('SELECT triggered_by FROM server_deployments WHERE id=?').get(operationId) as { triggered_by: string } | undefined;
+    if (!operation?.triggered_by.startsWith('rebalance:')) return null;
+    const migration = db().prepare('SELECT remove_operation_id,removed_at FROM ec_host_migrations WHERE deployment_id=?')
+        .get(operationId) as { remove_operation_id: string | null; removed_at: string | null } | undefined;
+    if (migration?.remove_operation_id || migration?.removed_at) return null;
+    const row = db().prepare(`SELECT m.candidate,r.settings,r.state FROM rebalancing_moves m
+        JOIN rebalancing_runs r ON r.id=m.run_id WHERE m.operation_id=? AND m.state='moving'`).get(operationId) as
+        { candidate: string; settings: string; state: string } | undefined;
+    if (!row || row.state !== 'running') return 'Rebalance run is no longer requesting this move.';
+    const settings = JSON.parse(row.settings) as RebalanceSettings;
+    return rebalanceMoveBlocker(rebalanceSnapshot(settings, operationId), JSON.parse(row.candidate) as RebalanceCandidate);
+}
+
+/** Cancelling an individual uncommitted move also withdraws the rest of that run. */
+export function requestRebalanceStopForOperation(operationId: string): void {
+    db().prepare(`UPDATE rebalancing_runs SET state='stopping',updated_at=?,message='Stopping after the current move.'
+        WHERE state='running' AND id IN (SELECT run_id FROM rebalancing_moves WHERE operation_id=?)`)
+        .run(new Date().toISOString(), operationId);
+}

src/lib/server/rebalancing/index.ts

@@ -0,0 +1,214 @@
+import { randomUUID } from 'node:crypto';
+import { db } from '../db/index.js';
+import { getHost } from '../hosts/index.js';
+import { requestInstanceMove } from '../provisioning/migrations.js';
+import { cancelProvisioningJob } from '../provisioning/cancellation.js';
+import { rebalanceSettings, type RebalanceCandidate, type RebalanceMove, type RebalancePreview,
+    type RebalanceReport, type RebalanceRun } from '../../rebalancing.js';
+import { planRebalance, rebalanceMoveBlocker } from './planner.js';
+import { rebalanceHosts, rebalanceSnapshot } from './snapshot.js';
+import { rebalanceAdmission } from './admission.js';
+
+export class RebalanceError extends Error { constructor(message: string, public status = 400) { super(message); } }
+interface RunRow { id: string; settings: string; state: RebalanceRun['state']; created_at: string; updated_at: string; message: string }
+interface MoveRow { id: string; candidate: string; state: RebalanceMove['state']; operation_id: string | null; message: string; updated_at: string }
+const atNow = () => new Date().toISOString();
+const movesFor = (id: string): RebalanceMove[] => (db().prepare('SELECT * FROM rebalancing_moves WHERE run_id=? ORDER BY ordinal').all(id) as MoveRow[])
+    .map(row => ({ ...JSON.parse(row.candidate), id: row.id, state: row.state, operationId: row.operation_id, message: row.message, updatedAt: row.updated_at }));
+function present(row: RunRow): RebalanceRun {
+    return { id: row.id, settings: JSON.parse(row.settings), state: row.state, createdAt: row.created_at,
+        updatedAt: row.updated_at, message: row.message, moves: movesFor(row.id) };
+}
+export function getRebalanceRun(id: string): RebalanceRun | null {
+    const row = db().prepare('SELECT * FROM rebalancing_runs WHERE id=?').get(id) as RunRow | undefined;
+    return row ? present(row) : null;
+}
+export function rebalanceReport(): RebalanceReport {
+    return { hosts: rebalanceHosts(), runs: (db().prepare(`SELECT * FROM rebalancing_runs
+        ORDER BY state IN ('running','stopping') DESC,created_at DESC,rowid DESC LIMIT 20`).all() as RunRow[]).map(present) };
+}
+
+/** Preview has no resource reservations or host side effects. */
+export function previewRebalance(input: unknown): RebalancePreview {
+    const settings = rebalanceSettings(input), now = Date.now();
+    const plan = planRebalance(rebalanceSnapshot(settings), settings);
+    const preview: RebalancePreview = { ...plan, settings, id: randomUUID(), createdAt: new Date(now).toISOString(),
+        expiresAt: new Date(now + 5 * 60_000).toISOString() };
+    db().prepare('INSERT INTO rebalancing_previews(id,preview,expires_at) VALUES(?,?,?)').run(preview.id, JSON.stringify(preview), preview.expiresAt);
+    return preview;
+}
+
+function hasOutstandingRun(): boolean {
+    return !!db().prepare(`SELECT 1 FROM rebalancing_runs WHERE state IN ('running','stopping')
+        UNION ALL SELECT 1 FROM rebalancing_moves m JOIN server_deployments d ON d.id=m.operation_id
+        WHERE d.state IN ('queued','waiting','running','awaiting-health','interrupted')
+        UNION ALL SELECT 1 FROM rebalancing_moves m JOIN native_provisioning n ON n.deployment_id=m.operation_id
+        LEFT JOIN ec_host_migrations migration ON migration.deployment_id=m.operation_id
+        JOIN ec_host_operations o ON o.id IN (n.operation_id,n.stage_operation_id,migration.remove_operation_id)
+        WHERE o.state NOT IN ('completed','failed') LIMIT 1`).get();
+}
+const moveSignature = (moves: RebalanceCandidate[]) => JSON.stringify(moves.map(move =>
+    [move.instanceId, move.generation, move.sourceHostId, move.targetHostId]));
+
+export function startRebalance(previewId: string): RebalanceRun {
+    return db().transaction(() => {
+        const row = db().prepare('SELECT preview,expires_at,run_id FROM rebalancing_previews WHERE id=?').get(previewId) as
+            { preview: string; expires_at: string; run_id: string | null } | undefined;
+        if (!row) throw new RebalanceError('Preview not found. Preview the distribution again.', 404);
+        // A retried request returns the original bounded run, even after its preview expires.
+        if (row.run_id) return getRebalanceRun(row.run_id)!;
+        if (Date.parse(row.expires_at) <= Date.now()) throw new RebalanceError('Preview expired. Preview the distribution again.', 409);
+        if (hasOutstandingRun()) throw new RebalanceError('Finish or stop the current rebalance and settle its operations before starting another.', 409);
+        const preview = JSON.parse(row.preview) as RebalancePreview;
+        const fresh = planRebalance(rebalanceSnapshot(preview.settings), preview.settings);
+        if (!fresh.moves.length || moveSignature(fresh.moves) !== moveSignature(preview.moves)) {
+            throw new RebalanceError('The fleet changed. Preview the distribution again before starting.', 409);
+        }
+        const id = randomUUID(), at = atNow();
+        db().prepare(`INSERT INTO rebalancing_runs(id,settings,state,created_at,updated_at,message)
+            VALUES(?,?,'running',?,?,'Waiting to begin the first move.')`).run(id, JSON.stringify(preview.settings), at, at);
+        for (const [ordinal, move] of fresh.moves.entries()) db().prepare(`INSERT INTO rebalancing_moves
+            (id,run_id,ordinal,server,instance_id,generation,source_host_id,target_host_id,candidate,state,message,updated_at)
+            VALUES(?,?,?,?,?,?,?,?,?,'planned','Waiting for earlier moves.',?)`)
+            .run(randomUUID(), id, ordinal, move.server, move.instanceId, move.generation, move.sourceHostId, move.targetHostId, JSON.stringify(move), at);
+        db().prepare('UPDATE rebalancing_previews SET run_id=? WHERE id=?').run(id, previewId);
+        return getRebalanceRun(id)!;
+    })();
+}
+
+function updateRun(id: string, state: RebalanceRun['state'], message: string): void {
+    db().prepare('UPDATE rebalancing_runs SET state=?,message=?,updated_at=? WHERE id=? AND (state!=? OR message!=?)')
+        .run(state, message, atNow(), id, state, message);
+}
+function updateMove(id: string, state: RebalanceMove['state'], message: string): void {
+    db().prepare('UPDATE rebalancing_moves SET state=?,message=?,updated_at=? WHERE id=? AND (state!=? OR message!=?)')
+        .run(state, message, atNow(), id, state, message);
+}
+function skipRemaining(id: string, reason: string): void {
+    db().prepare("UPDATE rebalancing_moves SET state='skipped',message=?,updated_at=? WHERE run_id=? AND state='planned'").run(reason, atNow(), id);
+}
+function committed(operationId: string): boolean {
+    return !!db().prepare('SELECT 1 FROM ec_host_migrations WHERE deployment_id=? AND (remove_operation_id IS NOT NULL OR removed_at IS NOT NULL)').get(operationId);
+}
+
+export function stopRebalance(id: string): RebalanceRun {
+    return db().transaction(() => {
+        const run = getRebalanceRun(id);
+        if (!run) throw new RebalanceError('Rebalance run not found.', 404);
+        if (!['running', 'stopping'].includes(run.state)) return run;
+        updateRun(id, 'stopping', 'Stopping after the current move.');
+        skipRemaining(id, 'Stopped before this move began.');
+        for (const move of run.moves.filter(move => move.state === 'moving')) {
+            if (move.operationId && !committed(move.operationId)
+                && cancelProvisioningJob(move.operationId, 'Rebalance stopped before source removal.')) {
+                updateMove(move.id, 'skipped', 'Stopped before source removal.');
+            }
+        }
+        if (!movesFor(id).some(move => move.state === 'moving')) updateRun(id, 'stopped', 'Stopped. Completed moves are retained.');
+        return getRebalanceRun(id)!;
+    })();
+}
+
+function refreshedAfterMove(move: RebalanceMove): boolean {
+    return [move.sourceHostId, move.targetHostId].every(id => {
+        const heartbeat = getHost(id)?.lastHeartbeat;
+        return !!heartbeat && Date.parse(heartbeat) > Date.parse(move.updatedAt);
+    });
+}
+
+/** Only a single approved move is admitted; Provisioning owns every host command. */
+function reconcileRun(run: RebalanceRun): boolean {
+    let active = run.moves.find(move => move.state === 'moving');
+    if (active) {
+        const operation = active.operationId && db().prepare(`SELECT d.state,d.stage,d.error,n.verified_at
+            FROM server_deployments d LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE d.id=?`).get(active.operationId) as
+            { state: string; stage: string; error: string | null; verified_at: string | null } | undefined;
+        if (!operation) {
+            updateMove(active.id, 'failed', 'Provisioning operation is unavailable. Inspect the instance before starting another run.');
+            skipRemaining(run.id, 'Stopped after a move could not be reconciled.');
+            updateRun(run.id, 'failed', 'A move could not be reconciled.'); return false;
+        }
+        if (operation.state === 'cancelled') {
+            updateMove(active.id, 'skipped', operation.stage);
+            if (run.state === 'running') updateRun(run.id, 'stopping', 'An operation was cancelled; stopping the remaining moves.');
+        } else if (['done', 'warning'].includes(operation.state) && operation.verified_at) {
+            updateMove(active.id, 'done', 'Healthy on the destination host.');
+        } else if (['failed', 'warning', 'interrupted'].includes(operation.state)) {
+            updateMove(active.id, 'failed', operation.error ?? operation.stage);
+            skipRemaining(run.id, 'Stopped after a move needs attention.');
+            updateRun(run.id, 'failed', 'A move needs attention. Use its Provisioning operation to recover.'); return false;
+        } else {
+            if (active.operationId && !committed(active.operationId)) {
+                const reason = run.state === 'stopping' ? 'Rebalance stopped before source removal.' : rebalanceAdmission(active.operationId);
+                if (reason && cancelProvisioningJob(active.operationId, reason, { preserveRebalanceRun: true })) {
+                    updateMove(active.id, 'skipped', reason);
+                } else {
+                    updateMove(active.id, 'moving', operation.stage);
+                    updateRun(run.id, run.state, operation.stage); return false;
+                }
+            } else {
+                updateMove(active.id, 'moving', operation.stage);
+                updateRun(run.id, run.state, run.state === 'stopping' ? 'Finishing the current move before stopping.' : operation.stage); return false;
+            }
+        }
+        run = getRebalanceRun(run.id)!;
+        active = undefined;
+    }
+    if (run.state === 'stopping') {
+        skipRemaining(run.id, 'Stopped before this move began.');
+        updateRun(run.id, 'stopped', 'Stopped. Completed moves are retained.'); return false;
+    }
+    if (!run.moves.some(move => move.state === 'planned')) {
+        const done = run.moves.filter(move => move.state === 'done').length;
+        const skipped = run.moves.filter(move => move.state === 'skipped').length;
+        updateRun(run.id, 'done', `Finished: ${done} moved${skipped ? `, ${skipped} skipped` : ''}.`); return false;
+    }
+    const latest = run.moves.filter(move => move.state === 'done').sort((a, b) => b.updatedAt.localeCompare(a.updatedAt))[0];
+    if (latest && !refreshedAfterMove(latest)) {
+        updateRun(run.id, 'running', 'Waiting for fresh source and destination measurements after the last move.'); return false;
+    }
+    const snapshot = rebalanceSnapshot(run.settings);
+    for (const move of run.moves.filter(move => move.state === 'planned')) {
+        const reason = rebalanceMoveBlocker(snapshot, move);
+        if (reason) { updateMove(move.id, 'skipped', reason); continue; }
+        // This transaction includes the durable move-to-operation link. A restarted
+        // worker sees either both records or neither and cannot repeat admission.
+        let operationId: string;
+        try { operationId = requestInstanceMove(move.server, move.targetHostId, { triggeredBy: `rebalance:${run.id}` }); }
+        catch (error) { updateMove(move.id, 'skipped', (error as Error).message); continue; }
+        db().prepare("UPDATE rebalancing_moves SET state='moving',operation_id=?,message='Waiting for safe migration admission.',updated_at=? WHERE id=?")
+            .run(operationId, atNow(), move.id);
+        updateRun(run.id, 'running', `Moving ${move.server} to ${move.targetHostName}.`);
+        return true;
+    }
+    const done = run.moves.filter(move => move.state === 'done').length;
+    updateRun(run.id, 'done', `Finished: ${done} moved. Remaining moves no longer qualify.`);
+    return false;
+}
+
+export async function runRebalancing(): Promise<boolean> {
+    return db().transaction(() => {
+        // Recovery is explicit in Provisioning. Reflect a recovered move, but
+        // never restart the other moves from the failed parent run.
+        const recovered = db().prepare(`SELECT m.id,m.run_id FROM rebalancing_moves m
+            JOIN rebalancing_runs r ON r.id=m.run_id JOIN server_deployments d ON d.id=m.operation_id
+            JOIN native_provisioning n ON n.deployment_id=d.id
+            WHERE r.state='failed' AND m.state='failed' AND d.state IN ('done','warning') AND n.verified_at IS NOT NULL`)
+            .all() as { id: string; run_id: string }[];
+        for (const move of recovered) {
+            updateMove(move.id, 'done', 'Recovered through Provisioning; healthy on the destination host.');
+            updateRun(move.run_id, 'stopped', 'The failed move was recovered. Remaining moves stay stopped.');
+        }
+        const row = db().prepare("SELECT * FROM rebalancing_runs WHERE state IN ('running','stopping') ORDER BY created_at LIMIT 1").get() as RunRow | undefined;
+        if (!row) return false;
+        return reconcileRun(present(row));
+    })();
+}
+
+export function pruneRebalancing(): void {
+    const now = atNow(), cutoff = new Date(Date.now() - 31 * 86_400_000).toISOString();
+    db().prepare('DELETE FROM rebalancing_previews WHERE expires_at<? AND run_id IS NULL').run(now);
+    db().prepare(`DELETE FROM rebalancing_runs WHERE state IN ('done','stopped') AND updated_at<?
+        AND NOT EXISTS (SELECT 1 FROM rebalancing_moves m JOIN server_deployments d ON d.id=m.operation_id
+            WHERE m.run_id=rebalancing_runs.id AND d.state NOT IN ('done','warning','cancelled'))`).run(cutoff);
+}

src/lib/server/rebalancing/planner.ts

@@ -0,0 +1,96 @@
+import type { RebalanceCandidate, RebalancePlan, RebalanceSettings } from '../../rebalancing.js';
+
+/** Difference in the pair's peak dominant utilization, in percentage points. */
+export const REBALANCE_MIN_IMPROVEMENT = 5;
+export const REBALANCE_COOLDOWN_MS = 30 * 60_000;
+export interface Resources { cpuCores: number; memoryBytes: number }
+export interface RebalancingHost {
+    id: string; name: string; cpuCount: number; memoryTotal: number;
+    used: Resources; running: Resources; startup: Resources;
+    instances: number; ports: number; reason: string | null;
+}
+export interface RebalancingInstance {
+    server: string; instanceId: string; generation: number; sourceHostId: string;
+    game: string; relief: Resources; demand: Resources;
+    releasesPort?: boolean;
+    destinations: Record<string, string | null>;
+}
+export interface RebalancingSnapshot {
+    hosts: RebalancingHost[]; instances: RebalancingInstance[];
+    exclusions: RebalancePlan['exclusions']; warnings: string[];
+}
+
+const pressure = (host: RebalancingHost, used = host.used) => Math.max(used.cpuCores / host.cpuCount, used.memoryBytes / host.memoryTotal);
+const add = (a: Resources, b: Resources): Resources => ({ cpuCores: a.cpuCores + b.cpuCores, memoryBytes: a.memoryBytes + b.memoryBytes });
+const subtract = (a: Resources, b: Resources): Resources => ({ cpuCores: Math.max(0, a.cpuCores - b.cpuCores), memoryBytes: Math.max(0, a.memoryBytes - b.memoryBytes) });
+const maximum = (a: Resources, b: Resources): Resources => ({ cpuCores: Math.max(a.cpuCores, b.cpuCores), memoryBytes: Math.max(a.memoryBytes, b.memoryBytes) });
+
+function evaluate(source: RebalancingHost, target: RebalancingHost, instance: RebalancingInstance): { reason: string | null; improvement: number; demand: Resources } {
+    const demand = maximum(instance.demand, target.running);
+    const startup = add(target.used, maximum(demand, target.startup));
+    const improvement = 100 * (Math.max(pressure(source), pressure(target))
+        - Math.max(pressure(source, subtract(source.used, instance.relief)), pressure(target, add(target.used, demand))));
+    let reason: string | null = source.reason ?? target.reason;
+    if (!reason && source.id === target.id) reason = 'Source and destination must be different hosts.';
+    if (!reason && !(target.id in instance.destinations)) reason = 'Destination compatibility has not been verified.';
+    if (!reason) reason = instance.destinations[target.id] ?? null;
+    if (!reason && target.ports < 1) reason = 'No available UDP/TCP game port.';
+    if (!reason && startup.cpuCores > target.cpuCount * 0.95) reason = 'Destination has insufficient startup CPU capacity.';
+    if (!reason && startup.memoryBytes > target.memoryTotal - 500_000_000) reason = 'Destination has insufficient startup memory capacity.';
+    if (!reason && (!Number.isFinite(improvement) || improvement + 1e-8 < REBALANCE_MIN_IMPROVEMENT)) reason = `Move would not reduce the pair’s peak utilization by at least ${REBALANCE_MIN_IMPROVEMENT} percentage points.`;
+    return { reason, improvement, demand };
+}
+
+/** Validate the approved pair against fresh evidence without replacing it with a different move. */
+export function rebalanceMoveBlocker(snapshot: RebalancingSnapshot, move: Pick<RebalanceCandidate, 'server' | 'instanceId' | 'generation' | 'sourceHostId' | 'targetHostId'>): string | null {
+    const instance = snapshot.instances.find(item => item.server === move.server && item.instanceId === move.instanceId
+        && item.generation === move.generation && item.sourceHostId === move.sourceHostId);
+    if (!instance) return snapshot.exclusions.find(item => item.server === move.server)?.reason ?? 'The instance or its generation has changed.';
+    const source = snapshot.hosts.find(host => host.id === move.sourceHostId), target = snapshot.hosts.find(host => host.id === move.targetHostId);
+    if (!source || !target) return 'The selected source or destination is unavailable.';
+    return evaluate(source, target, instance).reason;
+}
+
+/** Greedy largest improvements minimize churn; simulated relief never changes live capacity. */
+export function planRebalance(snapshot: RebalancingSnapshot, settings: RebalanceSettings): RebalancePlan {
+    const hosts = snapshot.hosts.filter(host => settings.hostIds.includes(host.id)).map(host => ({ ...host, used: { ...host.used } }));
+    const before = new Map(hosts.map(host => [host.id, { ...host, used: { ...host.used } }]));
+    const instances = snapshot.instances.filter(instance => settings.hostIds.includes(instance.sourceHostId)
+        && (settings.game === 'all' || settings.game === instance.game));
+    const moved = new Set<string>(), moves: RebalanceCandidate[] = [];
+    const limit = Math.max(1, Math.min(20, Math.floor(settings.maxMoves) || 1));
+    while (moves.length < limit) {
+        let best: { instance: RebalancingInstance; source: RebalancingHost; target: RebalancingHost; improvement: number; demand: Resources } | undefined;
+        for (const instance of [...instances].sort((a, b) => a.server.localeCompare(b.server))) {
+            if (moved.has(instance.instanceId)) continue;
+            const source = hosts.find(host => host.id === instance.sourceHostId);
+            if (!source) continue;
+            for (const target of [...hosts].sort((a, b) => a.id.localeCompare(b.id))) {
+                const result = evaluate(source, target, instance);
+                if (!result.reason && (!best || result.improvement > best.improvement + 1e-8)) best = { instance, source, target, ...result };
+            }
+        }
+        if (!best) break;
+        const { instance, source, target, improvement, demand } = best;
+        moves.push({ server: instance.server, instanceId: instance.instanceId, generation: instance.generation,
+            sourceHostId: source.id, sourceHostName: source.name, targetHostId: target.id, targetHostName: target.name, improvement });
+        source.used = subtract(source.used, instance.relief); source.instances--;
+        if (instance.releasesPort) source.ports++;
+        target.used = add(target.used, demand); target.instances++; target.ports--;
+        moved.add(instance.instanceId);
+    }
+    const exclusions = [...snapshot.exclusions];
+    for (const instance of instances.filter(item => !moved.has(item.instanceId))) {
+        const source = hosts.find(host => host.id === instance.sourceHostId)!;
+        const results = hosts.filter(host => host.id !== source.id).map(target => ({ target, ...evaluate(source, target, instance) }));
+        const fits = results.some(result => !result.reason);
+        const reason = fits && moves.length === limit ? 'Move limit reached.' : results.length
+            ? results.map(result => `${result.target.name}: ${result.reason ?? 'A different move provided more improvement.'}`).join(' ')
+            : 'No other participating host is available.';
+        exclusions.push({ server: instance.server, reason });
+    }
+    const projection = (host: RebalancingHost) => ({ cpuPercent: host.cpuCount > 0 ? host.used.cpuCores / host.cpuCount * 100 : 0,
+        memoryPercent: host.memoryTotal > 0 ? host.used.memoryBytes / host.memoryTotal * 100 : 0, instances: host.instances });
+    return { hosts: hosts.map(host => ({ id: host.id, name: host.name, before: projection(before.get(host.id)!), after: projection(host) })),
+        moves, exclusions, warnings: [...snapshot.warnings] };
+}

src/lib/server/rebalancing/schema.ts

@@ -0,0 +1,29 @@
+import type Database from 'better-sqlite3';
+
+export function ensureRebalancingSchema(handle: Database.Database): void {
+    handle.exec(`
+        CREATE TABLE IF NOT EXISTS rebalancing_runs (
+            id TEXT PRIMARY KEY, settings TEXT NOT NULL,
+            state TEXT NOT NULL CHECK(state IN ('running','stopping','done','stopped','failed')),
+            created_at TEXT NOT NULL, updated_at TEXT NOT NULL, message TEXT NOT NULL
+        );
+        CREATE UNIQUE INDEX IF NOT EXISTS rebalancing_one_run ON rebalancing_runs((1))
+            WHERE state IN ('running','stopping');
+        CREATE TABLE IF NOT EXISTS rebalancing_moves (
+            id TEXT PRIMARY KEY, run_id TEXT NOT NULL REFERENCES rebalancing_runs(id) ON DELETE CASCADE,
+            ordinal INTEGER NOT NULL, server TEXT NOT NULL, instance_id TEXT NOT NULL, generation INTEGER NOT NULL,
+            source_host_id TEXT NOT NULL, target_host_id TEXT NOT NULL,
+            candidate TEXT NOT NULL, state TEXT NOT NULL CHECK(state IN ('planned','moving','done','skipped','failed')),
+            operation_id TEXT UNIQUE REFERENCES server_deployments(id) ON DELETE SET NULL,
+            message TEXT NOT NULL, updated_at TEXT NOT NULL, UNIQUE(run_id,ordinal)
+        );
+        CREATE INDEX IF NOT EXISTS rebalancing_move_run ON rebalancing_moves(run_id,state,ordinal);
+        CREATE INDEX IF NOT EXISTS rebalancing_move_instance ON rebalancing_moves(instance_id,state,updated_at);
+        CREATE TABLE IF NOT EXISTS rebalancing_previews (
+            id TEXT PRIMARY KEY, preview TEXT NOT NULL, expires_at TEXT NOT NULL,
+            run_id TEXT REFERENCES rebalancing_runs(id) ON DELETE CASCADE
+        );
+        CREATE INDEX IF NOT EXISTS rebalancing_preview_expiry ON rebalancing_previews(expires_at);
+        CREATE INDEX IF NOT EXISTS rebalancing_preview_run ON rebalancing_previews(run_id);
+    `);
+}

src/lib/server/rebalancing/snapshot.ts

@@ -0,0 +1,144 @@
+import { db } from '../db/index.js';
+import { listHosts, HEARTBEAT_MAX_AGE_MS, requireReadyHost } from '../hosts/index.js';
+import { capacityCalibration } from '../hosts/calibration.js';
+import { assertReadyLibrary } from '../game-files/index.js';
+import { listServers } from '../fleet/servers.js';
+import { getServerKey } from '../fleet/server-keys.js';
+import { observationReader, serverObservation } from '../fleet/status.js';
+import { latestGameSubmission, telemetryReport } from '../fleet/status-evidence.js';
+import { mapIdentity, protectedWork } from '../autoscaling/snapshot.js';
+import { captureMoveRecipe } from '../provisioning/move-recipes.js';
+import { freshStatusTimestamp } from '../../server-freshness.js';
+import type { Host } from '../../hosts.js';
+import type { NativeInstance } from '../backends/native.js';
+import type { RebalanceHostOption, RebalanceSettings } from '../../rebalancing.js';
+import { REBALANCE_COOLDOWN_MS, type RebalancingSnapshot, type Resources } from './planner.js';
+
+const HISTORY_MS = 5 * 60_000;
+const nonnegative = (value: unknown): value is number => typeof value === 'number' && Number.isFinite(value) && value >= 0;
+
+function hostBlocker(host: Host, now: number): string | null {
+    if (host.mode !== 'active' || host.decommission) return 'An active host is required.';
+    const stamp = Date.parse(host.lastHeartbeat ?? '');
+    if (!host.connected || !Number.isFinite(stamp) || stamp > now || now - stamp > HEARTBEAT_MAX_AGE_MS) return 'Waiting for a fresh host report.';
+    if (host.readiness !== 'ready') return host.readinessReasons.join(' ') || 'The host is not ready.';
+    const { cpuCount, cpuUsagePercent, memoryTotal, memoryAvailable } = host.inventory;
+    if (!nonnegative(cpuCount) || !cpuCount || !nonnegative(cpuUsagePercent) || cpuUsagePercent > 100
+        || !nonnegative(memoryTotal) || !memoryTotal || !nonnegative(memoryAvailable) || memoryAvailable > memoryTotal) return 'Waiting for valid CPU and memory measurements.';
+    return null;
+}
+
+export function rebalanceHosts(): RebalanceHostOption[] {
+    const now = Date.now();
+    return listHosts().map(host => { const reason = hostBlocker(host, now); return { id: host.id, name: host.name, eligible: !reason, reason }; });
+}
+
+/** Database evidence only: no probes, writes, reservations, or source capacity credits. */
+export function rebalanceSnapshot(settings: RebalanceSettings, exceptOperationId = ''): RebalancingSnapshot {
+    const handle = db(), now = Date.now(), at = new Date(now).toISOString(), since = new Date(now - HISTORY_MS).toISOString();
+    const hosts = listHosts().filter(host => settings.hostIds.includes(host.id));
+    const natives = handle.prepare('SELECT * FROM native_instances WHERE retired_at IS NULL').all() as NativeInstance[];
+    const operations = handle.prepare(`SELECT id,server,stage FROM server_deployments WHERE server_deleted_at IS NULL
+        AND state IN ('queued','waiting','running','awaiting-health','interrupted') AND id!=?`).all(exceptOperationId) as { id: string; server: string; stage: string }[];
+    const ownServer = exceptOperationId ? (handle.prepare('SELECT server FROM server_deployments WHERE id=?').get(exceptOperationId) as { server: string } | undefined)?.server : undefined;
+    const observe = observationReader();
+    const snapshot: RebalancingSnapshot = { hosts: [], instances: [], exclusions: [], warnings: [] };
+    for (const id of settings.hostIds) if (!hosts.some(host => host.id === id)) snapshot.warnings.push(`Participating host ${id} is unavailable.`);
+    for (const host of hosts) {
+        const reason = hostBlocker(host, now), { running, startup } = capacityCalibration(host.id);
+        if (reason) snapshot.warnings.push(`${host.name}: ${reason}`);
+        const inventory = host.inventory;
+        const valid = !reason || nonnegative(inventory.cpuCount) && inventory.cpuCount > 0 && nonnegative(inventory.cpuUsagePercent)
+            && nonnegative(inventory.memoryTotal) && inventory.memoryTotal > 0 && nonnegative(inventory.memoryAvailable) && inventory.memoryAvailable <= inventory.memoryTotal;
+        const history = handle.prepare(`SELECT AVG(cpu_usage_percent) AS cpu,AVG(memory_available) AS memory FROM ec_host_resource_samples
+            WHERE host_id=? AND sampled_at>=? AND sampled_at<=? AND cpu_count=? AND memory_total=?`)
+            .get(host.id, since, at, inventory.cpuCount ?? 0, inventory.memoryTotal ?? 0) as { cpu: number | null; memory: number | null };
+        const used: Resources = { cpuCores: valid ? inventory.cpuCount! * Math.max(inventory.cpuUsagePercent!, history.cpu ?? 0) / 100 : 0,
+            memoryBytes: valid ? inventory.memoryTotal! - Math.min(inventory.memoryAvailable!, history.memory ?? inventory.memoryAvailable!) : 0 };
+        const hostInstances = natives.filter(instance => instance.host_id === host.id), measured = new Map(inventory.instanceResources?.map(sample => [sample.instanceId, sample]));
+        const reserved = new Set<string>();
+        const reserve = (id: string, allowance: Resources) => {
+            if (reserved.has(id)) return;
+            reserved.add(id); used.cpuCores += allowance.cpuCores; used.memoryBytes += allowance.memoryBytes;
+        };
+        for (const instance of hostInstances) {
+            const sample = measured.get(instance.id);
+            if (sample?.generation === instance.generation) continue; // Already in the host measurement.
+            const latest = handle.prepare(`SELECT d.state,d.submitted_at,n.operation_id,n.stage_operation_id FROM native_provisioning n
+                JOIN server_deployments d ON d.id=n.deployment_id WHERE n.instance_id=? AND n.generation=? AND d.id!=?
+                AND d.action IN ('create','rebuild') AND d.server_deleted_at IS NULL ORDER BY d.rowid DESC LIMIT 1`)
+                .get(instance.id, instance.generation, exceptOperationId) as { state: string; submitted_at: string | null; operation_id: string | null; stage_operation_id: string | null } | undefined;
+            const accounted = handle.prepare(`SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
+                WHERE n.instance_id=? AND n.generation=? AND d.action IN ('create','rebuild') AND d.state IN ('done','warning') AND d.finished_at<?`)
+                .get(instance.id, instance.generation, host.lastHeartbeat ?? '');
+            if (accounted) continue;
+            reserve(instance.id, latest?.state === 'queued' && !latest.submitted_at && !latest.operation_id && !latest.stage_operation_id ? running : startup);
+        }
+        const incoming = handle.prepare(`SELECT m.instance_id,m.removed_at,i.generation FROM ec_host_migrations m
+            JOIN server_deployments d ON d.id=m.deployment_id JOIN native_instances i ON i.id=m.instance_id
+            WHERE m.target_host_id=? AND m.deployment_id!=? AND d.server_deleted_at IS NULL AND i.retired_at IS NULL
+            AND d.state NOT IN ('done','warning','cancelled')`).all(host.id, exceptOperationId) as { instance_id: string; removed_at: string | null; generation: number }[];
+        for (const migration of incoming) {
+            const generation = migration.generation + (migration.removed_at ? 0 : 1);
+            if (measured.get(migration.instance_id)?.generation !== generation) reserve(migration.instance_id, startup);
+        }
+        const ports = handle.prepare('SELECT port,bind_address FROM ec_host_ports WHERE host_id=?').all(host.id) as { port: number; bind_address: string }[];
+        const usedPorts = new Set(ports.filter(port => port.port >= host.portMin && port.port <= host.portMax
+            && (port.bind_address === host.bindAddress || ['0.0.0.0', '::'].includes(port.bind_address) || ['0.0.0.0', '::'].includes(host.bindAddress))).map(port => port.port));
+        snapshot.hosts.push({ id: host.id, name: host.name, cpuCount: valid ? inventory.cpuCount! : 0, memoryTotal: valid ? inventory.memoryTotal! : 0,
+            used, running, startup, reason, instances: hostInstances.length + incoming.filter(row => !hostInstances.some(instance => instance.id === row.instance_id)).length,
+            ports: Math.max(0, host.portMax - host.portMin + 1 - usedPorts.size) });
+    }
+    const destinations = new Map<string, Record<string, string | null>>();
+    for (const row of listServers()) {
+        const identity = mapIdentity(row);
+        if (!row.host_id || !settings.hostIds.includes(row.host_id) || settings.game !== 'all' && identity?.game !== settings.game) continue;
+        const source = hosts.find(host => host.id === row.host_id), native = natives.find(instance => instance.id === row.native_instance_id);
+        // The admitted rebuild itself must not hide the pre-removal empty evidence.
+        const observation = row.name === ownServer ? serverObservation(row, { provisioning: false, settling: false,
+            telemetry: telemetryReport(row.name), submission: latestGameSubmission(row.name) }) : observe(row);
+        let reason = protectedWork(row.name, exceptOperationId);
+        if (!reason && (!identity || !native || !row.map_slug)) reason = 'A native instance with a known map is required.';
+        if (!reason && (!source || hostBlocker(source, now))) reason = source ? hostBlocker(source, now) : 'Source host is unavailable.';
+        const operation = operations.find(item => item.server === row.name);
+        if (!reason && operation) reason = `Provisioning ${operation.id}: ${operation.stage}`;
+        if (!reason && native && handle.prepare(`SELECT 1 FROM ec_host_operations WHERE instance_id=? AND state IN ('pending','accepted','running','unknown')
+            AND kind IN ('instance.create','instance.rebuild','instance.start','instance.stop','instance.restart','instance.remove','instance.cancel-create')`).get(native.id)) reason = 'An instance lifecycle command is unsettled.';
+        if (!reason && (row.desired_state === 'stopped' || ['stopped','exited','created','dead','missing'].includes(row.container_state ?? ''))) reason = 'Stopped instances are not eligible.';
+        if (!reason && (row.last_players ?? 0) > 0 && freshStatusTimestamp(row.last_seen_at, now)
+            && (!observation.telemetry_after || Date.parse(row.last_seen_at!) > Date.parse(observation.telemetry_after))) reason = 'Players are present according to a current RCON report.';
+        if (!reason && (observation.health !== 'online' || observation.last_players === null || !freshStatusTimestamp(observation.last_seen_at, now))) reason = 'Waiting for healthy, fresh occupancy evidence.';
+        if (!reason && observation.last_players !== 0) reason = 'Players are present.';
+        if (!reason && !getServerKey(row.name)) reason = 'Save a server key before moving this instance.';
+        if (!reason && handle.prepare(`SELECT 1 FROM rebalancing_moves WHERE instance_id=? AND state='done' AND updated_at>=?`)
+            .get(native!.id, new Date(now - REBALANCE_COOLDOWN_MS).toISOString())) reason = 'This instance was rebalanced less than 30 minutes ago.';
+        const sample = source?.inventory.instanceResources?.find(item => item.instanceId === native?.id && item.generation === native.generation);
+        if (!reason && (!sample || source?.inventory.instanceResourcesError || !nonnegative(sample.cpuCores) || !nonnegative(sample.memoryBytes) || !sample.memoryBytes)) reason = 'Waiting for resource measurements for the current instance generation.';
+        if (!reason && sample!.uptimeSeconds < 300) reason = 'Waiting for this instance to finish its first five minutes of startup.';
+        if (!reason) { try { captureMoveRecipe(row.name); } catch (error) { reason = (error as Error).message; } }
+        if (reason) { snapshot.exclusions.push({ server: row.name, reason }); continue; }
+        const key = JSON.stringify([native!.library_id, native!.image]);
+        if (!destinations.has(key)) {
+            const eligibility: Record<string, string | null> = {};
+            for (const host of hosts) {
+                try { requireReadyHost(host.id, { image: native!.image, requiredBytes: 1024 ** 3 }); assertReadyLibrary(host.id, native!.library_id); eligibility[host.id] = null; }
+                catch (error) { eligibility[host.id] = (error as Error).message; }
+            }
+            destinations.set(key, eligibility);
+        }
+        const history = handle.prepare(`SELECT AVG(cpu_cores) AS cpu,AVG(memory_bytes) AS memory FROM ec_instance_resource_samples
+            WHERE host_id=? AND instance_id=? AND generation=? AND phase='running' AND bucket>=? AND bucket<=?`)
+            .get(source!.id, native!.id, native!.generation, since, at) as { cpu: number | null; memory: number | null };
+        const bindConflicts = (bind: string) => bind === source!.bindAddress || ['0.0.0.0', '::'].includes(bind) || ['0.0.0.0', '::'].includes(source!.bindAddress);
+        const portOwners = handle.prepare('SELECT instance_id,bind_address FROM ec_host_ports WHERE host_id=? AND port=?')
+            .all(source!.id, native!.port) as { instance_id: string; bind_address: string }[];
+        snapshot.instances.push({ server: row.name, instanceId: native!.id, generation: native!.generation, sourceHostId: source!.id, game: identity!.game,
+            releasesPort: native!.port >= source!.portMin && native!.port <= source!.portMax && bindConflicts(native!.bind_address)
+                && portOwners.some(owner => owner.instance_id === native!.id && bindConflicts(owner.bind_address))
+                && !portOwners.some(owner => owner.instance_id !== native!.id && bindConflicts(owner.bind_address)),
+            relief: { cpuCores: Math.min(sample!.cpuCores, history.cpu ?? sample!.cpuCores), memoryBytes: Math.min(sample!.memoryBytes, history.memory ?? sample!.memoryBytes) },
+            demand: { cpuCores: Math.max(sample!.cpuCores, history.cpu ?? sample!.cpuCores), memoryBytes: Math.max(sample!.memoryBytes, history.memory ?? sample!.memoryBytes) },
+            destinations: destinations.get(key)! });
+    }
+    return snapshot;
+}

src/lib/server/retention.ts

@@ -56,6 +56,7 @@ function policies(): Policy[] {
             AND NOT EXISTS (SELECT 1 FROM watchdog_actions s WHERE s.operation_id=o.id)
             AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations c WHERE o.id IN (c.creation_id,c.cleanup_id))
             AND NOT EXISTS (SELECT 1 FROM server_creation_dependencies c WHERE c.removal_id=o.id)
+            AND NOT EXISTS (SELECT 1 FROM rebalancing_moves m WHERE m.operation_id=o.id AND m.state IN ('moving','failed'))
             AND NOT EXISTS (SELECT 1 FROM server_collection_members m JOIN server_collection_members sibling ON sibling.collection_id=m.collection_id
                 JOIN server_deployments d ON d.id=sibling.deployment_id WHERE m.deployment_id=o.id AND (${unsettledDeployment}))`,
             bytes: "COALESCE(length(o.observation_json),0)+(SELECT COALESCE(SUM(length(recipe)),0) FROM server_launch_recipes WHERE deployment_id=o.id)" },
@@ -88,6 +89,7 @@ function policies(): Policy[] {
         { category: 'workflow-payload', label: 'Completed launch file bodies', table: 'server_deployments', id: 'id', clock: 'finished_at', days: 'payloadKeepDays',
             failed: "o.state='failed'",
             eligible: `o.state IN ${terminal} AND NOT (${unsettledDeployment.replaceAll('d.', 'o.')})
+                AND NOT EXISTS (SELECT 1 FROM rebalancing_moves m WHERE m.operation_id=o.id AND m.state IN ('moving','failed'))
                 AND NOT EXISTS (SELECT 1 FROM retention_controls c WHERE c.category='provisioning' AND c.record_id=o.id AND c.protected=1)
                 AND NOT EXISTS (SELECT 1 FROM retention_expirations e WHERE e.category='workflow-payload' AND e.record_id=o.id)`,
             bytes: "(SELECT COALESCE(SUM(b.size),0) FROM file_blob_references r JOIN file_blobs b ON b.sha256=r.sha256 WHERE r.owner_type='recipe' AND r.owner_id=o.id)" },