Migrate instances when draining and verify hosts before enabling

src/lib/hosts.ts

@@ -94,6 +94,7 @@ export interface Host {
     enrolled: boolean; credentialActive: boolean;
     decommission: HostDecommission | null;
     preparation?: HostPreparation;
+    enablingSince?: string | null;
 }
 export interface HostOperation {
     id: string; hostId: string; kind: string; instanceId: string | null; generation: number | null;

src/lib/server/backends/schema.ts

@@ -23,6 +23,14 @@ export function migrateNativeInstances(handle: Database.Database): void {
             applied_at TEXT, verified_at TEXT
         );
         CREATE INDEX IF NOT EXISTS native_provisioning_instance ON native_provisioning(instance_id);
+        CREATE TABLE IF NOT EXISTS ec_host_migrations (
+            deployment_id TEXT PRIMARY KEY REFERENCES server_deployments(id),
+            source_host_id TEXT NOT NULL, instance_id TEXT NOT NULL REFERENCES native_instances(id),
+            target_host_id TEXT, target_port INTEGER, target_bind TEXT,
+            remove_operation_id TEXT, removed_at TEXT
+        );
+        CREATE INDEX IF NOT EXISTS ec_host_migrations_source ON ec_host_migrations(source_host_id);
+        CREATE INDEX IF NOT EXISTS ec_host_migrations_target ON ec_host_migrations(target_host_id);
         CREATE TABLE IF NOT EXISTS native_creation_cancellations (
             creation_id TEXT PRIMARY KEY REFERENCES server_deployments(id),
             cleanup_id TEXT NOT NULL UNIQUE REFERENCES server_deployments(id)

src/lib/server/game-files/index.ts

@@ -386,7 +386,7 @@ export function queueLibraryDownload(hostId: string, versionId: string, options:
     if (libraryFilesRetired(versionId)) throw new Error('These outdated game files have been removed. Upload them again to use them.');
     if (database().prepare("SELECT 1 FROM ec_host_operations WHERE host_id=? AND kind='library.remove' AND json_extract(payload,'$.libraryVersionId')=? AND state IN ('pending','accepted','running','unknown')").get(hostId, versionId)) throw new Error('This host copy is being removed.');
     const host = getHost(hostId);
-    if (!host || !host.connected || !host.credentialActive || host.mode === 'disabled') throw new Error('Choose an online, enrolled host.');
+    if (!host || !host.connected || !host.credentialActive || (host.mode === 'disabled' && !host.enablingSince)) throw new Error('Choose an online, enrolled host.');
     const prior = getHostLibraryCache(hostId, versionId);
     if (prior?.operationId) {
         const operation = getHostOperation(prior.operationId);

src/lib/server/hosts/availability.ts

@@ -52,7 +52,9 @@ function observeHost(host: Host): HostAvailability {
         JOIN native_provisioning n ON n.instance_id=i.id JOIN server_deployments d ON d.id=n.deployment_id
         WHERE i.host_id=? AND i.retired_at IS NULL AND d.action IN ('create','rebuild')
         AND d.state IN ('running','awaiting-health')`).all(host.id) as { id: string; generation: number }[];
-    const unmeasured = pending.filter(instance => measured.get(instance.id) !== instance.generation).length;
+    const incoming = handle.prepare(`SELECT COUNT(*) AS n FROM ec_host_migrations m JOIN server_deployments d ON d.id=m.deployment_id
+        WHERE m.target_host_id=? AND (m.removed_at IS NULL OR d.state IN ('queued','waiting','interrupted'))`).get(host.id) as { n: number };
+    const unmeasured = pending.filter(instance => measured.get(instance.id) !== instance.generation).length + incoming.n;
     const { cpuCount, cpuUsagePercent, memoryTotal, memoryAvailable } = host.inventory;
     const freeCpu = cpuCount && cpuUsagePercent !== undefined
         ? Math.max(0, cpuCount * (0.95 - cpuUsagePercent / 100) - unmeasured * startup.cpuCores) : null;

src/lib/server/hosts/capacity.ts

@@ -46,7 +46,7 @@ function projectedLoad(host: Host, options: PlacementOptions): number {
     }
     const instances = db().prepare(`SELECT i.id, i.generation,
         NOT EXISTS (SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
-            WHERE n.instance_id=i.id AND d.action='create' AND d.state IN ('done','warning') AND d.finished_at<?) AS pending,
+            WHERE n.instance_id=i.id AND n.generation=i.generation AND d.action IN ('create','rebuild') AND d.state IN ('done','warning') AND d.finished_at<?) AS pending,
         EXISTS (SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
             WHERE n.instance_id=i.id AND d.action='create' AND d.state='queued' AND d.submitted_at IS NULL
             AND n.operation_id IS NULL AND n.stage_operation_id IS NULL) AS unsubmitted
@@ -54,13 +54,15 @@ function projectedLoad(host: Host, options: PlacementOptions): number {
         .all(host.lastHeartbeat, host.id, options.excludeInstanceId ?? '') as
         { id: string; generation: number; pending: number; unsubmitted: number }[];
     const { running, startup } = capacityCalibration(host.id);
+    const incoming = db().prepare(`SELECT COUNT(*) AS n FROM ec_host_migrations
+        WHERE target_host_id=? AND removed_at IS NULL AND instance_id!=?`).get(host.id, options.excludeInstanceId ?? '') as { n: number };
     const planned = options.planned?.filter(item => item.hostId === host.id).length ?? 0;
     // Admission budgets the eventual running collection. Dispatch budgets only
     // the next startup; the rest of the queue waits for live capacity to recover.
     const launch = options.dispatch ? startup : running;
-    let extraCpu = planned * running.cpuCores + launch.cpuCores;
-    let extraMemory = planned * running.memoryBytes + launch.memoryBytes;
-    let reservations = planned + 1;
+    let extraCpu = planned * running.cpuCores + launch.cpuCores + incoming.n * startup.cpuCores;
+    let extraMemory = planned * running.memoryBytes + launch.memoryBytes + incoming.n * startup.memoryBytes;
+    let reservations = planned + 1 + incoming.n;
     const measurements = new Map(host.inventory.instanceResources?.map(sample => [sample.instanceId, sample]));
     for (const instance of instances) {
         if (options.dispatch && options.excludeInstanceId && instance.unsubmitted) continue;

src/lib/server/hosts/index.ts

@@ -37,6 +37,7 @@ function presentDecommission(row: DecommissionRow | undefined): HostDecommission
         error: row.error, retryable: row.stage === 'failed' && getHostOperationStatus(row.operation_id)?.state === 'failed' } : null;
 }
 function hostFromRow(row: HostRow): Host {
+    const enabling = database().prepare('SELECT requested_at FROM ec_host_enable_requests WHERE host_id=?').get(row.id) as { requested_at: string } | undefined;
     const inventory: HostInventory = JSON.parse(row.inventory);
     const connected = !!row.credential_active && !!row.last_heartbeat && Date.now() - Date.parse(row.last_heartbeat) <= HEARTBEAT_MAX_AGE_MS;
     const reasons: string[] = [];
@@ -53,13 +54,14 @@ function hostFromRow(row: HostRow): Host {
     if (!runtimeReady) reasons.push('The Plutainer runtime image is not ready yet.');
     if (!libraryReady) reasons.push('No verified game-file library is available yet.');
     if (row.mode === 'draining') reasons.push('Host is draining; new deployments are blocked.');
-    if (row.mode === 'disabled') reasons.push('Host is disabled.');
+    if (row.mode === 'disabled') reasons.push(enabling ? 'Host setup must refresh the runtime and verify both BO1 and BO2 game files before enabling.' : 'Host is disabled.');
     const decommission = presentDecommission(decommissionRow(row.id));
     if (decommission) reasons.push('Host deletion is in progress; waiting for confirmed cleanup and agent removal.');
     return { id: row.id, name: row.name, hostname: row.hostname, mode: row.mode, portMin: row.port_min, portMax: row.port_max,
         bindAddress: row.bind_address, publicAddress: row.public_address, minimumFreeBytes: row.minimum_free_bytes,
         createdAt: row.created_at, lastHeartbeat: row.last_heartbeat, connected, connectionState: connected ? 'online' : 'offline',
-        readiness: decommission ? 'deleting' : row.mode === 'draining' ? 'draining' : row.mode === 'disabled' ? 'disabled' : !row.enrolled_at ? 'preparing' : !infrastructureReady ? 'degraded' : !runtimeReady ? 'needs-runtime' : !libraryReady ? 'needs-game-files' : 'ready',
+        enablingSince: enabling?.requested_at ?? null,
+        readiness: decommission ? 'deleting' : enabling ? 'preparing' : row.mode === 'draining' ? 'draining' : row.mode === 'disabled' ? 'disabled' : !row.enrolled_at ? 'preparing' : !infrastructureReady ? 'degraded' : !runtimeReady ? 'needs-runtime' : !libraryReady ? 'needs-game-files' : 'ready',
         readinessReasons: reasons, inventory, enrolled: !!row.enrolled_at, credentialActive: !!row.credential_active, decommission };
 }
 export function listHosts(): Host[] { return (database().prepare(hostSql + ' WHERE h.deleted_at IS NULL ORDER BY h.name, h.id').all() as HostRow[]).map(hostFromRow); }
@@ -197,7 +199,7 @@ export function updateHost(id: string, input: Record<string, unknown>): Host {
     const host = requireHost(id);
     if (host.decommission) throw new Error('Host cleanup is in progress; its settings cannot change.');
     const name = input.name === undefined ? host.name : textField(input.name, 'Host name');
-    const mode = input.mode ?? host.mode;
+    let mode = input.mode ?? host.mode;
     if (!['active', 'draining', 'disabled'].includes(mode as string)) throw new Error('Host mode must be active, draining, or disabled.');
     const min = port(input.portMin ?? host.portMin); const max = port(input.portMax ?? host.portMax);
     if (max < min) throw new Error('The last game port must not precede the first.');
@@ -206,7 +208,20 @@ export function updateHost(id: string, input: Record<string, unknown>): Host {
     const publicAddress = input.publicAddress === undefined ? host.publicAddress : input.publicAddress === '' ? '' : textField(input.publicAddress, 'Public address');
     const minimum = input.minimumFreeBytes ?? host.minimumFreeBytes;
     if (typeof minimum !== 'number' || !Number.isSafeInteger(minimum) || minimum < 0) throw new Error('Minimum free storage must be a nonnegative byte count.');
-    database().prepare('UPDATE ec_hosts SET name=?,mode=?,port_min=?,port_max=?,bind_address=?,public_address=?,minimum_free_bytes=? WHERE id=?').run(name, mode, min, max, bind, publicAddress, minimum, id);
+    database().transaction(() => {
+        if (input.mode === 'active' && host.mode === 'draining') throw new Error('Wait for this host to finish draining before enabling it.');
+        if (input.mode === 'active' && host.mode === 'disabled') {
+            database().prepare('INSERT OR IGNORE INTO ec_host_enable_requests(host_id,requested_at,operation_floor) VALUES(?,?,(SELECT COALESCE(MAX(rowid),0) FROM ec_host_operations))').run(id, now());
+            mode = 'disabled';
+        }
+        if (input.mode === 'draining') {
+            if (host.enablingSince) throw new Error('Wait for automatic host setup to finish before draining.');
+            const incoming = database().prepare(`SELECT 1 FROM ec_host_migrations m JOIN server_deployments d ON d.id=m.deployment_id
+                WHERE m.target_host_id=? AND d.state NOT IN ('done','warning') AND d.server_deleted_at IS NULL LIMIT 1`).get(id);
+            if (incoming) throw new Error('Wait for incoming instance migrations to finish before draining this host.');
+        }
+        database().prepare('UPDATE ec_hosts SET name=?,mode=?,port_min=?,port_max=?,bind_address=?,public_address=?,minimum_free_bytes=? WHERE id=?').run(name, mode, min, max, bind, publicAddress, minimum, id);
+    })();
     scheduleAvailabilityRefresh();
     return requireHost(id);
 }
@@ -435,7 +450,7 @@ function enqueueHostOperation(hostId: string, kind: string, payload: string | Bu
             }
         }
         if (['instance.create','create'].includes(kind)) requireReadyHost(hostId);
-        if (host.mode === 'disabled' && kind !== 'instance.cancel-create') throw new Error('Host is disabled. Enable it before requesting operations.');
+        if (host.mode === 'disabled' && kind !== 'instance.cancel-create' && !(host.enablingSince && ['runtime.pull','library.download','host.inspect'].includes(kind))) throw new Error('Host is disabled. Enable it before requesting operations.');
         const id = options.id ?? randomUUID(); const timestamp = now();
         const serverName = options.instanceId ? (database().prepare('SELECT server FROM native_instances WHERE id=?')
             .get(options.instanceId) as { server: string } | undefined)?.server ?? null : null;
@@ -461,7 +476,7 @@ export function pendingHostOperationRows(hostId: string, lane: OperationLane, in
 /** Delivery snapshots retain status/results while large immutable payloads stay in SQLite. */
 export function pendingHostOperationRows(hostId: string, lane: OperationLane = 'all', includePayload = true): OperationDeliveryRow[] {
     const host = requireHost(hostId);
-    if (host.mode === 'disabled' || !host.connected) return [];
+    if ((host.mode === 'disabled' && !host.enablingSince) || !host.connected) return [];
     // Older agents hold activity.lock across the entire delivered batch, including
     // observations. Filtering only the unsupported cleanup can keep the updater
     // busy indefinitely. Drain the old batch by withholding ALL further work;
@@ -484,12 +499,13 @@ export function pendingHostOperationRows(hostId: string, lane: OperationLane = '
         const columns = includePayload ? `${operationStatusColumns},${operationPayloadSql} AS payload` : operationStatusColumns;
         const rows = database().prepare(`SELECT ${columns} FROM ec_host_operations o WHERE host_id=? AND state IN ('pending','accepted','running','unknown')
             ${laneFilter}
+            ${host.mode === 'disabled' ? "AND kind IN ('runtime.pull','library.download','host.inspect')" : ''}
             AND (? OR state != 'pending' OR kind NOT IN ('instance.create','create'))
             AND (kind!='instance.cancel-create' OR ?)
             AND NOT EXISTS (SELECT 1 FROM ec_host_operations cancellation WHERE cancellation.host_id=o.host_id
                 AND cancellation.kind='instance.cancel-create' AND json_extract(cancellation.payload,'$.createOperationId')=o.id)
             ORDER BY CASE WHEN kind='instance.cancel-create' THEN 0 ELSE 1 END,COALESCE(last_delivered_at,''),created_at,rowid LIMIT ${limit}`)
-            .all(hostId, Number(host.readiness === 'ready'), Number(host.inventory.capabilities?.includes('instance.cancel-create') ?? false)) as OperationDeliveryRow[];
+            .all(hostId, Number(host.readiness === 'ready' || host.mode === 'draining'), Number(host.inventory.capabilities?.includes('instance.cancel-create') ?? false)) as OperationDeliveryRow[];
         const deliveredAt = now();
         for (const operation of rows) database().prepare('UPDATE ec_host_operations SET last_delivered_at=? WHERE id=?').run(deliveredAt, operation.id);
         return rows;

src/lib/server/hosts/preparation.ts

@@ -4,10 +4,11 @@ import { AGENT_PROTOCOL_VERSION, getHost, queueHostOperation } from './index.js'
 import { getHostLibraryCache, latestPublishedLibraries, queueLibraryDownload } from '../game-files/index.js';
 import { GAME_FILE_GAMES } from '../../game-files.js';
 import type { Host, HostOperationState, HostPreparation, HostPreparationItem } from '../../hosts.js';
+import { scheduleAvailabilityRefresh } from './availability.js';
 
 const activeStates = ['pending', 'accepted', 'running', 'unknown'];
 const supersededDownload = 'Superseded by newer game files before delivery to the host.';
-type PreparationOperation = { id: string; state: HostOperationState; updated_at: string; error: string | null };
+type PreparationOperation = { id: string; sequence: number; state: HostOperationState; updated_at: string; error: string | null };
 type Target = { item: HostPreparationItem; image?: string; versionId?: string; requiredBytes: number; operation?: PreparationOperation };
 type QueuedSelection = { libraryId: string; image: string | null };
 type LibraryTarget = Pick<ReturnType<typeof latestPublishedLibraries>[number], 'id' | 'sizeBytes' | 'manifestSha256'> & { name: string; queued?: boolean };
@@ -23,7 +24,12 @@ function queuedSelections(hostId: string): QueuedSelection[] {
         WHERE d.action='create' AND d.server_deleted_at IS NULL AND n.deployment_id IS NULL
         AND d.state IN ('queued','waiting','running','awaiting-health','interrupted')
         AND (json_extract(q.selection,'$.automatic')=1 OR json_extract(q.selection,'$.hostId') IS NULL
-            OR json_extract(q.selection,'$.hostId')=?)`).all(hostId) as QueuedSelection[];
+            OR json_extract(q.selection,'$.hostId')=?)
+        UNION SELECT i.library_id AS libraryId,i.image FROM ec_host_migrations m
+            JOIN native_instances i ON i.id=m.instance_id JOIN server_deployments d ON d.id=m.deployment_id
+            WHERE m.removed_at IS NULL AND d.server_deleted_at IS NULL
+            AND d.state IN ('queued','waiting','running','awaiting-health','interrupted')
+            AND (m.target_host_id IS NULL OR m.target_host_id=?)`).all(hostId, hostId) as QueuedSelection[];
 }
 
 function targetLibraries(selections: QueuedSelection[]): LibraryTarget[] {
@@ -44,7 +50,7 @@ function targetLibraries(selections: QueuedSelection[]): LibraryTarget[] {
 }
 
 function latestOperation(hostId: string, kind: string, field: string, value: string): PreparationOperation | undefined {
-    return db().prepare(`SELECT id,state,updated_at,error FROM ec_host_operations
+    return db().prepare(`SELECT id,rowid AS sequence,state,updated_at,error FROM ec_host_operations
         WHERE host_id=? AND kind=? AND json_extract(payload,?)=? ORDER BY created_at DESC,rowid DESC LIMIT 1`)
         .get(hostId, kind, `$.${field}`, value) as PreparationOperation | undefined;
 }
@@ -67,7 +73,7 @@ function retryAt(hostId: string, target: Target): number {
 
 function hostBlocker(host: Host): { state: 'waiting' | 'blocked'; detail: string } | null {
     if (host.decommission) return { state: 'waiting', detail: 'Automatic preparation is paused during host deletion.' };
-    if (host.mode !== 'active') return { state: 'waiting', detail: `Automatic preparation is paused while this host is ${host.mode}.` };
+    if (host.mode !== 'active' && !host.enablingSince) return { state: 'waiting', detail: `Automatic preparation is paused while this host is ${host.mode}.` };
     if (!host.enrolled) return { state: 'waiting', detail: 'Automatic preparation starts after agent installation.' };
     if (!host.connected || !host.credentialActive) return { state: 'waiting', detail: 'Waiting for the host to reconnect.' };
     if (host.inventory.protocolVersion !== AGENT_PROTOCOL_VERSION) return { state: 'blocked', detail: 'A compatible host agent is required.' };
@@ -77,20 +83,26 @@ function hostBlocker(host: Host): { state: 'waiting' | 'blocked'; detail: string
 }
 
 function targets(host: Host): Target[] {
+    const enabling = db().prepare('SELECT operation_floor FROM ec_host_enable_requests WHERE host_id=?').get(host.id) as { operation_floor: number } | undefined;
+    const verifiedThisEnable = (operation: PreparationOperation | undefined) => !enabling || !!operation
+        && operation.sequence > enabling.operation_floor && operation.state === 'completed'
+        && Date.parse(host.lastHeartbeat ?? '') > Date.parse(operation.updated_at);
     const selections = queuedSelections(host.id);
     const images = new Set([config.komodo.defaultImage, ...selections.flatMap(selection => selection.image ? [selection.image] : [])]);
     const result: Target[] = [...images].map(image => {
-        const runtimeReady = host.inventory.runtimeImages?.includes(image);
+        const operation = latestOperation(host.id, 'runtime.pull', 'image', image);
+        const runtimeReady = host.inventory.runtimeImages?.includes(image) && verifiedThisEnable(operation);
         return { image, requiredBytes: host.minimumFreeBytes,
-            operation: latestOperation(host.id, 'runtime.pull', 'image', image),
+            operation,
             item: { kind: 'runtime', label: image, state: runtimeReady ? 'ready' : 'waiting', detail: runtimeReady ? 'Runtime is cached.' : 'Runtime will be downloaded automatically.' } };
     });
     for (const version of targetLibraries(selections)) {
         const cache = getHostLibraryCache(host.id, version.id);
         const matches = cache?.manifestSha256 === version.manifestSha256;
-        const ready = matches && ['ready', 'outdated'].includes(cache!.status) && cache!.bytesDownloaded === version.sizeBytes;
+        const operation = latestOperation(host.id, 'library.download', 'libraryVersionId', version.id);
+        const ready = matches && ['ready', 'outdated'].includes(cache!.status) && cache!.bytesDownloaded === version.sizeBytes && verifiedThisEnable(operation);
         result.push({ versionId: version.id, requiredBytes: version.sizeBytes - (matches ? cache!.bytesDownloaded : 0) + host.minimumFreeBytes,
-            operation: latestOperation(host.id, 'library.download', 'libraryVersionId', version.id),
+            operation,
             item: { kind: 'library', label: version.name, state: ready ? 'ready' : 'waiting',
                 detail: version.queued
                     ? ready ? 'Game files selected by queued launches are downloaded and verified.' : 'Game files selected by queued launches will be downloaded and verified automatically.'
@@ -110,7 +122,7 @@ function targets(host: Host): Target[] {
                 : target.item.cache?.status === 'verifying' ? 'Download complete. Verifying all game files.' : 'Downloading game files. Verification starts after all files arrive.';
         } else if (operation && Date.parse(host.lastHeartbeat!) <= Date.parse(operation.updated_at)) {
             target.item.state = 'preparing'; target.item.detail = 'Waiting for a fresh host inventory after preparation.';
-        } else if (retryAt(host.id, target) > Date.now()) {
+        } else if ((!enabling || (operation?.sequence ?? 0) > enabling.operation_floor) && retryAt(host.id, target) > Date.now()) {
             target.item.state = 'blocked';
             target.item.detail = `${operation?.error ?? 'Preparation failed.'} Automatic retry after ${new Date(retryAt(host.id, target)).toISOString()}.`;
         } else if (Number(host.inventory.storageAvailable ?? 0) < target.requiredBytes) {
@@ -122,7 +134,12 @@ function targets(host: Host): Target[] {
 
 export function hostPreparationStatus(host: Host): HostPreparation {
     const items = targets(host).map(target => target.item);
-    if (!items.some(item => item.kind === 'library')) items.push({ kind: 'library', label: 'Game files', state: 'waiting',
+    if (host.enablingSince) {
+        const published = latestPublishedLibraries();
+        for (const game of GAME_FILE_GAMES) if (!published.some(version => version.game === game.id)) {
+            items.push({ kind: 'library', label: game.name, state: 'blocked', detail: `Upload and publish ${game.name} game files before this host can be enabled.` });
+        }
+    } else if (!items.some(item => item.kind === 'library')) items.push({ kind: 'library', label: 'Game files', state: 'waiting',
         detail: 'Upload T5 or T6 game files; hosts download the latest available files automatically.' });
     const state = items.some(item => item.state === 'blocked') ? 'blocked'
         : items.some(item => item.state === 'preparing') ? 'preparing'
@@ -178,6 +195,12 @@ export function reconcileHostPreparation(hostId: string): HostPreparation {
                 else if (target?.versionId) queueLibraryDownload(host.id, target.versionId, { automatic: true });
             }
         }
-        return hostPreparationStatus(host);
+        const status = hostPreparationStatus(host);
+        if (host.enablingSince && !hostBlocker(host) && status.state === 'ready') {
+            db().prepare("UPDATE ec_hosts SET mode='active' WHERE id=? AND mode='disabled'").run(hostId);
+            db().prepare('DELETE FROM ec_host_enable_requests WHERE host_id=?').run(hostId);
+            scheduleAvailabilityRefresh();
+        }
+        return status;
     }).immediate();
 }

src/lib/server/hosts/schema.ts

@@ -19,6 +19,9 @@ export function ensureHostSchema(handle: Database.Database): void {
             id TEXT PRIMARY KEY, host_id TEXT NOT NULL REFERENCES ec_hosts(id), token_hash TEXT NOT NULL UNIQUE,
             created_at TEXT NOT NULL, expires_at TEXT NOT NULL, consumed_at TEXT
         );
+        CREATE TABLE IF NOT EXISTS ec_host_enable_requests (
+            host_id TEXT PRIMARY KEY REFERENCES ec_hosts(id), requested_at TEXT NOT NULL, operation_floor INTEGER NOT NULL
+        );
         CREATE TABLE IF NOT EXISTS ec_host_credentials (
             id TEXT PRIMARY KEY, host_id TEXT NOT NULL REFERENCES ec_hosts(id), token_hash TEXT NOT NULL UNIQUE,
             created_at TEXT NOT NULL, revoked_at TEXT, last_used_at TEXT

src/lib/server/provisioning/agent-events.ts

@@ -5,10 +5,11 @@ import { recordProgress } from './progress.js';
 export function recordAgentReceipt(operationId: string, state: string, at: string): void {
     // Host operations also serve installations that have no provisioning schema yet.
     if (!db().prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='native_provisioning'").get()) return;
-    const job = db().prepare(`SELECT d.id, d.phase_key, n.operation_id FROM native_provisioning n
+    const job = db().prepare(`SELECT d.id, d.phase_key, COALESCE(n.operation_id,m.remove_operation_id) AS operation_id FROM native_provisioning n
         JOIN server_deployments d ON d.id=n.deployment_id
-        WHERE (n.operation_id=? OR n.stage_operation_id=?) AND d.server_deleted_at IS NULL
-        AND d.state IN ('queued','waiting','running','awaiting-health','interrupted')`).get(operationId, operationId) as
+        LEFT JOIN ec_host_migrations m ON m.deployment_id=n.deployment_id
+        WHERE (n.operation_id=? OR n.stage_operation_id=? OR m.remove_operation_id=?) AND d.server_deleted_at IS NULL
+        AND d.state IN ('queued','waiting','running','awaiting-health','interrupted')`).get(operationId, operationId, operationId) as
         { id: string; phase_key: string | null; operation_id: string | null } | undefined;
     if (!job) return;
     const staging = job.operation_id !== operationId;

src/lib/server/provisioning/history.ts

@@ -106,7 +106,8 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
         const generatedAt = new Date().toISOString();
         const native = nativeDeployment(id);
         const agentOperation = native ? db().prepare(`SELECT o.id, o.host_id AS hostId, o.state, o.updated_at AS updatedAt, o.created_at AS createdAt, o.error
-            FROM native_provisioning n JOIN ec_host_operations o ON o.id=n.operation_id WHERE n.deployment_id=?`).get(id) as ProvisioningDetail['agentOperation'] : null;
+            FROM native_provisioning n LEFT JOIN ec_host_migrations m ON m.deployment_id=n.deployment_id
+            JOIN ec_host_operations o ON o.id=COALESCE(n.operation_id,m.remove_operation_id) WHERE n.deployment_id=?`).get(id) as ProvisioningDetail['agentOperation'] : null;
         const cancellation = db().prepare(`SELECT c.cleanup_id AS cleanupId,d.state,d.stage,d.error FROM native_creation_cancellations c
             JOIN server_deployments d ON d.id=c.cleanup_id WHERE c.creation_id=?`).get(id) as ProvisioningDetail['cancellation'];
         const { observation_json: _snapshot, phase_key: _phase, phase_started_at: _phaseStarted, observed_at: _observed, ...deployment } = row;
@@ -182,8 +183,10 @@ export function canResumeVerification(id: string): boolean {
     return !!db().prepare(`SELECT d.id FROM server_deployments d
         JOIN server_creation_jobs j ON j.deployment_id = d.id
         JOIN servers s ON s.name = j.server AND s.game_port = j.port
-        WHERE d.id = ? AND d.server_deleted_at IS NULL AND (d.action = 'create' OR (d.action = 'rebuild' AND EXISTS (
-            SELECT 1 FROM server_rebuild_jobs r WHERE r.deployment_id = d.id AND r.removed_at IS NOT NULL))) AND d.state IN ('failed', 'interrupted')
+        WHERE d.id = ? AND d.server_deleted_at IS NULL AND (d.action = 'create' OR (d.action = 'rebuild' AND (EXISTS (
+            SELECT 1 FROM server_rebuild_jobs r WHERE r.deployment_id = d.id AND r.removed_at IS NOT NULL) OR EXISTS (
+            SELECT 1 FROM ec_host_migrations m JOIN native_provisioning n ON n.deployment_id=m.deployment_id
+                WHERE m.deployment_id=d.id AND m.removed_at IS NOT NULL AND n.operation_id IS NOT NULL)))) AND d.state IN ('failed', 'interrupted')
         AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
         .get(id);
 }

src/lib/server/provisioning/host-drain.ts

@@ -0,0 +1,151 @@
+import { db } from '../db/index.js';
+import { nativeInstance, nativeRequest, type NativeInstance } from '../backends/native.js';
+import { getHost, getHostOperationStatus, listHosts, queueHostOperation, reserveHostPort, updateHost } from '../hosts/index.js';
+import { choosePlacement } from '../hosts/capacity.js';
+import { scheduleAvailabilityRefresh } from '../hosts/availability.js';
+import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
+import { getServerKey } from '../fleet/server-keys.js';
+import { captureRecipePlan, recipeForDeployment, recipeStorage, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
+import { probePlayerEvidence } from '../fleet/status-evidence.js';
+import { beginDeployment, deploymentStage } from './history.js';
+import { fleetSettings } from '../settings.js';
+import type { NativePlacement } from '../../hosts.js';
+
+interface Migration {
+    deployment_id: string; source_host_id: string; instance_id: string;
+    target_host_id: string | null; target_port: number | null; target_bind: string | null;
+    remove_operation_id: string | null; removed_at: string | null;
+}
+interface Job { id: string; server: string; action: string; state: string; stage: string }
+type Recipe = LaunchRecipePlan & { native: NativePlacement };
+
+export function hostMigration(id: string): Migration | undefined {
+    return db().prepare('SELECT * FROM ec_host_migrations WHERE deployment_id=?').get(id) as Migration | undefined;
+}
+
+/** Drain requests survive controller restarts and never make a source slot available early. */
+export function reconcileHostDrains(): void {
+    for (const host of listHosts().filter(host => host.mode === 'draining' && !host.decommission)) {
+        db().transaction(() => {
+            // Accepted launches that have no instance yet can be placed elsewhere.
+            const queued = db().prepare(`SELECT q.deployment_id,q.selection FROM native_creation_intents q
+                JOIN server_deployments d ON d.id=q.deployment_id
+                LEFT JOIN native_provisioning n ON n.deployment_id=d.id
+                WHERE d.state='queued' AND d.server_deleted_at IS NULL AND d.submitted_at IS NULL
+                AND n.operation_id IS NULL AND n.stage_operation_id IS NULL
+                AND json_extract(q.selection,'$.hostId')=?`).all(host.id) as { deployment_id: string; selection: string }[];
+            for (const row of queued) {
+                const selection: NativePlacement = JSON.parse(row.selection);
+                db().prepare('UPDATE native_creation_intents SET selection=? WHERE deployment_id=?')
+                    .run(JSON.stringify({ libraryId: selection.libraryId, image: selection.image, automatic: true, network: {} }), row.deployment_id);
+            }
+            const instances = db().prepare('SELECT * FROM native_instances WHERE host_id=? AND retired_at IS NULL').all(host.id) as NativeInstance[];
+            for (const instance of instances) {
+                const server = getServer(instance.server);
+                if (!server?.map_slug || activeBetaRemovalBlocker(instance.server)) continue;
+                if (db().prepare(`SELECT 1 FROM server_deployments WHERE server=? AND server_deleted_at IS NULL
+                    AND state IN ('queued','waiting','running','awaiting-health','interrupted')`).get(instance.server)) continue;
+                const recipe = { ...captureRecipePlan(server.game as 't5' | 't6', server.map_slug), image: instance.image,
+                    native: { libraryId: instance.library_id, image: instance.image } };
+                const id = beginDeployment(instance.server, 'rebuild', 'host-drain');
+                db().prepare("UPDATE server_deployments SET state='waiting' WHERE id=?").run(id);
+                db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,NULL,?,?,?)')
+                    .run(id, instance.server, 'zombie', recipe.mapSlug, recipe.expectedMap);
+                const stored = recipeStorage(recipe);
+                db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary) VALUES(?,?,?)').run(id, stored.recipe, stored.recipeSummary);
+                db().prepare('INSERT INTO ec_host_migrations(deployment_id,source_host_id,instance_id) VALUES(?,?,?)').run(id, host.id, instance.id);
+                deploymentStage(id, 'Host draining; waiting for an empty lobby and destination capacity');
+            }
+            const unfinished = db().prepare(`SELECT 1 FROM ec_host_migrations m JOIN server_deployments d ON d.id=m.deployment_id
+                WHERE m.source_host_id=? AND d.state NOT IN ('done','warning','cancelled') AND d.server_deleted_at IS NULL LIMIT 1`).get(host.id);
+            const ports = db().prepare('SELECT 1 FROM ec_host_ports WHERE host_id=? LIMIT 1').get(host.id);
+            if (!instances.length && !unfinished && !ports) updateHost(host.id, { mode: 'disabled' });
+        })();
+    }
+}
+
+/** Reserve a destination, remove the source once, then hand creation to the normal health-verified rebuild queue. */
+export async function processMigrationRemoval(job: Job, instance: NativeInstance, migration: Migration,
+    eligibility: () => string | null): Promise<void> {
+    const hold = (message: string) => deploymentStage(job.id, message, { blocker: { code: 'native-host', message, automatic: true } });
+    if (migration.remove_operation_id) {
+        const operation = getHostOperationStatus(migration.remove_operation_id);
+        if (operation?.state === 'failed' && Date.now() - Date.parse(operation.updatedAt) >= 60_000) {
+            // A failed receipt certifies no mutation began. Unknown results keep
+            // their reservation and original operation ID until reconciled.
+            db().transaction(() => {
+                db().prepare('DELETE FROM ec_host_ports WHERE host_id=? AND instance_id=?').run(migration.target_host_id, instance.id);
+                db().prepare(`UPDATE ec_host_migrations SET target_host_id=NULL,target_port=NULL,target_bind=NULL,
+                    remove_operation_id=NULL WHERE deployment_id=?`).run(job.id);
+                db().prepare("UPDATE server_deployments SET state='waiting',submitted_at=NULL WHERE id=?").run(job.id);
+                scheduleAvailabilityRefresh();
+            })();
+            hold('Source removal failed before mutation; checking destination capacity again before retrying.');
+            return;
+        }
+        if (operation?.state !== 'completed') {
+            hold(operation?.error ?? 'Waiting for confirmed source removal. The original agent operation will reconcile.');
+            return;
+        }
+        db().transaction(() => {
+            if (hostMigration(job.id)?.removed_at) return;
+            const current = nativeInstance(job.server);
+            if (current?.id !== instance.id || current.generation !== instance.generation || current.host_id !== migration.source_host_id) throw new Error('Migration source changed.');
+            const target = getHost(migration.target_host_id!);
+            if (!target) throw new Error('Migration destination is unavailable.');
+            const recipe = recipeForDeployment(job.id)!.recipe as Recipe;
+            recipe.native = { libraryId: instance.library_id, image: instance.image, hostId: target.id,
+                bindAddress: migration.target_bind!, publicAddress: target.publicAddress };
+            const stored = recipeStorage(recipe);
+            db().prepare('UPDATE server_launch_recipes SET recipe=?,recipe_summary=? WHERE deployment_id=?').run(stored.recipe, stored.recipeSummary, job.id);
+            db().prepare('DELETE FROM ec_host_ports WHERE host_id=? AND instance_id=?').run(instance.host_id, instance.id);
+            db().prepare(`UPDATE native_instances SET host_id=?,generation=generation+1,bind_address=?,port=?,public_address=?,public_port=NULL WHERE id=?`)
+                .run(target.id, migration.target_bind, migration.target_port, target.publicAddress, instance.id);
+            db().prepare('UPDATE native_provisioning SET generation=generation+1 WHERE deployment_id=?').run(job.id);
+            db().prepare('UPDATE server_creation_jobs SET port=? WHERE deployment_id=?').run(migration.target_port, job.id);
+            db().prepare(`UPDATE servers SET game_port=?,container_state='starting',container_checked_at=NULL,
+                last_players=NULL,last_roster=NULL,last_seen_at=NULL,last_error=NULL,active_since=NULL,inactive_since=NULL,
+                game_started_at=NULL,game_ended_at=NULL WHERE name=?`).run(migration.target_port, job.server);
+            db().prepare('UPDATE ec_host_migrations SET removed_at=? WHERE deployment_id=?').run(operation.updatedAt, job.id);
+            db().prepare("UPDATE server_deployments SET state='waiting',submitted_at=NULL WHERE id=?").run(job.id);
+            deploymentStage(job.id, `Source removed; rebuilding on ${target.name}`, { phase: 'startup-queue', blocker: null });
+            scheduleAvailabilityRefresh();
+        })();
+        return;
+    }
+    if (!getHost(instance.host_id)?.connected) { hold('Waiting for the source host to reconnect.'); return; }
+    const reason = eligibility() ?? activeBetaRemovalBlocker(job.server);
+    if (reason) { hold(reason); return; }
+    if (!getServerKey(job.server)) { hold('Save a server key before migrating this instance.'); return; }
+    const options = { libraryId: instance.library_id, image: instance.image, thorough: true, dispatch: true };
+    // No source mutation is submitted unless another host can take this instance.
+    choosePlacement(options);
+    const health = await nativeRequest<{ state: string }>(instance, 'instance.inspect');
+    if (!['stopped','exited','created','dead','missing'].includes(health.state)) {
+        const occupancy = await probePlayerEvidence(job.server);
+        if (occupancy.players !== 0) { hold(occupancy.players === null ? 'Waiting for a fresh local player count.' : `Waiting for ${occupancy.players} player(s) to leave.`); return; }
+    }
+    db().transaction(() => {
+        const state = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined;
+        if (!state || !['waiting','queued','interrupted'].includes(state.state) || hostMigration(job.id)?.remove_operation_id) return;
+        const current = nativeInstance(job.server);
+        if (current?.id !== instance.id || current.generation !== instance.generation || getHost(instance.host_id)?.mode !== 'draining') return;
+        const blocker = eligibility() ?? activeBetaRemovalBlocker(job.server);
+        if (blocker) { hold(blocker); return; }
+        const removing = db().prepare("SELECT COUNT(*) AS n FROM ec_host_operations WHERE kind='instance.remove' AND state IN ('pending','accepted','running','unknown')").get() as { n: number };
+        if (removing.n >= fleetSettings().destroyConcurrency) { hold('Waiting for destruction capacity.'); return; }
+        const { host, port } = choosePlacement(options);
+        reserveHostPort(host.id, instance.id, { port, bindAddress: host.bindAddress, protocol: 'udp' });
+        reserveHostPort(host.id, instance.id, { port, bindAddress: host.bindAddress, protocol: 'tcp' });
+        const attempt = (db().prepare(`SELECT COUNT(*) AS n FROM ec_host_operations
+            WHERE host_id=? AND instance_id=? AND kind='instance.remove' AND idempotency_key LIKE ?`)
+            .get(instance.host_id, instance.id, `${job.id}:migrate-remove:%`) as { n: number }).n;
+        const operation = queueHostOperation(instance.host_id, 'instance.remove', {}, {
+            instanceId: instance.id, generation: instance.generation, idempotencyKey: `${job.id}:migrate-remove:${attempt}`
+        });
+        db().prepare('UPDATE ec_host_migrations SET target_host_id=?,target_port=?,target_bind=?,remove_operation_id=? WHERE deployment_id=?')
+            .run(host.id, port, host.bindAddress, operation.id, job.id);
+        db().prepare("UPDATE server_deployments SET state='running',submitted_at=? WHERE id=?").run(new Date().toISOString(), job.id);
+        deploymentStage(job.id, `Capacity reserved on ${host.name}; waiting for source removal`, { phase: 'container-action', blocker: null });
+    })();
+}

src/lib/server/provisioning/native.ts

@@ -6,7 +6,7 @@ import { posix } from 'node:path';
 import { db } from '../db/index.js';
 import { config, relDestForKind, serverDir } from '../config.js';
 import { nativeInstance, nativeRequest, unplacedNativeCreation, type NativeInstance } from '../backends/native.js';
-import { getHost, getHostOperationStatus, listHosts, queueHostOperation, queueHostOperationSerialized, releaseHostPorts, reserveHostPort } from '../hosts/index.js';
+import { getHost, getHostOperationStatus, queueHostOperation, queueHostOperationSerialized, releaseHostPorts, reserveHostPort } from '../hosts/index.js';
 import { newestReadyLibraryId, assertReadyLibrary } from '../game-files/index.js';
 import { blueprintRecipe, captureRecipePlan, recipeForDeployment, recipeStorage, summarizeRecipe, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
 import { prepareNativeLaunch, evaluateNativePlacement, type PreparedNativeLaunch } from './launch-preparation.js';
@@ -27,6 +27,7 @@ import type { LaunchPreview } from '../../deployment-plan.js';
 import type { ProvisioningAction } from '../../provisioning.js';
 import { hostAgentUpdatesEnabled } from '../../hosts.js';
 import { cancelCreation, finishCreationCancellation } from './cancel-creation.js';
+import { hostMigration, processMigrationRemoval, reconcileHostDrains } from './host-drain.js';
 
 export type NativeSelection = import('../../hosts.js').NativePlacement;
 type NativeRecipe = LaunchRecipePlan & { native?: NativeSelection };
@@ -258,17 +259,7 @@ function complete(job: Job) {
 
 export async function processNativeProvisioning(eligibility: (operation: { id: string; server: string; action: string; state: string; stage: string }) => string | null,
     options: { wait?: boolean } = {}): Promise<void> {
-    // Draining prevents admission immediately and retires existing lobbies only through the normal safety queue.
-    for (const host of listHosts().filter(host => host.mode === 'draining' && host.connected)) {
-        const instances = db().prepare('SELECT * FROM native_instances WHERE host_id=? AND retired_at IS NULL').all(host.id) as NativeInstance[];
-        for (const instance of instances) {
-            if (!getServer(instance.server) || activeBetaRemovalBlocker(instance.server)) continue;
-            if (db().prepare("SELECT 1 FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND state IN ('queued','waiting','running','awaiting-health')").get(instance.server)) continue;
-            const id = beginDeployment(instance.server, 'destroy', 'host-drain');
-            db().prepare("UPDATE server_deployments SET state='waiting' WHERE id=?").run(id);
-            deploymentStage(id, 'Host draining; waiting for a fresh empty lobby');
-        }
-    }
+    reconcileHostDrains();
     const jobs = db().prepare(`SELECT d.*,n.instance_id,n.generation,n.stage_operation_id,n.operation_id,n.files_json,n.applied_at,q.selection
         FROM server_deployments d LEFT JOIN native_provisioning n ON n.deployment_id=d.id
         LEFT JOIN native_creation_intents q ON q.deployment_id=d.id
@@ -285,7 +276,7 @@ export async function processNativeProvisioning(eligibility: (operation: { id: s
         activeJobs.set(job.id, work);
         return work;
     });
-    if (options.wait !== false) await Promise.all(work);
+    if (options.wait !== false) { await Promise.all(work); reconcileHostDrains(); }
 }
 
 const activeJobs = new Map<string, Promise<void>>();
@@ -372,9 +363,14 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
         if (!instance) return;
         job.instance_id = instance.id; job.generation = instance.generation;
     }
-    if (!instance || instance.id !== job.instance_id || (instance.generation !== job.generation && !(job.action === 'rebuild' && instance.generation === job.generation + 1))) {
+    const migration = hostMigration(job.id);
+    if (!instance || instance.id !== job.instance_id || (instance.generation !== job.generation && !(job.action === 'rebuild' && !migration && instance.generation === job.generation + 1))) {
         finishDeployment(job.id, { ok: false, message: 'Original instance was retired or replaced; no command was sent.' }); return;
     }
+    if (migration && !migration.removed_at) {
+        await processMigrationRemoval(job, instance, migration, () => eligibility(job));
+        return;
+    }
     const host = getHost(instance.host_id);
     if (!host?.connected || host.mode === 'disabled') {
         db().prepare("UPDATE servers SET container_state='unknown',last_players=NULL,last_roster=NULL WHERE name=?").run(job.server);
@@ -389,6 +385,17 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
         }
         if (!operation || operation.state === 'unknown') { hold(job, 'Agent result is unknown. Reconciling the original operation; no replacement or repeated command will be sent.'); return; }
         if (operation.state === 'failed') {
+            if (migration) {
+                if (Date.now() - Date.parse(operation.updatedAt) >= 60_000) {
+                    db().transaction(() => {
+                        db().prepare('DELETE FROM restart_rollout WHERE operation_id=?').run(job.id);
+                        db().prepare('UPDATE native_provisioning SET operation_id=NULL WHERE deployment_id=?').run(job.id);
+                        db().prepare("UPDATE server_deployments SET state='waiting',submitted_at=NULL WHERE id=?").run(job.id);
+                    })();
+                }
+                hold(job, `${operation.error ?? 'Destination creation failed before mutation.'} Retrying after fresh capacity checks.`);
+                return;
+            }
             // The agent reports failed only when no mutation began. Ambiguous
             // post-mutation failures remain unknown and retain their capacity.
             db().transaction(() => {
@@ -415,7 +422,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             })();
             return;
         }
-        if (job.action === 'rebuild' && instance.generation === job.generation) {
+        if (job.action === 'rebuild' && !migration && instance.generation === job.generation) {
             db().prepare('UPDATE native_instances SET generation=? WHERE id=? AND generation=?').run(job.generation + 1, instance.id, job.generation);
             instance = nativeInstance(job.server)!;
         }
@@ -503,7 +510,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
     }
     if (job.action === 'destroy' && count(['destroy']) >= settings.destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; }
     deploymentStage(job.id, 'Checking host eligibility', { phase: 'eligibility', blocker: null });
-    if (job.action === 'create') choosePlacement({ hostId: instance.host_id, libraryId: instance.library_id, image: instance.image,
+    if (job.action === 'create' || migration) choosePlacement({ hostId: instance.host_id, libraryId: instance.library_id, image: instance.image,
         bindAddress: instance.bind_address, excludeInstanceId: instance.id, thorough: false, dispatch: true });
     if (job.action === 'create' || job.action === 'rebuild') assertReadyLibrary(instance.host_id, instance.library_id);
     const checkOccupancy = async (): Promise<boolean> => {
@@ -523,7 +530,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
         }
         return true;
     };
-    if (job.action !== 'create' && job.action !== 'start' && !await checkOccupancy()) return;
+    if (!migration && job.action !== 'create' && job.action !== 'start' && !await checkOccupancy()) return;
     const current = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined;
     if (!current || !['queued','waiting','interrupted'].includes(current.state) || eligibility(job)) return;
     let prepared: PreparedNativeLaunch | undefined;
@@ -541,8 +548,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
                 checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } });
             prepared = await prepareNativeLaunch({ game: prior.game, mapSlug: prior.mapSlug, native: prior.native,
                 instance, serverKey: preparedKey, rconPassword: config.rcon.password || randomBytes(32).toString('hex'),
-                rebuild: job.action === 'rebuild' });
-            if (job.action === 'rebuild' && !await checkOccupancy()) return;
+                rebuild: job.action === 'rebuild' && !migration });
+            if (job.action === 'rebuild' && !migration && !await checkOccupancy()) return;
         }
         const commit = db().transaction(() => {
             const state = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined;
@@ -569,7 +576,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
                     hold(job, 'Launch files or credentials changed during preparation; preparing the current selection next pass.'); return;
                 }
                 assertReadyLibrary(instance!.host_id, instance!.library_id);
-                if (job.action === 'create') choosePlacement({ hostId: instance!.host_id, libraryId: instance!.library_id, image: instance!.image,
+                if (job.action === 'create' || migration) choosePlacement({ hostId: instance!.host_id, libraryId: instance!.library_id, image: instance!.image,
                     bindAddress: instance!.bind_address, excludeInstanceId: instance!.id, thorough: false, dispatch: true });
             }
             if (job.action === 'start' || job.action === 'restart' || job.action === 'rebuild') {
@@ -590,8 +597,11 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
                 for (const file of recipe.files) recordScript(job.server, file.path, file.versionId, file.sha256, file.size, 'native-create');
                 recordProgress(job.id, { checkpoint: { key: 'prepare', label: 'Launch files selected and command payload prepared' } });
             }
-            const operationOptions = { instanceId: instance!.id, generation: instance!.generation, idempotencyKey: job.id };
-            const kind = `instance.${job.action === 'destroy' ? 'remove' : job.action}`;
+            const attempt = migration ? (db().prepare(`SELECT COUNT(*) AS n FROM ec_host_operations
+                WHERE host_id=? AND instance_id=? AND generation=? AND kind='instance.create'`).get(instance!.host_id, instance!.id, instance!.generation) as { n: number }).n : 0;
+            const operationOptions = { instanceId: instance!.id, generation: instance!.generation,
+                idempotencyKey: migration ? `${job.id}:migration-create:${attempt}` : job.id };
+            const kind = `instance.${migration ? 'create' : job.action === 'destroy' ? 'remove' : job.action}`;
             const operation = prepared ? queueHostOperationSerialized(instance!.host_id, kind,
                 Buffer.from(prepared.payloadJson.buffer, prepared.payloadJson.byteOffset, prepared.payloadJson.byteLength), operationOptions,
                 { name: instance!.server, game: prepared.recipe.game, libraryId: instance!.library_id })

src/lib/server/provisioning/requests.ts

@@ -120,7 +120,8 @@ export function cancelProvisioning(id: string, reason = 'Cancelled by requester'
     return db().transaction(() => {
         const finishedAt = new Date().toISOString();
         const result = db().prepare(`UPDATE server_deployments SET state = 'cancelled', finished_at = ?, stage = ?, error = NULL
-            WHERE id = ? AND action IN ('start','restart','stop','rebuild') AND state = 'waiting' AND submitted_at IS NULL`)
+            WHERE id = ? AND action IN ('start','restart','stop','rebuild') AND state = 'waiting' AND submitted_at IS NULL
+            AND NOT EXISTS (SELECT 1 FROM ec_host_migrations m WHERE m.deployment_id=server_deployments.id AND m.remove_operation_id IS NOT NULL)`)
             .run(finishedAt, reason, id);
         if (result.changes) recordProgress(id, { blocker: null, checks: [], event: { kind: 'outcome', message: reason, details: { state: 'cancelled', finishedAt, error: null, warnings: [] } } });
         return result.changes > 0;

src/lib/ui/components/HostPreparation.svelte

@@ -6,7 +6,7 @@
 </script>
 
 <div class="preparation" aria-live="polite">
-    <p>The default runtime and the latest available T5 and T6 game files are prepared automatically on active hosts. Running servers keep their current files.</p>
+    <p>The default runtime and the latest available T5 and T6 game files are prepared automatically. Re-enabling a host requires a fresh runtime pull and verification of both BO1 (T5) and BO2 (T6) libraries before it accepts deployments. Running servers keep their current files.</p>
     {#if host.preparation}
         <p class="status" class:blocked={host.preparation.state === 'blocked'}>Automatic setup: {host.preparation.state === 'ready' ? 'Ready' : host.preparation.state === 'preparing' ? 'In progress' : host.preparation.state === 'blocked' ? 'Needs attention' : 'Waiting'}</p>
         <ul>

src/routes/api/hosts/[id]/+server.ts

@@ -5,6 +5,8 @@ import { hostApiError, objectBody, requireManagementOrigin } from '$lib/server/h
 import { presentHostPreparation } from '$lib/server/hosts/preparation.js';
 import { config } from '$lib/server/config.js';
 import { hostInstances } from '$lib/server/hosts/instances.js';
+import { reconcileHostPreparation } from '$lib/server/hosts/preparation.js';
+import { kickProvisioning } from '$lib/server/pipeline/worker.js';
 type Event = { request: Request; params: { id: string } };
 export function GET({ request, params }: Event) {
     requireManagementOrigin(request); const host = getHost(params.id); if (!host) error(404, 'Host not found.');
@@ -12,7 +14,14 @@ export function GET({ request, params }: Event) {
 }
 export async function PATCH({ request, params }: Event) {
     requireManagementOrigin(request);
-    try { return json({ host: updateHost(params.id, await objectBody(request)) }); } catch (cause) { return hostApiError(cause); }
+    try {
+        const body = await objectBody(request);
+        if (body.mode !== undefined && body.mode !== 'active' && body.mode !== 'draining') throw new Error('Choose Enable or Drain host. Hosts disable automatically after draining.');
+        updateHost(params.id, body);
+        if (body.mode === 'active') reconcileHostPreparation(params.id);
+        if (body.mode === 'draining') void kickProvisioning().catch(cause => console.error('[host-drain]', cause));
+        return json({ host: presentHostPreparation(getHost(params.id)!) });
+    } catch (cause) { return hostApiError(cause); }
 }
 
 export function DELETE({ request, params }: Event) {

src/routes/hosts/[id]/+page.svelte

@@ -4,7 +4,7 @@
     let requestSequence = 0;
     import { page } from '$app/stores';
     import { goto } from '$app/navigation';
-    import type { Host, HostMode, CapacityCalibration } from '$lib/hosts.js';
+    import type { Host, CapacityCalibration } from '$lib/hosts.js';
     import { hostAgentUpdateLabel, hostAgentUpdateMessage, hostAgentUpdatesEnabled, hostDeletionMessage, hostReadinessLabel } from '$lib/hosts.js';
     import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
     import Button from '$ui/components/primitives/Button.svelte';
@@ -14,11 +14,13 @@
     import ServerCard from '$ui/components/ServerCard.svelte';
     import type { ServerObservation } from '$lib/server-observation.js';
     import { syncServerTime } from '$ui/stores/servers.js';
+    import Dialog from '$ui/components/primitives/Dialog.svelte';
     let host = $state<Host | null>(null);
     let instances = $state<ServerObservation[]>([]);
     let calibration = $state<CapacityCalibration | null>(null);
     let loading = $state(true); let busy = $state(false); let error = $state(''); let notice = $state('');
     let deleteOpen = $state(false);
+    let drainOpen = $state(false);
     let name = $state(''); let portMin = $state(28960); let portMax = $state(28980); let bindAddress = $state('0.0.0.0'); let publicAddress = $state(''); let minimumFreeGB = $state(1);
     let runtimeImage = $state('ghcr.io/ayymoss/plutainer:latest'); let enrollment = $state<{ command: string; expiresAt: string } | null>(null);
     const endpoint = $derived(`/api/hosts/${$page.params.id}`);
@@ -38,7 +40,12 @@
         catch (cause) { error = (cause as Error).message; } finally { busy = false; }
     }
     async function save(event: SubmitEvent) { event.preventDefault(); await action('', 'PATCH', { name, portMin, portMax, bindAddress, publicAddress, minimumFreeBytes: Math.round(minimumFreeGB * 1073741824) }, 'Host settings saved.'); }
-    async function mode(value: HostMode) { await action('', 'PATCH', { mode: value }, value === 'draining' ? 'Drain enabled. Existing games continue until safe retirement.' : `Host ${value === 'active' ? 'enabled' : 'disabled'}.`); }
+    async function mode(value: 'active' | 'draining') {
+        await action('', 'PATCH', { mode: value }, value === 'draining'
+            ? 'Drain started. Empty instances migrate as destination capacity becomes available.'
+            : 'Automatic setup started. This host will enable after the runtime and both game libraries are verified.');
+        if (!error) drainOpen = false;
+    }
     $effect(() => {
         void endpoint;
         return untrack(() => {
@@ -57,8 +64,8 @@
         {#if host.decommission}<section class="panel" aria-live="polite"><h2>Host cleanup</h2><p>{hostDeletionMessage(host)}</p>{#if host.decommission.error}<p class="ui-error">{host.decommission.error}</p>{/if}<p>Requested {new Date(host.decommission.requestedAt).toLocaleString()}. This page closes only after the cleanup worker confirms removal.</p>{#if host.decommission.stage === 'cleaned'}<p>The agent is expected to go offline during uninstall. If final confirmation never arrives, inspect the machine; loss of contact alone does not prove completion.</p>{/if}</section>{/if}
         <section class="panel"><div class="row"><h2>{host.connectionState} · {hostReadinessLabel(host.readiness)}</h2><span>Last heartbeat: {host.lastHeartbeat ? new Date(host.lastHeartbeat).toLocaleString() : 'Never'}</span></div>
             {#if host.readinessReasons.length}<ul>{#each host.readinessReasons as reason}<li>{reason}</li>{/each}</ul>{:else}<p>Connected and ready. Each deployment also checks its latest game files, runtime, and ports.</p>{/if}
-            {#if canManage && !cleaning}<div class="actions"><Button disabled={busy || host.mode === 'active'} onclick={() => mode('active')}>Enable</Button><Button disabled={busy || host.mode === 'draining'} onclick={() => mode('draining')}>Drain host</Button><Button disabled={busy || host.mode === 'disabled'} onclick={() => mode('disabled')}>Disable host</Button></div>{/if}
-            <p>Draining prevents new deployments and retires instances as they become safe to remove. Disabled hosts receive no new commands; running game containers continue.</p>
+            {#if canManage && !cleaning}<div class="actions"><Button disabled={busy || host.mode !== 'disabled' || !!host.enablingSince} onclick={() => mode('active')}>{host.enablingSince ? 'Enabling…' : 'Enable'}</Button><Button disabled={busy || host.mode !== 'active'} onclick={() => drainOpen = true}>Drain host</Button></div>{/if}
+            <p>Draining blocks new deployments and migrates empty instances to other hosts with capacity. The host disables when migration is complete and it is empty. Enabling refreshes the runtime and verifies BO1 and BO2 game files before returning the host to the capacity pool.</p>
         </section>
         <section class="panel" aria-labelledby="running-instances-heading">
             <h2 id="running-instances-heading">Running instances · {instances.length}</h2>
@@ -108,6 +115,12 @@
     {/if}
 </section>
 <DeleteHostDialog bind:open={deleteOpen} {host} ondeleted={async (result) => { if (result.deleted) await goto('/hosts'); else { enrollment = null; await load(); } }} />
+<Dialog bind:open={drainOpen} title="Drain {host?.name ?? 'host'}?" {busy}>
+    <p>New deployments will stop on this host. Each instance will wait until its lobby is empty and another host has capacity, then be destroyed here and rebuilt there with the same server identity and map. Its address or port may change.</p>
+    <p>If capacity is unavailable, instances stay here until space becomes available. After all migrations finish, this host will be disabled.</p>
+    {#if error}<p class="ui-error" role="alert">{error}</p>{/if}
+    {#snippet actions()}<Button disabled={busy} onclick={() => drainOpen = false}>Cancel</Button><Button variant="danger" disabled={busy} onclick={() => mode('draining')}>{busy ? 'Starting drain…' : 'Confirm drain'}</Button>{/snippet}
+</Dialog>
 
 <style>
     .instances { display:grid; grid-template-columns:repeat(auto-fit,minmax(min(100%,300px),1fr)); gap:16px; }