Migrate instances when draining and verify hosts before enabling

src/lib/hosts.ts

@@ -94,6 +94,7 @@ export interface Host {
     enrolled: boolean; credentialActive: boolean;
     decommission: HostDecommission | null;
     preparation?: HostPreparation;
+    enablingSince?: string | null;
 }
 export interface HostOperation {
     id: string; hostId: string; kind: string; instanceId: string | null; generation: number | null;

src/lib/server/backends/schema.ts

@@ -23,6 +23,14 @@ export function migrateNativeInstances(handle: Database.Database): void {
             applied_at TEXT, verified_at TEXT
         );
         CREATE INDEX IF NOT EXISTS native_provisioning_instance ON native_provisioning(instance_id);
+        CREATE TABLE IF NOT EXISTS ec_host_migrations (
+            deployment_id TEXT PRIMARY KEY REFERENCES server_deployments(id),
+            source_host_id TEXT NOT NULL, instance_id TEXT NOT NULL REFERENCES native_instances(id),
+            target_host_id TEXT, target_port INTEGER, target_bind TEXT,
+            remove_operation_id TEXT, removed_at TEXT
+        );
+        CREATE INDEX IF NOT EXISTS ec_host_migrations_source ON ec_host_migrations(source_host_id);
+        CREATE INDEX IF NOT EXISTS ec_host_migrations_target ON ec_host_migrations(target_host_id);
         CREATE TABLE IF NOT EXISTS native_creation_cancellations (
             creation_id TEXT PRIMARY KEY REFERENCES server_deployments(id),
             cleanup_id TEXT NOT NULL UNIQUE REFERENCES server_deployments(id)

src/lib/server/game-files/index.ts

@@ -386,7 +386,7 @@ export function queueLibraryDownload(hostId: string, versionId: string, options:
     if (libraryFilesRetired(versionId)) throw new Error('These outdated game files have been removed. Upload them again to use them.');
     if (database().prepare("SELECT 1 FROM ec_host_operations WHERE host_id=? AND kind='library.remove' AND json_extract(payload,'$.libraryVersionId')=? AND state IN ('pending','accepted','running','unknown')").get(hostId, versionId)) throw new Error('This host copy is being removed.');
     const host = getHost(hostId);
-    if (!host || !host.connected || !host.credentialActive || host.mode === 'disabled') throw new Error('Choose an online, enrolled host.');
+    if (!host || !host.connected || !host.credentialActive || (host.mode === 'disabled' && !host.enablingSince)) throw new Error('Choose an online, enrolled host.');
     const prior = getHostLibraryCache(hostId, versionId);
     if (prior?.operationId) {
         const operation = getHostOperation(prior.operationId);

src/lib/server/hosts/availability.ts

@@ -52,7 +52,9 @@ function observeHost(host: Host): HostAvailability {
         JOIN native_provisioning n ON n.instance_id=i.id JOIN server_deployments d ON d.id=n.deployment_id
         WHERE i.host_id=? AND i.retired_at IS NULL AND d.action IN ('create','rebuild')
         AND d.state IN ('running','awaiting-health')`).all(host.id) as { id: string; generation: number }[];
-    const unmeasured = pending.filter(instance => measured.get(instance.id) !== instance.generation).length;
+    const incoming = handle.prepare(`SELECT COUNT(*) AS n FROM ec_host_migrations m JOIN server_deployments d ON d.id=m.deployment_id
+        WHERE m.target_host_id=? AND (m.removed_at IS NULL OR d.state IN ('queued','waiting','interrupted'))`).get(host.id) as { n: number };
+    const unmeasured = pending.filter(instance => measured.get(instance.id) !== instance.generation).length + incoming.n;
     const { cpuCount, cpuUsagePercent, memoryTotal, memoryAvailable } = host.inventory;
     const freeCpu = cpuCount && cpuUsagePercent !== undefined
         ? Math.max(0, cpuCount * (0.95 - cpuUsagePercent / 100) - unmeasured * startup.cpuCores) : null;

src/lib/server/hosts/capacity.ts

@@ -46,7 +46,7 @@ function projectedLoad(host: Host, options: PlacementOptions): number {
     }
     const instances = db().prepare(`SELECT i.id, i.generation,
         NOT EXISTS (SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
-            WHERE n.instance_id=i.id AND d.action='create' AND d.state IN ('done','warning') AND d.finished_at<?) AS pending,
+            WHERE n.instance_id=i.id AND n.generation=i.generation AND d.action IN ('create','rebuild') AND d.state IN ('done','warning') AND d.finished_at<?) AS pending,
         EXISTS (SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
             WHERE n.instance_id=i.id AND d.action='create' AND d.state='queued' AND d.submitted_at IS NULL
             AND n.operation_id IS NULL AND n.stage_operation_id IS NULL) AS unsubmitted
@@ -54,13 +54,15 @@ function projectedLoad(host: Host, options: PlacementOptions): number {
         .all(host.lastHeartbeat, host.id, options.excludeInstanceId ?? '') as
         { id: string; generation: number; pending: number; unsubmitted: number }[];
     const { running, startup } = capacityCalibration(host.id);
+    const incoming = db().prepare(`SELECT COUNT(*) AS n FROM ec_host_migrations
+        WHERE target_host_id=? AND removed_at IS NULL AND instance_id!=?`).get(host.id, options.excludeInstanceId ?? '') as { n: number };
     const planned = options.planned?.filter(item => item.hostId === host.id).length ?? 0;
     // Admission budgets the eventual running collection. Dispatch budgets only
     // the next startup; the rest of the queue waits for live capacity to recover.
     const launch = options.dispatch ? startup : running;
-    let extraCpu = planned * running.cpuCores + launch.cpuCores;
-    let extraMemory = planned * running.memoryBytes + launch.memoryBytes;
-    let reservations = planned + 1;
+    let extraCpu = planned * running.cpuCores + launch.cpuCores + incoming.n * startup.cpuCores;
+    let extraMemory = planned * running.memoryBytes + launch.memoryBytes + incoming.n * startup.memoryBytes;
+    let reservations = planned + 1 + incoming.n;
     const measurements = new Map(host.inventory.instanceResources?.map(sample => [sample.instanceId, sample]));
     for (const instance of instances) {
         if (options.dispatch && options.excludeInstanceId && instance.unsubmitted) continue;

src/lib/server/hosts/index.ts

@@ -37,6 +37,7 @@ function presentDecommission(row: DecommissionRow | undefined): HostDecommission
         error: row.error, retryable: row.stage === 'failed' && getHostOperationStatus(row.operation_id)?.state === 'failed' } : null;
 }
 function hostFromRow(row: HostRow): Host {
+    const enabling = database().prepare('SELECT requested_at FROM ec_host_enable_requests WHERE host_id=?').get(row.id) as { requested_at: string } | undefined;
     const inventory: HostInventory = JSON.parse(row.inventory);
     const connected = !!row.credential_active && !!row.last_heartbeat && Date.now() - Date.parse(row.last_heartbeat) <= HEARTBEAT_MAX_AGE_MS;
     const reasons: string[] = [];
@@ -53,13 +54,14 @@ function hostFromRow(row: HostRow): Host {
     if (!runtimeReady) reasons.push('The Plutainer runtime image is not ready yet.');
     if (!libraryReady) reasons.push('No verified game-file library is available yet.');
     if (row.mode === 'draining') reasons.push('Host is draining; new deployments are blocked.');
-    if (row.mode === 'disabled') reasons.push('Host is disabled.');
+    if (row.mode === 'disabled') reasons.push(enabling ? 'Host setup must refresh the runtime and verify both BO1 and BO2 game files before enabling.' : 'Host is disabled.');
     const decommission = presentDecommission(decommissionRow(row.id));
     if (decommission) reasons.push('Host deletion is in progress; waiting for confirmed cleanup and agent removal.');
     return { id: row.id, name: row.name, hostname: row.hostname, mode: row.mode, portMin: row.port_min, portMax: row.port_max,
         bindAddress: row.bind_address, publicAddress: row.public_address, minimumFreeBytes: row.minimum_free_bytes,
         createdAt: row.created_at, lastHeartbeat: row.last_heartbeat, connected, connectionState: connected ? 'online' : 'offline',
-        readiness: decommission ? 'deleting' : row.mode === 'draining' ? 'draining' : row.mode === 'disabled' ? 'disabled' : !row.enrolled_at ? 'preparing' : !infrastructureReady ? 'degraded' : !runtimeReady ? 'needs-runtime' : !libraryReady ? 'needs-game-files' : 'ready',
+        enablingSince: enabling?.requested_at ?? null,
+        readiness: decommission ? 'deleting' : enabling ? 'preparing' : row.mode === 'draining' ? 'draining' : row.mode === 'disabled' ? 'disabled' : !row.enrolled_at ? 'preparing' : !infrastructureReady ? 'degraded' : !runtimeReady ? 'needs-runtime' : !libraryReady ? 'needs-game-files' : 'ready',
         readinessReasons: reasons, inventory, enrolled: !!row.enrolled_at, credentialActive: !!row.credential_active, decommission };
 }
 export function listHosts(): Host[] { return (database().prepare(hostSql + ' WHERE h.deleted_at IS NULL ORDER BY h.name, h.id').all() as HostRow[]).map(hostFromRow); }
@@ -197,7 +199,7 @@ export function updateHost(id: string, input: Record<string, unknown>): Host {
     const host = requireHost(id);
     if (host.decommission) throw new Error('Host cleanup is in progress; its settings cannot change.');
     const name = input.name === undefined ? host.name : textField(input.name, 'Host name');
-    const mode = input.mode ?? host.mode;
+    let mode = input.mode ?? host.mode;
     if (!['active', 'draining', 'disabled'].includes(mode as string)) throw new Error('Host mode must be active, draining, or disabled.');
     const min = port(input.portMin ?? host.portMin); const max = port(input.portMax ?? host.portMax);
     if (max < min) throw new Error('The last game port must not precede the first.');
@@ -206,7 +208,20 @@ export function updateHost(id: string, input: Record<string, unknown>): Host {
     const publicAddress = input.publicAddress === undefined ? host.publicAddress : input.publicAddress === '' ? '' : textField(input.publicAddress, 'Public address');
     const minimum = input.minimumFreeBytes ?? host.minimumFreeBytes;
     if (typeof minimum !== 'number' || !Number.isSafeInteger(minimum) || minimum < 0) throw new Error('Minimum free storage must be a nonnegative byte count.');
-    database().prepare('UPDATE ec_hosts SET name=?,mode=?,port_min=?,port_max=?,bind_address=?,public_address=?,minimum_free_bytes=? WHERE id=?').run(name, mode, min, max, bind, publicAddress, minimum, id);
+    database().transaction(() => {
+        if (input.mode === 'active' && host.mode === 'draining') throw new Error('Wait for this host to finish draining before enabling it.');
+        if (input.mode === 'active' && host.mode === 'disabled') {
+            database().prepare('INSERT OR IGNORE INTO ec_host_enable_requests(host_id,requested_at,operation_floor) VALUES(?,?,(SELECT COALESCE(MAX(rowid),0) FROM ec_host_operations))').run(id, now());
+            mode = 'disabled';
+        }
+        if (input.mode === 'draining') {
+            if (host.enablingSince) throw new Error('Wait for automatic host setup to finish before draining.');
+            const incoming = database().prepare(`SELECT 1 FROM ec_host_migrations m JOIN server_deployments d ON d.id=m.deployment_id
+                WHERE m.target_host_id=? AND d.state NOT IN ('done','warning') AND d.server_deleted_at IS NULL LIMIT 1`).get(id);
+            if (incoming) throw new Error('Wait for incoming instance migrations to finish before draining this host.');
+        }
+        database().prepare('UPDATE ec_hosts SET name=?,mode=?,port_min=?,port_max=?,bind_address=?,public_address=?,minimum_free_bytes=? WHERE id=?').run(name, mode, min, max, bind, publicAddress, minimum, id);
+    })();
     scheduleAvailabilityRefresh();
     return requireHost(id);
 }
@@ -435,7 +450,7 @@ function enqueueHostOperation(hostId: string, kind: string, payload: string | Bu
             }
         }
         if (['instance.create','create'].includes(kind)) requireReadyHost(hostId);
-        if (host.mode === 'disabled' && kind !== 'instance.cancel-create') throw new Error('Host is disabled. Enable it before requesting operations.');
+        if (host.mode === 'disabled' && kind !== 'instance.cancel-create' && !(host.enablingSince && ['runtime.pull','library.download','host.inspect'].includes(kind))) throw new Error('Host is disabled. Enable it before requesting operations.');
         const id = options.id ?? randomUUID(); const timestamp = now();
         const serverName = options.instanceId ? (database().prepare('SELECT server FROM native_instances WHERE id=?')
             .get(options.instanceId) as { server: string } | undefined)?.server ?? null : null;
@@ -461,7 +476,7 @@ export function pendingHostOperationRows(hostId: string, lane: OperationLane, in
 /** Delivery snapshots retain status/results while large immutable payloads stay in SQLite. */
 export function pendingHostOperationRows(hostId: string, lane: OperationLane = 'all', includePayload = true): OperationDeliveryRow[] {
     const host = requireHost(hostId);
-    if (host.mode === 'disabled' || !host.connected) return [];
+    if ((host.mode === 'disabled' && !host.enablingSince) || !host.connected) return [];
     // Older agents hold activity.lock across the entire delivered batch, including
     // observations. Filtering only the unsupported cleanup can keep the updater
     // busy indefinitely. Drain the old batch by withholding ALL further work;
@@ -484,12 +499,13 @@ export function pendingHostOperationRows(hostId: string, lane: OperationLane = '
         const columns = includePayload ? `${operationStatusColumns},${operationPayloadSql} AS payload` : operationStatusColumns;
         const rows = database().prepare(`SELECT ${columns} FROM ec_host_operations o WHERE host_id=? AND state IN ('pending','accepted','running','unknown')
             ${laneFilter}
+            ${host.mode === 'disabled' ? "AND kind IN ('runtime.pull','library.download','host.inspect')" : ''}
             AND (? OR state != 'pending' OR kind NOT IN ('instance.create','create'))
             AND (kind!='instance.cancel-create' OR ?)
             AND NOT EXISTS (SELECT 1 FROM ec_host_operations cancellation WHERE cancellation.host_id=o.host_id
                 AND cancellation.kind='instance.cancel-create' AND json_extract(cancellation.payload,'$.createOperationId')=o.id)
             ORDER BY CASE WHEN kind='instance.cancel-create' THEN 0 ELSE 1 END,COALESCE(last_delivered_at,''),created_at,rowid LIMIT ${limit}`)
-            .all(hostId, Number(host.readiness === 'ready'), Number(host.inventory.capabilities?.includes('instance.cancel-create') ?? false)) as OperationDeliveryRow[];
+            .all(hostId, Number(host.readiness === 'ready' || host.mode === 'draining'), Number(host.inventory.capabilities?.includes('instance.cancel-create') ?? false)) as OperationDeliveryRow[];
         const deliveredAt = now();
         for (const operation of rows) database().prepare('UPDATE ec_host_operations SET last_delivered_at=? WHERE id=?').run(deliveredAt, operation.id);
         return rows;

src/lib/server/hosts/preparation.ts

@@ -4,10 +4,11 @@ import { AGENT_PROTOCOL_VERSION, getHost, queueHostOperation } from './index.js'
 import { getHostLibraryCache, latestPublishedLibraries, queueLibraryDownload } from '../game-files/index.js';
 import { GAME_FILE_GAMES } from '../../game-files.js';
 import type { Host, HostOperationState, HostPreparation, HostPreparationItem } from '../../hosts.js';
+import { scheduleAvailabilityRefresh } from './availability.js';
 
 const activeStates = ['pending', 'accepted', 'running', 'unknown'];
 const supersededDownload = 'Superseded by newer game files before delivery to the host.';
-type PreparationOperation = { id: string; state: HostOperationState; updated_at: string; error: string | null };
+type PreparationOperation = { id: string; sequence: number; state: HostOperationState; updated_at: string; error: string | null };
 type Target = { item: HostPreparationItem; image?: string; versionId?: string; requiredBytes: number; operation?: PreparationOperation };
 type QueuedSelection = { libraryId: string; image: string | null };
 type LibraryTarget = Pick<ReturnType<typeof latestPublishedLibraries>[number], 'id' | 'sizeBytes' | 'manifestSha256'> & { name: string; queued?: boolean };
@@ -23,7 +24,12 @@ function queuedSelections(hostId: string): QueuedSelection[] {
         WHERE d.action='create' AND d.server_deleted_at IS NULL AND n.deployment_id IS NULL
         AND d.state IN ('queued','waiting','running','awaiting-health','interrupted')
         AND (json_extract(q.selection,'$.automatic')=1 OR json_extract(q.selection,'$.hostId') IS NULL
-            OR json_extract(q.selection,'$.hostId')=?)`).all(hostId) as QueuedSelection[];
+            OR json_extract(q.selection,'$.hostId')=?)
+        UNION SELECT i.library_id AS libraryId,i.image FROM ec_host_migrations m
+            JOIN native_instances i ON i.id=m.instance_id JOIN server_deployments d ON d.id=m.deployment_id
+            WHERE m.removed_at IS NULL AND d.server_deleted_at IS NULL
+            AND d.state IN ('queued','waiting','running','awaiting-health','interrupted')
+            AND (m.target_host_id IS NULL OR m.target_host_id=?)`).all(hostId, hostId) as QueuedSelection[];
 }
 
 function targetLibraries(selections: QueuedSelection[]): LibraryTarget[] {
@@ -44,7 +50,7 @@ function targetLibraries(selections: QueuedSelection[]): LibraryTarget[] {
 }
 
 function latestOperation(hostId: string, kind: string, field: string, value: string): PreparationOperation | undefined {
-    return db().prepare(`SELECT id,state,updated_at,error FROM ec_host_operations
+    return db().prepare(`SELECT id,rowid AS sequence,state,updated_at,error FROM ec_host_operations
         WHERE host_id=? AND kind=? AND json_extract(payload,?)=? ORDER BY created_at DESC,rowid DESC LIMIT 1`)
         .get(hostId, kind, `$.${field}`, value) as PreparationOperation | undefined;
 }
@@ -67,7 +73,7 @@ function retryAt(hostId: string, target: Target): number {
 
 function hostBlocker(host: Host): { state: 'waiting' | 'blocked'; detail: string } | null {
     if (host.decommission) return { state: 'waiting', detail: 'Automatic preparation is paused during host deletion.' };
-    if (host.mode !== 'active') return { state: 'waiting', detail: `Automatic preparation is paused while this host is ${host.mode}.` };
+    if (host.mode !== 'active' && !host.enablingSince) return { state: 'waiting', detail: `Automatic preparation is paused while this host is ${host.mode}.` };
     if (!host.enrolled) return { state: 'waiting', detail: 'Automatic preparation starts after agent installation.' };
     if (!host.connected || !host.credentialActive) return { state: 'waiting', detail: 'Waiting for the host to reconnect.' };
     if (host.inventory.protocolVersion !== AGENT_PROTOCOL_VERSION) return { state: 'blocked', detail: 'A compatible host agent is required.' };
@@ -77,20 +83,26 @@ function hostBlocker(host: Host): { state: 'waiting' | 'blocked'; detail: string
 }
 
 function targets(host: Host): Target[] {
+    const enabling = db().prepare('SELECT operation_floor FROM ec_host_enable_requests WHERE host_id=?').get(host.id) as { operation_floor: number } | undefined;
+    const verifiedThisEnable = (operation: PreparationOperation | undefined) => !enabling || !!operation
+        && operation.sequence > enabling.operation_floor && operation.state === 'completed'
+        && Date.parse(host.lastHeartbeat ?? '') > Date.parse(operation.updated_at);
     const selections = queuedSelections(host.id);
     const images = new Set([config.komodo.defaultImage, ...selections.flatMap(selection => selection.image ? [selection.image] : [])]);
     const result: Target[] = [...images].map(image => {
-        const runtimeReady = host.inventory.runtimeImages?.includes(image);
+        const operation = latestOperation(host.id, 'runtime.pull', 'image', image);
+        const runtimeReady = host.inventory.runtimeImages?.includes(image) && verifiedThisEnable(operation);
         return { image, requiredBytes: host.minimumFreeBytes,
-            operation: latestOperation(host.id, 'runtime.pull', 'image', image),
+            operation,
             item: { kind: 'runtime', label: image, state: runtimeReady ? 'ready' : 'waiting', detail: runtimeReady ? 'Runtime is cached.' : 'Runtime will be downloaded automatically.' } };
     });
     for (const version of targetLibraries(selections)) {
         const cache = getHostLibraryCache(host.id, version.id);
         const matches = cache?.manifestSha256 === version.manifestSha256;
-        const ready = matches && ['ready', 'outdated'].includes(cache!.status) && cache!.bytesDownloaded === version.sizeBytes;
+        const operation = latestOperation(host.id, 'library.download', 'libraryVersionId', version.id);
+        const ready = matches && ['ready', 'outdated'].includes(cache!.status) && cache!.bytesDownloaded === version.sizeBytes && verifiedThisEnable(operation);
         result.push({ versionId: version.id, requiredBytes: version.sizeBytes - (matches ? cache!.bytesDownloaded : 0) + host.minimumFreeBytes,
-            operation: latestOperation(host.id, 'library.download', 'libraryVersionId', version.id),
+            operation,
             item: { kind: 'library', label: version.name, state: ready ? 'ready' : 'waiting',
                 detail: version.queued
                     ? ready ? 'Game files selected by queued launches are downloaded and verified.' : 'Game files selected by queued launches will be downloaded and verified automatically.'
@@ -110,7 +122,7 @@ function targets(host: Host): Target[] {
                 : target.item.cache?.status === 'verifying' ? 'Download complete. Verifying all game files.' : 'Downloading game files. Verification starts after all files arrive.';
         } else if (operation && Date.parse(host.lastHeartbeat!) <= Date.parse(operation.updated_at)) {
             target.item.state = 'preparing'; target.item.detail = 'Waiting for a fresh host inventory after preparation.';
-        } else if (retryAt(host.id, target) > Date.now()) {
+        } else if ((!enabling || (operation?.sequence ?? 0) > enabling.operation_floor) && retryAt(host.id, target) > Date.now()) {
             target.item.state = 'blocked';
             target.item.detail = `${operation?.error ?? 'Preparation failed.'} Automatic retry after ${new Date(retryAt(host.id, target)).toISOString()}.`;
         } else if (Number(host.inventory.storageAvailable ?? 0) < target.requiredBytes) {
@@ -122,7 +134,12 @@ function targets(host: Host): Target[] {
 
 export function hostPreparationStatus(host: Host): HostPreparation {
     const items = targets(host).map(target => target.item);
-    if (!items.some(item => item.kind === 'library')) items.push({ kind: 'library', label: 'Game files', state: 'waiting',
+    if (host.enablingSince) {
+        const published = latestPublishedLibraries();
+        for (const game of GAME_FILE_GAMES) if (!published.some(version => version.game === game.id)) {
+            items.push({ kind: 'library', label: game.name, state: 'blocked', detail: `Upload and publish ${game.name} game files before this host can be enabled.` });
+        }
+    } else if (!items.some(item => item.kind === 'library')) items.push({ kind: 'library', label: 'Game files', state: 'waiting',
         detail: 'Upload T5 or T6 game files; hosts download the latest available files automatically.' });
     const state = items.some(item => item.state === 'blocked') ? 'blocked'
         : items.some(item => item.state === 'preparing') ? 'preparing'
@@ -178,6 +195,12 @@ export function reconcileHostPreparation(hostId: string): HostPreparation {
                 else if (target?.versionId) queueLibraryDownload(host.id, target.versionId, { automatic: true });
             }
         }
-        return hostPreparationStatus(host);
+        const status = hostPreparationStatus(host);
+        if (host.enablingSince && !hostBlocker(host) && status.state === 'ready') {
+            db().prepare("UPDATE ec_hosts SET mode='active' WHERE id=? AND mode='disabled'").run(hostId);
+            db().prepare('DELETE FROM ec_host_enable_requests WHERE host_id=?').run(hostId);
+            scheduleAvailabilityRefresh();
+        }
+        return status;
     }).immediate();
 }

src/lib/server/hosts/schema.ts

@@ -19,6 +19,9 @@ export function ensureHostSchema(handle: Database.Database): void {
             id TEXT PRIMARY KEY, host_id TEXT NOT NULL REFERENCES ec_hosts(id), token_hash TEXT NOT NULL UNIQUE,
             created_at TEXT NOT NULL, expires_at TEXT NOT NULL, consumed_at TEXT
         );
+        CREATE TABLE IF NOT EXISTS ec_host_enable_requests (
+            host_id TEXT PRIMARY KEY REFERENCES ec_hosts(id), requested_at TEXT NOT NULL, operation_floor INTEGER NOT NULL
+        );
         CREATE TABLE IF NOT EXISTS ec_host_credentials (
             id TEXT PRIMARY KEY, host_id TEXT NOT NULL REFERENCES ec_hosts(id), token_hash TEXT NOT NULL UNIQUE,
             created_at TEXT NOT NULL, revoked_at TEXT, last_used_at TEXT

src/lib/server/provisioning/agent-events.ts

@@ -5,10 +5,11 @@ import { recordProgress } from './progress.js';
 export function recordAgentReceipt(operationId: string, state: string, at: string): void {
     // Host operations also serve installations that have no provisioning schema yet.
     if (!db().prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='native_provisioning'").get()) return;
-    const job = db().prepare(`SELECT d.id, d.phase_key, n.operation_id FROM native_provisioning n
+    const job = db().prepare(`SELECT d.id, d.phase_key, COALESCE(n.operation_id,m.remove_operation_id) AS operation_id FROM native_provisioning n
         JOIN server_deployments d ON d.id=n.deployment_id
-        WHERE (n.operation_id=? OR n.stage_operation_id=?) AND d.server_deleted_at IS NULL
-        AND d.state IN ('queued','waiting','running','awaiting-health','interrupted')`).get(operationId, operationId) as
+        LEFT JOIN ec_host_migrations m ON m.deployment_id=n.deployment_id
+        WHERE (n.operation_id=? OR n.stage_operation_id=? OR m.remove_operation_id=?) AND d.server_deleted_at IS NULL
+        AND d.state IN ('queued','waiting','running','awaiting-health','interrupted')`).get(operationId, operationId, operationId) as
         { id: string; phase_key: string | null; operation_id: string | null } | undefined;
     if (!job) return;
     const staging = job.operation_id !== operationId;

src/lib/server/provisioning/history.ts

@@ -106,7 +106,8 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
         const generatedAt = new Date().toISOString();
         const native = nativeDeployment(id);
         const agentOperation = native ? db().prepare(`SELECT o.id, o.host_id AS hostId, o.state, o.updated_at AS updatedAt, o.created_at AS createdAt, o.error
-            FROM native_provisioning n JOIN ec_host_operations o ON o.id=n.operation_id WHERE n.deployment_id=?`).get(id) as ProvisioningDetail['agentOperation'] : null;
+            FROM native_provisioning n LEFT JOIN ec_host_migrations m ON m.deployment_id=n.deployment_id
+            JOIN ec_host_operations o ON o.id=COALESCE(n.operation_id,m.remove_operation_id) WHERE n.deployment_id=?`).get(id) as ProvisioningDetail['agentOperation'] : null;
         const cancellation = db().prepare(`SELECT c.cleanup_id AS cleanupId,d.state,d.stage,d.error FROM native_creation_cancellations c
             JOIN server_deployments d ON d.id=c.cleanup_id WHERE c.creation_id=?`).get(id) as ProvisioningDetail['cancellation'];
         const { observation_json: _snapshot, phase_key: _phase, phase_started_at: _phaseStarted, observed_at: _observed, ...deployment } = row;
@@ -182,8 +183,10 @@ export function canResumeVerification(id: string): boolean {
     return !!db().prepare(`SELECT d.id FROM server_deployments d
         JOIN server_creation_jobs j ON j.deployment_id = d.id
         JOIN servers s ON s.name = j.server AND s.game_port = j.port
-        WHERE d.id = ? AND d.server_deleted_at IS NULL AND (d.action = 'create' OR (d.action = 'rebuild' AND EXISTS (
-            SELECT 1 FROM server_rebuild_jobs r WHERE r.deployment_id = d.id AND r.removed_at IS NOT NULL))) AND d.state IN ('failed', 'interrupted')
+        WHERE d.id = ? AND d.server_deleted_at IS NULL AND (d.action = 'create' OR (d.action = 'rebuild' AND (EXISTS (
+            SELECT 1 FROM server_rebuild_jobs r WHERE r.deployment_id = d.id AND r.removed_at IS NOT NULL) OR EXISTS (
+            SELECT 1 FROM ec_host_migrations m JOIN native_provisioning n ON n.deployment_id=m.deployment_id
+                WHERE m.deployment_id=d.id AND m.removed_at IS NOT NULL AND n.operation_id IS NOT NULL)))) AND d.state IN ('failed', 'interrupted')
         AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = d.server AND newer.rowid > d.rowid)`)
         .get(id);
 }

src/lib/server/provisioning/host-drain.ts

@@ -0,0 +1,151 @@
+import { db } from '../db/index.js';
+import { nativeInstance, nativeRequest, type NativeInstance } from '../backends/native.js';
+import { getHost, getHostOperationStatus, listHosts, queueHostOperation, reserveHostPort, updateHost } from '../hosts/index.js';
+import { choosePlacement } from '../hosts/capacity.js';
+import { scheduleAvailabilityRefresh } from '../hosts/availability.js';
+import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
+import { getServerKey } from '../fleet/server-keys.js';
+import { captureRecipePlan, recipeForDeployment, recipeStorage, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
+import { probePlayerEvidence } from '../fleet/status-evidence.js';
+import { beginDeployment, deploymentStage } from './history.js';
+import { fleetSettings } from '../settings.js';
+import type { NativePlacement } from '../../hosts.js';
+
+interface Migration {
+    deployment_id: string; source_host_id: string; instance_id: string;
+    target_host_id: string | null; target_port: number | null; target_bind: string | null;
+    remove_operation_id: string | null; removed_at: string | null;
+}
+interface Job { id: string; server: string; action: string; state: string; stage: string }
+type Recipe = LaunchRecipePlan & { native: NativePlacement };
+
+export function hostMigration(id: string): Migration | undefined {
+    return db().prepare('SELECT * FROM ec_host_migrations WHERE deployment_id=?').get(id) as Migration | undefined;
+}
+
+/** Drain requests survive controller restarts and never make a source slot available early. */
+export function reconcileHostDrains(): void {
+    for (const host of listHosts().filter(host => host.mode === 'draining' && !host.decommission)) {
+        db().transaction(() => {
+            // Accepted launches that have no instance yet can be placed elsewhere.
+            const queued = db().prepare(`SELECT q.deployment_id,q.selection FROM native_creation_intents q
+                JOIN server_deployments d ON d.id=q.deployment_id
+                LEFT JOIN native_provisioning n ON n.deployment_id=d.id
+                WHERE d.state='queued' AND d.server_deleted_at IS NULL AND d.submitted_at IS NULL
+                AND n.operation_id IS NULL AND n.stage_operation_id IS NULL
+                AND json_extract(q.selection,'$.hostId')=?`).all(host.id) as { deployment_id: string; selection: string }[];
+            for (const row of queued) {
+                const selection: NativePlacement = JSON.parse(row.selection);
+                db().prepare('UPDATE native_creation_intents SET selection=? WHERE deployment_id=?')
+                    .run(JSON.stringify({ libraryId: selection.libraryId, image: selection.image, automatic: true, network: {} }), row.deployment_id);
+            }
+            const instances = db().prepare('SELECT * FROM native_instances WHERE host_id=? AND retired_at IS NULL').all(host.id) as NativeInstance[];
+            for (const instance of instances) {
+                const server = getServer(instance.server);
+                if (!server?.map_slug || activeBetaRemovalBlocker(instance.server)) continue;
+                if (db().prepare(`SELECT 1 FROM server_deployments WHERE server=? AND server_deleted_at IS NULL
+                    AND state IN ('queued','waiting','running','awaiting-health','interrupted')`).get(instance.server)) continue;
+                const recipe = { ...captureRecipePlan(server.game as 't5' | 't6', server.map_slug), image: instance.image,
+                    native: { libraryId: instance.library_id, image: instance.image } };
+                const id = beginDeployment(instance.server, 'rebuild', 'host-drain');
+                db().prepare("UPDATE server_deployments SET state='waiting' WHERE id=?").run(id);
+                db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,NULL,?,?,?)')
+                    .run(id, instance.server, 'zombie', recipe.mapSlug, recipe.expectedMap);
+                const stored = recipeStorage(recipe);
+                db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary) VALUES(?,?,?)').run(id, stored.recipe, stored.recipeSummary);
+                db().prepare('INSERT INTO ec_host_migrations(deployment_id,source_host_id,instance_id) VALUES(?,?,?)').run(id, host.id, instance.id);
+                deploymentStage(id, 'Host draining; waiting for an empty lobby and destination capacity');
+            }
+            const unfinished = db().prepare(`SELECT 1 FROM ec_host_migrations m JOIN server_deployments d ON d.id=m.deployment_id
+                WHERE m.source_host_id=? AND d.state NOT IN ('done','warning','cancelled') AND d.server_deleted_at IS NULL LIMIT 1`).get(host.id);
+            const ports = db().prepare('SELECT 1 FROM ec_host_ports WHERE host_id=? LIMIT 1').get(host.id);
+            if (!instances.length && !unfinished && !ports) updateHost(host.id, { mode: 'disabled' });
+        })();
+    }
+}
+
+/** Reserve a destination, remove the source once, then hand creation to the normal health-verified rebuild queue. */
+export async function processMigrationRemoval(job: Job, instance: NativeInstance, migration: Migration,
+    eligibility: () => string | null): Promise<void> {
+    const hold = (message: string) => deploymentStage(job.id, message, { blocker: { code: 'native-host', message, automatic: true } });
+    if (migration.remove_operation_id) {
+        const operation = getHostOperationStatus(migration.remove_operation_id);
+        if (operation?.state === 'failed' && Date.now() - Date.parse(operation.updatedAt) >= 60_000) {
+            // A failed receipt certifies no mutation began. Unknown results keep
+            // their reservation and original operation ID until reconciled.
+            db().transaction(() => {
+                db().prepare('DELETE FROM ec_host_ports WHERE host_id=? AND instance_id=?').run(migration.target_host_id, instance.id);
+                db().prepare(`UPDATE ec_host_migrations SET target_host_id=NULL,target_port=NULL,target_bind=NULL,
+                    remove_operation_id=NULL WHERE deployment_id=?`).run(job.id);
+                db().prepare("UPDATE server_deployments SET state='waiting',submitted_at=NULL WHERE id=?").run(job.id);
+                scheduleAvailabilityRefresh();
+            })();
+            hold('Source removal failed before mutation; checking destination capacity again before retrying.');
+            return;
+        }
+        if (operation?.state !== 'completed') {
+            hold(operation?.error ?? 'Waiting for confirmed source removal. The original agent operation will reconcile.');
+            return;
+        }
+        db().transaction(() => {
+            if (hostMigration(job.id)?.removed_at) return;
+            const current = nativeInstance(job.server);
+            if (current?.id !== instance.id || current.generation !== instance.generation || current.host_id !== migration.source_host_id) throw new Error('Migration source changed.');
+            const target = getHost(migration.target_host_id!);
+            if (!target) throw new Error('Migration destination is unavailable.');
+            const recipe = recipeForDeployment(job.id)!.recipe as Recipe;
+            recipe.native = { libraryId: instance.library_id, image: instance.image, hostId: target.id,
+                bindAddress: migration.target_bind!, publicAddress: target.publicAddress };
+            const stored = recipeStorage(recipe);
+            db().prepare('UPDATE server_launch_recipes SET recipe=?,recipe_summary=? WHERE deployment_id=?').run(stored.recipe, stored.recipeSummary, job.id);
+            db().prepare('DELETE FROM ec_host_ports WHERE host_id=? AND instance_id=?').run(instance.host_id, instance.id);
+            db().prepare(`UPDATE native_instances SET host_id=?,generation=generation+1,bind_address=?,port=?,public_address=?,public_port=NULL WHERE id=?`)
+                .run(target.id, migration.target_bind, migration.target_port, target.publicAddress, instance.id);
+            db().prepare('UPDATE native_provisioning SET generation=generation+1 WHERE deployment_id=?').run(job.id);
+            db().prepare('UPDATE server_creation_jobs SET port=? WHERE deployment_id=?').run(migration.target_port, job.id);
+            db().prepare(`UPDATE servers SET game_port=?,container_state='starting',container_checked_at=NULL,
+                last_players=NULL,last_roster=NULL,last_seen_at=NULL,last_error=NULL,active_since=NULL,inactive_since=NULL,
+                game_started_at=NULL,game_ended_at=NULL WHERE name=?`).run(migration.target_port, job.server);
+            db().prepare('UPDATE ec_host_migrations SET removed_at=? WHERE deployment_id=?').run(operation.updatedAt, job.id);
+            db().prepare("UPDATE server_deployments SET state='waiting',submitted_at=NULL WHERE id=?").run(job.id);
+            deploymentStage(job.id, `Source removed; rebuilding on ${target.name}`, { phase: 'startup-queue', blocker: null });
+            scheduleAvailabilityRefresh();
+        })();
+        return;
+    }
+    if (!getHost(instance.host_id)?.connected) { hold('Waiting for the source host to reconnect.'); return; }
+    const reason = eligibility() ?? activeBetaRemovalBlocker(job.server);
+    if (reason) { hold(reason); return; }
+    if (!getServerKey(job.server)) { hold('Save a server key before migrating this instance.'); return; }
+    const options = { libraryId: instance.library_id, image: instance.image, thorough: true, dispatch: true };
+    // No source mutation is submitted unless another host can take this instance.
+    choosePlacement(options);
+    const health = await nativeRequest<{ state: string }>(instance, 'instance.inspect');
+    if (!['stopped','exited','created','dead','missing'].includes(health.state)) {
+        const occupancy = await probePlayerEvidence(job.server);
+        if (occupancy.players !== 0) { hold(occupancy.players === null ? 'Waiting for a fresh local player count.' : `Waiting for ${occupancy.players} player(s) to leave.`); return; }
+    }
+    db().transaction(() => {
+        const state = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined;
+        if (!state || !['waiting','queued','interrupted'].includes(state.state) || hostMigration(job.id)?.remove_operation_id) return;
+        const current = nativeInstance(job.server);
+        if (current?.id !== instance.id || current.generation !== instance.generation || getHost(instance.host_id)?.mode !== 'draining') return;
+        const blocker = eligibility() ?? activeBetaRemovalBlocker(job.server);
+        if (blocker) { hold(blocker); return; }
+        const removing = db().prepare("SELECT COUNT(*) AS n FROM ec_host_operations WHERE kind='instance.remove' AND state IN ('pending','accepted','running','unknown')").get() as { n: number };
+        if (removing.n >= fleetSettings().destroyConcurrency) { hold('Waiting for destruction capacity.'); return; }
+        const { host, port } = choosePlacement(options);
+        reserveHostPort(host.id, instance.id, { port, bindAddress: host.bindAddress, protocol: 'udp' });
+        reserveHostPort(host.id, instance.id, { port, bindAddress: host.bindAddress, protocol: 'tcp' });
+        const attempt = (db().prepare(`SELECT COUNT(*) AS n FROM ec_host_operations
+            WHERE host_id=? AND instance_id=? AND kind='instance.remove' AND idempotency_key LIKE ?`)
+            .get(instance.host_id, instance.id, `${job.id}:migrate-remove:%`) as { n: number }).n;
+        const operation = queueHostOperation(instance.host_id, 'instance.remove', {}, {
+            instanceId: instance.id, generation: instance.generation, idempotencyKey: `${job.id}:migrate-remove:${attempt}`
+        });
+        db().prepare('UPDATE ec_host_migrations SET target_host_id=?,target_port=?,target_bind=?,remove_operation_id=? WHERE deployment_id=?')
+            .run(host.id, port, host.bindAddress, operation.id, job.id);
+        db().prepare("UPDATE server_deployments SET state='running',submitted_at=? WHERE id=?").run(new Date().toISOString(), job.id);
+        deploymentStage(job.id, `Capacity reserved on ${host.name}; waiting for source removal`, { phase: 'container-action', blocker: null });
+    })();
+}

src/lib/server/provisioning/native.ts

@@ -6,7 +6,7 @@ import { posix } from 'node:path';
 import { db } from '../db/index.js';
 import { config, relDestForKind, serverDir } from '../config.js';
 import { nativeInstance, nativeRequest, unplacedNativeCreation, type NativeInstance } from '../backends/native.js';
-import { getHost, getHostOperationStatus, listHosts, queueHostOperation, queueHostOperationSerialized, releaseHostPorts, reserveHostPort } from '../hosts/index.js';
+import { getHost, getHostOperationStatus, queueHostOperation, queueHostOperationSerialized, releaseHostPorts, reserveHostPort } from '../hosts/index.js';
 import { newestReadyLibraryId, assertReadyLibrary } from '../game-files/index.js';
 import { blueprintRecipe, captureRecipePlan, recipeForDeployment, recipeStorage, summarizeRecipe, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
 import { prepareNativeLaunch, evaluateNativePlacement, type PreparedNativeLaunch } from './launch-preparation.js';
@@ -27,6 +27,7 @@ import type { LaunchPreview } from '../../deployment-plan.js';
 import type { ProvisioningAction } from '../../provisioning.js';
 import { hostAgentUpdatesEnabled } from '../../hosts.js';
 import { cancelCreation, finishCreationCancellation } from './cancel-creation.js';
+import { hostMigration, processMigrationRemoval, reconcileHostDrains } from './host-drain.js';
 
 export type NativeSelection = import('../../hosts.js').NativePlacement;
 type NativeRecipe = LaunchRecipePlan & { native?: NativeSelection };
@@ -258,17 +259,7 @@ function complete(job: Job) {
 
 export async function processNativeProvisioning(eligibility: (operation: { id: string; server: string; action: string; state: string; stage: string }) => string | null,
     options: { wait?: boolean } = {}): Promise<void> {
-    // Draining prevents admission immediately and retires existing lobbies only through the normal safety queue.
-    for (const host of listHosts().filter(host => host.mode === 'draining' && host.connected)) {
-        const instances = db().prepare('SELECT * FROM native_instances WHERE host_id=? AND retired_at IS NULL').all(host.id) as NativeInstance[];
-        for (const instance of instances) {
-            if (!getServer(instance.server) || activeBetaRemovalBlocker(instance.server)) continue;
-            if (db().prepare("SELECT 1 FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND state IN ('queued','waiting','running','awaiting-health')").get(instance.server)) continue;
-            const id = beginDeployment(instance.server, 'destroy', 'host-drain');
-            db().prepare("UPDATE server_deployments SET state='waiting' WHERE id=?").run(id);
-            deploymentStage(id, 'Host draining; waiting for a fresh empty lobby');
-        }
-    }
+    reconcileHostDrains();
     const jobs = db().prepare(`SELECT d.*,n.instance_id,n.generation,n.stage_operation_id,n.operation_id,n.files_json,n.applied_at,q.selection
         FROM server_deployments d LEFT JOIN native_provisioning n ON n.deployment_id=d.id
         LEFT JOIN native_creation_intents q ON q.deployment_id=d.id
@@ -285,7 +276,7 @@ export async function processNativeProvisioning(eligibility: (operation: { id: s
         activeJobs.set(job.id, work);
         return work;
     });
-    if (options.wait !== false) await Promise.all(work);
+    if (options.wait !== false) { await Promise.all(work); reconcileHostDrains(); }
 }
 
 const activeJobs = new Map<string, Promise<void>>();
@@ -372,9 +363,14 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
         if (!instance) return;
         job.instance_id = instance.id; job.generation = instance.generation;
     }
-    if (!instance || instance.id !== job.instance_id || (instance.generation !== job.generation && !(job.action === 'rebuild' && instance.generation === job.generation + 1))) {
+    const migration = hostMigration(job.id);
+    if (!instance || instance.id !== job.instance_id || (instance.generation !== job.generation && !(job.action === 'rebuild' && !migration && instance.generation === job.generation + 1))) {
         finishDeployment(job.id, { ok: false, message: 'Original instance was retired or replaced; no command was sent.' }); return;
     }
+    if (migration && !migration.removed_at) {
+        await processMigrationRemoval(job, instance, migration, () => eligibility(job));
+        return;
+    }
     const host = getHost(instance.host_id);
     if (!host?.connected || host.mode === 'disabled') {
         db().prepare("UPDATE servers SET container_state='unknown',last_players=NULL,last_roster=NULL WHERE name=?").run(job.server);
@@ -389,6 +385,17 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
         }
         if (!operation || operation.state === 'unknown') { hold(job, 'Agent result is unknown. Reconciling the original operation; no replacement or repeated command will be sent.'); return; }
         if (operation.state === 'failed') {
+            if (migration) {
+                if (Date.now() - Date.parse(operation.updatedAt) >= 60_000) {
+                    db().transaction(() => {
+                        db().prepare('DELETE FROM restart_rollout WHERE operation_id=?').run(job.id);
+                        db().prepare('UPDATE native_provisioning SET operation_id=NULL WHERE deployment_id=?').run(job.id);
+                        db().prepare("UPDATE server_deployments SET state='waiting',submitted_at=NULL WHERE id=?").run(job.id);
+                    })();
+                }
+                hold(job, `${operation.error ?? 'Destination creation failed before mutation.'} Retrying after fresh capacity checks.`);
+                return;
+            }
             // The agent reports failed only when no mutation began. Ambiguous
             // post-mutation failures remain unknown and retain their capacity.
             db().transaction(() => {
@@ -415,7 +422,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
             })();
             return;
         }
-        if (job.action === 'rebuild' && instance.generation === job.generation) {
+        if (job.action === 'rebuild' && !migration && instance.generation === job.generation) {
             db().prepare('UPDATE native_instances SET generation=? WHERE id=? AND generation=?').run(job.generation + 1, instance.id, job.generation);
             instance = nativeInstance(job.server)!;
         }
@@ -503,7 +510,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
     }
     if (job.action === 'destroy' && count(['destroy']) >= settings.destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; }
     deploymentStage(job.id, 'Checking host eligibility', { phase: 'eligibility', blocker: null });
-    if (job.action === 'create') choosePlacement({ hostId: instance.host_id, libraryId: instance.library_id, image: instance.image,
+    if (job.action === 'create' || migration) choosePlacement({ hostId: instance.host_id, libraryId: instance.library_id, image: instance.image,
         bindAddress: instance.bind_address, excludeInstanceId: instance.id, thorough: false, dispatch: true });
     if (job.action === 'create' || job.action === 'rebuild') assertReadyLibrary(instance.host_id, instance.library_id);
     const checkOccupancy = async (): Promise<boolean> => {
@@ -523,7 +530,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
         }
         return true;
     };
-    if (job.action !== 'create' && job.action !== 'start' && !await checkOccupancy()) return;
+    if (!migration && job.action !== 'create' && job.action !== 'start' && !await checkOccupancy()) return;
     const current = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined;
     if (!current || !['queued','waiting','interrupted'].includes(current.state) || eligibility(job)) return;
     let prepared: PreparedNativeLaunch | undefined;
@@ -541,8 +548,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
                 checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } });
             prepared = await prepareNativeLaunch({ game: prior.game, mapSlug: prior.mapSlug, native: prior.native,
                 instance, serverKey: preparedKey, rconPassword: config.rcon.password || randomBytes(32).toString('hex'),
-                rebuild: job.action === 'rebuild' });
-            if (job.action === 'rebuild' && !await checkOccupancy()) return;
+                rebuild: job.action === 'rebuild' && !migration });
+            if (job.action === 'rebuild' && !migration && !await checkOccupancy()) return;
         }
         const commit = db().transaction(() => {
             const state = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined;
@@ -569,7 +576,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
                     hold(job, 'Launch files or credentials changed during preparation; preparing the current selection next pass.'); return;
                 }
                 assertReadyLibrary(instance!.host_id, instance!.library_id);
-                if (job.action === 'create') choosePlacement({ hostId: instance!.host_id, libraryId: instance!.library_id, image: instance!.image,
+                if (job.action === 'create' || migration) choosePlacement({ hostId: instance!.host_id, libraryId: instance!.library_id, image: instance!.image,
                     bindAddress: instance!.bind_address, excludeInstanceId: instance!.id, thorough: false, dispatch: true });
             }
             if (job.action === 'start' || job.action === 'restart' || job.action === 'rebuild') {
@@ -590,8 +597,11 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
                 for (const file of recipe.files) recordScript(job.server, file.path, file.versionId, file.sha256, file.size, 'native-create');
                 recordProgress(job.id, { checkpoint: { key: 'prepare', label: 'Launch files selected and command payload prepared' } });
             }
-            const operationOptions = { instanceId: instance!.id, generation: instance!.generation, idempotencyKey: job.id };
-            const kind = `instance.${job.action === 'destroy' ? 'remove' : job.action}`;
+            const attempt = migration ? (db().prepare(`SELECT COUNT(*) AS n FROM ec_host_operations
+                WHERE host_id=? AND instance_id=? AND generation=? AND kind='instance.create'`).get(instance!.host_id, instance!.id, instance!.generation) as { n: number }).n : 0;
+            const operationOptions = { instanceId: instance!.id, generation: instance!.generation,
+                idempotencyKey: migration ? `${job.id}:migration-create:${attempt}` : job.id };
+            const kind = `instance.${migration ? 'create' : job.action === 'destroy' ? 'remove' : job.action}`;
             const operation = prepared ? queueHostOperationSerialized(instance!.host_id, kind,
                 Buffer.from(prepared.payloadJson.buffer, prepared.payloadJson.byteOffset, prepared.payloadJson.byteLength), operationOptions,
                 { name: instance!.server, game: prepared.recipe.game, libraryId: instance!.library_id })

src/lib/server/provisioning/requests.ts

@@ -120,7 +120,8 @@ export function cancelProvisioning(id: string, reason = 'Cancelled by requester'
     return db().transaction(() => {
         const finishedAt = new Date().toISOString();
         const result = db().prepare(`UPDATE server_deployments SET state = 'cancelled', finished_at = ?, stage = ?, error = NULL
-            WHERE id = ? AND action IN ('start','restart','stop','rebuild') AND state = 'waiting' AND submitted_at IS NULL`)
+            WHERE id = ? AND action IN ('start','restart','stop','rebuild') AND state = 'waiting' AND submitted_at IS NULL
+            AND NOT EXISTS (SELECT 1 FROM ec_host_migrations m WHERE m.deployment_id=server_deployments.id AND m.remove_operation_id IS NOT NULL)`)
             .run(finishedAt, reason, id);
         if (result.changes) recordProgress(id, { blocker: null, checks: [], event: { kind: 'outcome', message: reason, details: { state: 'cancelled', finishedAt, error: null, warnings: [] } } });
         return result.changes > 0;

src/lib/ui/components/HostPreparation.svelte

@@ -6,7 +6,7 @@
 </script>
 
 <div class="preparation" aria-live="polite">
-    <p>The default runtime and the latest available T5 and T6 game files are prepared automatically on active hosts. Running servers keep their current files.</p>
+    <p>The default runtime and the latest available T5 and T6 game files are prepared automatically. Re-enabling a host requires a fresh runtime pull and verification of both BO1 (T5) and BO2 (T6) libraries before it accepts deployments. Running servers keep their current files.</p>
     {#if host.preparation}
         <p class="status" class:blocked={host.preparation.state === 'blocked'}>Automatic setup: {host.preparation.state === 'ready' ? 'Ready' : host.preparation.state === 'preparing' ? 'In progress' : host.preparation.state === 'blocked' ? 'Needs attention' : 'Waiting'}</p>
         <ul>