import { db } from '../db/index.js'; import { nativeInstance, nativeRequest, type NativeInstance } from '../backends/native.js'; import { getHost, getHostOperationStatus, listHosts, queueHostOperation, reserveHostPort, updateHost } from '../hosts/index.js'; import { choosePlacement } from '../hosts/capacity.js'; import { scheduleAvailabilityRefresh } from '../hosts/availability.js'; import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js'; import { getServerKey } from '../fleet/server-keys.js'; import { captureRecipePlan, recipeForDeployment, recipeStorage, type LaunchRecipePlan } from '../fleet/launch-recipes.js'; import { probePlayerEvidence } from '../fleet/status-evidence.js'; import { beginDeployment, deploymentStage } from './history.js'; import { fleetSettings } from '../settings.js'; import type { NativePlacement } from '../../hosts.js'; interface Migration { deployment_id: string; source_host_id: string; instance_id: string; target_host_id: string | null; target_port: number | null; target_bind: string | null; remove_operation_id: string | null; removed_at: string | null; } interface Job { id: string; server: string; action: string; state: string; stage: string } type Recipe = LaunchRecipePlan & { native: NativePlacement }; export function hostMigration(id: string): Migration | undefined { return db().prepare('SELECT * FROM ec_host_migrations WHERE deployment_id=?').get(id) as Migration | undefined; } /** Drain requests survive controller restarts and never make a source slot available early. */ export function reconcileHostDrains(): void { for (const host of listHosts().filter(host => host.mode === 'draining' && !host.decommission)) { db().transaction(() => { // Accepted launches that have no instance yet can be placed elsewhere. const queued = db().prepare(`SELECT q.deployment_id,q.selection FROM native_creation_intents q JOIN server_deployments d ON d.id=q.deployment_id LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE d.state='queued' AND d.server_deleted_at IS NULL AND d.submitted_at IS NULL AND n.operation_id IS NULL AND n.stage_operation_id IS NULL AND json_extract(q.selection,'$.hostId')=?`).all(host.id) as { deployment_id: string; selection: string }[]; for (const row of queued) { const selection: NativePlacement = JSON.parse(row.selection); db().prepare('UPDATE native_creation_intents SET selection=? WHERE deployment_id=?') .run(JSON.stringify({ libraryId: selection.libraryId, image: selection.image, automatic: true, network: {} }), row.deployment_id); } const instances = db().prepare('SELECT * FROM native_instances WHERE host_id=? AND retired_at IS NULL').all(host.id) as NativeInstance[]; for (const instance of instances) { const server = getServer(instance.server); if (!server?.map_slug || activeBetaRemovalBlocker(instance.server)) continue; if (db().prepare(`SELECT 1 FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND state IN ('queued','waiting','running','awaiting-health','interrupted')`).get(instance.server)) continue; const recipe = { ...captureRecipePlan(server.game as 't5' | 't6', server.map_slug), image: instance.image, native: { libraryId: instance.library_id, image: instance.image } }; const id = beginDeployment(instance.server, 'rebuild', 'host-drain'); db().prepare("UPDATE server_deployments SET state='waiting' WHERE id=?").run(id); db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,NULL,?,?,?)') .run(id, instance.server, 'zombie', recipe.mapSlug, recipe.expectedMap); const stored = recipeStorage(recipe); db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary) VALUES(?,?,?)').run(id, stored.recipe, stored.recipeSummary); db().prepare('INSERT INTO ec_host_migrations(deployment_id,source_host_id,instance_id) VALUES(?,?,?)').run(id, host.id, instance.id); deploymentStage(id, 'Host draining; waiting for an empty lobby and destination capacity'); } const unfinished = db().prepare(`SELECT 1 FROM ec_host_migrations m JOIN server_deployments d ON d.id=m.deployment_id WHERE m.source_host_id=? AND d.state NOT IN ('done','warning','cancelled') AND d.server_deleted_at IS NULL LIMIT 1`).get(host.id); const ports = db().prepare('SELECT 1 FROM ec_host_ports WHERE host_id=? LIMIT 1').get(host.id); if (!instances.length && !unfinished && !ports) updateHost(host.id, { mode: 'disabled' }); })(); } } /** Reserve a destination, remove the source once, then hand creation to the normal health-verified rebuild queue. */ export async function processMigrationRemoval(job: Job, instance: NativeInstance, migration: Migration, eligibility: () => string | null): Promise { const hold = (message: string) => deploymentStage(job.id, message, { blocker: { code: 'native-host', message, automatic: true } }); if (migration.remove_operation_id) { const operation = getHostOperationStatus(migration.remove_operation_id); if (operation?.state === 'failed' && Date.now() - Date.parse(operation.updatedAt) >= 60_000) { // A failed receipt certifies no mutation began. Unknown results keep // their reservation and original operation ID until reconciled. db().transaction(() => { db().prepare('DELETE FROM ec_host_ports WHERE host_id=? AND instance_id=?').run(migration.target_host_id, instance.id); db().prepare(`UPDATE ec_host_migrations SET target_host_id=NULL,target_port=NULL,target_bind=NULL, remove_operation_id=NULL WHERE deployment_id=?`).run(job.id); db().prepare("UPDATE server_deployments SET state='waiting',submitted_at=NULL WHERE id=?").run(job.id); scheduleAvailabilityRefresh(); })(); hold('Source removal failed before mutation; checking destination capacity again before retrying.'); return; } if (operation?.state !== 'completed') { hold(operation?.error ?? 'Waiting for confirmed source removal. The original agent operation will reconcile.'); return; } db().transaction(() => { if (hostMigration(job.id)?.removed_at) return; const current = nativeInstance(job.server); if (current?.id !== instance.id || current.generation !== instance.generation || current.host_id !== migration.source_host_id) throw new Error('Migration source changed.'); const target = getHost(migration.target_host_id!); if (!target) throw new Error('Migration destination is unavailable.'); const recipe = recipeForDeployment(job.id)!.recipe as Recipe; recipe.native = { libraryId: instance.library_id, image: instance.image, hostId: target.id, bindAddress: migration.target_bind!, publicAddress: target.publicAddress }; const stored = recipeStorage(recipe); db().prepare('UPDATE server_launch_recipes SET recipe=?,recipe_summary=? WHERE deployment_id=?').run(stored.recipe, stored.recipeSummary, job.id); db().prepare('DELETE FROM ec_host_ports WHERE host_id=? AND instance_id=?').run(instance.host_id, instance.id); db().prepare(`UPDATE native_instances SET host_id=?,generation=generation+1,bind_address=?,port=?,public_address=?,public_port=NULL WHERE id=?`) .run(target.id, migration.target_bind, migration.target_port, target.publicAddress, instance.id); db().prepare('UPDATE native_provisioning SET generation=generation+1 WHERE deployment_id=?').run(job.id); db().prepare('UPDATE server_creation_jobs SET port=? WHERE deployment_id=?').run(migration.target_port, job.id); db().prepare(`UPDATE servers SET game_port=?,container_state='starting',container_checked_at=NULL, last_players=NULL,last_roster=NULL,last_seen_at=NULL,last_error=NULL,rcon_failures=0,active_since=NULL,inactive_since=NULL, game_started_at=NULL,game_ended_at=NULL WHERE name=?`).run(migration.target_port, job.server); db().prepare('UPDATE ec_host_migrations SET removed_at=? WHERE deployment_id=?').run(operation.updatedAt, job.id); db().prepare("UPDATE server_deployments SET state='waiting',submitted_at=NULL WHERE id=?").run(job.id); deploymentStage(job.id, `Source removed; rebuilding on ${target.name}`, { phase: 'startup-queue', blocker: null }); scheduleAvailabilityRefresh(); })(); return; } if (!getHost(instance.host_id)?.connected) { hold('Waiting for the source host to reconnect.'); return; } const reason = eligibility() ?? activeBetaRemovalBlocker(job.server); if (reason) { hold(reason); return; } if (!getServerKey(job.server)) { hold('Save a server key before migrating this instance.'); return; } const options = { libraryId: instance.library_id, image: instance.image, thorough: true, dispatch: true }; // No source mutation is submitted unless another host can take this instance. choosePlacement(options); const health = await nativeRequest<{ state: string }>(instance, 'instance.inspect'); if (!['stopped','exited','created','dead','missing'].includes(health.state)) { const occupancy = await probePlayerEvidence(job.server); if (occupancy.players !== 0) { hold(occupancy.players === null ? 'Waiting for a fresh local player count.' : `Waiting for ${occupancy.players} player(s) to leave.`); return; } } db().transaction(() => { const state = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined; if (!state || !['waiting','queued','interrupted'].includes(state.state) || hostMigration(job.id)?.remove_operation_id) return; const current = nativeInstance(job.server); if (current?.id !== instance.id || current.generation !== instance.generation || getHost(instance.host_id)?.mode !== 'draining') return; const blocker = eligibility() ?? activeBetaRemovalBlocker(job.server); if (blocker) { hold(blocker); return; } const removing = db().prepare("SELECT COUNT(*) AS n FROM ec_host_operations WHERE kind='instance.remove' AND state IN ('pending','accepted','running','unknown')").get() as { n: number }; if (removing.n >= fleetSettings().destroyConcurrency) { hold('Waiting for destruction capacity.'); return; } const { host, port } = choosePlacement(options); reserveHostPort(host.id, instance.id, { port, bindAddress: host.bindAddress, protocol: 'udp' }); reserveHostPort(host.id, instance.id, { port, bindAddress: host.bindAddress, protocol: 'tcp' }); const attempt = (db().prepare(`SELECT COUNT(*) AS n FROM ec_host_operations WHERE host_id=? AND instance_id=? AND kind='instance.remove' AND idempotency_key LIKE ?`) .get(instance.host_id, instance.id, `${job.id}:migrate-remove:%`) as { n: number }).n; const operation = queueHostOperation(instance.host_id, 'instance.remove', {}, { instanceId: instance.id, generation: instance.generation, idempotencyKey: `${job.id}:migrate-remove:${attempt}` }); db().prepare('UPDATE ec_host_migrations SET target_host_id=?,target_port=?,target_bind=?,remove_operation_id=? WHERE deployment_id=?') .run(host.id, port, host.bindAddress, operation.id, job.id); db().prepare("UPDATE server_deployments SET state='running',submitted_at=? WHERE id=?").run(new Date().toISOString(), job.id); deploymentStage(job.id, `Capacity reserved on ${host.name}; waiting for source removal`, { phase: 'container-action', blocker: null }); })(); }