import { db } from '../db/index.js'; import { listHosts, HEARTBEAT_MAX_AGE_MS, requireReadyHost } from '../hosts/index.js'; import { capacityCalibration } from '../hosts/calibration.js'; import { assertReadyLibrary } from '../game-files/index.js'; import { listServers } from '../fleet/servers.js'; import { getServerKey } from '../fleet/server-keys.js'; import { observationReader, serverObservation } from '../fleet/status.js'; import { latestGameSubmission, telemetryReport } from '../fleet/status-evidence.js'; import { mapIdentity, protectedWork } from '../autoscaling/snapshot.js'; import { captureMoveRecipe } from '../provisioning/move-recipes.js'; import { freshStatusTimestamp } from '../../server-freshness.js'; import type { Host } from '../../hosts.js'; import type { NativeInstance } from '../backends/native.js'; import type { RebalanceHostOption, RebalanceSettings } from '../../rebalancing.js'; import { REBALANCE_COOLDOWN_MS, type RebalancingSnapshot, type Resources } from './planner.js'; const HISTORY_MS = 5 * 60_000; const nonnegative = (value: unknown): value is number => typeof value === 'number' && Number.isFinite(value) && value >= 0; function hostBlocker(host: Host, now: number): string | null { if (host.mode !== 'active' || host.decommission) return 'An active host is required.'; const stamp = Date.parse(host.lastHeartbeat ?? ''); if ( !host.connected || !Number.isFinite(stamp) || stamp > now || now - stamp > HEARTBEAT_MAX_AGE_MS ) return 'Waiting for a fresh host report.'; if (host.readiness !== 'ready') return host.readinessReasons.join(' ') || 'The host is not ready.'; const { cpuCount, cpuUsagePercent, memoryTotal, memoryAvailable } = host.inventory; if ( !nonnegative(cpuCount) || !cpuCount || !nonnegative(cpuUsagePercent) || cpuUsagePercent > 100 || !nonnegative(memoryTotal) || !memoryTotal || !nonnegative(memoryAvailable) || memoryAvailable > memoryTotal ) return 'Waiting for valid CPU and memory measurements.'; return null; } export function rebalanceHosts(): RebalanceHostOption[] { const now = Date.now(); return listHosts().map((host) => { const reason = hostBlocker(host, now); return { id: host.id, name: host.name, eligible: !reason, reason }; }); } /** Database evidence only: no probes, writes, reservations, or source capacity credits. */ export function rebalanceSnapshot( settings: RebalanceSettings, exceptOperationId = '' ): RebalancingSnapshot { const handle = db(), now = Date.now(), at = new Date(now).toISOString(), since = new Date(now - HISTORY_MS).toISOString(); const hosts = listHosts().filter((host) => settings.hostIds.includes(host.id)); const natives = handle .prepare('SELECT * FROM native_instances WHERE retired_at IS NULL') .all() as NativeInstance[]; const operations = handle .prepare( `SELECT id,server,stage FROM server_deployments WHERE server_deleted_at IS NULL AND state IN ('queued','waiting','running','awaiting-health','interrupted') AND id!=?` ) .all(exceptOperationId) as { id: string; server: string; stage: string }[]; const ownServer = exceptOperationId ? ( handle .prepare('SELECT server FROM server_deployments WHERE id=?') .get(exceptOperationId) as { server: string } | undefined )?.server : undefined; const observe = observationReader(); const snapshot: RebalancingSnapshot = { hosts: [], instances: [], exclusions: [], warnings: [] }; for (const id of settings.hostIds) if (!hosts.some((host) => host.id === id)) snapshot.warnings.push(`Participating host ${id} is unavailable.`); for (const host of hosts) { const reason = hostBlocker(host, now), { running, startup } = capacityCalibration(host.id); if (reason) snapshot.warnings.push(`${host.name}: ${reason}`); const inventory = host.inventory; const valid = !reason || (nonnegative(inventory.cpuCount) && inventory.cpuCount > 0 && nonnegative(inventory.cpuUsagePercent) && nonnegative(inventory.memoryTotal) && inventory.memoryTotal > 0 && nonnegative(inventory.memoryAvailable) && inventory.memoryAvailable <= inventory.memoryTotal); const history = handle .prepare( `SELECT AVG(cpu_usage_percent) AS cpu,AVG(memory_available) AS memory FROM ec_host_resource_samples WHERE host_id=? AND sampled_at>=? AND sampled_at<=? AND cpu_count=? AND memory_total=?` ) .get(host.id, since, at, inventory.cpuCount ?? 0, inventory.memoryTotal ?? 0) as { cpu: number | null; memory: number | null; }; const used: Resources = { cpuCores: valid ? (inventory.cpuCount! * Math.max(inventory.cpuUsagePercent!, history.cpu ?? 0)) / 100 : 0, memoryBytes: valid ? inventory.memoryTotal! - Math.min(inventory.memoryAvailable!, history.memory ?? inventory.memoryAvailable!) : 0 }; const hostInstances = natives.filter((instance) => instance.host_id === host.id), measured = new Map( inventory.instanceResources?.map((sample) => [sample.instanceId, sample]) ); const reserved = new Set(); const reserve = (id: string, allowance: Resources) => { if (reserved.has(id)) return; reserved.add(id); used.cpuCores += allowance.cpuCores; used.memoryBytes += allowance.memoryBytes; }; for (const instance of hostInstances) { const sample = measured.get(instance.id); if (sample?.generation === instance.generation) continue; // Already in the host measurement. const latest = handle .prepare( `SELECT d.state,d.submitted_at,n.operation_id,n.stage_operation_id FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id WHERE n.instance_id=? AND n.generation=? AND d.id!=? AND d.action IN ('create','rebuild') AND d.server_deleted_at IS NULL ORDER BY d.rowid DESC LIMIT 1` ) .get(instance.id, instance.generation, exceptOperationId) as | { state: string; submitted_at: string | null; operation_id: string | null; stage_operation_id: string | null; } | undefined; const accounted = handle .prepare( `SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id WHERE n.instance_id=? AND n.generation=? AND d.action IN ('create','rebuild') AND d.state IN ('done','warning') AND d.finished_at port.port >= host.portMin && port.port <= host.portMax && (port.bind_address === host.bindAddress || ['0.0.0.0', '::'].includes(port.bind_address) || ['0.0.0.0', '::'].includes(host.bindAddress)) ) .map((port) => port.port) ); snapshot.hosts.push({ id: host.id, name: host.name, cpuCount: valid ? inventory.cpuCount! : 0, memoryTotal: valid ? inventory.memoryTotal! : 0, used, running, startup, reason, instances: hostInstances.length + incoming.filter( (row) => !hostInstances.some((instance) => instance.id === row.instance_id) ).length, ports: Math.max(0, host.portMax - host.portMin + 1 - usedPorts.size) }); } const destinations = new Map>(); for (const row of listServers()) { const identity = mapIdentity(row); if ( !row.host_id || !settings.hostIds.includes(row.host_id) || (settings.game !== 'all' && identity?.game !== settings.game) ) continue; const source = hosts.find((host) => host.id === row.host_id), native = natives.find((instance) => instance.id === row.native_instance_id); // The admitted rebuild itself must not hide the pre-removal empty evidence. const observation = row.name === ownServer ? serverObservation(row, { provisioning: false, settling: false, telemetry: telemetryReport(row.name), submission: latestGameSubmission(row.name) }) : observe(row); let reason = protectedWork(row.name, exceptOperationId); if (!reason && (!identity || !native || !row.map_slug)) reason = 'A native instance with a known map is required.'; if (!reason && (!source || hostBlocker(source, now))) reason = source ? hostBlocker(source, now) : 'Source host is unavailable.'; const operation = operations.find((item) => item.server === row.name); if (!reason && operation) reason = `Provisioning ${operation.id}: ${operation.stage}`; if ( !reason && native && handle .prepare( `SELECT 1 FROM ec_host_operations WHERE instance_id=? AND state IN ('pending','accepted','running','unknown') AND kind IN ('instance.create','instance.rebuild','instance.start','instance.stop','instance.restart','instance.remove','instance.cancel-create')` ) .get(native.id) ) reason = 'An instance lifecycle command is unsettled.'; if ( !reason && (row.desired_state === 'stopped' || ['stopped', 'exited', 'created', 'dead', 'missing'].includes( row.container_state ?? '' )) ) reason = 'Stopped instances are not eligible.'; if ( !reason && (row.last_players ?? 0) > 0 && freshStatusTimestamp(row.last_seen_at, now) && (!observation.telemetry_after || Date.parse(row.last_seen_at!) > Date.parse(observation.telemetry_after)) ) reason = 'Players are present according to a current RCON report.'; if ( !reason && (observation.health !== 'online' || observation.last_players === null || !freshStatusTimestamp(observation.last_seen_at, now)) ) reason = 'Waiting for healthy, fresh occupancy evidence.'; if (!reason && observation.last_players !== 0) reason = 'Players are present.'; if (!reason && !getServerKey(row.name)) reason = 'Save a server key before moving this instance.'; if ( !reason && handle .prepare( `SELECT 1 FROM rebalancing_moves WHERE instance_id=? AND state='done' AND updated_at>=?` ) .get(native!.id, new Date(now - REBALANCE_COOLDOWN_MS).toISOString()) ) reason = 'This instance was rebalanced less than 30 minutes ago.'; const sample = source?.inventory.instanceResources?.find( (item) => item.instanceId === native?.id && item.generation === native.generation ); if ( !reason && (!sample || source?.inventory.instanceResourcesError || !nonnegative(sample.cpuCores) || !nonnegative(sample.memoryBytes) || !sample.memoryBytes) ) reason = 'Waiting for resource measurements for the current instance generation.'; if (!reason && sample!.uptimeSeconds < 300) reason = 'Waiting for this instance to finish its first five minutes of startup.'; if (!reason) { try { captureMoveRecipe(row.name); } catch (error) { reason = (error as Error).message; } } if (reason) { snapshot.exclusions.push({ server: row.name, reason }); continue; } const key = JSON.stringify([native!.library_id, native!.image]); if (!destinations.has(key)) { const eligibility: Record = {}; for (const host of hosts) { try { requireReadyHost(host.id, { image: native!.image, requiredBytes: 1024 ** 3 }); assertReadyLibrary(host.id, native!.library_id); eligibility[host.id] = null; } catch (error) { eligibility[host.id] = (error as Error).message; } } destinations.set(key, eligibility); } const history = handle .prepare( `SELECT AVG(cpu_cores) AS cpu,AVG(memory_bytes) AS memory FROM ec_instance_resource_samples WHERE host_id=? AND instance_id=? AND generation=? AND phase='running' AND bucket>=? AND bucket<=?` ) .get(source!.id, native!.id, native!.generation, since, at) as { cpu: number | null; memory: number | null; }; const bindConflicts = (bind: string) => bind === source!.bindAddress || ['0.0.0.0', '::'].includes(bind) || ['0.0.0.0', '::'].includes(source!.bindAddress); const portOwners = handle .prepare( 'SELECT instance_id,bind_address FROM ec_host_ports WHERE host_id=? AND port=?' ) .all(source!.id, native!.port) as { instance_id: string; bind_address: string }[]; snapshot.instances.push({ server: row.name, instanceId: native!.id, generation: native!.generation, sourceHostId: source!.id, game: identity!.game, releasesPort: native!.port >= source!.portMin && native!.port <= source!.portMax && bindConflicts(native!.bind_address) && portOwners.some( (owner) => owner.instance_id === native!.id && bindConflicts(owner.bind_address) ) && !portOwners.some( (owner) => owner.instance_id !== native!.id && bindConflicts(owner.bind_address) ), relief: { cpuCores: Math.min(sample!.cpuCores, history.cpu ?? sample!.cpuCores), memoryBytes: Math.min(sample!.memoryBytes, history.memory ?? sample!.memoryBytes) }, demand: { cpuCores: Math.max(sample!.cpuCores, history.cpu ?? sample!.cpuCores), memoryBytes: Math.max(sample!.memoryBytes, history.memory ?? sample!.memoryBytes) }, destinations: destinations.get(key)! }); } return snapshot; }