Add safe manual instance rebalancing
This commit is contained in:
@@ -0,0 +1,144 @@
|
||||
import { db } from '../db/index.js';
|
||||
import { listHosts, HEARTBEAT_MAX_AGE_MS, requireReadyHost } from '../hosts/index.js';
|
||||
import { capacityCalibration } from '../hosts/calibration.js';
|
||||
import { assertReadyLibrary } from '../game-files/index.js';
|
||||
import { listServers } from '../fleet/servers.js';
|
||||
import { getServerKey } from '../fleet/server-keys.js';
|
||||
import { observationReader, serverObservation } from '../fleet/status.js';
|
||||
import { latestGameSubmission, telemetryReport } from '../fleet/status-evidence.js';
|
||||
import { mapIdentity, protectedWork } from '../autoscaling/snapshot.js';
|
||||
import { captureMoveRecipe } from '../provisioning/move-recipes.js';
|
||||
import { freshStatusTimestamp } from '../../server-freshness.js';
|
||||
import type { Host } from '../../hosts.js';
|
||||
import type { NativeInstance } from '../backends/native.js';
|
||||
import type { RebalanceHostOption, RebalanceSettings } from '../../rebalancing.js';
|
||||
import { REBALANCE_COOLDOWN_MS, type RebalancingSnapshot, type Resources } from './planner.js';
|
||||
|
||||
const HISTORY_MS = 5 * 60_000;
|
||||
const nonnegative = (value: unknown): value is number => typeof value === 'number' && Number.isFinite(value) && value >= 0;
|
||||
|
||||
function hostBlocker(host: Host, now: number): string | null {
|
||||
if (host.mode !== 'active' || host.decommission) return 'An active host is required.';
|
||||
const stamp = Date.parse(host.lastHeartbeat ?? '');
|
||||
if (!host.connected || !Number.isFinite(stamp) || stamp > now || now - stamp > HEARTBEAT_MAX_AGE_MS) return 'Waiting for a fresh host report.';
|
||||
if (host.readiness !== 'ready') return host.readinessReasons.join(' ') || 'The host is not ready.';
|
||||
const { cpuCount, cpuUsagePercent, memoryTotal, memoryAvailable } = host.inventory;
|
||||
if (!nonnegative(cpuCount) || !cpuCount || !nonnegative(cpuUsagePercent) || cpuUsagePercent > 100
|
||||
|| !nonnegative(memoryTotal) || !memoryTotal || !nonnegative(memoryAvailable) || memoryAvailable > memoryTotal) return 'Waiting for valid CPU and memory measurements.';
|
||||
return null;
|
||||
}
|
||||
|
||||
export function rebalanceHosts(): RebalanceHostOption[] {
|
||||
const now = Date.now();
|
||||
return listHosts().map(host => { const reason = hostBlocker(host, now); return { id: host.id, name: host.name, eligible: !reason, reason }; });
|
||||
}
|
||||
|
||||
/** Database evidence only: no probes, writes, reservations, or source capacity credits. */
|
||||
export function rebalanceSnapshot(settings: RebalanceSettings, exceptOperationId = ''): RebalancingSnapshot {
|
||||
const handle = db(), now = Date.now(), at = new Date(now).toISOString(), since = new Date(now - HISTORY_MS).toISOString();
|
||||
const hosts = listHosts().filter(host => settings.hostIds.includes(host.id));
|
||||
const natives = handle.prepare('SELECT * FROM native_instances WHERE retired_at IS NULL').all() as NativeInstance[];
|
||||
const operations = handle.prepare(`SELECT id,server,stage FROM server_deployments WHERE server_deleted_at IS NULL
|
||||
AND state IN ('queued','waiting','running','awaiting-health','interrupted') AND id!=?`).all(exceptOperationId) as { id: string; server: string; stage: string }[];
|
||||
const ownServer = exceptOperationId ? (handle.prepare('SELECT server FROM server_deployments WHERE id=?').get(exceptOperationId) as { server: string } | undefined)?.server : undefined;
|
||||
const observe = observationReader();
|
||||
const snapshot: RebalancingSnapshot = { hosts: [], instances: [], exclusions: [], warnings: [] };
|
||||
for (const id of settings.hostIds) if (!hosts.some(host => host.id === id)) snapshot.warnings.push(`Participating host ${id} is unavailable.`);
|
||||
for (const host of hosts) {
|
||||
const reason = hostBlocker(host, now), { running, startup } = capacityCalibration(host.id);
|
||||
if (reason) snapshot.warnings.push(`${host.name}: ${reason}`);
|
||||
const inventory = host.inventory;
|
||||
const valid = !reason || nonnegative(inventory.cpuCount) && inventory.cpuCount > 0 && nonnegative(inventory.cpuUsagePercent)
|
||||
&& nonnegative(inventory.memoryTotal) && inventory.memoryTotal > 0 && nonnegative(inventory.memoryAvailable) && inventory.memoryAvailable <= inventory.memoryTotal;
|
||||
const history = handle.prepare(`SELECT AVG(cpu_usage_percent) AS cpu,AVG(memory_available) AS memory FROM ec_host_resource_samples
|
||||
WHERE host_id=? AND sampled_at>=? AND sampled_at<=? AND cpu_count=? AND memory_total=?`)
|
||||
.get(host.id, since, at, inventory.cpuCount ?? 0, inventory.memoryTotal ?? 0) as { cpu: number | null; memory: number | null };
|
||||
const used: Resources = { cpuCores: valid ? inventory.cpuCount! * Math.max(inventory.cpuUsagePercent!, history.cpu ?? 0) / 100 : 0,
|
||||
memoryBytes: valid ? inventory.memoryTotal! - Math.min(inventory.memoryAvailable!, history.memory ?? inventory.memoryAvailable!) : 0 };
|
||||
const hostInstances = natives.filter(instance => instance.host_id === host.id), measured = new Map(inventory.instanceResources?.map(sample => [sample.instanceId, sample]));
|
||||
const reserved = new Set<string>();
|
||||
const reserve = (id: string, allowance: Resources) => {
|
||||
if (reserved.has(id)) return;
|
||||
reserved.add(id); used.cpuCores += allowance.cpuCores; used.memoryBytes += allowance.memoryBytes;
|
||||
};
|
||||
for (const instance of hostInstances) {
|
||||
const sample = measured.get(instance.id);
|
||||
if (sample?.generation === instance.generation) continue; // Already in the host measurement.
|
||||
const latest = handle.prepare(`SELECT d.state,d.submitted_at,n.operation_id,n.stage_operation_id FROM native_provisioning n
|
||||
JOIN server_deployments d ON d.id=n.deployment_id WHERE n.instance_id=? AND n.generation=? AND d.id!=?
|
||||
AND d.action IN ('create','rebuild') AND d.server_deleted_at IS NULL ORDER BY d.rowid DESC LIMIT 1`)
|
||||
.get(instance.id, instance.generation, exceptOperationId) as { state: string; submitted_at: string | null; operation_id: string | null; stage_operation_id: string | null } | undefined;
|
||||
const accounted = handle.prepare(`SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
|
||||
WHERE n.instance_id=? AND n.generation=? AND d.action IN ('create','rebuild') AND d.state IN ('done','warning') AND d.finished_at<?`)
|
||||
.get(instance.id, instance.generation, host.lastHeartbeat ?? '');
|
||||
if (accounted) continue;
|
||||
reserve(instance.id, latest?.state === 'queued' && !latest.submitted_at && !latest.operation_id && !latest.stage_operation_id ? running : startup);
|
||||
}
|
||||
const incoming = handle.prepare(`SELECT m.instance_id,m.removed_at,i.generation FROM ec_host_migrations m
|
||||
JOIN server_deployments d ON d.id=m.deployment_id JOIN native_instances i ON i.id=m.instance_id
|
||||
WHERE m.target_host_id=? AND m.deployment_id!=? AND d.server_deleted_at IS NULL AND i.retired_at IS NULL
|
||||
AND d.state NOT IN ('done','warning','cancelled')`).all(host.id, exceptOperationId) as { instance_id: string; removed_at: string | null; generation: number }[];
|
||||
for (const migration of incoming) {
|
||||
const generation = migration.generation + (migration.removed_at ? 0 : 1);
|
||||
if (measured.get(migration.instance_id)?.generation !== generation) reserve(migration.instance_id, startup);
|
||||
}
|
||||
const ports = handle.prepare('SELECT port,bind_address FROM ec_host_ports WHERE host_id=?').all(host.id) as { port: number; bind_address: string }[];
|
||||
const usedPorts = new Set(ports.filter(port => port.port >= host.portMin && port.port <= host.portMax
|
||||
&& (port.bind_address === host.bindAddress || ['0.0.0.0', '::'].includes(port.bind_address) || ['0.0.0.0', '::'].includes(host.bindAddress))).map(port => port.port));
|
||||
snapshot.hosts.push({ id: host.id, name: host.name, cpuCount: valid ? inventory.cpuCount! : 0, memoryTotal: valid ? inventory.memoryTotal! : 0,
|
||||
used, running, startup, reason, instances: hostInstances.length + incoming.filter(row => !hostInstances.some(instance => instance.id === row.instance_id)).length,
|
||||
ports: Math.max(0, host.portMax - host.portMin + 1 - usedPorts.size) });
|
||||
}
|
||||
const destinations = new Map<string, Record<string, string | null>>();
|
||||
for (const row of listServers()) {
|
||||
const identity = mapIdentity(row);
|
||||
if (!row.host_id || !settings.hostIds.includes(row.host_id) || settings.game !== 'all' && identity?.game !== settings.game) continue;
|
||||
const source = hosts.find(host => host.id === row.host_id), native = natives.find(instance => instance.id === row.native_instance_id);
|
||||
// The admitted rebuild itself must not hide the pre-removal empty evidence.
|
||||
const observation = row.name === ownServer ? serverObservation(row, { provisioning: false, settling: false,
|
||||
telemetry: telemetryReport(row.name), submission: latestGameSubmission(row.name) }) : observe(row);
|
||||
let reason = protectedWork(row.name, exceptOperationId);
|
||||
if (!reason && (!identity || !native || !row.map_slug)) reason = 'A native instance with a known map is required.';
|
||||
if (!reason && (!source || hostBlocker(source, now))) reason = source ? hostBlocker(source, now) : 'Source host is unavailable.';
|
||||
const operation = operations.find(item => item.server === row.name);
|
||||
if (!reason && operation) reason = `Provisioning ${operation.id}: ${operation.stage}`;
|
||||
if (!reason && native && handle.prepare(`SELECT 1 FROM ec_host_operations WHERE instance_id=? AND state IN ('pending','accepted','running','unknown')
|
||||
AND kind IN ('instance.create','instance.rebuild','instance.start','instance.stop','instance.restart','instance.remove','instance.cancel-create')`).get(native.id)) reason = 'An instance lifecycle command is unsettled.';
|
||||
if (!reason && (row.desired_state === 'stopped' || ['stopped','exited','created','dead','missing'].includes(row.container_state ?? ''))) reason = 'Stopped instances are not eligible.';
|
||||
if (!reason && (row.last_players ?? 0) > 0 && freshStatusTimestamp(row.last_seen_at, now)
|
||||
&& (!observation.telemetry_after || Date.parse(row.last_seen_at!) > Date.parse(observation.telemetry_after))) reason = 'Players are present according to a current RCON report.';
|
||||
if (!reason && (observation.health !== 'online' || observation.last_players === null || !freshStatusTimestamp(observation.last_seen_at, now))) reason = 'Waiting for healthy, fresh occupancy evidence.';
|
||||
if (!reason && observation.last_players !== 0) reason = 'Players are present.';
|
||||
if (!reason && !getServerKey(row.name)) reason = 'Save a server key before moving this instance.';
|
||||
if (!reason && handle.prepare(`SELECT 1 FROM rebalancing_moves WHERE instance_id=? AND state='done' AND updated_at>=?`)
|
||||
.get(native!.id, new Date(now - REBALANCE_COOLDOWN_MS).toISOString())) reason = 'This instance was rebalanced less than 30 minutes ago.';
|
||||
const sample = source?.inventory.instanceResources?.find(item => item.instanceId === native?.id && item.generation === native.generation);
|
||||
if (!reason && (!sample || source?.inventory.instanceResourcesError || !nonnegative(sample.cpuCores) || !nonnegative(sample.memoryBytes) || !sample.memoryBytes)) reason = 'Waiting for resource measurements for the current instance generation.';
|
||||
if (!reason && sample!.uptimeSeconds < 300) reason = 'Waiting for this instance to finish its first five minutes of startup.';
|
||||
if (!reason) { try { captureMoveRecipe(row.name); } catch (error) { reason = (error as Error).message; } }
|
||||
if (reason) { snapshot.exclusions.push({ server: row.name, reason }); continue; }
|
||||
const key = JSON.stringify([native!.library_id, native!.image]);
|
||||
if (!destinations.has(key)) {
|
||||
const eligibility: Record<string, string | null> = {};
|
||||
for (const host of hosts) {
|
||||
try { requireReadyHost(host.id, { image: native!.image, requiredBytes: 1024 ** 3 }); assertReadyLibrary(host.id, native!.library_id); eligibility[host.id] = null; }
|
||||
catch (error) { eligibility[host.id] = (error as Error).message; }
|
||||
}
|
||||
destinations.set(key, eligibility);
|
||||
}
|
||||
const history = handle.prepare(`SELECT AVG(cpu_cores) AS cpu,AVG(memory_bytes) AS memory FROM ec_instance_resource_samples
|
||||
WHERE host_id=? AND instance_id=? AND generation=? AND phase='running' AND bucket>=? AND bucket<=?`)
|
||||
.get(source!.id, native!.id, native!.generation, since, at) as { cpu: number | null; memory: number | null };
|
||||
const bindConflicts = (bind: string) => bind === source!.bindAddress || ['0.0.0.0', '::'].includes(bind) || ['0.0.0.0', '::'].includes(source!.bindAddress);
|
||||
const portOwners = handle.prepare('SELECT instance_id,bind_address FROM ec_host_ports WHERE host_id=? AND port=?')
|
||||
.all(source!.id, native!.port) as { instance_id: string; bind_address: string }[];
|
||||
snapshot.instances.push({ server: row.name, instanceId: native!.id, generation: native!.generation, sourceHostId: source!.id, game: identity!.game,
|
||||
releasesPort: native!.port >= source!.portMin && native!.port <= source!.portMax && bindConflicts(native!.bind_address)
|
||||
&& portOwners.some(owner => owner.instance_id === native!.id && bindConflicts(owner.bind_address))
|
||||
&& !portOwners.some(owner => owner.instance_id !== native!.id && bindConflicts(owner.bind_address)),
|
||||
relief: { cpuCores: Math.min(sample!.cpuCores, history.cpu ?? sample!.cpuCores), memoryBytes: Math.min(sample!.memoryBytes, history.memory ?? sample!.memoryBytes) },
|
||||
demand: { cpuCores: Math.max(sample!.cpuCores, history.cpu ?? sample!.cpuCores), memoryBytes: Math.max(sample!.memoryBytes, history.memory ?? sample!.memoryBytes) },
|
||||
destinations: destinations.get(key)! });
|
||||
}
|
||||
return snapshot;
|
||||
}
|
||||
Reference in New Issue
Block a user