120 lines
8.0 KiB
TypeScript
120 lines
8.0 KiB
TypeScript
import { db } from '../db/index.js';
|
|
import { listHosts, requireReadyHost } from './index.js';
|
|
import { assertReadyLibrary } from '../game-files/index.js';
|
|
import type { Host } from '../../hosts.js';
|
|
import { capacityCalibration } from './calibration.js';
|
|
|
|
// Storage and OS headroom are separate from learned instance CPU/memory usage.
|
|
const INSTANCE_STORAGE_BYTES = 1024 ** 3;
|
|
const MEMORY_HEADROOM = 500_000_000;
|
|
const CPU_USABLE_FRACTION = 0.95;
|
|
export class CapacityError extends Error {}
|
|
export interface PlacementOptions {
|
|
hostId?: string; libraryId: string; image: string; bindAddress?: string;
|
|
excludeHostIds?: string[];
|
|
thorough?: boolean; excludeInstanceId?: string;
|
|
// Execution only: not-yet-submitted launches wait their turn instead of
|
|
// requiring their entire backlog to fit before any one can start.
|
|
dispatch?: boolean;
|
|
planned?: { hostId: string; port: number }[];
|
|
}
|
|
|
|
/** One indexed port read per host, including both protocols and wildcard binds. */
|
|
export function availableHostPort(host: Host, options: Pick<PlacementOptions, 'bindAddress' | 'excludeInstanceId' | 'planned'> = {}): number | null {
|
|
const bind = options.bindAddress ?? host.bindAddress;
|
|
const rows = db().prepare('SELECT port,bind_address FROM ec_host_ports WHERE host_id=? AND instance_id!=?')
|
|
.all(host.id, options.excludeInstanceId ?? '') as { port: number; bind_address: string }[];
|
|
const used = new Set(rows.filter(row => row.bind_address === bind || ['0.0.0.0', '::'].includes(bind)
|
|
|| ['0.0.0.0', '::'].includes(row.bind_address)).map(row => row.port));
|
|
for (const item of options.planned ?? []) if (item.hostId === host.id) used.add(item.port);
|
|
for (let port = host.portMin; port <= host.portMax; port++) if (!used.has(port)) return port;
|
|
return null;
|
|
}
|
|
|
|
function projectedLoad(host: Host, options: PlacementOptions): number {
|
|
const { cpuCount, cpuUsagePercent, memoryTotal, memoryAvailable } = host.inventory;
|
|
if (!cpuCount || cpuUsagePercent === undefined || !memoryTotal || memoryAvailable === undefined || memoryAvailable > memoryTotal) {
|
|
throw new Error('Waiting for valid CPU and memory measurements.');
|
|
}
|
|
let historicalCpu = cpuUsagePercent;
|
|
let historicalAvailable = memoryAvailable;
|
|
if (options.thorough) {
|
|
const history = db().prepare(`SELECT AVG(cpu_usage_percent) AS cpu, AVG(memory_available) AS memory
|
|
FROM ec_host_resource_samples WHERE host_id=? AND sampled_at>=? AND cpu_count=? AND memory_total=?`)
|
|
.get(host.id, new Date(Date.now() - 5 * 60_000).toISOString(), cpuCount, memoryTotal) as { cpu: number | null; memory: number | null };
|
|
historicalCpu = Math.max(cpuUsagePercent, history.cpu ?? cpuUsagePercent);
|
|
historicalAvailable = Math.min(memoryAvailable, history.memory ?? memoryAvailable);
|
|
}
|
|
const instances = db().prepare(`SELECT i.id, i.generation,
|
|
NOT EXISTS (SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
|
|
WHERE n.instance_id=i.id AND n.generation=i.generation AND d.action IN ('create','rebuild') AND d.state IN ('done','warning') AND d.finished_at<?) AS pending,
|
|
EXISTS (SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
|
|
WHERE n.instance_id=i.id AND d.action='create' AND d.state='queued' AND d.submitted_at IS NULL
|
|
AND n.operation_id IS NULL AND n.stage_operation_id IS NULL) AS unsubmitted
|
|
FROM native_instances i WHERE i.host_id=? AND i.retired_at IS NULL AND i.id!=?`)
|
|
.all(host.lastHeartbeat, host.id, options.excludeInstanceId ?? '') as
|
|
{ id: string; generation: number; pending: number; unsubmitted: number }[];
|
|
const { running, startup } = capacityCalibration(host.id);
|
|
const incoming = db().prepare(`SELECT COUNT(*) AS n FROM ec_host_migrations
|
|
WHERE target_host_id=? AND removed_at IS NULL AND instance_id!=?`).get(host.id, options.excludeInstanceId ?? '') as { n: number };
|
|
const planned = options.planned?.filter(item => item.hostId === host.id).length ?? 0;
|
|
// Admission budgets the eventual running collection. Dispatch budgets only
|
|
// the next startup; the rest of the queue waits for live capacity to recover.
|
|
const launch = options.dispatch ? startup : running;
|
|
let extraCpu = planned * running.cpuCores + launch.cpuCores + incoming.n * startup.cpuCores;
|
|
let extraMemory = planned * running.memoryBytes + launch.memoryBytes + incoming.n * startup.memoryBytes;
|
|
let reservations = planned + 1 + incoming.n;
|
|
const measurements = new Map(host.inventory.instanceResources?.map(sample => [sample.instanceId, sample]));
|
|
for (const instance of instances) {
|
|
if (options.dispatch && options.excludeInstanceId && instance.unsubmitted) continue;
|
|
if (instance.unsubmitted) {
|
|
extraCpu += running.cpuCores;
|
|
extraMemory += running.memoryBytes;
|
|
reservations++;
|
|
continue;
|
|
}
|
|
const sample = measurements.get(instance.id);
|
|
// Current host usage already includes observed containers, including
|
|
// those still awaiting health verification. Do not top up to old peaks.
|
|
if (sample?.generation === instance.generation || !instance.pending) continue;
|
|
// A submitted launch absent from current telemetry may still start.
|
|
extraCpu += startup.cpuCores;
|
|
extraMemory += startup.memoryBytes;
|
|
reservations++;
|
|
}
|
|
const cores = cpuCount * cpuUsagePercent / 100 + extraCpu;
|
|
const usedMemory = memoryTotal - memoryAvailable;
|
|
const memory = usedMemory + extraMemory;
|
|
const usableMemory = memoryTotal - MEMORY_HEADROOM;
|
|
const gib = (bytes: number) => `${(bytes / 1024 ** 3).toFixed(2)} GiB`;
|
|
if (cores > cpuCount * CPU_USABLE_FRACTION) throw new Error(`Insufficient CPU capacity: ${cores.toFixed(2)} cores projected, ${(cpuCount * CPU_USABLE_FRACTION).toFixed(2)} usable; ${reservations} reservations including this launch.`);
|
|
if (memory > usableMemory) throw new Error(`Insufficient memory capacity: ${gib(memory)} projected, ${gib(usableMemory)} usable after host headroom. `
|
|
+ `${gib(usedMemory)} currently used + ${gib(extraMemory)} additional reservations; ${reservations} reservations including this launch (${gib(launch.memoryBytes)} ${options.dispatch ? 'startup' : 'running'} allowance).`);
|
|
// Recovered resources must not remain unavailable until the historical
|
|
// average catches up. History ranks hosts after current capacity passes.
|
|
const historicalMemory = memoryTotal - historicalAvailable + extraMemory;
|
|
const historicalCores = cpuCount * historicalCpu / 100 + extraCpu;
|
|
return Math.max(historicalCores / cpuCount, historicalMemory / memoryTotal);
|
|
}
|
|
|
|
/** Admission stops at the first fit; placement evaluates and ranks every fit. No remote I/O. */
|
|
export function choosePlacement(options: PlacementOptions): { host: Host; port: number; load: number } {
|
|
const candidates: { host: Host; port: number; load: number }[] = [];
|
|
const reasons: string[] = [];
|
|
for (const host of listHosts().filter(host => (!options.hostId || host.id === options.hostId) && !options.excludeHostIds?.includes(host.id))) {
|
|
try {
|
|
requireReadyHost(host.id, { image: options.image, requiredBytes: INSTANCE_STORAGE_BYTES });
|
|
assertReadyLibrary(host.id, options.libraryId);
|
|
const port = availableHostPort(host, options);
|
|
if (port === null) throw new Error('No available UDP/TCP game port.');
|
|
const load = projectedLoad(host, options);
|
|
const candidate = { host, port, load };
|
|
if (!options.thorough) return candidate;
|
|
candidates.push(candidate);
|
|
} catch (error) { reasons.push(`${host.name}: ${(error as Error).message}`); }
|
|
}
|
|
candidates.sort((a, b) => a.load - b.load || a.host.id.localeCompare(b.host.id));
|
|
if (candidates.length) return candidates[0];
|
|
throw new CapacityError(`No host has capacity for this server. ${reasons.join(' ') || 'Add and prepare a host.'}`);
|
|
}
|