379 lines
17 KiB
TypeScript
379 lines
17 KiB
TypeScript
import { db } from '../db/index.js';
|
|
import { listHosts, HEARTBEAT_MAX_AGE_MS, requireReadyHost } from '../hosts/index.js';
|
|
import { capacityCalibration } from '../hosts/calibration.js';
|
|
import { assertReadyLibrary } from '../game-files/index.js';
|
|
import { listServers } from '../fleet/servers.js';
|
|
import { getServerKey } from '../fleet/server-keys.js';
|
|
import { observationReader, serverObservation } from '../fleet/status.js';
|
|
import { latestGameSubmission, telemetryReport } from '../fleet/status-evidence.js';
|
|
import { mapIdentity, protectedWork } from '../autoscaling/snapshot.js';
|
|
import { captureMoveRecipe } from '../provisioning/move-recipes.js';
|
|
import { freshStatusTimestamp } from '../../server-freshness.js';
|
|
import type { Host } from '../../hosts.js';
|
|
import type { NativeInstance } from '../backends/native.js';
|
|
import type { RebalanceHostOption, RebalanceSettings } from '../../rebalancing.js';
|
|
import { REBALANCE_COOLDOWN_MS, type RebalancingSnapshot, type Resources } from './planner.js';
|
|
|
|
const HISTORY_MS = 5 * 60_000;
|
|
const nonnegative = (value: unknown): value is number =>
|
|
typeof value === 'number' && Number.isFinite(value) && value >= 0;
|
|
|
|
function hostBlocker(host: Host, now: number): string | null {
|
|
if (host.mode !== 'active' || host.decommission) return 'An active host is required.';
|
|
const stamp = Date.parse(host.lastHeartbeat ?? '');
|
|
if (
|
|
!host.connected ||
|
|
!Number.isFinite(stamp) ||
|
|
stamp > now ||
|
|
now - stamp > HEARTBEAT_MAX_AGE_MS
|
|
)
|
|
return 'Waiting for a fresh host report.';
|
|
if (host.readiness !== 'ready')
|
|
return host.readinessReasons.join(' ') || 'The host is not ready.';
|
|
const { cpuCount, cpuUsagePercent, memoryTotal, memoryAvailable } = host.inventory;
|
|
if (
|
|
!nonnegative(cpuCount) ||
|
|
!cpuCount ||
|
|
!nonnegative(cpuUsagePercent) ||
|
|
cpuUsagePercent > 100 ||
|
|
!nonnegative(memoryTotal) ||
|
|
!memoryTotal ||
|
|
!nonnegative(memoryAvailable) ||
|
|
memoryAvailable > memoryTotal
|
|
)
|
|
return 'Waiting for valid CPU and memory measurements.';
|
|
return null;
|
|
}
|
|
|
|
export function rebalanceHosts(): RebalanceHostOption[] {
|
|
const now = Date.now();
|
|
return listHosts().map((host) => {
|
|
const reason = hostBlocker(host, now);
|
|
return { id: host.id, name: host.name, eligible: !reason, reason };
|
|
});
|
|
}
|
|
|
|
/** Database evidence only: no probes, writes, reservations, or source capacity credits. */
|
|
export function rebalanceSnapshot(
|
|
settings: RebalanceSettings,
|
|
exceptOperationId = ''
|
|
): RebalancingSnapshot {
|
|
const handle = db(),
|
|
now = Date.now(),
|
|
at = new Date(now).toISOString(),
|
|
since = new Date(now - HISTORY_MS).toISOString();
|
|
const hosts = listHosts().filter((host) => settings.hostIds.includes(host.id));
|
|
const natives = handle
|
|
.prepare('SELECT * FROM native_instances WHERE retired_at IS NULL')
|
|
.all() as NativeInstance[];
|
|
const operations = handle
|
|
.prepare(
|
|
`SELECT id,server,stage FROM server_deployments WHERE server_deleted_at IS NULL
|
|
AND state IN ('queued','waiting','running','awaiting-health','interrupted') AND id!=?`
|
|
)
|
|
.all(exceptOperationId) as { id: string; server: string; stage: string }[];
|
|
const ownServer = exceptOperationId
|
|
? (
|
|
handle
|
|
.prepare('SELECT server FROM server_deployments WHERE id=?')
|
|
.get(exceptOperationId) as { server: string } | undefined
|
|
)?.server
|
|
: undefined;
|
|
const observe = observationReader();
|
|
const snapshot: RebalancingSnapshot = {
|
|
hosts: [],
|
|
instances: [],
|
|
exclusions: [],
|
|
warnings: []
|
|
};
|
|
for (const id of settings.hostIds)
|
|
if (!hosts.some((host) => host.id === id))
|
|
snapshot.warnings.push(`Participating host ${id} is unavailable.`);
|
|
for (const host of hosts) {
|
|
const reason = hostBlocker(host, now),
|
|
{ running, startup } = capacityCalibration(host.id);
|
|
if (reason) snapshot.warnings.push(`${host.name}: ${reason}`);
|
|
const inventory = host.inventory;
|
|
const valid =
|
|
!reason ||
|
|
(nonnegative(inventory.cpuCount) &&
|
|
inventory.cpuCount > 0 &&
|
|
nonnegative(inventory.cpuUsagePercent) &&
|
|
nonnegative(inventory.memoryTotal) &&
|
|
inventory.memoryTotal > 0 &&
|
|
nonnegative(inventory.memoryAvailable) &&
|
|
inventory.memoryAvailable <= inventory.memoryTotal);
|
|
const history = handle
|
|
.prepare(
|
|
`SELECT AVG(cpu_usage_percent) AS cpu,AVG(memory_available) AS memory FROM ec_host_resource_samples
|
|
WHERE host_id=? AND sampled_at>=? AND sampled_at<=? AND cpu_count=? AND memory_total=?`
|
|
)
|
|
.get(host.id, since, at, inventory.cpuCount ?? 0, inventory.memoryTotal ?? 0) as {
|
|
cpu: number | null;
|
|
memory: number | null;
|
|
};
|
|
const used: Resources = {
|
|
cpuCores: valid
|
|
? (inventory.cpuCount! * Math.max(inventory.cpuUsagePercent!, history.cpu ?? 0)) /
|
|
100
|
|
: 0,
|
|
memoryBytes: valid
|
|
? inventory.memoryTotal! -
|
|
Math.min(inventory.memoryAvailable!, history.memory ?? inventory.memoryAvailable!)
|
|
: 0
|
|
};
|
|
const hostInstances = natives.filter((instance) => instance.host_id === host.id),
|
|
measured = new Map(
|
|
inventory.instanceResources?.map((sample) => [sample.instanceId, sample])
|
|
);
|
|
const reserved = new Set<string>();
|
|
const reserve = (id: string, allowance: Resources) => {
|
|
if (reserved.has(id)) return;
|
|
reserved.add(id);
|
|
used.cpuCores += allowance.cpuCores;
|
|
used.memoryBytes += allowance.memoryBytes;
|
|
};
|
|
for (const instance of hostInstances) {
|
|
const sample = measured.get(instance.id);
|
|
if (sample?.generation === instance.generation) continue; // Already in the host measurement.
|
|
const latest = handle
|
|
.prepare(
|
|
`SELECT d.state,d.submitted_at,n.operation_id,n.stage_operation_id FROM native_provisioning n
|
|
JOIN server_deployments d ON d.id=n.deployment_id WHERE n.instance_id=? AND n.generation=? AND d.id!=?
|
|
AND d.action IN ('create','rebuild') AND d.server_deleted_at IS NULL ORDER BY d.rowid DESC LIMIT 1`
|
|
)
|
|
.get(instance.id, instance.generation, exceptOperationId) as
|
|
| {
|
|
state: string;
|
|
submitted_at: string | null;
|
|
operation_id: string | null;
|
|
stage_operation_id: string | null;
|
|
}
|
|
| undefined;
|
|
const accounted = handle
|
|
.prepare(
|
|
`SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
|
|
WHERE n.instance_id=? AND n.generation=? AND d.action IN ('create','rebuild') AND d.state IN ('done','warning') AND d.finished_at<?`
|
|
)
|
|
.get(instance.id, instance.generation, host.lastHeartbeat ?? '');
|
|
if (accounted) continue;
|
|
reserve(
|
|
instance.id,
|
|
latest?.state === 'queued' &&
|
|
!latest.submitted_at &&
|
|
!latest.operation_id &&
|
|
!latest.stage_operation_id
|
|
? running
|
|
: startup
|
|
);
|
|
}
|
|
const incoming = handle
|
|
.prepare(
|
|
`SELECT m.instance_id,m.removed_at,i.generation FROM ec_host_migrations m
|
|
JOIN server_deployments d ON d.id=m.deployment_id JOIN native_instances i ON i.id=m.instance_id
|
|
WHERE m.target_host_id=? AND m.deployment_id!=? AND d.server_deleted_at IS NULL AND i.retired_at IS NULL
|
|
AND d.state NOT IN ('done','warning','cancelled')`
|
|
)
|
|
.all(host.id, exceptOperationId) as {
|
|
instance_id: string;
|
|
removed_at: string | null;
|
|
generation: number;
|
|
}[];
|
|
for (const migration of incoming) {
|
|
const generation = migration.generation + (migration.removed_at ? 0 : 1);
|
|
if (measured.get(migration.instance_id)?.generation !== generation)
|
|
reserve(migration.instance_id, startup);
|
|
}
|
|
const ports = handle
|
|
.prepare('SELECT port,bind_address FROM ec_host_ports WHERE host_id=?')
|
|
.all(host.id) as { port: number; bind_address: string }[];
|
|
const usedPorts = new Set(
|
|
ports
|
|
.filter(
|
|
(port) =>
|
|
port.port >= host.portMin &&
|
|
port.port <= host.portMax &&
|
|
(port.bind_address === host.bindAddress ||
|
|
['0.0.0.0', '::'].includes(port.bind_address) ||
|
|
['0.0.0.0', '::'].includes(host.bindAddress))
|
|
)
|
|
.map((port) => port.port)
|
|
);
|
|
snapshot.hosts.push({
|
|
id: host.id,
|
|
name: host.name,
|
|
cpuCount: valid ? inventory.cpuCount! : 0,
|
|
memoryTotal: valid ? inventory.memoryTotal! : 0,
|
|
used,
|
|
running,
|
|
startup,
|
|
reason,
|
|
instances:
|
|
hostInstances.length +
|
|
incoming.filter(
|
|
(row) => !hostInstances.some((instance) => instance.id === row.instance_id)
|
|
).length,
|
|
ports: Math.max(0, host.portMax - host.portMin + 1 - usedPorts.size)
|
|
});
|
|
}
|
|
const destinations = new Map<string, Record<string, string | null>>();
|
|
for (const row of listServers()) {
|
|
const identity = mapIdentity(row);
|
|
if (
|
|
!row.host_id ||
|
|
!settings.hostIds.includes(row.host_id) ||
|
|
(settings.game !== 'all' && identity?.game !== settings.game)
|
|
)
|
|
continue;
|
|
const source = hosts.find((host) => host.id === row.host_id),
|
|
native = natives.find((instance) => instance.id === row.native_instance_id);
|
|
// The admitted rebuild itself must not hide the pre-removal empty evidence.
|
|
const observation =
|
|
row.name === ownServer
|
|
? serverObservation(row, {
|
|
provisioning: false,
|
|
settling: false,
|
|
telemetry: telemetryReport(row.name),
|
|
submission: latestGameSubmission(row.name)
|
|
})
|
|
: observe(row);
|
|
let reason = protectedWork(row.name, exceptOperationId);
|
|
if (!reason && (!identity || !native || !row.map_slug))
|
|
reason = 'A native instance with a known map is required.';
|
|
if (!reason && (!source || hostBlocker(source, now)))
|
|
reason = source ? hostBlocker(source, now) : 'Source host is unavailable.';
|
|
const operation = operations.find((item) => item.server === row.name);
|
|
if (!reason && operation) reason = `Provisioning ${operation.id}: ${operation.stage}`;
|
|
if (
|
|
!reason &&
|
|
native &&
|
|
handle
|
|
.prepare(
|
|
`SELECT 1 FROM ec_host_operations WHERE instance_id=? AND state IN ('pending','accepted','running','unknown')
|
|
AND kind IN ('instance.create','instance.rebuild','instance.start','instance.stop','instance.restart','instance.remove','instance.cancel-create')`
|
|
)
|
|
.get(native.id)
|
|
)
|
|
reason = 'An instance lifecycle command is unsettled.';
|
|
if (
|
|
!reason &&
|
|
(row.desired_state === 'stopped' ||
|
|
['stopped', 'exited', 'created', 'dead', 'missing'].includes(
|
|
row.container_state ?? ''
|
|
))
|
|
)
|
|
reason = 'Stopped instances are not eligible.';
|
|
if (
|
|
!reason &&
|
|
(row.last_players ?? 0) > 0 &&
|
|
freshStatusTimestamp(row.last_seen_at, now) &&
|
|
(!observation.telemetry_after ||
|
|
Date.parse(row.last_seen_at!) > Date.parse(observation.telemetry_after))
|
|
)
|
|
reason = 'Players are present according to a current RCON report.';
|
|
if (
|
|
!reason &&
|
|
(observation.health !== 'online' ||
|
|
observation.last_players === null ||
|
|
!freshStatusTimestamp(observation.last_seen_at, now))
|
|
)
|
|
reason = 'Waiting for healthy, fresh occupancy evidence.';
|
|
if (!reason && observation.last_players !== 0) reason = 'Players are present.';
|
|
if (!reason && !getServerKey(row.name))
|
|
reason = 'Save a server key before moving this instance.';
|
|
if (
|
|
!reason &&
|
|
handle
|
|
.prepare(
|
|
`SELECT 1 FROM rebalancing_moves WHERE instance_id=? AND state='done' AND updated_at>=?`
|
|
)
|
|
.get(native!.id, new Date(now - REBALANCE_COOLDOWN_MS).toISOString())
|
|
)
|
|
reason = 'This instance was rebalanced less than 30 minutes ago.';
|
|
const sample = source?.inventory.instanceResources?.find(
|
|
(item) => item.instanceId === native?.id && item.generation === native.generation
|
|
);
|
|
if (
|
|
!reason &&
|
|
(!sample ||
|
|
source?.inventory.instanceResourcesError ||
|
|
!nonnegative(sample.cpuCores) ||
|
|
!nonnegative(sample.memoryBytes) ||
|
|
!sample.memoryBytes)
|
|
)
|
|
reason = 'Waiting for resource measurements for the current instance generation.';
|
|
if (!reason && sample!.uptimeSeconds < 300)
|
|
reason = 'Waiting for this instance to finish its first five minutes of startup.';
|
|
if (!reason) {
|
|
try {
|
|
captureMoveRecipe(row.name);
|
|
} catch (error) {
|
|
reason = (error as Error).message;
|
|
}
|
|
}
|
|
if (reason) {
|
|
snapshot.exclusions.push({ server: row.name, reason });
|
|
continue;
|
|
}
|
|
const key = JSON.stringify([native!.library_id, native!.image]);
|
|
if (!destinations.has(key)) {
|
|
const eligibility: Record<string, string | null> = {};
|
|
for (const host of hosts) {
|
|
try {
|
|
requireReadyHost(host.id, { image: native!.image, requiredBytes: 1024 ** 3 });
|
|
assertReadyLibrary(host.id, native!.library_id);
|
|
eligibility[host.id] = null;
|
|
} catch (error) {
|
|
eligibility[host.id] = (error as Error).message;
|
|
}
|
|
}
|
|
destinations.set(key, eligibility);
|
|
}
|
|
const history = handle
|
|
.prepare(
|
|
`SELECT AVG(cpu_cores) AS cpu,AVG(memory_bytes) AS memory FROM ec_instance_resource_samples
|
|
WHERE host_id=? AND instance_id=? AND generation=? AND phase='running' AND bucket>=? AND bucket<=?`
|
|
)
|
|
.get(source!.id, native!.id, native!.generation, since, at) as {
|
|
cpu: number | null;
|
|
memory: number | null;
|
|
};
|
|
const bindConflicts = (bind: string) =>
|
|
bind === source!.bindAddress ||
|
|
['0.0.0.0', '::'].includes(bind) ||
|
|
['0.0.0.0', '::'].includes(source!.bindAddress);
|
|
const portOwners = handle
|
|
.prepare(
|
|
'SELECT instance_id,bind_address FROM ec_host_ports WHERE host_id=? AND port=?'
|
|
)
|
|
.all(source!.id, native!.port) as { instance_id: string; bind_address: string }[];
|
|
snapshot.instances.push({
|
|
server: row.name,
|
|
instanceId: native!.id,
|
|
generation: native!.generation,
|
|
sourceHostId: source!.id,
|
|
game: identity!.game,
|
|
releasesPort:
|
|
native!.port >= source!.portMin &&
|
|
native!.port <= source!.portMax &&
|
|
bindConflicts(native!.bind_address) &&
|
|
portOwners.some(
|
|
(owner) => owner.instance_id === native!.id && bindConflicts(owner.bind_address)
|
|
) &&
|
|
!portOwners.some(
|
|
(owner) => owner.instance_id !== native!.id && bindConflicts(owner.bind_address)
|
|
),
|
|
relief: {
|
|
cpuCores: Math.min(sample!.cpuCores, history.cpu ?? sample!.cpuCores),
|
|
memoryBytes: Math.min(sample!.memoryBytes, history.memory ?? sample!.memoryBytes)
|
|
},
|
|
demand: {
|
|
cpuCores: Math.max(sample!.cpuCores, history.cpu ?? sample!.cpuCores),
|
|
memoryBytes: Math.max(sample!.memoryBytes, history.memory ?? sample!.memoryBytes)
|
|
},
|
|
destinations: destinations.get(key)!
|
|
});
|
|
}
|
|
return snapshot;
|
|
}
|