Automatically place servers using learned resource capacity
This commit is contained in:
@@ -45,7 +45,7 @@ export interface CollectionPreview {
|
||||
removals: { name: string; reason: string }[];
|
||||
entries: { count: number; existing: number; missing: number; existingServers: string[];
|
||||
coverage: { name: string; health: string; href: string }[]; recipe: LaunchRecipeSummary }[];
|
||||
servers: { entryIndex: number; name: string | null; port: number | null; hasSavedKey: boolean }[];
|
||||
servers: { native?: import('./hosts.js').NativePlacement; entryIndex: number; name: string | null; port: number | null; hasSavedKey: boolean }[];
|
||||
blockers: string[];
|
||||
ready: boolean;
|
||||
}
|
||||
|
||||
+18
-1
@@ -68,7 +68,16 @@ export interface HostInventory {
|
||||
currentOperation?: string | null; [key: string]: unknown;
|
||||
capabilities?: string[];
|
||||
agentUpdate?: HostAgentUpdate;
|
||||
instanceResources?: InstanceResources[];
|
||||
}
|
||||
export interface InstanceResources {
|
||||
instanceId: string; generation: number; uptimeSeconds: number; cpuCores: number; memoryBytes: number;
|
||||
}
|
||||
export interface ResourceAllowance {
|
||||
cpuCores: number; memoryBytes: number; source: 'host' | 'fleet' | 'bootstrap';
|
||||
samples: number; observedAt: string | null;
|
||||
}
|
||||
export interface CapacityCalibration { running: ResourceAllowance; startup: ResourceAllowance }
|
||||
export interface Host {
|
||||
id: string; name: string; hostname: string; mode: HostMode; portMin: number; portMax: number;
|
||||
bindAddress: string; publicAddress: string; minimumFreeBytes: number; createdAt: string;
|
||||
@@ -95,6 +104,14 @@ export interface HostPreparation {
|
||||
}
|
||||
|
||||
export interface NativeDeploymentSelection {
|
||||
hostId: string; libraryIds: Partial<Record<'t5' | 't6', string>>; image: string;
|
||||
libraryIds: Partial<Record<'t5' | 't6', string>>; image: string;
|
||||
bindAddress?: string; publicAddress?: string; publicPort?: number;
|
||||
}
|
||||
|
||||
/** The host/port are provisional until the creation command is submitted. */
|
||||
export interface NativePlacement {
|
||||
hostId: string; libraryId: string; image?: string;
|
||||
bindAddress?: string; publicAddress?: string; publicPort?: number;
|
||||
automatic?: boolean;
|
||||
network?: { bindAddress?: string; publicAddress?: string; publicPort?: number };
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ export function migrateNativeInstances(handle: Database.Database): void {
|
||||
port INTEGER NOT NULL, public_address TEXT, public_port INTEGER,
|
||||
created_at TEXT NOT NULL, retired_at TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS native_instance_host ON native_instances(host_id) WHERE retired_at IS NULL;
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS native_instance_live_name ON native_instances(server) WHERE retired_at IS NULL;
|
||||
CREATE TABLE IF NOT EXISTS native_provisioning (
|
||||
deployment_id TEXT PRIMARY KEY REFERENCES server_deployments(id),
|
||||
@@ -16,6 +17,7 @@ export function migrateNativeInstances(handle: Database.Database): void {
|
||||
stage_operation_id TEXT, operation_id TEXT, files_json TEXT,
|
||||
applied_at TEXT, verified_at TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS native_provisioning_instance ON native_provisioning(instance_id);
|
||||
CREATE TABLE IF NOT EXISTS native_creation_cancellations (
|
||||
creation_id TEXT PRIMARY KEY REFERENCES server_deployments(id),
|
||||
cleanup_id TEXT NOT NULL UNIQUE REFERENCES server_deployments(id)
|
||||
|
||||
@@ -12,6 +12,7 @@ import { serverObservation } from './status.js';
|
||||
import { activeBetaRemovalBlocker, getServer } from './servers.js';
|
||||
import { parseServerName, zombieMapBySlug, zombieMaps } from './map-aliases.js';
|
||||
import { nativeSelection, nextNativePort } from '../provisioning/native.js';
|
||||
import { listHosts } from '../hosts/index.js';
|
||||
import { nativeInstance } from '../backends/native.js';
|
||||
|
||||
function coverage(entries: BlueprintEntry[]) {
|
||||
@@ -73,15 +74,24 @@ function collectionPlan(entries: BlueprintEntry[], mode: CollectionMode) {
|
||||
function allocate(entries: BlueprintEntry[], releasingServers: string[] = []): CollectionPreview['servers'] {
|
||||
const names: string[] = [];
|
||||
const ports: number[] = [];
|
||||
const planned: { hostId: string; port: number }[] = [];
|
||||
const counts = coverage(entries);
|
||||
return entries.flatMap((entry, entryIndex) => Array.from({ length: counts[entryIndex].missing }, () => {
|
||||
const base = `${entry.recipe.game}_zombie_${entry.recipe.mapSlug}`;
|
||||
const lobby = nextLobbyNumber(base, names);
|
||||
const name = lobby === null ? null : `${base}_${lobby}`;
|
||||
const port = entry.recipe.native ? nextNativePort(entry.recipe.native, ports) : nextAvailablePort(ports, releasingServers);
|
||||
let native = entry.recipe.native;
|
||||
let port: number | null;
|
||||
if (native) {
|
||||
try {
|
||||
native = nativeSelection(native as unknown as Record<string, unknown>, entry.recipe.game, { planned });
|
||||
port = nextNativePort(native, planned.filter(item => item.hostId === native!.hostId).map(item => item.port));
|
||||
if (port !== null) planned.push({ hostId: native.hostId, port });
|
||||
} catch { port = null; }
|
||||
} else port = nextAvailablePort(ports, releasingServers);
|
||||
if (name) names.push(name);
|
||||
if (port !== null) ports.push(port);
|
||||
return { entryIndex, name, port, hasSavedKey: !!name && !!getServerKey(name) };
|
||||
return { entryIndex, name, port, ...(native ? { native } : {}), hasSavedKey: !!name && !!getServerKey(name) };
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -98,7 +108,7 @@ function blockersFor(servers: CollectionPreview['servers'], entries: BlueprintEn
|
||||
if (!rconConfigured()) blockers.push('Configure RCON_PASSWORD so every new server can be verified.');
|
||||
}
|
||||
if (servers.some(server => !server.name)) blockers.push('There are not enough free lobbies for one or more maps (20 per map). Reduce their counts or free capacity.');
|
||||
if (servers.some(server => server.port === null)) blockers.push(`There are not enough free game ports in ${config.portRange.min}–${config.portRange.max} for the whole collection.`);
|
||||
if (servers.some(server => server.port === null)) blockers.push(entries.some(entry => entry.recipe.native) ? 'There is not enough host CPU, memory or port capacity for the whole collection.' : `There are not enough free game ports in ${config.portRange.min}–${config.portRange.max} for the whole collection.`);
|
||||
return blockers;
|
||||
}
|
||||
|
||||
@@ -117,7 +127,7 @@ export function previewCollection(blueprintId: unknown, mode: unknown = 'fill',
|
||||
if (mode !== 'fill' && mode !== 'exact') throw new Error('Choose fill or exact collection mode.');
|
||||
return db().transaction((): CollectionPreview => {
|
||||
const { entries, blueprintName } = blueprintEntries(blueprintId);
|
||||
if (selection?.hostId !== undefined || !config.komodo.address) {
|
||||
if (selection?.hostId !== undefined || !config.komodo.address || listHosts().length > 0) {
|
||||
const libraries = selection?.libraryIds as Record<string, unknown> | undefined;
|
||||
for (const entry of entries) {
|
||||
entry.recipe.native = nativeSelection({ ...selection, libraryId: libraries?.[entry.recipe.game] ?? selection?.libraryId }, entry.recipe.game);
|
||||
@@ -159,7 +169,7 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
|
||||
const servers = allocate(entries, releasingServers);
|
||||
const blockers = [...blockersFor(servers, entries), ...removalBlockersFor(removals)];
|
||||
if (blockers.length) throw new Error(blockers.join(' '));
|
||||
if (servers.length !== reviewed.length || servers.some((server, i) => server.entryIndex !== reviewed[i]?.entryIndex || server.name !== reviewed[i]?.name || server.port !== reviewed[i]?.port)) {
|
||||
if (servers.length !== reviewed.length || servers.some((server, i) => server.entryIndex !== reviewed[i]?.entryIndex || server.name !== reviewed[i]?.name || (!server.native?.automatic && server.port !== reviewed[i]?.port))) {
|
||||
throw new Error('Fleet counts, lobby or port availability changed. Nothing was queued. Refresh the preview and review each server key.');
|
||||
}
|
||||
const expected = new Set(servers.map(server => server.name));
|
||||
@@ -196,7 +206,7 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
|
||||
return { id, ports };
|
||||
});
|
||||
servers.forEach((server, index) => {
|
||||
const recipe = entries[server.entryIndex].recipe;
|
||||
const recipe = { ...entries[server.entryIndex].recipe, ...(server.native ? { native: server.native } : {}) };
|
||||
const childPreviewId = randomUUID();
|
||||
db().prepare('INSERT INTO server_launch_previews (id, expires_at, server, port, recipe, blueprint_name) VALUES (?, ?, ?, ?, ?, ?)')
|
||||
.run(childPreviewId, preview.expires_at, server.name, server.port, JSON.stringify(recipe), preview.blueprint_name);
|
||||
|
||||
@@ -9,7 +9,7 @@ import { getBaselineForScope, getVersion, listPromotedOverlaysForScope } from '.
|
||||
import { sha256Buffer } from './files.js';
|
||||
|
||||
export interface LaunchRecipe {
|
||||
native?: { hostId: string; libraryId: string; image?: string; bindAddress?: string; publicAddress?: string; publicPort?: number };
|
||||
native?: import('../../hosts.js').NativePlacement;
|
||||
format: 1;
|
||||
game: ZombieGame;
|
||||
mapSlug: string;
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
import { db } from '../db/index.js';
|
||||
import { ensureHostSchema } from './schema.js';
|
||||
import type { CapacityCalibration, HostInventory, ResourceAllowance } from '../../hosts.js';
|
||||
|
||||
const BOOTSTRAP_CPU = 0.5;
|
||||
const BOOTSTRAP_MEMORY = 1024 ** 3;
|
||||
const MARGIN = 1.25;
|
||||
const WINDOW_MS = 24 * 60 * 60_000;
|
||||
const MAX_SAMPLES = 10_000;
|
||||
type Phase = 'startup' | 'running';
|
||||
interface AllowanceRow {
|
||||
scope: string; phase: Phase; cpu_cores: number; memory_bytes: number; samples: number;
|
||||
window_start: string; observed_at: string; computed_at: string;
|
||||
}
|
||||
function database() { const handle = db(); ensureHostSchema(handle); return handle; }
|
||||
|
||||
export function validateInstanceResources(inventory: HostInventory): void {
|
||||
const samples = inventory.instanceResources;
|
||||
if (samples === undefined) return;
|
||||
if (!Array.isArray(samples) || samples.length > 1000) throw new Error('Invalid instance resource samples.');
|
||||
const seen = new Set<string>();
|
||||
for (const sample of samples) {
|
||||
if (!sample || typeof sample !== 'object' || typeof sample.instanceId !== 'string' || !sample.instanceId || sample.instanceId.length > 100
|
||||
|| !Number.isSafeInteger(sample.generation) || sample.generation < 1
|
||||
|| !Number.isSafeInteger(sample.uptimeSeconds) || sample.uptimeSeconds < 0
|
||||
|| typeof sample.cpuCores !== 'number' || !Number.isFinite(sample.cpuCores) || sample.cpuCores < 0
|
||||
|| !Number.isSafeInteger(sample.memoryBytes) || sample.memoryBytes <= 0
|
||||
|| (inventory.cpuCount !== undefined && sample.cpuCores > inventory.cpuCount + 0.01)
|
||||
|| (inventory.memoryTotal !== undefined && sample.memoryBytes > inventory.memoryTotal)
|
||||
|| seen.has(sample.instanceId)) throw new Error('Invalid instance resource sample.');
|
||||
seen.add(sample.instanceId);
|
||||
}
|
||||
}
|
||||
|
||||
/** Heartbeat ingestion owns all learning work. Launch requests only read cached allowances. */
|
||||
export function recordInstanceResources(hostId: string, inventory: HostInventory, timestamp: string): void {
|
||||
if (!inventory.instanceResources?.length || !inventory.dockerRunning || !inventory.cpuCount || !inventory.memoryTotal) return;
|
||||
const handle = database();
|
||||
const bucket = timestamp.slice(0, 16) + ':00.000Z';
|
||||
const observed = new Map<Phase, { cpu: number; memory: number }>();
|
||||
const owned = handle.prepare('SELECT 1 FROM native_instances WHERE id=? AND host_id=? AND generation=? AND retired_at IS NULL');
|
||||
const insert = handle.prepare(`INSERT INTO ec_instance_resource_samples
|
||||
(host_id,instance_id,generation,phase,bucket,cpu_cores,memory_bytes) VALUES(?,?,?,?,?,?,?)
|
||||
ON CONFLICT(host_id,instance_id,generation,phase,bucket) DO UPDATE SET
|
||||
cpu_cores=MAX(cpu_cores,excluded.cpu_cores), memory_bytes=MAX(memory_bytes,excluded.memory_bytes)`);
|
||||
for (const sample of inventory.instanceResources) {
|
||||
// An agent can only train from its currently owned generation; foreign,
|
||||
// retired and delayed pre-rebuild samples cannot contaminate the model.
|
||||
if (!owned.get(sample.instanceId, hostId, sample.generation)) continue;
|
||||
const phase: Phase = sample.uptimeSeconds < 300 ? 'startup' : 'running';
|
||||
insert.run(hostId, sample.instanceId, sample.generation, phase, bucket, sample.cpuCores, sample.memoryBytes);
|
||||
const prior = observed.get(phase);
|
||||
observed.set(phase, { cpu: Math.max(prior?.cpu ?? 0, sample.cpuCores), memory: Math.max(prior?.memory ?? 0, sample.memoryBytes) });
|
||||
}
|
||||
for (const [phase, peak] of observed) for (const scope of [hostId, '*']) {
|
||||
const previous = handle.prepare('SELECT * FROM ec_resource_allowances WHERE scope=? AND phase=?').get(scope, phase) as AllowanceRow | undefined;
|
||||
if (!previous || previous.computed_at < bucket) {
|
||||
const rows = handle.prepare(`SELECT cpu_cores, memory_bytes, bucket FROM ec_instance_resource_samples
|
||||
WHERE ${scope === '*' ? '' : 'host_id=? AND '}phase=? AND bucket>=? ORDER BY bucket DESC LIMIT ?`)
|
||||
.all(...(scope === '*' ? [] : [scope]), phase, new Date(Date.parse(timestamp) - WINDOW_MS).toISOString(), MAX_SAMPLES) as
|
||||
{ cpu_cores: number; memory_bytes: number; bucket: string }[];
|
||||
const percentile = (values: number[]) => values.sort((a, b) => a - b)[phase === 'startup' ? values.length - 1 : Math.ceil(values.length * 0.95) - 1];
|
||||
const cpu = Math.max(percentile(rows.map(row => row.cpu_cores)), peak.cpu) * MARGIN;
|
||||
const memory = Math.ceil(Math.max(percentile(rows.map(row => row.memory_bytes)), peak.memory) * MARGIN);
|
||||
handle.prepare(`INSERT INTO ec_resource_allowances
|
||||
(scope,phase,cpu_cores,memory_bytes,samples,window_start,observed_at,computed_at) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(scope,phase) DO UPDATE SET
|
||||
cpu_cores=excluded.cpu_cores,memory_bytes=excluded.memory_bytes,samples=excluded.samples,
|
||||
window_start=excluded.window_start,observed_at=excluded.observed_at,computed_at=excluded.computed_at`)
|
||||
.run(scope, phase, cpu, memory, rows.length, rows.at(-1)!.bucket, timestamp, timestamp);
|
||||
if (scope === '*') handle.prepare('DELETE FROM ec_instance_resource_samples WHERE bucket<?')
|
||||
.run(new Date(Date.parse(timestamp) - WINDOW_MS).toISOString());
|
||||
} else {
|
||||
// Raise the allowance immediately on a new peak; a lower sample
|
||||
// cannot reduce it until the next scheduled percentile calculation.
|
||||
handle.prepare(`UPDATE ec_resource_allowances SET cpu_cores=MAX(cpu_cores,?),memory_bytes=MAX(memory_bytes,?),observed_at=? WHERE scope=? AND phase=?`)
|
||||
.run(peak.cpu * MARGIN, Math.ceil(peak.memory * MARGIN), timestamp, scope, phase);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Host measurements account for hardware differences; fleet evidence covers new hosts. */
|
||||
export function capacityCalibration(hostId: string): CapacityCalibration {
|
||||
const rows = database().prepare('SELECT * FROM ec_resource_allowances WHERE scope IN (?,?) AND observed_at>=?')
|
||||
.all(hostId, '*', new Date(Date.now() - WINDOW_MS).toISOString()) as AllowanceRow[];
|
||||
const allowance = (phase: Phase): ResourceAllowance => {
|
||||
const local = rows.find(row => row.scope === hostId && row.phase === phase);
|
||||
const fleet = rows.find(row => row.scope === '*' && row.phase === phase);
|
||||
const learned = (row?: AllowanceRow) => !!row && row.samples >= (phase === 'startup' ? 3 : 20)
|
||||
&& Date.parse(row.observed_at) - Date.parse(row.window_start) >= (phase === 'startup' ? 120_000 : 300_000);
|
||||
const selected = learned(local) ? local : learned(fleet) ? fleet : undefined;
|
||||
return {
|
||||
// A few higher observations can increase a fleet/bootstrap allowance
|
||||
// immediately. Decreasing it requires the evidence thresholds above.
|
||||
cpuCores: Math.max(0.01, selected?.cpu_cores ?? BOOTSTRAP_CPU, local?.cpu_cores ?? 0, selected ? 0 : fleet?.cpu_cores ?? 0),
|
||||
memoryBytes: Math.max(selected?.memory_bytes ?? BOOTSTRAP_MEMORY, local?.memory_bytes ?? 0, selected ? 0 : fleet?.memory_bytes ?? 0),
|
||||
source: selected ? selected.scope === hostId ? 'host' : 'fleet' : 'bootstrap',
|
||||
samples: selected?.samples ?? local?.samples ?? fleet?.samples ?? 0,
|
||||
observedAt: selected?.observed_at ?? local?.observed_at ?? fleet?.observed_at ?? null
|
||||
};
|
||||
};
|
||||
const running = allowance('running'); const startup = allowance('startup');
|
||||
if (running.source === 'bootstrap' && (running.cpuCores > startup.cpuCores || running.memoryBytes > startup.memoryBytes)) startup.source = 'bootstrap';
|
||||
startup.cpuCores = Math.max(startup.cpuCores, running.cpuCores);
|
||||
startup.memoryBytes = Math.max(startup.memoryBytes, running.memoryBytes);
|
||||
return { running, startup };
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
import { db } from '../db/index.js';
|
||||
import { listHosts, requireReadyHost } from './index.js';
|
||||
import { requireReadyLibrary } from '../game-files/index.js';
|
||||
import type { Host } from '../../hosts.js';
|
||||
import { capacityCalibration } from './calibration.js';
|
||||
|
||||
// Storage and OS headroom are separate from learned instance CPU/memory usage.
|
||||
const INSTANCE_STORAGE_BYTES = 1024 ** 3;
|
||||
const MEMORY_HEADROOM = 512 * 1024 ** 2;
|
||||
export class CapacityError extends Error {}
|
||||
export interface PlacementOptions {
|
||||
hostId?: string; libraryId: string; image: string; bindAddress?: string;
|
||||
thorough?: boolean; excludeInstanceId?: string;
|
||||
planned?: { hostId: string; port: number }[];
|
||||
}
|
||||
|
||||
/** One indexed port read per host, including both protocols and wildcard binds. */
|
||||
export function availableHostPort(host: Host, options: Pick<PlacementOptions, 'bindAddress' | 'excludeInstanceId' | 'planned'> = {}): number | null {
|
||||
const bind = options.bindAddress ?? host.bindAddress;
|
||||
const rows = db().prepare('SELECT port,bind_address FROM ec_host_ports WHERE host_id=? AND instance_id!=?')
|
||||
.all(host.id, options.excludeInstanceId ?? '') as { port: number; bind_address: string }[];
|
||||
const used = new Set(rows.filter(row => row.bind_address === bind || ['0.0.0.0', '::'].includes(bind)
|
||||
|| ['0.0.0.0', '::'].includes(row.bind_address)).map(row => row.port));
|
||||
for (const item of options.planned ?? []) if (item.hostId === host.id) used.add(item.port);
|
||||
for (let port = host.portMin; port <= host.portMax; port++) if (!used.has(port)) return port;
|
||||
return null;
|
||||
}
|
||||
|
||||
function projectedLoad(host: Host, options: PlacementOptions): number {
|
||||
const { cpuCount, cpuUsagePercent, memoryTotal, memoryAvailable } = host.inventory;
|
||||
if (!cpuCount || cpuUsagePercent === undefined || !memoryTotal || memoryAvailable === undefined || memoryAvailable > memoryTotal) {
|
||||
throw new Error('Waiting for valid CPU and memory measurements.');
|
||||
}
|
||||
let cpu = cpuUsagePercent;
|
||||
let available = memoryAvailable;
|
||||
if (options.thorough) {
|
||||
const history = db().prepare(`SELECT AVG(cpu_usage_percent) AS cpu, AVG(memory_available) AS memory
|
||||
FROM ec_host_resource_samples WHERE host_id=? AND sampled_at>=? AND cpu_count=? AND memory_total=?`)
|
||||
.get(host.id, new Date(Date.now() - 5 * 60_000).toISOString(), cpuCount, memoryTotal) as { cpu: number | null; memory: number | null };
|
||||
cpu = Math.max(cpu, history.cpu ?? cpu);
|
||||
available = Math.min(available, history.memory ?? available);
|
||||
}
|
||||
// Unfinished/uncertain launches remain charged until recovery and a later
|
||||
// heartbeat. Observed usage accounts for established servers; the count
|
||||
// floor also reserves their minimum footprint while they are idle/stopped.
|
||||
const instances = db().prepare(`SELECT COUNT(*) AS total, COALESCE(SUM(NOT EXISTS (
|
||||
SELECT 1 FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
|
||||
WHERE n.instance_id=i.id AND d.action='create' AND d.state IN ('done','warning') AND d.finished_at<?
|
||||
)),0) AS pending FROM native_instances i WHERE i.host_id=? AND i.retired_at IS NULL AND i.id!=?`)
|
||||
.get(host.lastHeartbeat, host.id, options.excludeInstanceId ?? '') as { total: number; pending: number };
|
||||
const planned = options.planned?.filter(item => item.hostId === host.id).length ?? 0;
|
||||
const count = instances.total + planned + 1;
|
||||
const pending = instances.pending + planned + 1;
|
||||
const { running, startup } = capacityCalibration(host.id);
|
||||
const cores = Math.max(cpuCount * cpu / 100 + pending * startup.cpuCores,
|
||||
(count - pending) * running.cpuCores + pending * startup.cpuCores);
|
||||
const memory = Math.max(memoryTotal - available + pending * startup.memoryBytes,
|
||||
(count - pending) * running.memoryBytes + pending * startup.memoryBytes);
|
||||
if (cores > cpuCount * 0.9) throw new Error('Insufficient CPU capacity.');
|
||||
if (memory > memoryTotal - MEMORY_HEADROOM) throw new Error('Insufficient memory capacity.');
|
||||
return Math.max(cores / cpuCount, memory / memoryTotal);
|
||||
}
|
||||
|
||||
/** Admission stops at the first fit; placement evaluates and ranks every fit. No remote I/O. */
|
||||
export function choosePlacement(options: PlacementOptions): { host: Host; port: number; load: number } {
|
||||
const candidates: { host: Host; port: number; load: number }[] = [];
|
||||
const reasons: string[] = [];
|
||||
for (const host of listHosts().filter(host => !options.hostId || host.id === options.hostId)) {
|
||||
try {
|
||||
requireReadyHost(host.id, { image: options.image, requiredBytes: INSTANCE_STORAGE_BYTES });
|
||||
requireReadyLibrary(host.id, options.libraryId);
|
||||
const port = availableHostPort(host, options);
|
||||
if (port === null) throw new Error('No available UDP/TCP game port.');
|
||||
const load = projectedLoad(host, options);
|
||||
const candidate = { host, port, load };
|
||||
if (!options.thorough) return candidate;
|
||||
candidates.push(candidate);
|
||||
} catch (error) { reasons.push(`${host.name}: ${(error as Error).message}`); }
|
||||
}
|
||||
candidates.sort((a, b) => a.load - b.load || a.host.id.localeCompare(b.host.id));
|
||||
if (candidates.length) return candidates[0];
|
||||
throw new CapacityError(`No host has capacity for this server. ${reasons.join(' ') || 'Add and prepare a host.'}`);
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'node:crypt
|
||||
import { isIP } from 'node:net';
|
||||
import { db } from '../db/index.js';
|
||||
import { ensureHostSchema } from './schema.js';
|
||||
import { recordInstanceResources, validateInstanceResources } from './calibration.js';
|
||||
import type { Host, HostDecommission, HostDeletion, HostInventory, HostMode, HostOperation, HostOperationState } from '../../hosts.js';
|
||||
|
||||
export const AGENT_PROTOCOL_VERSION = 1;
|
||||
@@ -219,6 +220,7 @@ export function validateInventory(value: unknown): HostInventory {
|
||||
if (inventory[key] !== undefined && (typeof inventory[key] !== 'number' || !Number.isSafeInteger(inventory[key]) || Number(inventory[key]) < 0)) throw new Error(`Invalid inventory ${key}.`);
|
||||
}
|
||||
if (inventory.cpuUsagePercent !== undefined && (typeof inventory.cpuUsagePercent !== 'number' || !Number.isFinite(inventory.cpuUsagePercent) || inventory.cpuUsagePercent < 0 || inventory.cpuUsagePercent > 100)) throw new Error('Invalid inventory cpuUsagePercent.');
|
||||
validateInstanceResources(inventory);
|
||||
for (const key of ['dockerRunning','directoriesWritable']) if (inventory[key] !== undefined && typeof inventory[key] !== 'boolean') throw new Error(`Invalid inventory ${key}.`);
|
||||
if (inventory.runtimeImages !== undefined && (!Array.isArray(inventory.runtimeImages) || inventory.runtimeImages.some(image => typeof image !== 'string'))) throw new Error('Invalid runtime image inventory.');
|
||||
if (inventory.capabilities !== undefined && (!Array.isArray(inventory.capabilities) || inventory.capabilities.length > 32 || inventory.capabilities.some(value => typeof value !== 'string' || value.length > 80))) throw new Error('Invalid agent capability inventory.');
|
||||
@@ -253,6 +255,7 @@ export function enrollHost(token: unknown, rawInventory: unknown): { hostId: str
|
||||
revokeCredentials(enrollment.host_id);
|
||||
const credential = issueCredential(enrollment.host_id);
|
||||
database().prepare('UPDATE ec_hosts SET enrolled_at=?,last_heartbeat=?,hostname=?,inventory=? WHERE id=?').run(now(), now(), inventory.hostname ?? '', JSON.stringify(inventory), enrollment.host_id);
|
||||
recordResources(enrollment.host_id, inventory, now());
|
||||
return { hostId: enrollment.host_id, credential, heartbeatSeconds: 15 };
|
||||
})();
|
||||
}
|
||||
@@ -284,9 +287,23 @@ export function rotateCredential(hostId: string, nextCredential?: string): strin
|
||||
}
|
||||
export function heartbeat(hostId: string, value: unknown): Host {
|
||||
requireHost(hostId); const inventory = validateInventory(value);
|
||||
database().prepare('UPDATE ec_hosts SET last_heartbeat=?,hostname=?,inventory=? WHERE id=?').run(now(), inventory.hostname ?? '', JSON.stringify(inventory), hostId);
|
||||
database().transaction(() => {
|
||||
const timestamp = now();
|
||||
database().prepare('UPDATE ec_hosts SET last_heartbeat=?,hostname=?,inventory=? WHERE id=?').run(timestamp, inventory.hostname ?? '', JSON.stringify(inventory), hostId);
|
||||
recordResources(hostId, inventory, timestamp);
|
||||
})();
|
||||
return requireHost(hostId);
|
||||
}
|
||||
|
||||
function recordResources(hostId: string, inventory: HostInventory, timestamp: string): void {
|
||||
recordInstanceResources(hostId, inventory, timestamp);
|
||||
database().prepare(`INSERT OR REPLACE INTO ec_host_resource_samples
|
||||
(host_id,sampled_at,cpu_count,cpu_usage_percent,memory_total,memory_available) VALUES(?,?,?,?,?,?)`)
|
||||
.run(hostId, timestamp, inventory.cpuCount ?? null, inventory.cpuUsagePercent ?? null,
|
||||
inventory.memoryTotal ?? null, inventory.memoryAvailable ?? null);
|
||||
database().prepare('DELETE FROM ec_host_resource_samples WHERE host_id=? AND sampled_at<?')
|
||||
.run(hostId, new Date(Date.parse(timestamp) - 7 * 86400_000).toISOString());
|
||||
}
|
||||
export function requireReadyHost(hostId: string, options: { libraryId?: string; requiredBytes?: number; image?: string; port?: number; bindAddress?: string; protocol?: 'udp' | 'tcp' } = {}): Host {
|
||||
const host = requireHost(hostId); const reasons = [...host.readinessReasons];
|
||||
if (options.requiredBytes !== undefined && (host.inventory.storageAvailable ?? 0) < options.requiredBytes + host.minimumFreeBytes) reasons.push('Insufficient storage for this deployment.');
|
||||
|
||||
@@ -40,6 +40,25 @@ export function ensureHostSchema(handle: Database.Database): void {
|
||||
PRIMARY KEY(host_id,bind_address,protocol,port)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS ec_host_ports_instance ON ec_host_ports(instance_id);
|
||||
CREATE TABLE IF NOT EXISTS ec_host_resource_samples (
|
||||
host_id TEXT NOT NULL REFERENCES ec_hosts(id), sampled_at TEXT NOT NULL,
|
||||
cpu_count REAL, cpu_usage_percent REAL, memory_total REAL, memory_available REAL,
|
||||
PRIMARY KEY(host_id, sampled_at)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS ec_instance_resource_samples (
|
||||
host_id TEXT NOT NULL, instance_id TEXT NOT NULL, generation INTEGER NOT NULL,
|
||||
phase TEXT NOT NULL CHECK(phase IN ('startup','running')), bucket TEXT NOT NULL,
|
||||
cpu_cores REAL NOT NULL, memory_bytes REAL NOT NULL,
|
||||
PRIMARY KEY(host_id,instance_id,generation,phase,bucket)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS ec_instance_resources_host ON ec_instance_resource_samples(host_id,phase,bucket DESC);
|
||||
CREATE INDEX IF NOT EXISTS ec_instance_resources_fleet ON ec_instance_resource_samples(phase,bucket DESC);
|
||||
CREATE INDEX IF NOT EXISTS ec_instance_resources_retention ON ec_instance_resource_samples(bucket);
|
||||
CREATE TABLE IF NOT EXISTS ec_resource_allowances (
|
||||
scope TEXT NOT NULL, phase TEXT NOT NULL, cpu_cores REAL NOT NULL, memory_bytes REAL NOT NULL,
|
||||
samples INTEGER NOT NULL, window_start TEXT NOT NULL, observed_at TEXT NOT NULL, computed_at TEXT NOT NULL,
|
||||
PRIMARY KEY(scope,phase)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS ec_host_decommissions (
|
||||
host_id TEXT PRIMARY KEY REFERENCES ec_hosts(id),
|
||||
operation_id TEXT NOT NULL UNIQUE REFERENCES ec_host_operations(id),
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { listHosts } from '../hosts/index.js';
|
||||
import { fleetSettings } from '../settings.js';
|
||||
import { probePlayerEvidence, probeServerStatus } from '../fleet/status-evidence.js';
|
||||
import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent, REMOVING_SERVER, resumeCleanup, resumeVerification, claimRemoval } from './history.js';
|
||||
@@ -176,7 +177,7 @@ interface PreviewRow {
|
||||
|
||||
/** Read-only checks; a preview never reserves a slot or writes credentials. */
|
||||
export function previewCreation(input: { game?: unknown; mapSlug?: unknown; blueprintId?: unknown; deploymentId?: unknown; hostId?: unknown; libraryId?: unknown }): LaunchPreview {
|
||||
if (input.hostId !== undefined || !config.komodo.address) return previewNativeCreation(input);
|
||||
if (input.hostId !== undefined || !config.komodo.address || listHosts().length > 0) return previewNativeCreation(input);
|
||||
if (input.blueprintId !== undefined && input.deploymentId !== undefined) throw new Error('Choose one starting point.');
|
||||
let recipe: LaunchRecipe;
|
||||
let blueprintName: string | null = null;
|
||||
@@ -218,7 +219,7 @@ export function previewCreation(input: { game?: unknown; mapSlug?: unknown; blue
|
||||
// releasingServers is internal to atomic collection queueing; HTTP callers cannot set it.
|
||||
export function queueCreateServer(input: CreateServerInput, releasingServers: string[] = []): CreateServerResult {
|
||||
const nativePreview = input.previewId ? db().prepare("SELECT 1 FROM server_launch_previews WHERE id=? AND json_extract(recipe,'$.native.hostId') IS NOT NULL").get(input.previewId) : null;
|
||||
if (input.hostId || nativePreview || !config.komodo.address) return queueNativeCreation(input);
|
||||
if (input.hostId !== undefined || nativePreview || !config.komodo.address || listHosts().length > 0) return queueNativeCreation(input);
|
||||
return db().transaction((): CreateServerResult => {
|
||||
const preview = input.previewId ? db().prepare('SELECT * FROM server_launch_previews WHERE id = ?').get(input.previewId) as PreviewRow | undefined : undefined;
|
||||
if (input.previewId && !preview) return { ok: false, kind: 'bad-request', message: 'Launch preview expired or was not found. Review a fresh preview.' };
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
import { isIP } from 'node:net';
|
||||
import { CapacityError, choosePlacement, availableHostPort, type PlacementOptions } from '../hosts/capacity.js';
|
||||
import { randomBytes, randomUUID } from 'node:crypto';
|
||||
import { posix } from 'node:path';
|
||||
import { db } from '../db/index.js';
|
||||
import { config, relDestForKind, serverDir } from '../config.js';
|
||||
import { nativeInstance, nativeRequest, type NativeInstance } from '../backends/native.js';
|
||||
import { getHost, getHostOperation, listHosts, nextHostPort, queueHostOperation, releaseHostPorts, requireReadyHost, reserveHostPort } from '../hosts/index.js';
|
||||
import { getHost, getHostOperation, listHosts, queueHostOperation, releaseHostPorts, reserveHostPort } from '../hosts/index.js';
|
||||
import { newestReadyLibrary, requireReadyLibrary } from '../game-files/index.js';
|
||||
import { blueprintRecipe, captureRecipe, recipeForDeployment, summarizeRecipe, type LaunchRecipe } from '../fleet/launch-recipes.js';
|
||||
import { getServerKey, saveServerKey } from '../fleet/server-keys.js';
|
||||
@@ -24,14 +26,14 @@ import type { ProvisioningAction } from '../../provisioning.js';
|
||||
import { hostAgentUpdatesEnabled } from '../../hosts.js';
|
||||
import { cancelCreation, finishCreationCancellation } from './cancel-creation.js';
|
||||
|
||||
export interface NativeSelection { hostId: string; libraryId: string; image?: string; bindAddress?: string; publicAddress?: string; publicPort?: number }
|
||||
export type NativeSelection = import('../../hosts.js').NativePlacement;
|
||||
type NativeRecipe = LaunchRecipe & { native?: NativeSelection };
|
||||
interface Job { id: string; server: string; action: ProvisioningAction; state: string; submitted_at: string | null; stage: string;
|
||||
instance_id: string; generation: number; stage_operation_id: string | null; operation_id: string | null; files_json: string | null; applied_at: string | null }
|
||||
interface FilePlan { id: number; path: string; contentBase64: string | null; sha256: string | null; checkBefore: boolean; expectedBeforeSha: string | null }
|
||||
|
||||
export function nativeSelection(input: Record<string, unknown>, game: string): NativeSelection {
|
||||
if (typeof input.hostId !== 'string' || !input.hostId) throw new Error('Select a ready host.');
|
||||
export function nativeSelection(input: Record<string, unknown>, game: string, options: Pick<PlacementOptions, 'planned'> = {}): NativeSelection {
|
||||
if (input.hostId !== undefined && typeof input.hostId !== 'string') throw new Error('Invalid host.');
|
||||
const version = newestReadyLibrary(game);
|
||||
if (!version) throw new Error(`Upload ${game.toUpperCase()} game files before launching a server.`);
|
||||
if (input.libraryId !== undefined && input.libraryId !== '' && input.libraryId !== version.id) {
|
||||
@@ -39,14 +41,17 @@ export function nativeSelection(input: Record<string, unknown>, game: string): N
|
||||
}
|
||||
const image = input.image === undefined || input.image === '' ? config.komodo.defaultImage : input.image;
|
||||
if (typeof image !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9._/:@-]{0,499}$/.test(image)) throw new Error('Invalid runtime image.');
|
||||
const host = requireReadyHost(input.hostId, { image, requiredBytes: 1024 ** 3 });
|
||||
requireReadyLibrary(host.id, version.id);
|
||||
const bindAddress = input.bindAddress === undefined || input.bindAddress === '' ? host.bindAddress : input.bindAddress;
|
||||
if (typeof bindAddress !== 'string') throw new Error('Invalid bind address.');
|
||||
if (input.publicAddress !== undefined && (typeof input.publicAddress !== 'string' || input.publicAddress.length > 253)) throw new Error('Invalid public address.');
|
||||
if (input.publicPort !== undefined && (!Number.isInteger(input.publicPort) || Number(input.publicPort) < 1 || Number(input.publicPort) > 65535)) throw new Error('Invalid public port.');
|
||||
return { hostId: host.id, libraryId: version.id, image, bindAddress, publicAddress: input.publicAddress as string | undefined ?? host.publicAddress,
|
||||
publicPort: input.publicPort as number | undefined };
|
||||
const automatic = input.automatic === true || !input.hostId;
|
||||
const rawNetwork = input.automatic === true ? input.network ?? {} : input;
|
||||
if (!rawNetwork || typeof rawNetwork !== 'object' || Array.isArray(rawNetwork)) throw new Error('Invalid network overrides.');
|
||||
const network = rawNetwork as NativeSelection;
|
||||
const bindAddress = network.bindAddress || undefined;
|
||||
if (bindAddress !== undefined && (typeof bindAddress !== 'string' || !isIP(bindAddress))) throw new Error('Invalid bind address.');
|
||||
const { host } = choosePlacement({ hostId: automatic ? undefined : input.hostId as string, libraryId: version.id, image, bindAddress, ...options });
|
||||
if (network.publicAddress !== undefined && (typeof network.publicAddress !== 'string' || network.publicAddress.length > 253)) throw new Error('Invalid public address.');
|
||||
if (network.publicPort !== undefined && (!Number.isInteger(network.publicPort) || Number(network.publicPort) < 1 || Number(network.publicPort) > 65535)) throw new Error('Invalid public port.');
|
||||
return { hostId: host.id, libraryId: version.id, image, bindAddress: bindAddress ?? host.bindAddress, publicAddress: network.publicAddress ?? host.publicAddress,
|
||||
publicPort: network.publicPort, automatic, network: { bindAddress, publicAddress: network.publicAddress, publicPort: network.publicPort } };
|
||||
}
|
||||
|
||||
export function previewNativeCreation(input: Record<string, unknown>): LaunchPreview {
|
||||
@@ -72,10 +77,10 @@ export function previewNativeCreation(input: Record<string, unknown>): LaunchPre
|
||||
db().prepare('INSERT INTO server_launch_previews(id,expires_at,server,port,recipe,blueprint_name) VALUES(?,?,?,?,?,?)')
|
||||
.run(id, expiresAt, name, port, JSON.stringify(recipe), blueprintName);
|
||||
const checks: LaunchPreview['checks'] = [
|
||||
{ label: 'Host readiness', state: 'pass', detail: `${getHost(selection.hostId)!.name}: connected, Docker ready, storage and runtime verified.` },
|
||||
{ label: 'Game files', state: 'pass', detail: `The latest ${recipe.game.toUpperCase()} game files are verified on the selected host.` },
|
||||
{ label: 'Host capacity', state: 'pass', detail: selection.automatic ? 'Capacity available. The least loaded eligible host is selected when provisioning starts.' : `${getHost(selection.hostId)!.name}: capacity available.` },
|
||||
{ label: 'Game files', state: 'pass', detail: `The latest ${recipe.game.toUpperCase()} game files are verified on an eligible host.` },
|
||||
{ label: 'Lobby', state: name ? 'pass' : 'blocked', detail: name ?? 'All lobby IDs for this map are reserved.' },
|
||||
{ label: 'Host port', state: port ? 'pass' : 'blocked', detail: port ? `${selection.bindAddress}:${port} / UDP and TCP` : 'No available UDP/TCP port on this host.' },
|
||||
{ label: 'Host port', state: port ? 'pass' : 'blocked', detail: port ? selection.automatic ? 'UDP/TCP capacity available; port assigned at placement.' : `${selection.bindAddress}:${port} / UDP and TCP` : 'No available UDP/TCP port on this host.' },
|
||||
{ label: 'Configuration', state: 'pass', detail: recipe.configSource }
|
||||
];
|
||||
return { id, expiresAt, name, port, hasSavedKey: !!name && !!getServerKey(name), recipe: summarizeRecipe(recipe), blueprintName, checks,
|
||||
@@ -83,16 +88,9 @@ export function previewNativeCreation(input: Record<string, unknown>): LaunchPre
|
||||
}
|
||||
|
||||
export function nextNativePort(selection: NativeSelection, taken: number[] = []): number | null {
|
||||
const excluded = [...taken];
|
||||
for (;;) {
|
||||
const port = nextHostPort(selection.hostId, excluded, selection.bindAddress, 'udp');
|
||||
if (port === null) return null;
|
||||
// Both protocols are published by Plutainer and reserved together.
|
||||
const tcp = nextHostPort(selection.hostId, excluded, selection.bindAddress, 'tcp');
|
||||
if (tcp === port) return port;
|
||||
if (tcp !== null && tcp < port) { excluded.push(tcp); continue; }
|
||||
excluded.push(port);
|
||||
}
|
||||
const host = getHost(selection.hostId);
|
||||
return host ? availableHostPort(host, { bindAddress: selection.bindAddress,
|
||||
planned: taken.map(port => ({ hostId: host.id, port })) }) : null;
|
||||
}
|
||||
|
||||
export function queueNativeCreation(input: CreateServerInput): CreateServerResult {
|
||||
@@ -113,7 +111,7 @@ export function queueNativeCreation(input: CreateServerInput): CreateServerResul
|
||||
const name = `${game}_zombie_${input.mapSlug}_${lobby}`;
|
||||
const port = nextNativePort(selection);
|
||||
if (!port) throw new Error('No available host port.');
|
||||
if ((input.expectedName && input.expectedName !== name) || (preview && (preview.server !== name || preview.port !== port))) throw new Error('Available lobby or host port changed. Review a fresh preview.');
|
||||
if ((input.expectedName && input.expectedName !== name) || (preview && (preview.server !== name || (!selection.automatic && preview.port !== port)))) throw new Error('Available lobby or host port changed. Review a fresh preview.');
|
||||
const key = input.plutoKey?.trim() || getServerKey(name);
|
||||
if (!key) throw new Error(`Save a Plutonium server key for ${name}.`);
|
||||
saveServerKey(name, key);
|
||||
@@ -128,11 +126,11 @@ export function queueNativeCreation(input: CreateServerInput): CreateServerResul
|
||||
db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,?,?,?,?)')
|
||||
.run(id, name, port, 'zombie', recipe.mapSlug, recipe.expectedMap);
|
||||
db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,blueprint_name) VALUES(?,?,?)').run(id, JSON.stringify(recipe), preview?.blueprint_name ?? null);
|
||||
if (preview) db().prepare('UPDATE server_launch_previews SET deployment_id=? WHERE id=?').run(id, preview.id);
|
||||
if (preview) db().prepare('UPDATE server_launch_previews SET deployment_id=?,port=? WHERE id=?').run(id, port, preview.id);
|
||||
deploymentStage(id, 'Waiting for provisioning evaluation', { phase: 'eligibility', recipeStatus: 'queued' });
|
||||
return { ok: true as const, name, port, deploymentId: id };
|
||||
})();
|
||||
} catch (error) { return { ok: false, kind: 'bad-request', message: (error as Error).message }; }
|
||||
} catch (error) { return { ok: false, kind: error instanceof CapacityError ? 'conflict' : 'bad-request', message: (error as Error).message }; }
|
||||
}
|
||||
|
||||
export function bindNativeOperation(id: string, server: string): void {
|
||||
@@ -270,12 +268,43 @@ export async function processNativeProvisioning(eligibility: (operation: { id: s
|
||||
|
||||
const activeJobs = new Map<string, Promise<void>>();
|
||||
|
||||
/** Reservations can move only while no remote work has ever been submitted. */
|
||||
function placeQueuedCreation(job: Job, instance: NativeInstance): NativeInstance {
|
||||
return db().transaction(() => {
|
||||
const recipe = recipeForDeployment(job.id)!.recipe as NativeRecipe;
|
||||
const prior = recipe.native!;
|
||||
const network = prior.automatic ? prior.network ?? {} : prior;
|
||||
const { host, port } = choosePlacement({ hostId: prior.automatic ? undefined : instance.host_id,
|
||||
libraryId: instance.library_id, image: instance.image, bindAddress: network.bindAddress,
|
||||
thorough: true, excludeInstanceId: instance.id });
|
||||
const bind = network.bindAddress ?? host.bindAddress;
|
||||
releaseHostPorts(instance.id);
|
||||
reserveHostPort(host.id, instance.id, { port, bindAddress: bind, protocol: 'udp' });
|
||||
reserveHostPort(host.id, instance.id, { port, bindAddress: bind, protocol: 'tcp' });
|
||||
db().prepare(`UPDATE native_instances SET host_id=?,bind_address=?,port=?,public_address=?,public_port=? WHERE id=?`)
|
||||
.run(host.id, bind, port, network.publicAddress ?? host.publicAddress, network.publicPort ?? null, instance.id);
|
||||
db().prepare('UPDATE server_creation_jobs SET port=? WHERE deployment_id=?').run(port, job.id);
|
||||
recipe.native = { ...prior, hostId: host.id, bindAddress: bind, publicAddress: network.publicAddress ?? host.publicAddress, publicPort: network.publicPort };
|
||||
db().prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?').run(JSON.stringify(recipe), job.id);
|
||||
db().prepare('UPDATE server_launch_previews SET port=? WHERE deployment_id=?').run(port, job.id);
|
||||
deploymentStage(job.id, `Placed on ${host.name} after CPU, memory, history and port checks`, { phase: 'eligibility' });
|
||||
return nativeInstance(job.server)!;
|
||||
})();
|
||||
}
|
||||
|
||||
async function processJob(job: Job, eligibility: (job: Job) => string | null) {
|
||||
if (!stillActive(job)) return;
|
||||
let instance = nativeInstance(job.server);
|
||||
if (!instance || instance.id !== job.instance_id || (instance.generation !== job.generation && !(job.action === 'rebuild' && instance.generation === job.generation + 1))) {
|
||||
finishDeployment(job.id, { ok: false, message: 'Original instance was retired or replaced; no command was sent.' }); return;
|
||||
}
|
||||
if (job.action === 'create' && !job.operation_id && !job.stage_operation_id && !job.submitted_at) {
|
||||
const settings = fleetSettings();
|
||||
const active = db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL
|
||||
AND action IN ('create','rebuild') AND state IN ('running','awaiting-health')`).get(job.id) as { n: number };
|
||||
if (active.n >= settings.createConcurrency) { hold(job, 'Waiting for startup capacity.'); return; }
|
||||
instance = placeQueuedCreation(job, instance);
|
||||
}
|
||||
const host = getHost(instance.host_id);
|
||||
if (!host?.connected || host.mode === 'disabled') {
|
||||
db().prepare("UPDATE servers SET container_state='unknown',last_players=NULL,last_roster=NULL WHERE name=?").run(job.server);
|
||||
@@ -403,7 +432,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
|
||||
}
|
||||
if (job.action === 'destroy' && count(['destroy']) >= settings.destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; }
|
||||
deploymentStage(job.id, 'Checking host eligibility', { phase: 'eligibility', blocker: null });
|
||||
if (job.action === 'create') requireReadyHost(instance.host_id, { image: instance.image, requiredBytes: 1024 ** 3 });
|
||||
if (job.action === 'create') choosePlacement({ hostId: instance.host_id, libraryId: instance.library_id, image: instance.image,
|
||||
bindAddress: instance.bind_address, excludeInstanceId: instance.id, thorough: true });
|
||||
if (job.action === 'create' || job.action === 'rebuild') requireReadyLibrary(instance.host_id, instance.library_id);
|
||||
if (job.action !== 'create' && job.action !== 'start') {
|
||||
// Fast controller check keeps occupied lobbies out of the operation slots.
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
let mode = $state<CollectionMode>('fill');
|
||||
let games = $state<ZombieGame[]>([]);
|
||||
let collectionName = $state('Launch collection');
|
||||
let hostSelection = $state<NativeDeploymentSelection>({ hostId: '', libraryIds: {}, image: '' });
|
||||
let hostSelection = $state<NativeDeploymentSelection>({ libraryIds: {}, image: '' });
|
||||
let hostReady = $state(false);
|
||||
let confirmRemovals = $state(false);
|
||||
let keys = $state<Record<string, string>>({});
|
||||
@@ -41,7 +41,7 @@
|
||||
|
||||
async function initialize() {
|
||||
const current = ++session; busy = true; error = ''; games = []; hostReady = false;
|
||||
hostSelection = { hostId: '', libraryIds: {}, image: '' };
|
||||
hostSelection = { libraryIds: {}, image: '' };
|
||||
try {
|
||||
const response = await fetch('/api/blueprints'); const body = await response.json();
|
||||
if (!response.ok) throw new Error(body.error ?? 'Could not load collection.');
|
||||
@@ -54,12 +54,12 @@
|
||||
}
|
||||
|
||||
async function review() {
|
||||
if (!hostReady) { error = 'Choose a host and wait for its latest game files and runtime to finish automatic setup.'; return; }
|
||||
if (!hostReady) { error = 'Wait for an eligible host and wait for its latest game files and runtime to finish automatic setup.'; return; }
|
||||
const current = ++session;
|
||||
const previous = preview;
|
||||
busy = true; error = ''; keyNotice = ''; confirmRemovals = false;
|
||||
try {
|
||||
const res = await fetch('/api/deployment-collections/preview', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ blueprintId, mode, hostId: hostSelection.hostId, libraryIds: hostSelection.libraryIds, image: hostSelection.image }) });
|
||||
const res = await fetch('/api/deployment-collections/preview', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ blueprintId, mode, libraryIds: hostSelection.libraryIds, image: hostSelection.image }) });
|
||||
const body = await res.json();
|
||||
if (!res.ok) throw new Error(body.error ?? 'Could not preview the collection.');
|
||||
if (current === session) {
|
||||
@@ -105,7 +105,7 @@
|
||||
{#if open && games.length}<NativeHostSelection {games} bind:selection={hostSelection} bind:ready={hostReady} disabled={busy} onchange={() => { preview = null; confirmRemovals = false; }} />{/if}
|
||||
<p class="muted">{mode === 'exact' ? 'Keep matching lobbies, add missing ones, and remove excess lobbies and maps outside this collection across both games.' : 'Create missing lobbies and keep all existing servers.'}</p>
|
||||
{#if error}<p class="error" role="alert">{error}</p>{/if}
|
||||
{#if !preview}<p class="muted">{busy ? 'Preparing the collection…' : 'Choose a host. Its latest game files sync automatically before you review capacity and server keys.'}</p>{#if !busy}<Button disabled={!hostReady} onclick={review}>Preview launch</Button>{/if}
|
||||
{#if !preview}<p class="muted">{busy ? 'Preparing the collection…' : 'Wait for an eligible host. Its latest game files sync automatically before you review capacity and server keys.'}</p>{#if !busy}<Button disabled={!hostReady} onclick={review}>Preview launch</Button>{/if}
|
||||
{:else}
|
||||
{#if keyNotice}<p role="status" class="muted">{keyNotice}</p>{/if}
|
||||
{#if expired}<p class="error" role="status">This preview expired. Refresh before launch; keys remain entered for unchanged IDs.</p>{/if}
|
||||
@@ -133,7 +133,7 @@
|
||||
{#each preview.servers as server, index (server.name ?? `unavailable-${index}`)}
|
||||
<li class="server-row">
|
||||
<span class="order" aria-hidden="true">{String(index + 1).padStart(2, '0')}</span>
|
||||
<div class="server-identity">{#if index === 0 || preview.servers[index - 1].entryIndex !== server.entryIndex}<span class="map-group">{preview.entries[server.entryIndex].recipe.game.toUpperCase()} · {preview.entries[server.entryIndex].recipe.mapName}</span>{/if}<strong>{#if server.name}<ServerIdentity name={server.name} />{:else}{preview.entries[server.entryIndex].recipe.mapName}{/if}</strong><span class="server-id">{server.name ?? 'No available lobby'} · {server.port === null ? 'No available port' : `Port ${server.port}`}</span></div>
|
||||
<div class="server-identity">{#if index === 0 || preview.servers[index - 1].entryIndex !== server.entryIndex}<span class="map-group">{preview.entries[server.entryIndex].recipe.game.toUpperCase()} · {preview.entries[server.entryIndex].recipe.mapName}</span>{/if}<strong>{#if server.name}<ServerIdentity name={server.name} />{:else}{preview.entries[server.entryIndex].recipe.mapName}{/if}</strong><span class="server-id">{server.name ?? 'No available lobby'} · {server.port === null ? 'No available capacity' : server.native?.automatic ? 'Host and port assigned automatically' : `Port ${server.port}`}</span></div>
|
||||
{#if server.name}
|
||||
{#if server.hasSavedKey}<p class="key-ready"><span aria-hidden="true">✓</span> Saved key ready</p>
|
||||
{:else}<label><span>Server key required</span><input aria-label={`Key for ${server.name}`} data-missing={!keys[server.name]?.trim()} type="password" bind:value={keys[server.name]} disabled={busy} maxlength="512" autocomplete="off" placeholder="Paste this server’s key" /></label>{/if}
|
||||
@@ -145,7 +145,7 @@
|
||||
<div class="launch-footer"><div>{#if preview.servers.length}<p role="status"><strong>{keysReady} / {preview.servers.length} keys available</strong></p>{/if}<p class="caption">Key validity is checked at startup. Preview expires at {$timeFormat.minuteTime(preview.expiresAt)}.</p>{#if keysReady < preview.servers.length}<button class="text-button" type="button" onclick={nextMissing}>Jump to next missing key</button>{/if}</div><div class="actions"><Button variant="ghost" disabled={busy} onclick={() => open = false}>Cancel</Button><Button type="submit" variant={preview.removals.length ? 'danger' : 'primary'} loading={busy} disabled={busy || expired || !canLaunch}>{preview.mode === 'exact' ? `Apply: ${preview.servers.length} create, ${preview.removals.length} remove` : `Queue ${preview.servers.length} server${preview.servers.length === 1 ? '' : 's'} →`}</Button></div></div>
|
||||
</form>
|
||||
{:else}<p class="muted">{preview.mode === 'exact' ? 'Your fleet already matches this collection.' : 'Your fleet already meets or exceeds every target in this collection.'} No servers will be created or removed.</p><div class="actions"><Button onclick={() => open = false}>Done</Button></div>{/if}
|
||||
<details class="recipes"><summary>Setup and launch requirements</summary><p class="muted requirements">Each new server uses the selected host, latest game files, and prepared runtime. Host readiness, synced files, runtime availability, and local port reservations are checked again before deployment. Existing servers keep their current files.</p>{#each preview.entries as entry}<div class="recipe"><LaunchRecipe recipe={entry.recipe} /></div>{/each}</details>
|
||||
<details class="recipes"><summary>Setup and launch requirements</summary><p class="muted requirements">Each new server is automatically placed on an eligible host with the latest game files and prepared runtime. CPU, memory, host readiness, synced files, runtime availability, and local port reservations are checked again before deployment. Existing servers keep their current files.</p>{#each preview.entries as entry}<div class="recipe"><LaunchRecipe recipe={entry.recipe} /></div>{/each}</details>
|
||||
{/if}
|
||||
</div>
|
||||
</dialog>
|
||||
|
||||
@@ -23,7 +23,7 @@
|
||||
let maps = $derived(catalogs[game]);
|
||||
let selectedBlueprint = $derived(blueprints.find(b => b.id === blueprintId));
|
||||
let preview = $state<LaunchPreview | null>(null);
|
||||
let hostSelection = $state<NativeDeploymentSelection>({ hostId: '', libraryIds: {}, image: '' });
|
||||
let hostSelection = $state<NativeDeploymentSelection>({ libraryIds: {}, image: '' });
|
||||
let hostReady = $state(false);
|
||||
let plutoKey = $state('');
|
||||
let busy = $state(false);
|
||||
@@ -55,7 +55,7 @@
|
||||
blueprintId = initialBlueprintId;
|
||||
game = initialGame;
|
||||
mapSlug = initialMapSlug;
|
||||
hostSelection = { hostId: '', libraryIds: {}, image: '' }; hostReady = false;
|
||||
hostSelection = { libraryIds: {}, image: '' }; hostReady = false;
|
||||
try {
|
||||
const [mapRes, blueprintRes] = await Promise.all([fetch('/api/maps'), fetch('/api/blueprints')]);
|
||||
if (!mapRes.ok || !blueprintRes.ok) throw new Error('Could not load maps and collections. Try again.');
|
||||
@@ -76,7 +76,7 @@
|
||||
}
|
||||
|
||||
function selectionChanged() {
|
||||
if (preview) selectionNotice = 'The host or game files changed. Review the launch again with the latest files.';
|
||||
if (preview) selectionNotice = 'The launch settings or game files changed. Review the launch again with the latest files.';
|
||||
preview = null; plutoKey = ''; keyNotice = '';
|
||||
}
|
||||
|
||||
@@ -85,14 +85,14 @@
|
||||
if (!sourceDeploymentId && selectedBlueprint) {
|
||||
open = false; collectionOpen = true; return;
|
||||
}
|
||||
if (!hostReady) { error = 'Choose a host and wait for its latest game files and runtime to finish automatic setup.'; return; }
|
||||
if (!hostReady) { error = 'Wait for a host to finish preparing the latest game files and runtime.'; return; }
|
||||
const current = session;
|
||||
const previousName = preview?.name;
|
||||
busy = true; error = ''; savedName = ''; keyNotice = ''; selectionNotice = '';
|
||||
try {
|
||||
const res = await fetch('/api/provisioning/preview', { method: 'POST', headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ ...(sourceDeploymentId ? { deploymentId: sourceDeploymentId } : { game, mapSlug }),
|
||||
hostId: hostSelection.hostId, libraryId: hostSelection.libraryIds[game], image: hostSelection.image,
|
||||
libraryId: hostSelection.libraryIds[game], image: hostSelection.image,
|
||||
bindAddress: hostSelection.bindAddress || undefined, publicAddress: hostSelection.publicAddress || undefined, publicPort: hostSelection.publicPort }) });
|
||||
const body = await res.json();
|
||||
if (!res.ok) throw new Error(body.error ?? 'Could not prepare launch preview.');
|
||||
@@ -154,13 +154,13 @@
|
||||
<ul>{#each preview.checks as check}<li class:blocked={check.state === 'blocked'}><span class="check" aria-label={check.state}>{check.state === 'pass' ? '✓' : check.state === 'blocked' ? '!' : 'i'}</span><div><strong>{check.label}</strong><p>{check.detail}</p></div></li>{/each}</ul>
|
||||
</section>
|
||||
<form onsubmit={event => { event.preventDefault(); void launch(); }}>
|
||||
<div class="identity"><span class="eyebrow">Your new server</span><strong>{preview.name ?? 'Waiting for an available lobby'}</strong><p>{preview.port ? `Port ${preview.port} · ` : ''}Fresh identity, with its own server key.</p></div>
|
||||
<div class="identity"><span class="eyebrow">Your new server</span><strong>{preview.name ?? 'Waiting for an available lobby'}</strong><p>Fresh identity, with its own server key.</p></div>
|
||||
{#if preview.hasSavedKey}<p class="key-ready"><span aria-hidden="true">✓</span> Saved key ready</p>
|
||||
{:else}<label>Plutonium server key<input type="password" bind:value={plutoKey} disabled={busy || saving || !preview.name} maxlength="512" autocomplete="off" placeholder="Paste the key for this server ID" /></label>{/if}
|
||||
<p class="muted">{preview.hasSavedKey ? 'The saved key for this server ID will be reused automatically. Electric Cherry does not check whether Plutonium accepts it. Update it in Keys.' : 'Add the key here. It is saved for this exact server ID when you launch, including if provisioning fails.'} <a href={`/server-keys?server=${encodeURIComponent(preview.name ?? '')}`} target="_blank" rel="noopener">Manage saved keys in a new tab</a>; your launch stays open here.</p>
|
||||
<div class="launch-actions"><Button variant="ghost" disabled={busy || saving} onclick={() => { preview = null; plutoKey = ''; error = ''; }}>Back</Button><Button type="submit" variant="primary" loading={busy} disabled={busy || saving || expired || !hostReady || !preview.ready || (!preview.hasSavedKey && !plutoKey.trim())}>Launch server</Button></div>
|
||||
{#if !preview.hasSavedKey && !plutoKey.trim()}<p class="muted">Enter this server’s key to enable launch.</p>{/if}
|
||||
<p class="caption">Preview valid until {$timeFormat.minuteTime(preview.expiresAt)}. Lobby and port availability are checked again at launch.</p>
|
||||
<p class="caption">Preview valid until {$timeFormat.minuteTime(preview.expiresAt)}. Capacity is checked again at launch. The host and port are assigned automatically.</p>
|
||||
</form>
|
||||
<details class="save"><summary>Make this a reusable collection</summary><p class="muted">Save this game and map with a target of one lobby. Future launches use the latest promoted setup and create only missing lobbies.</p>
|
||||
{#if savedName}<p class="success" role="status">“{savedName}” is ready in your collection library.</p>{:else}
|
||||
@@ -168,7 +168,7 @@
|
||||
{/if}
|
||||
</details>
|
||||
{:else if sourceDeploymentId}
|
||||
<p class="muted">Choose a host for the latest promoted {game.toUpperCase()} setup, with a fresh lobby and port.</p><Button disabled={busy || !hostReady} onclick={review}>Review launch</Button>
|
||||
<p class="muted">Launch the latest promoted {game.toUpperCase()} setup, with a fresh lobby and port.</p><Button disabled={busy || !hostReady} onclick={review}>Review launch</Button>
|
||||
{:else}
|
||||
<form onsubmit={event => { event.preventDefault(); void review(); }}>
|
||||
<label>Start with<select bind:value={blueprintId} disabled={busy}><option value="">Current promoted setup</option>{#each blueprints as blueprint}<option value={blueprint.id}>{blueprint.name} · {blueprint.totalServers} server{blueprint.totalServers === 1 ? '' : 's'}</option>{/each}</select></label>
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
<script lang="ts">
|
||||
import { onMount } from 'svelte';
|
||||
import type { Host, NativeDeploymentSelection } from '$lib/hosts.js';
|
||||
import { hostReadinessLabel } from '$lib/hosts.js';
|
||||
import type { GameLibrary } from '$lib/game-files.js';
|
||||
import type { ZombieGame } from '$lib/map-aliases.js';
|
||||
import { pollVisible } from '$ui/poll.js';
|
||||
@@ -12,11 +11,11 @@
|
||||
let hosts = $state<Host[]>([]); let libraries = $state<GameLibrary[]>([]); let loading = $state(true); let error = $state('');
|
||||
let defaultRuntimeImage = $state('');
|
||||
let runtimeSelection: NativeDeploymentSelection | null = null;
|
||||
const host = $derived(hosts.find(item => item.id === selection.hostId));
|
||||
const eligible = $derived(hosts.filter(host => host.readiness === 'ready' && host.inventory.runtimeImages?.includes(selection.image)));
|
||||
const latest = $derived(latestGameFiles(libraries));
|
||||
const imageReady = $derived(!!host?.inventory.runtimeImages?.includes(selection.image));
|
||||
function cacheReady(game: ZombieGame) { return !!latest[game]?.caches.some(cache => cache.hostId === selection.hostId && ['ready', 'outdated'].includes(cache.status)); }
|
||||
$effect(() => { ready = !loading && !error && host?.readiness === 'ready' && imageReady && games.length > 0 && games.every(game => selection.libraryIds[game] === latest[game]?.id && cacheReady(game)); });
|
||||
|
||||
function cacheReady(game: ZombieGame) { return !!latest[game]?.caches.some(cache => eligible.some(host => host.id === cache.hostId) && ['ready', 'outdated'].includes(cache.status)); }
|
||||
$effect(() => { ready = !loading && !error && games.length > 0 && games.every(game => selection.libraryIds[game] === latest[game]?.id && cacheReady(game)); });
|
||||
$effect(() => {
|
||||
if (disabled || !defaultRuntimeImage || runtimeSelection === selection) return;
|
||||
if (!selection.image) selection.image = defaultRuntimeImage;
|
||||
@@ -43,16 +42,14 @@
|
||||
<div class="title"><h3>Host and game files</h3><button type="button" disabled={disabled || loading} onclick={() => { onchange?.(); void load(); }}>Refresh hosts</button></div>
|
||||
{#if error}<p role="alert" class="error">{error}</p>{/if}
|
||||
{#if loading}<p role="status">Loading host readiness and game files…</p>{/if}
|
||||
<label>Game host<select bind:value={selection.hostId} disabled={disabled || loading} onchange={() => onchange?.()}><option value="">Choose a host</option>{#each hosts as candidate}<option value={candidate.id} disabled={candidate.mode !== 'active' || !!candidate.decommission || !candidate.credentialActive}>{candidate.name} · {candidate.connectionState} · {candidate.preparation?.state === 'preparing' ? 'Preparing automatically' : hostReadinessLabel(candidate.readiness)}</option>{/each}</select></label>
|
||||
<p>Hosts are assigned automatically. We check available capacity before accepting a launch and choose the least loaded eligible host when provisioning starts.</p>
|
||||
{#if !loading && !hosts.length}<p><a href="/hosts" target="_blank" rel="noopener">Add a host</a>. Its runtime and latest T5 and T6 files will be prepared automatically.</p>{/if}
|
||||
{#if host}<p>{host.hostname} · ports {host.portMin}–{host.portMax} · {host.publicAddress || host.bindAddress}</p>{/if}
|
||||
{#if host?.preparation && host.preparation.state !== 'ready'}<p class:error={host.preparation.state === 'blocked'}>{host.preparation.items.find(item => item.state === 'blocked' || item.state === 'preparing' || item.state === 'waiting')?.detail} <a href={`/hosts/${host.id}`} target="_blank" rel="noopener">View automatic setup</a></p>{/if}
|
||||
{#if !loading}{#each games as game}
|
||||
<div class="game-files">
|
||||
<strong>{game === 't5' ? 'Black Ops I (T5)' : 'Black Ops II (T6)'} files</strong>
|
||||
{#if latest[game]}
|
||||
<p>Latest upload · {$timeFormat.dateTime(latest[game]?.publishedAt ?? latest[game]?.createdAt)}</p>
|
||||
<p role="status">{!host ? 'Choose a host to check its files.' : cacheReady(game) ? 'Latest files ready on this host.' : 'Waiting for this host to sync the latest files automatically.'}</p>
|
||||
<p role="status">{cacheReady(game) ? 'Latest files ready on an eligible host.' : 'Waiting for a host to prepare the runtime and latest files automatically.'}</p>
|
||||
{:else}
|
||||
<p>No files uploaded yet. <a href="/game-files" target="_blank" rel="noopener">Upload {game.toUpperCase()} files</a> to prepare your hosts.</p>
|
||||
{/if}
|
||||
@@ -60,11 +57,10 @@
|
||||
{/each}{/if}
|
||||
<p>Hosts keep the latest T5 and T6 files ready automatically. <a href="/game-files" target="_blank" rel="noopener">Upload game files</a> when they change. Existing servers keep their current files.</p>
|
||||
<details><summary>Advanced: runtime override</summary><p>The configured default runtime is prepared automatically. Custom images must be prepared from the host page.</p><label>Plutainer runtime image<input required bind:value={selection.image} disabled={disabled} onchange={() => onchange?.()} /></label></details>
|
||||
{#if host && !imageReady}<p>Waiting for runtime preparation. <a href={`/hosts/${host.id}`} target="_blank" rel="noopener">View host setup</a>.</p>{/if}
|
||||
{#if network}<details><summary>Network overrides</summary><label>Local bind address<input bind:value={selection.bindAddress} placeholder={host?.bindAddress ?? 'Host default'} disabled={disabled} onchange={() => onchange?.()} /></label><label>Public / NAT address<input bind:value={selection.publicAddress} placeholder={host?.publicAddress || 'Host default'} disabled={disabled} onchange={() => onchange?.()} /></label><label>Public / NAT port<input type="number" min="1" max="65535" bind:value={selection.publicPort} placeholder="Same as local port" disabled={disabled} onchange={() => onchange?.()} /></label><p>Public mapping does not change the local Docker port reservation.</p></details>{/if}
|
||||
{#if network}<details><summary>Network overrides</summary><label>Local bind address<input bind:value={selection.bindAddress} placeholder="Host default" disabled={disabled} onchange={() => onchange?.()} /></label><label>Public / NAT address<input bind:value={selection.publicAddress} placeholder="Host default" disabled={disabled} onchange={() => onchange?.()} /></label><label>Public / NAT port<input type="number" min="1" max="65535" bind:value={selection.publicPort} placeholder="Same as local port" disabled={disabled} onchange={() => onchange?.()} /></label><p>Public mapping does not change the local Docker port reservation.</p></details>{/if}
|
||||
</section>
|
||||
|
||||
<style>
|
||||
.game-files { display:grid; gap:5px; } .game-files strong { font-size:13px; font-weight:600; }
|
||||
.selection { display:grid; gap:14px; padding:20px; border:1px solid var(--border-subtle); border-radius:var(--radius-md); background:var(--surface-sunken); } .title { display:flex; gap:12px; justify-content:space-between; align-items:center; } h3 { font-size:var(--text-sm); margin:0; } label { display:grid; gap:7px; font-size:13px; } input,select { width:100%; box-sizing:border-box; padding:11px; color:var(--text-primary); background:var(--surface); border:1px solid var(--border-control); border-radius:var(--radius-sm); font:inherit; } p { color:var(--text-secondary); font-size:12px; line-height:1.6; margin:0; } a,button { color:var(--accent-primary-hover); } button { background:none; border:0; cursor:pointer; font:inherit; font-size:12px; } button:disabled { opacity:.5; cursor:default; } .error { color:var(--state-danger); } summary { cursor:pointer; font-size:13px; } details label { margin-top:14px; } details p { margin-top:12px; }
|
||||
.selection { display:grid; gap:14px; padding:20px; border:1px solid var(--border-subtle); border-radius:var(--radius-md); background:var(--surface-sunken); } .title { display:flex; gap:12px; justify-content:space-between; align-items:center; } h3 { font-size:var(--text-sm); margin:0; } label { display:grid; gap:7px; font-size:13px; } input { width:100%; box-sizing:border-box; padding:11px; color:var(--text-primary); background:var(--surface); border:1px solid var(--border-control); border-radius:var(--radius-sm); font:inherit; } p { color:var(--text-secondary); font-size:12px; line-height:1.6; margin:0; } a,button { color:var(--accent-primary-hover); } button { background:none; border:0; cursor:pointer; font:inherit; font-size:12px; } button:disabled { opacity:.5; cursor:default; } .error { color:var(--state-danger); } summary { cursor:pointer; font-size:13px; } details label { margin-top:14px; } details p { margin-top:12px; }
|
||||
</style>
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { capacityCalibration } from '$lib/server/hosts/calibration.js';
|
||||
import { error, json } from '@sveltejs/kit';
|
||||
import { deleteHost, getHost, hostDeletionStatus, HostDeletionError, listHostOperations, updateHost } from '$lib/server/hosts/index.js';
|
||||
import { hostApiError, objectBody, requireManagementOrigin } from '$lib/server/hosts/http.js';
|
||||
@@ -8,7 +9,7 @@ export function GET({ request, params }: Event) {
|
||||
requireManagementOrigin(request); const host = getHost(params.id); if (!host) error(404, 'Host not found.');
|
||||
// Payloads and results can contain RCON keys or staged credentials. History
|
||||
// exposes metadata only; agent APIs remain the only command delivery path.
|
||||
return json({ host: presentHostPreparation(host), defaultRuntimeImage: config.komodo.defaultImage, deletion: hostDeletionStatus(host.id), operations: listHostOperations(host.id, 100, true).map(({ payload: _payload, result: _result, ...operation }) => operation) });
|
||||
return json({ host: presentHostPreparation(host), calibration: capacityCalibration(host.id), defaultRuntimeImage: config.komodo.defaultImage, deletion: hostDeletionStatus(host.id), operations: listHostOperations(host.id, 100, true).map(({ payload: _payload, result: _result, ...operation }) => operation) });
|
||||
}
|
||||
export async function PATCH({ request, params }: Event) {
|
||||
requireManagementOrigin(request);
|
||||
|
||||
@@ -33,7 +33,7 @@ export async function POST({ request }: { request: Request }) {
|
||||
});
|
||||
if (!result.ok) {
|
||||
const status = result.kind === 'bad-request' ? 400 : result.kind === 'unconfigured' ? 503
|
||||
: result.kind === 'unknown-map' || result.kind === 'no-slot' || result.kind === 'no-port' ? 409
|
||||
: result.kind === 'conflict' || result.kind === 'unknown-map' || result.kind === 'no-slot' || result.kind === 'no-port' ? 409
|
||||
: 500;
|
||||
return json({ ok: false, error: result.message, kind: result.kind }, { status });
|
||||
}
|
||||
|
||||
@@ -4,13 +4,14 @@
|
||||
let requestSequence = 0;
|
||||
import { page } from '$app/stores';
|
||||
import { goto } from '$app/navigation';
|
||||
import type { Host, HostOperation, HostMode } from '$lib/hosts.js';
|
||||
import type { Host, HostOperation, HostMode, CapacityCalibration } from '$lib/hosts.js';
|
||||
import { hostAgentUpdateLabel, hostAgentUpdateMessage, hostAgentUpdatesEnabled, hostDeletionMessage, hostReadinessLabel } from '$lib/hosts.js';
|
||||
import SectionHeader from '$ui/components/primitives/SectionHeader.svelte';
|
||||
import Button from '$ui/components/primitives/Button.svelte';
|
||||
import DeleteHostDialog from '$ui/components/DeleteHostDialog.svelte';
|
||||
import HostPreparation from '$ui/components/HostPreparation.svelte';
|
||||
let host = $state<Host | null>(null); let operations = $state<HostOperation[]>([]);
|
||||
let calibration = $state<CapacityCalibration | null>(null);
|
||||
let loading = $state(true); let busy = $state(false); let error = $state(''); let notice = $state('');
|
||||
let deleteOpen = $state(false);
|
||||
let name = $state(''); let portMin = $state(28960); let portMax = $state(28980); let bindAddress = $state('0.0.0.0'); let publicAddress = $state(''); let minimumFreeGB = $state(1);
|
||||
@@ -22,7 +23,7 @@
|
||||
async function load(initialize = false, signal?: AbortSignal) {
|
||||
const target = endpoint; const sequence = ++requestSequence;
|
||||
try {
|
||||
const response = await fetch(target, { signal: signal ?? AbortSignal.timeout(15000) }); const body = await response.json(); if (signal?.aborted || target !== endpoint || sequence !== requestSequence) return; if (response.status === 404 && host?.decommission) { await goto('/hosts'); return; } if (!response.ok) throw new Error(body.error ?? body.message ?? 'Could not load host.'); host = body.host; operations = body.operations; error = '';
|
||||
const response = await fetch(target, { signal: signal ?? AbortSignal.timeout(15000) }); const body = await response.json(); if (signal?.aborted || target !== endpoint || sequence !== requestSequence) return; if (response.status === 404 && host?.decommission) { await goto('/hosts'); return; } if (!response.ok) throw new Error(body.error ?? body.message ?? 'Could not load host.'); host = body.host; operations = body.operations; calibration = body.calibration ?? null; error = '';
|
||||
if (initialize && host) { name = host.name; portMin = host.portMin; portMax = host.portMax; bindAddress = host.bindAddress; publicAddress = host.publicAddress; minimumFreeGB = host.minimumFreeBytes / 1073741824; runtimeImage = body.defaultRuntimeImage ?? runtimeImage; }
|
||||
} catch (cause) { if (!signal?.aborted && sequence === requestSequence) error = (cause as Error).message; return false; } finally { if (sequence === requestSequence) loading = false; }
|
||||
}
|
||||
@@ -55,6 +56,14 @@
|
||||
<p>Draining prevents new deployments and retires instances as they become safe to remove. Disabled hosts receive no new commands; running game containers continue.</p>
|
||||
</section>
|
||||
{#if !cleaning}<section class="panel"><h2>Automatic host setup</h2><HostPreparation {host} /></section>{/if}
|
||||
{#if calibration}<section class="panel"><h2>Learned server capacity</h2>
|
||||
<p>Allowances update from observed container usage with 25% headroom. Startup and running usage are tracked separately.</p>
|
||||
<dl>{#each ['startup', 'running'] as phase}{@const allowance = calibration[phase as keyof CapacityCalibration]}
|
||||
<dt>{phase === 'startup' ? 'Starting a server' : 'Running a server'}</dt>
|
||||
<dd>{allowance.cpuCores.toFixed(2)} CPU cores · {bytes(allowance.memoryBytes)} · {allowance.source === 'host' ? 'Learned on this host' : allowance.source === 'fleet' ? 'Learned from the fleet' : 'Calibrating; temporary allowance'} ({allowance.samples} samples)</dd>
|
||||
{/each}</dl>
|
||||
{#if calibration.startup.source === 'bootstrap' || calibration.running.source === 'bootstrap'}<p>Learning begins as agent 1.4.3 or later reports running containers. Temporary allowances stay in use until enough measurements are available.</p>{/if}
|
||||
</section>{/if}
|
||||
<div class="grid"><section class="panel"><h2>Inventory</h2><dl>
|
||||
<dt>Hostname</dt><dd>{host.hostname || 'Pending'}</dd><dt>Agent</dt><dd>{host.inventory.agentVersion ?? 'Unknown'}</dd><dt>Platform</dt><dd>{host.inventory.os ?? 'Unknown'} · {host.inventory.architecture ?? 'Unknown'}</dd><dt>CPU</dt><dd>{host.inventory.cpu ?? 'Unknown'} ({host.inventory.cpuCount ?? '?'} cores)</dd><dt>CPU usage</dt><dd title="Average across all cores since the previous inventory sample">{host.inventory.cpuUsagePercent === undefined ? 'Not reported' : `${host.inventory.cpuUsagePercent.toFixed(1)}%`}{#if !host.connected && host.inventory.cpuUsagePercent !== undefined} (last reported){/if}</dd><dt>Memory available / total</dt><dd>{bytes(host.inventory.memoryAvailable)} / {bytes(host.inventory.memoryTotal)}</dd><dt>Game storage available / total</dt><dd>{bytes(host.inventory.storageAvailable)} / {bytes(host.inventory.storageTotal)}</dd><dt>Docker</dt><dd>{host.inventory.dockerVersion ?? 'Unknown'} · {host.inventory.dockerRunning ? 'Running' : 'Unavailable'}</dd><dt>Current operation</dt><dd>{host.inventory.currentOperation ?? 'None reported'}</dd>
|
||||
</dl></section><section class="panel"><h2>Host configuration</h2>{#if canManage}<form onsubmit={save}><fieldset disabled={busy || cleaning}>
|
||||
|
||||
Reference in New Issue
Block a user