josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Automatically place servers using learned resource capacity

Browse Source
This commit is contained in:
josh
2026-10-04 21:04:09 -04:00
parent a1b957392f
commit e6cdf9eb7f
25 changed files with 927 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/provisioning/native.ts
+59 -29
View File
@@ -1,9 +1,11 @@
import { isIP } from 'node:net';
import { CapacityError, choosePlacement, availableHostPort, type PlacementOptions } from '../hosts/capacity.js';
import { randomBytes, randomUUID } from 'node:crypto';
import { posix } from 'node:path';
import { db } from '../db/index.js';
import { config, relDestForKind, serverDir } from '../config.js';
import { nativeInstance, nativeRequest, type NativeInstance } from '../backends/native.js';
import { getHost, getHostOperation, listHosts, nextHostPort, queueHostOperation, releaseHostPorts, requireReadyHost, reserveHostPort } from '../hosts/index.js';
import { getHost, getHostOperation, listHosts, queueHostOperation, releaseHostPorts, reserveHostPort } from '../hosts/index.js';
import { newestReadyLibrary, requireReadyLibrary } from '../game-files/index.js';
import { blueprintRecipe, captureRecipe, recipeForDeployment, summarizeRecipe, type LaunchRecipe } from '../fleet/launch-recipes.js';
import { getServerKey, saveServerKey } from '../fleet/server-keys.js';
@@ -24,14 +26,14 @@ import type { ProvisioningAction } from '../../provisioning.js';
import { hostAgentUpdatesEnabled } from '../../hosts.js';
import { cancelCreation, finishCreationCancellation } from './cancel-creation.js';
export interface NativeSelection { hostId: string; libraryId: string; image?: string; bindAddress?: string; publicAddress?: string; publicPort?: number }
export type NativeSelection = import('../../hosts.js').NativePlacement;
type NativeRecipe = LaunchRecipe & { native?: NativeSelection };
interface Job { id: string; server: string; action: ProvisioningAction; state: string; submitted_at: string | null; stage: string;
instance_id: string; generation: number; stage_operation_id: string | null; operation_id: string | null; files_json: string | null; applied_at: string | null }
interface FilePlan { id: number; path: string; contentBase64: string | null; sha256: string | null; checkBefore: boolean; expectedBeforeSha: string | null }
export function nativeSelection(input: Record<string, unknown>, game: string): NativeSelection {
if (typeof input.hostId !== 'string' || !input.hostId) throw new Error('Select a ready host.');
export function nativeSelection(input: Record<string, unknown>, game: string, options: Pick<PlacementOptions, 'planned'> = {}): NativeSelection {
if (input.hostId !== undefined && typeof input.hostId !== 'string') throw new Error('Invalid host.');
const version = newestReadyLibrary(game);
if (!version) throw new Error(`Upload ${game.toUpperCase()} game files before launching a server.`);
if (input.libraryId !== undefined && input.libraryId !== '' && input.libraryId !== version.id) {
@@ -39,14 +41,17 @@ export function nativeSelection(input: Record<string, unknown>, game: string): N
}
const image = input.image === undefined || input.image === '' ? config.komodo.defaultImage : input.image;
if (typeof image !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9._/:@-]{0,499}$/.test(image)) throw new Error('Invalid runtime image.');
const host = requireReadyHost(input.hostId, { image, requiredBytes: 1024 ** 3 });
requireReadyLibrary(host.id, version.id);
const bindAddress = input.bindAddress === undefined || input.bindAddress === '' ? host.bindAddress : input.bindAddress;
if (typeof bindAddress !== 'string') throw new Error('Invalid bind address.');
if (input.publicAddress !== undefined && (typeof input.publicAddress !== 'string' || input.publicAddress.length > 253)) throw new Error('Invalid public address.');
if (input.publicPort !== undefined && (!Number.isInteger(input.publicPort) || Number(input.publicPort) < 1 || Number(input.publicPort) > 65535)) throw new Error('Invalid public port.');
return { hostId: host.id, libraryId: version.id, image, bindAddress, publicAddress: input.publicAddress as string | undefined ?? host.publicAddress,
publicPort: input.publicPort as number | undefined };
const automatic = input.automatic === true || !input.hostId;
const rawNetwork = input.automatic === true ? input.network ?? {} : input;
if (!rawNetwork || typeof rawNetwork !== 'object' || Array.isArray(rawNetwork)) throw new Error('Invalid network overrides.');
const network = rawNetwork as NativeSelection;
const bindAddress = network.bindAddress || undefined;
if (bindAddress !== undefined && (typeof bindAddress !== 'string' || !isIP(bindAddress))) throw new Error('Invalid bind address.');
const { host } = choosePlacement({ hostId: automatic ? undefined : input.hostId as string, libraryId: version.id, image, bindAddress, ...options });
if (network.publicAddress !== undefined && (typeof network.publicAddress !== 'string' || network.publicAddress.length > 253)) throw new Error('Invalid public address.');
if (network.publicPort !== undefined && (!Number.isInteger(network.publicPort) || Number(network.publicPort) < 1 || Number(network.publicPort) > 65535)) throw new Error('Invalid public port.');
return { hostId: host.id, libraryId: version.id, image, bindAddress: bindAddress ?? host.bindAddress, publicAddress: network.publicAddress ?? host.publicAddress,
publicPort: network.publicPort, automatic, network: { bindAddress, publicAddress: network.publicAddress, publicPort: network.publicPort } };
}
export function previewNativeCreation(input: Record<string, unknown>): LaunchPreview {
@@ -72,10 +77,10 @@ export function previewNativeCreation(input: Record<string, unknown>): LaunchPre
db().prepare('INSERT INTO server_launch_previews(id,expires_at,server,port,recipe,blueprint_name) VALUES(?,?,?,?,?,?)')
.run(id, expiresAt, name, port, JSON.stringify(recipe), blueprintName);
const checks: LaunchPreview['checks'] = [
{ label: 'Host readiness', state: 'pass', detail: `${getHost(selection.hostId)!.name}: connected, Docker ready, storage and runtime verified.` },
{ label: 'Game files', state: 'pass', detail: `The latest ${recipe.game.toUpperCase()} game files are verified on the selected host.` },
{ label: 'Host capacity', state: 'pass', detail: selection.automatic ? 'Capacity available. The least loaded eligible host is selected when provisioning starts.' : `${getHost(selection.hostId)!.name}: capacity available.` },
{ label: 'Game files', state: 'pass', detail: `The latest ${recipe.game.toUpperCase()} game files are verified on an eligible host.` },
{ label: 'Lobby', state: name ? 'pass' : 'blocked', detail: name ?? 'All lobby IDs for this map are reserved.' },
{ label: 'Host port', state: port ? 'pass' : 'blocked', detail: port ? `${selection.bindAddress}:${port} / UDP and TCP` : 'No available UDP/TCP port on this host.' },
{ label: 'Host port', state: port ? 'pass' : 'blocked', detail: port ? selection.automatic ? 'UDP/TCP capacity available; port assigned at placement.' : `${selection.bindAddress}:${port} / UDP and TCP` : 'No available UDP/TCP port on this host.' },
{ label: 'Configuration', state: 'pass', detail: recipe.configSource }
];
return { id, expiresAt, name, port, hasSavedKey: !!name && !!getServerKey(name), recipe: summarizeRecipe(recipe), blueprintName, checks,
@@ -83,16 +88,9 @@ export function previewNativeCreation(input: Record<string, unknown>): LaunchPre
}
export function nextNativePort(selection: NativeSelection, taken: number[] = []): number | null {
const excluded = [...taken];
for (;;) {
const port = nextHostPort(selection.hostId, excluded, selection.bindAddress, 'udp');
if (port === null) return null;
// Both protocols are published by Plutainer and reserved together.
const tcp = nextHostPort(selection.hostId, excluded, selection.bindAddress, 'tcp');
if (tcp === port) return port;
if (tcp !== null && tcp < port) { excluded.push(tcp); continue; }
excluded.push(port);
}
const host = getHost(selection.hostId);
return host ? availableHostPort(host, { bindAddress: selection.bindAddress,
planned: taken.map(port => ({ hostId: host.id, port })) }) : null;
}
export function queueNativeCreation(input: CreateServerInput): CreateServerResult {
@@ -113,7 +111,7 @@ export function queueNativeCreation(input: CreateServerInput): CreateServerResul
const name = `${game}_zombie_${input.mapSlug}_${lobby}`;
const port = nextNativePort(selection);
if (!port) throw new Error('No available host port.');
if ((input.expectedName && input.expectedName !== name) || (preview && (preview.server !== name || preview.port !== port))) throw new Error('Available lobby or host port changed. Review a fresh preview.');
if ((input.expectedName && input.expectedName !== name) || (preview && (preview.server !== name || (!selection.automatic && preview.port !== port)))) throw new Error('Available lobby or host port changed. Review a fresh preview.');
const key = input.plutoKey?.trim() || getServerKey(name);
if (!key) throw new Error(`Save a Plutonium server key for ${name}.`);
saveServerKey(name, key);
@@ -128,11 +126,11 @@ export function queueNativeCreation(input: CreateServerInput): CreateServerResul
db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,?,?,?,?)')
.run(id, name, port, 'zombie', recipe.mapSlug, recipe.expectedMap);
db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,blueprint_name) VALUES(?,?,?)').run(id, JSON.stringify(recipe), preview?.blueprint_name ?? null);
if (preview) db().prepare('UPDATE server_launch_previews SET deployment_id=? WHERE id=?').run(id, preview.id);
if (preview) db().prepare('UPDATE server_launch_previews SET deployment_id=?,port=? WHERE id=?').run(id, port, preview.id);
deploymentStage(id, 'Waiting for provisioning evaluation', { phase: 'eligibility', recipeStatus: 'queued' });
return { ok: true as const, name, port, deploymentId: id };
})();
} catch (error) { return { ok: false, kind: 'bad-request', message: (error as Error).message }; }
} catch (error) { return { ok: false, kind: error instanceof CapacityError ? 'conflict' : 'bad-request', message: (error as Error).message }; }
}
export function bindNativeOperation(id: string, server: string): void {
@@ -270,12 +268,43 @@ export async function processNativeProvisioning(eligibility: (operation: { id: s
const activeJobs = new Map<string, Promise<void>>();
/** Reservations can move only while no remote work has ever been submitted. */
function placeQueuedCreation(job: Job, instance: NativeInstance): NativeInstance {
return db().transaction(() => {
const recipe = recipeForDeployment(job.id)!.recipe as NativeRecipe;
const prior = recipe.native!;
const network = prior.automatic ? prior.network ?? {} : prior;
const { host, port } = choosePlacement({ hostId: prior.automatic ? undefined : instance.host_id,
libraryId: instance.library_id, image: instance.image, bindAddress: network.bindAddress,
thorough: true, excludeInstanceId: instance.id });
const bind = network.bindAddress ?? host.bindAddress;
releaseHostPorts(instance.id);
reserveHostPort(host.id, instance.id, { port, bindAddress: bind, protocol: 'udp' });
reserveHostPort(host.id, instance.id, { port, bindAddress: bind, protocol: 'tcp' });
db().prepare(`UPDATE native_instances SET host_id=?,bind_address=?,port=?,public_address=?,public_port=? WHERE id=?`)
.run(host.id, bind, port, network.publicAddress ?? host.publicAddress, network.publicPort ?? null, instance.id);
db().prepare('UPDATE server_creation_jobs SET port=? WHERE deployment_id=?').run(port, job.id);
recipe.native = { ...prior, hostId: host.id, bindAddress: bind, publicAddress: network.publicAddress ?? host.publicAddress, publicPort: network.publicPort };
db().prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?').run(JSON.stringify(recipe), job.id);
db().prepare('UPDATE server_launch_previews SET port=? WHERE deployment_id=?').run(port, job.id);
deploymentStage(job.id, `Placed on ${host.name} after CPU, memory, history and port checks`, { phase: 'eligibility' });
return nativeInstance(job.server)!;
})();
}
async function processJob(job: Job, eligibility: (job: Job) => string | null) {
if (!stillActive(job)) return;
let instance = nativeInstance(job.server);
if (!instance || instance.id !== job.instance_id || (instance.generation !== job.generation && !(job.action === 'rebuild' && instance.generation === job.generation + 1))) {
finishDeployment(job.id, { ok: false, message: 'Original instance was retired or replaced; no command was sent.' }); return;
}
if (job.action === 'create' && !job.operation_id && !job.stage_operation_id && !job.submitted_at) {
const settings = fleetSettings();
const active = db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL
AND action IN ('create','rebuild') AND state IN ('running','awaiting-health')`).get(job.id) as { n: number };
if (active.n >= settings.createConcurrency) { hold(job, 'Waiting for startup capacity.'); return; }
instance = placeQueuedCreation(job, instance);
}
const host = getHost(instance.host_id);
if (!host?.connected || host.mode === 'disabled') {
db().prepare("UPDATE servers SET container_state='unknown',last_players=NULL,last_roster=NULL WHERE name=?").run(job.server);
@@ -403,7 +432,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
}
if (job.action === 'destroy' && count(['destroy']) >= settings.destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; }
deploymentStage(job.id, 'Checking host eligibility', { phase: 'eligibility', blocker: null });
if (job.action === 'create') requireReadyHost(instance.host_id, { image: instance.image, requiredBytes: 1024 ** 3 });
if (job.action === 'create') choosePlacement({ hostId: instance.host_id, libraryId: instance.library_id, image: instance.image,
bindAddress: instance.bind_address, excludeInstanceId: instance.id, thorough: true });
if (job.action === 'create' || job.action === 'rebuild') requireReadyLibrary(instance.host_id, instance.library_id);
if (job.action !== 'create' && job.action !== 'start') {
// Fast controller check keeps occupied lobbies out of the operation slots.