Queue native collection launches before async placement
This commit is contained in:
@@ -15,7 +15,19 @@ export function nativeInstance(server: string): NativeInstance | null {
|
||||
}
|
||||
|
||||
export function nativeDeployment(id: string): boolean {
|
||||
return !!db().prepare('SELECT 1 FROM native_provisioning WHERE deployment_id = ?').get(id);
|
||||
return !!db().prepare(`SELECT 1 FROM native_provisioning WHERE deployment_id = ?
|
||||
UNION ALL SELECT 1 FROM native_creation_intents WHERE deployment_id = ?
|
||||
UNION ALL SELECT 1 FROM native_creation_cancellations c
|
||||
JOIN native_creation_intents i ON i.deployment_id=c.creation_id WHERE c.cleanup_id = ? LIMIT 1`).get(id, id, id);
|
||||
}
|
||||
|
||||
export function unplacedNativeCreation(server: string): string | null {
|
||||
const row = db().prepare(`SELECT d.id FROM server_deployments d
|
||||
JOIN native_creation_intents i ON i.deployment_id=d.id
|
||||
LEFT JOIN native_provisioning n ON n.deployment_id=d.id
|
||||
WHERE d.server=? AND d.server_deleted_at IS NULL AND d.action='create' AND d.state='queued'
|
||||
AND n.deployment_id IS NULL ORDER BY d.rowid DESC LIMIT 1`).get(server) as { id: string } | undefined;
|
||||
return row?.id ?? null;
|
||||
}
|
||||
|
||||
const observations = new Map<string, Promise<unknown>>();
|
||||
|
||||
@@ -1,7 +1,12 @@
|
||||
import type Database from 'better-sqlite3';
|
||||
|
||||
export function migrateNativeInstances(handle: Database.Database): void {
|
||||
migrateUnplacedCreationPort(handle);
|
||||
handle.exec(`
|
||||
CREATE TABLE IF NOT EXISTS native_creation_intents (
|
||||
deployment_id TEXT PRIMARY KEY REFERENCES server_deployments(id) ON DELETE CASCADE,
|
||||
selection TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS native_instances (
|
||||
id TEXT PRIMARY KEY, server TEXT NOT NULL, host_id TEXT NOT NULL,
|
||||
generation INTEGER NOT NULL DEFAULT 1, library_id TEXT NOT NULL,
|
||||
@@ -24,3 +29,27 @@ export function migrateNativeInstances(handle: Database.Database): void {
|
||||
);
|
||||
`);
|
||||
}
|
||||
|
||||
/** Accepted native creations receive their host port only when provisioning places them. */
|
||||
function migrateUnplacedCreationPort(handle: Database.Database): void {
|
||||
const columns = handle.prepare('PRAGMA table_info(server_creation_jobs)').all() as { name: string; notnull: number }[];
|
||||
if (!columns.find(column => column.name === 'port')?.notnull) return;
|
||||
const { sql } = handle.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='server_creation_jobs'").get() as { sql: string };
|
||||
const replacement = sql.replace(/\bport\s+INTEGER\s+NOT\s+NULL\b/i, 'port INTEGER');
|
||||
if (replacement === sql) throw new Error('Could not migrate unplaced creation ports.');
|
||||
const dependent = handle.prepare("SELECT sql FROM sqlite_master WHERE tbl_name='server_creation_jobs' AND type IN ('index','trigger') AND sql IS NOT NULL")
|
||||
.all() as { sql: string }[];
|
||||
const foreignKeys = handle.pragma('foreign_keys', { simple: true });
|
||||
handle.pragma('foreign_keys = OFF');
|
||||
try {
|
||||
handle.transaction(() => {
|
||||
handle.exec(replacement.replace('server_creation_jobs', 'server_creation_jobs_unplaced'));
|
||||
const names = columns.map(column => `"${column.name.replaceAll('"', '""')}"`).join(', ');
|
||||
handle.exec(`INSERT INTO server_creation_jobs_unplaced(rowid,${names}) SELECT rowid,${names} FROM server_creation_jobs;
|
||||
DROP TABLE server_creation_jobs;
|
||||
ALTER TABLE server_creation_jobs_unplaced RENAME TO server_creation_jobs;`);
|
||||
for (const object of dependent) handle.exec(object.sql);
|
||||
if ((handle.pragma('foreign_key_check') as unknown[]).length) throw new Error('Unplaced creation migration found broken references.');
|
||||
})();
|
||||
} finally { handle.pragma(`foreign_keys = ${foreignKeys ? 'ON' : 'OFF'}`); }
|
||||
}
|
||||
|
||||
@@ -475,7 +475,7 @@ CREATE INDEX IF NOT EXISTS idx_server_deployment_events ON server_deployment_eve
|
||||
CREATE TABLE IF NOT EXISTS server_creation_jobs (
|
||||
deployment_id TEXT PRIMARY KEY REFERENCES server_deployments(id) ON DELETE CASCADE,
|
||||
server TEXT NOT NULL,
|
||||
port INTEGER NOT NULL,
|
||||
port INTEGER,
|
||||
game_mode TEXT NOT NULL,
|
||||
map_slug TEXT NOT NULL,
|
||||
expected_map TEXT NOT NULL
|
||||
|
||||
@@ -4,18 +4,18 @@ import type { CollectionDetail, CollectionMode, CollectionPreview, CollectionSum
|
||||
import { db } from '../db/index.js';
|
||||
import { config, komodoConfigured, rconConfigured } from '../config.js';
|
||||
import { blueprintEntries, summarizeRecipe, type BlueprintEntry } from './launch-recipes.js';
|
||||
import { creationQueue, destroyServer, nextAvailablePort, nextLobbyNumber, queueCreateServer } from '../provisioning/index.js';
|
||||
import { creationQueue, destroyServer, nextAvailablePort, queueCreateServer } from '../provisioning/index.js';
|
||||
import { deploymentStage, finishDeployment, recordDeploymentEvent } from './deployments.js';
|
||||
import { housePartyTargets } from './house-party.js';
|
||||
import { getServerKey } from './server-keys.js';
|
||||
import { serverObservation } from './status.js';
|
||||
import { activeBetaRemovalBlocker, getServer } from './servers.js';
|
||||
import { parseServerName, zombieMapBySlug, zombieMaps } from './map-aliases.js';
|
||||
import { nativeSelection, nextNativePort } from '../provisioning/native.js';
|
||||
import { nativeLaunchSelection, queueNativeCreationIntent } from '../provisioning/native.js';
|
||||
import { listHosts } from '../hosts/index.js';
|
||||
import { nativeInstance } from '../backends/native.js';
|
||||
import { nativeDeployment, nativeInstance } from '../backends/native.js';
|
||||
import { launchAvailability } from '../hosts/availability.js';
|
||||
|
||||
function coverage(entries: BlueprintEntry[]) {
|
||||
function coverage(entries: BlueprintEntry[], includeHealth = true) {
|
||||
type Server = { name: string; game: string | null; map_slug: string | null; map: string | null };
|
||||
const registered = db().prepare('SELECT name, game, map_slug, map FROM servers WHERE destroyed_at IS NULL').all() as Server[];
|
||||
const queued = db().prepare(`SELECT j.server AS name, substr(j.server, 1, 2) AS game, j.map_slug, j.expected_map AS map
|
||||
@@ -37,14 +37,14 @@ function coverage(entries: BlueprintEntry[]) {
|
||||
const matches = zombieMaps(game).filter(map => map.canonical === server.map);
|
||||
return matches.length === 1 && matches[0].slug === entry.recipe.mapSlug;
|
||||
}).map(server => server.name).sort();
|
||||
const coverage = existingServers.map(name => {
|
||||
const coverage = includeHealth ? existingServers.map(name => {
|
||||
const operation = db().prepare("SELECT id, state FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('create','rebuild') AND state IN ('queued', 'running', 'awaiting-health') ORDER BY rowid DESC LIMIT 1").get(name) as { id: string; state: string } | undefined;
|
||||
const server = getServer(name);
|
||||
return { name, health: operation ? operation.state === 'queued' ? 'queued' : 'starting'
|
||||
: server?.container_state && ['stopped', 'exited', 'created', 'dead'].includes(server.container_state) ? 'stopped'
|
||||
: server ? serverObservation(server).health : 'offline',
|
||||
href: operation ? `/provisioning/${encodeURIComponent(operation.id)}` : `/servers/${encodeURIComponent(name)}` };
|
||||
});
|
||||
}) : [];
|
||||
return { existingServers, coverage, existing: existingServers.length, missing: Math.max(0, entry.count - existingServers.length) };
|
||||
});
|
||||
}
|
||||
@@ -57,8 +57,8 @@ function fleetSnapshot() {
|
||||
}));
|
||||
}
|
||||
|
||||
function collectionPlan(entries: BlueprintEntry[], mode: CollectionMode) {
|
||||
const counts = coverage(entries);
|
||||
function collectionPlan(entries: BlueprintEntry[], mode: CollectionMode, includeHealth = true) {
|
||||
const counts = coverage(entries, includeHealth);
|
||||
// Preserve deployed lobbies before queued ones, then the lowest lobby IDs.
|
||||
const retainedServers = counts.flatMap((entry, index) => [...entry.existingServers].sort((a, b) =>
|
||||
Number(!!getServer(b)) - Number(!!getServer(a)) || a.localeCompare(b, 'en', { numeric: true })
|
||||
@@ -71,50 +71,56 @@ function collectionPlan(entries: BlueprintEntry[], mode: CollectionMode) {
|
||||
return { counts, retainedServers, removals };
|
||||
}
|
||||
|
||||
function allocate(entries: BlueprintEntry[], releasingServers: string[] = []): CollectionPreview['servers'] {
|
||||
const names: string[] = [];
|
||||
function allocate(entries: BlueprintEntry[], releasingServers: string[] = [], counts = coverage(entries)): CollectionPreview['servers'] {
|
||||
const occupied = db().prepare(`SELECT name FROM servers UNION
|
||||
SELECT j.server AS name FROM server_creation_jobs j JOIN server_deployments d ON d.id=j.deployment_id
|
||||
WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued','running','awaiting-health')
|
||||
OR (d.action='rebuild' AND (d.state='waiting' OR (d.state IN ('failed','interrupted') AND d.stage='Removing server files and registry'))))`)
|
||||
.all() as { name: string }[];
|
||||
const lobbies = new Map<string, Set<number>>();
|
||||
for (const { name } of occupied) {
|
||||
const match = /^(.*)_(\d+)$/.exec(name);
|
||||
if (!match) continue;
|
||||
const taken = lobbies.get(match[1]) ?? new Set<number>();
|
||||
taken.add(Number(match[2])); lobbies.set(match[1], taken);
|
||||
}
|
||||
const savedKeys = new Set((db().prepare("SELECT server_id FROM server_keys WHERE length(trim(pluto_key))>0").all() as { server_id: string }[]).map(row => row.server_id));
|
||||
const ports: number[] = [];
|
||||
const planned: { hostId: string; port: number }[] = [];
|
||||
const counts = coverage(entries);
|
||||
return entries.flatMap((entry, entryIndex) => Array.from({ length: counts[entryIndex].missing }, () => {
|
||||
const base = `${entry.recipe.game}_zombie_${entry.recipe.mapSlug}`;
|
||||
const lobby = nextLobbyNumber(base, names);
|
||||
const taken = lobbies.get(base) ?? new Set<number>();
|
||||
const lobby = Array.from({ length: 20 }, (_, index) => index + 1).find(number => !taken.has(number)) ?? null;
|
||||
const name = lobby === null ? null : `${base}_${lobby}`;
|
||||
let native = entry.recipe.native;
|
||||
let port: number | null;
|
||||
if (native) {
|
||||
try {
|
||||
native = nativeSelection(native as unknown as Record<string, unknown>, entry.recipe.game, { planned });
|
||||
port = nextNativePort(native, planned.filter(item => item.hostId === native!.hostId).map(item => item.port));
|
||||
if (port !== null) planned.push({ hostId: native.hostId, port });
|
||||
} catch { port = null; }
|
||||
} else port = nextAvailablePort(ports, releasingServers);
|
||||
if (name) names.push(name);
|
||||
const native = entry.recipe.native;
|
||||
const port = native ? null : nextAvailablePort(ports, releasingServers);
|
||||
if (lobby !== null) { taken.add(lobby); lobbies.set(base, taken); }
|
||||
if (port !== null) ports.push(port);
|
||||
return { entryIndex, name, port, ...(native ? { native } : {}), hasSavedKey: !!name && !!getServerKey(name) };
|
||||
return { entryIndex, name, port, ...(native ? { native } : {}), hasSavedKey: !!name && savedKeys.has(name) };
|
||||
}));
|
||||
}
|
||||
|
||||
function blockersFor(servers: CollectionPreview['servers'], entries: BlueprintEntry[] = []): string[] {
|
||||
const blockers: string[] = [];
|
||||
if (!servers.length) return blockers;
|
||||
if (entries.some(entry => entry.recipe.native)) {
|
||||
for (const entry of entries) {
|
||||
try { nativeSelection(entry.recipe.native as unknown as Record<string, unknown>, entry.recipe.game); }
|
||||
catch (error) { blockers.push((error as Error).message); }
|
||||
}
|
||||
} else {
|
||||
const legacy = servers.filter(server => !server.native && !entries[server.entryIndex]?.recipe.native);
|
||||
if (legacy.length) {
|
||||
if (!komodoConfigured() || !config.komodo.targetServer) blockers.push('Select and prepare a host before launching.');
|
||||
if (!rconConfigured()) blockers.push('Configure RCON_PASSWORD so every new server can be verified.');
|
||||
}
|
||||
if (servers.some(server => !server.name)) blockers.push('There are not enough free lobbies for one or more maps (20 per map). Reduce their counts or free capacity.');
|
||||
if (servers.some(server => server.port === null)) blockers.push(entries.some(entry => entry.recipe.native) ? 'There is not enough host CPU, memory or port capacity for the whole collection.' : `There are not enough free game ports in ${config.portRange.min}–${config.portRange.max} for the whole collection.`);
|
||||
if (legacy.some(server => server.port === null)) blockers.push(`There are not enough free game ports in ${config.portRange.min}–${config.portRange.max} for the whole collection.`);
|
||||
return blockers;
|
||||
}
|
||||
|
||||
function removalBlockersFor(removals: CollectionPreview['removals']): string[] {
|
||||
const blockers: string[] = [];
|
||||
if (removals.some(server => !nativeInstance(server.name)) && (!komodoConfigured() || !rconConfigured())) blockers.push('Configure the legacy backend before removing its servers.');
|
||||
const native = (name: string) => {
|
||||
if (nativeInstance(name)) return true;
|
||||
const creation = db().prepare(`SELECT id FROM server_deployments WHERE server=? AND server_deleted_at IS NULL
|
||||
AND action='create' AND state IN ('queued','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(name) as { id: string } | undefined;
|
||||
return !!creation && nativeDeployment(creation.id);
|
||||
};
|
||||
if (removals.some(server => !native(server.name)) && (!komodoConfigured() || !rconConfigured())) blockers.push('Configure the legacy backend before removing its servers.');
|
||||
for (const server of removals) {
|
||||
const blocker = activeBetaRemovalBlocker(server.name);
|
||||
if (blocker) blockers.push(blocker);
|
||||
@@ -130,24 +136,26 @@ export function previewCollection(blueprintId: unknown, mode: unknown = 'fill',
|
||||
if (selection?.hostId !== undefined || !config.komodo.address || listHosts().length > 0) {
|
||||
const libraries = selection?.libraryIds as Record<string, unknown> | undefined;
|
||||
for (const entry of entries) {
|
||||
entry.recipe.native = nativeSelection({ ...selection, libraryId: libraries?.[entry.recipe.game] ?? selection?.libraryId }, entry.recipe.game);
|
||||
entry.recipe.native = nativeLaunchSelection({ ...selection, libraryId: libraries?.[entry.recipe.game] ?? selection?.libraryId }, entry.recipe.game);
|
||||
entry.recipe.image = entry.recipe.native.image!;
|
||||
}
|
||||
}
|
||||
const { counts, retainedServers, removals } = collectionPlan(entries, mode);
|
||||
const servers = allocate(entries, removals.map(server => server.name));
|
||||
const servers = allocate(entries, removals.map(server => server.name), counts);
|
||||
const blockers = [...blockersFor(servers, entries), ...removalBlockersFor(removals)];
|
||||
const selections = servers.flatMap(server => server.native ? [server.native] : []);
|
||||
const availability = selections.length ? launchAvailability(selections) : {};
|
||||
const id = randomUUID();
|
||||
const expiresAt = new Date(Date.now() + 30 * 60_000).toISOString();
|
||||
db().prepare('DELETE FROM server_collection_previews WHERE expires_at < ? AND collection_id IS NULL').run(new Date().toISOString());
|
||||
db().prepare('INSERT INTO server_collection_previews (id, expires_at, blueprint_name, entries, servers, mode, fleet_snapshot) VALUES (?, ?, ?, ?, ?, ?, ?)')
|
||||
.run(id, expiresAt, blueprintName, JSON.stringify(entries), JSON.stringify(servers), mode, mode === 'exact' ? JSON.stringify(fleetSnapshot()) : null);
|
||||
return { id, expiresAt, blueprintName, mode, retainedServers, removals, launchConcurrency: fleetSettings().createConcurrency, servers, blockers, ready: blockers.length === 0,
|
||||
return { id, expiresAt, blueprintName, mode, retainedServers, removals, launchConcurrency: fleetSettings().createConcurrency, servers, blockers, ...availability, ready: blockers.length === 0,
|
||||
entries: entries.map((entry, index) => ({ count: entry.count, ...counts[index], recipe: summarizeRecipe(entry.recipe) })) };
|
||||
})();
|
||||
}
|
||||
|
||||
/** Reserve and enqueue the entire collection atomically. No external operations here. */
|
||||
/** Save all reviewed identities and keys atomically; native placement belongs to provisioning. */
|
||||
export function queueCollection(input: { previewId?: unknown; keys?: unknown }): { ok: true; collectionId: string | null } {
|
||||
if (typeof input.previewId !== 'string' || !input.previewId) throw new Error('Review a collection preview before launching.');
|
||||
if (input.keys !== undefined && (!input.keys || typeof input.keys !== 'object' || Array.isArray(input.keys))) throw new Error('Provide keys indexed by the exact server ID.');
|
||||
@@ -160,19 +168,35 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
|
||||
if (preview.collection_id) return { ok: true as const, collectionId: preview.collection_id };
|
||||
if (preview.expires_at <= new Date().toISOString()) throw new Error('Collection preview expired. Refresh it before launching.');
|
||||
const entries: BlueprintEntry[] = JSON.parse(preview.entries);
|
||||
// Older previews embedded plugin bytes. Child previews need only the
|
||||
// Older previews embedded plugin bytes. Queued jobs need only the
|
||||
// reviewed metadata; provisioning captures the current files at dispatch.
|
||||
for (const entry of entries) entry.recipe.files = entry.recipe.files.map(({ content: _content, ...file }) => file);
|
||||
const reviewed: CollectionPreview['servers'] = JSON.parse(preview.servers);
|
||||
if (preview.mode === 'exact' && preview.fleet_snapshot !== JSON.stringify(fleetSnapshot())) {
|
||||
throw new Error('The fleet changed. Nothing was queued. Refresh the preview and review the servers to keep and remove.');
|
||||
}
|
||||
const { retainedServers, removals } = collectionPlan(entries, preview.mode);
|
||||
const { counts, retainedServers, removals } = collectionPlan(entries, preview.mode, false);
|
||||
const releasingServers = removals.map(server => server.name);
|
||||
const servers = allocate(entries, releasingServers);
|
||||
const nativeReview = reviewed.every(server => server.native || entries[server.entryIndex]?.recipe.native);
|
||||
// Native Apply accepts the reviewed logical identities. Capacity and
|
||||
// physical ports can change while this request waits in the queue.
|
||||
const servers = nativeReview ? reviewed : allocate(entries, releasingServers, counts);
|
||||
const validatedSelections = new Set<string>();
|
||||
for (const server of servers) {
|
||||
const recipe = entries[server.entryIndex].recipe;
|
||||
const selection = server.native ?? recipe.native;
|
||||
if (!selection) continue;
|
||||
const identity = JSON.stringify([recipe.game, selection]);
|
||||
if (validatedSelections.has(identity)) continue;
|
||||
// Recheck saved configuration once per selection without allocating
|
||||
// physical capacity or changing the names and files already reviewed.
|
||||
nativeLaunchSelection({ ...selection }, recipe.game);
|
||||
validatedSelections.add(identity);
|
||||
}
|
||||
const blockers = [...blockersFor(servers, entries), ...removalBlockersFor(removals)];
|
||||
if (blockers.length) throw new Error(blockers.join(' '));
|
||||
if (servers.length !== reviewed.length || servers.some((server, i) => server.entryIndex !== reviewed[i]?.entryIndex || server.name !== reviewed[i]?.name || (!server.native?.automatic && server.port !== reviewed[i]?.port))) {
|
||||
if (entries.some((_entry, index) => counts[index].missing !== reviewed.filter(server => server.entryIndex === index).length)
|
||||
|| servers.length !== reviewed.length || servers.some((server, i) => server.entryIndex !== reviewed[i]?.entryIndex || server.name !== reviewed[i]?.name || (!server.native && server.port !== reviewed[i]?.port))) {
|
||||
throw new Error('Fleet counts, lobby or port availability changed. Nothing was queued. Refresh the preview and review each server key.');
|
||||
}
|
||||
const expected = new Set(servers.map(server => server.name));
|
||||
@@ -210,16 +234,21 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
|
||||
});
|
||||
servers.forEach((server, index) => {
|
||||
const recipe = { ...entries[server.entryIndex].recipe, ...(server.native ? { native: server.native } : {}) };
|
||||
const childPreviewId = randomUUID();
|
||||
db().prepare('INSERT INTO server_launch_previews (id, expires_at, server, port, recipe, blueprint_name) VALUES (?, ?, ?, ?, ?, ?)')
|
||||
.run(childPreviewId, preview.expires_at, server.name, server.port, JSON.stringify(recipe), preview.blueprint_name);
|
||||
const result = queueCreateServer({ game: recipe.game, gameMode: 'zombie', mapSlug: recipe.mapSlug,
|
||||
expectedName: server.name!, previewId: childPreviewId, plutoKey: keys[server.name!] as string | undefined, createdBy: 'ui' }, releasingServers);
|
||||
let result;
|
||||
if (recipe.native) result = queueNativeCreationIntent({ server: server.name!, game: recipe.game, mapSlug: recipe.mapSlug,
|
||||
selection: recipe.native, recipe, plutoKey: keys[server.name!] as string | undefined, createdBy: 'ui', blueprintName: preview.blueprint_name });
|
||||
else {
|
||||
const childPreviewId = randomUUID();
|
||||
db().prepare('INSERT INTO server_launch_previews (id, expires_at, server, port, recipe, blueprint_name) VALUES (?, ?, ?, ?, ?, ?)')
|
||||
.run(childPreviewId, preview.expires_at, server.name, server.port, JSON.stringify(recipe), preview.blueprint_name);
|
||||
result = queueCreateServer({ game: recipe.game, gameMode: 'zombie', mapSlug: recipe.mapSlug,
|
||||
expectedName: server.name!, previewId: childPreviewId, plutoKey: keys[server.name!] as string | undefined, createdBy: 'ui' }, releasingServers);
|
||||
}
|
||||
// Throw so the outer transaction also reverses all earlier keys and children.
|
||||
if (!result.ok) throw new Error(`${server.name}: ${result.message} Nothing was queued.`);
|
||||
db().prepare('INSERT INTO server_collection_members (collection_id, deployment_id, position) VALUES (?, ?, ?)')
|
||||
.run(collectionId, result.deploymentId, index);
|
||||
for (const removal of removalJobs.filter(removal => removal.ports.includes(server.port))) {
|
||||
for (const removal of recipe.native ? [] : removalJobs.filter(removal => removal.ports.includes(server.port))) {
|
||||
db().prepare('INSERT INTO server_creation_dependencies (deployment_id, removal_id) VALUES (?, ?)').run(result.deploymentId, removal.id);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -439,6 +439,18 @@ function libraryInUse(versionId: string, hostId?: string): boolean {
|
||||
if (!handle.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='native_instances'").get()) return true;
|
||||
if (handle.prepare(`SELECT 1 FROM native_instances WHERE library_id=? AND retired_at IS NULL ${hostId ? 'AND host_id=?' : ''} LIMIT 1`)
|
||||
.get(versionId, ...(hostId ? [hostId] : []))) return true;
|
||||
// Accepted jobs pin their game files before placement creates an instance.
|
||||
// Automatic intents may still choose any host, so retain each available copy.
|
||||
if (handle.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='native_creation_intents'").get()
|
||||
&& handle.prepare(`SELECT 1 FROM server_deployments d
|
||||
JOIN native_creation_intents q ON q.deployment_id=d.id
|
||||
LEFT JOIN native_provisioning n ON n.deployment_id=d.id
|
||||
WHERE d.action='create' AND d.server_deleted_at IS NULL
|
||||
AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') AND n.deployment_id IS NULL
|
||||
AND json_extract(q.selection,'$.libraryId')=?
|
||||
${hostId ? `AND (json_extract(q.selection,'$.automatic')=1 OR json_extract(q.selection,'$.hostId') IS NULL
|
||||
OR json_extract(q.selection,'$.hostId')=?)` : ''} LIMIT 1`)
|
||||
.get(versionId, ...(hostId ? [hostId] : []))) return true;
|
||||
return !!handle.prepare(`SELECT 1 FROM ec_host_operations WHERE state IN ('pending','accepted','running','unknown')
|
||||
AND (json_extract(payload,'$.libraryVersionId')=? OR json_extract(payload,'$.libraryId')=? OR json_extract(payload,'$.importId')=?)
|
||||
${hostId ? 'AND host_id=?' : ''} LIMIT 1`).get(versionId, versionId, versionId, ...(hostId ? [hostId] : []));
|
||||
|
||||
@@ -3,6 +3,11 @@ import type Database from 'better-sqlite3';
|
||||
const initialized = new WeakSet<Database.Database>();
|
||||
export function ensureGameFileSchema(handle: Database.Database): void {
|
||||
if (initialized.has(handle)) return;
|
||||
// Worker readers open only after the controller has initialized the schema.
|
||||
if (handle.readonly) {
|
||||
initialized.add(handle);
|
||||
return;
|
||||
}
|
||||
handle.exec(`
|
||||
CREATE TABLE IF NOT EXISTS game_file_libraries (
|
||||
id TEXT PRIMARY KEY, name TEXT NOT NULL COLLATE NOCASE UNIQUE,
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
import { db } from '../db/index.js';
|
||||
import { fleetSettings } from '../settings.js';
|
||||
import { subscribe } from '../pipeline/bus.js';
|
||||
import { capacityCalibration } from './calibration.js';
|
||||
import { HEARTBEAT_MAX_AGE_MS, listHosts } from './index.js';
|
||||
import type { Host, NativePlacement } from '../../hosts.js';
|
||||
import type { LaunchAvailability } from '../../deployment-plan.js';
|
||||
|
||||
interface HostAvailability {
|
||||
host: Host;
|
||||
freeCpu: number | null;
|
||||
freeMemory: number | null;
|
||||
runningCpu: number;
|
||||
runningMemory: number;
|
||||
ports: { port: number; bind_address: string }[];
|
||||
}
|
||||
interface Snapshot {
|
||||
updatedAt: string;
|
||||
hosts: Map<string, HostAvailability>;
|
||||
startupLimit: number;
|
||||
active: number;
|
||||
queued: number;
|
||||
}
|
||||
const snapshots = new WeakMap<ReturnType<typeof db>, Snapshot>();
|
||||
const scheduled = new WeakSet<ReturnType<typeof db>>();
|
||||
|
||||
function snapshotSchemaReady(handle: ReturnType<typeof db>): boolean {
|
||||
// Host enrollment and game-file maintenance can run before provisioning is
|
||||
// initialized. Missing queue data means unknown capacity, not an empty fleet.
|
||||
const { count } = handle.prepare(`SELECT COUNT(*) AS count FROM sqlite_master WHERE type='table'
|
||||
AND name IN ('native_instances','native_provisioning','server_deployments','settings')`).get() as { count: number };
|
||||
if (count === 4) return true;
|
||||
snapshots.delete(handle);
|
||||
return false;
|
||||
}
|
||||
|
||||
function queueSnapshot(handle: ReturnType<typeof db>, hosts: Map<string, HostAvailability>): Snapshot {
|
||||
const counts = handle.prepare(`SELECT
|
||||
COALESCE(SUM(state IN ('running','awaiting-health')),0) AS active,
|
||||
COALESCE(SUM(state='queued'),0) AS queued FROM server_deployments
|
||||
WHERE action IN ('create','rebuild') AND state IN ('queued','running','awaiting-health')
|
||||
AND server_deleted_at IS NULL`).get() as { active: number; queued: number };
|
||||
return { hosts, ...counts, startupLimit: fleetSettings().createConcurrency, updatedAt: new Date().toISOString() };
|
||||
}
|
||||
|
||||
function observeHost(host: Host): HostAvailability {
|
||||
const handle = db();
|
||||
const { running, startup } = capacityCalibration(host.id);
|
||||
const measured = new Map(host.inventory.instanceResources?.map(sample => [sample.instanceId, sample.generation]));
|
||||
const pending = handle.prepare(`SELECT DISTINCT i.id,i.generation FROM native_instances i
|
||||
JOIN native_provisioning n ON n.instance_id=i.id JOIN server_deployments d ON d.id=n.deployment_id
|
||||
WHERE i.host_id=? AND i.retired_at IS NULL AND d.action IN ('create','rebuild')
|
||||
AND d.state IN ('running','awaiting-health')`).all(host.id) as { id: string; generation: number }[];
|
||||
const unmeasured = pending.filter(instance => measured.get(instance.id) !== instance.generation).length;
|
||||
const { cpuCount, cpuUsagePercent, memoryTotal, memoryAvailable } = host.inventory;
|
||||
const freeCpu = cpuCount && cpuUsagePercent !== undefined
|
||||
? Math.max(0, cpuCount * (0.95 - cpuUsagePercent / 100) - unmeasured * startup.cpuCores) : null;
|
||||
const freeMemory = memoryTotal && memoryAvailable !== undefined && memoryAvailable <= memoryTotal
|
||||
? Math.max(0, memoryAvailable - 500_000_000 - unmeasured * startup.memoryBytes) : null;
|
||||
return { host, freeCpu, freeMemory, runningCpu: running.cpuCores, runningMemory: running.memoryBytes,
|
||||
ports: handle.prepare('SELECT port,bind_address FROM ec_host_ports WHERE host_id=?').all(host.id) as HostAvailability['ports'] };
|
||||
}
|
||||
|
||||
/** Host telemetry updates the estimate once, independently of how many lobbies are previewed. */
|
||||
export function refreshHostAvailability(host: Host): void {
|
||||
const handle = db();
|
||||
if (!snapshotSchemaReady(handle)) return;
|
||||
const hosts = new Map(snapshots.get(handle)?.hosts);
|
||||
hosts.set(host.id, observeHost(host));
|
||||
snapshots.set(handle, queueSnapshot(handle, hosts));
|
||||
}
|
||||
|
||||
/** Lifecycle events refresh the shared snapshot after their transaction and request have yielded. */
|
||||
export function scheduleAvailabilityRefresh(): void {
|
||||
const handle = db();
|
||||
if (scheduled.has(handle)) return;
|
||||
scheduled.add(handle);
|
||||
setImmediate(() => {
|
||||
scheduled.delete(handle);
|
||||
if (!handle.open) return;
|
||||
try {
|
||||
if (!snapshotSchemaReady(handle)) return;
|
||||
const hosts = new Map(listHosts().map(host => [host.id, observeHost(host)]));
|
||||
snapshots.set(handle, queueSnapshot(handle, hosts));
|
||||
} catch (error) { console.error('[availability] snapshot refresh failed:', error); }
|
||||
});
|
||||
}
|
||||
|
||||
subscribe(event => {
|
||||
if (event.type === 'fleet-invalidated' || event.type === 'server-status') scheduleAvailabilityRefresh();
|
||||
});
|
||||
|
||||
function freePorts(snapshot: HostAvailability, bindAddress?: string): number {
|
||||
const bind = bindAddress ?? snapshot.host.bindAddress;
|
||||
const used = new Set(snapshot.ports.filter(row => row.port >= snapshot.host.portMin && row.port <= snapshot.host.portMax
|
||||
&& (row.bind_address === bind || ['0.0.0.0', '::'].includes(bind) || ['0.0.0.0', '::'].includes(row.bind_address))).map(row => row.port));
|
||||
return Math.max(0, snapshot.host.portMax - snapshot.host.portMin + 1 - used.size);
|
||||
}
|
||||
|
||||
/** Estimates only: no placement, historical scans, reservations, or live host requests. */
|
||||
export function launchAvailability(selections: NativePlacement[]): { availability: LaunchAvailability; warnings: string[] } {
|
||||
const snapshot = snapshots.get(db());
|
||||
if (!snapshot) {
|
||||
scheduleAvailabilityRefresh();
|
||||
return { availability: { updatedAt: null, requested: selections.length, estimatedAvailable: null, portsAvailable: null,
|
||||
cpuCoresAvailable: null, memoryBytesAvailable: null, startupSlots: null, startupLimit: null, queued: null },
|
||||
warnings: selections.length ? ['Capacity measurements are updating. You can queue this launch; provisioning waits for an eligible host.'] : [] };
|
||||
}
|
||||
const candidates = [...snapshot.hosts.values()].filter(({ host }) => host.readiness === 'ready'
|
||||
&& host.lastHeartbeat && Date.now() - Date.parse(host.lastHeartbeat) <= HEARTBEAT_MAX_AGE_MS
|
||||
&& (host.inventory.storageAvailable ?? 0) >= host.minimumFreeBytes + 1024 ** 3);
|
||||
const eligible = (candidate: HostAvailability, selection: NativePlacement) => (!selection.hostId || selection.automatic || candidate.host.id === selection.hostId)
|
||||
&& !!candidate.host.inventory.runtimeImages?.includes(selection.image ?? '')
|
||||
&& !!candidate.host.inventory.libraries?.some(library => library.id === selection.libraryId && ['ready', 'outdated'].includes(library.state));
|
||||
const relevant = candidates.filter(candidate => selections.some(selection => eligible(candidate, selection)));
|
||||
const planned = new Map<string, number>();
|
||||
let estimatedAvailable = 0;
|
||||
for (const selection of selections) {
|
||||
const candidate = relevant.find(candidate => {
|
||||
const used = planned.get(candidate.host.id) ?? 0;
|
||||
return eligible(candidate, selection) && freePorts(candidate, selection.network?.bindAddress ?? selection.bindAddress) > used
|
||||
&& candidate.freeCpu !== null && candidate.freeMemory !== null
|
||||
&& candidate.freeCpu >= (used + 1) * candidate.runningCpu
|
||||
&& candidate.freeMemory >= (used + 1) * candidate.runningMemory;
|
||||
});
|
||||
if (candidate) { estimatedAvailable++; planned.set(candidate.host.id, (planned.get(candidate.host.id) ?? 0) + 1); }
|
||||
}
|
||||
const warnings: string[] = [];
|
||||
const unknown = relevant.some(candidate => candidate.freeCpu === null || candidate.freeMemory === null);
|
||||
if (selections.length && !relevant.length) warnings.push('Waiting for an eligible host with the selected runtime and game files. The launch can be queued now.');
|
||||
else if (unknown) warnings.push('Some host CPU or memory measurements are unavailable. Provisioning waits for fresh capacity measurements.');
|
||||
else if (estimatedAvailable < selections.length) warnings.push(`Current capacity is estimated for ${estimatedAvailable} of ${selections.length} servers. Remaining launches wait for capacity, including any removals in this collection.`);
|
||||
const startupSlots = Math.max(0, snapshot.startupLimit - snapshot.active);
|
||||
if (selections.length && !startupSlots) warnings.push('All startup slots are in use. New launches will wait in the queue.');
|
||||
if (selections.length && snapshot.queued) warnings.push(`${snapshot.queued} server${snapshot.queued === 1 ? ' is' : 's are'} already queued.`);
|
||||
return { availability: { updatedAt: snapshot.updatedAt, requested: selections.length, estimatedAvailable: unknown ? null : estimatedAvailable,
|
||||
portsAvailable: relevant.reduce((total, candidate) => total + freePorts(candidate), 0),
|
||||
cpuCoresAvailable: unknown ? null : relevant.reduce((total, candidate) => total + (candidate.freeCpu ?? 0), 0),
|
||||
memoryBytesAvailable: unknown ? null : relevant.reduce((total, candidate) => total + (candidate.freeMemory ?? 0), 0),
|
||||
startupSlots, startupLimit: snapshot.startupLimit, queued: snapshot.queued }, warnings };
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import { isIP } from 'node:net';
|
||||
import { db } from '../db/index.js';
|
||||
import { ensureHostSchema } from './schema.js';
|
||||
import { recordInstanceResources, validateInstanceResources } from './calibration.js';
|
||||
import { refreshHostAvailability, scheduleAvailabilityRefresh } from './availability.js';
|
||||
import type { Host, HostDecommission, HostDeletion, HostInventory, HostMode, HostOperation, HostOperationState } from '../../hosts.js';
|
||||
|
||||
export const AGENT_PROTOCOL_VERSION = 1;
|
||||
@@ -83,6 +84,17 @@ export function hostDeletionStatus(id: string): HostDeletion {
|
||||
const names = instances.slice(0, 5).map(instance => instance.server).join(', ') + (instances.length > 5 ? ', …' : '');
|
||||
blockers.push(`Remove the ${instances.length} server instance${instances.length === 1 ? '' : 's'} assigned to this host first (${names}). Queued and failed deployments also require removal.`);
|
||||
}
|
||||
const pinned = handle.prepare(`SELECT d.server FROM server_deployments d
|
||||
JOIN native_creation_intents q ON q.deployment_id=d.id
|
||||
LEFT JOIN native_provisioning n ON n.deployment_id=d.id
|
||||
WHERE d.action='create' AND d.server_deleted_at IS NULL
|
||||
AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') AND n.deployment_id IS NULL
|
||||
AND json_extract(q.selection,'$.hostId')=? AND COALESCE(json_extract(q.selection,'$.automatic'),0)!=1
|
||||
ORDER BY d.server LIMIT 6`).all(id) as { server: string }[];
|
||||
if (pinned.length) {
|
||||
const names = pinned.slice(0, 5).map(job => job.server).join(', ') + (pinned.length > 5 ? ', …' : '');
|
||||
blockers.push(`Cancel the queued server launches pinned to this host before deleting it (${names}).`);
|
||||
}
|
||||
const ports = handle.prepare('SELECT COUNT(*) AS count FROM ec_host_ports WHERE host_id=?').get(id) as { count: number };
|
||||
if (ports.count && !instances.length) blockers.push(`This host still has ${ports.count} reserved game port${ports.count === 1 ? '' : 's'}. Finish or reconcile the corresponding server removals first.`);
|
||||
// A pending command may have reached the agent before its acknowledgement.
|
||||
@@ -174,6 +186,7 @@ export function createHost(input: Record<string, unknown>): Host {
|
||||
const bind = input.bindAddress === undefined ? '0.0.0.0' : textField(input.bindAddress, 'Bind address');
|
||||
if (!isIP(bind)) throw new Error('Bind address must be an IPv4 or IPv6 address.');
|
||||
database().prepare('INSERT INTO ec_hosts(id,name,port_min,port_max,bind_address,created_at) VALUES(?,?,?,?,?,?)').run(id, name, min, max, bind, now());
|
||||
scheduleAvailabilityRefresh();
|
||||
return requireHost(id);
|
||||
}
|
||||
export function updateHost(id: string, input: Record<string, unknown>): Host {
|
||||
@@ -190,6 +203,7 @@ export function updateHost(id: string, input: Record<string, unknown>): Host {
|
||||
const minimum = input.minimumFreeBytes ?? host.minimumFreeBytes;
|
||||
if (typeof minimum !== 'number' || !Number.isSafeInteger(minimum) || minimum < 0) throw new Error('Minimum free storage must be a nonnegative byte count.');
|
||||
database().prepare('UPDATE ec_hosts SET name=?,mode=?,port_min=?,port_max=?,bind_address=?,public_address=?,minimum_free_bytes=? WHERE id=?').run(name, mode, min, max, bind, publicAddress, minimum, id);
|
||||
scheduleAvailabilityRefresh();
|
||||
return requireHost(id);
|
||||
}
|
||||
|
||||
@@ -256,6 +270,7 @@ export function enrollHost(token: unknown, rawInventory: unknown): { hostId: str
|
||||
const credential = issueCredential(enrollment.host_id);
|
||||
database().prepare('UPDATE ec_hosts SET enrolled_at=?,last_heartbeat=?,hostname=?,inventory=? WHERE id=?').run(now(), now(), inventory.hostname ?? '', JSON.stringify(inventory), enrollment.host_id);
|
||||
recordResources(enrollment.host_id, inventory, now());
|
||||
scheduleAvailabilityRefresh();
|
||||
return { hostId: enrollment.host_id, credential, heartbeatSeconds: 15 };
|
||||
})();
|
||||
}
|
||||
@@ -270,6 +285,7 @@ export function revokeCredentials(hostId: string): void {
|
||||
const host = requireHost(hostId);
|
||||
if (host.decommission && !host.decommission.retryable) throw new Error('Host cleanup needs its credentials until removal is confirmed.');
|
||||
database().prepare('UPDATE ec_host_credentials SET revoked_at=? WHERE host_id=? AND revoked_at IS NULL').run(now(), hostId);
|
||||
scheduleAvailabilityRefresh();
|
||||
}
|
||||
export function rotateCredential(hostId: string, nextCredential?: string): string {
|
||||
requireHost(hostId);
|
||||
@@ -292,7 +308,9 @@ export function heartbeat(hostId: string, value: unknown): Host {
|
||||
database().prepare('UPDATE ec_hosts SET last_heartbeat=?,hostname=?,inventory=? WHERE id=?').run(timestamp, inventory.hostname ?? '', JSON.stringify(inventory), hostId);
|
||||
recordResources(hostId, inventory, timestamp);
|
||||
})();
|
||||
return requireHost(hostId);
|
||||
const host = requireHost(hostId);
|
||||
refreshHostAvailability(host);
|
||||
return host;
|
||||
}
|
||||
|
||||
function recordResources(hostId: string, inventory: HostInventory, timestamp: string): void {
|
||||
@@ -504,7 +522,11 @@ export function reserveHostPort(hostId: string, instanceId: string, options: { p
|
||||
const allocated = options.port ?? nextHostPort(hostId, [], bind, protocol);
|
||||
if (allocated === null || !hostPortAvailable(hostId, allocated, bind, protocol, instanceId)) throw new Error('No available game port on the selected host.');
|
||||
database().prepare('INSERT INTO ec_host_ports(host_id,instance_id,bind_address,protocol,port,created_at) VALUES(?,?,?,?,?,?)').run(hostId, instanceId, bind, protocol, allocated, now());
|
||||
scheduleAvailabilityRefresh();
|
||||
return allocated;
|
||||
})();
|
||||
}
|
||||
export function releaseHostPorts(instanceId: string): void { database().prepare('DELETE FROM ec_host_ports WHERE instance_id=?').run(instanceId); }
|
||||
export function releaseHostPorts(instanceId: string): void {
|
||||
database().prepare('DELETE FROM ec_host_ports WHERE instance_id=?').run(instanceId);
|
||||
scheduleAvailabilityRefresh();
|
||||
}
|
||||
|
||||
@@ -2,12 +2,46 @@ import { db } from '../db/index.js';
|
||||
import { config } from '../config.js';
|
||||
import { AGENT_PROTOCOL_VERSION, getHost, queueHostOperation } from './index.js';
|
||||
import { getHostLibraryCache, latestPublishedLibraries, queueLibraryDownload } from '../game-files/index.js';
|
||||
import { GAME_FILE_GAMES } from '../../game-files.js';
|
||||
import type { Host, HostOperationState, HostPreparation, HostPreparationItem } from '../../hosts.js';
|
||||
|
||||
const activeStates = ['pending', 'accepted', 'running', 'unknown'];
|
||||
const supersededDownload = 'Superseded by newer game files before delivery to the host.';
|
||||
type PreparationOperation = { id: string; state: HostOperationState; updated_at: string; error: string | null };
|
||||
type Target = { item: HostPreparationItem; image?: string; versionId?: string; requiredBytes: number; operation?: PreparationOperation };
|
||||
type QueuedSelection = { libraryId: string; image: string | null };
|
||||
type LibraryTarget = Pick<ReturnType<typeof latestPublishedLibraries>[number], 'id' | 'sizeBytes' | 'manifestSha256'> & { name: string; queued?: boolean };
|
||||
|
||||
/** Accepted requests retain their prerequisites until placement owns an instance. */
|
||||
function queuedSelections(hostId: string): QueuedSelection[] {
|
||||
// Standalone host/library readers can initialize before provisioning migrations.
|
||||
if (!db().prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='native_creation_intents'").get()) return [];
|
||||
return db().prepare(`SELECT DISTINCT json_extract(q.selection,'$.libraryId') AS libraryId,
|
||||
json_extract(q.selection,'$.image') AS image FROM native_creation_intents q
|
||||
JOIN server_deployments d ON d.id=q.deployment_id
|
||||
LEFT JOIN native_provisioning n ON n.deployment_id=d.id
|
||||
WHERE d.action='create' AND d.server_deleted_at IS NULL AND n.deployment_id IS NULL
|
||||
AND d.state IN ('queued','waiting','running','awaiting-health','interrupted')
|
||||
AND (json_extract(q.selection,'$.automatic')=1 OR json_extract(q.selection,'$.hostId') IS NULL
|
||||
OR json_extract(q.selection,'$.hostId')=?)`).all(hostId) as QueuedSelection[];
|
||||
}
|
||||
|
||||
function targetLibraries(selections: QueuedSelection[]): LibraryTarget[] {
|
||||
const latest = latestPublishedLibraries();
|
||||
const latestIds = new Set(latest.map(version => version.id));
|
||||
const queued = new Map<string, LibraryTarget>();
|
||||
const get = db().prepare(`SELECT v.id,l.game,v.version,v.size_bytes AS sizeBytes,v.manifest_sha256 AS manifestSha256
|
||||
FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id
|
||||
WHERE v.id=? AND v.state='ready' AND NOT EXISTS(SELECT 1 FROM game_file_cleanup c WHERE c.version_id=v.id)`);
|
||||
for (const selection of selections) {
|
||||
if (latestIds.has(selection.libraryId) || queued.has(selection.libraryId)) continue;
|
||||
const version = get.get(selection.libraryId) as { id: string; game: string; version: string; sizeBytes: number; manifestSha256: string } | undefined;
|
||||
if (version) queued.set(version.id, { ...version, queued: true,
|
||||
name: `${GAME_FILE_GAMES.find(game => game.id === version.game)?.name ?? version.game} · Queued version ${version.version}` });
|
||||
}
|
||||
// Finish prerequisites for accepted work before downloading newer optional files.
|
||||
return [...queued.values(), ...latest];
|
||||
}
|
||||
|
||||
function latestOperation(hostId: string, kind: string, field: string, value: string): PreparationOperation | undefined {
|
||||
return db().prepare(`SELECT id,state,updated_at,error FROM ec_host_operations
|
||||
@@ -43,19 +77,24 @@ function hostBlocker(host: Host): { state: 'waiting' | 'blocked'; detail: string
|
||||
}
|
||||
|
||||
function targets(host: Host): Target[] {
|
||||
const image = config.komodo.defaultImage;
|
||||
const runtimeReady = host.inventory.runtimeImages?.includes(image);
|
||||
const result: Target[] = [{ image, requiredBytes: host.minimumFreeBytes,
|
||||
operation: latestOperation(host.id, 'runtime.pull', 'image', image),
|
||||
item: { kind: 'runtime', label: image, state: runtimeReady ? 'ready' : 'waiting', detail: runtimeReady ? 'Runtime is cached.' : 'Runtime will be downloaded automatically.' } }];
|
||||
for (const version of latestPublishedLibraries()) {
|
||||
const selections = queuedSelections(host.id);
|
||||
const images = new Set([config.komodo.defaultImage, ...selections.flatMap(selection => selection.image ? [selection.image] : [])]);
|
||||
const result: Target[] = [...images].map(image => {
|
||||
const runtimeReady = host.inventory.runtimeImages?.includes(image);
|
||||
return { image, requiredBytes: host.minimumFreeBytes,
|
||||
operation: latestOperation(host.id, 'runtime.pull', 'image', image),
|
||||
item: { kind: 'runtime', label: image, state: runtimeReady ? 'ready' : 'waiting', detail: runtimeReady ? 'Runtime is cached.' : 'Runtime will be downloaded automatically.' } };
|
||||
});
|
||||
for (const version of targetLibraries(selections)) {
|
||||
const cache = getHostLibraryCache(host.id, version.id);
|
||||
const matches = cache?.manifestSha256 === version.manifestSha256;
|
||||
const ready = matches && ['ready', 'outdated'].includes(cache!.status) && cache!.bytesDownloaded === version.sizeBytes;
|
||||
result.push({ versionId: version.id, requiredBytes: version.sizeBytes - (matches ? cache!.bytesDownloaded : 0) + host.minimumFreeBytes,
|
||||
operation: latestOperation(host.id, 'library.download', 'libraryVersionId', version.id),
|
||||
item: { kind: 'library', label: version.name, state: ready ? 'ready' : 'waiting',
|
||||
detail: ready ? 'The latest game files are downloaded and verified.' : 'The latest game files will be downloaded and verified automatically.',
|
||||
detail: version.queued
|
||||
? ready ? 'Game files selected by queued launches are downloaded and verified.' : 'Game files selected by queued launches will be downloaded and verified automatically.'
|
||||
: ready ? 'The latest game files are downloaded and verified.' : 'The latest game files will be downloaded and verified automatically.',
|
||||
bytesDownloaded: cache?.bytesDownloaded ?? 0, bytesTotal: version.sizeBytes, ...(cache ? { cache } : {}) } });
|
||||
}
|
||||
const blocker = hostBlocker(host);
|
||||
@@ -100,13 +139,13 @@ export function presentHostPreparation(host: Host): Host {
|
||||
}
|
||||
|
||||
function supersedeUndeliveredDownloads(hostId: string): void {
|
||||
const latest = new Set(latestPublishedLibraries().map(version => version.id));
|
||||
const required = new Set(targetLibraries(queuedSelections(hostId)).map(version => version.id));
|
||||
const pending = db().prepare(`SELECT id,json_extract(payload,'$.libraryVersionId') AS versionId
|
||||
FROM ec_host_operations WHERE host_id=? AND kind='library.download' AND state='pending'
|
||||
AND last_delivered_at IS NULL AND json_extract(payload,'$.automatic')=1`).all(hostId) as { id: string; versionId: string }[];
|
||||
const now = new Date().toISOString();
|
||||
for (const operation of pending) {
|
||||
if (latest.has(operation.versionId)) continue;
|
||||
if (required.has(operation.versionId)) continue;
|
||||
// Delivered operations may already be writing files. Only unsubmitted
|
||||
// automatic work can be replaced without consulting the agent journal.
|
||||
db().prepare("UPDATE ec_host_operations SET state='failed',error=?,updated_at=? WHERE id=?").run(supersededDownload, now, operation.id);
|
||||
|
||||
@@ -3,6 +3,8 @@ import type Database from 'better-sqlite3';
|
||||
const initialized = new WeakSet<Database.Database>();
|
||||
export function ensureHostSchema(handle: Database.Database): void {
|
||||
if (initialized.has(handle)) return;
|
||||
// Background readers open only after controller migrations have completed.
|
||||
if (handle.readonly) { initialized.add(handle); return; }
|
||||
handle.exec(`
|
||||
CREATE TABLE IF NOT EXISTS ec_hosts (
|
||||
id TEXT PRIMARY KEY, name TEXT NOT NULL, hostname TEXT NOT NULL DEFAULT '',
|
||||
|
||||
@@ -6,9 +6,13 @@ import { recordProgress } from './progress.js';
|
||||
|
||||
export function canCancelCreation(id: string): boolean {
|
||||
return !!db().prepare(`SELECT 1 FROM server_deployments d
|
||||
JOIN native_provisioning n ON n.deployment_id=d.id
|
||||
JOIN native_instances i ON i.id=n.instance_id AND i.generation=n.generation AND i.retired_at IS NULL
|
||||
LEFT JOIN native_provisioning n ON n.deployment_id=d.id
|
||||
LEFT JOIN native_instances i ON i.id=n.instance_id AND i.generation=n.generation AND i.retired_at IS NULL
|
||||
WHERE d.id=? AND d.action='create' AND d.server_deleted_at IS NULL
|
||||
AND (i.id IS NOT NULL OR (n.deployment_id IS NULL
|
||||
AND EXISTS (SELECT 1 FROM native_creation_intents intent WHERE intent.deployment_id=d.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM native_instances replacement WHERE replacement.server=d.server AND replacement.retired_at IS NULL)
|
||||
AND NOT EXISTS (SELECT 1 FROM servers replacement WHERE replacement.name=d.server AND replacement.destroyed_at IS NULL)))
|
||||
AND d.state IN ('queued','waiting','running','awaiting-health','interrupted','failed')
|
||||
AND NOT EXISTS (SELECT 1 FROM server_deployments other WHERE other.server=d.server
|
||||
AND other.server_deleted_at IS NULL AND other.rowid>d.rowid AND other.state!='cancelled'
|
||||
@@ -23,9 +27,9 @@ export function cancelCreation(id: string, triggeredBy = 'ui'): string | null {
|
||||
if (prior) return prior.cleanup_id;
|
||||
if (!canCancelCreation(id)) return null;
|
||||
const job = db().prepare(`SELECT d.server,n.operation_id,n.stage_operation_id FROM server_deployments d
|
||||
JOIN native_provisioning n ON n.deployment_id=d.id WHERE d.id=?`).get(id) as
|
||||
LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE d.id=?`).get(id) as
|
||||
{ server: string; operation_id: string | null; stage_operation_id: string | null };
|
||||
const instance = nativeInstance(job.server)!;
|
||||
const instance = nativeInstance(job.server);
|
||||
// Replace a removal still waiting on occupancy, including one whose asynchronous probe is outstanding.
|
||||
for (const removal of db().prepare("SELECT id FROM server_deployments WHERE server=? AND action='destroy' AND state='waiting' AND server_deleted_at IS NULL").all(job.server) as { id: string }[]) {
|
||||
cancelRemoval(removal.id);
|
||||
@@ -37,13 +41,16 @@ export function cancelCreation(id: string, triggeredBy = 'ui'): string | null {
|
||||
event: { kind: 'cancelled', message: 'Creation cancelled. Its partial instance will be removed without a player-count check.', details: { cleanupId } } });
|
||||
if (!job.operation_id && !job.stage_operation_id) {
|
||||
// No remote mutation was ever queued. The reservation is the only resource to remove.
|
||||
db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(new Date().toISOString(), instance.id);
|
||||
releaseHostPorts(instance.id);
|
||||
if (instance) {
|
||||
db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(new Date().toISOString(), instance.id);
|
||||
releaseHostPorts(instance.id);
|
||||
}
|
||||
deploymentStage(cleanupId, 'Unsubmitted creation cancelled; reservations released', { phase: 'cleanup',
|
||||
checkpoint: { key: 'cleanup', label: 'No host command submitted; local reservations released' } });
|
||||
finishDeployment(cleanupId, { ok: true });
|
||||
finishCreationCancellation(cleanupId);
|
||||
} else {
|
||||
if (!instance) throw new Error('The original native instance is unavailable.');
|
||||
const operation = queueHostOperation(instance.host_id, 'instance.cancel-create', { createOperationId: job.operation_id ?? id },
|
||||
{ instanceId: instance.id, generation: instance.generation, idempotencyKey: cleanupId });
|
||||
db().prepare('UPDATE native_provisioning SET operation_id=? WHERE deployment_id=?').run(operation.id, cleanupId);
|
||||
@@ -57,12 +64,12 @@ export function cancelCreation(id: string, triggeredBy = 'ui'): string | null {
|
||||
|
||||
export function finishCreationCancellation(cleanupId: string): void {
|
||||
const cancellation = db().prepare(`SELECT c.creation_id,n.operation_id FROM native_creation_cancellations c
|
||||
JOIN native_provisioning n ON n.deployment_id=c.creation_id WHERE c.cleanup_id=?`).get(cleanupId) as
|
||||
LEFT JOIN native_provisioning n ON n.deployment_id=c.creation_id WHERE c.cleanup_id=?`).get(cleanupId) as
|
||||
{ creation_id: string; operation_id: string | null } | undefined;
|
||||
if (!cancellation) return;
|
||||
const now = new Date().toISOString();
|
||||
db().prepare('UPDATE server_deployments SET server_deleted_at=? WHERE id IN (?,?)').run(now, cancellation.creation_id, cleanupId);
|
||||
deploymentStage(cancellation.creation_id, 'Creation cancelled; partial instance removed', { blocker: null,
|
||||
deploymentStage(cancellation.creation_id, 'Creation cancelled; cleanup completed', { blocker: null,
|
||||
checkpoint: { key: 'cleanup', phase: 'cleanup', label: 'Cancellation cleanup confirmed; instance and reservations removed' } });
|
||||
// The original command may have lost its receipt. Cleanup proves that its effects are now gone.
|
||||
if (cancellation.operation_id) {
|
||||
|
||||
@@ -9,6 +9,7 @@ import { readProgress, recordProgress, type ProgressUpdate } from './progress.js
|
||||
import { bindNativeOperation } from './native.js';
|
||||
import { workerActivity } from '../pipeline/activity.js';
|
||||
import { canCancelCreation } from './cancel-creation.js';
|
||||
import { nativeDeployment } from '../backends/native.js';
|
||||
|
||||
export function beginDeployment(server: string, action: ProvisioningAction, triggeredBy = 'ui'): string {
|
||||
const id = randomUUID();
|
||||
@@ -103,7 +104,7 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
|
||||
(ProvisioningDeployment & { observation_json: string | null; phase_key: string | null; phase_started_at: string | null; observed_at: string | null }) | undefined;
|
||||
if (!row) return null;
|
||||
const generatedAt = new Date().toISOString();
|
||||
const native = db().prepare('SELECT instance_id FROM native_provisioning WHERE deployment_id=?').get(id);
|
||||
const native = nativeDeployment(id);
|
||||
const agentOperation = native ? db().prepare(`SELECT o.id, o.host_id AS hostId, o.state, o.updated_at AS updatedAt, o.created_at AS createdAt, o.error
|
||||
FROM native_provisioning n JOIN ec_host_operations o ON o.id=n.operation_id WHERE n.deployment_id=?`).get(id) as ProvisioningDetail['agentOperation'] : null;
|
||||
const cancellation = db().prepare(`SELECT c.cleanup_id AS cleanupId,d.state,d.stage,d.error FROM native_creation_cancellations c
|
||||
@@ -254,10 +255,11 @@ export function listDeployments() {
|
||||
|
||||
export function interruptDeployments(): void {
|
||||
db().transaction(() => {
|
||||
const running = db().prepare("SELECT id FROM server_deployments WHERE state = 'running' AND id NOT IN (SELECT deployment_id FROM native_provisioning)").all() as { id: string }[];
|
||||
const native = 'SELECT deployment_id FROM native_provisioning UNION ALL SELECT deployment_id FROM native_creation_intents';
|
||||
const running = db().prepare(`SELECT id FROM server_deployments WHERE state = 'running' AND id NOT IN (${native})`).all() as { id: string }[];
|
||||
const finishedAt = new Date().toISOString();
|
||||
const error = 'The application restarted before this operation finished. Check the server before retrying.';
|
||||
db().prepare(`UPDATE server_deployments SET state = 'interrupted', finished_at = ?, error = ? WHERE state = 'running' AND id NOT IN (SELECT deployment_id FROM native_provisioning)`).run(finishedAt, error);
|
||||
db().prepare(`UPDATE server_deployments SET state = 'interrupted', finished_at = ?, error = ? WHERE state = 'running' AND id NOT IN (${native})`).run(finishedAt, error);
|
||||
for (const { id } of running) recordProgress(id, { blocker: null, event: { kind: 'outcome',
|
||||
message: 'Interrupted by application restart. Check the server before retrying.', details: { state: 'interrupted', finishedAt, error } } });
|
||||
})();
|
||||
|
||||
@@ -44,7 +44,7 @@ export type CreateServerFailure =
|
||||
| 'db'; // shouldn't happen, but wraps INSERT failures
|
||||
|
||||
export type CreateServerResult =
|
||||
| { ok: true; name: string; port: number; deploymentId: string }
|
||||
| { ok: true; name: string; port: number | null; deploymentId: string }
|
||||
| { ok: false; kind: CreateServerFailure; message: string };
|
||||
|
||||
export type DestroyServerResult =
|
||||
|
||||
@@ -4,6 +4,7 @@ import { config } from '../config.js';
|
||||
import { db } from '../db/index.js';
|
||||
import type { NativeInstance } from '../backends/native.js';
|
||||
import type { LaunchRecipePlan } from '../fleet/launch-recipes.js';
|
||||
import type { PlacementOptions } from '../hosts/capacity.js';
|
||||
|
||||
export interface NativeLaunchInput {
|
||||
game: 't5' | 't6';
|
||||
@@ -25,7 +26,7 @@ export interface PreparedNativeLaunch {
|
||||
let worker: Worker | null = null;
|
||||
let sequence = 0;
|
||||
const pending = new Map<number, {
|
||||
resolve: (value: PreparedNativeLaunch) => void;
|
||||
resolve: (value: unknown) => void;
|
||||
reject: (error: Error) => void;
|
||||
timer: NodeJS.Timeout;
|
||||
}>();
|
||||
@@ -39,14 +40,14 @@ function stop(error: Error, current = worker): Promise<number> | undefined {
|
||||
}
|
||||
|
||||
/** The reader and JSON encoder run off the controller's request thread. */
|
||||
export function prepareNativeLaunch(input: NativeLaunchInput): Promise<PreparedNativeLaunch> {
|
||||
function run<T>(kind: 'launch' | 'placement', input: NativeLaunchInput | PlacementOptions): Promise<T> {
|
||||
db(); // Migrations must finish before the independent read-only connection opens.
|
||||
if (!worker) {
|
||||
const current = worker = new Worker(resolve(process.env.NODE_ENV === 'production'
|
||||
? 'build/launch/worker.mjs' : '.svelte-kit/launch/worker.mjs'), {
|
||||
workerData: { dbPath: resolve(config.dbPath) }
|
||||
});
|
||||
current.on('message', ({ id, value, error }: { id: number; value: PreparedNativeLaunch; error?: string }) => {
|
||||
current.on('message', ({ id, value, error }: { id: number; value: unknown; error?: string }) => {
|
||||
if (current !== worker) return;
|
||||
const request = pending.get(id);
|
||||
if (!request) return;
|
||||
@@ -59,13 +60,18 @@ export function prepareNativeLaunch(input: NativeLaunchInput): Promise<PreparedN
|
||||
}
|
||||
worker.ref();
|
||||
const id = ++sequence;
|
||||
return new Promise((resolve, reject) => {
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
const timer = setTimeout(() => { void stop(new Error('Launch preparation timed out; preparation will retry.')); }, 60_000);
|
||||
pending.set(id, { resolve, reject, timer });
|
||||
worker!.postMessage({ id, input });
|
||||
pending.set(id, { resolve: value => resolve(value as T), reject, timer });
|
||||
worker!.postMessage({ id, kind, input });
|
||||
});
|
||||
}
|
||||
|
||||
export function prepareNativeLaunch(input: NativeLaunchInput): Promise<PreparedNativeLaunch> { return run('launch', input); }
|
||||
export function evaluateNativePlacement(input: PlacementOptions): Promise<{ hostId: string; lastHeartbeat: string | null }> {
|
||||
return run('placement', input);
|
||||
}
|
||||
|
||||
export async function closeLaunchPreparation(): Promise<void> {
|
||||
await stop(new Error('Launch preparation stopped.'));
|
||||
}
|
||||
|
||||
@@ -5,9 +5,17 @@ import { relDestForKind } from '../config.js';
|
||||
import { captureRecipe, summarizeRecipe } from '../fleet/launch-recipes.js';
|
||||
import { sha256Buffer } from '../fleet/files.js';
|
||||
import type { NativeLaunchInput } from './launch-preparation.js';
|
||||
import { choosePlacement, type PlacementOptions } from '../hosts/capacity.js';
|
||||
|
||||
parentPort!.on('message', ({ id, input }: { id: number; input: NativeLaunchInput }) => {
|
||||
parentPort!.on('message', (message: { id: number; kind: 'launch'; input: NativeLaunchInput } | { id: number; kind: 'placement'; input: PlacementOptions }) => {
|
||||
const { id } = message;
|
||||
try {
|
||||
if (message.kind === 'placement') {
|
||||
const decision = db().transaction(() => choosePlacement(message.input))();
|
||||
parentPort!.postMessage({ id, value: { hostId: decision.host.id, lastHeartbeat: decision.host.lastHeartbeat } });
|
||||
return;
|
||||
}
|
||||
const input = message.input;
|
||||
// All file versions and composed configuration belong to one snapshot.
|
||||
const recipe = db().transaction(() => ({ ...captureRecipe(input.game, input.mapSlug),
|
||||
native: input.native, image: input.instance.image }))();
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
import { isIP } from 'node:net';
|
||||
import { setImmediate as yieldToController } from 'node:timers/promises';
|
||||
import { CapacityError, choosePlacement, availableHostPort, type PlacementOptions } from '../hosts/capacity.js';
|
||||
import { choosePlacement, availableHostPort, type PlacementOptions } from '../hosts/capacity.js';
|
||||
import { randomBytes, randomUUID } from 'node:crypto';
|
||||
import { posix } from 'node:path';
|
||||
import { db } from '../db/index.js';
|
||||
import { config, relDestForKind, serverDir } from '../config.js';
|
||||
import { nativeInstance, nativeRequest, type NativeInstance } from '../backends/native.js';
|
||||
import { nativeInstance, nativeRequest, unplacedNativeCreation, type NativeInstance } from '../backends/native.js';
|
||||
import { getHost, getHostOperationStatus, listHosts, queueHostOperation, queueHostOperationSerialized, releaseHostPorts, reserveHostPort } from '../hosts/index.js';
|
||||
import { newestReadyLibraryId, assertReadyLibrary } from '../game-files/index.js';
|
||||
import { blueprintRecipe, captureRecipePlan, recipeForDeployment, recipeStorage, summarizeRecipe, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
|
||||
import { prepareNativeLaunch, type PreparedNativeLaunch } from './launch-preparation.js';
|
||||
import { prepareNativeLaunch, evaluateNativePlacement, type PreparedNativeLaunch } from './launch-preparation.js';
|
||||
import { getServerKey, saveServerKey } from '../fleet/server-keys.js';
|
||||
import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
|
||||
import { nextLobbyNumber, type CreateServerInput, type CreateServerResult } from './instances.js';
|
||||
@@ -30,11 +30,12 @@ import { cancelCreation, finishCreationCancellation } from './cancel-creation.js
|
||||
|
||||
export type NativeSelection = import('../../hosts.js').NativePlacement;
|
||||
type NativeRecipe = LaunchRecipePlan & { native?: NativeSelection };
|
||||
interface Job { id: string; server: string; action: ProvisioningAction; state: string; submitted_at: string | null; stage: string;
|
||||
interface Job { selection?: string | null; id: string; server: string; action: ProvisioningAction; state: string; submitted_at: string | null; stage: string;
|
||||
instance_id: string; generation: number; stage_operation_id: string | null; operation_id: string | null; files_json: string | null; applied_at: string | null }
|
||||
interface FilePlan { id: number; path: string; contentBase64: string | null; sha256: string | null; checkBefore: boolean; expectedBeforeSha: string | null }
|
||||
|
||||
export function nativeSelection(input: Record<string, unknown>, game: string, options: Pick<PlacementOptions, 'planned'> = {}): NativeSelection {
|
||||
/** Validate a launch request without selecting a host or consuming capacity. */
|
||||
export function nativeLaunchSelection(input: Record<string, unknown>, game: string): NativeSelection {
|
||||
if (input.hostId !== undefined && typeof input.hostId !== 'string') throw new Error('Invalid host.');
|
||||
const libraryId = newestReadyLibraryId(game);
|
||||
if (!libraryId) throw new Error(`Upload ${game.toUpperCase()} game files before launching a server.`);
|
||||
@@ -44,16 +45,29 @@ export function nativeSelection(input: Record<string, unknown>, game: string, op
|
||||
const image = input.image === undefined || input.image === '' ? config.komodo.defaultImage : input.image;
|
||||
if (typeof image !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9._/:@-]{0,499}$/.test(image)) throw new Error('Invalid runtime image.');
|
||||
const automatic = input.automatic === true || !input.hostId;
|
||||
if (!automatic) {
|
||||
const host = getHost(input.hostId as string);
|
||||
if (!host) throw new Error('The selected host was not found.');
|
||||
if (host.decommission) throw new Error('The selected host is being removed. Choose another host.');
|
||||
}
|
||||
const rawNetwork = input.automatic === true ? input.network ?? {} : input;
|
||||
if (!rawNetwork || typeof rawNetwork !== 'object' || Array.isArray(rawNetwork)) throw new Error('Invalid network overrides.');
|
||||
const network = rawNetwork as NativeSelection;
|
||||
const bindAddress = network.bindAddress || undefined;
|
||||
if (bindAddress !== undefined && (typeof bindAddress !== 'string' || !isIP(bindAddress))) throw new Error('Invalid bind address.');
|
||||
const { host } = choosePlacement({ hostId: automatic ? undefined : input.hostId as string, libraryId, image, bindAddress, ...options });
|
||||
if (network.publicAddress !== undefined && (typeof network.publicAddress !== 'string' || network.publicAddress.length > 253)) throw new Error('Invalid public address.');
|
||||
if (network.publicPort !== undefined && (!Number.isInteger(network.publicPort) || Number(network.publicPort) < 1 || Number(network.publicPort) > 65535)) throw new Error('Invalid public port.');
|
||||
return { hostId: host.id, libraryId, image, bindAddress: bindAddress ?? host.bindAddress, publicAddress: network.publicAddress ?? host.publicAddress,
|
||||
publicPort: network.publicPort, automatic, network: { bindAddress, publicAddress: network.publicAddress, publicPort: network.publicPort } };
|
||||
return { hostId: automatic ? undefined : input.hostId as string, libraryId, image, automatic,
|
||||
bindAddress, publicAddress: network.publicAddress, publicPort: network.publicPort,
|
||||
network: { bindAddress, publicAddress: network.publicAddress, publicPort: network.publicPort } };
|
||||
}
|
||||
|
||||
export function nativeSelection(input: Record<string, unknown>, game: string, options: Pick<PlacementOptions, 'planned'> = {}): NativeSelection {
|
||||
const selection = nativeLaunchSelection(input, game);
|
||||
const { host } = choosePlacement({ hostId: selection.hostId, libraryId: selection.libraryId,
|
||||
image: selection.image!, bindAddress: selection.bindAddress, ...options });
|
||||
return { ...selection, hostId: host.id, bindAddress: selection.bindAddress ?? host.bindAddress,
|
||||
publicAddress: selection.publicAddress ?? host.publicAddress };
|
||||
}
|
||||
|
||||
export function previewNativeCreation(input: Record<string, unknown>): LaunchPreview {
|
||||
@@ -79,7 +93,7 @@ export function previewNativeCreation(input: Record<string, unknown>): LaunchPre
|
||||
db().prepare('INSERT INTO server_launch_previews(id,expires_at,server,port,recipe,blueprint_name) VALUES(?,?,?,?,?,?)')
|
||||
.run(id, expiresAt, name, port, JSON.stringify(recipe), blueprintName);
|
||||
const checks: LaunchPreview['checks'] = [
|
||||
{ label: 'Host capacity', state: 'pass', detail: selection.automatic ? 'Capacity available. The least loaded eligible host is selected when provisioning starts.' : `${getHost(selection.hostId)!.name}: capacity available.` },
|
||||
{ label: 'Host capacity', state: 'pass', detail: selection.automatic ? 'Capacity available. The least loaded eligible host is selected when provisioning starts.' : `${getHost(selection.hostId!)!.name}: capacity available.` },
|
||||
{ label: 'Game files', state: 'pass', detail: `The latest ${recipe.game.toUpperCase()} game files are verified on an eligible host.` },
|
||||
{ label: 'Lobby', state: name ? 'pass' : 'blocked', detail: name ?? 'All lobby IDs for this map are reserved.' },
|
||||
{ label: 'Host port', state: port ? 'pass' : 'blocked', detail: port ? selection.automatic ? 'UDP/TCP capacity available; port assigned at placement.' : `${selection.bindAddress}:${port} / UDP and TCP` : 'No available UDP/TCP port on this host.' },
|
||||
@@ -90,7 +104,7 @@ export function previewNativeCreation(input: Record<string, unknown>): LaunchPre
|
||||
}
|
||||
|
||||
export function nextNativePort(selection: NativeSelection, taken: number[] = []): number | null {
|
||||
const host = getHost(selection.hostId);
|
||||
const host = selection.hostId ? getHost(selection.hostId) : null;
|
||||
return host ? availableHostPort(host, { bindAddress: selection.bindAddress,
|
||||
planned: taken.map(port => ({ hostId: host.id, port })) }) : null;
|
||||
}
|
||||
@@ -99,42 +113,51 @@ export function queueNativeCreation(input: CreateServerInput): CreateServerResul
|
||||
try {
|
||||
return db().transaction(() => {
|
||||
const preview = input.previewId ? db().prepare('SELECT * FROM server_launch_previews WHERE id=?').get(input.previewId) as
|
||||
{ id: string; recipe: string; server: string; port: number; expires_at: string; deployment_id: string | null; blueprint_name: string | null } | undefined : undefined;
|
||||
{ id: string; recipe: string; server: string; port: number | null; expires_at: string; deployment_id: string | null; blueprint_name: string | null } | undefined : undefined;
|
||||
if (input.previewId && !preview) throw new Error('Review a fresh launch preview.');
|
||||
if (preview?.deployment_id) return { ok: true as const, name: preview.server, port: preview.port, deploymentId: preview.deployment_id };
|
||||
if (preview && preview.expires_at <= new Date().toISOString()) throw new Error('Launch preview expired.');
|
||||
const saved = preview ? JSON.parse(preview.recipe) as NativeRecipe : undefined;
|
||||
const game = input.game ?? 't5';
|
||||
if (input.gameMode !== 'zombie' || (saved && (saved.game !== game || saved.mapSlug !== input.mapSlug))) throw new Error('Selection changed. Review a fresh launch preview.');
|
||||
const selection = nativeSelection((saved?.native ?? input) as unknown as Record<string, unknown>, game);
|
||||
const recipe: NativeRecipe = { ...captureRecipePlan(game, input.mapSlug), native: selection, image: selection.image! };
|
||||
const selection = nativeLaunchSelection((saved?.native ?? input) as unknown as Record<string, unknown>, game);
|
||||
const lobby = nextLobbyNumber(`${game}_zombie_${input.mapSlug}`);
|
||||
if (lobby === null) throw new Error('All lobby IDs are reserved.');
|
||||
const name = `${game}_zombie_${input.mapSlug}_${lobby}`;
|
||||
const port = nextNativePort(selection);
|
||||
if (!port) throw new Error('No available host port.');
|
||||
if ((input.expectedName && input.expectedName !== name) || (preview && (preview.server !== name || (!selection.automatic && preview.port !== port)))) throw new Error('Available lobby or host port changed. Review a fresh preview.');
|
||||
const key = input.plutoKey?.trim() || getServerKey(name);
|
||||
if (!key) throw new Error(`Save a Plutonium server key for ${name}.`);
|
||||
saveServerKey(name, key);
|
||||
const instanceId = randomUUID();
|
||||
reserveHostPort(selection.hostId, instanceId, { port, bindAddress: selection.bindAddress, protocol: 'udp' });
|
||||
reserveHostPort(selection.hostId, instanceId, { port, bindAddress: selection.bindAddress, protocol: 'tcp' });
|
||||
db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,bind_address,port,public_address,public_port,created_at)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?)`).run(instanceId, name, selection.hostId, selection.libraryId, selection.image, selection.bindAddress, port,
|
||||
selection.publicAddress ?? null, selection.publicPort ?? null, new Date().toISOString());
|
||||
const id = beginDeployment(name, 'create', input.createdBy);
|
||||
if ((input.expectedName && input.expectedName !== name) || (preview && preview.server !== name)) throw new Error('Available lobby changed. Review a fresh preview.');
|
||||
const result = queueNativeCreationIntent({ server: name, game, mapSlug: input.mapSlug, selection,
|
||||
plutoKey: input.plutoKey, createdBy: input.createdBy, blueprintName: preview?.blueprint_name,
|
||||
recipe: saved ?? captureRecipePlan(game, input.mapSlug) });
|
||||
if (result.ok && preview) db().prepare('UPDATE server_launch_previews SET deployment_id=?,port=NULL WHERE id=?').run(result.deploymentId, preview.id);
|
||||
return result;
|
||||
})();
|
||||
} catch (error) { return { ok: false, kind: 'bad-request', message: (error as Error).message }; }
|
||||
}
|
||||
|
||||
/** Persist the reviewed logical identity; all host/port/resource decisions belong to provisioning. */
|
||||
export function queueNativeCreationIntent(input: { server: string; game: 't5' | 't6'; mapSlug: string; selection: NativeSelection;
|
||||
plutoKey?: string; createdBy?: string; blueprintName?: string | null; recipe?: LaunchRecipePlan }): CreateServerResult {
|
||||
try {
|
||||
return db().transaction(() => {
|
||||
const expected = `${input.game}_zombie_${input.mapSlug}_${nextLobbyNumber(`${input.game}_zombie_${input.mapSlug}`)}`;
|
||||
if (input.server !== expected || nativeInstance(input.server)) throw new Error('Available lobby changed. Review a fresh preview.');
|
||||
const key = input.plutoKey?.trim() || getServerKey(input.server);
|
||||
if (!key) throw new Error(`Save a Plutonium server key for ${input.server}.`);
|
||||
saveServerKey(input.server, key);
|
||||
const recipe: NativeRecipe = { ...(input.recipe ?? captureRecipePlan(input.game, input.mapSlug)),
|
||||
native: input.selection, image: input.selection.image! };
|
||||
const id = beginDeployment(input.server, 'create', input.createdBy ?? 'ui');
|
||||
db().prepare("UPDATE server_deployments SET state='queued' WHERE id=?").run(id);
|
||||
db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,?,?,?,?)')
|
||||
.run(id, name, port, 'zombie', recipe.mapSlug, recipe.expectedMap);
|
||||
db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,NULL,?,?,?)')
|
||||
.run(id, input.server, 'zombie', input.mapSlug, recipe.expectedMap);
|
||||
db().prepare('INSERT INTO native_creation_intents(deployment_id,selection) VALUES(?,?)').run(id, JSON.stringify(input.selection));
|
||||
const stored = recipeStorage(recipe);
|
||||
db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary,blueprint_name) VALUES(?,?,?,?)')
|
||||
.run(id, stored.recipe, stored.recipeSummary, preview?.blueprint_name ?? null);
|
||||
if (preview) db().prepare('UPDATE server_launch_previews SET deployment_id=?,port=? WHERE id=?').run(id, port, preview.id);
|
||||
deploymentStage(id, 'Waiting for provisioning evaluation', { phase: 'eligibility', recipeStatus: 'queued' });
|
||||
return { ok: true as const, name, port, deploymentId: id };
|
||||
.run(id, stored.recipe, stored.recipeSummary, input.blueprintName ?? null);
|
||||
deploymentStage(id, 'Awaiting placement', { phase: 'eligibility', recipeStatus: 'queued' });
|
||||
return { ok: true as const, name: input.server, port: null, deploymentId: id };
|
||||
})();
|
||||
} catch (error) { return { ok: false, kind: error instanceof CapacityError ? 'conflict' : 'bad-request', message: (error as Error).message }; }
|
||||
} catch (error) { return { ok: false, kind: 'bad-request', message: (error as Error).message }; }
|
||||
}
|
||||
|
||||
export function bindNativeOperation(id: string, server: string): void {
|
||||
@@ -144,8 +167,11 @@ export function bindNativeOperation(id: string, server: string): void {
|
||||
|
||||
/** Cancelling an unsubmitted launch releases only its local reservation. */
|
||||
export function cancelQueuedNativeCreation(server: string, triggeredBy: string): string | null {
|
||||
if (getServer(server)) return null;
|
||||
const unplaced = unplacedNativeCreation(server);
|
||||
if (unplaced) return cancelCreation(unplaced, triggeredBy);
|
||||
const instance = nativeInstance(server);
|
||||
if (!instance || getServer(server)) return null;
|
||||
if (!instance) return null;
|
||||
return db().transaction(() => {
|
||||
const creation = db().prepare(`SELECT d.id FROM server_deployments d JOIN native_provisioning n ON n.deployment_id=d.id
|
||||
WHERE n.instance_id=? AND d.action='create' AND d.state='queued' AND n.operation_id IS NULL`).get(instance.id) as { id: string } | undefined;
|
||||
@@ -243,9 +269,10 @@ export async function processNativeProvisioning(eligibility: (operation: { id: s
|
||||
deploymentStage(id, 'Host draining; waiting for a fresh empty lobby');
|
||||
}
|
||||
}
|
||||
const jobs = db().prepare(`SELECT d.*,n.instance_id,n.generation,n.stage_operation_id,n.operation_id,n.files_json,n.applied_at
|
||||
FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
|
||||
WHERE d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') ORDER BY d.started_at,d.rowid`).all() as Job[];
|
||||
const jobs = db().prepare(`SELECT d.*,n.instance_id,n.generation,n.stage_operation_id,n.operation_id,n.files_json,n.applied_at,q.selection
|
||||
FROM server_deployments d LEFT JOIN native_provisioning n ON n.deployment_id=d.id
|
||||
LEFT JOIN native_creation_intents q ON q.deployment_id=d.id
|
||||
WHERE (n.deployment_id IS NOT NULL OR q.deployment_id IS NOT NULL) AND d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') ORDER BY d.started_at,d.rowid`).all() as Job[];
|
||||
// One host's observation must not prevent another job being admitted.
|
||||
// Per-job ownership also covers overlapping direct calls and worker passes.
|
||||
const work = jobs.map(job => {
|
||||
@@ -254,7 +281,7 @@ export async function processNativeProvisioning(eligibility: (operation: { id: s
|
||||
const work = trackActivity(`provisioning:${job.id}`, `Evaluating ${job.action} for ${job.server}`,
|
||||
() => processJob(job, eligibility))
|
||||
.catch(error => { if (stillActive(job)) hold(job, (error as Error).message); })
|
||||
.finally(() => { activeJobs.delete(job.id); });
|
||||
.finally(() => { activeJobs.delete(job.id); preparingJobs.delete(job.id); });
|
||||
activeJobs.set(job.id, work);
|
||||
return work;
|
||||
});
|
||||
@@ -281,27 +308,47 @@ function sameLaunchFiles(first: LaunchRecipePlan, second: LaunchRecipePlan): boo
|
||||
return JSON.stringify(selected(first)) === JSON.stringify(selected(second));
|
||||
}
|
||||
|
||||
/** Reservations can move only while no remote work has ever been submitted. */
|
||||
function placeQueuedCreation(job: Job, instance: NativeInstance): NativeInstance {
|
||||
/** Capacity history and host ranking run in the background reader, one job at a time. */
|
||||
async function placeQueuedCreation(job: Job, instance: NativeInstance | null): Promise<NativeInstance | null> {
|
||||
const recipe = recipeForDeployment(job.id)!.recipe as NativeRecipe;
|
||||
const prior: NativeSelection = job.selection ? JSON.parse(job.selection) : recipe.native!;
|
||||
const network = prior.automatic ? prior.network ?? {} : prior;
|
||||
const options: PlacementOptions = { hostId: prior.automatic ? undefined : prior.hostId,
|
||||
libraryId: prior.libraryId, image: prior.image ?? recipe.image, bindAddress: network.bindAddress,
|
||||
thorough: true, dispatch: true, excludeInstanceId: instance?.id };
|
||||
const decision = await evaluateNativePlacement(options);
|
||||
return db().transaction(() => {
|
||||
const recipe = recipeForDeployment(job.id)!.recipe as NativeRecipe;
|
||||
const prior = recipe.native!;
|
||||
const network = prior.automatic ? prior.network ?? {} : prior;
|
||||
const { host, port } = choosePlacement({ hostId: prior.automatic ? undefined : instance.host_id,
|
||||
libraryId: instance.library_id, image: instance.image, bindAddress: network.bindAddress,
|
||||
thorough: true, dispatch: true, excludeInstanceId: instance.id });
|
||||
const state = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined;
|
||||
if (state?.state !== 'queued') return null;
|
||||
const live = nativeInstance(job.server);
|
||||
if (instance ? live?.id !== instance.id || live.generation !== instance.generation : !!live || !!getServer(job.server)) return null;
|
||||
const active = (db().prepare(`SELECT count(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL
|
||||
AND action IN ('create','rebuild') AND state IN ('running','awaiting-health')`).get(job.id) as { n: number }).n;
|
||||
if (active + otherPreparations(job.id) >= fleetSettings().createConcurrency) { hold(job, 'Waiting for startup capacity.'); return null; }
|
||||
if (getHost(decision.hostId)?.lastHeartbeat !== decision.lastHeartbeat) { hold(job, 'Host measurements changed; waiting for another placement check.'); return null; }
|
||||
// A short current-state check protects against reservations made while the
|
||||
// reader ranked hosts. It never scans resource history or contacts agents.
|
||||
const { host, port } = choosePlacement({ ...options, hostId: decision.hostId, thorough: false });
|
||||
const bind = network.bindAddress ?? host.bindAddress;
|
||||
releaseHostPorts(instance.id);
|
||||
reserveHostPort(host.id, instance.id, { port, bindAddress: bind, protocol: 'udp' });
|
||||
reserveHostPort(host.id, instance.id, { port, bindAddress: bind, protocol: 'tcp' });
|
||||
db().prepare(`UPDATE native_instances SET host_id=?,bind_address=?,port=?,public_address=?,public_port=? WHERE id=?`)
|
||||
.run(host.id, bind, port, network.publicAddress ?? host.publicAddress, network.publicPort ?? null, instance.id);
|
||||
const instanceId = instance?.id ?? randomUUID();
|
||||
if (instance) releaseHostPorts(instanceId);
|
||||
reserveHostPort(host.id, instanceId, { port, bindAddress: bind, protocol: 'udp' });
|
||||
reserveHostPort(host.id, instanceId, { port, bindAddress: bind, protocol: 'tcp' });
|
||||
if (instance) db().prepare(`UPDATE native_instances SET host_id=?,bind_address=?,port=?,public_address=?,public_port=? WHERE id=?`)
|
||||
.run(host.id, bind, port, network.publicAddress ?? host.publicAddress, network.publicPort ?? null, instanceId);
|
||||
else {
|
||||
db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,bind_address,port,public_address,public_port,created_at)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?)`).run(instanceId, job.server, host.id, prior.libraryId, options.image, bind, port,
|
||||
network.publicAddress ?? host.publicAddress, network.publicPort ?? null, new Date().toISOString());
|
||||
db().prepare('INSERT INTO native_provisioning(deployment_id,instance_id,generation) VALUES(?,?,1)').run(job.id, instanceId);
|
||||
}
|
||||
db().prepare('UPDATE server_creation_jobs SET port=? WHERE deployment_id=?').run(port, job.id);
|
||||
recipe.native = { ...prior, hostId: host.id, bindAddress: bind, publicAddress: network.publicAddress ?? host.publicAddress, publicPort: network.publicPort };
|
||||
const stored = recipeStorage(recipe);
|
||||
db().prepare('UPDATE server_launch_recipes SET recipe=?,recipe_summary=? WHERE deployment_id=?').run(stored.recipe, stored.recipeSummary, job.id);
|
||||
db().prepare('UPDATE server_launch_previews SET port=? WHERE deployment_id=?').run(port, job.id);
|
||||
deploymentStage(job.id, `Placed on ${host.name} after CPU, memory, history and port checks`, { phase: 'eligibility' });
|
||||
preparingJobs.add(job.id);
|
||||
return nativeInstance(job.server)!;
|
||||
})();
|
||||
}
|
||||
@@ -309,15 +356,24 @@ function placeQueuedCreation(job: Job, instance: NativeInstance): NativeInstance
|
||||
async function processJob(job: Job, eligibility: (job: Job) => string | null) {
|
||||
if (!stillActive(job)) return;
|
||||
let instance = nativeInstance(job.server);
|
||||
if (!instance || instance.id !== job.instance_id || (instance.generation !== job.generation && !(job.action === 'rebuild' && instance.generation === job.generation + 1))) {
|
||||
finishDeployment(job.id, { ok: false, message: 'Original instance was retired or replaced; no command was sent.' }); return;
|
||||
}
|
||||
if (job.action === 'create' && !job.operation_id && !job.stage_operation_id && !job.submitted_at) {
|
||||
// A resumed bound job owns its original identity. An unplaced intent
|
||||
// cannot adopt an instance or registered server that appeared later.
|
||||
if (job.instance_id ? !instance || instance.id !== job.instance_id || instance.generation !== job.generation
|
||||
: !!instance || !!getServer(job.server)) {
|
||||
finishDeployment(job.id, { ok: false, message: 'Original instance was retired or replaced; no command was sent.' }); return;
|
||||
}
|
||||
const settings = fleetSettings();
|
||||
const active = db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL
|
||||
AND action IN ('create','rebuild') AND state IN ('running','awaiting-health')`).get(job.id) as { n: number };
|
||||
if (active.n + otherPreparations(job.id) >= settings.createConcurrency) { hold(job, 'Waiting for startup capacity.'); return; }
|
||||
instance = placeQueuedCreation(job, instance);
|
||||
if (eligibility(job)) return;
|
||||
instance = await placeQueuedCreation(job, instance);
|
||||
if (!instance) return;
|
||||
job.instance_id = instance.id; job.generation = instance.generation;
|
||||
}
|
||||
if (!instance || instance.id !== job.instance_id || (instance.generation !== job.generation && !(job.action === 'rebuild' && instance.generation === job.generation + 1))) {
|
||||
finishDeployment(job.id, { ok: false, message: 'Original instance was retired or replaced; no command was sent.' }); return;
|
||||
}
|
||||
const host = getHost(instance.host_id);
|
||||
if (!host?.connected || host.mode === 'disabled') {
|
||||
@@ -448,7 +504,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
|
||||
if (job.action === 'destroy' && count(['destroy']) >= settings.destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; }
|
||||
deploymentStage(job.id, 'Checking host eligibility', { phase: 'eligibility', blocker: null });
|
||||
if (job.action === 'create') choosePlacement({ hostId: instance.host_id, libraryId: instance.library_id, image: instance.image,
|
||||
bindAddress: instance.bind_address, excludeInstanceId: instance.id, thorough: true, dispatch: true });
|
||||
bindAddress: instance.bind_address, excludeInstanceId: instance.id, thorough: false, dispatch: true });
|
||||
if (job.action === 'create' || job.action === 'rebuild') assertReadyLibrary(instance.host_id, instance.library_id);
|
||||
const checkOccupancy = async (): Promise<boolean> => {
|
||||
// Fast controller check keeps occupied lobbies out of the operation slots.
|
||||
@@ -514,7 +570,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
|
||||
}
|
||||
assertReadyLibrary(instance!.host_id, instance!.library_id);
|
||||
if (job.action === 'create') choosePlacement({ hostId: instance!.host_id, libraryId: instance!.library_id, image: instance!.image,
|
||||
bindAddress: instance!.bind_address, excludeInstanceId: instance!.id, thorough: true, dispatch: true });
|
||||
bindAddress: instance!.bind_address, excludeInstanceId: instance!.id, thorough: false, dispatch: true });
|
||||
}
|
||||
if (job.action === 'start' || job.action === 'restart' || job.action === 'rebuild') {
|
||||
db().prepare(`INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES(?,?,?) ON CONFLICT(server)
|
||||
|
||||
@@ -7,7 +7,7 @@ import { queueRebuildServer } from './instances.js';
|
||||
import { restartRequestInFlight } from './restarts.js';
|
||||
import { runtimeActions, type RuntimeAction } from '../../provisioning.js';
|
||||
import { recordProgress } from './progress.js';
|
||||
import { nativeInstance } from '../backends/native.js';
|
||||
import { nativeInstance, unplacedNativeCreation } from '../backends/native.js';
|
||||
|
||||
export interface LifecycleOperation {
|
||||
id: string; server: string; action: string; state: string; stage: string;
|
||||
@@ -137,7 +137,7 @@ export function releaseRestartRequest(server: string, source: RestartSource): vo
|
||||
|
||||
export function requestLifecycle(server: string, action: RuntimeAction) {
|
||||
if (!runtimeActions.includes(action)) return { ok: false as const, status: 400, error: 'Choose start, restart, rebuild, or stop.' };
|
||||
if (!nativeInstance(server) && (!komodoConfigured() || !rconConfigured())) return { ok: false as const, status: 503, error: 'Configure a host and RCON before provisioning.' };
|
||||
if (!nativeInstance(server) && !unplacedNativeCreation(server) && (!komodoConfigured() || !rconConfigured())) return { ok: false as const, status: 503, error: 'Configure a host and RCON before provisioning.' };
|
||||
return db().transaction(() => {
|
||||
const blocker = action === 'rebuild' ? activeBetaRemovalBlocker(server) : null;
|
||||
if (blocker) return { ok: false as const, status: 409, error: blocker };
|
||||
|
||||
Reference in New Issue
Block a user