josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Queue native collection launches before async placement

Browse Source
This commit is contained in:
josh
2026-10-05 11:35:50 -04:00
parent 1405114a14
commit 7c76851252
34 changed files with 1324 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/fleet/collections.ts
+76 -47
View File
@@ -4,18 +4,18 @@ import type { CollectionDetail, CollectionMode, CollectionPreview, CollectionSum
import { db } from '../db/index.js';
import { config, komodoConfigured, rconConfigured } from '../config.js';
import { blueprintEntries, summarizeRecipe, type BlueprintEntry } from './launch-recipes.js';
import { creationQueue, destroyServer, nextAvailablePort, nextLobbyNumber, queueCreateServer } from '../provisioning/index.js';
import { creationQueue, destroyServer, nextAvailablePort, queueCreateServer } from '../provisioning/index.js';
import { deploymentStage, finishDeployment, recordDeploymentEvent } from './deployments.js';
import { housePartyTargets } from './house-party.js';
import { getServerKey } from './server-keys.js';
import { serverObservation } from './status.js';
import { activeBetaRemovalBlocker, getServer } from './servers.js';
import { parseServerName, zombieMapBySlug, zombieMaps } from './map-aliases.js';
import { nativeSelection, nextNativePort } from '../provisioning/native.js';
import { nativeLaunchSelection, queueNativeCreationIntent } from '../provisioning/native.js';
import { listHosts } from '../hosts/index.js';
import { nativeInstance } from '../backends/native.js';
import { nativeDeployment, nativeInstance } from '../backends/native.js';
import { launchAvailability } from '../hosts/availability.js';
function coverage(entries: BlueprintEntry[]) {
function coverage(entries: BlueprintEntry[], includeHealth = true) {
type Server = { name: string; game: string | null; map_slug: string | null; map: string | null };
const registered = db().prepare('SELECT name, game, map_slug, map FROM servers WHERE destroyed_at IS NULL').all() as Server[];
const queued = db().prepare(`SELECT j.server AS name, substr(j.server, 1, 2) AS game, j.map_slug, j.expected_map AS map
@@ -37,14 +37,14 @@ function coverage(entries: BlueprintEntry[]) {
const matches = zombieMaps(game).filter(map => map.canonical === server.map);
return matches.length === 1 && matches[0].slug === entry.recipe.mapSlug;
}).map(server => server.name).sort();
const coverage = existingServers.map(name => {
const coverage = includeHealth ? existingServers.map(name => {
const operation = db().prepare("SELECT id, state FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('create','rebuild') AND state IN ('queued', 'running', 'awaiting-health') ORDER BY rowid DESC LIMIT 1").get(name) as { id: string; state: string } | undefined;
const server = getServer(name);
return { name, health: operation ? operation.state === 'queued' ? 'queued' : 'starting'
: server?.container_state && ['stopped', 'exited', 'created', 'dead'].includes(server.container_state) ? 'stopped'
: server ? serverObservation(server).health : 'offline',
href: operation ? `/provisioning/${encodeURIComponent(operation.id)}` : `/servers/${encodeURIComponent(name)}` };
});
}) : [];
return { existingServers, coverage, existing: existingServers.length, missing: Math.max(0, entry.count - existingServers.length) };
});
}
@@ -57,8 +57,8 @@ function fleetSnapshot() {
}));
}
function collectionPlan(entries: BlueprintEntry[], mode: CollectionMode) {
const counts = coverage(entries);
function collectionPlan(entries: BlueprintEntry[], mode: CollectionMode, includeHealth = true) {
const counts = coverage(entries, includeHealth);
// Preserve deployed lobbies before queued ones, then the lowest lobby IDs.
const retainedServers = counts.flatMap((entry, index) => [...entry.existingServers].sort((a, b) =>
Number(!!getServer(b)) - Number(!!getServer(a)) || a.localeCompare(b, 'en', { numeric: true })
@@ -71,50 +71,56 @@ function collectionPlan(entries: BlueprintEntry[], mode: CollectionMode) {
return { counts, retainedServers, removals };
}
function allocate(entries: BlueprintEntry[], releasingServers: string[] = []): CollectionPreview['servers'] {
const names: string[] = [];
function allocate(entries: BlueprintEntry[], releasingServers: string[] = [], counts = coverage(entries)): CollectionPreview['servers'] {
const occupied = db().prepare(`SELECT name FROM servers UNION
SELECT j.server AS name FROM server_creation_jobs j JOIN server_deployments d ON d.id=j.deployment_id
WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued','running','awaiting-health')
OR (d.action='rebuild' AND (d.state='waiting' OR (d.state IN ('failed','interrupted') AND d.stage='Removing server files and registry'))))`)
.all() as { name: string }[];
const lobbies = new Map<string, Set<number>>();
for (const { name } of occupied) {
const match = /^(.*)_(\d+)$/.exec(name);
if (!match) continue;
const taken = lobbies.get(match[1]) ?? new Set<number>();
taken.add(Number(match[2])); lobbies.set(match[1], taken);
}
const savedKeys = new Set((db().prepare("SELECT server_id FROM server_keys WHERE length(trim(pluto_key))>0").all() as { server_id: string }[]).map(row => row.server_id));
const ports: number[] = [];
const planned: { hostId: string; port: number }[] = [];
const counts = coverage(entries);
return entries.flatMap((entry, entryIndex) => Array.from({ length: counts[entryIndex].missing }, () => {
const base = `${entry.recipe.game}_zombie_${entry.recipe.mapSlug}`;
const lobby = nextLobbyNumber(base, names);
const taken = lobbies.get(base) ?? new Set<number>();
const lobby = Array.from({ length: 20 }, (_, index) => index + 1).find(number => !taken.has(number)) ?? null;
const name = lobby === null ? null : `${base}_${lobby}`;
let native = entry.recipe.native;
let port: number | null;
if (native) {
try {
native = nativeSelection(native as unknown as Record<string, unknown>, entry.recipe.game, { planned });
port = nextNativePort(native, planned.filter(item => item.hostId === native!.hostId).map(item => item.port));
if (port !== null) planned.push({ hostId: native.hostId, port });
} catch { port = null; }
} else port = nextAvailablePort(ports, releasingServers);
if (name) names.push(name);
const native = entry.recipe.native;
const port = native ? null : nextAvailablePort(ports, releasingServers);
if (lobby !== null) { taken.add(lobby); lobbies.set(base, taken); }
if (port !== null) ports.push(port);
return { entryIndex, name, port, ...(native ? { native } : {}), hasSavedKey: !!name && !!getServerKey(name) };
return { entryIndex, name, port, ...(native ? { native } : {}), hasSavedKey: !!name && savedKeys.has(name) };
}));
}
function blockersFor(servers: CollectionPreview['servers'], entries: BlueprintEntry[] = []): string[] {
const blockers: string[] = [];
if (!servers.length) return blockers;
if (entries.some(entry => entry.recipe.native)) {
for (const entry of entries) {
try { nativeSelection(entry.recipe.native as unknown as Record<string, unknown>, entry.recipe.game); }
catch (error) { blockers.push((error as Error).message); }
}
} else {
const legacy = servers.filter(server => !server.native && !entries[server.entryIndex]?.recipe.native);
if (legacy.length) {
if (!komodoConfigured() || !config.komodo.targetServer) blockers.push('Select and prepare a host before launching.');
if (!rconConfigured()) blockers.push('Configure RCON_PASSWORD so every new server can be verified.');
}
if (servers.some(server => !server.name)) blockers.push('There are not enough free lobbies for one or more maps (20 per map). Reduce their counts or free capacity.');
if (servers.some(server => server.port === null)) blockers.push(entries.some(entry => entry.recipe.native) ? 'There is not enough host CPU, memory or port capacity for the whole collection.' : `There are not enough free game ports in ${config.portRange.min}–${config.portRange.max} for the whole collection.`);
if (legacy.some(server => server.port === null)) blockers.push(`There are not enough free game ports in ${config.portRange.min}–${config.portRange.max} for the whole collection.`);
return blockers;
}
function removalBlockersFor(removals: CollectionPreview['removals']): string[] {
const blockers: string[] = [];
if (removals.some(server => !nativeInstance(server.name)) && (!komodoConfigured() || !rconConfigured())) blockers.push('Configure the legacy backend before removing its servers.');
const native = (name: string) => {
if (nativeInstance(name)) return true;
const creation = db().prepare(`SELECT id FROM server_deployments WHERE server=? AND server_deleted_at IS NULL
AND action='create' AND state IN ('queued','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(name) as { id: string } | undefined;
return !!creation && nativeDeployment(creation.id);
};
if (removals.some(server => !native(server.name)) && (!komodoConfigured() || !rconConfigured())) blockers.push('Configure the legacy backend before removing its servers.');
for (const server of removals) {
const blocker = activeBetaRemovalBlocker(server.name);
if (blocker) blockers.push(blocker);
@@ -130,24 +136,26 @@ export function previewCollection(blueprintId: unknown, mode: unknown = 'fill',
if (selection?.hostId !== undefined || !config.komodo.address || listHosts().length > 0) {
const libraries = selection?.libraryIds as Record<string, unknown> | undefined;
for (const entry of entries) {
entry.recipe.native = nativeSelection({ ...selection, libraryId: libraries?.[entry.recipe.game] ?? selection?.libraryId }, entry.recipe.game);
entry.recipe.native = nativeLaunchSelection({ ...selection, libraryId: libraries?.[entry.recipe.game] ?? selection?.libraryId }, entry.recipe.game);
entry.recipe.image = entry.recipe.native.image!;
}
}
const { counts, retainedServers, removals } = collectionPlan(entries, mode);
const servers = allocate(entries, removals.map(server => server.name));
const servers = allocate(entries, removals.map(server => server.name), counts);
const blockers = [...blockersFor(servers, entries), ...removalBlockersFor(removals)];
const selections = servers.flatMap(server => server.native ? [server.native] : []);
const availability = selections.length ? launchAvailability(selections) : {};
const id = randomUUID();
const expiresAt = new Date(Date.now() + 30 * 60_000).toISOString();
db().prepare('DELETE FROM server_collection_previews WHERE expires_at < ? AND collection_id IS NULL').run(new Date().toISOString());
db().prepare('INSERT INTO server_collection_previews (id, expires_at, blueprint_name, entries, servers, mode, fleet_snapshot) VALUES (?, ?, ?, ?, ?, ?, ?)')
.run(id, expiresAt, blueprintName, JSON.stringify(entries), JSON.stringify(servers), mode, mode === 'exact' ? JSON.stringify(fleetSnapshot()) : null);
return { id, expiresAt, blueprintName, mode, retainedServers, removals, launchConcurrency: fleetSettings().createConcurrency, servers, blockers, ready: blockers.length === 0,
return { id, expiresAt, blueprintName, mode, retainedServers, removals, launchConcurrency: fleetSettings().createConcurrency, servers, blockers, ...availability, ready: blockers.length === 0,
entries: entries.map((entry, index) => ({ count: entry.count, ...counts[index], recipe: summarizeRecipe(entry.recipe) })) };
})();
}
/** Reserve and enqueue the entire collection atomically. No external operations here. */
/** Save all reviewed identities and keys atomically; native placement belongs to provisioning. */
export function queueCollection(input: { previewId?: unknown; keys?: unknown }): { ok: true; collectionId: string | null } {
if (typeof input.previewId !== 'string' || !input.previewId) throw new Error('Review a collection preview before launching.');
if (input.keys !== undefined && (!input.keys || typeof input.keys !== 'object' || Array.isArray(input.keys))) throw new Error('Provide keys indexed by the exact server ID.');
@@ -160,19 +168,35 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
if (preview.collection_id) return { ok: true as const, collectionId: preview.collection_id };
if (preview.expires_at <= new Date().toISOString()) throw new Error('Collection preview expired. Refresh it before launching.');
const entries: BlueprintEntry[] = JSON.parse(preview.entries);
// Older previews embedded plugin bytes. Child previews need only the
// Older previews embedded plugin bytes. Queued jobs need only the
// reviewed metadata; provisioning captures the current files at dispatch.
for (const entry of entries) entry.recipe.files = entry.recipe.files.map(({ content: _content, ...file }) => file);
const reviewed: CollectionPreview['servers'] = JSON.parse(preview.servers);
if (preview.mode === 'exact' && preview.fleet_snapshot !== JSON.stringify(fleetSnapshot())) {
throw new Error('The fleet changed. Nothing was queued. Refresh the preview and review the servers to keep and remove.');
}
const { retainedServers, removals } = collectionPlan(entries, preview.mode);
const { counts, retainedServers, removals } = collectionPlan(entries, preview.mode, false);
const releasingServers = removals.map(server => server.name);
const servers = allocate(entries, releasingServers);
const nativeReview = reviewed.every(server => server.native || entries[server.entryIndex]?.recipe.native);
// Native Apply accepts the reviewed logical identities. Capacity and
// physical ports can change while this request waits in the queue.
const servers = nativeReview ? reviewed : allocate(entries, releasingServers, counts);
const validatedSelections = new Set<string>();
for (const server of servers) {
const recipe = entries[server.entryIndex].recipe;
const selection = server.native ?? recipe.native;
if (!selection) continue;
const identity = JSON.stringify([recipe.game, selection]);
if (validatedSelections.has(identity)) continue;
// Recheck saved configuration once per selection without allocating
// physical capacity or changing the names and files already reviewed.
nativeLaunchSelection({ ...selection }, recipe.game);
validatedSelections.add(identity);
}
const blockers = [...blockersFor(servers, entries), ...removalBlockersFor(removals)];
if (blockers.length) throw new Error(blockers.join(' '));
if (servers.length !== reviewed.length || servers.some((server, i) => server.entryIndex !== reviewed[i]?.entryIndex || server.name !== reviewed[i]?.name || (!server.native?.automatic && server.port !== reviewed[i]?.port))) {
if (entries.some((_entry, index) => counts[index].missing !== reviewed.filter(server => server.entryIndex === index).length)
|| servers.length !== reviewed.length || servers.some((server, i) => server.entryIndex !== reviewed[i]?.entryIndex || server.name !== reviewed[i]?.name || (!server.native && server.port !== reviewed[i]?.port))) {
throw new Error('Fleet counts, lobby or port availability changed. Nothing was queued. Refresh the preview and review each server key.');
}
const expected = new Set(servers.map(server => server.name));
@@ -210,16 +234,21 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
});
servers.forEach((server, index) => {
const recipe = { ...entries[server.entryIndex].recipe, ...(server.native ? { native: server.native } : {}) };
const childPreviewId = randomUUID();
db().prepare('INSERT INTO server_launch_previews (id, expires_at, server, port, recipe, blueprint_name) VALUES (?, ?, ?, ?, ?, ?)')
.run(childPreviewId, preview.expires_at, server.name, server.port, JSON.stringify(recipe), preview.blueprint_name);
const result = queueCreateServer({ game: recipe.game, gameMode: 'zombie', mapSlug: recipe.mapSlug,
expectedName: server.name!, previewId: childPreviewId, plutoKey: keys[server.name!] as string | undefined, createdBy: 'ui' }, releasingServers);
let result;
if (recipe.native) result = queueNativeCreationIntent({ server: server.name!, game: recipe.game, mapSlug: recipe.mapSlug,
selection: recipe.native, recipe, plutoKey: keys[server.name!] as string | undefined, createdBy: 'ui', blueprintName: preview.blueprint_name });
else {
const childPreviewId = randomUUID();
db().prepare('INSERT INTO server_launch_previews (id, expires_at, server, port, recipe, blueprint_name) VALUES (?, ?, ?, ?, ?, ?)')
.run(childPreviewId, preview.expires_at, server.name, server.port, JSON.stringify(recipe), preview.blueprint_name);
result = queueCreateServer({ game: recipe.game, gameMode: 'zombie', mapSlug: recipe.mapSlug,
expectedName: server.name!, previewId: childPreviewId, plutoKey: keys[server.name!] as string | undefined, createdBy: 'ui' }, releasingServers);
}
// Throw so the outer transaction also reverses all earlier keys and children.
if (!result.ok) throw new Error(`${server.name}: ${result.message} Nothing was queued.`);
db().prepare('INSERT INTO server_collection_members (collection_id, deployment_id, position) VALUES (?, ?, ?)')
.run(collectionId, result.deploymentId, index);
for (const removal of removalJobs.filter(removal => removal.ports.includes(server.port))) {
for (const removal of recipe.native ? [] : removalJobs.filter(removal => removal.ports.includes(server.port))) {
db().prepare('INSERT INTO server_creation_dependencies (deployment_id, removal_id) VALUES (?, ?)').run(result.deploymentId, removal.id);
}
});