feat: match fleet to blueprint targets exactly
Add an optional exact blueprint mode with keep/create/remove previews, atomic queueing, cancellation of unstarted excess creations, and persistent dependencies for safe port reuse. Track removals alongside creations and preserve fill mode as the default. Validation: type check, production build, and 103 deployment tests passed. The full suite has one telemetry assertion failure also reproduced on the previous commit. Fixes #42
This commit is contained in:
@@ -32,11 +32,16 @@ export interface BlueprintSummary {
|
||||
totalServers: number;
|
||||
}
|
||||
|
||||
export type CollectionMode = 'fill' | 'exact';
|
||||
|
||||
export interface CollectionPreview {
|
||||
id: string;
|
||||
expiresAt: string;
|
||||
blueprintName: string;
|
||||
launchConcurrency: number;
|
||||
mode: CollectionMode;
|
||||
retainedServers: string[];
|
||||
removals: { name: string; reason: string }[];
|
||||
entries: { count: number; existing: number; missing: number; existingServers: string[];
|
||||
coverage: { name: string; health: string; href: string }[]; recipe: LaunchRecipeSummary }[];
|
||||
servers: { entryIndex: number; name: string | null; port: number | null; hasSavedKey: boolean }[];
|
||||
@@ -55,12 +60,14 @@ export interface CollectionSummary {
|
||||
queued: number;
|
||||
provisioning: number;
|
||||
verifying: number;
|
||||
mode: CollectionMode;
|
||||
}
|
||||
|
||||
export interface CollectionDetail {
|
||||
collection: CollectionSummary;
|
||||
retainedServers: string[];
|
||||
queue: { concurrency: number; active: { id: string; server: string; state: string; stage: string }[]; queued: number };
|
||||
servers: { id: string; server: string; port: number; map_slug: string; state: string; stage: string;
|
||||
servers: { id: string; server: string; action: 'create' | 'destroy'; port: number | null; map_slug: string | null; state: string; stage: string;
|
||||
started_at: string; finished_at: string | null; error: string | null; server_exists: number; queuePosition: number | null }[];
|
||||
}
|
||||
|
||||
|
||||
@@ -249,6 +249,14 @@ export function db(): Database.Database {
|
||||
migrateTable(handle, 'beta_scripts', BETA_SCRIPTS_COLUMNS);
|
||||
migrateTable(handle, 'deployment_scripts', DEPLOYMENT_SCRIPTS_COLUMNS);
|
||||
migrateTable(handle, 'deployments', DEPLOYMENTS_COLUMNS);
|
||||
migrateTable(handle, 'server_launch_collections', [
|
||||
{ name: 'mode', ddl: "ALTER TABLE server_launch_collections ADD COLUMN mode TEXT NOT NULL DEFAULT 'fill'" },
|
||||
{ name: 'retained_servers', ddl: "ALTER TABLE server_launch_collections ADD COLUMN retained_servers TEXT NOT NULL DEFAULT '[]'" }
|
||||
]);
|
||||
migrateTable(handle, 'server_collection_previews', [
|
||||
{ name: 'mode', ddl: "ALTER TABLE server_collection_previews ADD COLUMN mode TEXT NOT NULL DEFAULT 'fill'" },
|
||||
{ name: 'fleet_snapshot', ddl: 'ALTER TABLE server_collection_previews ADD COLUMN fleet_snapshot TEXT' }
|
||||
]);
|
||||
migrateTable(handle, 'server_telemetry', [{ name: 'payload', ddl: 'ALTER TABLE server_telemetry ADD COLUMN payload TEXT' }]);
|
||||
// Kind-referencing indexes: safe here because migrateTable above just
|
||||
// ALTER'd the column into existence if it was missing.
|
||||
|
||||
@@ -467,7 +467,9 @@ CREATE TABLE IF NOT EXISTS server_launch_recipes (
|
||||
CREATE TABLE IF NOT EXISTS server_launch_collections (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL
|
||||
created_at TEXT NOT NULL,
|
||||
mode TEXT NOT NULL DEFAULT 'fill',
|
||||
retained_servers TEXT NOT NULL DEFAULT '[]'
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS server_collection_members (
|
||||
@@ -483,9 +485,26 @@ CREATE TABLE IF NOT EXISTS server_collection_previews (
|
||||
blueprint_name TEXT NOT NULL,
|
||||
entries TEXT NOT NULL,
|
||||
servers TEXT NOT NULL,
|
||||
mode TEXT NOT NULL DEFAULT 'fill',
|
||||
fleet_snapshot TEXT,
|
||||
collection_id TEXT REFERENCES server_launch_collections(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
-- A pending removal can be shared by multiple reviewed collection requests.
|
||||
CREATE TABLE IF NOT EXISTS server_collection_removals (
|
||||
collection_id TEXT NOT NULL REFERENCES server_launch_collections(id) ON DELETE CASCADE,
|
||||
deployment_id TEXT NOT NULL REFERENCES server_deployments(id) ON DELETE CASCADE,
|
||||
position INTEGER NOT NULL,
|
||||
PRIMARY KEY (collection_id, deployment_id)
|
||||
);
|
||||
|
||||
-- Reusing a port is safe only after the reviewed removal has completed.
|
||||
CREATE TABLE IF NOT EXISTS server_creation_dependencies (
|
||||
deployment_id TEXT NOT NULL REFERENCES server_deployments(id) ON DELETE CASCADE,
|
||||
removal_id TEXT NOT NULL REFERENCES server_deployments(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (deployment_id, removal_id)
|
||||
);
|
||||
|
||||
-- Persist state transitions atomically, including changes made by background workers.
|
||||
-- Polls with unchanged status/error do not produce duplicate activity entries.
|
||||
CREATE TABLE IF NOT EXISTS beta_events (
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import type { CollectionDetail, CollectionPreview, CollectionSummary } from '$lib/deployment-plan.js';
|
||||
import type { CollectionDetail, CollectionMode, CollectionPreview, CollectionSummary } from '$lib/deployment-plan.js';
|
||||
import { db } from '../db/index.js';
|
||||
import { config, komodoConfigured, rconConfigured } from '../config.js';
|
||||
import { blueprintEntries, summarizeRecipe, type BlueprintEntry } from './launch-recipes.js';
|
||||
import { creationQueue, nextAvailablePort, nextLobbyNumber, queueCreateServer } from './create.js';
|
||||
import { creationQueue, destroyServer, nextAvailablePort, nextLobbyNumber, queueCreateServer } from './create.js';
|
||||
import { deploymentStage, finishDeployment, recordDeploymentEvent } from './deployments.js';
|
||||
import { housePartyTargets } from './house-party.js';
|
||||
import { getServerKey } from './server-keys.js';
|
||||
import { classifyHealth } from './health.js';
|
||||
import { getServer } from './servers.js';
|
||||
@@ -43,7 +45,29 @@ function coverage(entries: BlueprintEntry[]) {
|
||||
});
|
||||
}
|
||||
|
||||
function allocate(entries: BlueprintEntry[]): CollectionPreview['servers'] {
|
||||
function fleetSnapshot() {
|
||||
return housePartyTargets().map(name => ({ name,
|
||||
server: db().prepare('SELECT game, map_slug, map, game_port FROM servers WHERE name = ?').get(name) ?? null,
|
||||
creation: db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'create' ORDER BY rowid DESC LIMIT 1").get(name) ?? null,
|
||||
removal: db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) ?? null
|
||||
}));
|
||||
}
|
||||
|
||||
function collectionPlan(entries: BlueprintEntry[], mode: CollectionMode) {
|
||||
const counts = coverage(entries);
|
||||
// Preserve deployed lobbies before queued ones, then the lowest lobby IDs.
|
||||
const retainedServers = counts.flatMap((entry, index) => [...entry.existingServers].sort((a, b) =>
|
||||
Number(!!getServer(b)) - Number(!!getServer(a)) || a.localeCompare(b, 'en', { numeric: true })
|
||||
).slice(0, mode === 'exact' ? entries[index].count : undefined));
|
||||
const retained = new Set(retainedServers);
|
||||
const matching = new Set(counts.flatMap(entry => entry.existingServers));
|
||||
const removals = mode === 'exact' ? housePartyTargets().filter(name => !retained.has(name)).map(name => ({ name,
|
||||
reason: matching.has(name) ? 'Exceeds the target lobby count' : 'Outside this blueprint or already awaiting removal'
|
||||
})) : [];
|
||||
return { counts, retainedServers, removals };
|
||||
}
|
||||
|
||||
function allocate(entries: BlueprintEntry[], releasingServers: string[] = []): CollectionPreview['servers'] {
|
||||
const names: string[] = [];
|
||||
const ports: number[] = [];
|
||||
const counts = coverage(entries);
|
||||
@@ -51,7 +75,7 @@ function allocate(entries: BlueprintEntry[]): CollectionPreview['servers'] {
|
||||
const base = `${entry.recipe.game}_zombie_${entry.recipe.mapSlug}`;
|
||||
const lobby = nextLobbyNumber(base, names);
|
||||
const name = lobby === null ? null : `${base}_${lobby}`;
|
||||
const port = nextAvailablePort(ports);
|
||||
const port = nextAvailablePort(ports, releasingServers);
|
||||
if (name) names.push(name);
|
||||
if (port !== null) ports.push(port);
|
||||
return { entryIndex, name, port, hasSavedKey: !!name && !!getServerKey(name) };
|
||||
@@ -68,19 +92,21 @@ function blockersFor(servers: CollectionPreview['servers']): string[] {
|
||||
return blockers;
|
||||
}
|
||||
|
||||
export function previewCollection(blueprintId: unknown): CollectionPreview {
|
||||
export function previewCollection(blueprintId: unknown, mode: unknown = 'fill'): CollectionPreview {
|
||||
if (typeof blueprintId !== 'string' || !blueprintId) throw new Error('Choose a blueprint to launch.');
|
||||
return db().transaction(() => {
|
||||
if (mode !== 'fill' && mode !== 'exact') throw new Error('Choose fill or exact blueprint mode.');
|
||||
return db().transaction((): CollectionPreview => {
|
||||
const { entries, blueprintName } = blueprintEntries(blueprintId);
|
||||
const servers = allocate(entries);
|
||||
const counts = coverage(entries);
|
||||
const { counts, retainedServers, removals } = collectionPlan(entries, mode);
|
||||
const servers = allocate(entries, removals.map(server => server.name));
|
||||
const blockers = blockersFor(servers);
|
||||
if (removals.length && (!komodoConfigured() || !rconConfigured())) blockers.push('Configure Komodo and RCON before removing servers.');
|
||||
const id = randomUUID();
|
||||
const expiresAt = new Date(Date.now() + 30 * 60_000).toISOString();
|
||||
db().prepare('DELETE FROM server_collection_previews WHERE expires_at < ? AND collection_id IS NULL').run(new Date().toISOString());
|
||||
db().prepare('INSERT INTO server_collection_previews (id, expires_at, blueprint_name, entries, servers) VALUES (?, ?, ?, ?, ?)')
|
||||
.run(id, expiresAt, blueprintName, JSON.stringify(entries), JSON.stringify(servers));
|
||||
return { id, expiresAt, blueprintName, launchConcurrency: config.createConcurrency, servers, blockers, ready: blockers.length === 0,
|
||||
db().prepare('INSERT INTO server_collection_previews (id, expires_at, blueprint_name, entries, servers, mode, fleet_snapshot) VALUES (?, ?, ?, ?, ?, ?, ?)')
|
||||
.run(id, expiresAt, blueprintName, JSON.stringify(entries), JSON.stringify(servers), mode, mode === 'exact' ? JSON.stringify(fleetSnapshot()) : null);
|
||||
return { id, expiresAt, blueprintName, mode, retainedServers, removals, launchConcurrency: config.createConcurrency, servers, blockers, ready: blockers.length === 0,
|
||||
entries: entries.map((entry, index) => ({ count: entry.count, ...counts[index], recipe: summarizeRecipe(entry.recipe) })) };
|
||||
})();
|
||||
}
|
||||
@@ -92,15 +118,21 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
|
||||
const keys = (input.keys ?? {}) as Record<string, unknown>;
|
||||
return db().transaction(() => {
|
||||
const preview = db().prepare('SELECT * FROM server_collection_previews WHERE id = ?').get(input.previewId) as {
|
||||
id: string; expires_at: string; blueprint_name: string; entries: string; servers: string; collection_id: string | null;
|
||||
id: string; expires_at: string; blueprint_name: string; entries: string; servers: string; collection_id: string | null; mode: CollectionMode; fleet_snapshot: string | null;
|
||||
} | undefined;
|
||||
if (!preview) throw new Error('Collection preview expired or was not found. Refresh the preview.');
|
||||
if (preview.collection_id) return { ok: true as const, collectionId: preview.collection_id };
|
||||
if (preview.expires_at <= new Date().toISOString()) throw new Error('Collection preview expired. Refresh it before launching.');
|
||||
const entries: BlueprintEntry[] = JSON.parse(preview.entries);
|
||||
const reviewed: CollectionPreview['servers'] = JSON.parse(preview.servers);
|
||||
const servers = allocate(entries);
|
||||
if (preview.mode === 'exact' && preview.fleet_snapshot !== JSON.stringify(fleetSnapshot())) {
|
||||
throw new Error('The fleet changed. Nothing was queued. Refresh the preview and review the servers to keep and remove.');
|
||||
}
|
||||
const { retainedServers, removals } = collectionPlan(entries, preview.mode);
|
||||
const releasingServers = removals.map(server => server.name);
|
||||
const servers = allocate(entries, releasingServers);
|
||||
const blockers = blockersFor(servers);
|
||||
if (removals.length && (!komodoConfigured() || !rconConfigured())) blockers.push('Configure Komodo and RCON before removing servers.');
|
||||
if (blockers.length) throw new Error(blockers.join(' '));
|
||||
if (servers.length !== reviewed.length || servers.some((server, i) => server.entryIndex !== reviewed[i]?.entryIndex || server.name !== reviewed[i]?.name || server.port !== reviewed[i]?.port)) {
|
||||
throw new Error('Fleet counts, lobby or port availability changed. Nothing was queued. Refresh the preview and review each server key.');
|
||||
@@ -109,36 +141,65 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
|
||||
for (const [name, value] of Object.entries(keys)) {
|
||||
if (!expected.has(name) || typeof value !== 'string') throw new Error('Keys must be strings for server IDs shown in this preview.');
|
||||
}
|
||||
if (!servers.length) return { ok: true as const, collectionId: null };
|
||||
if (!servers.length && !removals.length) return { ok: true as const, collectionId: null };
|
||||
const collectionId = randomUUID();
|
||||
db().prepare('INSERT INTO server_launch_collections (id, name, created_at) VALUES (?, ?, ?)')
|
||||
.run(collectionId, preview.blueprint_name, new Date().toISOString());
|
||||
db().prepare('INSERT INTO server_launch_collections (id, name, created_at, mode, retained_servers) VALUES (?, ?, ?, ?, ?)')
|
||||
.run(collectionId, preview.blueprint_name, new Date().toISOString(), preview.mode, JSON.stringify(retainedServers));
|
||||
const removalJobs = removals.map((server, position) => {
|
||||
const ports = (db().prepare(`SELECT game_port AS port FROM servers WHERE name = ? UNION
|
||||
SELECT j.port FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
|
||||
WHERE j.server = ? AND d.state IN ('queued', 'running', 'awaiting-health')`).all(server.name, server.name) as { port: number | null }[]).map(row => row.port);
|
||||
const queued = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'create' AND state = 'queued'").get(server.name) as { id: string } | undefined;
|
||||
let id: string;
|
||||
if (queued && !getServer(server.name)) {
|
||||
// No container or files exist yet: cancel the reservation instead of booting just to remove it.
|
||||
const result = destroyServer(server.name, 'blueprint-exact');
|
||||
if (!result.ok || !result.deploymentId) throw new Error('Could not queue removal for ' + server.name);
|
||||
id = result.deploymentId;
|
||||
db().prepare("UPDATE server_deployments SET state = 'cancelled', stage = 'Cancelled by exact blueprint', finished_at = ? WHERE id = ?").run(new Date().toISOString(), queued.id);
|
||||
recordDeploymentEvent(queued.id, 'Queued creation cancelled by exact blueprint; saved key retained.');
|
||||
deploymentStage(id, 'Cancelled queued creation; no server was started');
|
||||
finishDeployment(id, { ok: true });
|
||||
} else {
|
||||
const result = destroyServer(server.name, 'blueprint-exact');
|
||||
if (!result.ok || !result.deploymentId) throw new Error('Could not queue removal for ' + server.name);
|
||||
id = result.deploymentId;
|
||||
}
|
||||
db().prepare('INSERT INTO server_collection_removals (collection_id, deployment_id, position) VALUES (?, ?, ?)').run(collectionId, id, position);
|
||||
return { id, ports };
|
||||
});
|
||||
servers.forEach((server, index) => {
|
||||
const recipe = entries[server.entryIndex].recipe;
|
||||
const childPreviewId = randomUUID();
|
||||
db().prepare('INSERT INTO server_launch_previews (id, expires_at, server, port, recipe, blueprint_name) VALUES (?, ?, ?, ?, ?, ?)')
|
||||
.run(childPreviewId, preview.expires_at, server.name, server.port, JSON.stringify(recipe), preview.blueprint_name);
|
||||
const result = queueCreateServer({ game: recipe.game, gameMode: 'zombie', mapSlug: recipe.mapSlug,
|
||||
expectedName: server.name!, previewId: childPreviewId, plutoKey: keys[server.name!] as string | undefined, createdBy: 'ui' });
|
||||
expectedName: server.name!, previewId: childPreviewId, plutoKey: keys[server.name!] as string | undefined, createdBy: 'ui' }, releasingServers);
|
||||
// Throw so the outer transaction also reverses all earlier keys and children.
|
||||
if (!result.ok) throw new Error(`${server.name}: ${result.message} Nothing was queued.`);
|
||||
db().prepare('INSERT INTO server_collection_members (collection_id, deployment_id, position) VALUES (?, ?, ?)')
|
||||
.run(collectionId, result.deploymentId, index);
|
||||
for (const removal of removalJobs.filter(removal => removal.ports.includes(server.port))) {
|
||||
db().prepare('INSERT INTO server_creation_dependencies (deployment_id, removal_id) VALUES (?, ?)').run(result.deploymentId, removal.id);
|
||||
}
|
||||
});
|
||||
db().prepare('UPDATE server_collection_previews SET collection_id = ? WHERE id = ?').run(collectionId, preview.id);
|
||||
return { ok: true as const, collectionId };
|
||||
})();
|
||||
}
|
||||
|
||||
const summarySql = `SELECT c.id, c.name, c.created_at, COUNT(d.id) AS total,
|
||||
const membershipSql = `SELECT collection_id, deployment_id, position FROM server_collection_members
|
||||
UNION ALL SELECT collection_id, deployment_id, position FROM server_collection_removals`;
|
||||
|
||||
const summarySql = `SELECT c.id, c.name, c.created_at, c.mode, COUNT(d.id) AS total,
|
||||
COALESCE(SUM(d.state IN ('done', 'warning')), 0) AS done,
|
||||
COALESCE(SUM(d.state IN ('failed', 'interrupted')), 0) AS failed,
|
||||
COALESCE(SUM(d.state IN ('failed', 'interrupted', 'cancelled')), 0) AS failed,
|
||||
COALESCE(SUM(d.state IN ('queued', 'running', 'awaiting-health', 'waiting')), 0) AS active,
|
||||
COALESCE(SUM(d.state = 'queued'), 0) AS queued,
|
||||
COALESCE(SUM(d.state IN ('queued', 'waiting')), 0) AS queued,
|
||||
COALESCE(SUM(d.state = 'running'), 0) AS provisioning,
|
||||
COALESCE(SUM(d.state = 'awaiting-health'), 0) AS verifying
|
||||
FROM server_launch_collections c
|
||||
LEFT JOIN server_collection_members m ON m.collection_id = c.id
|
||||
LEFT JOIN (${membershipSql}) m ON m.collection_id = c.id
|
||||
LEFT JOIN server_deployments d ON d.id = m.deployment_id`;
|
||||
|
||||
export function listCollections(): CollectionSummary[] {
|
||||
@@ -148,12 +209,13 @@ export function listCollections(): CollectionSummary[] {
|
||||
export function getCollection(id: string): CollectionDetail | null {
|
||||
const collection = db().prepare(summarySql + ' WHERE c.id = ? GROUP BY c.id').get(id) as CollectionSummary | undefined;
|
||||
if (!collection) return null;
|
||||
const servers = db().prepare(`SELECT d.id, d.server, d.state, d.stage, d.started_at, d.finished_at, d.error, j.port, j.map_slug,
|
||||
const servers = db().prepare(`SELECT d.id, d.server, d.action, d.state, d.stage, d.started_at, d.finished_at, d.error, j.port, j.map_slug,
|
||||
EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server) AS server_exists
|
||||
FROM server_collection_members m JOIN server_deployments d ON d.id = m.deployment_id
|
||||
JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE m.collection_id = ? ORDER BY m.position`).all(id) as CollectionDetail['servers'];
|
||||
FROM (${membershipSql}) m JOIN server_deployments d ON d.id = m.deployment_id
|
||||
LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE m.collection_id = ? ORDER BY d.action DESC, m.position`).all(id) as CollectionDetail['servers'];
|
||||
const queue = creationQueue();
|
||||
const positions = new Map(queue.queued.map((job, index) => [job.id, index + 1]));
|
||||
return { collection, queue: { concurrency: config.createConcurrency, active: queue.active, queued: queue.queued.length },
|
||||
const stored = db().prepare('SELECT retained_servers FROM server_launch_collections WHERE id = ?').get(id) as { retained_servers: string };
|
||||
return { collection, retainedServers: JSON.parse(stored.retained_servers), queue: { concurrency: config.createConcurrency, active: queue.active, queued: queue.queued.length },
|
||||
servers: servers.map(server => ({ ...server, queuePosition: positions.get(server.id) ?? null })) };
|
||||
}
|
||||
|
||||
@@ -83,13 +83,13 @@ export function nextLobbyNumber(baseName: string, additionalNames: string[] = []
|
||||
* from the DB (which EC controls end-to-end under the new naming scheme, so
|
||||
* the DB is authoritative — no need to ask Komodo).
|
||||
*/
|
||||
export function nextAvailablePort(additionalPorts: number[] = []): number | null {
|
||||
export function nextAvailablePort(additionalPorts: number[] = [], releasingServers: string[] = []): number | null {
|
||||
const rows = db()
|
||||
.prepare(`SELECT game_port FROM servers WHERE game_port IS NOT NULL UNION
|
||||
SELECT j.port AS game_port FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
|
||||
.prepare(`SELECT name, game_port FROM servers WHERE game_port IS NOT NULL UNION
|
||||
SELECT j.server AS name, j.port AS game_port FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
|
||||
WHERE d.state IN ('queued', 'running', 'awaiting-health')`)
|
||||
.all() as Array<{ game_port: number }>;
|
||||
const taken = new Set([...rows.map((r) => r.game_port), ...additionalPorts]);
|
||||
.all() as Array<{ name: string; game_port: number }>;
|
||||
const taken = new Set([...rows.filter(row => !releasingServers.includes(row.name)).map((r) => r.game_port), ...additionalPorts]);
|
||||
for (let p = config.portRange.min; p <= config.portRange.max; p++) {
|
||||
if (!taken.has(p)) return p;
|
||||
}
|
||||
@@ -195,7 +195,8 @@ export function previewCreation(input: { game?: unknown; mapSlug?: unknown; blue
|
||||
checks, ready: checks.every(check => check.state !== 'blocked') };
|
||||
}
|
||||
|
||||
export function queueCreateServer(input: CreateServerInput): CreateServerResult {
|
||||
// releasingServers is internal to atomic collection queueing; HTTP callers cannot set it.
|
||||
export function queueCreateServer(input: CreateServerInput, releasingServers: string[] = []): CreateServerResult {
|
||||
return db().transaction((): CreateServerResult => {
|
||||
const preview = input.previewId ? db().prepare('SELECT * FROM server_launch_previews WHERE id = ?').get(input.previewId) as PreviewRow | undefined : undefined;
|
||||
if (input.previewId && !preview) return { ok: false, kind: 'bad-request', message: 'Launch preview expired or was not found. Review a fresh preview.' };
|
||||
@@ -206,13 +207,13 @@ export function queueCreateServer(input: CreateServerInput): CreateServerResult
|
||||
// A repeated submit after a lost response returns the same operation.
|
||||
if (preview?.deployment_id) return { ok: true, name: preview.server!, port: preview.port!, deploymentId: preview.deployment_id };
|
||||
if (preview && preview.expires_at <= new Date().toISOString()) return { ok: false, kind: 'bad-request', message: 'Launch preview expired. Review a fresh preview.' };
|
||||
if (preview && (!preview.server || !preview.port || preview.server !== nextName(input) || preview.port !== nextAvailablePort())) {
|
||||
if (preview && (!preview.server || !preview.port || preview.server !== nextName(input) || preview.port !== nextAvailablePort([], releasingServers))) {
|
||||
return { ok: false, kind: 'no-slot', message: 'Available lobby or port changed. Refresh the preview and review the new server key.' };
|
||||
}
|
||||
if (recipe && !config.rcon.password) {
|
||||
return { ok: false, kind: 'unconfigured', message: 'Health-check settings changed. Refresh the preview.' };
|
||||
}
|
||||
const prepared = prepareCreation(input);
|
||||
const prepared = prepareCreation(input, releasingServers);
|
||||
if (!prepared.ok) return prepared;
|
||||
// A preview records choices, not version pins. Promotions remain authoritative.
|
||||
const currentRecipe = captureRecipe(input.game ?? 't5', prepared.mapEntry.slug);
|
||||
@@ -247,7 +248,7 @@ export async function createServer(input: CreateServerInput): Promise<CreateServ
|
||||
return outcome ?? queued;
|
||||
}
|
||||
|
||||
function prepareCreation(input: CreateServerInput): PreparedCreation | Extract<CreateServerResult, { ok: false }> {
|
||||
function prepareCreation(input: CreateServerInput, releasingServers: string[] = []): PreparedCreation | Extract<CreateServerResult, { ok: false }> {
|
||||
const game = input.game ?? 't5';
|
||||
if (game !== 't5' && game !== 't6') {
|
||||
return { ok: false, kind: 'bad-request', message: 'Supported games are t5 and t6' };
|
||||
@@ -279,7 +280,7 @@ function prepareCreation(input: CreateServerInput): PreparedCreation | Extract<C
|
||||
const plutoKey = input.plutoKey?.trim() || getServerKey(name);
|
||||
if (!plutoKey) return { ok: false, kind: 'bad-request', message: `No saved key for ${name}. Enter a Plutonium server key.` };
|
||||
|
||||
const port = nextAvailablePort();
|
||||
const port = nextAvailablePort([], releasingServers);
|
||||
if (port === null) {
|
||||
return {
|
||||
ok: false,
|
||||
@@ -640,6 +641,16 @@ export function creationQueue() {
|
||||
return { active: jobs.filter(job => job.state !== 'queued'), queued: jobs.filter(job => job.state === 'queued') };
|
||||
}
|
||||
|
||||
function creationBlocker(id: string): string | null {
|
||||
const dependency = db().prepare(`SELECT d.server, d.state FROM server_creation_dependencies dep
|
||||
JOIN server_deployments d ON d.id = dep.removal_id
|
||||
WHERE dep.deployment_id = ? AND d.state NOT IN ('done', 'warning') LIMIT 1`).get(id) as { server: string; state: string } | undefined;
|
||||
if (dependency) return `Waiting for removal of ${dependency.server} (${dependency.state}) before reusing its port`;
|
||||
const occupied = db().prepare(`SELECT s.name FROM servers s JOIN server_creation_jobs j ON s.game_port = j.port
|
||||
WHERE j.deployment_id = ? LIMIT 1`).get(id) as { name: string } | undefined;
|
||||
return occupied ? `Waiting for port to be released by ${occupied.name}` : null;
|
||||
}
|
||||
|
||||
async function runCreations(): Promise<CreationOutcomes> {
|
||||
const outcomes: CreationOutcomes = new Map();
|
||||
if (creating || destroying.size > 0) return outcomes;
|
||||
@@ -696,12 +707,22 @@ async function runCreations(): Promise<CreationOutcomes> {
|
||||
// worker pass into an unbounded stream of expensive container startups.
|
||||
const queue = creationQueue();
|
||||
const available = Math.max(0, config.createConcurrency - queue.active.length);
|
||||
for (const { id } of queue.queued.slice(0, available)) {
|
||||
const eligible = queue.queued.filter(({ id }) => {
|
||||
const blocker = creationBlocker(id);
|
||||
if (blocker) deploymentStage(id, blocker);
|
||||
return !blocker;
|
||||
});
|
||||
for (const { id } of eligible.slice(0, available)) {
|
||||
if (creating || destroying.size > 0) break;
|
||||
// API calls can cancel a selected job or resume an older startup while
|
||||
// a previous provision awaits Komodo. Recheck without expanding this pass.
|
||||
if (creationQueue().active.length >= config.createConcurrency) break;
|
||||
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND action = 'create' AND state = 'queued'").get(id)) continue;
|
||||
const blocker = creationBlocker(id);
|
||||
if (blocker) {
|
||||
deploymentStage(id, blocker);
|
||||
continue;
|
||||
}
|
||||
try { outcomes.set(id, await provisionCreation(id)); }
|
||||
catch (err) {
|
||||
const failure = err instanceof Error ? err : new Error(String(err));
|
||||
|
||||
@@ -27,7 +27,9 @@ export function getDeployment(id: string) {
|
||||
if (!deployment) return null;
|
||||
const events = db().prepare('SELECT id, created_at, message FROM server_deployment_events WHERE deployment_id = ? ORDER BY id').all(id);
|
||||
const launch = recipeForDeployment(id);
|
||||
const collection = db().prepare(`SELECT c.id, c.name FROM server_collection_members m
|
||||
const collection = db().prepare(`SELECT c.id, c.name FROM (
|
||||
SELECT collection_id, deployment_id FROM server_collection_members
|
||||
UNION ALL SELECT collection_id, deployment_id FROM server_collection_removals) m
|
||||
JOIN server_launch_collections c ON c.id = m.collection_id WHERE m.deployment_id = ?`).get(id) ?? null;
|
||||
return { deployment, events, recipe: launch ? summarizeRecipe(launch.recipe) : null, blueprintName: launch?.blueprintName ?? null,
|
||||
canResumeVerification: canResumeVerification(id), canResumeCleanup: canResumeCleanup(id), canCancelRemoval: canCancelRemoval(id), collection };
|
||||
|
||||
@@ -6,11 +6,13 @@
|
||||
import Button from './primitives/Button.svelte';
|
||||
import LaunchRecipe from './LaunchRecipe.svelte';
|
||||
import ServerIdentity from './ServerIdentity.svelte';
|
||||
import type { CollectionPreview } from '$lib/deployment-plan.js';
|
||||
import type { CollectionMode, CollectionPreview } from '$lib/deployment-plan.js';
|
||||
|
||||
let { open = $bindable(false), blueprintId = '' }: { open?: boolean; blueprintId?: string } = $props();
|
||||
let dialog: HTMLDialogElement | undefined = $state();
|
||||
let preview = $state<CollectionPreview | null>(null);
|
||||
let mode = $state<CollectionMode>('fill');
|
||||
let confirmRemovals = $state(false);
|
||||
let keys = $state<Record<string, string>>({});
|
||||
let busy = $state(false);
|
||||
let error = $state('');
|
||||
@@ -21,20 +23,21 @@
|
||||
const expired = $derived(!!preview && new Date(preview.expiresAt).getTime() <= now);
|
||||
const keysReady = $derived(preview?.servers.filter(server => server.name && (server.hasSavedKey || keys[server.name]?.trim())).length ?? 0);
|
||||
onMount(() => { const timer = setInterval(() => { now = Date.now(); }, 10_000); return () => clearInterval(timer); });
|
||||
let canLaunch = $derived(preview?.ready && preview.servers.length > 0 && preview.servers.every(server => server.name && (server.hasSavedKey || keys[server.name]?.trim())));
|
||||
let canLaunch = $derived(preview?.ready && preview.mode === mode && (preview.servers.length > 0 || preview.removals.length > 0)
|
||||
&& (!preview.removals.length || confirmRemovals) && preview.servers.every(server => server.name && (server.hasSavedKey || keys[server.name]?.trim())));
|
||||
|
||||
$effect(() => {
|
||||
if (!dialog) return;
|
||||
if (open && !wasOpen) { wasOpen = true; keys = {}; preview = null; keyNotice = ''; dialog.showModal(); void review(); }
|
||||
if (open && !wasOpen) { wasOpen = true; mode = 'fill'; keys = {}; preview = null; keyNotice = ''; dialog.showModal(); void review(); }
|
||||
else if (!open && wasOpen) { wasOpen = false; session++; dialog.close(); }
|
||||
});
|
||||
|
||||
async function review() {
|
||||
const current = ++session;
|
||||
const previous = preview;
|
||||
busy = true; error = ''; keyNotice = '';
|
||||
busy = true; error = ''; keyNotice = ''; confirmRemovals = false;
|
||||
try {
|
||||
const res = await fetch('/api/deployment-collections/preview', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ blueprintId }) });
|
||||
const res = await fetch('/api/deployment-collections/preview', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ blueprintId, mode }) });
|
||||
const body = await res.json();
|
||||
if (!res.ok) throw new Error(body.error ?? 'Could not preview the collection.');
|
||||
if (current === session) {
|
||||
@@ -71,21 +74,38 @@
|
||||
<dialog bind:this={dialog} aria-label="Launch collection" onclick={event => { if (isDialogBackdrop(event, dialog) && !busy) open = false; }} oncancel={event => { if (busy) event.preventDefault(); }} onclose={() => open = false}>
|
||||
<div class="composer">
|
||||
<header><div><span class="eyebrow">Collection launch</span><h2>{preview?.blueprintName ?? 'Preparing your collection'}</h2></div><button class="close" disabled={busy} onclick={() => open = false} aria-label="Close collection preview">×</button></header>
|
||||
<label class="mode">Apply blueprint
|
||||
<select bind:value={mode} disabled={busy} onchange={event => { mode = event.currentTarget.value as CollectionMode; void review(); }}>
|
||||
<option value="fill">Fill missing servers</option>
|
||||
<option value="exact">Match blueprint exactly</option>
|
||||
</select>
|
||||
</label>
|
||||
<p class="muted">{mode === 'exact' ? 'Keep matching lobbies, add missing ones, and remove excess lobbies and maps outside this blueprint across both games.' : 'Create missing lobbies and keep all existing servers.'}</p>
|
||||
{#if error}<p class="error" role="alert">{error}</p>{/if}
|
||||
{#if !preview}<p class="muted">{busy ? 'Finding a lobby and port for every server…' : 'Refresh to prepare another preview.'}</p>{#if !busy}<Button onclick={review}>Refresh preview</Button>{/if}
|
||||
{:else}
|
||||
{#if keyNotice}<p role="status" class="muted">{keyNotice}</p>{/if}
|
||||
{#if expired}<p class="error" role="status">This preview expired. Refresh before launch; keys remain entered for unchanged IDs.</p>{/if}
|
||||
<div class="lineup"><strong>{preview.servers.length ? `${preview.servers.length} missing server${preview.servers.length === 1 ? '' : 's'} to create` : 'Blueprint already satisfied'}</strong>
|
||||
<div class="lineup"><strong>{preview.mode === 'exact' ? `${preview.retainedServers.length} keep · ${preview.servers.length} create · ${preview.removals.length} remove` : preview.servers.length ? `${preview.servers.length} missing server${preview.servers.length === 1 ? '' : 's'} to create` : 'Blueprint already satisfied'}</strong>
|
||||
<table><thead><tr><th>Map</th><th>Target</th><th>Existing / queued</th><th>To create</th></tr></thead><tbody>{#each preview.entries as entry}<tr><th>{entry.recipe.mapName} <small>{entry.recipe.game.toUpperCase()}</small></th><td>{entry.count}</td><td title={entry.existingServers.join(', ')}>{entry.existing}</td><td class="quantity">{entry.missing}</td></tr>{/each}</tbody></table>
|
||||
<p class="muted">Targets count inventory, including stopped servers; they do not mean every server is ready to play. Only missing servers are created. Extra servers remain and existing servers are not updated.</p>
|
||||
{#if preview.entries.some(entry => entry.existing)}<details class="coverage" open={!preview.servers.length}><summary>Review existing servers</summary>{#each preview.entries.filter(entry => entry.existing) as entry}<div class="coverage-map"><strong>{entry.recipe.mapName}: {entry.existing} existing</strong><p class="muted">{[...new Set((entry.coverage ?? []).map(server => server.health))].map(health => `${entry.coverage.filter(server => server.health === health).length} ${health}`).join(', ')}</p><ul>{#each entry.coverage ?? [] as server}<li><a href={server.href}><ServerIdentity name={server.name} /></a><span>{server.health === 'online' ? 'Healthy' : server.health}</span></li>{/each}</ul></div>{/each}</details>{/if}
|
||||
<p class="muted">Targets count inventory, including stopped servers; they do not mean every server is ready to play. {preview.mode === 'exact' ? 'Existing deployed lobbies are kept before queued creations, then the lowest lobby numbers. Retained servers are not changed or restarted.' : 'Only missing servers are created. Extra servers remain and existing servers are not updated.'}</p>
|
||||
{#if preview.mode === 'exact' && preview.retainedServers.length}<details class="coverage"><summary>Servers to keep ({preview.retainedServers.length})</summary><ul>{#each preview.retainedServers as name}<li><ServerIdentity {name} /><span>{name}</span></li>{/each}</ul></details>{/if}
|
||||
{#if preview.mode === 'fill' && preview.entries.some(entry => entry.existing)}<details class="coverage" open={!preview.servers.length}><summary>Review existing servers</summary>{#each preview.entries.filter(entry => entry.existing) as entry}<div class="coverage-map"><strong>{entry.recipe.mapName}: {entry.existing} existing</strong><p class="muted">{[...new Set((entry.coverage ?? []).map(server => server.health))].map(health => `${entry.coverage.filter(server => server.health === health).length} ${health}`).join(', ')}</p><ul>{#each entry.coverage ?? [] as server}<li><a href={server.href}><ServerIdentity name={server.name} /></a><span>{server.health === 'online' ? 'Healthy' : server.health}</span></li>{/each}</ul></div>{/each}</details>{/if}
|
||||
</div>
|
||||
<div class="section-title"><h3>{preview.servers.length ? 'Your launch order' : 'Nothing to create'}</h3><button class="text-button" disabled={busy} onclick={review}>Refresh preview</button></div>
|
||||
<div class="section-title"><h3>{preview.mode === 'exact' ? 'Review fleet changes' : preview.servers.length ? 'Your launch order' : 'Nothing to create'}</h3><button class="text-button" disabled={busy} onclick={review}>Refresh preview</button></div>
|
||||
{#each preview.blockers as blocker}<p class="error" role="alert">{blocker}</p>{/each}
|
||||
{#if preview.servers.length}
|
||||
<p class="muted">Saved keys are reused automatically. Add keys only where required below. Update saved keys in <a href="/server-keys" target="_blank" rel="noopener">Keys (opens in a new tab)</a>. Fleet counts and available slots are checked again when you queue the launch.</p>
|
||||
{#if preview.servers.length || preview.removals.length}
|
||||
{#if preview.servers.length}<p class="muted">Saved keys are reused automatically. Add keys only where required below. Update saved keys in <a href="/server-keys" target="_blank" rel="noopener">Keys (opens in a new tab)</a>. Fleet counts and available slots are checked again when you queue the launch.</p>{/if}
|
||||
<form onsubmit={event => { event.preventDefault(); void launch(); }}>
|
||||
{#if preview.removals.length}
|
||||
<section class="removals" aria-label="Servers to remove">
|
||||
<h3>Remove {preview.removals.length} server{preview.removals.length === 1 ? '' : 's'}</h3>
|
||||
<p class="muted">Containers, files, and registrations will be removed after a fresh zero-player check. Busy or unreachable servers wait. Unstarted creations are cancelled. Saved keys are kept. New servers reusing these ports wait for removal to finish.</p>
|
||||
<ul>{#each preview.removals as server}<li><strong><ServerIdentity name={server.name} /></strong><span class="server-id">{server.name}</span><span class="muted">{server.reason}</span></li>{/each}</ul>
|
||||
<label class="confirm-removals"><input type="checkbox" bind:checked={confirmRemovals} disabled={busy || preview.mode !== mode} />Remove the {preview.removals.length} listed servers and apply this blueprint.</label>
|
||||
</section>
|
||||
{/if}
|
||||
{#if preview.servers.length}
|
||||
<ol class="servers" aria-label="Server launch order">
|
||||
{#each preview.servers as server, index (server.name ?? `unavailable-${index}`)}
|
||||
<li class="server-row">
|
||||
@@ -98,15 +118,22 @@
|
||||
</li>
|
||||
{/each}
|
||||
</ol>
|
||||
<div class="launch-footer"><div><p role="status"><strong>{keysReady} / {preview.servers.length} keys available</strong></p><p class="caption">Validity is checked at startup. Preview expires at {$timeFormat.minuteTime(preview.expiresAt)}.</p>{#if keysReady < preview.servers.length}<button class="text-button" type="button" onclick={nextMissing}>Jump to next missing key</button>{/if}</div><div class="actions"><Button variant="ghost" disabled={busy} onclick={() => open = false}>Cancel</Button><Button type="submit" variant="primary" loading={busy} disabled={busy || expired || !canLaunch}>Queue {preview.servers.length} server{preview.servers.length === 1 ? '' : 's'} →</Button></div></div>
|
||||
{/if}
|
||||
<div class="launch-footer"><div>{#if preview.servers.length}<p role="status"><strong>{keysReady} / {preview.servers.length} keys available</strong></p>{/if}<p class="caption">Key validity is checked at startup. Preview expires at {$timeFormat.minuteTime(preview.expiresAt)}.</p>{#if keysReady < preview.servers.length}<button class="text-button" type="button" onclick={nextMissing}>Jump to next missing key</button>{/if}</div><div class="actions"><Button variant="ghost" disabled={busy} onclick={() => open = false}>Cancel</Button><Button type="submit" variant={preview.removals.length ? 'danger' : 'primary'} loading={busy} disabled={busy || expired || !canLaunch}>{preview.mode === 'exact' ? `Apply: ${preview.servers.length} create, ${preview.removals.length} remove` : `Queue ${preview.servers.length} server${preview.servers.length === 1 ? '' : 's'} →`}</Button></div></div>
|
||||
</form>
|
||||
{:else}<p class="muted">Your fleet already meets or exceeds every target in this blueprint. No servers will be created or removed.</p><div class="actions"><Button onclick={() => open = false}>Done</Button></div>{/if}
|
||||
{:else}<p class="muted">{preview.mode === 'exact' ? 'Your fleet already matches this blueprint.' : 'Your fleet already meets or exceeds every target in this blueprint.'} No servers will be created or removed.</p><div class="actions"><Button onclick={() => open = false}>Done</Button></div>{/if}
|
||||
<details class="recipes"><summary>Setup and launch requirements</summary><p class="muted requirements">Each server uses the latest promoted setup when its turn begins. The host must have the selected games and DLC. This preview checks settings and capacity; it does not probe game files, images, credentials, or network access.</p>{#each preview.entries as entry}<div class="recipe"><LaunchRecipe recipe={entry.recipe} /></div>{/each}</details>
|
||||
{/if}
|
||||
</div>
|
||||
</dialog>
|
||||
|
||||
<style>
|
||||
select { padding: 10px; border: 1px solid var(--border-control); border-radius: var(--radius-md); background: var(--surface-sunken); color: var(--text-primary); font: inherit; }
|
||||
.removals { display: grid; gap: 12px; padding: 16px; border: 1px solid var(--state-danger); border-radius: var(--radius-md); }
|
||||
.removals ul { list-style: none; margin: 0; padding: 0; display: grid; gap: 16px; }
|
||||
.removals li { display: grid; gap: 4px; font-size: 13px; }
|
||||
.confirm-removals { display: flex; align-items: flex-start; gap: 10px; font-size: 13px; }
|
||||
.confirm-removals input { width: auto; margin-top: 2px; }
|
||||
.key-ready { color: var(--state-online); font-size: 13px; font-weight: 600; }
|
||||
a { color: var(--accent-primary-hover); }
|
||||
@media (max-width: 600px) { .server-row .key-ready { grid-column: 2; } }
|
||||
|
||||
Reference in New Issue
Block a user