josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: match fleet to blueprint targets exactly

Browse Source 
Add an optional exact blueprint mode with keep/create/remove previews, atomic queueing, cancellation of unstarted excess creations, and persistent dependencies for safe port reuse. Track removals alongside creations and preserve fill mode as the default.

Validation: type check, production build, and 103 deployment tests passed. The full suite has one telemetry assertion failure also reproduced on the previous commit.

Fixes #42
This commit is contained in:
josh
2026-09-27 13:36:08 -04:00
parent 1aa7f15371
commit 0b0e5b5bdb
13 changed files with 517 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/fleet/collections.ts
+87 -25
View File
@@ -1,9 +1,11 @@
import { randomUUID } from 'node:crypto';
import type { CollectionDetail, CollectionPreview, CollectionSummary } from '$lib/deployment-plan.js';
import type { CollectionDetail, CollectionMode, CollectionPreview, CollectionSummary } from '$lib/deployment-plan.js';
import { db } from '../db/index.js';
import { config, komodoConfigured, rconConfigured } from '../config.js';
import { blueprintEntries, summarizeRecipe, type BlueprintEntry } from './launch-recipes.js';
import { creationQueue, nextAvailablePort, nextLobbyNumber, queueCreateServer } from './create.js';
import { creationQueue, destroyServer, nextAvailablePort, nextLobbyNumber, queueCreateServer } from './create.js';
import { deploymentStage, finishDeployment, recordDeploymentEvent } from './deployments.js';
import { housePartyTargets } from './house-party.js';
import { getServerKey } from './server-keys.js';
import { classifyHealth } from './health.js';
import { getServer } from './servers.js';
@@ -43,7 +45,29 @@ function coverage(entries: BlueprintEntry[]) {
});
}
function allocate(entries: BlueprintEntry[]): CollectionPreview['servers'] {
function fleetSnapshot() {
return housePartyTargets().map(name => ({ name,
server: db().prepare('SELECT game, map_slug, map, game_port FROM servers WHERE name = ?').get(name) ?? null,
creation: db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'create' ORDER BY rowid DESC LIMIT 1").get(name) ?? null,
removal: db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) ?? null
}));
}
function collectionPlan(entries: BlueprintEntry[], mode: CollectionMode) {
const counts = coverage(entries);
// Preserve deployed lobbies before queued ones, then the lowest lobby IDs.
const retainedServers = counts.flatMap((entry, index) => [...entry.existingServers].sort((a, b) =>
Number(!!getServer(b)) - Number(!!getServer(a)) || a.localeCompare(b, 'en', { numeric: true })
).slice(0, mode === 'exact' ? entries[index].count : undefined));
const retained = new Set(retainedServers);
const matching = new Set(counts.flatMap(entry => entry.existingServers));
const removals = mode === 'exact' ? housePartyTargets().filter(name => !retained.has(name)).map(name => ({ name,
reason: matching.has(name) ? 'Exceeds the target lobby count' : 'Outside this blueprint or already awaiting removal'
})) : [];
return { counts, retainedServers, removals };
}
function allocate(entries: BlueprintEntry[], releasingServers: string[] = []): CollectionPreview['servers'] {
const names: string[] = [];
const ports: number[] = [];
const counts = coverage(entries);
@@ -51,7 +75,7 @@ function allocate(entries: BlueprintEntry[]): CollectionPreview['servers'] {
const base = `${entry.recipe.game}_zombie_${entry.recipe.mapSlug}`;
const lobby = nextLobbyNumber(base, names);
const name = lobby === null ? null : `${base}_${lobby}`;
const port = nextAvailablePort(ports);
const port = nextAvailablePort(ports, releasingServers);
if (name) names.push(name);
if (port !== null) ports.push(port);
return { entryIndex, name, port, hasSavedKey: !!name && !!getServerKey(name) };
@@ -68,19 +92,21 @@ function blockersFor(servers: CollectionPreview['servers']): string[] {
return blockers;
}
export function previewCollection(blueprintId: unknown): CollectionPreview {
export function previewCollection(blueprintId: unknown, mode: unknown = 'fill'): CollectionPreview {
if (typeof blueprintId !== 'string' || !blueprintId) throw new Error('Choose a blueprint to launch.');
return db().transaction(() => {
if (mode !== 'fill' && mode !== 'exact') throw new Error('Choose fill or exact blueprint mode.');
return db().transaction((): CollectionPreview => {
const { entries, blueprintName } = blueprintEntries(blueprintId);
const servers = allocate(entries);
const counts = coverage(entries);
const { counts, retainedServers, removals } = collectionPlan(entries, mode);
const servers = allocate(entries, removals.map(server => server.name));
const blockers = blockersFor(servers);
if (removals.length && (!komodoConfigured() || !rconConfigured())) blockers.push('Configure Komodo and RCON before removing servers.');
const id = randomUUID();
const expiresAt = new Date(Date.now() + 30 * 60_000).toISOString();
db().prepare('DELETE FROM server_collection_previews WHERE expires_at < ? AND collection_id IS NULL').run(new Date().toISOString());
db().prepare('INSERT INTO server_collection_previews (id, expires_at, blueprint_name, entries, servers) VALUES (?, ?, ?, ?, ?)')
.run(id, expiresAt, blueprintName, JSON.stringify(entries), JSON.stringify(servers));
return { id, expiresAt, blueprintName, launchConcurrency: config.createConcurrency, servers, blockers, ready: blockers.length === 0,
db().prepare('INSERT INTO server_collection_previews (id, expires_at, blueprint_name, entries, servers, mode, fleet_snapshot) VALUES (?, ?, ?, ?, ?, ?, ?)')
.run(id, expiresAt, blueprintName, JSON.stringify(entries), JSON.stringify(servers), mode, mode === 'exact' ? JSON.stringify(fleetSnapshot()) : null);
return { id, expiresAt, blueprintName, mode, retainedServers, removals, launchConcurrency: config.createConcurrency, servers, blockers, ready: blockers.length === 0,
entries: entries.map((entry, index) => ({ count: entry.count, ...counts[index], recipe: summarizeRecipe(entry.recipe) })) };
})();
}
@@ -92,15 +118,21 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
const keys = (input.keys ?? {}) as Record<string, unknown>;
return db().transaction(() => {
const preview = db().prepare('SELECT * FROM server_collection_previews WHERE id = ?').get(input.previewId) as {
id: string; expires_at: string; blueprint_name: string; entries: string; servers: string; collection_id: string | null;
id: string; expires_at: string; blueprint_name: string; entries: string; servers: string; collection_id: string | null; mode: CollectionMode; fleet_snapshot: string | null;
} | undefined;
if (!preview) throw new Error('Collection preview expired or was not found. Refresh the preview.');
if (preview.collection_id) return { ok: true as const, collectionId: preview.collection_id };
if (preview.expires_at <= new Date().toISOString()) throw new Error('Collection preview expired. Refresh it before launching.');
const entries: BlueprintEntry[] = JSON.parse(preview.entries);
const reviewed: CollectionPreview['servers'] = JSON.parse(preview.servers);
const servers = allocate(entries);
if (preview.mode === 'exact' && preview.fleet_snapshot !== JSON.stringify(fleetSnapshot())) {
throw new Error('The fleet changed. Nothing was queued. Refresh the preview and review the servers to keep and remove.');
}
const { retainedServers, removals } = collectionPlan(entries, preview.mode);
const releasingServers = removals.map(server => server.name);
const servers = allocate(entries, releasingServers);
const blockers = blockersFor(servers);
if (removals.length && (!komodoConfigured() || !rconConfigured())) blockers.push('Configure Komodo and RCON before removing servers.');
if (blockers.length) throw new Error(blockers.join(' '));
if (servers.length !== reviewed.length || servers.some((server, i) => server.entryIndex !== reviewed[i]?.entryIndex || server.name !== reviewed[i]?.name || server.port !== reviewed[i]?.port)) {
throw new Error('Fleet counts, lobby or port availability changed. Nothing was queued. Refresh the preview and review each server key.');
@@ -109,36 +141,65 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
for (const [name, value] of Object.entries(keys)) {
if (!expected.has(name) || typeof value !== 'string') throw new Error('Keys must be strings for server IDs shown in this preview.');
}
if (!servers.length) return { ok: true as const, collectionId: null };
if (!servers.length && !removals.length) return { ok: true as const, collectionId: null };
const collectionId = randomUUID();
db().prepare('INSERT INTO server_launch_collections (id, name, created_at) VALUES (?, ?, ?)')
.run(collectionId, preview.blueprint_name, new Date().toISOString());
db().prepare('INSERT INTO server_launch_collections (id, name, created_at, mode, retained_servers) VALUES (?, ?, ?, ?, ?)')
.run(collectionId, preview.blueprint_name, new Date().toISOString(), preview.mode, JSON.stringify(retainedServers));
const removalJobs = removals.map((server, position) => {
const ports = (db().prepare(`SELECT game_port AS port FROM servers WHERE name = ? UNION
SELECT j.port FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
WHERE j.server = ? AND d.state IN ('queued', 'running', 'awaiting-health')`).all(server.name, server.name) as { port: number | null }[]).map(row => row.port);
const queued = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND action = 'create' AND state = 'queued'").get(server.name) as { id: string } | undefined;
let id: string;
if (queued && !getServer(server.name)) {
// No container or files exist yet: cancel the reservation instead of booting just to remove it.
const result = destroyServer(server.name, 'blueprint-exact');
if (!result.ok || !result.deploymentId) throw new Error('Could not queue removal for ' + server.name);
id = result.deploymentId;
db().prepare("UPDATE server_deployments SET state = 'cancelled', stage = 'Cancelled by exact blueprint', finished_at = ? WHERE id = ?").run(new Date().toISOString(), queued.id);
recordDeploymentEvent(queued.id, 'Queued creation cancelled by exact blueprint; saved key retained.');
deploymentStage(id, 'Cancelled queued creation; no server was started');
finishDeployment(id, { ok: true });
} else {
const result = destroyServer(server.name, 'blueprint-exact');
if (!result.ok || !result.deploymentId) throw new Error('Could not queue removal for ' + server.name);
id = result.deploymentId;
}
db().prepare('INSERT INTO server_collection_removals (collection_id, deployment_id, position) VALUES (?, ?, ?)').run(collectionId, id, position);
return { id, ports };
});
servers.forEach((server, index) => {
const recipe = entries[server.entryIndex].recipe;
const childPreviewId = randomUUID();
db().prepare('INSERT INTO server_launch_previews (id, expires_at, server, port, recipe, blueprint_name) VALUES (?, ?, ?, ?, ?, ?)')
.run(childPreviewId, preview.expires_at, server.name, server.port, JSON.stringify(recipe), preview.blueprint_name);
const result = queueCreateServer({ game: recipe.game, gameMode: 'zombie', mapSlug: recipe.mapSlug,
expectedName: server.name!, previewId: childPreviewId, plutoKey: keys[server.name!] as string | undefined, createdBy: 'ui' });
expectedName: server.name!, previewId: childPreviewId, plutoKey: keys[server.name!] as string | undefined, createdBy: 'ui' }, releasingServers);
// Throw so the outer transaction also reverses all earlier keys and children.
if (!result.ok) throw new Error(`${server.name}: ${result.message} Nothing was queued.`);
db().prepare('INSERT INTO server_collection_members (collection_id, deployment_id, position) VALUES (?, ?, ?)')
.run(collectionId, result.deploymentId, index);
for (const removal of removalJobs.filter(removal => removal.ports.includes(server.port))) {
db().prepare('INSERT INTO server_creation_dependencies (deployment_id, removal_id) VALUES (?, ?)').run(result.deploymentId, removal.id);
}
});
db().prepare('UPDATE server_collection_previews SET collection_id = ? WHERE id = ?').run(collectionId, preview.id);
return { ok: true as const, collectionId };
})();
}
const summarySql = `SELECT c.id, c.name, c.created_at, COUNT(d.id) AS total,
const membershipSql = `SELECT collection_id, deployment_id, position FROM server_collection_members
UNION ALL SELECT collection_id, deployment_id, position FROM server_collection_removals`;
const summarySql = `SELECT c.id, c.name, c.created_at, c.mode, COUNT(d.id) AS total,
COALESCE(SUM(d.state IN ('done', 'warning')), 0) AS done,
COALESCE(SUM(d.state IN ('failed', 'interrupted')), 0) AS failed,
COALESCE(SUM(d.state IN ('failed', 'interrupted', 'cancelled')), 0) AS failed,
COALESCE(SUM(d.state IN ('queued', 'running', 'awaiting-health', 'waiting')), 0) AS active,
COALESCE(SUM(d.state = 'queued'), 0) AS queued,
COALESCE(SUM(d.state IN ('queued', 'waiting')), 0) AS queued,
COALESCE(SUM(d.state = 'running'), 0) AS provisioning,
COALESCE(SUM(d.state = 'awaiting-health'), 0) AS verifying
FROM server_launch_collections c
LEFT JOIN server_collection_members m ON m.collection_id = c.id
LEFT JOIN (${membershipSql}) m ON m.collection_id = c.id
LEFT JOIN server_deployments d ON d.id = m.deployment_id`;
export function listCollections(): CollectionSummary[] {
@@ -148,12 +209,13 @@ export function listCollections(): CollectionSummary[] {
export function getCollection(id: string): CollectionDetail | null {
const collection = db().prepare(summarySql + ' WHERE c.id = ? GROUP BY c.id').get(id) as CollectionSummary | undefined;
if (!collection) return null;
const servers = db().prepare(`SELECT d.id, d.server, d.state, d.stage, d.started_at, d.finished_at, d.error, j.port, j.map_slug,
const servers = db().prepare(`SELECT d.id, d.server, d.action, d.state, d.stage, d.started_at, d.finished_at, d.error, j.port, j.map_slug,
EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server) AS server_exists
FROM server_collection_members m JOIN server_deployments d ON d.id = m.deployment_id
JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE m.collection_id = ? ORDER BY m.position`).all(id) as CollectionDetail['servers'];
FROM (${membershipSql}) m JOIN server_deployments d ON d.id = m.deployment_id
LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE m.collection_id = ? ORDER BY d.action DESC, m.position`).all(id) as CollectionDetail['servers'];
const queue = creationQueue();
const positions = new Map(queue.queued.map((job, index) => [job.id, index + 1]));
return { collection, queue: { concurrency: config.createConcurrency, active: queue.active, queued: queue.queued.length },
const stored = db().prepare('SELECT retained_servers FROM server_launch_collections WHERE id = ?').get(id) as { retained_servers: string };
return { collection, retainedServers: JSON.parse(stored.retained_servers), queue: { concurrency: config.createConcurrency, active: queue.active, queued: queue.queued.length },
servers: servers.map(server => ({ ...server, queuePosition: positions.get(server.id) ?? null })) };
}