262 lines
20 KiB
TypeScript
262 lines
20 KiB
TypeScript
import { fleetSettings } from '../settings.js';
|
|
import { randomUUID } from 'node:crypto';
|
|
import type { CollectionDetail, CollectionMode, CollectionPreview, CollectionSummary } from '$lib/deployment-plan.js';
|
|
import { db } from '../db/index.js';
|
|
import { blueprintEntries, summarizeRecipe, type BlueprintEntry } from './launch-recipes.js';
|
|
import { creationQueue, destroyServer } from '../provisioning/index.js';
|
|
import { deploymentStage, finishDeployment, recordDeploymentEvent } from './deployments.js';
|
|
import { housePartyTargets } from './house-party.js';
|
|
import { serverObservation } from './status.js';
|
|
import { activeBetaRemovalBlocker, getServer } from './servers.js';
|
|
import { parseServerName, zombieMapBySlug, zombieMaps } from './map-aliases.js';
|
|
import { nativeLaunchSelection, queueNativeCreationIntent } from '../provisioning/native.js';
|
|
import { nativeDeployment, nativeInstance } from '../backends/native.js';
|
|
import { launchAvailability } from '../hosts/availability.js';
|
|
|
|
function coverage(entries: BlueprintEntry[], includeHealth = true) {
|
|
type Server = { name: string; game: string | null; map_slug: string | null; map: string | null };
|
|
const registered = db().prepare('SELECT name, game, map_slug, map FROM servers WHERE destroyed_at IS NULL').all() as Server[];
|
|
const queued = db().prepare(`SELECT j.server AS name, substr(j.server, 1, 2) AS game, j.map_slug, j.expected_map AS map
|
|
FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
|
|
WHERE d.server_deleted_at IS NULL AND d.state IN ('queued', 'running', 'awaiting-health')`).all() as Server[];
|
|
const removing = new Set((db().prepare("SELECT server FROM server_deployments WHERE server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')").all() as { server: string }[]).map(row => row.server));
|
|
// A provisioning server can appear in both tables; count its identity once.
|
|
const servers = [...new Map([...queued, ...registered].map(server => [server.name, server])).values()];
|
|
return entries.map(entry => {
|
|
const existingServers = servers.filter(server => {
|
|
if (removing.has(server.name)) return false;
|
|
const parsed = parseServerName(server.name);
|
|
const game = server.game ?? parsed?.game;
|
|
if (game !== entry.recipe.game) return false;
|
|
if (server.map_slug) return server.map_slug === entry.recipe.mapSlug;
|
|
const namedMap = parsed?.mapCanonical ? zombieMapBySlug(game, parsed.mapSlug) : undefined;
|
|
if (namedMap && (!server.map || server.map === namedMap.canonical)) return namedMap.slug === entry.recipe.mapSlug;
|
|
// Legacy canonical maps are usable only when they identify one variant.
|
|
const matches = zombieMaps(game).filter(map => map.canonical === server.map);
|
|
return matches.length === 1 && matches[0].slug === entry.recipe.mapSlug;
|
|
}).map(server => server.name).sort();
|
|
const coverage = includeHealth ? existingServers.map(name => {
|
|
const operation = db().prepare("SELECT id, state FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('create','rebuild') AND state IN ('queued', 'running', 'awaiting-health') ORDER BY rowid DESC LIMIT 1").get(name) as { id: string; state: string } | undefined;
|
|
const server = getServer(name);
|
|
return { name, health: operation ? operation.state === 'queued' ? 'queued' : 'starting'
|
|
: server?.container_state && ['stopped', 'exited', 'created', 'dead'].includes(server.container_state) ? 'stopped'
|
|
: server ? serverObservation(server).health : 'offline',
|
|
href: operation ? `/provisioning/${encodeURIComponent(operation.id)}` : `/servers/${encodeURIComponent(name)}` };
|
|
}) : [];
|
|
return { existingServers, coverage, existing: existingServers.length, missing: Math.max(0, entry.count - existingServers.length) };
|
|
});
|
|
}
|
|
|
|
function fleetSnapshot() {
|
|
return housePartyTargets().map(name => ({ name,
|
|
server: db().prepare('SELECT game, map_slug, map, game_port FROM servers WHERE name = ?').get(name) ?? null,
|
|
creation: db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('create','rebuild') ORDER BY rowid DESC LIMIT 1").get(name) ?? null,
|
|
removal: db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')").get(name) ?? null
|
|
}));
|
|
}
|
|
|
|
function collectionPlan(entries: BlueprintEntry[], mode: CollectionMode, includeHealth = true) {
|
|
const counts = coverage(entries, includeHealth);
|
|
// Preserve deployed lobbies before queued ones, then the lowest lobby IDs.
|
|
const retainedServers = counts.flatMap((entry, index) => [...entry.existingServers].sort((a, b) =>
|
|
Number(!!getServer(b)) - Number(!!getServer(a)) || a.localeCompare(b, 'en', { numeric: true })
|
|
).slice(0, mode === 'exact' ? entries[index].count : undefined));
|
|
const retained = new Set(retainedServers);
|
|
const matching = new Set(counts.flatMap(entry => entry.existingServers));
|
|
const removals = mode === 'exact' ? housePartyTargets().filter(name => !retained.has(name)).map(name => ({ name,
|
|
reason: matching.has(name) ? 'Exceeds the target lobby count' : 'Outside this collection or already awaiting removal'
|
|
})) : [];
|
|
return { counts, retainedServers, removals };
|
|
}
|
|
|
|
function allocate(entries: BlueprintEntry[], counts = coverage(entries)): CollectionPreview['servers'] {
|
|
const occupied = db().prepare(`SELECT name FROM servers UNION
|
|
SELECT j.server AS name FROM server_creation_jobs j JOIN server_deployments d ON d.id=j.deployment_id
|
|
WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued','running','awaiting-health')
|
|
OR (d.action='rebuild' AND (d.state='waiting' OR (d.state IN ('failed','interrupted') AND d.stage='Removing server files and registry'))))`)
|
|
.all() as { name: string }[];
|
|
const lobbies = new Map<string, Set<number>>();
|
|
for (const { name } of occupied) {
|
|
const match = /^(.*)_(\d+)$/.exec(name);
|
|
if (!match) continue;
|
|
const taken = lobbies.get(match[1]) ?? new Set<number>();
|
|
taken.add(Number(match[2])); lobbies.set(match[1], taken);
|
|
}
|
|
const savedKeys = new Set((db().prepare("SELECT server_id FROM server_keys WHERE length(trim(pluto_key))>0").all() as { server_id: string }[]).map(row => row.server_id));
|
|
return entries.flatMap((entry, entryIndex) => Array.from({ length: counts[entryIndex].missing }, () => {
|
|
const base = `${entry.recipe.game}_zombie_${entry.recipe.mapSlug}`;
|
|
const taken = lobbies.get(base) ?? new Set<number>();
|
|
const lobby = Array.from({ length: 20 }, (_, index) => index + 1).find(number => !taken.has(number)) ?? null;
|
|
const name = lobby === null ? null : `${base}_${lobby}`;
|
|
const native = entry.recipe.native;
|
|
const port = null;
|
|
if (lobby !== null) { taken.add(lobby); lobbies.set(base, taken); }
|
|
return { entryIndex, name, port, ...(native ? { native } : {}), hasSavedKey: !!name && savedKeys.has(name) };
|
|
}));
|
|
}
|
|
|
|
function blockersFor(servers: CollectionPreview['servers'], entries: BlueprintEntry[] = []): string[] {
|
|
const blockers: string[] = [];
|
|
if (!servers.length) return blockers;
|
|
if (servers.some(server => !server.native && !entries[server.entryIndex]?.recipe.native)) blockers.push('Select and prepare a host before launching.');
|
|
if (servers.some(server => !server.name)) blockers.push('There are not enough free lobbies for one or more maps (20 per map). Reduce their counts or free capacity.');
|
|
return blockers;
|
|
}
|
|
|
|
function removalBlockersFor(removals: CollectionPreview['removals']): string[] {
|
|
const blockers: string[] = [];
|
|
const native = (name: string) => {
|
|
if (nativeInstance(name)) return true;
|
|
const creation = db().prepare(`SELECT id FROM server_deployments WHERE server=? AND server_deleted_at IS NULL
|
|
AND action='create' AND state IN ('queued','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(name) as { id: string } | undefined;
|
|
return !!creation && nativeDeployment(creation.id);
|
|
};
|
|
if (removals.some(server => !native(server.name))) blockers.push('Every server to remove must have a native host instance.');
|
|
for (const server of removals) {
|
|
const blocker = activeBetaRemovalBlocker(server.name);
|
|
if (blocker) blockers.push(blocker);
|
|
}
|
|
return blockers;
|
|
}
|
|
|
|
export function previewCollection(blueprintId: unknown, mode: unknown = 'fill', selection?: Record<string, unknown>): CollectionPreview {
|
|
if (typeof blueprintId !== 'string' || !blueprintId) throw new Error('Choose a collection to launch.');
|
|
if (mode !== 'fill' && mode !== 'exact') throw new Error('Choose fill or exact collection mode.');
|
|
return db().transaction((): CollectionPreview => {
|
|
const { entries, blueprintName } = blueprintEntries(blueprintId);
|
|
const libraries = selection?.libraryIds as Record<string, unknown> | undefined;
|
|
for (const entry of entries) {
|
|
entry.recipe.native = nativeLaunchSelection({ ...selection, libraryId: libraries?.[entry.recipe.game] ?? selection?.libraryId }, entry.recipe.game);
|
|
entry.recipe.image = entry.recipe.native.image!;
|
|
}
|
|
const { counts, retainedServers, removals } = collectionPlan(entries, mode);
|
|
const servers = allocate(entries, counts);
|
|
const blockers = [...blockersFor(servers, entries), ...removalBlockersFor(removals)];
|
|
const selections = servers.flatMap(server => server.native ? [server.native] : []);
|
|
const availability = selections.length ? launchAvailability(selections) : {};
|
|
const id = randomUUID();
|
|
const expiresAt = new Date(Date.now() + 30 * 60_000).toISOString();
|
|
db().prepare('DELETE FROM server_collection_previews WHERE expires_at < ? AND collection_id IS NULL').run(new Date().toISOString());
|
|
db().prepare('INSERT INTO server_collection_previews (id, expires_at, blueprint_name, entries, servers, mode, fleet_snapshot) VALUES (?, ?, ?, ?, ?, ?, ?)')
|
|
.run(id, expiresAt, blueprintName, JSON.stringify(entries), JSON.stringify(servers), mode, mode === 'exact' ? JSON.stringify(fleetSnapshot()) : null);
|
|
return { id, expiresAt, blueprintName, mode, retainedServers, removals, launchConcurrency: fleetSettings().createConcurrency, servers, blockers, ...availability, ready: blockers.length === 0,
|
|
entries: entries.map((entry, index) => ({ count: entry.count, ...counts[index], recipe: summarizeRecipe(entry.recipe) })) };
|
|
})();
|
|
}
|
|
|
|
/** Save all reviewed identities and keys atomically; native placement belongs to provisioning. */
|
|
export function queueCollection(input: { previewId?: unknown; keys?: unknown }): { ok: true; collectionId: string | null } {
|
|
if (typeof input.previewId !== 'string' || !input.previewId) throw new Error('Review a collection preview before launching.');
|
|
if (input.keys !== undefined && (!input.keys || typeof input.keys !== 'object' || Array.isArray(input.keys))) throw new Error('Provide keys indexed by the exact server ID.');
|
|
const keys = (input.keys ?? {}) as Record<string, unknown>;
|
|
return db().transaction(() => {
|
|
const preview = db().prepare('SELECT * FROM server_collection_previews WHERE id = ?').get(input.previewId) as {
|
|
id: string; expires_at: string; blueprint_name: string; entries: string; servers: string; collection_id: string | null; mode: CollectionMode; fleet_snapshot: string | null;
|
|
} | undefined;
|
|
if (!preview) throw new Error('Collection preview expired or was not found. Refresh the preview.');
|
|
if (preview.collection_id) return { ok: true as const, collectionId: preview.collection_id };
|
|
if (preview.expires_at <= new Date().toISOString()) throw new Error('Collection preview expired. Refresh it before launching.');
|
|
const entries: BlueprintEntry[] = JSON.parse(preview.entries);
|
|
// Older previews embedded plugin bytes. Queued jobs need only the
|
|
// reviewed metadata; provisioning captures the current files at dispatch.
|
|
for (const entry of entries) entry.recipe.files = entry.recipe.files.map(({ content: _content, ...file }) => file);
|
|
const reviewed: CollectionPreview['servers'] = JSON.parse(preview.servers);
|
|
if (preview.mode === 'exact' && preview.fleet_snapshot !== JSON.stringify(fleetSnapshot())) {
|
|
throw new Error('The fleet changed. Nothing was queued. Refresh the preview and review the servers to keep and remove.');
|
|
}
|
|
const { counts, retainedServers, removals } = collectionPlan(entries, preview.mode, false);
|
|
const servers = reviewed;
|
|
const validatedSelections = new Set<string>();
|
|
for (const server of servers) {
|
|
const recipe = entries[server.entryIndex].recipe;
|
|
const selection = server.native ?? recipe.native;
|
|
if (!selection) throw new Error('Select a native host before launching. Refresh the preview.');
|
|
const identity = JSON.stringify([recipe.game, selection]);
|
|
if (validatedSelections.has(identity)) continue;
|
|
// Recheck saved configuration once per selection without allocating
|
|
// physical capacity or changing the names and files already reviewed.
|
|
nativeLaunchSelection({ ...selection }, recipe.game);
|
|
validatedSelections.add(identity);
|
|
}
|
|
const blockers = [...blockersFor(servers, entries), ...removalBlockersFor(removals)];
|
|
if (blockers.length) throw new Error(blockers.join(' '));
|
|
if (entries.some((_entry, index) => counts[index].missing !== reviewed.filter(server => server.entryIndex === index).length)) {
|
|
throw new Error('Fleet counts changed. Nothing was queued. Refresh the preview and review each server key.');
|
|
}
|
|
const expected = new Set(servers.map(server => server.name));
|
|
for (const [name, value] of Object.entries(keys)) {
|
|
if (!expected.has(name) || typeof value !== 'string') throw new Error('Keys must be strings for server IDs shown in this preview.');
|
|
}
|
|
if (!servers.length && !removals.length) return { ok: true as const, collectionId: null };
|
|
const collectionId = randomUUID();
|
|
db().prepare('INSERT INTO server_launch_collections (id, name, created_at, mode, retained_servers) VALUES (?, ?, ?, ?, ?)')
|
|
.run(collectionId, preview.blueprint_name, new Date().toISOString(), preview.mode, JSON.stringify(retainedServers));
|
|
removals.forEach((server, position) => {
|
|
const queued = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'create' AND state = 'queued'").get(server.name) as { id: string } | undefined;
|
|
let id: string;
|
|
if (queued && !getServer(server.name)) {
|
|
// No container or files exist yet: cancel the reservation instead of booting just to remove it.
|
|
const result = destroyServer(server.name, 'blueprint-exact');
|
|
if (!result.ok) throw new Error(result.message);
|
|
if (!result.deploymentId) throw new Error('Could not queue removal for ' + server.name);
|
|
id = result.deploymentId;
|
|
db().prepare("UPDATE server_deployments SET state = 'cancelled', stage = 'Cancelled by exact collection', finished_at = ? WHERE id = ?").run(new Date().toISOString(), queued.id);
|
|
recordDeploymentEvent(queued.id, 'Queued creation cancelled by exact collection; saved key retained.');
|
|
deploymentStage(id, 'Cancelled queued creation; no server was started');
|
|
finishDeployment(id, { ok: true });
|
|
} else {
|
|
const result = destroyServer(server.name, 'blueprint-exact');
|
|
if (!result.ok) throw new Error(result.message);
|
|
if (!result.deploymentId) throw new Error('Could not queue removal for ' + server.name);
|
|
id = result.deploymentId;
|
|
}
|
|
db().prepare('INSERT INTO server_collection_removals (collection_id, deployment_id, position) VALUES (?, ?, ?)').run(collectionId, id, position);
|
|
});
|
|
servers.forEach((server, index) => {
|
|
const recipe = { ...entries[server.entryIndex].recipe, ...(server.native ? { native: server.native } : {}) };
|
|
if (!recipe.native) throw new Error('Select a native host before launching. Refresh the preview.');
|
|
const result = queueNativeCreationIntent({ server: server.name!, game: recipe.game, mapSlug: recipe.mapSlug,
|
|
selection: recipe.native, recipe, plutoKey: keys[server.name!] as string | undefined, createdBy: 'ui', blueprintName: preview.blueprint_name });
|
|
// Throw so the outer transaction also reverses all earlier keys and children.
|
|
if (!result.ok) throw new Error(`${server.name}: ${result.message} Nothing was queued.`);
|
|
db().prepare('INSERT INTO server_collection_members (collection_id, deployment_id, position) VALUES (?, ?, ?)')
|
|
.run(collectionId, result.deploymentId, index);
|
|
|
|
});
|
|
db().prepare('UPDATE server_collection_previews SET collection_id = ? WHERE id = ?').run(collectionId, preview.id);
|
|
return { ok: true as const, collectionId };
|
|
})();
|
|
}
|
|
|
|
const membershipSql = `SELECT collection_id, deployment_id, position FROM server_collection_members
|
|
UNION ALL SELECT collection_id, deployment_id, position FROM server_collection_removals`;
|
|
|
|
const summarySql = `SELECT c.id, c.name, c.created_at, c.mode, COUNT(d.id) AS total,
|
|
COALESCE(SUM(d.state IN ('done', 'warning')), 0) AS done,
|
|
COALESCE(SUM(d.state IN ('failed', 'interrupted', 'cancelled')), 0) AS failed,
|
|
COALESCE(SUM(d.state IN ('queued', 'running', 'awaiting-health', 'waiting')), 0) AS active,
|
|
COALESCE(SUM(d.state IN ('queued', 'waiting')), 0) AS queued,
|
|
COALESCE(SUM(d.state = 'running'), 0) AS provisioning,
|
|
COALESCE(SUM(d.state = 'awaiting-health'), 0) AS verifying
|
|
FROM server_launch_collections c
|
|
LEFT JOIN (${membershipSql}) m ON m.collection_id = c.id
|
|
LEFT JOIN server_deployments d ON d.id = m.deployment_id`;
|
|
|
|
export function listCollections(): CollectionSummary[] {
|
|
return db().prepare(summarySql + ' GROUP BY c.id ORDER BY c.created_at DESC, c.rowid DESC').all() as CollectionSummary[];
|
|
}
|
|
|
|
export function getCollection(id: string): CollectionDetail | null {
|
|
const collection = db().prepare(summarySql + ' WHERE c.id = ? GROUP BY c.id').get(id) as CollectionSummary | undefined;
|
|
if (!collection) return null;
|
|
const servers = db().prepare(`SELECT d.id, d.server, d.server_deleted_at, d.action, d.state, d.stage, d.started_at, d.finished_at, d.error, j.port, j.map_slug,
|
|
(d.server_deleted_at IS NULL AND EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server AND s.destroyed_at IS NULL)) AS server_exists
|
|
FROM (${membershipSql}) m JOIN server_deployments d ON d.id = m.deployment_id
|
|
LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE m.collection_id = ? ORDER BY d.action DESC, m.position`).all(id) as CollectionDetail['servers'];
|
|
const queue = creationQueue();
|
|
const positions = new Map(queue.queued.map((job, index) => [job.id, index + 1]));
|
|
const stored = db().prepare('SELECT retained_servers, retained_server_deletions FROM server_launch_collections WHERE id = ?').get(id) as { retained_servers: string; retained_server_deletions: string };
|
|
return { collection, retainedServers: JSON.parse(stored.retained_servers), retainedServerDeletedAt: JSON.parse(stored.retained_server_deletions), queue: { concurrency: fleetSettings().createConcurrency, active: queue.active, queued: queue.queued.length },
|
|
servers: servers.map(server => ({ ...server, queuePosition: positions.get(server.id) ?? null })) };
|
|
}
|