import { scalingOwnershipBlocker } from '../autoscaling/ownership.js'; import { fleetSettings } from '../settings.js'; import { randomUUID } from 'node:crypto'; import type { CollectionDetail, CollectionMode, CollectionPreview, CollectionSummary } from '$lib/deployment-plan.js'; import { db } from '../db/index.js'; import { blueprintEntries, summarizeRecipe, type BlueprintEntry } from './launch-recipes.js'; import { creationQueue, destroyServer } from '../provisioning/index.js'; import { deploymentStage, finishDeployment, recordDeploymentEvent } from './deployments.js'; import { housePartyTargets } from './house-party.js'; import { serverObservation } from './status.js'; import { activeBetaRemovalBlocker, getServer } from './servers.js'; import { parseServerName, zombieMapBySlug, zombieMaps } from './map-aliases.js'; import { nativeLaunchSelection, queueNativeCreationIntent } from '../provisioning/native.js'; import { nativeDeployment, nativeInstance } from '../backends/native.js'; import { launchAvailability } from '../hosts/availability.js'; import { availableServerKeyCount } from './server-keys.js'; import { canCancelCreation, cancelCreation } from '../provisioning/cancel-creation.js'; import { canCancelRemoval, cancelRemoval } from '../provisioning/history.js'; function coverage(entries: BlueprintEntry[], includeHealth = true) { type Server = { name: string; game: string | null; map_slug: string | null; map: string | null; }; const registered = db() .prepare('SELECT name, game, map_slug, map FROM servers WHERE destroyed_at IS NULL') .all() as Server[]; const queued = db() .prepare( `SELECT j.server AS name, substr(j.server, 1, 2) AS game, j.map_slug, j.expected_map AS map FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.server_deleted_at IS NULL AND d.state IN ('queued', 'running', 'awaiting-health')` ) .all() as Server[]; const removing = new Set( ( db() .prepare( "SELECT server FROM server_deployments WHERE server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')" ) .all() as { server: string }[] ).map((row) => row.server) ); // A provisioning server can appear in both tables; count its identity once. const servers = [ ...new Map([...queued, ...registered].map((server) => [server.name, server])).values() ]; return entries.map((entry) => { const existingServers = servers .filter((server) => { if (removing.has(server.name)) return false; const parsed = parseServerName(server.name); const game = server.game ?? parsed?.game; if (game !== entry.recipe.game) return false; if (server.map_slug) return server.map_slug === entry.recipe.mapSlug; const namedMap = parsed?.mapCanonical ? zombieMapBySlug(game, parsed.mapSlug) : undefined; if (namedMap && (!server.map || server.map === namedMap.canonical)) return namedMap.slug === entry.recipe.mapSlug; // Legacy canonical maps are usable only when they identify one variant. const matches = zombieMaps(game).filter((map) => map.canonical === server.map); return matches.length === 1 && matches[0].slug === entry.recipe.mapSlug; }) .map((server) => server.name) .sort(); const coverage = includeHealth ? existingServers.map((name) => { const operation = db() .prepare( "SELECT id, state FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('create','rebuild') AND state IN ('queued', 'running', 'awaiting-health') ORDER BY rowid DESC LIMIT 1" ) .get(name) as { id: string; state: string } | undefined; const server = getServer(name); return { name, health: operation ? operation.state === 'queued' ? 'queued' : 'starting' : server?.container_state && ['stopped', 'exited', 'created', 'dead'].includes( server.container_state ) ? 'stopped' : server ? serverObservation(server).health : 'offline', href: operation ? `/provisioning/${encodeURIComponent(operation.id)}` : `/servers/${encodeURIComponent(name)}` }; }) : []; return { existingServers, coverage, existing: existingServers.length, missing: Math.max(0, entry.count - existingServers.length) }; }); } function fleetSnapshot() { return housePartyTargets().map((name) => ({ name, server: db() .prepare('SELECT game, map_slug, map, game_port FROM servers WHERE name = ?') .get(name) ?? null, creation: db() .prepare( "SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action IN ('create','rebuild') ORDER BY rowid DESC LIMIT 1" ) .get(name) ?? null, removal: db() .prepare( "SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'destroy' AND state IN ('waiting', 'running')" ) .get(name) ?? null })); } function collectionPlan(entries: BlueprintEntry[], mode: CollectionMode, includeHealth = true) { const counts = coverage(entries, includeHealth); // Preserve deployed lobbies before queued ones, then the lowest lobby IDs. const retainedServers = counts.flatMap((entry, index) => [...entry.existingServers] .sort( (a, b) => Number(!!getServer(b)) - Number(!!getServer(a)) || a.localeCompare(b, 'en', { numeric: true }) ) .slice(0, mode === 'exact' ? entries[index].count : undefined) ); const retained = new Set(retainedServers); const matching = new Set(counts.flatMap((entry) => entry.existingServers)); const removals = mode === 'exact' ? housePartyTargets() .filter((name) => !retained.has(name)) .map((name) => ({ name, reason: matching.has(name) ? 'Exceeds the target lobby count' : 'Outside this collection or already awaiting removal' })) : []; return { counts, retainedServers, removals }; } function allocate( entries: BlueprintEntry[], counts = coverage(entries) ): CollectionPreview['servers'] { const occupied = db() .prepare( `SELECT name FROM servers UNION SELECT server AS name FROM native_instances WHERE retired_at IS NULL UNION SELECT server_id AS name FROM server_key_leases UNION SELECT j.server AS name FROM server_creation_jobs j JOIN server_deployments d ON d.id=j.deployment_id WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued','running','awaiting-health') OR (d.action='rebuild' AND (d.state='waiting' OR (d.state IN ('failed','interrupted') AND d.stage='Removing server files and registry'))))` ) .all() as { name: string }[]; const lobbies = new Map>(); for (const { name } of occupied) { const match = /^(.*)_(\d+)$/.exec(name); if (!match) continue; const taken = lobbies.get(match[1]) ?? new Set(); taken.add(Number(match[2])); lobbies.set(match[1], taken); } const availableKeys = { t5: availableServerKeyCount('t5'), t6: availableServerKeyCount('t6') }; return entries.flatMap((entry, entryIndex) => Array.from({ length: counts[entryIndex].missing }, () => { const base = `${entry.recipe.game}_zombie_${entry.recipe.mapSlug}`; const taken = lobbies.get(base) ?? new Set(); const lobby = Array.from({ length: 20 }, (_, index) => index + 1).find( (number) => !taken.has(number) ) ?? null; const name = lobby === null ? null : `${base}_${lobby}`; const native = entry.recipe.native; const port = null; if (lobby !== null) { taken.add(lobby); lobbies.set(base, taken); } const hasSavedKey = !!name && availableKeys[entry.recipe.game] > 0; if (hasSavedKey) availableKeys[entry.recipe.game]--; return { entryIndex, name, port, ...(native ? { native } : {}), hasSavedKey }; }) ); } function blockersFor( servers: CollectionPreview['servers'], entries: BlueprintEntry[] = [] ): string[] { const blockers: string[] = []; for (const server of servers) if (server.name) { const owned = scalingOwnershipBlocker(server.name); if (owned) blockers.push(owned); } if (!servers.length) return blockers; if (servers.some((server) => !server.native && !entries[server.entryIndex]?.recipe.native)) blockers.push('Select and prepare a host before launching.'); if (servers.some((server) => !server.name)) blockers.push( 'There are not enough free lobbies for one or more maps (20 per map). Reduce their counts or free capacity.' ); return blockers; } function removalBlockersFor(removals: CollectionPreview['removals']): string[] { const blockers: string[] = []; const native = (name: string) => { if (nativeInstance(name)) return true; const creation = db() .prepare( `SELECT id FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND action='create' AND state IN ('queued','running','awaiting-health') ORDER BY rowid DESC LIMIT 1` ) .get(name) as { id: string } | undefined; return !!creation && nativeDeployment(creation.id); }; if (removals.some((server) => !native(server.name))) blockers.push('Every server to remove must have a native host instance.'); for (const server of removals) { const owned = scalingOwnershipBlocker(server.name); if (owned) blockers.push(owned); const blocker = activeBetaRemovalBlocker(server.name); if (blocker) blockers.push(blocker); } return blockers; } export function previewCollection( blueprintId: unknown, mode: unknown = 'fill', selection?: Record ): CollectionPreview { if (typeof blueprintId !== 'string' || !blueprintId) throw new Error('Choose a collection to launch.'); if (mode !== 'fill' && mode !== 'exact') throw new Error('Choose fill or exact collection mode.'); return db().transaction((): CollectionPreview => { const { entries, blueprintName } = blueprintEntries(blueprintId); const libraries = selection?.libraryIds as Record | undefined; for (const entry of entries) { entry.recipe.native = nativeLaunchSelection( { ...selection, libraryId: libraries?.[entry.recipe.game] ?? selection?.libraryId }, entry.recipe.game ); entry.recipe.image = entry.recipe.native.image!; } const { counts, retainedServers, removals } = collectionPlan(entries, mode); const servers = allocate(entries, counts); const blockers = [...blockersFor(servers, entries), ...removalBlockersFor(removals)]; const selections = servers.flatMap((server) => (server.native ? [server.native] : [])); const availability = selections.length ? launchAvailability(selections) : {}; const id = randomUUID(); const expiresAt = new Date(Date.now() + 30 * 60_000).toISOString(); db() .prepare( 'DELETE FROM server_collection_previews WHERE expires_at < ? AND collection_id IS NULL' ) .run(new Date().toISOString()); db() .prepare( 'INSERT INTO server_collection_previews (id, expires_at, blueprint_name, entries, servers, mode, fleet_snapshot) VALUES (?, ?, ?, ?, ?, ?, ?)' ) .run( id, expiresAt, blueprintName, JSON.stringify(entries), JSON.stringify(servers), mode, mode === 'exact' ? JSON.stringify(fleetSnapshot()) : null ); return { id, expiresAt, blueprintName, mode, retainedServers, removals, launchConcurrency: fleetSettings().createConcurrency, servers, blockers, ...availability, ready: blockers.length === 0, entries: entries.map((entry, index) => ({ count: entry.count, ...counts[index], recipe: summarizeRecipe(entry.recipe) })) }; })(); } /** Save all reviewed identities and keys atomically; native placement belongs to provisioning. */ export function queueCollection(input: { previewId?: unknown; keys?: unknown }): { ok: true; collectionId: string | null; } { if (typeof input.previewId !== 'string' || !input.previewId) throw new Error('Review a collection preview before launching.'); if ( input.keys !== undefined && (!input.keys || typeof input.keys !== 'object' || Array.isArray(input.keys)) ) throw new Error('Provide keys indexed by the exact server ID.'); const keys = (input.keys ?? {}) as Record; return db().transaction(() => { const preview = db() .prepare('SELECT * FROM server_collection_previews WHERE id = ?') .get(input.previewId) as | { id: string; expires_at: string; blueprint_name: string; entries: string; servers: string; collection_id: string | null; mode: CollectionMode; fleet_snapshot: string | null; } | undefined; if (!preview) throw new Error('Collection preview expired or was not found. Refresh the preview.'); if (preview.collection_id) return { ok: true as const, collectionId: preview.collection_id }; if (preview.expires_at <= new Date().toISOString()) throw new Error('Collection preview expired. Refresh it before launching.'); const entries: BlueprintEntry[] = JSON.parse(preview.entries); // Older previews embedded plugin bytes. Queued jobs need only the // reviewed metadata; provisioning captures the current files at dispatch. for (const entry of entries) entry.recipe.files = entry.recipe.files.map(({ content: _content, ...file }) => file); const reviewed: CollectionPreview['servers'] = JSON.parse(preview.servers); if ( preview.mode === 'exact' && preview.fleet_snapshot !== JSON.stringify(fleetSnapshot()) ) { throw new Error( 'The fleet changed. Nothing was queued. Refresh the preview and review the servers to keep and remove.' ); } const { counts, retainedServers, removals } = collectionPlan(entries, preview.mode, false); const servers = reviewed; const validatedSelections = new Set(); for (const server of servers) { const recipe = entries[server.entryIndex].recipe; const selection = server.native ?? recipe.native; if (!selection) throw new Error('Select a native host before launching. Refresh the preview.'); const identity = JSON.stringify([recipe.game, selection]); if (validatedSelections.has(identity)) continue; // Recheck saved configuration once per selection without allocating // physical capacity or changing the names and files already reviewed. nativeLaunchSelection({ ...selection }, recipe.game); validatedSelections.add(identity); } const blockers = [...blockersFor(servers, entries), ...removalBlockersFor(removals)]; if (blockers.length) throw new Error(blockers.join(' ')); if ( entries.some( (_entry, index) => counts[index].missing !== reviewed.filter((server) => server.entryIndex === index).length ) ) { throw new Error( 'Fleet counts changed. Nothing was queued. Refresh the preview and review the launches.' ); } const expected = new Set(servers.map((server) => server.name)); for (const [name, value] of Object.entries(keys)) { if (!expected.has(name) || typeof value !== 'string') throw new Error('Keys must be strings for server IDs shown in this preview.'); } if (!servers.length && !removals.length) return { ok: true as const, collectionId: null }; const collectionId = randomUUID(); db() .prepare( 'INSERT INTO server_launch_collections (id, name, created_at, mode, retained_servers) VALUES (?, ?, ?, ?, ?)' ) .run( collectionId, preview.blueprint_name, new Date().toISOString(), preview.mode, JSON.stringify(retainedServers) ); removals.forEach((server, position) => { const queued = db() .prepare( "SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'create' AND state = 'queued'" ) .get(server.name) as { id: string } | undefined; let id: string; if (queued && !getServer(server.name)) { // No container or files exist yet: cancel the reservation instead of booting just to remove it. const result = destroyServer(server.name, 'blueprint-exact'); if (!result.ok) throw new Error(result.message); if (!result.deploymentId) throw new Error('Could not queue removal for ' + server.name); id = result.deploymentId; db() .prepare( "UPDATE server_deployments SET state = 'cancelled', stage = 'Cancelled by exact collection', finished_at = ? WHERE id = ?" ) .run(new Date().toISOString(), queued.id); recordDeploymentEvent( queued.id, 'Queued creation cancelled by exact collection; saved key retained.' ); deploymentStage(id, 'Cancelled queued creation; no server was started'); finishDeployment(id, { ok: true }); } else { const result = destroyServer(server.name, 'blueprint-exact'); if (!result.ok) throw new Error(result.message); if (!result.deploymentId) throw new Error('Could not queue removal for ' + server.name); id = result.deploymentId; } db() .prepare( 'INSERT INTO server_collection_removals (collection_id, deployment_id, position) VALUES (?, ?, ?)' ) .run(collectionId, id, position); }); servers.forEach((server, index) => { const recipe = { ...entries[server.entryIndex].recipe, ...(server.native ? { native: server.native } : {}) }; if (!recipe.native) throw new Error('Select a native host before launching. Refresh the preview.'); const result = queueNativeCreationIntent({ server: server.name!, game: recipe.game, mapSlug: recipe.mapSlug, selection: recipe.native, recipe, plutoKey: keys[server.name!] as string | undefined, createdBy: 'ui', blueprintName: preview.blueprint_name }); // Throw so the outer transaction also reverses all earlier keys and children. if (!result.ok) throw new Error(`${server.name}: ${result.message} Nothing was queued.`); db() .prepare( 'INSERT INTO server_collection_members (collection_id, deployment_id, position) VALUES (?, ?, ?)' ) .run(collectionId, result.deploymentId, index); }); db() .prepare('UPDATE server_collection_previews SET collection_id = ? WHERE id = ?') .run(collectionId, preview.id); return { ok: true as const, collectionId }; })(); } const membershipSql = `SELECT collection_id, deployment_id, position FROM server_collection_members UNION ALL SELECT collection_id, deployment_id, position FROM server_collection_removals`; function collectionCancellationCandidates(id: string) { const creations = ( db() .prepare( `SELECT d.id FROM server_collection_members m JOIN server_deployments d ON d.id=m.deployment_id WHERE m.collection_id=? AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations cc WHERE cc.creation_id=d.id)` ) .all(id) as { id: string }[] ).filter((job) => canCancelCreation(job.id)); // Exact launches can reuse manual removals or creation cleanup jobs. Membership // alone must never give a launch permission to withdraw those operations. const removals = ( db() .prepare( `SELECT d.id FROM server_collection_removals m JOIN server_deployments d ON d.id=m.deployment_id WHERE m.collection_id=? AND d.triggered_by='blueprint-exact' AND NOT EXISTS (SELECT 1 FROM native_provisioning n WHERE n.deployment_id=d.id AND n.operation_id IS NOT NULL) AND NOT EXISTS (SELECT 1 FROM native_creation_cancellations cc WHERE cc.cleanup_id=d.id)` ) .all(id) as { id: string }[] ).filter((job) => canCancelRemoval(job.id)); return { creations, removals }; } /** Withdraw the remaining launch atomically; partial instances keep durable cleanup jobs. */ export function cancelCollection(id: string): boolean { return db().transaction(() => { const collection = db() .prepare('SELECT cancelled_at FROM server_launch_collections WHERE id=?') .get(id) as { cancelled_at: string | null } | undefined; if (!collection) return false; if (collection.cancelled_at) return true; const { creations, removals } = collectionCancellationCandidates(id); if (!creations.length && !removals.length) return false; db() .prepare('UPDATE server_launch_collections SET cancelled_at=? WHERE id=?') .run(new Date().toISOString(), id); for (const creation of creations) { if (!cancelCreation(creation.id, 'collection-cancel')) throw new Error( 'Could not cancel a collection creation. Refresh the launch and try again.' ); } for (const removal of removals) { // A shared removal remains requested until every owning launch withdraws. const shared = db() .prepare( `SELECT 1 FROM server_collection_removals m JOIN server_launch_collections c ON c.id=m.collection_id WHERE m.deployment_id=? AND c.id!=? AND c.cancelled_at IS NULL LIMIT 1` ) .get(removal.id, id); if (!shared) cancelRemoval(removal.id); } return true; })(); } const summarySql = `SELECT c.id, c.name, c.created_at, c.cancelled_at, c.mode, COUNT(d.id) AS total, COALESCE(SUM(d.state IN ('done', 'warning')), 0) AS done, COALESCE(SUM(d.state IN ('failed', 'interrupted')), 0) AS failed, COALESCE(SUM(d.state = 'cancelled'), 0) AS cancelled, COALESCE(SUM(cleanup.state IN ('queued', 'running', 'awaiting-health', 'waiting')), 0) AS cleanupActive, COALESCE(SUM(cleanup.state IN ('failed', 'interrupted', 'warning', 'cancelled')), 0) AS cleanupFailed, COALESCE(SUM(d.state IN ('queued', 'running', 'awaiting-health', 'waiting')), 0) AS active, COALESCE(SUM(d.state IN ('queued', 'waiting')), 0) AS queued, COALESCE(SUM(d.state = 'running'), 0) AS provisioning, COALESCE(SUM(d.state = 'awaiting-health'), 0) AS verifying FROM server_launch_collections c LEFT JOIN (${membershipSql}) m ON m.collection_id = c.id LEFT JOIN server_deployments d ON d.id = m.deployment_id LEFT JOIN native_creation_cancellations cc ON cc.creation_id = d.id LEFT JOIN server_deployments cleanup ON cleanup.id = cc.cleanup_id`; export function listCollections(): CollectionSummary[] { return db() .prepare(summarySql + ' GROUP BY c.id ORDER BY c.created_at DESC, c.rowid DESC') .all() as CollectionSummary[]; } export function getCollection(id: string): CollectionDetail | null { const collection = db() .prepare(summarySql + ' WHERE c.id = ? GROUP BY c.id') .get(id) as CollectionSummary | undefined; if (!collection) return null; const servers = db() .prepare( `SELECT d.id, d.server, d.server_deleted_at, d.action, d.state, d.stage, d.started_at, d.finished_at, d.error, j.port, j.map_slug, (d.server_deleted_at IS NULL AND EXISTS(SELECT 1 FROM servers s WHERE s.name = d.server AND s.destroyed_at IS NULL)) AS server_exists FROM (${membershipSql}) m JOIN server_deployments d ON d.id = m.deployment_id LEFT JOIN server_creation_jobs j ON j.deployment_id = d.id WHERE m.collection_id = ? ORDER BY d.action DESC, m.position` ) .all(id) as CollectionDetail['servers']; const queue = creationQueue(); const positions = new Map(queue.queued.map((job, index) => [job.id, index + 1])); const stored = db() .prepare( 'SELECT retained_servers, retained_server_deletions FROM server_launch_collections WHERE id = ?' ) .get(id) as { retained_servers: string; retained_server_deletions: string }; const cancellations = db() .prepare( `SELECT cc.creation_id, cc.cleanup_id AS cleanupId, d.state, d.stage, d.error FROM server_collection_members m JOIN native_creation_cancellations cc ON cc.creation_id=m.deployment_id JOIN server_deployments d ON d.id=cc.cleanup_id WHERE m.collection_id=?` ) .all(id) as (NonNullable & { creation_id: string; })[]; const cleanupByCreation = new Map( cancellations.map(({ creation_id, ...cleanup }) => [creation_id, cleanup]) ); const candidates = collection.cancelled_at ? null : collectionCancellationCandidates(id); return { collection, canCancel: !!candidates && (candidates.creations.length > 0 || candidates.removals.length > 0), retainedServers: JSON.parse(stored.retained_servers), retainedServerDeletedAt: JSON.parse(stored.retained_server_deletions), queue: { concurrency: fleetSettings().createConcurrency, active: queue.active, queued: queue.queued.length }, servers: servers.map((server) => ({ ...server, queuePosition: positions.get(server.id) ?? null, cancellation: cleanupByCreation.get(server.id) ?? null })) }; }