From dbd2940d290d7eef5bb413c353a02772fc5c7591 Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Mon, 5 Oct 2026 10:15:48 -0400 Subject: [PATCH] Fix blocking collection matching with large plugins --- docs/deployment-blueprints.md | 3 + src/lib/server/db/index.ts | 7 + src/lib/server/db/schema.sql | 5 +- src/lib/server/fleet/collections.ts | 3 + src/lib/server/fleet/launch-recipes.ts | 47 ++++-- src/lib/server/fleet/promoted-artifacts.ts | 17 +- src/lib/server/fleet/scripts.ts | 9 ++ src/lib/server/provisioning/instances.ts | 16 +- src/lib/server/provisioning/native.ts | 14 +- .../api/deployment-collections/+server.ts | 8 +- .../unit/collection-match-performance.test.ts | 145 ++++++++++++++++++ tests/unit/deployment-collections.test.ts | 2 +- tests/unit/launch-recipes.test.ts | 3 +- .../provisioning-history-performance.test.ts | 79 ++++++++++ 14 files changed, 319 insertions(+), 39 deletions(-) create mode 100644 tests/unit/collection-match-performance.test.ts create mode 100644 tests/unit/provisioning-history-performance.test.ts diff --git a/docs/deployment-blueprints.md b/docs/deployment-blueprints.md index 3954c43..48af6af 100644 --- a/docs/deployment-blueprints.md +++ b/docs/deployment-blueprints.md @@ -165,6 +165,9 @@ resolve the promoted setup again when submitted and when provisioning begins, so promotions made after a preview or while a job is queued apply to that server. This applies to collection collections, single-server launches, and **Launch again**. +Previews and queued jobs store file metadata without copying script or plugin +contents for every lobby. Provisioning captures the bytes when each job starts. + The files selected at provisioning, their hashes and version metadata, composed configuration, selected map, and image reference are recorded in SQLite for deployment history. These records do not override promotions on future launches. diff --git a/src/lib/server/db/index.ts b/src/lib/server/db/index.ts index 8d8dfc3..e4adb39 100644 --- a/src/lib/server/db/index.ts +++ b/src/lib/server/db/index.ts @@ -369,6 +369,13 @@ export function db(): Database.Database { backfillLeaderboard(handle); initializeFleetSettings(handle); migrateNativeInstances(handle); + // Lifecycle history is retained indefinitely. Admission and status reads + // must narrow by server or active state instead of scanning that history. + // Install after migrations that rebuild this table or add server_deleted_at. + handle.exec(`CREATE INDEX IF NOT EXISTS idx_server_deployments_server + ON server_deployments(server, server_deleted_at); + CREATE INDEX IF NOT EXISTS idx_server_deployments_state + ON server_deployments(state, action, server_deleted_at);`); installReportRevisions(handle); } catch (err) { handle.close(); diff --git a/src/lib/server/db/schema.sql b/src/lib/server/db/schema.sql index 37f7fa5..cc8c118 100644 --- a/src/lib/server/db/schema.sql +++ b/src/lib/server/db/schema.sql @@ -516,8 +516,9 @@ CREATE TABLE IF NOT EXISTS server_launch_previews ( deployment_id TEXT REFERENCES server_deployments(id) ON DELETE CASCADE ); --- Historical launch files survive library deletion. New launches resolve current --- promotions instead of replaying these snapshots; credentials stay in their stores. +-- Queued plans contain metadata only. Attempted launch files survive library +-- deletion. New launches resolve current promotions instead of replaying these +-- snapshots; credentials stay in their stores. CREATE TABLE IF NOT EXISTS server_launch_recipes ( deployment_id TEXT PRIMARY KEY REFERENCES server_deployments(id) ON DELETE CASCADE, recipe TEXT NOT NULL, diff --git a/src/lib/server/fleet/collections.ts b/src/lib/server/fleet/collections.ts index 2bf8335..20495a5 100644 --- a/src/lib/server/fleet/collections.ts +++ b/src/lib/server/fleet/collections.ts @@ -160,6 +160,9 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }): if (preview.collection_id) return { ok: true as const, collectionId: preview.collection_id }; if (preview.expires_at <= new Date().toISOString()) throw new Error('Collection preview expired. Refresh it before launching.'); const entries: BlueprintEntry[] = JSON.parse(preview.entries); + // Older previews embedded plugin bytes. Child previews need only the + // reviewed metadata; provisioning captures the current files at dispatch. + for (const entry of entries) entry.recipe.files = entry.recipe.files.map(({ content: _content, ...file }) => file); const reviewed: CollectionPreview['servers'] = JSON.parse(preview.servers); if (preview.mode === 'exact' && preview.fleet_snapshot !== JSON.stringify(fleetSnapshot())) { throw new Error('The fleet changed. Nothing was queued. Refresh the preview and review the servers to keep and remove.'); diff --git a/src/lib/server/fleet/launch-recipes.ts b/src/lib/server/fleet/launch-recipes.ts index 23d9f6e..db3f979 100644 --- a/src/lib/server/fleet/launch-recipes.ts +++ b/src/lib/server/fleet/launch-recipes.ts @@ -3,12 +3,12 @@ import type { BlueprintSummary, LaunchRecipeSummary, LaunchFileSummary } from '$ import { config, configFileForGame } from '../config.js'; import { db } from '../db/index.js'; import { zombieMapBySlug, type ZombieGame } from './map-aliases.js'; -import { promotedArtifacts } from './promoted-artifacts.js'; +import { promotedArtifactMetadata, promotedArtifacts } from './promoted-artifacts.js'; import { getPromotedServerConfig, getServerMapConfig, MAP_CONFIG_FILE } from './server-config.js'; -import { getBaselineForScope, getVersion, listPromotedOverlaysForScope } from './scripts.js'; +import { getBaselineForScope, getVersionMetadata, listPromotedOverlaysForScope } from './scripts.js'; import { sha256Buffer } from './files.js'; -export interface LaunchRecipe { +export interface LaunchRecipePlan { native?: import('../../hosts.js').NativePlacement; format: 1; game: ZombieGame; @@ -19,10 +19,15 @@ export interface LaunchRecipe { image: string; configFile: string; configSource: string; + files: (LaunchFileSummary & { content?: string })[]; +} + +/** Attempted launches retain their exact bytes; queued plans need only metadata. */ +export interface LaunchRecipe extends LaunchRecipePlan { files: (LaunchFileSummary & { content: string })[]; } -export interface BlueprintEntry { count: number; recipe: LaunchRecipe } +export interface BlueprintEntry { count: number; recipe: LaunchRecipePlan } interface BlueprintSelection { count: number; game: ZombieGame; mapSlug: string } interface BlueprintRow { id: string; name: string; description: string; created_at: string; recipe: string } @@ -45,7 +50,7 @@ function summarizeBlueprint(row: BlueprintRow): BlueprintSummary { export function blueprintEntries(id: string): { entries: BlueprintEntry[]; blueprintName: string } { const row = db().prepare('SELECT * FROM server_blueprints WHERE id = ?').get(id) as BlueprintRow | undefined; if (!row) throw new Error('This collection is no longer available. Choose another starting point.'); - const entries = readEntries(row.recipe).map(({ count, game, mapSlug }) => ({ count, recipe: captureRecipe(game, mapSlug) })); + const entries = readEntries(row.recipe).map(({ count, game, mapSlug }) => ({ count, recipe: captureRecipePlan(game, mapSlug) })); return { entries, blueprintName: row.name }; } @@ -66,22 +71,36 @@ function buildEntries(input: unknown): BlueprintSelection[] { } export function captureRecipe(game: ZombieGame, mapSlug: string): LaunchRecipe { + return buildRecipe(game, mapSlug, true); +} + +/** Preview and admission keep file references; execution captures the current bytes. */ +export function captureRecipePlan(game: ZombieGame, mapSlug: string): LaunchRecipePlan { + return buildRecipe(game, mapSlug, false); +} + +function buildRecipe(game: ZombieGame, mapSlug: string, includeContent: true): LaunchRecipe; +function buildRecipe(game: ZombieGame, mapSlug: string, includeContent: false): LaunchRecipePlan; +function buildRecipe(game: ZombieGame, mapSlug: string, includeContent: boolean): LaunchRecipePlan { const map = zombieMapBySlug(game, mapSlug); if (!map) throw new Error('Choose a supported game and map.'); - const files: LaunchRecipe['files'] = promotedArtifacts(game, map.canonical).map(({ kind, script, version }) => ({ + const metadata = ({ kind, script, version }: ReturnType[number]): LaunchFileSummary => ({ name: script.name, kind, path: script.target_rel_path, versionId: version.id, - version: version.version, sha256: version.sha256, size: version.size, content: version.content.toString('base64') - })); + version: version.version, sha256: version.sha256, size: version.size + }); + const files: LaunchRecipePlan['files'] = includeContent + ? promotedArtifacts(game, map.canonical).map(artifact => ({ ...metadata(artifact), content: artifact.version.content.toString('base64') })) + : promotedArtifactMetadata(game, map.canonical).map(metadata); const addConfig = (name: string, path: string, content: Buffer) => files.push({ name, path, kind: 'config', versionId: null, version: null, - sha256: sha256Buffer(content), size: content.length, content: content.toString('base64') + sha256: sha256Buffer(content), size: content.length, ...(includeContent ? { content: content.toString('base64') } : {}) }); const composed = getPromotedServerConfig(game, map.canonical, map.slug); let configSource = 'Plutainer defaults, seeded on first boot'; if (composed) { const baseline = getBaselineForScope('game', game)!; const layers = [baseline, ...listPromotedOverlaysForScope('game', game), ...listPromotedOverlaysForScope('map', map.canonical)]; - configSource = layers.map(layer => `${layer.name} v${getVersion(layer.current_version_id!)!.version}`).join(' + '); + configSource = layers.map(layer => `${layer.name} v${getVersionMetadata(layer.current_version_id!)!.version}`).join(' + '); addConfig('Composed server configuration', configFileForGame(game), composed); } addConfig('Selected map', MAP_CONFIG_FILE, getServerMapConfig(game, map.canonical, map.slug)!); @@ -90,19 +109,19 @@ export function captureRecipe(game: ZombieGame, mapSlug: string): LaunchRecipe { configFile: configFileForGame(game), configSource, files }; } -export function summarizeRecipe(recipe: LaunchRecipe): LaunchRecipeSummary { +export function summarizeRecipe(recipe: LaunchRecipePlan): LaunchRecipeSummary { const { format: _format, files, ...summary } = recipe; return { ...summary, files: files.map(({ content: _content, ...file }) => file), fingerprint: sha256Buffer(Buffer.from(JSON.stringify(recipe))) }; } -export function recipeForDeployment(id: string): { recipe: LaunchRecipe; blueprintName: string | null } | null { +export function recipeForDeployment(id: string): { recipe: LaunchRecipePlan; blueprintName: string | null } | null { const row = db().prepare('SELECT recipe, blueprint_name FROM server_launch_recipes WHERE deployment_id = ?') .get(id) as { recipe: string; blueprint_name: string | null } | undefined; return row ? { recipe: JSON.parse(row.recipe), blueprintName: row.blueprint_name } : null; } -export function blueprintRecipe(id: string): { recipe: LaunchRecipe; blueprintName: string } { +export function blueprintRecipe(id: string): { recipe: LaunchRecipePlan; blueprintName: string } { const { entries, blueprintName } = blueprintEntries(id); if (entries.length !== 1 || entries[0].count !== 1) throw new Error('This collection launches multiple servers. Use the collection preview.'); return { recipe: entries[0].recipe, blueprintName }; @@ -122,7 +141,7 @@ function blueprintName(input: { name: unknown; description?: unknown }, exceptId export function saveBlueprint(input: { name: unknown; description?: unknown; previewId?: unknown; deploymentId?: unknown; entries?: unknown }): BlueprintSummary { const name = blueprintName(input); if ([input.previewId, input.deploymentId, input.entries].filter(value => value !== undefined).length !== 1) throw new Error('Choose exactly one launch preview, recorded deployment, or set of maps.'); - let recipe: LaunchRecipe | undefined; + let recipe: LaunchRecipePlan | undefined; if (typeof input.previewId === 'string') { const row = db().prepare('SELECT recipe FROM server_launch_previews WHERE id = ? AND expires_at > ?') .get(input.previewId, new Date().toISOString()) as { recipe: string } | undefined; diff --git a/src/lib/server/fleet/promoted-artifacts.ts b/src/lib/server/fleet/promoted-artifacts.ts index 69dc006..9261f6e 100644 --- a/src/lib/server/fleet/promoted-artifacts.ts +++ b/src/lib/server/fleet/promoted-artifacts.ts @@ -1,14 +1,23 @@ -import { getVersion, listScripts } from './scripts.js'; +import { getVersion, getVersionMetadata, listScripts } from './scripts.js'; -/** Published file artifacts shared by provisioning and drift reconciliation. */ -export function promotedArtifacts(game: string | null, map: string | null) { +function selectedArtifacts(game: string | null, map: string | null, readVersion: (id: number) => Version | undefined) { return (['script', 'plugin'] as const).flatMap((kind) => listScripts(kind).flatMap((script) => { const scopeMatches = script.scope_type === 'game' ? script.scope_value === game : script.scope_type === 'map' ? script.scope_value === map : false; if (!scopeMatches || !script.current_version_id) return []; - const version = getVersion(script.current_version_id); + const version = readVersion(script.current_version_id); return version ? [{ kind, script, version }] : []; }) ); } + +/** Published file artifacts shared by provisioning and drift reconciliation. */ +export function promotedArtifacts(game: string | null, map: string | null) { + return selectedArtifacts(game, map, getVersion); +} + +/** Metadata uses the same promotion and scope rules without reading file content. */ +export function promotedArtifactMetadata(game: string | null, map: string | null) { + return selectedArtifacts(game, map, getVersionMetadata); +} diff --git a/src/lib/server/fleet/scripts.ts b/src/lib/server/fleet/scripts.ts index 921ff54..a7dbf67 100644 --- a/src/lib/server/fleet/scripts.ts +++ b/src/lib/server/fleet/scripts.ts @@ -149,6 +149,15 @@ export function getVersion(id: number): ScriptVersionRow | undefined { | undefined; } +export type ScriptVersionMetadata = Pick; + +/** Planning a launch never needs to load potentially large script or plugin bytes. */ +export function getVersionMetadata(id: number): ScriptVersionMetadata | undefined { + return db().prepare('SELECT id, version, sha256, size FROM script_versions WHERE id = ?').get(id) as + | ScriptVersionMetadata + | undefined; +} + // Latest version is 'max(version)' — used everywhere we want the freshest // upload, whether or not it has been promoted (current_version_id). export function getLatestVersion(scriptId: number): ScriptVersionRow | undefined { diff --git a/src/lib/server/provisioning/instances.ts b/src/lib/server/provisioning/instances.ts index d2f3b0a..fdc0574 100644 --- a/src/lib/server/provisioning/instances.ts +++ b/src/lib/server/provisioning/instances.ts @@ -4,7 +4,7 @@ import { probePlayerEvidence, probeServerStatus } from '../fleet/status-evidence import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent, REMOVING_SERVER, resumeCleanup, resumeVerification, claimRemoval } from './history.js'; import { randomUUID } from 'node:crypto'; import type { LaunchPreview } from '$lib/deployment-plan.js'; -import { blueprintRecipe, captureRecipe, recipeForDeployment, summarizeRecipe, type LaunchRecipe } from '../fleet/launch-recipes.js'; +import { blueprintRecipe, captureRecipe, captureRecipePlan, recipeForDeployment, summarizeRecipe, type LaunchRecipe, type LaunchRecipePlan } from '../fleet/launch-recipes.js'; import { mkdir, chmod, rm, writeFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; import { config, destOfKind, serverDir, gameOfServer, configFileForGame } from '../config.js'; @@ -179,7 +179,7 @@ interface PreviewRow { export function previewCreation(input: { game?: unknown; mapSlug?: unknown; blueprintId?: unknown; deploymentId?: unknown; hostId?: unknown; libraryId?: unknown }): LaunchPreview { if (input.hostId !== undefined || !config.komodo.address || listHosts().length > 0) return previewNativeCreation(input); if (input.blueprintId !== undefined && input.deploymentId !== undefined) throw new Error('Choose one starting point.'); - let recipe: LaunchRecipe; + let recipe: LaunchRecipePlan; let blueprintName: string | null = null; if (input.blueprintId !== undefined) { if (typeof input.blueprintId !== 'string') throw new Error('Invalid blueprint.'); @@ -190,10 +190,10 @@ export function previewCreation(input: { game?: unknown; mapSlug?: unknown; blue if (!source) throw new Error('This deployment has no recorded launch recipe.'); blueprintName = source.blueprintName; // History describes the previous launch; only its map selection is reusable. - recipe = captureRecipe(source.recipe.game, source.recipe.mapSlug); + recipe = captureRecipePlan(source.recipe.game, source.recipe.mapSlug); } else { if ((input.game !== 't5' && input.game !== 't6') || typeof input.mapSlug !== 'string') throw new Error('Choose a supported game and map.'); - recipe = captureRecipe(input.game, input.mapSlug); + recipe = captureRecipePlan(input.game, input.mapSlug); } const lobby = nextLobbyNumber(`${recipe.game}_zombie_${recipe.mapSlug}`); const name = lobby === null ? null : `${recipe.game}_zombie_${recipe.mapSlug}_${lobby}`; @@ -223,7 +223,7 @@ export function queueCreateServer(input: CreateServerInput, releasingServers: st return db().transaction((): CreateServerResult => { const preview = input.previewId ? db().prepare('SELECT * FROM server_launch_previews WHERE id = ?').get(input.previewId) as PreviewRow | undefined : undefined; if (input.previewId && !preview) return { ok: false, kind: 'bad-request', message: 'Launch preview expired or was not found. Review a fresh preview.' }; - const recipe: LaunchRecipe | undefined = preview ? JSON.parse(preview.recipe) : undefined; + const recipe: LaunchRecipePlan | undefined = preview ? JSON.parse(preview.recipe) : undefined; if (recipe && (recipe.game !== (input.game ?? 't5') || recipe.mapSlug !== input.mapSlug || input.gameMode !== 'zombie')) { return { ok: false, kind: 'bad-request', message: 'The selection does not match the reviewed launch recipe.' }; } @@ -239,7 +239,7 @@ export function queueCreateServer(input: CreateServerInput, releasingServers: st const prepared = prepareCreation(input, releasingServers); if (!prepared.ok) return prepared; // A preview records choices, not version pins. Promotions remain authoritative. - const currentRecipe = captureRecipe(input.game ?? 't5', prepared.mapEntry.slug); + const currentRecipe = captureRecipePlan(input.game ?? 't5', prepared.mapEntry.slug); const id = beginDeployment(prepared.name, 'create', input.createdBy); db().prepare('INSERT INTO server_launch_recipes (deployment_id, recipe, blueprint_name) VALUES (?, ?, ?)') .run(id, JSON.stringify(currentRecipe), preview?.blueprint_name ?? null); @@ -268,7 +268,7 @@ export function queueRebuildServer(name: string, triggeredBy = 'ui'): CreateServ if (!config.komodo.targetServer || (!config.rcon.password && !triggeredBy.startsWith('watchdog:'))) return { ok: false, kind: 'unconfigured', message: 'Configure the Komodo target host and RCON before rebuilding.' }; if (!getServerKey(name)) return { ok: false, kind: 'bad-request', message: `Save a server key for ${name} in Keys before rebuilding.` }; // Validate that the replacement setup can be composed before destruction. - const recipe = captureRecipe(game, map.slug); + const recipe = captureRecipePlan(game, map.slug); return db().transaction(() => { const id = beginDeployment(name, 'rebuild', triggeredBy); db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id); @@ -304,7 +304,7 @@ export async function rebuildServer(id: string): Promise { } // Recheck prerequisites while the original instance still exists. if (!getServerKey(job.server)) { finishDeployment(id, { ok: false, message: 'Saved server key was removed; rebuild did not begin.' }); return; } - try { captureRecipe(gameOfServer(job.server), job.map_slug); } + try { captureRecipePlan(gameOfServer(job.server), job.map_slug); } catch (err) { finishDeployment(id, { ok: false, message: (err as Error).message }); return; } db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND state = 'waiting'").run(id); destroying.add(job.server); diff --git a/src/lib/server/provisioning/native.ts b/src/lib/server/provisioning/native.ts index a4c8964..d9b85b1 100644 --- a/src/lib/server/provisioning/native.ts +++ b/src/lib/server/provisioning/native.ts @@ -7,7 +7,7 @@ import { config, relDestForKind, serverDir } from '../config.js'; import { nativeInstance, nativeRequest, type NativeInstance } from '../backends/native.js'; import { getHost, getHostOperation, listHosts, queueHostOperation, releaseHostPorts, reserveHostPort } from '../hosts/index.js'; import { newestReadyLibraryId, assertReadyLibrary } from '../game-files/index.js'; -import { blueprintRecipe, captureRecipe, recipeForDeployment, summarizeRecipe, type LaunchRecipe } from '../fleet/launch-recipes.js'; +import { blueprintRecipe, captureRecipe, captureRecipePlan, recipeForDeployment, summarizeRecipe, type LaunchRecipe, type LaunchRecipePlan } from '../fleet/launch-recipes.js'; import { getServerKey, saveServerKey } from '../fleet/server-keys.js'; import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js'; import { nextLobbyNumber, type CreateServerInput, type CreateServerResult } from './instances.js'; @@ -27,7 +27,7 @@ import { hostAgentUpdatesEnabled } from '../../hosts.js'; import { cancelCreation, finishCreationCancellation } from './cancel-creation.js'; export type NativeSelection = import('../../hosts.js').NativePlacement; -type NativeRecipe = LaunchRecipe & { native?: NativeSelection }; +type NativeRecipe = LaunchRecipePlan & { native?: NativeSelection }; interface Job { id: string; server: string; action: ProvisioningAction; state: string; submitted_at: string | null; stage: string; instance_id: string; generation: number; stage_operation_id: string | null; operation_id: string | null; files_json: string | null; applied_at: string | null } interface FilePlan { id: number; path: string; contentBase64: string | null; sha256: string | null; checkBefore: boolean; expectedBeforeSha: string | null } @@ -60,12 +60,12 @@ export function previewNativeCreation(input: Record): LaunchPre if (typeof input.deploymentId === 'string') { const previous = recipeForDeployment(input.deploymentId); if (!previous) throw new Error('The recorded deployment is unavailable.'); - recipe = captureRecipe(previous.recipe.game, previous.recipe.mapSlug); + recipe = captureRecipePlan(previous.recipe.game, previous.recipe.mapSlug); blueprintName = previous.blueprintName; } else if (typeof input.blueprintId === 'string') ({ recipe, blueprintName } = blueprintRecipe(input.blueprintId)); else { if ((input.game !== 't5' && input.game !== 't6') || typeof input.mapSlug !== 'string') throw new Error('Choose a supported game and map.'); - recipe = captureRecipe(input.game, input.mapSlug); + recipe = captureRecipePlan(input.game, input.mapSlug); } const selection = nativeSelection(input, recipe.game); recipe.native = selection; recipe.image = selection.image!; @@ -105,7 +105,7 @@ export function queueNativeCreation(input: CreateServerInput): CreateServerResul const game = input.game ?? 't5'; if (input.gameMode !== 'zombie' || (saved && (saved.game !== game || saved.mapSlug !== input.mapSlug))) throw new Error('Selection changed. Review a fresh launch preview.'); const selection = nativeSelection((saved?.native ?? input) as unknown as Record, game); - const recipe: NativeRecipe = { ...captureRecipe(game, input.mapSlug), native: selection, image: selection.image! }; + const recipe: NativeRecipe = { ...captureRecipePlan(game, input.mapSlug), native: selection, image: selection.image! }; const lobby = nextLobbyNumber(`${game}_zombie_${input.mapSlug}`); if (lobby === null) throw new Error('All lobby IDs are reserved.'); const name = `${game}_zombie_${input.mapSlug}_${lobby}`; @@ -157,7 +157,7 @@ export function queueNativeRebuild(name: string, triggeredBy: string): CreateSer if (blocker) throw new Error(blocker); if (!server?.map_slug || !getServerKey(name)) throw new Error('A saved map and server key are required before rebuilding.'); assertReadyLibrary(instance.host_id, instance.library_id); - const recipe: NativeRecipe = { ...captureRecipe(server.game as 't5' | 't6', server.map_slug), image: instance.image, + const recipe: NativeRecipe = { ...captureRecipePlan(server.game as 't5' | 't6', server.map_slug), image: instance.image, native: { hostId: instance.host_id, libraryId: instance.library_id, image: instance.image, bindAddress: instance.bind_address } }; return db().transaction(() => { const id = beginDeployment(name, 'rebuild', triggeredBy); @@ -478,7 +478,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) { deploymentStage(job.id, 'Preparing launch files for the host agent', { phase: 'prepare', recipeStatus: 'attempted', checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } }); const prior = recipeForDeployment(job.id)!.recipe as NativeRecipe; - const recipe: NativeRecipe = { ...captureRecipe(prior.game, prior.mapSlug), native: prior.native, image: instance!.image }; + const recipe = { ...captureRecipe(prior.game, prior.mapSlug), native: prior.native, image: instance!.image }; payload = createPayload(instance!, recipe); if (job.action === 'rebuild') payload.nextGeneration = instance!.generation + 1; db().prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?').run(JSON.stringify(recipe), job.id); diff --git a/src/routes/api/deployment-collections/+server.ts b/src/routes/api/deployment-collections/+server.ts index 92d65e6..7bc46dc 100644 --- a/src/routes/api/deployment-collections/+server.ts +++ b/src/routes/api/deployment-collections/+server.ts @@ -1,6 +1,6 @@ import { json } from '@sveltejs/kit'; import { listCollections, queueCollection } from '$lib/server/fleet/collections.js'; -import { kick } from '$lib/server/pipeline/worker.js'; +import { kickProvisioning } from '$lib/server/pipeline/worker.js'; export function GET() { return json({ collections: listCollections() }); } @@ -9,7 +9,11 @@ export async function POST({ request }: { request: Request }) { const body = await request.json(); if (!body || typeof body !== 'object' || Array.isArray(body)) throw new Error('Provide a collection preview and its server keys.'); const result = queueCollection(body); - if (result.collectionId) void kick().catch(err => console.error('[collection] worker kick failed:', err)); + // The durable queue is committed. Let the response and pending I/O run + // before provisioning loads launch files or starts checking removals. + if (result.collectionId) setImmediate(() => { + void kickProvisioning().catch(err => console.error('[collection] worker kick failed:', err)); + }); return json(result, { status: result.collectionId ? 202 : 200 }); } catch (err) { return json({ error: (err as Error).message }, { status: 400 }); } } diff --git a/tests/unit/collection-match-performance.test.ts b/tests/unit/collection-match-performance.test.ts new file mode 100644 index 0000000..5470252 --- /dev/null +++ b/tests/unit/collection-match-performance.test.ts @@ -0,0 +1,145 @@ +import { randomUUID } from 'node:crypto'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { newFixture, type Fixture } from '../helpers/fixture.js'; + +vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ + rconGetPlayerCount: vi.fn(async () => 0), + rconPlutoniumVersion: vi.fn(async () => ({ revision: 5354, error: null })), + rconStatusResult: vi.fn(async () => ({ ok: true, endpoint: { host: 'agent', port: 28960 }, rawReply: '', + status: { players: 0, map: 'zm_transit', gametype: null, roster: [] } })) +})); + +let fixture: Fixture; +let hostId: string; +const image = 'example.test/plutainer:test'; + +beforeEach(async () => { + vi.resetModules(); vi.clearAllMocks(); + fixture = await newFixture([]); + vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries')); + vi.stubEnv('PLUTAINER_IMAGE', image); + vi.stubEnv('RCON_PASSWORD', 'test-password'); + vi.stubEnv('KOMODO_ADDRESS', ''); + vi.stubEnv('KOMODO_API_KEY', ''); + vi.stubEnv('KOMODO_API_SECRET', ''); + vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] }))); + const { createHost, createEnrollment, enrollHost, heartbeat, reserveHostPort } = await import('../../src/lib/server/hosts/index.js'); + hostId = createHost({ name: 'Match collection host', portMin: 28960, portMax: 28999 }).id; + const inventory = { + cpuCount: 64, cpuUsagePercent: 5, memoryTotal: 128 * 1024 ** 3, memoryAvailable: 120 * 1024 ** 3, + protocolVersion: 1, dockerRunning: true, directoriesWritable: true, + storageAvailable: 1_000_000_000_000, runtimeImages: [image] + }; + enrollHost(createEnrollment(hostId).token, inventory); + const { createLibraryImport, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + const libraries = ['t5', 't6'].map(game => { + const id = createLibraryImport({ hostId, game }).id; + db().prepare("UPDATE game_file_versions SET state='ready',manifest='[]',manifest_sha256=?,size_bytes=1,published_at=? WHERE id=?") + .run('a'.repeat(64), new Date().toISOString(), id); + return { id, state: 'ready' as const }; + }); + updateHostLibraryCaches(hostId, libraries.map(library => ({ libraryVersionId: library.id, status: 'ready', manifestSha256: 'a'.repeat(64) }))); + heartbeat(hostId, { ...inventory, libraries }); + const { T6_ZOMBIE_MAPS } = await import('../../src/lib/map-aliases.js'); + const now = new Date().toISOString(); + db().transaction(() => { + for (const [index, map] of T6_ZOMBIE_MAPS.slice(0, 9).entries()) { + const name = `t6_zombie_${map.slug}_1`; + const instanceId = randomUUID(); + const deploymentId = randomUUID(); + const port = 28960 + index; + db().prepare(`INSERT INTO servers(name,komodo_name,game,map,game_mode,map_slug,game_port,container_state,last_players,last_seen_at) + VALUES(?,'','t6',?,'zombie',?,?,'running',0,?)`).run(name, map.canonical, map.slug, port, now); + db().prepare('INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES(?,?,?,?,?,?,?)') + .run(instanceId, name, hostId, libraries[1].id, image, port, now); + db().prepare("INSERT INTO server_deployments(id,server,action,state,stage,triggered_by,started_at,finished_at) VALUES(?,?,'create','done','Verified','fixture',?,?)") + .run(deploymentId, name, now, now); + db().prepare('INSERT INTO native_provisioning(deployment_id,instance_id,generation) VALUES(?,?,1)').run(deploymentId, instanceId); + for (const protocol of ['tcp', 'udp'] as const) reserveHostPort(hostId, instanceId, { port, protocol }); + } + })(); + const backend = await import('../../src/lib/server/backends/native.js'); + vi.spyOn(backend, 'nativeRequest').mockImplementation(async (_instance, kind) => { + if (kind !== 'instance.inspect') throw new Error(`Unexpected request ${kind}`); + return { exists: true, healthy: true, state: 'running' } as never; + }); +}); + +afterEach(async () => { + await new Promise(resolve => setImmediate(resolve)); + const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); + await processProvisioning(); + await new Promise(resolve => setImmediate(resolve)); + vi.restoreAllMocks(); + await fixture.cleanup(); + vi.unstubAllEnvs(); vi.unstubAllGlobals(); +}); + +describe('matching a native collection', () => { + it('admits nine removals and ten creations without copying plugin bytes into every queued launch', async () => { + const { db } = await import('../../src/lib/server/db/index.js'); + const { createScript, setScope, deleteScript } = await import('../../src/lib/server/fleet/scripts.js'); + const content = Buffer.alloc(8 * 1024 ** 2, 42); + const { script, version } = createScript({ name: 'Promoted plugin', kind: 'plugin', targetRelPath: 'fixture.dll', + scopeType: 'game', scopeValue: 't5', content }); + db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(version.id, script.id); + const { saveBlueprint, recipeForDeployment } = await import('../../src/lib/server/fleet/launch-recipes.js'); + const { previewCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js'); + const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js'); + const { nativeRequest } = await import('../../src/lib/server/backends/native.js'); + const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js'); + const { POST } = await import('../../src/routes/api/deployment-collections/+server.js'); + const { T5_ZOMBIE_MAPS } = await import('../../src/lib/map-aliases.js'); + const blueprint = saveBlueprint({ name: 'Match ten T5 lobbies', entries: T5_ZOMBIE_MAPS.filter(map => map.slug !== 'zombietron') + .map(map => ({ game: 't5', mapSlug: map.slug, count: 1 })) }); + const preview = previewCollection(blueprint.id, 'exact'); + expect(preview).toMatchObject({ ready: true, mode: 'exact' }); + expect(preview.removals).toHaveLength(9); + expect(preview.servers).toHaveLength(10); + const storedPreview = db().prepare('SELECT length(entries) AS bytes FROM server_collection_previews WHERE id=?') + .get(preview.id) as { bytes: number }; + expect(storedPreview.bytes).toBeLessThan(100_000); + + const submit = (keys?: Record) => POST({ request: new Request('http://localhost/api/deployment-collections', { + method: 'POST', body: JSON.stringify({ previewId: preview.id, keys }) }) }); + const response = await submit(Object.fromEntries(preview.servers.map(server => [server.name!, `key-${server.name}`]))); + const body = await response.json(); + expect(response.status).toBe(202); + expect(body).toMatchObject({ ok: true, collectionId: expect.any(String) }); + expect(await (await submit()).json()).toEqual(body); + // Exercise the real route and worker: admission must return before + // loading launch payloads or inspecting the nine servers to remove. + expect(nativeRequest).not.toHaveBeenCalled(); + expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.create'").all()).toEqual([]); + for (const table of ['server_launch_previews', 'server_launch_recipes']) { + const rows = db().prepare(`SELECT length(recipe) AS bytes FROM ${table}`).all() as { bytes: number }[]; + expect(rows).toHaveLength(10); + expect(rows.every(row => row.bytes < 10_000)).toBe(true); + } + const collection = getCollection(body.collectionId)!; + expect(collection.collection).toMatchObject({ mode: 'exact', total: 19 }); + expect(collection.servers.filter(server => server.action === 'destroy')).toHaveLength(9); + expect(collection.servers.filter(server => server.action === 'create')).toHaveLength(10); + + await new Promise(resolve => setImmediate(resolve)); + await processProvisioning(); + const dispatched = db().prepare(`SELECT n.deployment_id,o.payload FROM native_provisioning n + JOIN ec_host_operations o ON o.id=n.operation_id WHERE o.kind='instance.create'`).all() as { deployment_id: string; payload: string }[]; + expect(dispatched).toHaveLength(1); + const payload = JSON.parse(dispatched[0].payload) as { files: { path: string; contentBase64: string; sha256: string }[] }; + expect(payload.files.find(file => file.path.endsWith('/fixture.dll'))).toMatchObject({ + sha256: version.sha256, contentBase64: content.toString('base64') + }); + + // Queued plans are descriptions, while an attempted launch preserves + // its exact bytes even after the promoted artifact is deleted. + setScope({ scriptId: script.id, scopeType: null, scopeValue: null }); + deleteScript(script.id); + const queued = collection.servers.find(server => server.action === 'create' && server.id !== dispatched[0].deployment_id)!; + expect(getDeployment(queued.id)?.recipe?.files).toContainEqual(expect.objectContaining({ path: 'fixture.dll', versionId: version.id, size: content.length })); + const attempted = recipeForDeployment(dispatched[0].deployment_id)!.recipe.files.find(file => file.path === 'fixture.dll'); + expect(attempted).toMatchObject({ sha256: version.sha256, content: content.toString('base64') }); + }, 30_000); +}); diff --git a/tests/unit/deployment-collections.test.ts b/tests/unit/deployment-collections.test.ts index e4d5077..a43da44 100644 --- a/tests/unit/deployment-collections.test.ts +++ b/tests/unit/deployment-collections.test.ts @@ -19,7 +19,7 @@ vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconStatusResult: vi.fn(async (server: string) => ({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: '', status: { players: 0, map: server.startsWith('t6_') ? 'zm_transit' : server.includes('_moon_') ? 'zombie_moon' : 'zombie_theater', gametype: null, roster: [] } })) })); -vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) })); +vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) })); let fixture: Fixture; const entries = [ diff --git a/tests/unit/launch-recipes.test.ts b/tests/unit/launch-recipes.test.ts index b4a1053..772ef42 100644 --- a/tests/unit/launch-recipes.test.ts +++ b/tests/unit/launch-recipes.test.ts @@ -100,7 +100,8 @@ describe('launch recipes and previews', () => { expect(getServerScript(queued.name, 'perks.gsc')).toMatchObject({ script_version_id: newScript.id, sha256: newScript.sha256 }); expect(getServerScript(queued.name, 'utilities.dll')).toMatchObject({ script_version_id: newPlugin.id, sha256: newPlugin.sha256 }); const recordedConfig = recipeForDeployment(queued.deploymentId)?.recipe.files.find(file => file.path === 'dedicated_sp.cfg'); - expect(Buffer.from(recordedConfig!.content, 'base64').toString()).toBe(deployedConfig); + if (recordedConfig?.content === undefined) throw new Error('Attempted launch must retain the configuration bytes.'); + expect(Buffer.from(recordedConfig.content, 'base64').toString()).toBe(deployedConfig); expect(getDeployment(queued.deploymentId)?.recipe?.fingerprint).not.toBe(preview.recipe.fingerprint); expect(getDeployment(queued.deploymentId)?.deployment).toMatchObject({ state: 'awaiting-health' }); await processCreations(); diff --git a/tests/unit/provisioning-history-performance.test.ts b/tests/unit/provisioning-history-performance.test.ts new file mode 100644 index 0000000..e43197b --- /dev/null +++ b/tests/unit/provisioning-history-performance.test.ts @@ -0,0 +1,79 @@ +import Database from 'better-sqlite3'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import schema from '../../src/lib/server/db/schema.sql?raw'; +import { newFixture, type Fixture } from '../helpers/fixture.js'; + +let fixture: Fixture; +beforeEach(async () => { + vi.resetModules(); + fixture = await newFixture([]); +}); +afterEach(async () => { + vi.restoreAllMocks(); + await fixture.cleanup(); +}); + +describe('provisioning reads with retained lifecycle history', () => { + it('indexes the server and queue reads used by collection matching and health polling', async () => { + const { db } = await import('../../src/lib/server/db/index.js'); + const { latestGameSubmission } = await import('../../src/lib/server/fleet/lifecycle-evidence.js'); + const { hasActiveProvisioning } = await import('../../src/lib/server/fleet/provisioning.js'); + const { housePartyTargets } = await import('../../src/lib/server/fleet/house-party.js'); + const { creationQueue } = await import('../../src/lib/server/provisioning/instances.js'); + const handle = db(); + handle.transaction(() => { + const insert = handle.prepare(`INSERT INTO server_deployments + (id,server,action,state,stage,started_at,triggered_by,server_deleted_at) + VALUES(?,?,'create','done','Complete','2025-01-01','fixture','2025-01-02')`); + for (let i = 0; i < 2_000; i++) insert.run(`history-${i}`, `retired-${i}`); + handle.prepare(`INSERT INTO server_deployments + (id,server,action,state,stage,started_at,triggered_by,submitted_at) + VALUES('current','current','create','queued','Queued','2026-01-01','fixture','2026-01-01')`).run(); + })(); + + // Inspect the SQL issued by the public reads, so the regression follows + // those implementations rather than keeping separate copies of them. + const expectIndexed = (read: () => unknown, bindings: string[] = []) => { + const prepare = vi.spyOn(handle, 'prepare'); + read(); + const queries = prepare.mock.calls.map(([sql]) => sql).filter(sql => sql.includes('server_deployments')); + prepare.mockRestore(); + expect(queries).toHaveLength(1); + const plan = handle.prepare(`EXPLAIN QUERY PLAN ${queries[0]}`).all(...bindings) as { detail: string }[]; + const details = plan.map(row => row.detail).join('\n'); + expect(details).toMatch(/SEARCH server_deployments USING (?:COVERING )?INDEX/); + expect(details).not.toMatch(/SCAN server_deployments\b/); + }; + expectIndexed(() => expect(latestGameSubmission('current')).toBe('2026-01-01'), ['current', 'current']); + expectIndexed(() => expect(hasActiveProvisioning('current')).toBe(true), ['current', '']); + expectIndexed(() => expect(housePartyTargets()).toEqual(['current'])); + expectIndexed(() => expect(creationQueue().queued.map(job => job.id)).toEqual(['current'])); + }); + + it('installs the indexes after legacy table rebuilds and keeps them on reopen', async () => { + const legacy = new Database(fixture.dbPath); + try { + legacy.exec(schema.replace(/CREATE TABLE IF NOT EXISTS server_deployments \([\s\S]*?\n\);/, table => table + .replace("'create', 'start', 'restart', 'rebuild', 'stop', 'destroy'", "'create', 'destroy'") + .replace(' server_deleted_at TEXT,\n', ''))); + legacy.exec(`INSERT INTO servers(name,komodo_name) VALUES('existing','existing'); + INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by) + VALUES('old','existing','create','done','Complete','2025-01-01','fixture'); + INSERT INTO server_deployment_events(deployment_id,created_at,message) + VALUES('old','2025-01-01','Creation complete');`); + } finally { legacy.close(); } + const { db, closeDb } = await import('../../src/lib/server/db/index.js'); + for (let open = 0; open < 2; open++) { + const handle = db(); + expect(handle.prepare('SELECT rowid,id,state FROM server_deployments').all()) + .toEqual([{ rowid: 1, id: 'old', state: 'done' }]); + expect(handle.prepare('SELECT message FROM server_deployment_events').all()).toEqual([{ message: 'Creation complete' }]); + const indexes = handle.pragma('index_list(server_deployments)') as { name: string }[]; + expect(indexes.map(index => index.name)).toEqual(expect.arrayContaining([ + 'idx_server_deployments_server', 'idx_server_deployments_state' + ])); + expect(handle.pragma('foreign_key_check')).toEqual([]); + closeDb(); + } + }); +});