Fix controller stalls during large collection launches

src/lib/server/game-files/index.ts

@@ -43,9 +43,14 @@ export function getLibraryFile(versionId: string, path: string) {
         FROM game_file_entries f JOIN game_file_versions v ON v.id=f.version_id WHERE f.version_id=? AND f.path=?`)
         .get(versionId, path) as (GameFileEntry & { state: GameLibraryVersion['state']; sourceHostId: string; storageLocation: string }) | undefined ?? null;
 }
+/** Launch selection needs only the ID; manifests can contain tens of thousands of files. */
+export function newestReadyLibraryId(game: string): string | null {
+    const row = database().prepare("SELECT v.id FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id WHERE l.game=? AND v.state='ready' ORDER BY v.published_at DESC, v.created_at DESC, v.rowid DESC LIMIT 1").get(game) as { id: string } | undefined;
+    return row?.id ?? null;
+}
 export function newestReadyLibrary(game: string): GameLibraryVersion | null {
-    const row = database().prepare("SELECT v.*, l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id WHERE l.game=? AND v.state='ready' ORDER BY v.published_at DESC, v.created_at DESC, v.rowid DESC LIMIT 1").get(game) as VersionRow | undefined;
-    return row ? versionFromRow(row) : null;
+    const id = newestReadyLibraryId(game);
+    return id ? getLibraryVersion(id) : null;
 }
 /** Hosts follow the latest successful publication for each game, including legacy libraries. */
 export function latestPublishedLibraries() {
@@ -79,13 +84,20 @@ export function getHostLibraryCache(hostId: string, versionId: string): HostLibr
     const row = database().prepare('SELECT * FROM host_library_caches WHERE host_id=? AND version_id=?').get(hostId, versionId) as CacheRow | undefined;
     return row ? cacheFromRow(row) : null;
 }
-export function requireReadyLibrary(hostId: string, versionId: string): GameLibraryVersion {
-    const version = getLibraryVersion(versionId);
+/** Admission and dispatch recheck readiness without loading or parsing the file manifest. */
+export function assertReadyLibrary(hostId: string, versionId: string): void {
+    const handle = database();
+    const version = handle.prepare('SELECT state, manifest_sha256 FROM game_file_versions WHERE id=?')
+        .get(versionId) as Pick<VersionRow, 'state' | 'manifest_sha256'> | undefined;
     if (!version || version.state !== 'ready' || libraryFilesRetired(versionId)) throw new Error('Select an available published game-file version.');
-    if (database().prepare("SELECT 1 FROM ec_host_operations WHERE host_id=? AND kind='library.remove' AND json_extract(payload,'$.libraryVersionId')=? AND state IN ('pending','accepted','running','unknown')").get(hostId, versionId)) throw new Error('This host copy is being removed.');
-    const cache = getHostLibraryCache(hostId, versionId);
-    if (!cache || !['ready', 'outdated'].includes(cache.status) || cache.manifestSha256 !== version.manifestSha256) throw new Error('The selected host must finish downloading and verifying this game-file version.');
-    return version;
+    if (handle.prepare("SELECT 1 FROM ec_host_operations WHERE host_id=? AND kind='library.remove' AND json_extract(payload,'$.libraryVersionId')=? AND state IN ('pending','accepted','running','unknown')").get(hostId, versionId)) throw new Error('This host copy is being removed.');
+    const cache = handle.prepare('SELECT status, manifest_sha256 FROM host_library_caches WHERE host_id=? AND version_id=?')
+        .get(hostId, versionId) as Pick<CacheRow, 'status' | 'manifest_sha256'> | undefined;
+    if (!cache || !['ready', 'outdated'].includes(cache.status) || cache.manifest_sha256 !== version.manifest_sha256) throw new Error('The selected host must finish downloading and verifying this game-file version.');
+}
+export function requireReadyLibrary(hostId: string, versionId: string): GameLibraryVersion {
+    assertReadyLibrary(hostId, versionId);
+    return getLibraryVersion(versionId)!;
 }
 
 function textField(input: unknown, label: string, max: number, required = true): string {

src/lib/server/hosts/capacity.ts

@@ -1,6 +1,6 @@
 import { db } from '../db/index.js';
 import { listHosts, requireReadyHost } from './index.js';
-import { requireReadyLibrary } from '../game-files/index.js';
+import { assertReadyLibrary } from '../game-files/index.js';
 import type { Host } from '../../hosts.js';
 import { capacityCalibration } from './calibration.js';
 
@@ -101,7 +101,7 @@ export function choosePlacement(options: PlacementOptions): { host: Host; port:
     for (const host of listHosts().filter(host => !options.hostId || host.id === options.hostId)) {
         try {
             requireReadyHost(host.id, { image: options.image, requiredBytes: INSTANCE_STORAGE_BYTES });
-            requireReadyLibrary(host.id, options.libraryId);
+            assertReadyLibrary(host.id, options.libraryId);
             const port = availableHostPort(host, options);
             if (port === null) throw new Error('No available UDP/TCP game port.');
             const load = projectedLoad(host, options);

src/lib/server/provisioning/native.ts

@@ -6,7 +6,7 @@ import { db } from '../db/index.js';
 import { config, relDestForKind, serverDir } from '../config.js';
 import { nativeInstance, nativeRequest, type NativeInstance } from '../backends/native.js';
 import { getHost, getHostOperation, listHosts, queueHostOperation, releaseHostPorts, reserveHostPort } from '../hosts/index.js';
-import { newestReadyLibrary, requireReadyLibrary } from '../game-files/index.js';
+import { newestReadyLibraryId, assertReadyLibrary } from '../game-files/index.js';
 import { blueprintRecipe, captureRecipe, recipeForDeployment, summarizeRecipe, type LaunchRecipe } from '../fleet/launch-recipes.js';
 import { getServerKey, saveServerKey } from '../fleet/server-keys.js';
 import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
@@ -34,9 +34,9 @@ interface FilePlan { id: number; path: string; contentBase64: string | null; sha
 
 export function nativeSelection(input: Record<string, unknown>, game: string, options: Pick<PlacementOptions, 'planned'> = {}): NativeSelection {
     if (input.hostId !== undefined && typeof input.hostId !== 'string') throw new Error('Invalid host.');
-    const version = newestReadyLibrary(game);
-    if (!version) throw new Error(`Upload ${game.toUpperCase()} game files before launching a server.`);
-    if (input.libraryId !== undefined && input.libraryId !== '' && input.libraryId !== version.id) {
+    const libraryId = newestReadyLibraryId(game);
+    if (!libraryId) throw new Error(`Upload ${game.toUpperCase()} game files before launching a server.`);
+    if (input.libraryId !== undefined && input.libraryId !== '' && input.libraryId !== libraryId) {
         throw new Error(`The current ${game.toUpperCase()} game files changed. Refresh the launch preview.`);
     }
     const image = input.image === undefined || input.image === '' ? config.komodo.defaultImage : input.image;
@@ -47,10 +47,10 @@ export function nativeSelection(input: Record<string, unknown>, game: string, op
     const network = rawNetwork as NativeSelection;
     const bindAddress = network.bindAddress || undefined;
     if (bindAddress !== undefined && (typeof bindAddress !== 'string' || !isIP(bindAddress))) throw new Error('Invalid bind address.');
-    const { host } = choosePlacement({ hostId: automatic ? undefined : input.hostId as string, libraryId: version.id, image, bindAddress, ...options });
+    const { host } = choosePlacement({ hostId: automatic ? undefined : input.hostId as string, libraryId, image, bindAddress, ...options });
     if (network.publicAddress !== undefined && (typeof network.publicAddress !== 'string' || network.publicAddress.length > 253)) throw new Error('Invalid public address.');
     if (network.publicPort !== undefined && (!Number.isInteger(network.publicPort) || Number(network.publicPort) < 1 || Number(network.publicPort) > 65535)) throw new Error('Invalid public port.');
-    return { hostId: host.id, libraryId: version.id, image, bindAddress: bindAddress ?? host.bindAddress, publicAddress: network.publicAddress ?? host.publicAddress,
+    return { hostId: host.id, libraryId, image, bindAddress: bindAddress ?? host.bindAddress, publicAddress: network.publicAddress ?? host.publicAddress,
         publicPort: network.publicPort, automatic, network: { bindAddress, publicAddress: network.publicAddress, publicPort: network.publicPort } };
 }
 
@@ -156,7 +156,7 @@ export function queueNativeRebuild(name: string, triggeredBy: string): CreateSer
         const blocker = activeBetaRemovalBlocker(name);
         if (blocker) throw new Error(blocker);
         if (!server?.map_slug || !getServerKey(name)) throw new Error('A saved map and server key are required before rebuilding.');
-        requireReadyLibrary(instance.host_id, instance.library_id);
+        assertReadyLibrary(instance.host_id, instance.library_id);
         const recipe: NativeRecipe = { ...captureRecipe(server.game as 't5' | 't6', server.map_slug), image: instance.image,
             native: { hostId: instance.host_id, libraryId: instance.library_id, image: instance.image, bindAddress: instance.bind_address } };
         return db().transaction(() => {
@@ -434,7 +434,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
     deploymentStage(job.id, 'Checking host eligibility', { phase: 'eligibility', blocker: null });
     if (job.action === 'create') choosePlacement({ hostId: instance.host_id, libraryId: instance.library_id, image: instance.image,
         bindAddress: instance.bind_address, excludeInstanceId: instance.id, thorough: true, dispatch: true });
-    if (job.action === 'create' || job.action === 'rebuild') requireReadyLibrary(instance.host_id, instance.library_id);
+    if (job.action === 'create' || job.action === 'rebuild') assertReadyLibrary(instance.host_id, instance.library_id);
     if (job.action !== 'create' && job.action !== 'start') {
         // Fast controller check keeps occupied lobbies out of the operation slots.
         // The agent repeats this immediately before its local mutation.

tests/unit/collection-launch-performance.test.ts

@@ -0,0 +1,101 @@
+import { createHash } from 'node:crypto';
+import { join } from 'node:path';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+
+vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) }));
+
+let fixture: Fixture;
+let hostId: string;
+let libraryId: string;
+let manifest: string;
+const image = 'example.test/plutainer:test';
+const serverCount = 100;
+
+beforeEach(async () => {
+    vi.resetModules(); vi.clearAllMocks();
+    fixture = await newFixture([]);
+    vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries'));
+    vi.stubEnv('PLUTAINER_IMAGE', image);
+    vi.stubEnv('RCON_PASSWORD', 'test-password');
+    vi.stubEnv('KOMODO_ADDRESS', '');
+    vi.stubEnv('KOMODO_API_KEY', '');
+    vi.stubEnv('KOMODO_API_SECRET', '');
+    const { createHost, createEnrollment, enrollHost, heartbeat } = await import('../../src/lib/server/hosts/index.js');
+    hostId = createHost({ name: 'Large native fleet', portMin: 28960, portMax: 29059 }).id;
+    const inventory = {
+        cpuCount: 256, cpuUsagePercent: 5, memoryTotal: 512 * 1024 ** 3, memoryAvailable: 500 * 1024 ** 3,
+        protocolVersion: 1, dockerRunning: true, directoriesWritable: true,
+        storageAvailable: 1_000_000_000_000, runtimeImages: [image]
+    };
+    enrollHost(createEnrollment(hostId).token, inventory);
+    const { createLibraryImport, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js');
+    libraryId = createLibraryImport({ hostId, game: 't5' }).id;
+    const fileCount = 20_000;
+    const fileSize = 1024 ** 2;
+    const sha256 = createHash('sha256').update('fixture game file').digest('hex');
+    manifest = JSON.stringify(Array.from({ length: fileCount }, (_, index) => ({
+        path: `zone/base-${String(index).padStart(5, '0')}.ff`, size: fileSize, sha256
+    })));
+    const manifestSha256 = createHash('sha256').update(manifest).digest('hex');
+    const { db } = await import('../../src/lib/server/db/index.js');
+    // Seed a published manifest without uploading gigabytes of fixture files.
+    db().transaction(() => {
+        db().prepare(`UPDATE game_file_versions SET state='ready',manifest=?,manifest_sha256=?,size_bytes=?,published_at=? WHERE id=?`)
+            .run(manifest, manifestSha256, fileCount * fileSize, new Date().toISOString(), libraryId);
+        db().prepare(`INSERT INTO game_file_entries(version_id,path,size,sha256)
+            SELECT ?,json_extract(value,'$.path'),json_extract(value,'$.size'),json_extract(value,'$.sha256') FROM json_each(?)`)
+            .run(libraryId, manifest);
+    })();
+    updateHostLibraryCaches(hostId, [{ libraryVersionId: libraryId, status: 'ready', manifestSha256 }]);
+    heartbeat(hostId, { ...inventory, libraries: [{ id: libraryId, state: 'ready' }] });
+});
+
+afterEach(async () => {
+    vi.restoreAllMocks();
+    vi.unstubAllGlobals();
+    await fixture.cleanup();
+    vi.unstubAllEnvs();
+});
+
+describe('large native collection admission', () => {
+    it('previews and queues every lobby without deserializing the game-file manifest', async () => {
+        const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
+        const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const blueprint = saveBlueprint({ name: 'One hundred lobbies', entries:
+            ['theater', 'cosmodrome', 'coast', 'temple', 'moon'].map(mapSlug => ({ game: 't5', mapSlug, count: 20 })) });
+
+        const parse = JSON.parse;
+        let manifestParses = 0;
+        const observeParse: typeof JSON.parse = (value, reviver) => {
+            if (value === manifest) manifestParses++;
+            return parse(value, reviver);
+        };
+        // A mock records all return values and would retain every parsed manifest.
+        vi.stubGlobal('JSON', Object.assign(Object.create(JSON), { parse: observeParse }));
+        const preview = previewCollection(blueprint.id);
+        const previewParses = manifestParses;
+        expect(preview.ready).toBe(true);
+        expect(preview.servers).toHaveLength(serverCount);
+        expect(new Set(preview.servers.map(server => server.name)).size).toBe(serverCount);
+        expect(new Set(preview.servers.map(server => server.port)).size).toBe(serverCount);
+        expect(db().prepare('SELECT * FROM ec_host_ports').all()).toHaveLength(0);
+
+        manifestParses = 0;
+        const result = queueCollection({ previewId: preview.id,
+            keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key-' + server.name])) });
+        const queueParses = manifestParses;
+        expect(result).toMatchObject({ ok: true, collectionId: expect.any(String) });
+        expect(queueCollection({ previewId: preview.id })).toEqual(result);
+        expect(getCollection(result.collectionId!)?.collection).toMatchObject({ total: serverCount, queued: serverCount });
+        const instances = db().prepare('SELECT host_id,library_id,port FROM native_instances ORDER BY port').all();
+        expect(instances).toEqual(Array.from({ length: serverCount }, (_, index) => ({
+            host_id: hostId, library_id: libraryId, port: 28960 + index
+        })));
+        expect(db().prepare('SELECT * FROM ec_host_ports').all()).toHaveLength(serverCount * 2);
+        expect(db().prepare('SELECT * FROM server_collection_members').all()).toHaveLength(serverCount);
+        expect(previewParses, 'Preview must use library metadata without reading the full manifest').toBe(0);
+        expect(queueParses, 'Queue admission must use library metadata without reading the full manifest').toBe(0);
+    }, 30_000);
+});

tests/unit/game-file-library.test.ts

@@ -6,8 +6,8 @@ import { join } from 'node:path';
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 const state = vi.hoisted(() => ({ handle: null as Database.Database | null }));
 vi.mock('../../src/lib/server/db/index.js', () => ({ db: () => state.handle }));
-import { heartbeat, createEnrollment, createHost, enrollHost, getHostOperation, reportHostOperation, revokeCredentials } from '../../src/lib/server/hosts/index.js';
-import { cleanupControllerLibraries, reconcileHostLibraryCleanup, createLibraryImport, getHostLibraryCache, getLibraryVersion, importPublicationStatus, latestPublishedLibraries, libraryOverview, listLibraries, newestReadyLibrary, publishImport, queueLibraryDownload, requestImportPublication, requireReadyLibrary, setImportManifest, updateHostLibraryCaches, uploadImportFile } from '../../src/lib/server/game-files/index.js';
+import { heartbeat, createEnrollment, createHost, enrollHost, getHostOperation, queueHostOperation, reportHostOperation, revokeCredentials } from '../../src/lib/server/hosts/index.js';
+import { assertReadyLibrary, cleanupControllerLibraries, reconcileHostLibraryCleanup, createLibraryImport, getHostLibraryCache, getLibraryVersion, importPublicationStatus, latestPublishedLibraries, libraryOverview, listLibraries, newestReadyLibrary, publishImport, queueLibraryDownload, requestImportPublication, requireReadyLibrary, setImportManifest, updateHostLibraryCaches, uploadImportFile } from '../../src/lib/server/game-files/index.js';
 import { serveGameFile } from '../../src/lib/server/game-files/http.js';
 import { GET as manifestRoute } from '../../src/routes/api/agent/libraries/[id]/manifest/+server.js';
 
@@ -180,6 +180,65 @@ describe('verified host distribution', () => {
     });
 });
 
+describe.each([
+    { name: 'metadata readiness checks', check: assertReadyLibrary },
+    { name: 'full library readiness checks', check: requireReadyLibrary }
+])('$name', ({ check }) => {
+    it('rejects missing, importing and failed versions', () => {
+        const version = begin();
+        expect(() => check(hostId, 'missing')).toThrow('Select an available published game-file version.');
+        expect(() => check(hostId, version.id)).toThrow('Select an available published game-file version.');
+        state.handle!.prepare("UPDATE game_file_versions SET state='failed' WHERE id=?").run(version.id);
+        expect(() => check(hostId, version.id)).toThrow('Select an available published game-file version.');
+    });
+
+    it('rejects controller files as soon as retirement starts and after removal', async () => {
+        const version = await published();
+        updateHostLibraryCaches(hostId, [{ libraryVersionId: version.id, status: 'ready', manifestSha256: version.manifestSha256 }]);
+        expect(() => check(hostId, version.id)).not.toThrow();
+        state.handle!.prepare('INSERT INTO game_file_cleanup(version_id,started_at) VALUES(?,?)').run(version.id, new Date().toISOString());
+        expect(() => check(hostId, version.id)).toThrow('Select an available published game-file version.');
+        state.handle!.prepare('UPDATE game_file_cleanup SET removed_at=? WHERE version_id=?').run(new Date().toISOString(), version.id);
+        expect(() => check(hostId, version.id)).toThrow('Select an available published game-file version.');
+    });
+
+    it('blocks unresolved removal only for the selected host and version', async () => {
+        const version = await published();
+        updateHostLibraryCaches(hostId, [{ libraryVersionId: version.id, status: 'ready', manifestSha256: version.manifestSha256 }]);
+        const otherHost = createHost({ name: 'Other host' });
+        queueHostOperation(otherHost.id, 'library.remove', { libraryVersionId: version.id });
+        queueHostOperation(hostId, 'library.remove', { libraryVersionId: begin().id });
+        expect(() => check(hostId, version.id)).not.toThrow();
+        const removal = queueHostOperation(hostId, 'library.remove', { libraryVersionId: version.id });
+        const updateState = state.handle!.prepare('UPDATE ec_host_operations SET state=? WHERE id=?');
+        for (const operationState of ['pending', 'accepted', 'running', 'unknown']) {
+            // Exercise every unresolved state without applying unrelated receipt side effects.
+            updateState.run(operationState, removal.id);
+            expect(() => check(hostId, version.id)).toThrow('This host copy is being removed.');
+        }
+        for (const operationState of ['completed', 'failed']) {
+            updateState.run(operationState, removal.id);
+            expect(() => check(hostId, version.id)).not.toThrow();
+        }
+    });
+
+    it('requires the exact checksum and accepts verified ready or outdated copies', async () => {
+        const version = await published();
+        expect(() => check(hostId, version.id)).toThrow('finish downloading and verifying');
+        updateHostLibraryCaches(hostId, [{ libraryVersionId: version.id, status: 'ready', manifestSha256: version.manifestSha256 }]);
+        for (const status of ['ready', 'outdated']) {
+            state.handle!.prepare('UPDATE host_library_caches SET status=?,manifest_sha256=? WHERE host_id=? AND version_id=?')
+                .run(status, 'different-manifest', hostId, version.id);
+            expect(() => check(hostId, version.id)).toThrow('finish downloading and verifying');
+            state.handle!.prepare('UPDATE host_library_caches SET manifest_sha256=? WHERE host_id=? AND version_id=?')
+                .run(version.manifestSha256, hostId, version.id);
+            expect(() => check(hostId, version.id)).not.toThrow();
+        }
+        expect(getLibraryVersion(version.id)?.files).toEqual([file]);
+        expect(requireReadyLibrary(hostId, version.id).files).toEqual([file]);
+    });
+});
+
 
 describe('obsolete game-file cleanup', () => {
     beforeEach(() => {