Fix controller stalls during large collection launches

tests/unit/collection-launch-performance.test.ts

@@ -0,0 +1,101 @@
+import { createHash } from 'node:crypto';
+import { join } from 'node:path';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+
+vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) }));
+
+let fixture: Fixture;
+let hostId: string;
+let libraryId: string;
+let manifest: string;
+const image = 'example.test/plutainer:test';
+const serverCount = 100;
+
+beforeEach(async () => {
+    vi.resetModules(); vi.clearAllMocks();
+    fixture = await newFixture([]);
+    vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries'));
+    vi.stubEnv('PLUTAINER_IMAGE', image);
+    vi.stubEnv('RCON_PASSWORD', 'test-password');
+    vi.stubEnv('KOMODO_ADDRESS', '');
+    vi.stubEnv('KOMODO_API_KEY', '');
+    vi.stubEnv('KOMODO_API_SECRET', '');
+    const { createHost, createEnrollment, enrollHost, heartbeat } = await import('../../src/lib/server/hosts/index.js');
+    hostId = createHost({ name: 'Large native fleet', portMin: 28960, portMax: 29059 }).id;
+    const inventory = {
+        cpuCount: 256, cpuUsagePercent: 5, memoryTotal: 512 * 1024 ** 3, memoryAvailable: 500 * 1024 ** 3,
+        protocolVersion: 1, dockerRunning: true, directoriesWritable: true,
+        storageAvailable: 1_000_000_000_000, runtimeImages: [image]
+    };
+    enrollHost(createEnrollment(hostId).token, inventory);
+    const { createLibraryImport, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js');
+    libraryId = createLibraryImport({ hostId, game: 't5' }).id;
+    const fileCount = 20_000;
+    const fileSize = 1024 ** 2;
+    const sha256 = createHash('sha256').update('fixture game file').digest('hex');
+    manifest = JSON.stringify(Array.from({ length: fileCount }, (_, index) => ({
+        path: `zone/base-${String(index).padStart(5, '0')}.ff`, size: fileSize, sha256
+    })));
+    const manifestSha256 = createHash('sha256').update(manifest).digest('hex');
+    const { db } = await import('../../src/lib/server/db/index.js');
+    // Seed a published manifest without uploading gigabytes of fixture files.
+    db().transaction(() => {
+        db().prepare(`UPDATE game_file_versions SET state='ready',manifest=?,manifest_sha256=?,size_bytes=?,published_at=? WHERE id=?`)
+            .run(manifest, manifestSha256, fileCount * fileSize, new Date().toISOString(), libraryId);
+        db().prepare(`INSERT INTO game_file_entries(version_id,path,size,sha256)
+            SELECT ?,json_extract(value,'$.path'),json_extract(value,'$.size'),json_extract(value,'$.sha256') FROM json_each(?)`)
+            .run(libraryId, manifest);
+    })();
+    updateHostLibraryCaches(hostId, [{ libraryVersionId: libraryId, status: 'ready', manifestSha256 }]);
+    heartbeat(hostId, { ...inventory, libraries: [{ id: libraryId, state: 'ready' }] });
+});
+
+afterEach(async () => {
+    vi.restoreAllMocks();
+    vi.unstubAllGlobals();
+    await fixture.cleanup();
+    vi.unstubAllEnvs();
+});
+
+describe('large native collection admission', () => {
+    it('previews and queues every lobby without deserializing the game-file manifest', async () => {
+        const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js');
+        const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const blueprint = saveBlueprint({ name: 'One hundred lobbies', entries:
+            ['theater', 'cosmodrome', 'coast', 'temple', 'moon'].map(mapSlug => ({ game: 't5', mapSlug, count: 20 })) });
+
+        const parse = JSON.parse;
+        let manifestParses = 0;
+        const observeParse: typeof JSON.parse = (value, reviver) => {
+            if (value === manifest) manifestParses++;
+            return parse(value, reviver);
+        };
+        // A mock records all return values and would retain every parsed manifest.
+        vi.stubGlobal('JSON', Object.assign(Object.create(JSON), { parse: observeParse }));
+        const preview = previewCollection(blueprint.id);
+        const previewParses = manifestParses;
+        expect(preview.ready).toBe(true);
+        expect(preview.servers).toHaveLength(serverCount);
+        expect(new Set(preview.servers.map(server => server.name)).size).toBe(serverCount);
+        expect(new Set(preview.servers.map(server => server.port)).size).toBe(serverCount);
+        expect(db().prepare('SELECT * FROM ec_host_ports').all()).toHaveLength(0);
+
+        manifestParses = 0;
+        const result = queueCollection({ previewId: preview.id,
+            keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key-' + server.name])) });
+        const queueParses = manifestParses;
+        expect(result).toMatchObject({ ok: true, collectionId: expect.any(String) });
+        expect(queueCollection({ previewId: preview.id })).toEqual(result);
+        expect(getCollection(result.collectionId!)?.collection).toMatchObject({ total: serverCount, queued: serverCount });
+        const instances = db().prepare('SELECT host_id,library_id,port FROM native_instances ORDER BY port').all();
+        expect(instances).toEqual(Array.from({ length: serverCount }, (_, index) => ({
+            host_id: hostId, library_id: libraryId, port: 28960 + index
+        })));
+        expect(db().prepare('SELECT * FROM ec_host_ports').all()).toHaveLength(serverCount * 2);
+        expect(db().prepare('SELECT * FROM server_collection_members').all()).toHaveLength(serverCount);
+        expect(previewParses, 'Preview must use library metadata without reading the full manifest').toBe(0);
+        expect(queueParses, 'Queue admission must use library metadata without reading the full manifest').toBe(0);
+    }, 30_000);
+});

tests/unit/game-file-library.test.ts

@@ -6,8 +6,8 @@ import { join } from 'node:path';
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 const state = vi.hoisted(() => ({ handle: null as Database.Database | null }));
 vi.mock('../../src/lib/server/db/index.js', () => ({ db: () => state.handle }));
-import { heartbeat, createEnrollment, createHost, enrollHost, getHostOperation, reportHostOperation, revokeCredentials } from '../../src/lib/server/hosts/index.js';
-import { cleanupControllerLibraries, reconcileHostLibraryCleanup, createLibraryImport, getHostLibraryCache, getLibraryVersion, importPublicationStatus, latestPublishedLibraries, libraryOverview, listLibraries, newestReadyLibrary, publishImport, queueLibraryDownload, requestImportPublication, requireReadyLibrary, setImportManifest, updateHostLibraryCaches, uploadImportFile } from '../../src/lib/server/game-files/index.js';
+import { heartbeat, createEnrollment, createHost, enrollHost, getHostOperation, queueHostOperation, reportHostOperation, revokeCredentials } from '../../src/lib/server/hosts/index.js';
+import { assertReadyLibrary, cleanupControllerLibraries, reconcileHostLibraryCleanup, createLibraryImport, getHostLibraryCache, getLibraryVersion, importPublicationStatus, latestPublishedLibraries, libraryOverview, listLibraries, newestReadyLibrary, publishImport, queueLibraryDownload, requestImportPublication, requireReadyLibrary, setImportManifest, updateHostLibraryCaches, uploadImportFile } from '../../src/lib/server/game-files/index.js';
 import { serveGameFile } from '../../src/lib/server/game-files/http.js';
 import { GET as manifestRoute } from '../../src/routes/api/agent/libraries/[id]/manifest/+server.js';
 
@@ -180,6 +180,65 @@ describe('verified host distribution', () => {
     });
 });
 
+describe.each([
+    { name: 'metadata readiness checks', check: assertReadyLibrary },
+    { name: 'full library readiness checks', check: requireReadyLibrary }
+])('$name', ({ check }) => {
+    it('rejects missing, importing and failed versions', () => {
+        const version = begin();
+        expect(() => check(hostId, 'missing')).toThrow('Select an available published game-file version.');
+        expect(() => check(hostId, version.id)).toThrow('Select an available published game-file version.');
+        state.handle!.prepare("UPDATE game_file_versions SET state='failed' WHERE id=?").run(version.id);
+        expect(() => check(hostId, version.id)).toThrow('Select an available published game-file version.');
+    });
+
+    it('rejects controller files as soon as retirement starts and after removal', async () => {
+        const version = await published();
+        updateHostLibraryCaches(hostId, [{ libraryVersionId: version.id, status: 'ready', manifestSha256: version.manifestSha256 }]);
+        expect(() => check(hostId, version.id)).not.toThrow();
+        state.handle!.prepare('INSERT INTO game_file_cleanup(version_id,started_at) VALUES(?,?)').run(version.id, new Date().toISOString());
+        expect(() => check(hostId, version.id)).toThrow('Select an available published game-file version.');
+        state.handle!.prepare('UPDATE game_file_cleanup SET removed_at=? WHERE version_id=?').run(new Date().toISOString(), version.id);
+        expect(() => check(hostId, version.id)).toThrow('Select an available published game-file version.');
+    });
+
+    it('blocks unresolved removal only for the selected host and version', async () => {
+        const version = await published();
+        updateHostLibraryCaches(hostId, [{ libraryVersionId: version.id, status: 'ready', manifestSha256: version.manifestSha256 }]);
+        const otherHost = createHost({ name: 'Other host' });
+        queueHostOperation(otherHost.id, 'library.remove', { libraryVersionId: version.id });
+        queueHostOperation(hostId, 'library.remove', { libraryVersionId: begin().id });
+        expect(() => check(hostId, version.id)).not.toThrow();
+        const removal = queueHostOperation(hostId, 'library.remove', { libraryVersionId: version.id });
+        const updateState = state.handle!.prepare('UPDATE ec_host_operations SET state=? WHERE id=?');
+        for (const operationState of ['pending', 'accepted', 'running', 'unknown']) {
+            // Exercise every unresolved state without applying unrelated receipt side effects.
+            updateState.run(operationState, removal.id);
+            expect(() => check(hostId, version.id)).toThrow('This host copy is being removed.');
+        }
+        for (const operationState of ['completed', 'failed']) {
+            updateState.run(operationState, removal.id);
+            expect(() => check(hostId, version.id)).not.toThrow();
+        }
+    });
+
+    it('requires the exact checksum and accepts verified ready or outdated copies', async () => {
+        const version = await published();
+        expect(() => check(hostId, version.id)).toThrow('finish downloading and verifying');
+        updateHostLibraryCaches(hostId, [{ libraryVersionId: version.id, status: 'ready', manifestSha256: version.manifestSha256 }]);
+        for (const status of ['ready', 'outdated']) {
+            state.handle!.prepare('UPDATE host_library_caches SET status=?,manifest_sha256=? WHERE host_id=? AND version_id=?')
+                .run(status, 'different-manifest', hostId, version.id);
+            expect(() => check(hostId, version.id)).toThrow('finish downloading and verifying');
+            state.handle!.prepare('UPDATE host_library_caches SET manifest_sha256=? WHERE host_id=? AND version_id=?')
+                .run(version.manifestSha256, hostId, version.id);
+            expect(() => check(hostId, version.id)).not.toThrow();
+        }
+        expect(getLibraryVersion(version.id)?.files).toEqual([file]);
+        expect(requireReadyLibrary(hostId, version.id).files).toEqual([file]);
+    });
+});
+
 
 describe('obsolete game-file cleanup', () => {
     beforeEach(() => {