Fix blocking collection matching with large plugins

tests/unit/collection-match-performance.test.ts

@@ -0,0 +1,145 @@
+import { randomUUID } from 'node:crypto';
+import { join } from 'node:path';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+
+vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
+    rconGetPlayerCount: vi.fn(async () => 0),
+    rconPlutoniumVersion: vi.fn(async () => ({ revision: 5354, error: null })),
+    rconStatusResult: vi.fn(async () => ({ ok: true, endpoint: { host: 'agent', port: 28960 }, rawReply: '',
+        status: { players: 0, map: 'zm_transit', gametype: null, roster: [] } }))
+}));
+
+let fixture: Fixture;
+let hostId: string;
+const image = 'example.test/plutainer:test';
+
+beforeEach(async () => {
+    vi.resetModules(); vi.clearAllMocks();
+    fixture = await newFixture([]);
+    vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries'));
+    vi.stubEnv('PLUTAINER_IMAGE', image);
+    vi.stubEnv('RCON_PASSWORD', 'test-password');
+    vi.stubEnv('KOMODO_ADDRESS', '');
+    vi.stubEnv('KOMODO_API_KEY', '');
+    vi.stubEnv('KOMODO_API_SECRET', '');
+    vi.stubGlobal('fetch', vi.fn(async () => Response.json({ product: 'plutonium-core-prod', revision: 5354, files: [] })));
+    const { createHost, createEnrollment, enrollHost, heartbeat, reserveHostPort } = await import('../../src/lib/server/hosts/index.js');
+    hostId = createHost({ name: 'Match collection host', portMin: 28960, portMax: 28999 }).id;
+    const inventory = {
+        cpuCount: 64, cpuUsagePercent: 5, memoryTotal: 128 * 1024 ** 3, memoryAvailable: 120 * 1024 ** 3,
+        protocolVersion: 1, dockerRunning: true, directoriesWritable: true,
+        storageAvailable: 1_000_000_000_000, runtimeImages: [image]
+    };
+    enrollHost(createEnrollment(hostId).token, inventory);
+    const { createLibraryImport, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js');
+    const { db } = await import('../../src/lib/server/db/index.js');
+    const libraries = ['t5', 't6'].map(game => {
+        const id = createLibraryImport({ hostId, game }).id;
+        db().prepare("UPDATE game_file_versions SET state='ready',manifest='[]',manifest_sha256=?,size_bytes=1,published_at=? WHERE id=?")
+            .run('a'.repeat(64), new Date().toISOString(), id);
+        return { id, state: 'ready' as const };
+    });
+    updateHostLibraryCaches(hostId, libraries.map(library => ({ libraryVersionId: library.id, status: 'ready', manifestSha256: 'a'.repeat(64) })));
+    heartbeat(hostId, { ...inventory, libraries });
+    const { T6_ZOMBIE_MAPS } = await import('../../src/lib/map-aliases.js');
+    const now = new Date().toISOString();
+    db().transaction(() => {
+        for (const [index, map] of T6_ZOMBIE_MAPS.slice(0, 9).entries()) {
+            const name = `t6_zombie_${map.slug}_1`;
+            const instanceId = randomUUID();
+            const deploymentId = randomUUID();
+            const port = 28960 + index;
+            db().prepare(`INSERT INTO servers(name,komodo_name,game,map,game_mode,map_slug,game_port,container_state,last_players,last_seen_at)
+                VALUES(?,'','t6',?,'zombie',?,?,'running',0,?)`).run(name, map.canonical, map.slug, port, now);
+            db().prepare('INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) VALUES(?,?,?,?,?,?,?)')
+                .run(instanceId, name, hostId, libraries[1].id, image, port, now);
+            db().prepare("INSERT INTO server_deployments(id,server,action,state,stage,triggered_by,started_at,finished_at) VALUES(?,?,'create','done','Verified','fixture',?,?)")
+                .run(deploymentId, name, now, now);
+            db().prepare('INSERT INTO native_provisioning(deployment_id,instance_id,generation) VALUES(?,?,1)').run(deploymentId, instanceId);
+            for (const protocol of ['tcp', 'udp'] as const) reserveHostPort(hostId, instanceId, { port, protocol });
+        }
+    })();
+    const backend = await import('../../src/lib/server/backends/native.js');
+    vi.spyOn(backend, 'nativeRequest').mockImplementation(async (_instance, kind) => {
+        if (kind !== 'instance.inspect') throw new Error(`Unexpected request ${kind}`);
+        return { exists: true, healthy: true, state: 'running' } as never;
+    });
+});
+
+afterEach(async () => {
+    await new Promise<void>(resolve => setImmediate(resolve));
+    const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
+    await processProvisioning();
+    await new Promise<void>(resolve => setImmediate(resolve));
+    vi.restoreAllMocks();
+    await fixture.cleanup();
+    vi.unstubAllEnvs(); vi.unstubAllGlobals();
+});
+
+describe('matching a native collection', () => {
+    it('admits nine removals and ten creations without copying plugin bytes into every queued launch', async () => {
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { createScript, setScope, deleteScript } = await import('../../src/lib/server/fleet/scripts.js');
+        const content = Buffer.alloc(8 * 1024 ** 2, 42);
+        const { script, version } = createScript({ name: 'Promoted plugin', kind: 'plugin', targetRelPath: 'fixture.dll',
+            scopeType: 'game', scopeValue: 't5', content });
+        db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(version.id, script.id);
+        const { saveBlueprint, recipeForDeployment } = await import('../../src/lib/server/fleet/launch-recipes.js');
+        const { previewCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js');
+        const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
+        const { nativeRequest } = await import('../../src/lib/server/backends/native.js');
+        const { processProvisioning } = await import('../../src/lib/server/provisioning/worker.js');
+        const { POST } = await import('../../src/routes/api/deployment-collections/+server.js');
+        const { T5_ZOMBIE_MAPS } = await import('../../src/lib/map-aliases.js');
+        const blueprint = saveBlueprint({ name: 'Match ten T5 lobbies', entries: T5_ZOMBIE_MAPS.filter(map => map.slug !== 'zombietron')
+            .map(map => ({ game: 't5', mapSlug: map.slug, count: 1 })) });
+        const preview = previewCollection(blueprint.id, 'exact');
+        expect(preview).toMatchObject({ ready: true, mode: 'exact' });
+        expect(preview.removals).toHaveLength(9);
+        expect(preview.servers).toHaveLength(10);
+        const storedPreview = db().prepare('SELECT length(entries) AS bytes FROM server_collection_previews WHERE id=?')
+            .get(preview.id) as { bytes: number };
+        expect(storedPreview.bytes).toBeLessThan(100_000);
+
+        const submit = (keys?: Record<string, string>) => POST({ request: new Request('http://localhost/api/deployment-collections', {
+            method: 'POST', body: JSON.stringify({ previewId: preview.id, keys }) }) });
+        const response = await submit(Object.fromEntries(preview.servers.map(server => [server.name!, `key-${server.name}`])));
+        const body = await response.json();
+        expect(response.status).toBe(202);
+        expect(body).toMatchObject({ ok: true, collectionId: expect.any(String) });
+        expect(await (await submit()).json()).toEqual(body);
+        // Exercise the real route and worker: admission must return before
+        // loading launch payloads or inspecting the nine servers to remove.
+        expect(nativeRequest).not.toHaveBeenCalled();
+        expect(db().prepare("SELECT id FROM ec_host_operations WHERE kind='instance.create'").all()).toEqual([]);
+        for (const table of ['server_launch_previews', 'server_launch_recipes']) {
+            const rows = db().prepare(`SELECT length(recipe) AS bytes FROM ${table}`).all() as { bytes: number }[];
+            expect(rows).toHaveLength(10);
+            expect(rows.every(row => row.bytes < 10_000)).toBe(true);
+        }
+        const collection = getCollection(body.collectionId)!;
+        expect(collection.collection).toMatchObject({ mode: 'exact', total: 19 });
+        expect(collection.servers.filter(server => server.action === 'destroy')).toHaveLength(9);
+        expect(collection.servers.filter(server => server.action === 'create')).toHaveLength(10);
+
+        await new Promise<void>(resolve => setImmediate(resolve));
+        await processProvisioning();
+        const dispatched = db().prepare(`SELECT n.deployment_id,o.payload FROM native_provisioning n
+            JOIN ec_host_operations o ON o.id=n.operation_id WHERE o.kind='instance.create'`).all() as { deployment_id: string; payload: string }[];
+        expect(dispatched).toHaveLength(1);
+        const payload = JSON.parse(dispatched[0].payload) as { files: { path: string; contentBase64: string; sha256: string }[] };
+        expect(payload.files.find(file => file.path.endsWith('/fixture.dll'))).toMatchObject({
+            sha256: version.sha256, contentBase64: content.toString('base64')
+        });
+
+        // Queued plans are descriptions, while an attempted launch preserves
+        // its exact bytes even after the promoted artifact is deleted.
+        setScope({ scriptId: script.id, scopeType: null, scopeValue: null });
+        deleteScript(script.id);
+        const queued = collection.servers.find(server => server.action === 'create' && server.id !== dispatched[0].deployment_id)!;
+        expect(getDeployment(queued.id)?.recipe?.files).toContainEqual(expect.objectContaining({ path: 'fixture.dll', versionId: version.id, size: content.length }));
+        const attempted = recipeForDeployment(dispatched[0].deployment_id)!.recipe.files.find(file => file.path === 'fixture.dll');
+        expect(attempted).toMatchObject({ sha256: version.sha256, content: content.toString('base64') });
+    }, 30_000);
+});

tests/unit/deployment-collections.test.ts

@@ -19,7 +19,7 @@ vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
     rconStatusResult: vi.fn(async (server: string) => ({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: '',
         status: { players: 0, map: server.startsWith('t6_') ? 'zm_transit' : server.includes('_moon_') ? 'zombie_moon' : 'zombie_theater', gametype: null, roster: [] } }))
 }));
-vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}) }));
+vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) }));
 
 let fixture: Fixture;
 const entries = [

tests/unit/launch-recipes.test.ts

@@ -100,7 +100,8 @@ describe('launch recipes and previews', () => {
         expect(getServerScript(queued.name, 'perks.gsc')).toMatchObject({ script_version_id: newScript.id, sha256: newScript.sha256 });
         expect(getServerScript(queued.name, 'utilities.dll')).toMatchObject({ script_version_id: newPlugin.id, sha256: newPlugin.sha256 });
         const recordedConfig = recipeForDeployment(queued.deploymentId)?.recipe.files.find(file => file.path === 'dedicated_sp.cfg');
-        expect(Buffer.from(recordedConfig!.content, 'base64').toString()).toBe(deployedConfig);
+        if (recordedConfig?.content === undefined) throw new Error('Attempted launch must retain the configuration bytes.');
+        expect(Buffer.from(recordedConfig.content, 'base64').toString()).toBe(deployedConfig);
         expect(getDeployment(queued.deploymentId)?.recipe?.fingerprint).not.toBe(preview.recipe.fingerprint);
         expect(getDeployment(queued.deploymentId)?.deployment).toMatchObject({ state: 'awaiting-health' });
         await processCreations();

tests/unit/provisioning-history-performance.test.ts

@@ -0,0 +1,79 @@
+import Database from 'better-sqlite3';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import schema from '../../src/lib/server/db/schema.sql?raw';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+
+let fixture: Fixture;
+beforeEach(async () => {
+    vi.resetModules();
+    fixture = await newFixture([]);
+});
+afterEach(async () => {
+    vi.restoreAllMocks();
+    await fixture.cleanup();
+});
+
+describe('provisioning reads with retained lifecycle history', () => {
+    it('indexes the server and queue reads used by collection matching and health polling', async () => {
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { latestGameSubmission } = await import('../../src/lib/server/fleet/lifecycle-evidence.js');
+        const { hasActiveProvisioning } = await import('../../src/lib/server/fleet/provisioning.js');
+        const { housePartyTargets } = await import('../../src/lib/server/fleet/house-party.js');
+        const { creationQueue } = await import('../../src/lib/server/provisioning/instances.js');
+        const handle = db();
+        handle.transaction(() => {
+            const insert = handle.prepare(`INSERT INTO server_deployments
+                (id,server,action,state,stage,started_at,triggered_by,server_deleted_at)
+                VALUES(?,?,'create','done','Complete','2025-01-01','fixture','2025-01-02')`);
+            for (let i = 0; i < 2_000; i++) insert.run(`history-${i}`, `retired-${i}`);
+            handle.prepare(`INSERT INTO server_deployments
+                (id,server,action,state,stage,started_at,triggered_by,submitted_at)
+                VALUES('current','current','create','queued','Queued','2026-01-01','fixture','2026-01-01')`).run();
+        })();
+
+        // Inspect the SQL issued by the public reads, so the regression follows
+        // those implementations rather than keeping separate copies of them.
+        const expectIndexed = (read: () => unknown, bindings: string[] = []) => {
+            const prepare = vi.spyOn(handle, 'prepare');
+            read();
+            const queries = prepare.mock.calls.map(([sql]) => sql).filter(sql => sql.includes('server_deployments'));
+            prepare.mockRestore();
+            expect(queries).toHaveLength(1);
+            const plan = handle.prepare(`EXPLAIN QUERY PLAN ${queries[0]}`).all(...bindings) as { detail: string }[];
+            const details = plan.map(row => row.detail).join('\n');
+            expect(details).toMatch(/SEARCH server_deployments USING (?:COVERING )?INDEX/);
+            expect(details).not.toMatch(/SCAN server_deployments\b/);
+        };
+        expectIndexed(() => expect(latestGameSubmission('current')).toBe('2026-01-01'), ['current', 'current']);
+        expectIndexed(() => expect(hasActiveProvisioning('current')).toBe(true), ['current', '']);
+        expectIndexed(() => expect(housePartyTargets()).toEqual(['current']));
+        expectIndexed(() => expect(creationQueue().queued.map(job => job.id)).toEqual(['current']));
+    });
+
+    it('installs the indexes after legacy table rebuilds and keeps them on reopen', async () => {
+        const legacy = new Database(fixture.dbPath);
+        try {
+            legacy.exec(schema.replace(/CREATE TABLE IF NOT EXISTS server_deployments \([\s\S]*?\n\);/, table => table
+                .replace("'create', 'start', 'restart', 'rebuild', 'stop', 'destroy'", "'create', 'destroy'")
+                .replace('    server_deleted_at TEXT,\n', '')));
+            legacy.exec(`INSERT INTO servers(name,komodo_name) VALUES('existing','existing');
+                INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by)
+                    VALUES('old','existing','create','done','Complete','2025-01-01','fixture');
+                INSERT INTO server_deployment_events(deployment_id,created_at,message)
+                    VALUES('old','2025-01-01','Creation complete');`);
+        } finally { legacy.close(); }
+        const { db, closeDb } = await import('../../src/lib/server/db/index.js');
+        for (let open = 0; open < 2; open++) {
+            const handle = db();
+            expect(handle.prepare('SELECT rowid,id,state FROM server_deployments').all())
+                .toEqual([{ rowid: 1, id: 'old', state: 'done' }]);
+            expect(handle.prepare('SELECT message FROM server_deployment_events').all()).toEqual([{ message: 'Creation complete' }]);
+            const indexes = handle.pragma('index_list(server_deployments)') as { name: string }[];
+            expect(indexes.map(index => index.name)).toEqual(expect.arrayContaining([
+                'idx_server_deployments_server', 'idx_server_deployments_state'
+            ]));
+            expect(handle.pragma('foreign_key_check')).toEqual([]);
+            closeDb();
+        }
+    });
+});