Keep native collection launches responsive

tests/helpers/fixture.ts

@@ -49,6 +49,8 @@ export async function newFixture(servers: string[] = ['t5_zombie_theater_1']): P
             try {
                 const { closeReports } = await import('../../src/lib/server/reports/index.js');
                 await closeReports();
+                const { closeLaunchPreparation } = await import('../../src/lib/server/provisioning/launch-preparation.js');
+                await closeLaunchPreparation();
                 const mod = (await import('../../src/lib/server/db/index.js')) as {
                     closeDb: () => void;
                 };

tests/unit/agent-operation-delivery.test.ts

@@ -0,0 +1,142 @@
+import { setImmediate } from 'node:timers/promises';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+
+vi.mock('../../src/lib/server/hosts/preparation.js', () => ({ reconcileHostPreparation: vi.fn() }));
+
+let fixture: Fixture;
+const inventory = {
+    protocolVersion: 1, dockerRunning: true, directoriesWritable: true, storageAvailable: 100 * 1024 ** 3,
+    runtimeImages: ['plutainer:test'], libraries: [{ id: 'test-library', state: 'ready' }],
+    capabilities: ['instance.cancel-create', 'operations.health-lane']
+};
+
+beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
+afterEach(async () => { vi.restoreAllMocks(); vi.useRealTimers(); await fixture.cleanup(); });
+
+async function enrolled() {
+    const hosts = await import('../../src/lib/server/hosts/index.js');
+    const host = hosts.createHost({ name: 'Delivery host' });
+    const credentials = hosts.enrollHost(hosts.createEnrollment(host.id).token, inventory);
+    const { GET } = await import('../../src/routes/api/agent/operations/+server.js');
+    const request = (query = '') => new Request(`https://ec.test/api/agent/operations${query}`, {
+        headers: { authorization: `Bearer ${credentials.credential}` }
+    });
+    return { ...hosts, host, GET, request };
+}
+
+describe('agent operation delivery', () => {
+    it('streams large persisted launch files without parsing payloads and yields to controller work', async () => {
+        const api = await enrolled();
+        const payload = {
+            // Place a Unicode character across the first payload byte chunk.
+            value: 'x'.repeat(64 * 1024 - '{"value":"'.length - 1) + '🧟 café "quoted"\n',
+            files: [{ path: 'plugins/runtime.dll', contentBase64: 'A'.repeat(8 * 1024 ** 2) }]
+        };
+        const operation = api.queueHostOperation(api.host.id, 'instance.create', payload,
+            { instanceId: 'instance-one', generation: 2 });
+        const result = { phase: 'prepared', description: '世界', details: [null, true, 42] };
+        api.reportHostOperation(api.host.id, operation.id, { state: 'running', result });
+        const expected = api.getHostOperation(operation.id)!;
+        const serialized = JSON.stringify(payload);
+        const originalParse = JSON.parse;
+        const parse = vi.spyOn(JSON, 'parse').mockImplementation((value, reviver) => {
+            if (value === serialized) throw new Error('Delivery must not deserialize launch files.');
+            return originalParse(value, reviver);
+        });
+
+        const response = await api.GET({ request: api.request('?lane=write') });
+        expect(response.headers.get('cache-control')).toBe('no-store');
+        expect(response.headers.get('content-type')).toContain('application/json');
+        let complete = false;
+        let interleaved = false;
+        const controllerWork = setImmediate().then(() => { interleaved = !complete; });
+        const chunks: Uint8Array[] = [];
+        const reader = response.body!.getReader();
+        while (true) {
+            const next = await reader.read();
+            if (next.done) break;
+            expect(next.value.byteLength).toBeLessThanOrEqual(64 * 1024);
+            chunks.push(next.value);
+        }
+        complete = true;
+        await controllerWork;
+        expect(interleaved).toBe(true);
+        expect(chunks.length).toBeGreaterThan(100);
+        parse.mockRestore();
+        const body = JSON.parse(Buffer.concat(chunks).toString('utf8'));
+        expect(body).toEqual({ operations: [expected], serverTime: expect.any(String) });
+        expect(Number.isFinite(Date.parse(body.serverTime))).toBe(true);
+    });
+
+    it('selects without payloads, loads one payload at a time, and retains the selected receipt snapshot', async () => {
+        const api = await enrolled();
+        const payload = Buffer.from(JSON.stringify({ file: 'A'.repeat(512 * 1024) }));
+        api.queueHostOperationSerialized(api.host.id, 'instance.create', payload);
+        const second = api.queueHostOperationSerialized(api.host.id, 'instance.create', payload);
+        api.reportHostOperationStatus(api.host.id, second.id, { state: 'accepted', result: { phase: 'selected' } });
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const prepare = vi.spyOn(db(), 'prepare');
+        const payloadReads = () => prepare.mock.calls.filter(([sql]) => sql.startsWith('SELECT CAST(') && sql.includes(' AS BLOB) AS payload')).length;
+
+        const response = await api.GET({ request: api.request('?lane=write') });
+        const selection = prepare.mock.calls.find(([sql]) => sql.includes('FROM ec_host_operations o'))![0];
+        const projection = selection.split(' FROM ')[0];
+        expect(projection).not.toContain('payload');
+        expect(projection).not.toContain('*');
+        expect(payloadReads()).toBe(0);
+        // A receipt arriving during a streamed response must not mix a newer
+        // result with the state and timestamps chosen for this delivery.
+        api.reportHostOperationStatus(api.host.id, second.id, { state: 'running', result: { phase: 'newer' } });
+        const reader = response.body!.getReader();
+        const first = await reader.read();
+        expect(first.done).toBe(false);
+        expect(payloadReads()).toBe(0);
+        const chunks: Uint8Array[] = [first.value!];
+        let checkedNextPayload = false;
+        while (true) {
+            const next = await reader.read();
+            if (next.done) break;
+            if (payloadReads() === 2 && !checkedNextPayload) {
+                expect(Buffer.concat(chunks).includes(payload)).toBe(true);
+                checkedNextPayload = true;
+            }
+            chunks.push(next.value);
+        }
+        expect(payloadReads()).toBe(2);
+        expect(checkedNextPayload).toBe(true);
+        const body = JSON.parse(Buffer.concat(chunks).toString('utf8'));
+        expect(body.operations).toHaveLength(2);
+        expect(body.operations[1]).toMatchObject({ id: second.id, state: 'accepted', result: { phase: 'selected' } });
+        expect(body.operations[0].payload).toEqual(JSON.parse(payload.toString('utf8')));
+        expect(body.operations[1].payload).toEqual(body.operations[0].payload);
+        const other = api.createHost({ name: 'Other host' });
+        expect(() => api.getHostOperationPayload(other.id, second.id)).toThrow('not found for this host');
+    });
+
+    it('retains health lanes, delivery rotation, and cancellation suppression through the streamed route', async () => {
+        const api = await enrolled();
+        const operations = Array.from({ length: 27 }, (_, index) => api.queueHostOperation(api.host.id, 'instance.restart', { index }));
+        const inspect = api.queueHostOperation(api.host.id, 'instance.inspect', {});
+        const first = await (await api.GET({ request: api.request('?lane=write') })).json();
+        expect(first.operations.map((operation: { id: string }) => operation.id)).toEqual(operations.slice(0, 25).map(operation => operation.id));
+        const second = await (await api.GET({ request: api.request('?lane=write') })).json();
+        expect(second.operations.slice(0, 2).map((operation: { id: string }) => operation.id)).toEqual(operations.slice(25).map(operation => operation.id));
+        const health = await (await api.GET({ request: api.request('?lane=health') })).json();
+        expect(health.operations).toEqual([inspect]);
+
+        const create = api.queueHostOperation(api.host.id, 'instance.create', {});
+        const cancellation = api.queueHostOperation(api.host.id, 'instance.cancel-create', { createOperationId: create.id });
+        const cancelled = await (await api.GET({ request: api.request('?lane=write') })).json();
+        expect(cancelled.operations[0].id).toBe(cancellation.id);
+        expect(cancelled.operations.some((operation: { id: string }) => operation.id === create.id)).toBe(false);
+    });
+
+    it('rechecks credentials after waking a long poll before sending work', async () => {
+        const api = await enrolled();
+        const waiting = api.GET({ request: api.request('?lane=health&wait=20') });
+        api.revokeCredentials(api.host.id);
+        api.queueHostOperation(api.host.id, 'instance.inspect', {});
+        await expect(waiting).rejects.toMatchObject({ status: 401 });
+    });
+});

tests/unit/collection-match-performance.test.ts

@@ -125,10 +125,11 @@ describe('matching a native collection', () => {
 
         await new Promise<void>(resolve => setImmediate(resolve));
         await processProvisioning();
-        const dispatched = db().prepare(`SELECT n.deployment_id,o.payload FROM native_provisioning n
-            JOIN ec_host_operations o ON o.id=n.operation_id WHERE o.kind='instance.create'`).all() as { deployment_id: string; payload: string }[];
+        const dispatched = db().prepare(`SELECT n.deployment_id,o.id FROM native_provisioning n
+            JOIN ec_host_operations o ON o.id=n.operation_id WHERE o.kind='instance.create'`).all() as { deployment_id: string; id: string }[];
         expect(dispatched).toHaveLength(1);
-        const payload = JSON.parse(dispatched[0].payload) as { files: { path: string; contentBase64: string; sha256: string }[] };
+        const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        const payload = getHostOperation(dispatched[0].id)!.payload as { files: { path: string; contentBase64: string; sha256: string }[] };
         expect(payload.files.find(file => file.path.endsWith('/fixture.dll'))).toMatchObject({
             sha256: version.sha256, contentBase64: content.toString('base64')
         });

tests/unit/launch-recipe-summary-migration.test.ts

@@ -0,0 +1,46 @@
+import Database from 'better-sqlite3';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import schema from '../../src/lib/server/db/schema.sql?raw';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+import type { LaunchRecipe } from '../../src/lib/server/fleet/launch-recipes.js';
+
+let fixture: Fixture;
+beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
+afterEach(async () => { vi.restoreAllMocks(); await fixture.cleanup(); });
+
+describe('persisted launch recipe summaries', () => {
+    it('lazily upgrades legacy snapshots once without changing fingerprints or retained bytes', async () => {
+        const recipe: LaunchRecipe = {
+            format: 1, game: 't5', mapSlug: 'theater', mapName: 'Kino der Toten', expectedMap: 'zombie_theater',
+            capturedAt: '2025-01-01T00:00:00.000Z', image: 'plutainer:test', configFile: 'dedicated_sp.cfg', configSource: 'Defaults',
+            files: [{ name: 'Old plugin', kind: 'plugin', path: 'legacy.dll', versionId: 1, version: 1,
+                sha256: 'a'.repeat(64), size: 2 * 1024 ** 2, content: Buffer.alloc(2 * 1024 ** 2, 42).toString('base64') }]
+        };
+        const legacy = new Database(fixture.dbPath);
+        const encoded = JSON.stringify(recipe, null, 2);
+        try {
+            legacy.exec(schema.replace('    recipe_summary TEXT,\n', ''));
+            legacy.exec(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by)
+                VALUES('legacy','retired-server','create','done','Complete','2025-01-01','fixture');`);
+            legacy.prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,blueprint_name) VALUES(?,?,?)')
+                .run('legacy', encoded, 'Old collection');
+        } finally { legacy.close(); }
+        const { db, closeDb } = await import('../../src/lib/server/db/index.js');
+        const { recipeSummaryForDeployment, recipeForDeployment, summarizeRecipe } = await import('../../src/lib/server/fleet/launch-recipes.js');
+        const expected = { recipe: summarizeRecipe(recipe), blueprintName: 'Old collection' };
+        expect(db().prepare('SELECT recipe_summary FROM server_launch_recipes').get()).toEqual({ recipe_summary: null });
+        expect(recipeSummaryForDeployment('legacy')).toEqual(expected);
+        closeDb();
+        const parse = JSON.parse;
+        const parsing = vi.spyOn(JSON, 'parse').mockImplementation((value, reviver) => {
+            if (value.length > 1024 * 1024) throw new Error('Legacy recipe was decoded again.');
+            return parse(value, reviver);
+        });
+        try {
+            expect(recipeSummaryForDeployment('legacy')).toEqual(expected);
+            expect(recipeSummaryForDeployment('missing')).toBeNull();
+        } finally { parsing.mockRestore(); }
+        expect(recipeForDeployment('legacy')).toEqual({ recipe, blueprintName: 'Old collection' });
+        expect(db().prepare('SELECT recipe FROM server_launch_recipes').get()).toEqual({ recipe: encoded });
+    });
+});

tests/unit/launch-recipes.test.ts

@@ -46,6 +46,37 @@ async function seedArtifacts() {
 }
 
 describe('launch recipes and previews', () => {
+    it('keeps detail polling independent of retained plugin bytes after launching', async () => {
+        const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
+        const { db, closeDb } = await import('../../src/lib/server/db/index.js');
+        const plugin = createScript({ name: 'Large utility plugin', kind: 'plugin', scopeType: 'game', scopeValue: 't5',
+            targetRelPath: 'large.dll', content: Buffer.alloc(8 * 1024 ** 2, 42) });
+        db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(plugin.version.id, plugin.script.id);
+        const { queueCreateServer, processCreations } = await import('../../src/lib/server/fleet/create.js');
+        const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
+        const { recipeForDeployment, summarizeRecipe } = await import('../../src/lib/server/fleet/launch-recipes.js');
+        const queued = queueCreateServer(input);
+        if (!queued.ok) throw new Error(queued.message);
+        const queuedSummary = getDeployment(queued.deploymentId)!.recipe!;
+        await processCreations();
+        const full = recipeForDeployment(queued.deploymentId)!.recipe;
+        const expected = summarizeRecipe(full);
+        expect(expected.fingerprint).not.toBe(queuedSummary.fingerprint);
+        expect(full.files.find(file => file.path === 'large.dll')?.content).toBe(plugin.version.content.toString('base64'));
+        closeDb();
+        const parse = JSON.parse;
+        const parsing = vi.spyOn(JSON, 'parse').mockImplementation((value, reviver) => {
+            if (value.length > 1024 * 1024) throw new Error('Detail polling parsed plugin bytes.');
+            return parse(value, reviver);
+        });
+        try {
+            for (let read = 0; read < 3; read++) expect(getDeployment(queued.deploymentId)?.recipe).toEqual(expected);
+            const row = db().prepare('SELECT length(recipe_summary) AS bytes FROM server_launch_recipes WHERE deployment_id = ?')
+                .get(queued.deploymentId) as { bytes: number };
+            expect(row.bytes).toBeLessThan(10_000);
+        } finally { parsing.mockRestore(); }
+    });
+
     it('previews without reserving capacity, saving keys, exposing bytes, or calling Komodo', async () => {
         await seedArtifacts();
         const { previewCreation } = await import('../../src/lib/server/fleet/create.js');

tests/unit/native-hosts.test.ts

@@ -15,6 +15,68 @@ async function enrolled() {
 }
 
 describe('native host enrollment and scoped authentication', () => {
+    it('queues serialized launch files and processes receipts without hydrating plugin payloads', async () => {
+        const api = await enrolled();
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const payload = { files: [{ path: 'plugins/fixture.dll', contentBase64: Buffer.alloc(8 * 1024 ** 2, 42).toString('base64') }] };
+        const bytes = Buffer.from(JSON.stringify(payload));
+        const options = { instanceId: 'immutable-instance', generation: 1, idempotencyKey: 'large-launch' };
+        const parse = JSON.parse;
+        const parsing = vi.spyOn(JSON, 'parse').mockImplementation((value, reviver) => {
+            if (value.length > 1024 * 1024) throw new Error('Launch payload was unnecessarily parsed.');
+            return parse(value, reviver);
+        });
+        let id: string;
+        try {
+            const operation = api.queueHostOperationSerialized(api.host.id, 'instance.create', bytes, options);
+            id = operation.id;
+            expect(operation).not.toHaveProperty('payload');
+            expect(api.queueHostOperationSerialized(api.host.id, 'instance.create', bytes, options).id).toBe(id);
+            expect(() => api.queueHostOperationSerialized(api.host.id, 'instance.create', Buffer.from('{}'), options)).toThrow('different request');
+            const stored = db().prepare('SELECT typeof(payload) AS kind,count(*) AS count FROM ec_host_operations WHERE idempotency_key=?').get(options.idempotencyKey);
+            expect(stored).toEqual({ kind: 'text', count: 1 });
+            const waiting = api.waitHostOperationStatus(id);
+            expect(api.reportHostOperationStatus(api.host.id, id, { state: 'accepted' }).state).toBe('accepted');
+            expect(api.reportHostOperationStatus(api.host.id, id, { state: 'running' }).state).toBe('running');
+            api.reportHostOperationStatus(api.host.id, id, { state: 'completed', result: { healthy: true } });
+            expect(await waiting).toMatchObject({ id, state: 'completed', result: { healthy: true } });
+            expect(api.reportHostOperationStatus(api.host.id, id, { state: 'running' }).state).toBe('completed');
+            expect(api.getHostOperationStatus(id)).not.toHaveProperty('payload');
+            const other = api.createHost({ name: 'Other host' });
+            expect(() => api.reportHostOperationStatus(other.id, id, { state: 'completed' })).toThrow('this host');
+        } finally { parsing.mockRestore(); }
+        // Full-data APIs retain their original contract for explicit readers.
+        expect(api.getHostOperation(id!)?.payload).toEqual(payload);
+        expect(api.reportHostOperation(api.host.id, id!, { state: 'running' }).payload).toEqual(payload);
+        expect((await api.waitHostOperation(id!)).payload).toEqual(payload);
+    });
+
+    it('keeps serialized launch routing rows small while full-data APIs hydrate separate immutable bytes', async () => {
+        const api = await enrolled();
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const routing = { libraryId: 'test-library', name: 't5_zombie_theater_1', game: 't5' };
+        const payload = { ...routing, files: [{ contentBase64: 'A'.repeat(512 * 1024) }] };
+        const bytes = Buffer.from(JSON.stringify(payload));
+        const launch = api.queueHostOperationSerialized(api.host.id, 'instance.create', bytes, {}, routing);
+        const inline = api.queueHostOperation(api.host.id, 'instance.inspect', { name: 'inline-legacy' });
+        expect(api.getHostOperation(launch.id)?.payload).toEqual(payload);
+        expect(api.listHostOperations(api.host.id).find(operation => operation.id === launch.id)?.payload).toEqual(payload);
+        expect(api.pendingHostOperations(api.host.id, 'write').map(operation => operation.payload)).toEqual([payload]);
+        expect(api.pendingHostOperations(api.host.id, 'read')).toEqual([inline]);
+        expect(api.pendingHostOperationRows(api.host.id, 'write', false)[0]).not.toHaveProperty('payload');
+        api.reportHostOperationStatus(api.host.id, launch.id, { state: 'accepted' });
+        api.reportHostOperationStatus(api.host.id, launch.id, { state: 'running', result: { phase: 'starting' } });
+        const stored = db().prepare(`SELECT payload,last_delivered_at FROM ec_host_operations WHERE id=?`).get(launch.id) as
+            { payload: string; last_delivered_at: string | null };
+        expect(stored.payload).toBe(JSON.stringify(routing));
+        expect(stored.last_delivered_at).not.toBeNull();
+        expect(db().prepare('SELECT length(CAST(payload AS BLOB)) AS bytes FROM ec_host_operation_payloads WHERE operation_id=?').get(launch.id))
+            .toEqual({ bytes: bytes.length });
+        expect(api.getHostOperationPayload(api.host.id, launch.id)).toEqual(bytes);
+        expect(api.getHostOperationPayload(api.host.id, inline.id).toString('utf8')).toBe(JSON.stringify(inline.payload));
+        expect(db().prepare('SELECT 1 FROM ec_host_operation_payloads WHERE operation_id=?').get(inline.id)).toBeUndefined();
+    });
+
     it('narrows cancellation lookups by kind instead of scanning completed health history', async () => {
         const api = await enrolled();
         api.heartbeat(api.host.id, { ...inventory, capabilities: ['instance.cancel-create', 'operations.health-lane'] });
@@ -29,7 +91,7 @@ describe('native host enrollment and scoped authentication', () => {
         const inspect = api.queueHostOperation(api.host.id, 'instance.inspect', {});
         const prepare = vi.spyOn(handle, 'prepare');
         expect(api.pendingHostOperations(api.host.id, 'health').map(op => op.id)).toEqual([inspect.id]);
-        const query = prepare.mock.calls.map(([sql]) => sql).find(sql => sql.includes('SELECT * FROM ec_host_operations o'))!;
+        const query = prepare.mock.calls.map(([sql]) => sql).find(sql => sql.includes('FROM ec_host_operations o WHERE host_id=?'))!;
         prepare.mockRestore();
         // Assert the actual poll's access path, avoiding hardware-dependent timing thresholds.
         const plan = handle.prepare(`EXPLAIN QUERY PLAN ${query}`).all(api.host.id, 1, 1) as { detail: string }[];

tests/unit/native-launch-preparation.test.ts

@@ -0,0 +1,227 @@
+import { join } from 'node:path';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+
+vi.mock('../../src/lib/server/integrations/plutonium.js', () => ({
+    checkRelease: vi.fn(async () => {}), freshRelease: () => 5354, probeVersion: vi.fn(async () => 5354)
+}));
+vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) }));
+
+let fixture: Fixture;
+let hostId: string;
+let libraryId: string;
+const image = 'example.test/plutainer:test';
+const releases: (() => void)[] = [];
+const passes: Promise<void>[] = [];
+
+beforeEach(async () => {
+    vi.resetModules(); vi.clearAllMocks();
+    fixture = await newFixture([]);
+    vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries'));
+    vi.stubEnv('PLUTAINER_IMAGE', image); vi.stubEnv('RCON_PASSWORD', 'test-password');
+    vi.stubEnv('KOMODO_ADDRESS', ''); vi.stubEnv('KOMODO_API_KEY', ''); vi.stubEnv('KOMODO_API_SECRET', '');
+    const { createHost, createEnrollment, enrollHost, heartbeat } = await import('../../src/lib/server/hosts/index.js');
+    hostId = createHost({ name: 'Preparation test host', portMin: 28960, portMax: 28980 }).id;
+    const inventory = { cpuCount: 32, cpuUsagePercent: 0, memoryTotal: 64 * 1024 ** 3,
+        memoryAvailable: 60 * 1024 ** 3, protocolVersion: 1, dockerRunning: true,
+        directoriesWritable: true, storageAvailable: 1_000_000_000_000, runtimeImages: [image] };
+    enrollHost(createEnrollment(hostId).token, inventory);
+    const { createLibraryImport, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js');
+    const { db } = await import('../../src/lib/server/db/index.js');
+    libraryId = createLibraryImport({ hostId, game: 't5' }).id;
+    db().prepare("UPDATE game_file_versions SET state='ready',manifest='[]',manifest_sha256=?,size_bytes=1,published_at=? WHERE id=?")
+        .run('a'.repeat(64), new Date().toISOString(), libraryId);
+    updateHostLibraryCaches(hostId, [{ libraryVersionId: libraryId, status: 'ready', manifestSha256: 'a'.repeat(64) }]);
+    heartbeat(hostId, { ...inventory, libraries: [{ id: libraryId, state: 'ready' }] });
+});
+
+afterEach(async () => {
+    for (const release of releases.splice(0)) release();
+    await Promise.allSettled(passes.splice(0));
+    vi.restoreAllMocks();
+    await fixture.cleanup();
+    vi.unstubAllEnvs();
+});
+
+async function queue() {
+    const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
+    const result = queueNativeCreation({ game: 't5', gameMode: 'zombie', mapSlug: 'theater',
+        hostId, libraryId, image, plutoKey: 'saved-key' });
+    if (!result.ok) throw new Error(result.message);
+    return result;
+}
+
+async function runPass() {
+    const { processNativeProvisioning } = await import('../../src/lib/server/provisioning/native.js');
+    const pass = processNativeProvisioning(() => null);
+    passes.push(pass);
+    await pass;
+}
+
+async function operationFor(deploymentId: string) {
+    const { db } = await import('../../src/lib/server/db/index.js');
+    return db().prepare(`SELECT o.id FROM native_provisioning n JOIN ec_host_operations o ON o.id=n.operation_id
+        WHERE n.deployment_id=?`).get(deploymentId) as { id: string } | undefined;
+}
+
+/** Pause after real bytes are captured so mutations exercise the final admission checks. */
+async function pausePreparation() {
+    const preparation = await import('../../src/lib/server/provisioning/launch-preparation.js');
+    const prepare = preparation.prepareNativeLaunch;
+    let started!: () => void;
+    let release!: () => void;
+    const ready = new Promise<void>(resolve => { started = resolve; });
+    const gate = new Promise<void>(resolve => { release = resolve; });
+    releases.push(release);
+    const spy = vi.spyOn(preparation, 'prepareNativeLaunch').mockImplementationOnce(async input => {
+        const result = await prepare(input);
+        started();
+        await gate;
+        return result;
+    });
+    return { ready, release, spy };
+}
+
+describe('asynchronous native launch preparation', () => {
+    it('keeps controller timers and launch details responsive while preparing multiple large plugins', async () => {
+        const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
+        const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
+        saveFleetSettings({ ...fleetSettings(), createConcurrency: 4 });
+        const { script, version } = createScript({ name: 'Large launch plugin', kind: 'plugin', targetRelPath: 'fixture.dll',
+            scopeType: 'game', scopeValue: 't5', content: Buffer.alloc(8 * 1024 ** 2, 42) });
+        db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(version.id, script.id);
+        const jobs = await Promise.all(Array.from({ length: 4 }, queue));
+        const hosts = await import('../../src/lib/server/hosts/index.js');
+        const queueCommand = hosts.queueHostOperationSerialized;
+        let yielded = true;
+        let uninterruptedCommits = 0;
+        const committing = vi.spyOn(hosts, 'queueHostOperationSerialized').mockImplementation((...args) => {
+            if (!yielded) uninterruptedCommits++;
+            yielded = false;
+            setImmediate(() => { yielded = true; });
+            return queueCommand(...args);
+        });
+        const parse = JSON.parse;
+        const stringify = JSON.stringify;
+        const parsing = vi.spyOn(JSON, 'parse').mockImplementation((raw, reviver) => {
+            if (typeof raw === 'string' && raw.length > 1_000_000) throw new Error('Large file payload parsed on the controller thread');
+            return parse(raw, reviver);
+        });
+        const encoding = vi.spyOn(JSON, 'stringify').mockImplementation((...args: Parameters<typeof JSON.stringify>) => {
+            const encoded = stringify(...args);
+            if (encoded && encoded.length > 1_000_000) throw new Error('Large file payload serialized on the controller thread');
+            return encoded;
+        });
+        let ticks = 0;
+        const timer = setInterval(() => { ticks++; }, 1);
+        try {
+            const pass = runPass();
+            await new Promise<void>(resolve => setTimeout(resolve, 0));
+            for (const job of jobs) expect(getDeployment(job.deploymentId)?.recipe?.files)
+                .toContainEqual(expect.objectContaining({ path: 'fixture.dll', size: 8 * 1024 ** 2 }));
+            await pass;
+            expect(ticks).toBeGreaterThan(1);
+            expect(committing).toHaveBeenCalledTimes(4);
+            expect(uninterruptedCommits).toBe(0);
+            for (const job of jobs) {
+                expect(await operationFor(job.deploymentId)).toBeDefined();
+                expect(getDeployment(job.deploymentId)?.recipe?.files)
+                    .toContainEqual(expect.objectContaining({ path: 'fixture.dll', sha256: version.sha256 }));
+            }
+        } finally {
+            clearInterval(timer); parsing.mockRestore(); encoding.mockRestore(); committing.mockRestore();
+        }
+    }, 30_000);
+
+    it('does not resurrect an unsubmitted creation cancelled while its files are prepared', async () => {
+        const job = await queue();
+        const paused = await pausePreparation();
+        const pass = runPass();
+        await paused.ready;
+        const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
+        expect(cancelCreation(job.deploymentId)).toEqual(expect.any(String));
+        paused.release();
+        await pass;
+        const { db } = await import('../../src/lib/server/db/index.js');
+        expect(await operationFor(job.deploymentId)).toBeUndefined();
+        expect(db().prepare('SELECT name FROM servers WHERE name=?').get(job.name)).toBeUndefined();
+        expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
+        expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(job.deploymentId)).toEqual({ state: 'cancelled' });
+    });
+
+    it('discards prepared bytes when the promoted version changes before dispatch', async () => {
+        const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { script, version } = createScript({ name: 'Launch plugin', kind: 'plugin', targetRelPath: 'fixture.dll',
+            scopeType: 'game', scopeValue: 't5', content: Buffer.from('before promotion') });
+        db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(version.id, script.id);
+        const job = await queue();
+        const paused = await pausePreparation();
+        const pass = runPass();
+        await paused.ready;
+        const content = Buffer.from('newly promoted plugin');
+        const latest = uploadNewVersion(script.id, content);
+        db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(latest.id, script.id);
+        paused.release();
+        await pass;
+        expect(await operationFor(job.deploymentId)).toBeUndefined();
+        await runPass();
+        const operation = await operationFor(job.deploymentId);
+        expect(operation).toBeDefined();
+        const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
+        const payload = getHostOperation(operation!.id)!.payload as { files: { contentBase64: string; sha256: string }[] };
+        expect(payload.files).toContainEqual(expect.objectContaining({ contentBase64: content.toString('base64'), sha256: latest.sha256 }));
+    });
+
+    it('prepares only one launch while the single startup slot is reserved', async () => {
+        const first = await queue();
+        const second = await queue();
+        const paused = await pausePreparation();
+        const pass = runPass();
+        await paused.ready;
+        expect(paused.spy).toHaveBeenCalledTimes(1);
+        const { processNativeProvisioning } = await import('../../src/lib/server/provisioning/native.js');
+        await processNativeProvisioning(() => null, { wait: false });
+        expect(paused.spy).toHaveBeenCalledTimes(1);
+        paused.release();
+        await pass;
+        expect(await operationFor(first.deploymentId)).toBeDefined();
+        expect(await operationFor(second.deploymentId)).toBeUndefined();
+    });
+
+    it('keeps a prepared launch queued if its host is disabled before dispatch', async () => {
+        const job = await queue();
+        const paused = await pausePreparation();
+        const pass = runPass();
+        await paused.ready;
+        const { updateHost } = await import('../../src/lib/server/hosts/index.js');
+        updateHost(hostId, { mode: 'disabled' });
+        paused.release();
+        await pass;
+        expect(await operationFor(job.deploymentId)).toBeUndefined();
+        updateHost(hostId, { mode: 'active' });
+        await runPass();
+        expect(await operationFor(job.deploymentId)).toBeDefined();
+    });
+
+    it('rechecks a reduced startup limit before submitting prepared files', async () => {
+        const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
+        saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 });
+        const first = await queue();
+        await runPass();
+        expect(await operationFor(first.deploymentId)).toBeDefined();
+        const second = await queue();
+        const paused = await pausePreparation();
+        const pass = runPass();
+        await paused.ready;
+        saveFleetSettings({ ...fleetSettings(), createConcurrency: 1 });
+        paused.release();
+        await pass;
+        expect(await operationFor(second.deploymentId)).toBeUndefined();
+        saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 });
+        await runPass();
+        expect(await operationFor(second.deploymentId)).toBeDefined();
+    });
+});

tests/unit/native-provisioning.test.ts

@@ -246,7 +246,9 @@ describe('native instance provisioning', () => {
         try {
             const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
             expect(getDeployment(first.deploymentId)!.progress?.phase).toBe('verify-container');
-            expect((await operationFor(second.deploymentId))?.kind).toBe('instance.create');
+            await vi.waitFor(async () => {
+                expect((await operationFor(second.deploymentId))?.kind).toBe('instance.create');
+            }, { timeout: 5_000 });
         } finally {
             release({ healthy: true, state: 'running' } as never);
             await pending;
@@ -268,7 +270,9 @@ describe('native instance provisioning', () => {
         try {
             const second = await queue();
             await processProvisioning({ wait: false });
-            expect((await operationFor(second.deploymentId))?.kind).toBe('instance.create');
+            await vi.waitFor(async () => {
+                expect((await operationFor(second.deploymentId))?.kind).toBe('instance.create');
+            }, { timeout: 5_000 });
             expect(vi.mocked(nativeRequest)).toHaveBeenCalledTimes(1);
         } finally {
             release({ healthy: true, state: 'running' } as never);
@@ -401,6 +405,32 @@ describe('native instance provisioning', () => {
         await advance();
         expect(nativeInstance(job.name)?.generation).toBe(2);
     });
+    it('holds a rebuild when players join while its launch files are being prepared', async () => {
+        const job = await queue(); await advance(); await acknowledge(job.deploymentId);
+        const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
+        const rebuild = queueNativeRebuild(job.name, 'test'); if (!rebuild.ok) throw new Error(rebuild.message);
+        const preparation = await import('../../src/lib/server/provisioning/launch-preparation.js');
+        const prepare = preparation.prepareNativeLaunch;
+        let started!: () => void;
+        let release!: () => void;
+        const ready = new Promise<void>(resolve => { started = resolve; });
+        const gate = new Promise<void>(resolve => { release = resolve; });
+        const spy = vi.spyOn(preparation, 'prepareNativeLaunch').mockImplementationOnce(async input => {
+            const result = await prepare(input);
+            started(); await gate;
+            return result;
+        });
+        const pending = advance();
+        try {
+            await ready;
+            remote.players = 2;
+            release(); await pending;
+            expect(await operationFor(rebuild.deploymentId)).toBeNull();
+            remote.players = 0;
+            await advance();
+            expect((await operationFor(rebuild.deploymentId))?.kind).toBe('instance.rebuild');
+        } finally { release(); await pending; spy.mockRestore(); }
+    });
     it('finishes native rebuild verification without waiting for its own recovery slot to be released', async () => {
         const job = await queue(); await advance(); await acknowledge(job.deploymentId);
         const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');