Keep native collection launches responsive

src/lib/server/provisioning/history.ts

@@ -1,6 +1,6 @@
 import { randomUUID } from 'node:crypto';
 import { db } from '../db/index.js';
-import { recipeForDeployment, summarizeRecipe } from '../fleet/launch-recipes.js';
+import { recipeSummaryForDeployment } from '../fleet/launch-recipes.js';
 import { actionLabels, type ProvisioningAction } from '../../provisioning.js';
 import type { ProvisioningCapacity, ProvisioningDeployment, ProvisioningDetail, ProvisioningSource } from '../../provisioning-detail.js';
 import { fleetSettings } from '../settings.js';
@@ -111,7 +111,7 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
         const { observation_json: _snapshot, phase_key: _phase, phase_started_at: _phaseStarted, observed_at: _observed, ...deployment } = row;
         const sources = (db().prepare('SELECT kind, source_id, server FROM provisioning_sources WHERE operation_id = ? ORDER BY kind, source_id')
             .all(id) as Omit<ProvisioningSource, 'label' | 'href'>[]).map(sourceDetails);
-        const launch = recipeForDeployment(id);
+        const launch = recipeSummaryForDeployment(id);
         const collection = db().prepare(`SELECT c.id, c.name FROM (
             SELECT collection_id, deployment_id FROM server_collection_members
             UNION ALL SELECT collection_id, deployment_id FROM server_collection_removals) m
@@ -120,7 +120,7 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
             canCancelCreation: canCancelCreation(id), cancellation: cancellation ?? null,
             workerActivity: workerActivity(), events: deploymentEvents(id, options.events === 'recent'),
             activity: { ...activitySummary(id), failureExcerpt: latestFailureEvent(id) }, sources,
-            recipe: launch ? summarizeRecipe(launch.recipe) : null, blueprintName: launch?.blueprintName ?? null,
+            recipe: launch?.recipe ?? null, blueprintName: launch?.blueprintName ?? null,
             canResumeVerification: canResumeVerification(id), canResumeCleanup: canResumeCleanup(id),
             canResumeRebuild: canResumeRebuild(id), canCancelRemoval: canCancelRemoval(id), collection: collection ?? null,
             progress: readProgress(row), capacity: capacityFor(deployment, generatedAt),

src/lib/server/provisioning/instances.ts

@@ -4,7 +4,7 @@ import { probePlayerEvidence, probeServerStatus } from '../fleet/status-evidence
 import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent, REMOVING_SERVER, resumeCleanup, resumeVerification, claimRemoval } from './history.js';
 import { randomUUID } from 'node:crypto';
 import type { LaunchPreview } from '$lib/deployment-plan.js';
-import { blueprintRecipe, captureRecipe, captureRecipePlan, recipeForDeployment, summarizeRecipe, type LaunchRecipe, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
+import { blueprintRecipe, captureRecipe, captureRecipePlan, recipeForDeployment, recipeStorage, summarizeRecipe, type LaunchRecipe, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
 import { mkdir, chmod, rm, writeFile } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
 import { config, destOfKind, serverDir, gameOfServer, configFileForGame } from '../config.js';
@@ -240,9 +240,10 @@ export function queueCreateServer(input: CreateServerInput, releasingServers: st
         if (!prepared.ok) return prepared;
         // A preview records choices, not version pins. Promotions remain authoritative.
         const currentRecipe = captureRecipePlan(input.game ?? 't5', prepared.mapEntry.slug);
+        const storedRecipe = recipeStorage(currentRecipe);
         const id = beginDeployment(prepared.name, 'create', input.createdBy);
-        db().prepare('INSERT INTO server_launch_recipes (deployment_id, recipe, blueprint_name) VALUES (?, ?, ?)')
-            .run(id, JSON.stringify(currentRecipe), preview?.blueprint_name ?? null);
+        db().prepare('INSERT INTO server_launch_recipes (deployment_id, recipe, recipe_summary, blueprint_name) VALUES (?, ?, ?, ?)')
+            .run(id, storedRecipe.recipe, storedRecipe.recipeSummary, preview?.blueprint_name ?? null);
         if (preview) db().prepare('UPDATE server_launch_previews SET deployment_id = ? WHERE id = ?').run(id, preview.id);
         db().prepare("UPDATE server_deployments SET state = 'queued' WHERE id = ?").run(id);
         db().prepare('INSERT INTO server_creation_jobs (deployment_id, server, port, game_mode, map_slug, expected_map) VALUES (?, ?, ?, ?, ?, ?)')
@@ -275,7 +276,9 @@ export function queueRebuildServer(name: string, triggeredBy = 'ui'): CreateServ
         db().prepare('INSERT INTO server_creation_jobs (deployment_id, server, port, game_mode, map_slug, expected_map) VALUES (?, ?, ?, ?, ?, ?)')
             .run(id, name, server.game_port, 'zombie', map.slug, map.canonical);
         db().prepare('INSERT INTO server_rebuild_jobs (deployment_id) VALUES (?)').run(id);
-        db().prepare('INSERT INTO server_launch_recipes (deployment_id, recipe) VALUES (?, ?)').run(id, JSON.stringify(recipe));
+        const storedRecipe = recipeStorage(recipe);
+        db().prepare('INSERT INTO server_launch_recipes (deployment_id, recipe, recipe_summary) VALUES (?, ?, ?)')
+            .run(id, storedRecipe.recipe, storedRecipe.recipeSummary);
         deploymentStage(id, 'Waiting to rebuild the empty server', { phase: 'eligibility', recipeStatus: 'queued', observedAt: null,
             blocker: { code: 'eligibility', message: 'Waiting for safe rebuild eligibility checks.', automatic: true } });
         return { ok: true as const, name, port: server.game_port!, deploymentId: id };
@@ -399,10 +402,11 @@ async function createServerUnlocked(input: CreateServerInput, prepared: Prepared
     // Resolve once more at provisioning so promotions made while queued also apply.
     // Persist the exact attempted files for history, never as input to future launches.
     const recipe = captureRecipe(game, mapEntry.slug);
+    const storedRecipe = recipeStorage(recipe);
     db().transaction(() => {
-        db().prepare(`INSERT INTO server_launch_recipes (deployment_id, recipe) VALUES (?, ?)
-            ON CONFLICT(deployment_id) DO UPDATE SET recipe = excluded.recipe`)
-            .run(operation, JSON.stringify(recipe));
+        db().prepare(`INSERT INTO server_launch_recipes (deployment_id, recipe, recipe_summary) VALUES (?, ?, ?)
+            ON CONFLICT(deployment_id) DO UPDATE SET recipe = excluded.recipe, recipe_summary = excluded.recipe_summary`)
+            .run(operation, storedRecipe.recipe, storedRecipe.recipeSummary);
         deploymentStage(operation, 'Preparing server files', { phase: 'prepare', recipeStatus: 'attempted', blocker: null, checks: [],
             checkpoint: { key: 'startup-queue', label: 'Admitted to startup' },
             event: { kind: 'recipe', message: 'Promoted files selected for this creation attempt.', details: { capturedAt: recipe.capturedAt } } });

src/lib/server/provisioning/launch-preparation.ts

@@ -0,0 +1,71 @@
+import { Worker } from 'node:worker_threads';
+import { resolve } from 'node:path';
+import { config } from '../config.js';
+import { db } from '../db/index.js';
+import type { NativeInstance } from '../backends/native.js';
+import type { LaunchRecipePlan } from '../fleet/launch-recipes.js';
+
+export interface NativeLaunchInput {
+    game: 't5' | 't6';
+    mapSlug: string;
+    native?: LaunchRecipePlan['native'];
+    instance: NativeInstance;
+    serverKey: string;
+    rconPassword: string;
+    rebuild: boolean;
+}
+
+export interface PreparedNativeLaunch {
+    recipe: LaunchRecipePlan;
+    recipeSummary: string;
+    recipeJson: Uint8Array;
+    payloadJson: Uint8Array;
+}
+
+let worker: Worker | null = null;
+let sequence = 0;
+const pending = new Map<number, {
+    resolve: (value: PreparedNativeLaunch) => void;
+    reject: (error: Error) => void;
+    timer: NodeJS.Timeout;
+}>();
+
+function stop(error: Error, current = worker): Promise<number> | undefined {
+    if (current !== worker) return;
+    worker = null;
+    for (const request of pending.values()) { clearTimeout(request.timer); request.reject(error); }
+    pending.clear();
+    return current?.terminate();
+}
+
+/** The reader and JSON encoder run off the controller's request thread. */
+export function prepareNativeLaunch(input: NativeLaunchInput): Promise<PreparedNativeLaunch> {
+    db(); // Migrations must finish before the independent read-only connection opens.
+    if (!worker) {
+        const current = worker = new Worker(resolve(process.env.NODE_ENV === 'production'
+            ? 'build/launch/worker.mjs' : '.svelte-kit/launch/worker.mjs'), {
+            workerData: { dbPath: resolve(config.dbPath) }
+        });
+        current.on('message', ({ id, value, error }: { id: number; value: PreparedNativeLaunch; error?: string }) => {
+            if (current !== worker) return;
+            const request = pending.get(id);
+            if (!request) return;
+            pending.delete(id); clearTimeout(request.timer);
+            if (error) request.reject(new Error(error)); else request.resolve(value);
+            if (!pending.size) current.unref();
+        });
+        current.on('error', error => { void stop(error, current); });
+        current.on('exit', () => { void stop(new Error('Launch preparation worker stopped; preparation will retry.'), current); });
+    }
+    worker.ref();
+    const id = ++sequence;
+    return new Promise((resolve, reject) => {
+        const timer = setTimeout(() => { void stop(new Error('Launch preparation timed out; preparation will retry.')); }, 60_000);
+        pending.set(id, { resolve, reject, timer });
+        worker!.postMessage({ id, input });
+    });
+}
+
+export async function closeLaunchPreparation(): Promise<void> {
+    await stop(new Error('Launch preparation stopped.'));
+}

src/lib/server/provisioning/launch-worker.ts

@@ -0,0 +1,38 @@
+import { parentPort } from 'node:worker_threads';
+import { posix } from 'node:path';
+import { db } from '../db/index.js';
+import { relDestForKind } from '../config.js';
+import { captureRecipe, summarizeRecipe } from '../fleet/launch-recipes.js';
+import { sha256Buffer } from '../fleet/files.js';
+import type { NativeLaunchInput } from './launch-preparation.js';
+
+parentPort!.on('message', ({ id, input }: { id: number; input: NativeLaunchInput }) => {
+    try {
+        // All file versions and composed configuration belong to one snapshot.
+        const recipe = db().transaction(() => ({ ...captureRecipe(input.game, input.mapSlug),
+            native: input.native, image: input.instance.image }))();
+        const instance = input.instance;
+        const payload = {
+            name: instance.server, game: recipe.game, libraryId: instance.library_id, image: instance.image,
+            port: instance.port, bindAddress: instance.bind_address, rconPassword: input.rconPassword,
+            env: { PLUTAINER_SERVER_NAME: instance.server, PLUTAINER_PORT: String(instance.port),
+                PLUTAINER_GAME: recipe.game === 't6' ? 't6zm' : 't5sp', PLUTAINER_CONFIG_FILE: recipe.configFile,
+                PLUTAINER_MAP_ROTATE: 'false',
+                PLUTAINER_EXTRA_ARGS: `+exec electric_cherry_map.cfg +set tws_server_id ${instance.server} +map_rotate`,
+                PLUTO_SERVER_KEY: input.serverKey },
+            files: recipe.files.map(file => ({ path: posix.join(relDestForKind(file.kind, recipe.game), file.path),
+                contentBase64: file.content, sha256: file.sha256 })),
+            ...(input.rebuild ? { nextGeneration: instance.generation + 1 } : {})
+        };
+        const encode = (value: unknown) => new TextEncoder().encode(JSON.stringify(value));
+        const recipeJson = encode(recipe);
+        const payloadJson = encode(payload);
+        const plan = { ...recipe, files: recipe.files.map(({ content: _content, ...file }) => file) };
+        const recipeSummary = JSON.stringify(summarizeRecipe(plan, sha256Buffer(Buffer.from(recipeJson.buffer))));
+        // Transfer ownership of the encoded bytes; do not clone giant strings or objects.
+        parentPort!.postMessage({ id, value: { recipe: plan, recipeSummary, recipeJson, payloadJson } },
+            [recipeJson.buffer, payloadJson.buffer]);
+    } catch (error) {
+        parentPort!.postMessage({ id, error: error instanceof Error ? error.message : 'Launch preparation failed.' });
+    }
+});

src/lib/server/provisioning/native.ts

@@ -1,13 +1,15 @@
 import { isIP } from 'node:net';
+import { setImmediate as yieldToController } from 'node:timers/promises';
 import { CapacityError, choosePlacement, availableHostPort, type PlacementOptions } from '../hosts/capacity.js';
 import { randomBytes, randomUUID } from 'node:crypto';
 import { posix } from 'node:path';
 import { db } from '../db/index.js';
 import { config, relDestForKind, serverDir } from '../config.js';
 import { nativeInstance, nativeRequest, type NativeInstance } from '../backends/native.js';
-import { getHost, getHostOperation, listHosts, queueHostOperation, releaseHostPorts, reserveHostPort } from '../hosts/index.js';
+import { getHost, getHostOperationStatus, listHosts, queueHostOperation, queueHostOperationSerialized, releaseHostPorts, reserveHostPort } from '../hosts/index.js';
 import { newestReadyLibraryId, assertReadyLibrary } from '../game-files/index.js';
-import { blueprintRecipe, captureRecipe, captureRecipePlan, recipeForDeployment, summarizeRecipe, type LaunchRecipe, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
+import { blueprintRecipe, captureRecipePlan, recipeForDeployment, recipeStorage, summarizeRecipe, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
+import { prepareNativeLaunch, type PreparedNativeLaunch } from './launch-preparation.js';
 import { getServerKey, saveServerKey } from '../fleet/server-keys.js';
 import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
 import { nextLobbyNumber, type CreateServerInput, type CreateServerResult } from './instances.js';
@@ -125,7 +127,9 @@ export function queueNativeCreation(input: CreateServerInput): CreateServerResul
             db().prepare("UPDATE server_deployments SET state='queued' WHERE id=?").run(id);
             db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,?,?,?,?)')
                 .run(id, name, port, 'zombie', recipe.mapSlug, recipe.expectedMap);
-            db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,blueprint_name) VALUES(?,?,?)').run(id, JSON.stringify(recipe), preview?.blueprint_name ?? null);
+            const stored = recipeStorage(recipe);
+            db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary,blueprint_name) VALUES(?,?,?,?)')
+                .run(id, stored.recipe, stored.recipeSummary, preview?.blueprint_name ?? null);
             if (preview) db().prepare('UPDATE server_launch_previews SET deployment_id=?,port=? WHERE id=?').run(id, port, preview.id);
             deploymentStage(id, 'Waiting for provisioning evaluation', { phase: 'eligibility', recipeStatus: 'queued' });
             return { ok: true as const, name, port, deploymentId: id };
@@ -164,7 +168,8 @@ export function queueNativeRebuild(name: string, triggeredBy: string): CreateSer
             db().prepare("UPDATE server_deployments SET state='waiting' WHERE id=?").run(id);
             db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,?,?,?,?)')
                 .run(id, name, instance.port, 'zombie', recipe.mapSlug, recipe.expectedMap);
-            db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe) VALUES(?,?)').run(id, JSON.stringify(recipe));
+            const stored = recipeStorage(recipe);
+            db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary) VALUES(?,?,?)').run(id, stored.recipe, stored.recipeSummary);
             deploymentStage(id, 'Waiting to rebuild when empty', { phase: 'eligibility', recipeStatus: 'queued' });
             return { ok: true as const, name, port: instance.port, deploymentId: id };
         })();
@@ -212,16 +217,6 @@ function applyReceipt(job: Job, result: Record<string, unknown>) {
     })();
 }
 
-function createPayload(instance: NativeInstance, recipe: LaunchRecipe) {
-    const key = getServerKey(instance.server);
-    if (!key) throw new Error('The saved server key was removed.');
-    return { name: instance.server, game: recipe.game, libraryId: instance.library_id, image: instance.image, port: instance.port, bindAddress: instance.bind_address,
-        rconPassword: config.rcon.password || randomBytes(32).toString('hex'),
-        env: { PLUTAINER_SERVER_NAME: instance.server, PLUTAINER_PORT: String(instance.port), PLUTAINER_GAME: recipe.game === 't6' ? 't6zm' : 't5sp',
-            PLUTAINER_CONFIG_FILE: recipe.configFile, PLUTAINER_MAP_ROTATE: 'false', PLUTAINER_EXTRA_ARGS: `+exec electric_cherry_map.cfg +set tws_server_id ${instance.server} +map_rotate`, PLUTO_SERVER_KEY: key },
-        files: recipe.files.map(file => ({ path: posix.join(relDestForKind(file.kind, recipe.game), file.path), contentBase64: file.content, sha256: file.sha256 })) };
-}
-
 function hold(job: Job, message: string) { deploymentStage(job.id, message, { blocker: { code: 'native-host', message, automatic: true } }); }
 function stillActive(job: Job): boolean {
     return !!db().prepare(`SELECT 1 FROM server_deployments WHERE id=? AND server_deleted_at IS NULL
@@ -267,6 +262,24 @@ export async function processNativeProvisioning(eligibility: (operation: { id: s
 }
 
 const activeJobs = new Map<string, Promise<void>>();
+// Preparation owns startup capacity until its command is committed or discarded.
+const preparingJobs = new Set<string>();
+function otherPreparations(id: string): number { return preparingJobs.size - Number(preparingJobs.has(id)); }
+
+let launchCommit: Promise<void> = Promise.resolve();
+function commitPreparedLaunch(commit: () => void): Promise<void> {
+    // Worker replies can arrive together. Persist one launch per event-loop turn
+    // so a batch of large SQLite writes cannot hold off requests and health timers.
+    const next = launchCommit.then(async () => { await yieldToController(); commit(); });
+    launchCommit = next.catch(() => {});
+    return next;
+}
+
+function sameLaunchFiles(first: LaunchRecipePlan, second: LaunchRecipePlan): boolean {
+    const selected = (recipe: LaunchRecipePlan) => ({ game: recipe.game, mapSlug: recipe.mapSlug,
+        expectedMap: recipe.expectedMap, configFile: recipe.configFile, configSource: recipe.configSource, files: recipe.files });
+    return JSON.stringify(selected(first)) === JSON.stringify(selected(second));
+}
 
 /** Reservations can move only while no remote work has ever been submitted. */
 function placeQueuedCreation(job: Job, instance: NativeInstance): NativeInstance {
@@ -285,7 +298,8 @@ function placeQueuedCreation(job: Job, instance: NativeInstance): NativeInstance
             .run(host.id, bind, port, network.publicAddress ?? host.publicAddress, network.publicPort ?? null, instance.id);
         db().prepare('UPDATE server_creation_jobs SET port=? WHERE deployment_id=?').run(port, job.id);
         recipe.native = { ...prior, hostId: host.id, bindAddress: bind, publicAddress: network.publicAddress ?? host.publicAddress, publicPort: network.publicPort };
-        db().prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?').run(JSON.stringify(recipe), job.id);
+        const stored = recipeStorage(recipe);
+        db().prepare('UPDATE server_launch_recipes SET recipe=?,recipe_summary=? WHERE deployment_id=?').run(stored.recipe, stored.recipeSummary, job.id);
         db().prepare('UPDATE server_launch_previews SET port=? WHERE deployment_id=?').run(port, job.id);
         deploymentStage(job.id, `Placed on ${host.name} after CPU, memory, history and port checks`, { phase: 'eligibility' });
         return nativeInstance(job.server)!;
@@ -302,7 +316,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
         const settings = fleetSettings();
         const active = db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL
             AND action IN ('create','rebuild') AND state IN ('running','awaiting-health')`).get(job.id) as { n: number };
-        if (active.n >= settings.createConcurrency) { hold(job, 'Waiting for startup capacity.'); return; }
+        if (active.n + otherPreparations(job.id) >= settings.createConcurrency) { hold(job, 'Waiting for startup capacity.'); return; }
         instance = placeQueuedCreation(job, instance);
     }
     const host = getHost(instance.host_id);
@@ -311,7 +325,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
         hold(job, 'Host is unavailable; existing server state is unknown. Waiting for reconnection.'); return;
     }
     if (job.operation_id) {
-        const operation = getHostOperation(job.operation_id);
+        const operation = getHostOperationStatus(job.operation_id);
         if (operation?.kind === 'instance.cancel-create' && !['completed', 'failed'].includes(operation.state) && !host.inventory.capabilities?.includes('instance.cancel-create')) {
             hold(job, hostAgentUpdatesEnabled(host.inventory)
                 ? 'Waiting for the host agent update. New command delivery is paused so its current batch can finish and the automatic updater can run. Cleanup resumes after the updated agent connects.'
@@ -381,7 +395,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
         if (!status.ok) { hold(job, `Waiting for game status: ${status.reason}`); return; }
         if (job.action === 'create' || job.action === 'rebuild') {
             deploymentStage(job.id, 'Checking selected engine map', { phase: 'verify-map', blocker: null });
-            const recipe = recipeForDeployment(job.id)!.recipe;
+            const recipe = db().prepare('SELECT expected_map AS expectedMap FROM server_creation_jobs WHERE deployment_id=?')
+                .get(job.id) as { expectedMap: string };
             check({ key: 'map', label: 'Selected engine map verified', source: status.source,
                 result: status.map === recipe.expectedMap ? 'pass' : 'waiting', expected: recipe.expectedMap, actual: status.map }, 'verify-map');
             if (status.map !== recipe.expectedMap) { hold(job, `Waiting for selected map ${recipe.expectedMap}; observed ${status.map ?? 'unknown'}.`); return; }
@@ -416,7 +431,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
         hold(job, 'Staging and verifying files on the host.'); return;
     }
     if (job.stage_operation_id) {
-        const stage = getHostOperation(job.stage_operation_id);
+        const stage = getHostOperationStatus(job.stage_operation_id);
         if (stage?.state === 'failed') { finishDeployment(job.id, { ok: false, message: stage.error ?? 'File staging failed.' }); return; }
         if (stage?.state !== 'completed') { hold(job, 'Waiting for verified remote file staging.'); return; }
         if (files.some(file => (db().prepare('SELECT state FROM staged_file_changes WHERE id=?').get(file.id) as { state: string } | undefined)?.state !== 'pending')) {
@@ -426,7 +441,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
     const settings = fleetSettings();
     const count = (actions: string[]) => (db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id != ? AND server_deleted_at IS NULL
         AND action IN (${actions.map(() => '?').join(',')}) AND state IN ('running','awaiting-health')`).get(job.id, ...actions) as { n: number }).n;
-    if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) >= settings.createConcurrency) {
+    if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) + otherPreparations(job.id) >= settings.createConcurrency) {
         deploymentStage(job.id, 'Waiting for startup capacity.', { phase: 'startup-queue',
             blocker: { code: 'startup-capacity', message: 'Waiting for startup capacity.', automatic: true } }); return;
     }
@@ -435,65 +450,103 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
     if (job.action === 'create') choosePlacement({ hostId: instance.host_id, libraryId: instance.library_id, image: instance.image,
         bindAddress: instance.bind_address, excludeInstanceId: instance.id, thorough: true, dispatch: true });
     if (job.action === 'create' || job.action === 'rebuild') assertReadyLibrary(instance.host_id, instance.library_id);
-    if (job.action !== 'create' && job.action !== 'start') {
+    const checkOccupancy = async (): Promise<boolean> => {
         // Fast controller check keeps occupied lobbies out of the operation slots.
         // The agent repeats this immediately before its local mutation.
         deploymentStage(job.id, 'Checking container and player occupancy before submission', { phase: 'eligibility', blocker: null });
         const health = await nativeRequest<{ state: string; healthy?: boolean }>(instance, 'instance.inspect');
-        if (!stillActive(job)) return;
+        if (!stillActive(job)) return false;
         if (!['stopped','exited','created','dead','missing'].includes(health.state)) {
             const occupancy = await probePlayerEvidence(job.server);
-            if (!stillActive(job)) return;
+            if (!stillActive(job)) return false;
             const recoveredWatchdog = health.healthy && occupancy.players !== null && db().prepare(`SELECT 1 FROM provisioning_sources s
                 JOIN watchdog_incidents i ON i.id=s.source_id JOIN server_deployments d ON d.id=s.operation_id
                 WHERE s.operation_id=? AND s.kind='watchdog' AND i.kind='health' AND d.triggered_by LIKE 'watchdog:%'`).get(job.id);
-            if (recoveredWatchdog && occupancy.players !== null) { resolveRecoveredHealthOperation(job.id, occupancy.players, occupancy.source); return; }
-            if (occupancy.players !== 0) { hold(job, occupancy.players !== null ? `Waiting for ${occupancy.players} player(s) to leave.` : 'Waiting for a fresh local player count.'); return; }
+            if (recoveredWatchdog && occupancy.players !== null) { resolveRecoveredHealthOperation(job.id, occupancy.players, occupancy.source); return false; }
+            if (occupancy.players !== 0) { hold(job, occupancy.players !== null ? `Waiting for ${occupancy.players} player(s) to leave.` : 'Waiting for a fresh local player count.'); return false; }
         }
-    }
+        return true;
+    };
+    if (job.action !== 'create' && job.action !== 'start' && !await checkOccupancy()) return;
     const current = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined;
     if (!current || !['queued','waiting','interrupted'].includes(current.state) || eligibility(job)) return;
-    db().transaction(() => {
-        const live = nativeInstance(job.server);
-        if (live?.id !== instance!.id || live.generation !== instance!.generation) throw new Error('Instance changed during preflight.');
-        if (job.action === 'destroy' || job.action === 'rebuild') {
-            const beta = activeBetaRemovalBlocker(job.server);
-            if (beta) { hold(job, beta); return; }
-        }
-        if (JSON.stringify(filesFor(job)) !== JSON.stringify(files)) {
-            finishDeployment(job.id, { ok: false, message: 'File requests changed during preflight. Review and submit a fresh activation.' }); return;
-        }
-        // Async occupancy checks can overlap another job claiming the final slot.
-        if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) >= settings.createConcurrency) {
-            deploymentStage(job.id, 'Waiting for startup capacity.', { phase: 'startup-queue', blocker: { code: 'startup-capacity', message: 'Waiting for startup capacity.', automatic: true } }); return;
-        }
-        if (job.action === 'destroy' && count(['destroy']) >= settings.destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; }
-        if (job.action === 'start' || job.action === 'restart' || job.action === 'rebuild') {
-            db().prepare(`INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES(?,?,?) ON CONFLICT(server)
-                DO UPDATE SET requested_at=excluded.requested_at,operation_id=excluded.operation_id,healthy_at=NULL,error=NULL`).run(job.server, new Date().toISOString(), job.id);
-        }
-        recordProgress(job.id, { blocker: null, checkpoint: { key: 'eligibility', label: 'Host eligibility and safety checks passed' } });
-        let payload: Record<string, unknown> = files.length ? { stageId: job.id } : {};
+    let prepared: PreparedNativeLaunch | undefined;
+    let preparedKey: string | null = null;
+    try {
         if (job.action === 'create' || job.action === 'rebuild') {
-            deploymentStage(job.id, 'Preparing launch files for the host agent', { phase: 'prepare', recipeStatus: 'attempted',
-                checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } });
+            if (count(['create','rebuild']) + otherPreparations(job.id) >= fleetSettings().createConcurrency) {
+                hold(job, 'Waiting for startup capacity.'); return;
+            }
+            preparingJobs.add(job.id);
+            preparedKey = getServerKey(job.server) ?? null;
+            if (!preparedKey) throw new Error('The saved server key was removed.');
             const prior = recipeForDeployment(job.id)!.recipe as NativeRecipe;
-            const recipe = { ...captureRecipe(prior.game, prior.mapSlug), native: prior.native, image: instance!.image };
-            payload = createPayload(instance!, recipe);
-            if (job.action === 'rebuild') payload.nextGeneration = instance!.generation + 1;
-            db().prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?').run(JSON.stringify(recipe), job.id);
-            if (job.action === 'create') db().prepare(`INSERT INTO servers(name,komodo_type,komodo_name,game,map,game_mode,map_slug,host_dir,config_file,game_id,image,game_port,created_by,container_state)
-                VALUES(?,'deployment','',?,?,'zombie',?,?,?,?,?,?,'native','starting')`).run(job.server, recipe.game, recipe.expectedMap, recipe.mapSlug, serverDir(job.server),
-                    recipe.configFile, recipe.game === 't6' ? 't6zm' : 't5sp', recipe.image, instance!.port);
-            for (const file of recipe.files) recordScript(job.server, file.path, file.versionId, file.sha256, file.size, 'native-create');
-            recordProgress(job.id, { checkpoint: { key: 'prepare', label: 'Launch files selected and command payload prepared' } });
+            deploymentStage(job.id, 'Preparing launch files for the host agent', { phase: 'prepare',
+                checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } });
+            prepared = await prepareNativeLaunch({ game: prior.game, mapSlug: prior.mapSlug, native: prior.native,
+                instance, serverKey: preparedKey, rconPassword: config.rcon.password || randomBytes(32).toString('hex'),
+                rebuild: job.action === 'rebuild' });
+            if (job.action === 'rebuild' && !await checkOccupancy()) return;
         }
-        const operation = queueHostOperation(instance!.host_id, `instance.${job.action === 'destroy' ? 'remove' : job.action}`, payload,
-            { instanceId: instance!.id, generation: instance!.generation, idempotencyKey: job.id });
-        db().prepare('UPDATE native_provisioning SET operation_id=? WHERE deployment_id=?').run(operation.id, job.id);
-        for (const file of files) db().prepare("UPDATE staged_file_changes SET state='applying' WHERE id=? AND state='pending'").run(file.id);
-        db().prepare("UPDATE server_deployments SET state='running',submitted_at=? WHERE id=?").run(new Date().toISOString(), job.id);
-        deploymentStage(job.id, `Waiting for host agent to acknowledge ${job.action}`, { phase: 'container-action', blocker: null,
-            command: { state: 'attempted', message: `Durable agent operation ${operation.id}; reconciliation always uses this ID.` } });
-    })();
+        const commit = db().transaction(() => {
+            const state = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined;
+            if (!state || !['queued','waiting','interrupted'].includes(state.state) || eligibility(job)) return;
+            const live = nativeInstance(job.server);
+            if (live?.id !== instance!.id || live.generation !== instance!.generation) throw new Error('Instance changed during preflight.');
+            const liveHost = getHost(instance!.host_id);
+            if (!liveHost?.connected || liveHost.mode === 'disabled') { hold(job, 'Host is unavailable; waiting for reconnection.'); return; }
+            if (job.action === 'destroy' || job.action === 'rebuild') {
+                const beta = activeBetaRemovalBlocker(job.server);
+                if (beta) { hold(job, beta); return; }
+            }
+            if (JSON.stringify(filesFor(job)) !== JSON.stringify(files)) {
+                finishDeployment(job.id, { ok: false, message: 'File requests changed during preflight. Review and submit a fresh activation.' }); return;
+            }
+            // Async occupancy checks can overlap another job claiming the final slot.
+            if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) + otherPreparations(job.id) >= fleetSettings().createConcurrency) {
+                deploymentStage(job.id, 'Waiting for startup capacity.', { phase: 'startup-queue', blocker: { code: 'startup-capacity', message: 'Waiting for startup capacity.', automatic: true } }); return;
+            }
+            if (job.action === 'destroy' && count(['destroy']) >= fleetSettings().destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; }
+            if (prepared) {
+                if (getServerKey(job.server) !== preparedKey || !sameLaunchFiles(prepared.recipe,
+                    captureRecipePlan(prepared.recipe.game, prepared.recipe.mapSlug))) {
+                    hold(job, 'Launch files or credentials changed during preparation; preparing the current selection next pass.'); return;
+                }
+                assertReadyLibrary(instance!.host_id, instance!.library_id);
+                if (job.action === 'create') choosePlacement({ hostId: instance!.host_id, libraryId: instance!.library_id, image: instance!.image,
+                    bindAddress: instance!.bind_address, excludeInstanceId: instance!.id, thorough: true, dispatch: true });
+            }
+            if (job.action === 'start' || job.action === 'restart' || job.action === 'rebuild') {
+                db().prepare(`INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES(?,?,?) ON CONFLICT(server)
+                    DO UPDATE SET requested_at=excluded.requested_at,operation_id=excluded.operation_id,healthy_at=NULL,error=NULL`).run(job.server, new Date().toISOString(), job.id);
+            }
+            recordProgress(job.id, { blocker: null, checkpoint: { key: 'eligibility', label: 'Host eligibility and safety checks passed' } });
+            const payload: Record<string, unknown> = files.length ? { stageId: job.id } : {};
+            if (prepared) {
+                deploymentStage(job.id, 'Preparing launch files for the host agent', { phase: 'prepare', recipeStatus: 'attempted',
+                    checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } });
+                const recipe = prepared.recipe;
+                db().prepare('UPDATE server_launch_recipes SET recipe=CAST(? AS TEXT),recipe_summary=? WHERE deployment_id=?')
+                    .run(Buffer.from(prepared.recipeJson.buffer, prepared.recipeJson.byteOffset, prepared.recipeJson.byteLength), prepared.recipeSummary, job.id);
+                if (job.action === 'create') db().prepare(`INSERT INTO servers(name,komodo_type,komodo_name,game,map,game_mode,map_slug,host_dir,config_file,game_id,image,game_port,created_by,container_state)
+                    VALUES(?,'deployment','',?,?,'zombie',?,?,?,?,?,?,'native','starting')`).run(job.server, recipe.game, recipe.expectedMap, recipe.mapSlug, serverDir(job.server),
+                        recipe.configFile, recipe.game === 't6' ? 't6zm' : 't5sp', recipe.image, instance!.port);
+                for (const file of recipe.files) recordScript(job.server, file.path, file.versionId, file.sha256, file.size, 'native-create');
+                recordProgress(job.id, { checkpoint: { key: 'prepare', label: 'Launch files selected and command payload prepared' } });
+            }
+            const operationOptions = { instanceId: instance!.id, generation: instance!.generation, idempotencyKey: job.id };
+            const kind = `instance.${job.action === 'destroy' ? 'remove' : job.action}`;
+            const operation = prepared ? queueHostOperationSerialized(instance!.host_id, kind,
+                Buffer.from(prepared.payloadJson.buffer, prepared.payloadJson.byteOffset, prepared.payloadJson.byteLength), operationOptions,
+                { name: instance!.server, game: prepared.recipe.game, libraryId: instance!.library_id })
+                : queueHostOperation(instance!.host_id, kind, payload, operationOptions);
+            db().prepare('UPDATE native_provisioning SET operation_id=? WHERE deployment_id=?').run(operation.id, job.id);
+            for (const file of files) db().prepare("UPDATE staged_file_changes SET state='applying' WHERE id=? AND state='pending'").run(file.id);
+            db().prepare("UPDATE server_deployments SET state='running',submitted_at=? WHERE id=?").run(new Date().toISOString(), job.id);
+            deploymentStage(job.id, `Waiting for host agent to acknowledge ${job.action}`, { phase: 'container-action', blocker: null,
+                command: { state: 'attempted', message: `Durable agent operation ${operation.id}; reconciliation always uses this ID.` } });
+        });
+        if (prepared) await commitPreparedLaunch(commit);
+        else commit();
+    } finally { preparingJobs.delete(job.id); }
 }