josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Keep native collection launches responsive

Browse Source
This commit is contained in:
josh
2026-10-05 10:48:38 -04:00
parent dbd2940d29
commit 1405114a14
26 changed files with 998 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/provisioning/native.ts
+119 -66
View File
@@ -1,13 +1,15 @@
import { isIP } from 'node:net';
import { setImmediate as yieldToController } from 'node:timers/promises';
import { CapacityError, choosePlacement, availableHostPort, type PlacementOptions } from '../hosts/capacity.js';
import { randomBytes, randomUUID } from 'node:crypto';
import { posix } from 'node:path';
import { db } from '../db/index.js';
import { config, relDestForKind, serverDir } from '../config.js';
import { nativeInstance, nativeRequest, type NativeInstance } from '../backends/native.js';
import { getHost, getHostOperation, listHosts, queueHostOperation, releaseHostPorts, reserveHostPort } from '../hosts/index.js';
import { getHost, getHostOperationStatus, listHosts, queueHostOperation, queueHostOperationSerialized, releaseHostPorts, reserveHostPort } from '../hosts/index.js';
import { newestReadyLibraryId, assertReadyLibrary } from '../game-files/index.js';
import { blueprintRecipe, captureRecipe, captureRecipePlan, recipeForDeployment, summarizeRecipe, type LaunchRecipe, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
import { blueprintRecipe, captureRecipePlan, recipeForDeployment, recipeStorage, summarizeRecipe, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
import { prepareNativeLaunch, type PreparedNativeLaunch } from './launch-preparation.js';
import { getServerKey, saveServerKey } from '../fleet/server-keys.js';
import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
import { nextLobbyNumber, type CreateServerInput, type CreateServerResult } from './instances.js';
@@ -125,7 +127,9 @@ export function queueNativeCreation(input: CreateServerInput): CreateServerResul
db().prepare("UPDATE server_deployments SET state='queued' WHERE id=?").run(id);
db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,?,?,?,?)')
.run(id, name, port, 'zombie', recipe.mapSlug, recipe.expectedMap);
db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,blueprint_name) VALUES(?,?,?)').run(id, JSON.stringify(recipe), preview?.blueprint_name ?? null);
const stored = recipeStorage(recipe);
db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary,blueprint_name) VALUES(?,?,?,?)')
.run(id, stored.recipe, stored.recipeSummary, preview?.blueprint_name ?? null);
if (preview) db().prepare('UPDATE server_launch_previews SET deployment_id=?,port=? WHERE id=?').run(id, port, preview.id);
deploymentStage(id, 'Waiting for provisioning evaluation', { phase: 'eligibility', recipeStatus: 'queued' });
return { ok: true as const, name, port, deploymentId: id };
@@ -164,7 +168,8 @@ export function queueNativeRebuild(name: string, triggeredBy: string): CreateSer
db().prepare("UPDATE server_deployments SET state='waiting' WHERE id=?").run(id);
db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,?,?,?,?)')
.run(id, name, instance.port, 'zombie', recipe.mapSlug, recipe.expectedMap);
db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe) VALUES(?,?)').run(id, JSON.stringify(recipe));
const stored = recipeStorage(recipe);
db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary) VALUES(?,?,?)').run(id, stored.recipe, stored.recipeSummary);
deploymentStage(id, 'Waiting to rebuild when empty', { phase: 'eligibility', recipeStatus: 'queued' });
return { ok: true as const, name, port: instance.port, deploymentId: id };
})();
@@ -212,16 +217,6 @@ function applyReceipt(job: Job, result: Record<string, unknown>) {
})();
}
function createPayload(instance: NativeInstance, recipe: LaunchRecipe) {
const key = getServerKey(instance.server);
if (!key) throw new Error('The saved server key was removed.');
return { name: instance.server, game: recipe.game, libraryId: instance.library_id, image: instance.image, port: instance.port, bindAddress: instance.bind_address,
rconPassword: config.rcon.password || randomBytes(32).toString('hex'),
env: { PLUTAINER_SERVER_NAME: instance.server, PLUTAINER_PORT: String(instance.port), PLUTAINER_GAME: recipe.game === 't6' ? 't6zm' : 't5sp',
PLUTAINER_CONFIG_FILE: recipe.configFile, PLUTAINER_MAP_ROTATE: 'false', PLUTAINER_EXTRA_ARGS: `+exec electric_cherry_map.cfg +set tws_server_id ${instance.server} +map_rotate`, PLUTO_SERVER_KEY: key },
files: recipe.files.map(file => ({ path: posix.join(relDestForKind(file.kind, recipe.game), file.path), contentBase64: file.content, sha256: file.sha256 })) };
}
function hold(job: Job, message: string) { deploymentStage(job.id, message, { blocker: { code: 'native-host', message, automatic: true } }); }
function stillActive(job: Job): boolean {
return !!db().prepare(`SELECT 1 FROM server_deployments WHERE id=? AND server_deleted_at IS NULL
@@ -267,6 +262,24 @@ export async function processNativeProvisioning(eligibility: (operation: { id: s
}
const activeJobs = new Map<string, Promise<void>>();
// Preparation owns startup capacity until its command is committed or discarded.
const preparingJobs = new Set<string>();
function otherPreparations(id: string): number { return preparingJobs.size - Number(preparingJobs.has(id)); }
let launchCommit: Promise<void> = Promise.resolve();
function commitPreparedLaunch(commit: () => void): Promise<void> {
// Worker replies can arrive together. Persist one launch per event-loop turn
// so a batch of large SQLite writes cannot hold off requests and health timers.
const next = launchCommit.then(async () => { await yieldToController(); commit(); });
launchCommit = next.catch(() => {});
return next;
}
function sameLaunchFiles(first: LaunchRecipePlan, second: LaunchRecipePlan): boolean {
const selected = (recipe: LaunchRecipePlan) => ({ game: recipe.game, mapSlug: recipe.mapSlug,
expectedMap: recipe.expectedMap, configFile: recipe.configFile, configSource: recipe.configSource, files: recipe.files });
return JSON.stringify(selected(first)) === JSON.stringify(selected(second));
}
/** Reservations can move only while no remote work has ever been submitted. */
function placeQueuedCreation(job: Job, instance: NativeInstance): NativeInstance {
@@ -285,7 +298,8 @@ function placeQueuedCreation(job: Job, instance: NativeInstance): NativeInstance
.run(host.id, bind, port, network.publicAddress ?? host.publicAddress, network.publicPort ?? null, instance.id);
db().prepare('UPDATE server_creation_jobs SET port=? WHERE deployment_id=?').run(port, job.id);
recipe.native = { ...prior, hostId: host.id, bindAddress: bind, publicAddress: network.publicAddress ?? host.publicAddress, publicPort: network.publicPort };
db().prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?').run(JSON.stringify(recipe), job.id);
const stored = recipeStorage(recipe);
db().prepare('UPDATE server_launch_recipes SET recipe=?,recipe_summary=? WHERE deployment_id=?').run(stored.recipe, stored.recipeSummary, job.id);
db().prepare('UPDATE server_launch_previews SET port=? WHERE deployment_id=?').run(port, job.id);
deploymentStage(job.id, `Placed on ${host.name} after CPU, memory, history and port checks`, { phase: 'eligibility' });
return nativeInstance(job.server)!;
@@ -302,7 +316,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
const settings = fleetSettings();
const active = db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL
AND action IN ('create','rebuild') AND state IN ('running','awaiting-health')`).get(job.id) as { n: number };
if (active.n >= settings.createConcurrency) { hold(job, 'Waiting for startup capacity.'); return; }
if (active.n + otherPreparations(job.id) >= settings.createConcurrency) { hold(job, 'Waiting for startup capacity.'); return; }
instance = placeQueuedCreation(job, instance);
}
const host = getHost(instance.host_id);
@@ -311,7 +325,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
hold(job, 'Host is unavailable; existing server state is unknown. Waiting for reconnection.'); return;
}
if (job.operation_id) {
const operation = getHostOperation(job.operation_id);
const operation = getHostOperationStatus(job.operation_id);
if (operation?.kind === 'instance.cancel-create' && !['completed', 'failed'].includes(operation.state) && !host.inventory.capabilities?.includes('instance.cancel-create')) {
hold(job, hostAgentUpdatesEnabled(host.inventory)
? 'Waiting for the host agent update. New command delivery is paused so its current batch can finish and the automatic updater can run. Cleanup resumes after the updated agent connects.'
@@ -381,7 +395,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
if (!status.ok) { hold(job, `Waiting for game status: ${status.reason}`); return; }
if (job.action === 'create' || job.action === 'rebuild') {
deploymentStage(job.id, 'Checking selected engine map', { phase: 'verify-map', blocker: null });
const recipe = recipeForDeployment(job.id)!.recipe;
const recipe = db().prepare('SELECT expected_map AS expectedMap FROM server_creation_jobs WHERE deployment_id=?')
.get(job.id) as { expectedMap: string };
check({ key: 'map', label: 'Selected engine map verified', source: status.source,
result: status.map === recipe.expectedMap ? 'pass' : 'waiting', expected: recipe.expectedMap, actual: status.map }, 'verify-map');
if (status.map !== recipe.expectedMap) { hold(job, `Waiting for selected map ${recipe.expectedMap}; observed ${status.map ?? 'unknown'}.`); return; }
@@ -416,7 +431,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
hold(job, 'Staging and verifying files on the host.'); return;
}
if (job.stage_operation_id) {
const stage = getHostOperation(job.stage_operation_id);
const stage = getHostOperationStatus(job.stage_operation_id);
if (stage?.state === 'failed') { finishDeployment(job.id, { ok: false, message: stage.error ?? 'File staging failed.' }); return; }
if (stage?.state !== 'completed') { hold(job, 'Waiting for verified remote file staging.'); return; }
if (files.some(file => (db().prepare('SELECT state FROM staged_file_changes WHERE id=?').get(file.id) as { state: string } | undefined)?.state !== 'pending')) {
@@ -426,7 +441,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
const settings = fleetSettings();
const count = (actions: string[]) => (db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id != ? AND server_deleted_at IS NULL
AND action IN (${actions.map(() => '?').join(',')}) AND state IN ('running','awaiting-health')`).get(job.id, ...actions) as { n: number }).n;
if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) >= settings.createConcurrency) {
if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) + otherPreparations(job.id) >= settings.createConcurrency) {
deploymentStage(job.id, 'Waiting for startup capacity.', { phase: 'startup-queue',
blocker: { code: 'startup-capacity', message: 'Waiting for startup capacity.', automatic: true } }); return;
}
@@ -435,65 +450,103 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
if (job.action === 'create') choosePlacement({ hostId: instance.host_id, libraryId: instance.library_id, image: instance.image,
bindAddress: instance.bind_address, excludeInstanceId: instance.id, thorough: true, dispatch: true });
if (job.action === 'create' || job.action === 'rebuild') assertReadyLibrary(instance.host_id, instance.library_id);
if (job.action !== 'create' && job.action !== 'start') {
const checkOccupancy = async (): Promise<boolean> => {
// Fast controller check keeps occupied lobbies out of the operation slots.
// The agent repeats this immediately before its local mutation.
deploymentStage(job.id, 'Checking container and player occupancy before submission', { phase: 'eligibility', blocker: null });
const health = await nativeRequest<{ state: string; healthy?: boolean }>(instance, 'instance.inspect');
if (!stillActive(job)) return;
if (!stillActive(job)) return false;
if (!['stopped','exited','created','dead','missing'].includes(health.state)) {
const occupancy = await probePlayerEvidence(job.server);
if (!stillActive(job)) return;
if (!stillActive(job)) return false;
const recoveredWatchdog = health.healthy && occupancy.players !== null && db().prepare(`SELECT 1 FROM provisioning_sources s
JOIN watchdog_incidents i ON i.id=s.source_id JOIN server_deployments d ON d.id=s.operation_id
WHERE s.operation_id=? AND s.kind='watchdog' AND i.kind='health' AND d.triggered_by LIKE 'watchdog:%'`).get(job.id);
if (recoveredWatchdog && occupancy.players !== null) { resolveRecoveredHealthOperation(job.id, occupancy.players, occupancy.source); return; }
if (occupancy.players !== 0) { hold(job, occupancy.players !== null ? `Waiting for ${occupancy.players} player(s) to leave.` : 'Waiting for a fresh local player count.'); return; }
if (recoveredWatchdog && occupancy.players !== null) { resolveRecoveredHealthOperation(job.id, occupancy.players, occupancy.source); return false; }
if (occupancy.players !== 0) { hold(job, occupancy.players !== null ? `Waiting for ${occupancy.players} player(s) to leave.` : 'Waiting for a fresh local player count.'); return false; }
}
}
return true;
};
if (job.action !== 'create' && job.action !== 'start' && !await checkOccupancy()) return;
const current = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined;
if (!current || !['queued','waiting','interrupted'].includes(current.state) || eligibility(job)) return;
db().transaction(() => {
const live = nativeInstance(job.server);
if (live?.id !== instance!.id || live.generation !== instance!.generation) throw new Error('Instance changed during preflight.');
if (job.action === 'destroy' || job.action === 'rebuild') {
const beta = activeBetaRemovalBlocker(job.server);
if (beta) { hold(job, beta); return; }
}
if (JSON.stringify(filesFor(job)) !== JSON.stringify(files)) {
finishDeployment(job.id, { ok: false, message: 'File requests changed during preflight. Review and submit a fresh activation.' }); return;
}
// Async occupancy checks can overlap another job claiming the final slot.
if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) >= settings.createConcurrency) {
deploymentStage(job.id, 'Waiting for startup capacity.', { phase: 'startup-queue', blocker: { code: 'startup-capacity', message: 'Waiting for startup capacity.', automatic: true } }); return;
}
if (job.action === 'destroy' && count(['destroy']) >= settings.destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; }
if (job.action === 'start' || job.action === 'restart' || job.action === 'rebuild') {
db().prepare(`INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES(?,?,?) ON CONFLICT(server)
DO UPDATE SET requested_at=excluded.requested_at,operation_id=excluded.operation_id,healthy_at=NULL,error=NULL`).run(job.server, new Date().toISOString(), job.id);
}
recordProgress(job.id, { blocker: null, checkpoint: { key: 'eligibility', label: 'Host eligibility and safety checks passed' } });
let payload: Record<string, unknown> = files.length ? { stageId: job.id } : {};
let prepared: PreparedNativeLaunch | undefined;
let preparedKey: string | null = null;
try {
if (job.action === 'create' || job.action === 'rebuild') {
deploymentStage(job.id, 'Preparing launch files for the host agent', { phase: 'prepare', recipeStatus: 'attempted',
checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } });
if (count(['create','rebuild']) + otherPreparations(job.id) >= fleetSettings().createConcurrency) {
hold(job, 'Waiting for startup capacity.'); return;
}
preparingJobs.add(job.id);
preparedKey = getServerKey(job.server) ?? null;
if (!preparedKey) throw new Error('The saved server key was removed.');
const prior = recipeForDeployment(job.id)!.recipe as NativeRecipe;
const recipe = { ...captureRecipe(prior.game, prior.mapSlug), native: prior.native, image: instance!.image };
payload = createPayload(instance!, recipe);
if (job.action === 'rebuild') payload.nextGeneration = instance!.generation + 1;
db().prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?').run(JSON.stringify(recipe), job.id);
if (job.action === 'create') db().prepare(`INSERT INTO servers(name,komodo_type,komodo_name,game,map,game_mode,map_slug,host_dir,config_file,game_id,image,game_port,created_by,container_state)
VALUES(?,'deployment','',?,?,'zombie',?,?,?,?,?,?,'native','starting')`).run(job.server, recipe.game, recipe.expectedMap, recipe.mapSlug, serverDir(job.server),
recipe.configFile, recipe.game === 't6' ? 't6zm' : 't5sp', recipe.image, instance!.port);
for (const file of recipe.files) recordScript(job.server, file.path, file.versionId, file.sha256, file.size, 'native-create');
recordProgress(job.id, { checkpoint: { key: 'prepare', label: 'Launch files selected and command payload prepared' } });
deploymentStage(job.id, 'Preparing launch files for the host agent', { phase: 'prepare',
checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } });
prepared = await prepareNativeLaunch({ game: prior.game, mapSlug: prior.mapSlug, native: prior.native,
instance, serverKey: preparedKey, rconPassword: config.rcon.password || randomBytes(32).toString('hex'),
rebuild: job.action === 'rebuild' });
if (job.action === 'rebuild' && !await checkOccupancy()) return;
}
const operation = queueHostOperation(instance!.host_id, `instance.${job.action === 'destroy' ? 'remove' : job.action}`, payload,
{ instanceId: instance!.id, generation: instance!.generation, idempotencyKey: job.id });
db().prepare('UPDATE native_provisioning SET operation_id=? WHERE deployment_id=?').run(operation.id, job.id);
for (const file of files) db().prepare("UPDATE staged_file_changes SET state='applying' WHERE id=? AND state='pending'").run(file.id);
db().prepare("UPDATE server_deployments SET state='running',submitted_at=? WHERE id=?").run(new Date().toISOString(), job.id);
deploymentStage(job.id, `Waiting for host agent to acknowledge ${job.action}`, { phase: 'container-action', blocker: null,
command: { state: 'attempted', message: `Durable agent operation ${operation.id}; reconciliation always uses this ID.` } });
})();
const commit = db().transaction(() => {
const state = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined;
if (!state || !['queued','waiting','interrupted'].includes(state.state) || eligibility(job)) return;
const live = nativeInstance(job.server);
if (live?.id !== instance!.id || live.generation !== instance!.generation) throw new Error('Instance changed during preflight.');
const liveHost = getHost(instance!.host_id);
if (!liveHost?.connected || liveHost.mode === 'disabled') { hold(job, 'Host is unavailable; waiting for reconnection.'); return; }
if (job.action === 'destroy' || job.action === 'rebuild') {
const beta = activeBetaRemovalBlocker(job.server);
if (beta) { hold(job, beta); return; }
}
if (JSON.stringify(filesFor(job)) !== JSON.stringify(files)) {
finishDeployment(job.id, { ok: false, message: 'File requests changed during preflight. Review and submit a fresh activation.' }); return;
}
// Async occupancy checks can overlap another job claiming the final slot.
if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) + otherPreparations(job.id) >= fleetSettings().createConcurrency) {
deploymentStage(job.id, 'Waiting for startup capacity.', { phase: 'startup-queue', blocker: { code: 'startup-capacity', message: 'Waiting for startup capacity.', automatic: true } }); return;
}
if (job.action === 'destroy' && count(['destroy']) >= fleetSettings().destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; }
if (prepared) {
if (getServerKey(job.server) !== preparedKey || !sameLaunchFiles(prepared.recipe,
captureRecipePlan(prepared.recipe.game, prepared.recipe.mapSlug))) {
hold(job, 'Launch files or credentials changed during preparation; preparing the current selection next pass.'); return;
}
assertReadyLibrary(instance!.host_id, instance!.library_id);
if (job.action === 'create') choosePlacement({ hostId: instance!.host_id, libraryId: instance!.library_id, image: instance!.image,
bindAddress: instance!.bind_address, excludeInstanceId: instance!.id, thorough: true, dispatch: true });
}
if (job.action === 'start' || job.action === 'restart' || job.action === 'rebuild') {
db().prepare(`INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES(?,?,?) ON CONFLICT(server)
DO UPDATE SET requested_at=excluded.requested_at,operation_id=excluded.operation_id,healthy_at=NULL,error=NULL`).run(job.server, new Date().toISOString(), job.id);
}
recordProgress(job.id, { blocker: null, checkpoint: { key: 'eligibility', label: 'Host eligibility and safety checks passed' } });
const payload: Record<string, unknown> = files.length ? { stageId: job.id } : {};
if (prepared) {
deploymentStage(job.id, 'Preparing launch files for the host agent', { phase: 'prepare', recipeStatus: 'attempted',
checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } });
const recipe = prepared.recipe;
db().prepare('UPDATE server_launch_recipes SET recipe=CAST(? AS TEXT),recipe_summary=? WHERE deployment_id=?')
.run(Buffer.from(prepared.recipeJson.buffer, prepared.recipeJson.byteOffset, prepared.recipeJson.byteLength), prepared.recipeSummary, job.id);
if (job.action === 'create') db().prepare(`INSERT INTO servers(name,komodo_type,komodo_name,game,map,game_mode,map_slug,host_dir,config_file,game_id,image,game_port,created_by,container_state)
VALUES(?,'deployment','',?,?,'zombie',?,?,?,?,?,?,'native','starting')`).run(job.server, recipe.game, recipe.expectedMap, recipe.mapSlug, serverDir(job.server),
recipe.configFile, recipe.game === 't6' ? 't6zm' : 't5sp', recipe.image, instance!.port);
for (const file of recipe.files) recordScript(job.server, file.path, file.versionId, file.sha256, file.size, 'native-create');
recordProgress(job.id, { checkpoint: { key: 'prepare', label: 'Launch files selected and command payload prepared' } });
}
const operationOptions = { instanceId: instance!.id, generation: instance!.generation, idempotencyKey: job.id };
const kind = `instance.${job.action === 'destroy' ? 'remove' : job.action}`;
const operation = prepared ? queueHostOperationSerialized(instance!.host_id, kind,
Buffer.from(prepared.payloadJson.buffer, prepared.payloadJson.byteOffset, prepared.payloadJson.byteLength), operationOptions,
{ name: instance!.server, game: prepared.recipe.game, libraryId: instance!.library_id })
: queueHostOperation(instance!.host_id, kind, payload, operationOptions);
db().prepare('UPDATE native_provisioning SET operation_id=? WHERE deployment_id=?').run(operation.id, job.id);
for (const file of files) db().prepare("UPDATE staged_file_changes SET state='applying' WHERE id=? AND state='pending'").run(file.id);
db().prepare("UPDATE server_deployments SET state='running',submitted_at=? WHERE id=?").run(new Date().toISOString(), job.id);
deploymentStage(job.id, `Waiting for host agent to acknowledge ${job.action}`, { phase: 'container-action', blocker: null,
command: { state: 'attempted', message: `Durable agent operation ${operation.id}; reconciliation always uses this ID.` } });
});
if (prepared) await commitPreparedLaunch(commit);
else commit();
} finally { preparingJobs.delete(job.id); }
}