import { isIP } from 'node:net'; import { CapacityError, choosePlacement, availableHostPort, type PlacementOptions } from '../hosts/capacity.js'; import { randomBytes, randomUUID } from 'node:crypto'; import { posix } from 'node:path'; import { db } from '../db/index.js'; import { config, relDestForKind, serverDir } from '../config.js'; import { nativeInstance, nativeRequest, type NativeInstance } from '../backends/native.js'; import { getHost, getHostOperation, listHosts, queueHostOperation, releaseHostPorts, reserveHostPort } from '../hosts/index.js'; import { newestReadyLibrary, requireReadyLibrary } from '../game-files/index.js'; import { blueprintRecipe, captureRecipe, recipeForDeployment, summarizeRecipe, type LaunchRecipe } from '../fleet/launch-recipes.js'; import { getServerKey, saveServerKey } from '../fleet/server-keys.js'; import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js'; import { nextLobbyNumber, type CreateServerInput, type CreateServerResult } from './instances.js'; import { beginDeployment, deploymentStage, finishDeployment, REMOVING_SERVER } from './history.js'; import { recordProgress, type CheckInput } from './progress.js'; import { trackActivity } from '../pipeline/activity.js'; import { fleetSettings } from '../settings.js'; import { resolveRecoveredHealthOperation } from '../maintenance/recovery.js'; import { probeServerStatus, probePlayerEvidence } from '../fleet/status-evidence.js'; import { checkRelease, freshRelease, probeVersion } from '../integrations/plutonium.js'; import { getServerScript, recordScript, removeScript } from '../fleet/inventory.js'; import { sha256Buffer } from '../fleet/files.js'; import type { StagedFileChange } from '../fleet/staged-files.js'; import type { LaunchPreview } from '../../deployment-plan.js'; import type { ProvisioningAction } from '../../provisioning.js'; import { hostAgentUpdatesEnabled } from '../../hosts.js'; import { cancelCreation, finishCreationCancellation } from './cancel-creation.js'; export type NativeSelection = import('../../hosts.js').NativePlacement; type NativeRecipe = LaunchRecipe & { native?: NativeSelection }; interface Job { id: string; server: string; action: ProvisioningAction; state: string; submitted_at: string | null; stage: string; instance_id: string; generation: number; stage_operation_id: string | null; operation_id: string | null; files_json: string | null; applied_at: string | null } interface FilePlan { id: number; path: string; contentBase64: string | null; sha256: string | null; checkBefore: boolean; expectedBeforeSha: string | null } export function nativeSelection(input: Record, game: string, options: Pick = {}): NativeSelection { if (input.hostId !== undefined && typeof input.hostId !== 'string') throw new Error('Invalid host.'); const version = newestReadyLibrary(game); if (!version) throw new Error(`Upload ${game.toUpperCase()} game files before launching a server.`); if (input.libraryId !== undefined && input.libraryId !== '' && input.libraryId !== version.id) { throw new Error(`The current ${game.toUpperCase()} game files changed. Refresh the launch preview.`); } const image = input.image === undefined || input.image === '' ? config.komodo.defaultImage : input.image; if (typeof image !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9._/:@-]{0,499}$/.test(image)) throw new Error('Invalid runtime image.'); const automatic = input.automatic === true || !input.hostId; const rawNetwork = input.automatic === true ? input.network ?? {} : input; if (!rawNetwork || typeof rawNetwork !== 'object' || Array.isArray(rawNetwork)) throw new Error('Invalid network overrides.'); const network = rawNetwork as NativeSelection; const bindAddress = network.bindAddress || undefined; if (bindAddress !== undefined && (typeof bindAddress !== 'string' || !isIP(bindAddress))) throw new Error('Invalid bind address.'); const { host } = choosePlacement({ hostId: automatic ? undefined : input.hostId as string, libraryId: version.id, image, bindAddress, ...options }); if (network.publicAddress !== undefined && (typeof network.publicAddress !== 'string' || network.publicAddress.length > 253)) throw new Error('Invalid public address.'); if (network.publicPort !== undefined && (!Number.isInteger(network.publicPort) || Number(network.publicPort) < 1 || Number(network.publicPort) > 65535)) throw new Error('Invalid public port.'); return { hostId: host.id, libraryId: version.id, image, bindAddress: bindAddress ?? host.bindAddress, publicAddress: network.publicAddress ?? host.publicAddress, publicPort: network.publicPort, automatic, network: { bindAddress, publicAddress: network.publicAddress, publicPort: network.publicPort } }; } export function previewNativeCreation(input: Record): LaunchPreview { let recipe: NativeRecipe; let blueprintName: string | null = null; if (typeof input.deploymentId === 'string') { const previous = recipeForDeployment(input.deploymentId); if (!previous) throw new Error('The recorded deployment is unavailable.'); recipe = captureRecipe(previous.recipe.game, previous.recipe.mapSlug); blueprintName = previous.blueprintName; } else if (typeof input.blueprintId === 'string') ({ recipe, blueprintName } = blueprintRecipe(input.blueprintId)); else { if ((input.game !== 't5' && input.game !== 't6') || typeof input.mapSlug !== 'string') throw new Error('Choose a supported game and map.'); recipe = captureRecipe(input.game, input.mapSlug); } const selection = nativeSelection(input, recipe.game); recipe.native = selection; recipe.image = selection.image!; const base = `${recipe.game}_zombie_${recipe.mapSlug}`; const lobby = nextLobbyNumber(base); const name = lobby === null ? null : `${base}_${lobby}`; const port = nextNativePort(selection); const id = randomUUID(); const expiresAt = new Date(Date.now() + 30 * 60_000).toISOString(); db().prepare('INSERT INTO server_launch_previews(id,expires_at,server,port,recipe,blueprint_name) VALUES(?,?,?,?,?,?)') .run(id, expiresAt, name, port, JSON.stringify(recipe), blueprintName); const checks: LaunchPreview['checks'] = [ { label: 'Host capacity', state: 'pass', detail: selection.automatic ? 'Capacity available. The least loaded eligible host is selected when provisioning starts.' : `${getHost(selection.hostId)!.name}: capacity available.` }, { label: 'Game files', state: 'pass', detail: `The latest ${recipe.game.toUpperCase()} game files are verified on an eligible host.` }, { label: 'Lobby', state: name ? 'pass' : 'blocked', detail: name ?? 'All lobby IDs for this map are reserved.' }, { label: 'Host port', state: port ? 'pass' : 'blocked', detail: port ? selection.automatic ? 'UDP/TCP capacity available; port assigned at placement.' : `${selection.bindAddress}:${port} / UDP and TCP` : 'No available UDP/TCP port on this host.' }, { label: 'Configuration', state: 'pass', detail: recipe.configSource } ]; return { id, expiresAt, name, port, hasSavedKey: !!name && !!getServerKey(name), recipe: summarizeRecipe(recipe), blueprintName, checks, ready: checks.every(check => check.state !== 'blocked') }; } export function nextNativePort(selection: NativeSelection, taken: number[] = []): number | null { const host = getHost(selection.hostId); return host ? availableHostPort(host, { bindAddress: selection.bindAddress, planned: taken.map(port => ({ hostId: host.id, port })) }) : null; } export function queueNativeCreation(input: CreateServerInput): CreateServerResult { try { return db().transaction(() => { const preview = input.previewId ? db().prepare('SELECT * FROM server_launch_previews WHERE id=?').get(input.previewId) as { id: string; recipe: string; server: string; port: number; expires_at: string; deployment_id: string | null; blueprint_name: string | null } | undefined : undefined; if (input.previewId && !preview) throw new Error('Review a fresh launch preview.'); if (preview?.deployment_id) return { ok: true as const, name: preview.server, port: preview.port, deploymentId: preview.deployment_id }; if (preview && preview.expires_at <= new Date().toISOString()) throw new Error('Launch preview expired.'); const saved = preview ? JSON.parse(preview.recipe) as NativeRecipe : undefined; const game = input.game ?? 't5'; if (input.gameMode !== 'zombie' || (saved && (saved.game !== game || saved.mapSlug !== input.mapSlug))) throw new Error('Selection changed. Review a fresh launch preview.'); const selection = nativeSelection((saved?.native ?? input) as unknown as Record, game); const recipe: NativeRecipe = { ...captureRecipe(game, input.mapSlug), native: selection, image: selection.image! }; const lobby = nextLobbyNumber(`${game}_zombie_${input.mapSlug}`); if (lobby === null) throw new Error('All lobby IDs are reserved.'); const name = `${game}_zombie_${input.mapSlug}_${lobby}`; const port = nextNativePort(selection); if (!port) throw new Error('No available host port.'); if ((input.expectedName && input.expectedName !== name) || (preview && (preview.server !== name || (!selection.automatic && preview.port !== port)))) throw new Error('Available lobby or host port changed. Review a fresh preview.'); const key = input.plutoKey?.trim() || getServerKey(name); if (!key) throw new Error(`Save a Plutonium server key for ${name}.`); saveServerKey(name, key); const instanceId = randomUUID(); reserveHostPort(selection.hostId, instanceId, { port, bindAddress: selection.bindAddress, protocol: 'udp' }); reserveHostPort(selection.hostId, instanceId, { port, bindAddress: selection.bindAddress, protocol: 'tcp' }); db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,bind_address,port,public_address,public_port,created_at) VALUES(?,?,?,?,?,?,?,?,?,?)`).run(instanceId, name, selection.hostId, selection.libraryId, selection.image, selection.bindAddress, port, selection.publicAddress ?? null, selection.publicPort ?? null, new Date().toISOString()); const id = beginDeployment(name, 'create', input.createdBy); db().prepare("UPDATE server_deployments SET state='queued' WHERE id=?").run(id); db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,?,?,?,?)') .run(id, name, port, 'zombie', recipe.mapSlug, recipe.expectedMap); db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,blueprint_name) VALUES(?,?,?)').run(id, JSON.stringify(recipe), preview?.blueprint_name ?? null); if (preview) db().prepare('UPDATE server_launch_previews SET deployment_id=?,port=? WHERE id=?').run(id, port, preview.id); deploymentStage(id, 'Waiting for provisioning evaluation', { phase: 'eligibility', recipeStatus: 'queued' }); return { ok: true as const, name, port, deploymentId: id }; })(); } catch (error) { return { ok: false, kind: error instanceof CapacityError ? 'conflict' : 'bad-request', message: (error as Error).message }; } } export function bindNativeOperation(id: string, server: string): void { const instance = nativeInstance(server); if (instance) db().prepare('INSERT OR IGNORE INTO native_provisioning(deployment_id,instance_id,generation) VALUES(?,?,?)').run(id, instance.id, instance.generation); } /** Cancelling an unsubmitted launch releases only its local reservation. */ export function cancelQueuedNativeCreation(server: string, triggeredBy: string): string | null { const instance = nativeInstance(server); if (!instance || getServer(server)) return null; return db().transaction(() => { const creation = db().prepare(`SELECT d.id FROM server_deployments d JOIN native_provisioning n ON n.deployment_id=d.id WHERE n.instance_id=? AND d.action='create' AND d.state='queued' AND n.operation_id IS NULL`).get(instance.id) as { id: string } | undefined; if (!creation) return null; return cancelCreation(creation.id, triggeredBy); })(); } export function queueNativeRebuild(name: string, triggeredBy: string): CreateServerResult { try { const instance = nativeInstance(name)!; const server = getServer(name); const blocker = activeBetaRemovalBlocker(name); if (blocker) throw new Error(blocker); if (!server?.map_slug || !getServerKey(name)) throw new Error('A saved map and server key are required before rebuilding.'); requireReadyLibrary(instance.host_id, instance.library_id); const recipe: NativeRecipe = { ...captureRecipe(server.game as 't5' | 't6', server.map_slug), image: instance.image, native: { hostId: instance.host_id, libraryId: instance.library_id, image: instance.image, bindAddress: instance.bind_address } }; return db().transaction(() => { const id = beginDeployment(name, 'rebuild', triggeredBy); db().prepare("UPDATE server_deployments SET state='waiting' WHERE id=?").run(id); db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,?,?,?,?)') .run(id, name, instance.port, 'zombie', recipe.mapSlug, recipe.expectedMap); db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe) VALUES(?,?)').run(id, JSON.stringify(recipe)); deploymentStage(id, 'Waiting to rebuild when empty', { phase: 'eligibility', recipeStatus: 'queued' }); return { ok: true as const, name, port: instance.port, deploymentId: id }; })(); } catch (error) { return { ok: false, kind: 'bad-request', message: (error as Error).message }; } } function filesFor(job: Job): FilePlan[] { const rows = db().prepare(`SELECT f.* FROM staged_file_changes f JOIN provisioning_sources s ON s.kind=f.source_kind AND s.source_id=f.source_id AND s.server=f.server WHERE s.operation_id=? AND f.server=? AND f.state='pending' ORDER BY f.id`).all(job.id, job.server) as StagedFileChange[]; return rows.map(row => ({ id: row.id, path: posix.join(relDestForKind(row.kind, job.server.startsWith('t6_') ? 't6' : 't5'), row.rel_path), contentBase64: row.content?.toString('base64') ?? null, sha256: row.content ? sha256Buffer(row.content) : null, checkBefore: !!row.check_before, expectedBeforeSha: row.expected_before_sha })); } /** Agent receipts are the authoritative before images; do not read a controller mount. */ function applyReceipt(job: Job, result: Record) { if (job.applied_at || !job.files_json) return; const plans = JSON.parse(job.files_json) as FilePlan[]; const before = result.before as { id?: number; path: string; contentBase64: string | null }[] | undefined; db().transaction(() => { for (const plan of plans) { const snapshot = before?.find(item => item.id === plan.id || item.path === plan.path); if (!snapshot) throw new Error('Agent activation receipt is missing a before image.'); const row = db().prepare('SELECT * FROM staged_file_changes WHERE id=?').get(plan.id) as StagedFileChange; if (!row || row.state === 'applied') continue; const bytes = snapshot.contentBase64 === null ? null : Buffer.from(snapshot.contentBase64, 'base64'); const prior = getServerScript(job.server, row.rel_path); if (row.source_kind === 'beta') { if (bytes !== null) db().prepare(`INSERT OR IGNORE INTO beta_file_snapshots(beta_id,server,rel_path,prior_content,prior_sha256,prior_size) VALUES(?,?,?,?,?,?)`) .run(row.source_id, job.server, row.rel_path, bytes, sha256Buffer(bytes), bytes.length); db().prepare(`UPDATE beta_server_scripts SET had_original=?,prior_version_id=? WHERE beta_id=? AND server=? AND script_id IN (SELECT id FROM scripts WHERE target_rel_path=? AND kind=?)`).run(bytes === null ? 0 : 1, prior?.script_version_id ?? null, row.source_id, job.server, row.rel_path, row.kind); db().prepare('INSERT OR IGNORE INTO beta_file_changes(beta_id,server,rel_path) VALUES(?,?,?)').run(row.source_id, job.server, row.rel_path); db().prepare('UPDATE beta_servers SET deployed_at=?,updated_at=? WHERE beta_id=? AND server=?').run(new Date().toISOString(), new Date().toISOString(), row.source_id, job.server); } if (row.deployment_file_id && !row.preserve_before_image) db().prepare('UPDATE deployment_files SET verb=?,content_before=?,sha256_before=? WHERE id=?') .run(row.content === null ? 'removed' : bytes === null ? 'added' : 'updated', bytes, bytes === null ? null : sha256Buffer(bytes), row.deployment_file_id); if (row.content === null) removeScript(job.server, row.rel_path); else recordScript(job.server, row.rel_path, row.script_version_id && db().prepare('SELECT 1 FROM script_versions WHERE id=?').get(row.script_version_id) ? row.script_version_id : null, sha256Buffer(row.content), row.content.length, row.inventory_source); db().prepare("UPDATE staged_file_changes SET state='applied',applied_at=? WHERE id=?").run(new Date().toISOString(), row.id); } db().prepare('UPDATE native_provisioning SET applied_at=? WHERE deployment_id=?').run(new Date().toISOString(), job.id); })(); } function createPayload(instance: NativeInstance, recipe: LaunchRecipe) { const key = getServerKey(instance.server); if (!key) throw new Error('The saved server key was removed.'); return { name: instance.server, game: recipe.game, libraryId: instance.library_id, image: instance.image, port: instance.port, bindAddress: instance.bind_address, rconPassword: config.rcon.password || randomBytes(32).toString('hex'), env: { PLUTAINER_SERVER_NAME: instance.server, PLUTAINER_PORT: String(instance.port), PLUTAINER_GAME: recipe.game === 't6' ? 't6zm' : 't5sp', PLUTAINER_CONFIG_FILE: recipe.configFile, PLUTAINER_MAP_ROTATE: 'false', PLUTAINER_EXTRA_ARGS: `+exec electric_cherry_map.cfg +set tws_server_id ${instance.server} +map_rotate`, PLUTO_SERVER_KEY: key }, files: recipe.files.map(file => ({ path: posix.join(relDestForKind(file.kind, recipe.game), file.path), contentBase64: file.content, sha256: file.sha256 })) }; } function hold(job: Job, message: string) { deploymentStage(job.id, message, { blocker: { code: 'native-host', message, automatic: true } }); } function stillActive(job: Job): boolean { return !!db().prepare(`SELECT 1 FROM server_deployments WHERE id=? AND server_deleted_at IS NULL AND state IN ('queued','waiting','running','awaiting-health','interrupted')`).get(job.id); } function complete(job: Job) { db().transaction(() => { db().prepare('UPDATE native_provisioning SET verified_at=? WHERE deployment_id=?').run(new Date().toISOString(), job.id); db().prepare('UPDATE restart_rollout SET healthy_at=?,error=NULL WHERE operation_id=?').run(new Date().toISOString(), job.id); finishDeployment(job.id, { ok: true }); })(); } export async function processNativeProvisioning(eligibility: (operation: { id: string; server: string; action: string; state: string; stage: string }) => string | null, options: { wait?: boolean } = {}): Promise { // Draining prevents admission immediately and retires existing lobbies only through the normal safety queue. for (const host of listHosts().filter(host => host.mode === 'draining' && host.connected)) { const instances = db().prepare('SELECT * FROM native_instances WHERE host_id=? AND retired_at IS NULL').all(host.id) as NativeInstance[]; for (const instance of instances) { if (!getServer(instance.server) || activeBetaRemovalBlocker(instance.server)) continue; if (db().prepare("SELECT 1 FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND state IN ('queued','waiting','running','awaiting-health')").get(instance.server)) continue; const id = beginDeployment(instance.server, 'destroy', 'host-drain'); db().prepare("UPDATE server_deployments SET state='waiting' WHERE id=?").run(id); deploymentStage(id, 'Host draining; waiting for a fresh empty lobby'); } } const jobs = db().prepare(`SELECT d.*,n.instance_id,n.generation,n.stage_operation_id,n.operation_id,n.files_json,n.applied_at FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id WHERE d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') ORDER BY d.started_at,d.rowid`).all() as Job[]; // One host's observation must not prevent another job being admitted. // Per-job ownership also covers overlapping direct calls and worker passes. const work = jobs.map(job => { const pending = activeJobs.get(job.id); if (pending) return pending; const work = trackActivity(`provisioning:${job.id}`, `Evaluating ${job.action} for ${job.server}`, () => processJob(job, eligibility)) .catch(error => { if (stillActive(job)) hold(job, (error as Error).message); }) .finally(() => { activeJobs.delete(job.id); }); activeJobs.set(job.id, work); return work; }); if (options.wait !== false) await Promise.all(work); } const activeJobs = new Map>(); /** Reservations can move only while no remote work has ever been submitted. */ function placeQueuedCreation(job: Job, instance: NativeInstance): NativeInstance { return db().transaction(() => { const recipe = recipeForDeployment(job.id)!.recipe as NativeRecipe; const prior = recipe.native!; const network = prior.automatic ? prior.network ?? {} : prior; const { host, port } = choosePlacement({ hostId: prior.automatic ? undefined : instance.host_id, libraryId: instance.library_id, image: instance.image, bindAddress: network.bindAddress, thorough: true, excludeInstanceId: instance.id }); const bind = network.bindAddress ?? host.bindAddress; releaseHostPorts(instance.id); reserveHostPort(host.id, instance.id, { port, bindAddress: bind, protocol: 'udp' }); reserveHostPort(host.id, instance.id, { port, bindAddress: bind, protocol: 'tcp' }); db().prepare(`UPDATE native_instances SET host_id=?,bind_address=?,port=?,public_address=?,public_port=? WHERE id=?`) .run(host.id, bind, port, network.publicAddress ?? host.publicAddress, network.publicPort ?? null, instance.id); db().prepare('UPDATE server_creation_jobs SET port=? WHERE deployment_id=?').run(port, job.id); recipe.native = { ...prior, hostId: host.id, bindAddress: bind, publicAddress: network.publicAddress ?? host.publicAddress, publicPort: network.publicPort }; db().prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?').run(JSON.stringify(recipe), job.id); db().prepare('UPDATE server_launch_previews SET port=? WHERE deployment_id=?').run(port, job.id); deploymentStage(job.id, `Placed on ${host.name} after CPU, memory, history and port checks`, { phase: 'eligibility' }); return nativeInstance(job.server)!; })(); } async function processJob(job: Job, eligibility: (job: Job) => string | null) { if (!stillActive(job)) return; let instance = nativeInstance(job.server); if (!instance || instance.id !== job.instance_id || (instance.generation !== job.generation && !(job.action === 'rebuild' && instance.generation === job.generation + 1))) { finishDeployment(job.id, { ok: false, message: 'Original instance was retired or replaced; no command was sent.' }); return; } if (job.action === 'create' && !job.operation_id && !job.stage_operation_id && !job.submitted_at) { const settings = fleetSettings(); const active = db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL AND action IN ('create','rebuild') AND state IN ('running','awaiting-health')`).get(job.id) as { n: number }; if (active.n >= settings.createConcurrency) { hold(job, 'Waiting for startup capacity.'); return; } instance = placeQueuedCreation(job, instance); } const host = getHost(instance.host_id); if (!host?.connected || host.mode === 'disabled') { db().prepare("UPDATE servers SET container_state='unknown',last_players=NULL,last_roster=NULL WHERE name=?").run(job.server); hold(job, 'Host is unavailable; existing server state is unknown. Waiting for reconnection.'); return; } if (job.operation_id) { const operation = getHostOperation(job.operation_id); if (operation?.kind === 'instance.cancel-create' && !['completed', 'failed'].includes(operation.state) && !host.inventory.capabilities?.includes('instance.cancel-create')) { hold(job, hostAgentUpdatesEnabled(host.inventory) ? 'Waiting for the host agent update. New command delivery is paused so its current batch can finish and the automatic updater can run. Cleanup resumes after the updated agent connects.' : 'Update the host agent to support creation cancellation: rerun the current installer on this host. Automatic updates are not enabled. New command delivery is paused; cleanup and port release await the updated agent.'); return; } if (!operation || operation.state === 'unknown') { hold(job, 'Agent result is unknown. Reconciling the original operation; no replacement or repeated command will be sent.'); return; } if (operation.state === 'failed') { // The agent reports failed only when no mutation began. Ambiguous // post-mutation failures remain unknown and retain their capacity. db().transaction(() => { db().prepare('DELETE FROM restart_rollout WHERE operation_id=?').run(job.id); const files = JSON.parse(job.files_json ?? '[]') as FilePlan[]; for (const file of files) db().prepare("UPDATE staged_file_changes SET state='pending' WHERE id=? AND state='applying'").run(file.id); finishDeployment(job.id, { ok: false, message: operation.error ?? 'Agent operation failed before mutation.' }); })(); return; } if (operation.state !== 'completed') { const blocked = operation.result as { message?: string } | null; hold(job, blocked?.message ?? `Agent operation ${operation.state}`); return; } if (job.action === 'destroy') { db().transaction(() => { deploymentStage(job.id, REMOVING_SERVER, { phase: 'cleanup', checkpoint: { key: 'container-action', label: 'Agent confirmed instance and container removal', at: operation.updatedAt } }); db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(new Date().toISOString(), instance!.id); releaseHostPorts(instance!.id); db().prepare('DELETE FROM servers WHERE name=?').run(job.server); recordProgress(job.id, { checkpoint: { key: 'cleanup', label: 'Instance retired and port reservations released' } }); complete(job); finishCreationCancellation(job.id); })(); return; } if (job.action === 'rebuild' && instance.generation === job.generation) { db().prepare('UPDATE native_instances SET generation=? WHERE id=? AND generation=?').run(job.generation + 1, instance.id, job.generation); instance = nativeInstance(job.server)!; } applyReceipt(job, (operation.result ?? {}) as Record); if (job.state !== 'awaiting-health') { db().prepare("UPDATE server_deployments SET state='awaiting-health',error=NULL,finished_at=NULL WHERE id=?").run(job.id); recordProgress(job.id, { command: { state: 'accepted', message: `Agent confirmed ${job.action}; verifying the resulting instance.` }, checkpoint: { key: 'container-action', label: `Agent confirmed ${job.action}`, at: operation.updatedAt } }); } const checks: CheckInput[] = []; const check = (value: CheckInput, phase: import('../../provisioning-detail.js').ProvisioningPhase) => { checks.push({ ...value, checkedAt: new Date().toISOString() }); recordProgress(job.id, { checks, ...(value.result === 'pass' ? { checkpoint: { key: phase, label: value.label } } : {}) }); }; deploymentStage(job.id, 'Checking container state with host agent', { phase: job.action === 'stop' ? 'verify-stopped' : 'verify-container', blocker: null, checks }); const health = await nativeRequest<{ healthy: boolean; state: string }>(instance, 'instance.inspect'); if (!stillActive(job)) return; if (job.action === 'stop') { const stopped = ['stopped','exited','created','dead'].includes(health.state); check({ key: 'stopped', label: 'Stopped container verified', source: 'agent', result: stopped ? 'pass' : 'waiting', actual: health.state }, 'verify-stopped'); if (!stopped) { hold(job, 'Waiting for the agent to verify a stopped container.'); return; } db().prepare("UPDATE servers SET container_state=?,last_players=NULL,last_roster=NULL WHERE name=?").run(health.state, job.server); complete(job); return; } check({ key: 'container', label: 'Container readiness verified', source: 'agent', result: health.healthy ? 'pass' : 'waiting', actual: health.state }, 'verify-container'); if (!health.healthy) { hold(job, `Waiting for container readiness: ${health.state}`); return; } deploymentStage(job.id, 'Checking game status', { phase: 'verify-game', blocker: null }); const status = await probeServerStatus(job.server, operation.updatedAt); if (!stillActive(job)) return; check({ key: 'game', label: 'Fresh game status verified', source: status.ok ? status.source : null, result: status.ok ? 'pass' : 'unknown', actual: status.ok ? status.players : null, message: status.ok ? null : status.reason }, 'verify-game'); if (!status.ok) { hold(job, `Waiting for game status: ${status.reason}`); return; } if (job.action === 'create' || job.action === 'rebuild') { deploymentStage(job.id, 'Checking selected engine map', { phase: 'verify-map', blocker: null }); const recipe = recipeForDeployment(job.id)!.recipe; check({ key: 'map', label: 'Selected engine map verified', source: status.source, result: status.map === recipe.expectedMap ? 'pass' : 'waiting', expected: recipe.expectedMap, actual: status.map }, 'verify-map'); if (status.map !== recipe.expectedMap) { hold(job, `Waiting for selected map ${recipe.expectedMap}; observed ${status.map ?? 'unknown'}.`); return; } deploymentStage(job.id, 'Checking production Plutonium release', { phase: 'verify-plutonium', blocker: null }); await checkRelease(); if (!stillActive(job)) return; const target = freshRelease(); checks.push({ key: 'plutonium-release', label: 'Production Plutonium release', source: 'manifest', result: target === null ? 'unknown' : 'pass', actual: target }); recordProgress(job.id, { checks }); deploymentStage(job.id, 'Checking running Plutonium revision', { phase: 'verify-plutonium', blocker: null }); const revision = await probeVersion(job.server, { operationId: job.id }); if (!stillActive(job)) return; check({ key: 'plutonium-version', label: 'Plutonium revision verified', source: 'rcon', expected: target, actual: revision, result: revision === null || target === null ? 'unknown' : revision < target ? 'fail' : 'pass' }, 'verify-plutonium'); if (revision === null || target === null) { hold(job, 'Waiting to verify the Plutonium runtime revision.'); return; } if (revision < target) { finishDeployment(job.id, { ok: false, message: `Running Plutonium r${revision} is older than production r${target}.` }); return; } } db().prepare('UPDATE servers SET container_state=?,container_checked_at=? WHERE name=?').run(health.state, new Date().toISOString(), job.server); recordProgress(job.id, { blocker: null, checks }); complete(job); return; } const reason = eligibility(job); if (reason) { hold(job, reason); return; } if (job.action === 'destroy' || job.action === 'rebuild') { const beta = activeBetaRemovalBlocker(job.server); if (beta) { hold(job, beta); return; } } const files = job.files_json ? JSON.parse(job.files_json) as FilePlan[] : filesFor(job); if (files.length && !job.stage_operation_id) { const stage = queueHostOperation(instance.host_id, 'files.stage', { stageId: job.id, files }, { instanceId: instance.id, generation: instance.generation, idempotencyKey: `${job.id}:stage` }); db().prepare('UPDATE native_provisioning SET stage_operation_id=?,files_json=? WHERE deployment_id=?').run(stage.id, JSON.stringify(files), job.id); hold(job, 'Staging and verifying files on the host.'); return; } if (job.stage_operation_id) { const stage = getHostOperation(job.stage_operation_id); if (stage?.state === 'failed') { finishDeployment(job.id, { ok: false, message: stage.error ?? 'File staging failed.' }); return; } if (stage?.state !== 'completed') { hold(job, 'Waiting for verified remote file staging.'); return; } if (files.some(file => (db().prepare('SELECT state FROM staged_file_changes WHERE id=?').get(file.id) as { state: string } | undefined)?.state !== 'pending')) { finishDeployment(job.id, { ok: false, message: 'Staged file request was cancelled or changed before activation.' }); return; } } const settings = fleetSettings(); const count = (actions: string[]) => (db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id != ? AND server_deleted_at IS NULL AND action IN (${actions.map(() => '?').join(',')}) AND state IN ('running','awaiting-health')`).get(job.id, ...actions) as { n: number }).n; if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) >= settings.createConcurrency) { deploymentStage(job.id, 'Waiting for startup capacity.', { phase: 'startup-queue', blocker: { code: 'startup-capacity', message: 'Waiting for startup capacity.', automatic: true } }); return; } if (job.action === 'destroy' && count(['destroy']) >= settings.destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; } deploymentStage(job.id, 'Checking host eligibility', { phase: 'eligibility', blocker: null }); if (job.action === 'create') choosePlacement({ hostId: instance.host_id, libraryId: instance.library_id, image: instance.image, bindAddress: instance.bind_address, excludeInstanceId: instance.id, thorough: true }); if (job.action === 'create' || job.action === 'rebuild') requireReadyLibrary(instance.host_id, instance.library_id); if (job.action !== 'create' && job.action !== 'start') { // Fast controller check keeps occupied lobbies out of the operation slots. // The agent repeats this immediately before its local mutation. deploymentStage(job.id, 'Checking container and player occupancy before submission', { phase: 'eligibility', blocker: null }); const health = await nativeRequest<{ state: string; healthy?: boolean }>(instance, 'instance.inspect'); if (!stillActive(job)) return; if (!['stopped','exited','created','dead','missing'].includes(health.state)) { const occupancy = await probePlayerEvidence(job.server); if (!stillActive(job)) return; const recoveredWatchdog = health.healthy && occupancy.players !== null && db().prepare(`SELECT 1 FROM provisioning_sources s JOIN watchdog_incidents i ON i.id=s.source_id JOIN server_deployments d ON d.id=s.operation_id WHERE s.operation_id=? AND s.kind='watchdog' AND i.kind='health' AND d.triggered_by LIKE 'watchdog:%'`).get(job.id); if (recoveredWatchdog && occupancy.players !== null) { resolveRecoveredHealthOperation(job.id, occupancy.players, occupancy.source); return; } if (occupancy.players !== 0) { hold(job, occupancy.players !== null ? `Waiting for ${occupancy.players} player(s) to leave.` : 'Waiting for a fresh local player count.'); return; } } } const current = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined; if (!current || !['queued','waiting','interrupted'].includes(current.state) || eligibility(job)) return; db().transaction(() => { const live = nativeInstance(job.server); if (live?.id !== instance!.id || live.generation !== instance!.generation) throw new Error('Instance changed during preflight.'); if (job.action === 'destroy' || job.action === 'rebuild') { const beta = activeBetaRemovalBlocker(job.server); if (beta) { hold(job, beta); return; } } if (JSON.stringify(filesFor(job)) !== JSON.stringify(files)) { finishDeployment(job.id, { ok: false, message: 'File requests changed during preflight. Review and submit a fresh activation.' }); return; } // Async occupancy checks can overlap another job claiming the final slot. if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) >= settings.createConcurrency) { deploymentStage(job.id, 'Waiting for startup capacity.', { phase: 'startup-queue', blocker: { code: 'startup-capacity', message: 'Waiting for startup capacity.', automatic: true } }); return; } if (job.action === 'destroy' && count(['destroy']) >= settings.destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; } if (job.action === 'start' || job.action === 'restart' || job.action === 'rebuild') { db().prepare(`INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES(?,?,?) ON CONFLICT(server) DO UPDATE SET requested_at=excluded.requested_at,operation_id=excluded.operation_id,healthy_at=NULL,error=NULL`).run(job.server, new Date().toISOString(), job.id); } recordProgress(job.id, { blocker: null, checkpoint: { key: 'eligibility', label: 'Host eligibility and safety checks passed' } }); let payload: Record = files.length ? { stageId: job.id } : {}; if (job.action === 'create' || job.action === 'rebuild') { deploymentStage(job.id, 'Preparing launch files for the host agent', { phase: 'prepare', recipeStatus: 'attempted', checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } }); const prior = recipeForDeployment(job.id)!.recipe as NativeRecipe; const recipe: NativeRecipe = { ...captureRecipe(prior.game, prior.mapSlug), native: prior.native, image: instance!.image }; payload = createPayload(instance!, recipe); if (job.action === 'rebuild') payload.nextGeneration = instance!.generation + 1; db().prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?').run(JSON.stringify(recipe), job.id); if (job.action === 'create') db().prepare(`INSERT INTO servers(name,komodo_type,komodo_name,game,map,game_mode,map_slug,host_dir,config_file,game_id,image,game_port,created_by,container_state) VALUES(?,'deployment','',?,?,'zombie',?,?,?,?,?,?,'native','starting')`).run(job.server, recipe.game, recipe.expectedMap, recipe.mapSlug, serverDir(job.server), recipe.configFile, recipe.game === 't6' ? 't6zm' : 't5sp', recipe.image, instance!.port); for (const file of recipe.files) recordScript(job.server, file.path, file.versionId, file.sha256, file.size, 'native-create'); recordProgress(job.id, { checkpoint: { key: 'prepare', label: 'Launch files selected and command payload prepared' } }); } const operation = queueHostOperation(instance!.host_id, `instance.${job.action === 'destroy' ? 'remove' : job.action}`, payload, { instanceId: instance!.id, generation: instance!.generation, idempotencyKey: job.id }); db().prepare('UPDATE native_provisioning SET operation_id=? WHERE deployment_id=?').run(operation.id, job.id); for (const file of files) db().prepare("UPDATE staged_file_changes SET state='applying' WHERE id=? AND state='pending'").run(file.id); db().prepare("UPDATE server_deployments SET state='running',submitted_at=? WHERE id=?").run(new Date().toISOString(), job.id); deploymentStage(job.id, `Waiting for host agent to acknowledge ${job.action}`, { phase: 'container-action', blocker: null, command: { state: 'attempted', message: `Durable agent operation ${operation.id}; reconciliation always uses this ID.` } }); })(); }