|
|
|
@@ -0,0 +1,404 @@
|
|
|
|
|
import { randomBytes, randomUUID } from 'node:crypto';
|
|
|
|
|
import { posix } from 'node:path';
|
|
|
|
|
import { db } from '../db/index.js';
|
|
|
|
|
import { config, relDestForKind, serverDir } from '../config.js';
|
|
|
|
|
import { nativeInstance, nativeRequest, type NativeInstance } from '../backends/native.js';
|
|
|
|
|
import { getHost, getHostOperation, listHosts, nextHostPort, queueHostOperation, releaseHostPorts, requireReadyHost, reserveHostPort } from '../hosts/index.js';
|
|
|
|
|
import { getLibraryVersion, requireReadyLibrary } from '../game-files/index.js';
|
|
|
|
|
import { blueprintRecipe, captureRecipe, recipeForDeployment, summarizeRecipe, type LaunchRecipe } from '../fleet/launch-recipes.js';
|
|
|
|
|
import { getServerKey, saveServerKey } from '../fleet/server-keys.js';
|
|
|
|
|
import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
|
|
|
|
|
import { nextLobbyNumber, type CreateServerInput, type CreateServerResult } from './instances.js';
|
|
|
|
|
import { beginDeployment, deploymentStage, finishDeployment, REMOVING_SERVER } from './history.js';
|
|
|
|
|
import { recordProgress } from './progress.js';
|
|
|
|
|
import { fleetSettings } from '../settings.js';
|
|
|
|
|
import { resolveRecoveredHealthOperation } from '../maintenance/recovery.js';
|
|
|
|
|
import { probeServerStatus, probePlayerEvidence } from '../fleet/status-evidence.js';
|
|
|
|
|
import { checkRelease, freshRelease, probeVersion } from '../integrations/plutonium.js';
|
|
|
|
|
import { getServerScript, recordScript, removeScript } from '../fleet/inventory.js';
|
|
|
|
|
import { sha256Buffer } from '../fleet/files.js';
|
|
|
|
|
import type { StagedFileChange } from '../fleet/staged-files.js';
|
|
|
|
|
import type { LaunchPreview } from '../../deployment-plan.js';
|
|
|
|
|
import type { ProvisioningAction } from '../../provisioning.js';
|
|
|
|
|
|
|
|
|
|
export interface NativeSelection { hostId: string; libraryId: string; image?: string; bindAddress?: string; publicAddress?: string; publicPort?: number }
|
|
|
|
|
type NativeRecipe = LaunchRecipe & { native?: NativeSelection };
|
|
|
|
|
interface Job { id: string; server: string; action: ProvisioningAction; state: string; submitted_at: string | null; stage: string;
|
|
|
|
|
instance_id: string; generation: number; stage_operation_id: string | null; operation_id: string | null; files_json: string | null; applied_at: string | null }
|
|
|
|
|
interface FilePlan { id: number; path: string; contentBase64: string | null; sha256: string | null; checkBefore: boolean; expectedBeforeSha: string | null }
|
|
|
|
|
|
|
|
|
|
export function nativeSelection(input: Record<string, unknown>, game: string): NativeSelection {
|
|
|
|
|
if (typeof input.hostId !== 'string' || !input.hostId) throw new Error('Select a ready host.');
|
|
|
|
|
if (typeof input.libraryId !== 'string' || !input.libraryId) throw new Error('Select a published game-file version.');
|
|
|
|
|
const version = getLibraryVersion(input.libraryId);
|
|
|
|
|
if (!version || version.game !== game || version.state !== 'ready') throw new Error('Choose a published game-file version for the selected game.');
|
|
|
|
|
const image = input.image === undefined || input.image === '' ? config.komodo.defaultImage : input.image;
|
|
|
|
|
if (typeof image !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9._/:@-]{0,499}$/.test(image)) throw new Error('Invalid runtime image.');
|
|
|
|
|
const host = requireReadyHost(input.hostId, { image, requiredBytes: 1024 ** 3 });
|
|
|
|
|
requireReadyLibrary(host.id, version.id);
|
|
|
|
|
const bindAddress = input.bindAddress === undefined || input.bindAddress === '' ? host.bindAddress : input.bindAddress;
|
|
|
|
|
if (typeof bindAddress !== 'string') throw new Error('Invalid bind address.');
|
|
|
|
|
if (input.publicAddress !== undefined && (typeof input.publicAddress !== 'string' || input.publicAddress.length > 253)) throw new Error('Invalid public address.');
|
|
|
|
|
if (input.publicPort !== undefined && (!Number.isInteger(input.publicPort) || Number(input.publicPort) < 1 || Number(input.publicPort) > 65535)) throw new Error('Invalid public port.');
|
|
|
|
|
return { hostId: host.id, libraryId: version.id, image, bindAddress, publicAddress: input.publicAddress as string | undefined ?? host.publicAddress,
|
|
|
|
|
publicPort: input.publicPort as number | undefined };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function previewNativeCreation(input: Record<string, unknown>): LaunchPreview {
|
|
|
|
|
let recipe: NativeRecipe;
|
|
|
|
|
let blueprintName: string | null = null;
|
|
|
|
|
if (typeof input.deploymentId === 'string') {
|
|
|
|
|
const previous = recipeForDeployment(input.deploymentId);
|
|
|
|
|
if (!previous) throw new Error('The recorded deployment is unavailable.');
|
|
|
|
|
recipe = captureRecipe(previous.recipe.game, previous.recipe.mapSlug);
|
|
|
|
|
blueprintName = previous.blueprintName;
|
|
|
|
|
} else if (typeof input.blueprintId === 'string') ({ recipe, blueprintName } = blueprintRecipe(input.blueprintId));
|
|
|
|
|
else {
|
|
|
|
|
if ((input.game !== 't5' && input.game !== 't6') || typeof input.mapSlug !== 'string') throw new Error('Choose a supported game and map.');
|
|
|
|
|
recipe = captureRecipe(input.game, input.mapSlug);
|
|
|
|
|
}
|
|
|
|
|
const selection = nativeSelection(input, recipe.game);
|
|
|
|
|
recipe.native = selection; recipe.image = selection.image!;
|
|
|
|
|
const base = `${recipe.game}_zombie_${recipe.mapSlug}`;
|
|
|
|
|
const lobby = nextLobbyNumber(base);
|
|
|
|
|
const name = lobby === null ? null : `${base}_${lobby}`;
|
|
|
|
|
const port = nextNativePort(selection);
|
|
|
|
|
const id = randomUUID(); const expiresAt = new Date(Date.now() + 30 * 60_000).toISOString();
|
|
|
|
|
db().prepare('INSERT INTO server_launch_previews(id,expires_at,server,port,recipe,blueprint_name) VALUES(?,?,?,?,?,?)')
|
|
|
|
|
.run(id, expiresAt, name, port, JSON.stringify(recipe), blueprintName);
|
|
|
|
|
const checks: LaunchPreview['checks'] = [
|
|
|
|
|
{ label: 'Host readiness', state: 'pass', detail: `${getHost(selection.hostId)!.name}: connected, Docker ready, storage and runtime verified.` },
|
|
|
|
|
{ label: 'Game files', state: 'pass', detail: `Immutable library version ${selection.libraryId} is ready on the selected host.` },
|
|
|
|
|
{ label: 'Lobby', state: name ? 'pass' : 'blocked', detail: name ?? 'All lobby IDs for this map are reserved.' },
|
|
|
|
|
{ label: 'Host port', state: port ? 'pass' : 'blocked', detail: port ? `${selection.bindAddress}:${port} / UDP and TCP` : 'No available UDP/TCP port on this host.' },
|
|
|
|
|
{ label: 'Configuration', state: 'pass', detail: recipe.configSource }
|
|
|
|
|
];
|
|
|
|
|
return { id, expiresAt, name, port, hasSavedKey: !!name && !!getServerKey(name), recipe: summarizeRecipe(recipe), blueprintName, checks,
|
|
|
|
|
ready: checks.every(check => check.state !== 'blocked') };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function nextNativePort(selection: NativeSelection, taken: number[] = []): number | null {
|
|
|
|
|
const excluded = [...taken];
|
|
|
|
|
for (;;) {
|
|
|
|
|
const port = nextHostPort(selection.hostId, excluded, selection.bindAddress, 'udp');
|
|
|
|
|
if (port === null) return null;
|
|
|
|
|
// Both protocols are published by Plutainer and reserved together.
|
|
|
|
|
const tcp = nextHostPort(selection.hostId, excluded, selection.bindAddress, 'tcp');
|
|
|
|
|
if (tcp === port) return port;
|
|
|
|
|
if (tcp !== null && tcp < port) { excluded.push(tcp); continue; }
|
|
|
|
|
excluded.push(port);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function queueNativeCreation(input: CreateServerInput): CreateServerResult {
|
|
|
|
|
try {
|
|
|
|
|
return db().transaction(() => {
|
|
|
|
|
const preview = input.previewId ? db().prepare('SELECT * FROM server_launch_previews WHERE id=?').get(input.previewId) as
|
|
|
|
|
{ id: string; recipe: string; server: string; port: number; expires_at: string; deployment_id: string | null; blueprint_name: string | null } | undefined : undefined;
|
|
|
|
|
if (input.previewId && !preview) throw new Error('Review a fresh launch preview.');
|
|
|
|
|
if (preview?.deployment_id) return { ok: true as const, name: preview.server, port: preview.port, deploymentId: preview.deployment_id };
|
|
|
|
|
if (preview && preview.expires_at <= new Date().toISOString()) throw new Error('Launch preview expired.');
|
|
|
|
|
const saved = preview ? JSON.parse(preview.recipe) as NativeRecipe : undefined;
|
|
|
|
|
const game = input.game ?? 't5';
|
|
|
|
|
if (input.gameMode !== 'zombie' || (saved && (saved.game !== game || saved.mapSlug !== input.mapSlug))) throw new Error('Selection changed. Review a fresh launch preview.');
|
|
|
|
|
const selection = nativeSelection((saved?.native ?? input) as unknown as Record<string, unknown>, game);
|
|
|
|
|
const recipe: NativeRecipe = { ...captureRecipe(game, input.mapSlug), native: selection, image: selection.image! };
|
|
|
|
|
const lobby = nextLobbyNumber(`${game}_zombie_${input.mapSlug}`);
|
|
|
|
|
if (lobby === null) throw new Error('All lobby IDs are reserved.');
|
|
|
|
|
const name = `${game}_zombie_${input.mapSlug}_${lobby}`;
|
|
|
|
|
const port = nextNativePort(selection);
|
|
|
|
|
if (!port) throw new Error('No available host port.');
|
|
|
|
|
if ((input.expectedName && input.expectedName !== name) || (preview && (preview.server !== name || preview.port !== port))) throw new Error('Available lobby or host port changed. Review a fresh preview.');
|
|
|
|
|
const key = input.plutoKey?.trim() || getServerKey(name);
|
|
|
|
|
if (!key) throw new Error(`Save a Plutonium server key for ${name}.`);
|
|
|
|
|
saveServerKey(name, key);
|
|
|
|
|
const instanceId = randomUUID();
|
|
|
|
|
reserveHostPort(selection.hostId, instanceId, { port, bindAddress: selection.bindAddress, protocol: 'udp' });
|
|
|
|
|
reserveHostPort(selection.hostId, instanceId, { port, bindAddress: selection.bindAddress, protocol: 'tcp' });
|
|
|
|
|
db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,bind_address,port,public_address,public_port,created_at)
|
|
|
|
|
VALUES(?,?,?,?,?,?,?,?,?,?)`).run(instanceId, name, selection.hostId, selection.libraryId, selection.image, selection.bindAddress, port,
|
|
|
|
|
selection.publicAddress ?? null, selection.publicPort ?? null, new Date().toISOString());
|
|
|
|
|
const id = beginDeployment(name, 'create', input.createdBy);
|
|
|
|
|
db().prepare("UPDATE server_deployments SET state='queued' WHERE id=?").run(id);
|
|
|
|
|
db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,?,?,?,?)')
|
|
|
|
|
.run(id, name, port, 'zombie', recipe.mapSlug, recipe.expectedMap);
|
|
|
|
|
db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,blueprint_name) VALUES(?,?,?)').run(id, JSON.stringify(recipe), preview?.blueprint_name ?? null);
|
|
|
|
|
if (preview) db().prepare('UPDATE server_launch_previews SET deployment_id=? WHERE id=?').run(id, preview.id);
|
|
|
|
|
deploymentStage(id, 'Queued for native host deployment', { phase: 'startup-queue', recipeStatus: 'queued' });
|
|
|
|
|
return { ok: true as const, name, port, deploymentId: id };
|
|
|
|
|
})();
|
|
|
|
|
} catch (error) { return { ok: false, kind: 'bad-request', message: (error as Error).message }; }
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function bindNativeOperation(id: string, server: string): void {
|
|
|
|
|
const instance = nativeInstance(server);
|
|
|
|
|
if (instance) db().prepare('INSERT OR IGNORE INTO native_provisioning(deployment_id,instance_id,generation) VALUES(?,?,?)').run(id, instance.id, instance.generation);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** Cancelling an unsubmitted launch releases only its local reservation. */
|
|
|
|
|
export function cancelQueuedNativeCreation(server: string, triggeredBy: string): string | null {
|
|
|
|
|
const instance = nativeInstance(server);
|
|
|
|
|
if (!instance || getServer(server)) return null;
|
|
|
|
|
return db().transaction(() => {
|
|
|
|
|
const creation = db().prepare(`SELECT d.id FROM server_deployments d JOIN native_provisioning n ON n.deployment_id=d.id
|
|
|
|
|
WHERE n.instance_id=? AND d.action='create' AND d.state='queued' AND n.operation_id IS NULL`).get(instance.id) as { id: string } | undefined;
|
|
|
|
|
if (!creation) return null;
|
|
|
|
|
const id = beginDeployment(server, 'destroy', triggeredBy);
|
|
|
|
|
const now = new Date().toISOString();
|
|
|
|
|
db().prepare("UPDATE server_deployments SET state='cancelled',finished_at=?,stage='Queued native creation cancelled',server_deleted_at=? WHERE id=?")
|
|
|
|
|
.run(now, now, creation.id);
|
|
|
|
|
db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(now, instance.id);
|
|
|
|
|
releaseHostPorts(instance.id);
|
|
|
|
|
deploymentStage(id, 'Cancelled queued creation; no agent mutation was submitted');
|
|
|
|
|
finishDeployment(id, { ok: true });
|
|
|
|
|
return id;
|
|
|
|
|
})();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function queueNativeRebuild(name: string, triggeredBy: string): CreateServerResult {
|
|
|
|
|
try {
|
|
|
|
|
const instance = nativeInstance(name)!; const server = getServer(name);
|
|
|
|
|
const blocker = activeBetaRemovalBlocker(name);
|
|
|
|
|
if (blocker) throw new Error(blocker);
|
|
|
|
|
if (!server?.map_slug || !getServerKey(name)) throw new Error('A saved map and server key are required before rebuilding.');
|
|
|
|
|
requireReadyLibrary(instance.host_id, instance.library_id);
|
|
|
|
|
const recipe: NativeRecipe = { ...captureRecipe(server.game as 't5' | 't6', server.map_slug), image: instance.image,
|
|
|
|
|
native: { hostId: instance.host_id, libraryId: instance.library_id, image: instance.image, bindAddress: instance.bind_address } };
|
|
|
|
|
return db().transaction(() => {
|
|
|
|
|
const id = beginDeployment(name, 'rebuild', triggeredBy);
|
|
|
|
|
db().prepare("UPDATE server_deployments SET state='waiting' WHERE id=?").run(id);
|
|
|
|
|
db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,?,?,?,?)')
|
|
|
|
|
.run(id, name, instance.port, 'zombie', recipe.mapSlug, recipe.expectedMap);
|
|
|
|
|
db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe) VALUES(?,?)').run(id, JSON.stringify(recipe));
|
|
|
|
|
deploymentStage(id, 'Waiting to rebuild when empty', { phase: 'eligibility', recipeStatus: 'queued' });
|
|
|
|
|
return { ok: true as const, name, port: instance.port, deploymentId: id };
|
|
|
|
|
})();
|
|
|
|
|
} catch (error) { return { ok: false, kind: 'bad-request', message: (error as Error).message }; }
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function filesFor(job: Job): FilePlan[] {
|
|
|
|
|
const rows = db().prepare(`SELECT f.* FROM staged_file_changes f JOIN provisioning_sources s
|
|
|
|
|
ON s.kind=f.source_kind AND s.source_id=f.source_id AND s.server=f.server
|
|
|
|
|
WHERE s.operation_id=? AND f.server=? AND f.state='pending' ORDER BY f.id`).all(job.id, job.server) as StagedFileChange[];
|
|
|
|
|
return rows.map(row => ({ id: row.id, path: posix.join(relDestForKind(row.kind, job.server.startsWith('t6_') ? 't6' : 't5'), row.rel_path),
|
|
|
|
|
contentBase64: row.content?.toString('base64') ?? null, sha256: row.content ? sha256Buffer(row.content) : null,
|
|
|
|
|
checkBefore: !!row.check_before, expectedBeforeSha: row.expected_before_sha }));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** Agent receipts are the authoritative before images; do not read a controller mount. */
|
|
|
|
|
function applyReceipt(job: Job, result: Record<string, unknown>) {
|
|
|
|
|
if (job.applied_at || !job.files_json) return;
|
|
|
|
|
const plans = JSON.parse(job.files_json) as FilePlan[];
|
|
|
|
|
const before = result.before as { id?: number; path: string; contentBase64: string | null }[] | undefined;
|
|
|
|
|
db().transaction(() => {
|
|
|
|
|
for (const plan of plans) {
|
|
|
|
|
const snapshot = before?.find(item => item.id === plan.id || item.path === plan.path);
|
|
|
|
|
if (!snapshot) throw new Error('Agent activation receipt is missing a before image.');
|
|
|
|
|
const row = db().prepare('SELECT * FROM staged_file_changes WHERE id=?').get(plan.id) as StagedFileChange;
|
|
|
|
|
if (!row || row.state === 'applied') continue;
|
|
|
|
|
const bytes = snapshot.contentBase64 === null ? null : Buffer.from(snapshot.contentBase64, 'base64');
|
|
|
|
|
const prior = getServerScript(job.server, row.rel_path);
|
|
|
|
|
if (row.source_kind === 'beta') {
|
|
|
|
|
if (bytes !== null) db().prepare(`INSERT OR IGNORE INTO beta_file_snapshots(beta_id,server,rel_path,prior_content,prior_sha256,prior_size) VALUES(?,?,?,?,?,?)`)
|
|
|
|
|
.run(row.source_id, job.server, row.rel_path, bytes, sha256Buffer(bytes), bytes.length);
|
|
|
|
|
db().prepare(`UPDATE beta_server_scripts SET had_original=?,prior_version_id=? WHERE beta_id=? AND server=?
|
|
|
|
|
AND script_id IN (SELECT id FROM scripts WHERE target_rel_path=? AND kind=?)`).run(bytes === null ? 0 : 1, prior?.script_version_id ?? null, row.source_id, job.server, row.rel_path, row.kind);
|
|
|
|
|
db().prepare('INSERT OR IGNORE INTO beta_file_changes(beta_id,server,rel_path) VALUES(?,?,?)').run(row.source_id, job.server, row.rel_path);
|
|
|
|
|
db().prepare('UPDATE beta_servers SET deployed_at=?,updated_at=? WHERE beta_id=? AND server=?').run(new Date().toISOString(), new Date().toISOString(), row.source_id, job.server);
|
|
|
|
|
}
|
|
|
|
|
if (row.deployment_file_id && !row.preserve_before_image) db().prepare('UPDATE deployment_files SET verb=?,content_before=?,sha256_before=? WHERE id=?')
|
|
|
|
|
.run(row.content === null ? 'removed' : bytes === null ? 'added' : 'updated', bytes, bytes === null ? null : sha256Buffer(bytes), row.deployment_file_id);
|
|
|
|
|
if (row.content === null) removeScript(job.server, row.rel_path);
|
|
|
|
|
else recordScript(job.server, row.rel_path, row.script_version_id && db().prepare('SELECT 1 FROM script_versions WHERE id=?').get(row.script_version_id) ? row.script_version_id : null,
|
|
|
|
|
sha256Buffer(row.content), row.content.length, row.inventory_source);
|
|
|
|
|
db().prepare("UPDATE staged_file_changes SET state='applied',applied_at=? WHERE id=?").run(new Date().toISOString(), row.id);
|
|
|
|
|
}
|
|
|
|
|
db().prepare('UPDATE native_provisioning SET applied_at=? WHERE deployment_id=?').run(new Date().toISOString(), job.id);
|
|
|
|
|
})();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function createPayload(instance: NativeInstance, recipe: LaunchRecipe) {
|
|
|
|
|
const key = getServerKey(instance.server);
|
|
|
|
|
if (!key) throw new Error('The saved server key was removed.');
|
|
|
|
|
return { name: instance.server, game: recipe.game, libraryId: instance.library_id, image: instance.image, port: instance.port, bindAddress: instance.bind_address,
|
|
|
|
|
rconPassword: config.rcon.password || randomBytes(32).toString('hex'),
|
|
|
|
|
env: { PLUTAINER_SERVER_NAME: instance.server, PLUTAINER_PORT: String(instance.port), PLUTAINER_GAME: recipe.game === 't6' ? 't6zm' : 't5sp',
|
|
|
|
|
PLUTAINER_CONFIG_FILE: recipe.configFile, PLUTAINER_MAP_ROTATE: 'false', PLUTAINER_EXTRA_ARGS: `+exec electric_cherry_map.cfg +set tws_server_id ${instance.server} +map_rotate`, PLUTO_SERVER_KEY: key },
|
|
|
|
|
files: recipe.files.map(file => ({ path: posix.join(relDestForKind(file.kind, recipe.game), file.path), contentBase64: file.content, sha256: file.sha256 })) };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function hold(job: Job, message: string) { deploymentStage(job.id, message, { blocker: { code: 'native-host', message, automatic: true } }); }
|
|
|
|
|
function complete(job: Job) {
|
|
|
|
|
db().transaction(() => {
|
|
|
|
|
db().prepare('UPDATE native_provisioning SET verified_at=? WHERE deployment_id=?').run(new Date().toISOString(), job.id);
|
|
|
|
|
db().prepare('UPDATE restart_rollout SET healthy_at=?,error=NULL WHERE operation_id=?').run(new Date().toISOString(), job.id);
|
|
|
|
|
finishDeployment(job.id, { ok: true });
|
|
|
|
|
})();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export async function processNativeProvisioning(eligibility: (operation: { id: string; server: string; action: string; state: string; stage: string }) => string | null): Promise<void> {
|
|
|
|
|
// Draining prevents admission immediately and retires existing lobbies only through the normal safety queue.
|
|
|
|
|
for (const host of listHosts().filter(host => host.mode === 'draining' && host.connected)) {
|
|
|
|
|
const instances = db().prepare('SELECT * FROM native_instances WHERE host_id=? AND retired_at IS NULL').all(host.id) as NativeInstance[];
|
|
|
|
|
for (const instance of instances) {
|
|
|
|
|
if (!getServer(instance.server) || activeBetaRemovalBlocker(instance.server)) continue;
|
|
|
|
|
if (db().prepare("SELECT 1 FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND state IN ('queued','waiting','running','awaiting-health')").get(instance.server)) continue;
|
|
|
|
|
const id = beginDeployment(instance.server, 'destroy', 'host-drain');
|
|
|
|
|
db().prepare("UPDATE server_deployments SET state='waiting' WHERE id=?").run(id);
|
|
|
|
|
deploymentStage(id, 'Host draining; waiting for a fresh empty lobby');
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
const jobs = db().prepare(`SELECT d.*,n.instance_id,n.generation,n.stage_operation_id,n.operation_id,n.files_json,n.applied_at
|
|
|
|
|
FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
|
|
|
|
|
WHERE d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') ORDER BY d.started_at,d.rowid`).all() as Job[];
|
|
|
|
|
for (const job of jobs) {
|
|
|
|
|
try { await processJob(job, eligibility); }
|
|
|
|
|
catch (error) { hold(job, (error as Error).message); }
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function processJob(job: Job, eligibility: (job: Job) => string | null) {
|
|
|
|
|
let instance = nativeInstance(job.server);
|
|
|
|
|
if (!instance || instance.id !== job.instance_id || (instance.generation !== job.generation && !(job.action === 'rebuild' && instance.generation === job.generation + 1))) {
|
|
|
|
|
finishDeployment(job.id, { ok: false, message: 'Original instance was retired or replaced; no command was sent.' }); return;
|
|
|
|
|
}
|
|
|
|
|
const host = getHost(instance.host_id);
|
|
|
|
|
if (!host?.connected || host.mode === 'disabled') {
|
|
|
|
|
db().prepare("UPDATE servers SET container_state='unknown',last_players=NULL,last_roster=NULL WHERE name=?").run(job.server);
|
|
|
|
|
hold(job, 'Host is unavailable; existing server state is unknown. Waiting for reconnection.'); return;
|
|
|
|
|
}
|
|
|
|
|
if (job.operation_id) {
|
|
|
|
|
const operation = getHostOperation(job.operation_id);
|
|
|
|
|
if (!operation || operation.state === 'unknown') { hold(job, 'Agent result is unknown. Reconciling the original operation; no replacement or repeated command will be sent.'); return; }
|
|
|
|
|
if (operation.state === 'failed') {
|
|
|
|
|
// The agent reports failed only when no mutation began. Ambiguous
|
|
|
|
|
// post-mutation failures remain unknown and retain their capacity.
|
|
|
|
|
db().transaction(() => {
|
|
|
|
|
db().prepare('DELETE FROM restart_rollout WHERE operation_id=?').run(job.id);
|
|
|
|
|
const files = JSON.parse(job.files_json ?? '[]') as FilePlan[];
|
|
|
|
|
for (const file of files) db().prepare("UPDATE staged_file_changes SET state='pending' WHERE id=? AND state='applying'").run(file.id);
|
|
|
|
|
finishDeployment(job.id, { ok: false, message: operation.error ?? 'Agent operation failed before mutation.' });
|
|
|
|
|
})();
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
if (operation.state !== 'completed') {
|
|
|
|
|
const blocked = operation.result as { message?: string } | null;
|
|
|
|
|
hold(job, blocked?.message ?? `Agent operation ${operation.state}`); return;
|
|
|
|
|
}
|
|
|
|
|
if (job.action === 'destroy') {
|
|
|
|
|
db().transaction(() => {
|
|
|
|
|
deploymentStage(job.id, REMOVING_SERVER, { checkpoint: { key: 'delete-deployment', label: 'Agent confirmed instance and container removal' } });
|
|
|
|
|
db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(new Date().toISOString(), instance!.id);
|
|
|
|
|
releaseHostPorts(instance!.id);
|
|
|
|
|
db().prepare('DELETE FROM servers WHERE name=?').run(job.server);
|
|
|
|
|
complete(job);
|
|
|
|
|
})();
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
if (job.action === 'rebuild' && instance.generation === job.generation) {
|
|
|
|
|
db().prepare('UPDATE native_instances SET generation=? WHERE id=? AND generation=?').run(job.generation + 1, instance.id, job.generation);
|
|
|
|
|
instance = nativeInstance(job.server)!;
|
|
|
|
|
}
|
|
|
|
|
applyReceipt(job, (operation.result ?? {}) as Record<string, unknown>);
|
|
|
|
|
if (job.state !== 'awaiting-health') {
|
|
|
|
|
db().prepare("UPDATE server_deployments SET state='awaiting-health',error=NULL,finished_at=NULL WHERE id=?").run(job.id);
|
|
|
|
|
recordProgress(job.id, { command: { state: 'accepted', message: `Agent confirmed ${job.action}; verifying the resulting instance.` }, phase: 'verify-container' });
|
|
|
|
|
}
|
|
|
|
|
const health = await nativeRequest<{ healthy: boolean; state: string }>(instance, 'instance.inspect');
|
|
|
|
|
if (job.action === 'stop') {
|
|
|
|
|
if (!['stopped','exited','created','dead'].includes(health.state)) { hold(job, 'Waiting for the agent to verify a stopped container.'); return; }
|
|
|
|
|
db().prepare("UPDATE servers SET container_state=?,last_players=NULL,last_roster=NULL WHERE name=?").run(health.state, job.server);
|
|
|
|
|
complete(job); return;
|
|
|
|
|
}
|
|
|
|
|
if (!health.healthy) { hold(job, `Waiting for container readiness: ${health.state}`); return; }
|
|
|
|
|
const status = await probeServerStatus(job.server, operation.updatedAt);
|
|
|
|
|
if (!status.ok) { hold(job, `Waiting for game status: ${status.reason}`); return; }
|
|
|
|
|
if (job.action === 'create' || job.action === 'rebuild') {
|
|
|
|
|
const recipe = recipeForDeployment(job.id)!.recipe;
|
|
|
|
|
if (status.map !== recipe.expectedMap) { hold(job, `Waiting for selected map ${recipe.expectedMap}; observed ${status.map ?? 'unknown'}.`); return; }
|
|
|
|
|
await checkRelease();
|
|
|
|
|
const revision = await probeVersion(job.server); const target = freshRelease();
|
|
|
|
|
if (revision === null || target === null) { hold(job, 'Waiting to verify the Plutonium runtime revision.'); return; }
|
|
|
|
|
if (revision < target) { finishDeployment(job.id, { ok: false, message: `Running Plutonium r${revision} is older than production r${target}.` }); return; }
|
|
|
|
|
}
|
|
|
|
|
db().prepare("UPDATE servers SET container_state='running',container_checked_at=? WHERE name=?").run(new Date().toISOString(), job.server);
|
|
|
|
|
recordProgress(job.id, { blocker: null, checks: [{ key: 'container', label: 'Container readiness', source: 'agent', result: 'pass', actual: health.state },
|
|
|
|
|
{ key: 'game', label: 'Game status', source: status.source, result: 'pass', actual: status.players }] });
|
|
|
|
|
complete(job); return;
|
|
|
|
|
}
|
|
|
|
|
const reason = eligibility(job);
|
|
|
|
|
if (reason) { hold(job, reason); return; }
|
|
|
|
|
if (job.action === 'destroy' || job.action === 'rebuild') {
|
|
|
|
|
const beta = activeBetaRemovalBlocker(job.server); if (beta) { hold(job, beta); return; }
|
|
|
|
|
}
|
|
|
|
|
const files = job.files_json ? JSON.parse(job.files_json) as FilePlan[] : filesFor(job);
|
|
|
|
|
if (files.length && !job.stage_operation_id) {
|
|
|
|
|
const stage = queueHostOperation(instance.host_id, 'files.stage', { stageId: job.id, files }, { instanceId: instance.id, generation: instance.generation, idempotencyKey: `${job.id}:stage` });
|
|
|
|
|
db().prepare('UPDATE native_provisioning SET stage_operation_id=?,files_json=? WHERE deployment_id=?').run(stage.id, JSON.stringify(files), job.id);
|
|
|
|
|
hold(job, 'Staging and verifying files on the host.'); return;
|
|
|
|
|
}
|
|
|
|
|
if (job.stage_operation_id) {
|
|
|
|
|
const stage = getHostOperation(job.stage_operation_id);
|
|
|
|
|
if (stage?.state === 'failed') { finishDeployment(job.id, { ok: false, message: stage.error ?? 'File staging failed.' }); return; }
|
|
|
|
|
if (stage?.state !== 'completed') { hold(job, 'Waiting for verified remote file staging.'); return; }
|
|
|
|
|
if (files.some(file => (db().prepare('SELECT state FROM staged_file_changes WHERE id=?').get(file.id) as { state: string } | undefined)?.state !== 'pending')) {
|
|
|
|
|
finishDeployment(job.id, { ok: false, message: 'Staged file request was cancelled or changed before activation.' }); return;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
const settings = fleetSettings();
|
|
|
|
|
const count = (actions: string[]) => (db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id != ? AND server_deleted_at IS NULL
|
|
|
|
|
AND action IN (${actions.map(() => '?').join(',')}) AND state IN ('running','awaiting-health')`).get(job.id, ...actions) as { n: number }).n;
|
|
|
|
|
if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) >= settings.createConcurrency) { hold(job, 'Waiting for startup capacity.'); return; }
|
|
|
|
|
if (job.action === 'destroy' && count(['destroy']) >= settings.destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; }
|
|
|
|
|
if (job.action === 'create') requireReadyHost(instance.host_id, { image: instance.image, requiredBytes: 1024 ** 3 });
|
|
|
|
|
if (job.action === 'create' || job.action === 'rebuild') requireReadyLibrary(instance.host_id, instance.library_id);
|
|
|
|
|
if (job.action !== 'create' && job.action !== 'start') {
|
|
|
|
|
// Fast controller check keeps occupied lobbies out of the operation slots.
|
|
|
|
|
// The agent repeats this immediately before its local mutation.
|
|
|
|
|
const health = await nativeRequest<{ state: string; healthy?: boolean }>(instance, 'instance.inspect');
|
|
|
|
|
if (!['stopped','exited','created','dead','missing'].includes(health.state)) {
|
|
|
|
|
const occupancy = await probePlayerEvidence(job.server);
|
|
|
|
|
const recoveredWatchdog = health.healthy && occupancy.players !== null && db().prepare(`SELECT 1 FROM provisioning_sources s
|
|
|
|
|
JOIN watchdog_incidents i ON i.id=s.source_id JOIN server_deployments d ON d.id=s.operation_id
|
|
|
|
|
WHERE s.operation_id=? AND s.kind='watchdog' AND i.kind='health' AND d.triggered_by LIKE 'watchdog:%'`).get(job.id);
|
|
|
|
|
if (recoveredWatchdog && occupancy.players !== null) { resolveRecoveredHealthOperation(job.id, occupancy.players, occupancy.source); return; }
|
|
|
|
|
if (occupancy.players !== 0) { hold(job, occupancy.players !== null ? `Waiting for ${occupancy.players} player(s) to leave.` : 'Waiting for a fresh local player count.'); return; }
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
const current = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined;
|
|
|
|
|
if (!current || !['queued','waiting','interrupted'].includes(current.state) || eligibility(job)) return;
|
|
|
|
|
db().transaction(() => {
|
|
|
|
|
const live = nativeInstance(job.server);
|
|
|
|
|
if (live?.id !== instance!.id || live.generation !== instance!.generation) throw new Error('Instance changed during preflight.');
|
|
|
|
|
if (job.action === 'destroy' || job.action === 'rebuild') {
|
|
|
|
|
const beta = activeBetaRemovalBlocker(job.server);
|
|
|
|
|
if (beta) { hold(job, beta); return; }
|
|
|
|
|
}
|
|
|
|
|
if (JSON.stringify(filesFor(job)) !== JSON.stringify(files)) {
|
|
|
|
|
finishDeployment(job.id, { ok: false, message: 'File requests changed during preflight. Review and submit a fresh activation.' }); return;
|
|
|
|
|
}
|
|
|
|
|
if (job.action === 'start' || job.action === 'restart' || job.action === 'rebuild') {
|
|
|
|
|
db().prepare(`INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES(?,?,?) ON CONFLICT(server)
|
|
|
|
|
DO UPDATE SET requested_at=excluded.requested_at,operation_id=excluded.operation_id,healthy_at=NULL,error=NULL`).run(job.server, new Date().toISOString(), job.id);
|
|
|
|
|
}
|
|
|
|
|
let payload: Record<string, unknown> = files.length ? { stageId: job.id } : {};
|
|
|
|
|
if (job.action === 'create' || job.action === 'rebuild') {
|
|
|
|
|
const prior = recipeForDeployment(job.id)!.recipe as NativeRecipe;
|
|
|
|
|
const recipe: NativeRecipe = { ...captureRecipe(prior.game, prior.mapSlug), native: prior.native, image: instance!.image };
|
|
|
|
|
payload = createPayload(instance!, recipe);
|
|
|
|
|
if (job.action === 'rebuild') payload.nextGeneration = instance!.generation + 1;
|
|
|
|
|
db().prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?').run(JSON.stringify(recipe), job.id);
|
|
|
|
|
if (job.action === 'create') db().prepare(`INSERT INTO servers(name,komodo_type,komodo_name,game,map,game_mode,map_slug,host_dir,config_file,game_id,image,game_port,created_by,container_state)
|
|
|
|
|
VALUES(?,'deployment','',?,?,'zombie',?,?,?,?,?,?,'native','starting')`).run(job.server, recipe.game, recipe.expectedMap, recipe.mapSlug, serverDir(job.server),
|
|
|
|
|
recipe.configFile, recipe.game === 't6' ? 't6zm' : 't5sp', recipe.image, instance!.port);
|
|
|
|
|
for (const file of recipe.files) recordScript(job.server, file.path, file.versionId, file.sha256, file.size, 'native-create');
|
|
|
|
|
}
|
|
|
|
|
const operation = queueHostOperation(instance!.host_id, `instance.${job.action === 'destroy' ? 'remove' : job.action}`, payload,
|
|
|
|
|
{ instanceId: instance!.id, generation: instance!.generation, idempotencyKey: job.id });
|
|
|
|
|
db().prepare('UPDATE native_provisioning SET operation_id=? WHERE deployment_id=?').run(operation.id, job.id);
|
|
|
|
|
for (const file of files) db().prepare("UPDATE staged_file_changes SET state='applying' WHERE id=? AND state='pending'").run(file.id);
|
|
|
|
|
db().prepare("UPDATE server_deployments SET state='running',submitted_at=? WHERE id=?").run(new Date().toISOString(), job.id);
|
|
|
|
|
deploymentStage(job.id, `Host agent ${job.action} queued`, { phase: 'container-action', blocker: null,
|
|
|
|
|
command: { state: 'attempted', message: `Durable agent operation ${operation.id}; reconciliation always uses this ID.` } });
|
|
|
|
|
})();
|
|
|
|
|
}
|
