Add native host management and role-based operator authentication
Introduce an outbound Python host agent, Debian installer, durable remote operations, immutable game-file libraries, host-aware deployments and ports, and occupancy-safe campaigns and draining. Native hosts do not require Komodo. Add administrator, operator and viewer accounts, protected management routes, secure sessions, password recovery, audit history and separate telemetry tokens. Document first-admin setup, reporter migration and live-host acceptance checks. Validation: 1,343 TypeScript tests and 39 agent tests passed, along with type checking, the production build, compiled SSR/API smoke tests and administrator recovery. Real Debian/Docker T5/T6 acceptance remains unrun.
This commit is contained in:
@@ -0,0 +1,404 @@
|
||||
import { randomBytes, randomUUID } from 'node:crypto';
|
||||
import { posix } from 'node:path';
|
||||
import { db } from '../db/index.js';
|
||||
import { config, relDestForKind, serverDir } from '../config.js';
|
||||
import { nativeInstance, nativeRequest, type NativeInstance } from '../backends/native.js';
|
||||
import { getHost, getHostOperation, listHosts, nextHostPort, queueHostOperation, releaseHostPorts, requireReadyHost, reserveHostPort } from '../hosts/index.js';
|
||||
import { getLibraryVersion, requireReadyLibrary } from '../game-files/index.js';
|
||||
import { blueprintRecipe, captureRecipe, recipeForDeployment, summarizeRecipe, type LaunchRecipe } from '../fleet/launch-recipes.js';
|
||||
import { getServerKey, saveServerKey } from '../fleet/server-keys.js';
|
||||
import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
|
||||
import { nextLobbyNumber, type CreateServerInput, type CreateServerResult } from './instances.js';
|
||||
import { beginDeployment, deploymentStage, finishDeployment, REMOVING_SERVER } from './history.js';
|
||||
import { recordProgress } from './progress.js';
|
||||
import { fleetSettings } from '../settings.js';
|
||||
import { resolveRecoveredHealthOperation } from '../maintenance/recovery.js';
|
||||
import { probeServerStatus, probePlayerEvidence } from '../fleet/status-evidence.js';
|
||||
import { checkRelease, freshRelease, probeVersion } from '../integrations/plutonium.js';
|
||||
import { getServerScript, recordScript, removeScript } from '../fleet/inventory.js';
|
||||
import { sha256Buffer } from '../fleet/files.js';
|
||||
import type { StagedFileChange } from '../fleet/staged-files.js';
|
||||
import type { LaunchPreview } from '../../deployment-plan.js';
|
||||
import type { ProvisioningAction } from '../../provisioning.js';
|
||||
|
||||
export interface NativeSelection { hostId: string; libraryId: string; image?: string; bindAddress?: string; publicAddress?: string; publicPort?: number }
|
||||
type NativeRecipe = LaunchRecipe & { native?: NativeSelection };
|
||||
interface Job { id: string; server: string; action: ProvisioningAction; state: string; submitted_at: string | null; stage: string;
|
||||
instance_id: string; generation: number; stage_operation_id: string | null; operation_id: string | null; files_json: string | null; applied_at: string | null }
|
||||
interface FilePlan { id: number; path: string; contentBase64: string | null; sha256: string | null; checkBefore: boolean; expectedBeforeSha: string | null }
|
||||
|
||||
export function nativeSelection(input: Record<string, unknown>, game: string): NativeSelection {
|
||||
if (typeof input.hostId !== 'string' || !input.hostId) throw new Error('Select a ready host.');
|
||||
if (typeof input.libraryId !== 'string' || !input.libraryId) throw new Error('Select a published game-file version.');
|
||||
const version = getLibraryVersion(input.libraryId);
|
||||
if (!version || version.game !== game || version.state !== 'ready') throw new Error('Choose a published game-file version for the selected game.');
|
||||
const image = input.image === undefined || input.image === '' ? config.komodo.defaultImage : input.image;
|
||||
if (typeof image !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9._/:@-]{0,499}$/.test(image)) throw new Error('Invalid runtime image.');
|
||||
const host = requireReadyHost(input.hostId, { image, requiredBytes: 1024 ** 3 });
|
||||
requireReadyLibrary(host.id, version.id);
|
||||
const bindAddress = input.bindAddress === undefined || input.bindAddress === '' ? host.bindAddress : input.bindAddress;
|
||||
if (typeof bindAddress !== 'string') throw new Error('Invalid bind address.');
|
||||
if (input.publicAddress !== undefined && (typeof input.publicAddress !== 'string' || input.publicAddress.length > 253)) throw new Error('Invalid public address.');
|
||||
if (input.publicPort !== undefined && (!Number.isInteger(input.publicPort) || Number(input.publicPort) < 1 || Number(input.publicPort) > 65535)) throw new Error('Invalid public port.');
|
||||
return { hostId: host.id, libraryId: version.id, image, bindAddress, publicAddress: input.publicAddress as string | undefined ?? host.publicAddress,
|
||||
publicPort: input.publicPort as number | undefined };
|
||||
}
|
||||
|
||||
export function previewNativeCreation(input: Record<string, unknown>): LaunchPreview {
|
||||
let recipe: NativeRecipe;
|
||||
let blueprintName: string | null = null;
|
||||
if (typeof input.deploymentId === 'string') {
|
||||
const previous = recipeForDeployment(input.deploymentId);
|
||||
if (!previous) throw new Error('The recorded deployment is unavailable.');
|
||||
recipe = captureRecipe(previous.recipe.game, previous.recipe.mapSlug);
|
||||
blueprintName = previous.blueprintName;
|
||||
} else if (typeof input.blueprintId === 'string') ({ recipe, blueprintName } = blueprintRecipe(input.blueprintId));
|
||||
else {
|
||||
if ((input.game !== 't5' && input.game !== 't6') || typeof input.mapSlug !== 'string') throw new Error('Choose a supported game and map.');
|
||||
recipe = captureRecipe(input.game, input.mapSlug);
|
||||
}
|
||||
const selection = nativeSelection(input, recipe.game);
|
||||
recipe.native = selection; recipe.image = selection.image!;
|
||||
const base = `${recipe.game}_zombie_${recipe.mapSlug}`;
|
||||
const lobby = nextLobbyNumber(base);
|
||||
const name = lobby === null ? null : `${base}_${lobby}`;
|
||||
const port = nextNativePort(selection);
|
||||
const id = randomUUID(); const expiresAt = new Date(Date.now() + 30 * 60_000).toISOString();
|
||||
db().prepare('INSERT INTO server_launch_previews(id,expires_at,server,port,recipe,blueprint_name) VALUES(?,?,?,?,?,?)')
|
||||
.run(id, expiresAt, name, port, JSON.stringify(recipe), blueprintName);
|
||||
const checks: LaunchPreview['checks'] = [
|
||||
{ label: 'Host readiness', state: 'pass', detail: `${getHost(selection.hostId)!.name}: connected, Docker ready, storage and runtime verified.` },
|
||||
{ label: 'Game files', state: 'pass', detail: `Immutable library version ${selection.libraryId} is ready on the selected host.` },
|
||||
{ label: 'Lobby', state: name ? 'pass' : 'blocked', detail: name ?? 'All lobby IDs for this map are reserved.' },
|
||||
{ label: 'Host port', state: port ? 'pass' : 'blocked', detail: port ? `${selection.bindAddress}:${port} / UDP and TCP` : 'No available UDP/TCP port on this host.' },
|
||||
{ label: 'Configuration', state: 'pass', detail: recipe.configSource }
|
||||
];
|
||||
return { id, expiresAt, name, port, hasSavedKey: !!name && !!getServerKey(name), recipe: summarizeRecipe(recipe), blueprintName, checks,
|
||||
ready: checks.every(check => check.state !== 'blocked') };
|
||||
}
|
||||
|
||||
export function nextNativePort(selection: NativeSelection, taken: number[] = []): number | null {
|
||||
const excluded = [...taken];
|
||||
for (;;) {
|
||||
const port = nextHostPort(selection.hostId, excluded, selection.bindAddress, 'udp');
|
||||
if (port === null) return null;
|
||||
// Both protocols are published by Plutainer and reserved together.
|
||||
const tcp = nextHostPort(selection.hostId, excluded, selection.bindAddress, 'tcp');
|
||||
if (tcp === port) return port;
|
||||
if (tcp !== null && tcp < port) { excluded.push(tcp); continue; }
|
||||
excluded.push(port);
|
||||
}
|
||||
}
|
||||
|
||||
export function queueNativeCreation(input: CreateServerInput): CreateServerResult {
|
||||
try {
|
||||
return db().transaction(() => {
|
||||
const preview = input.previewId ? db().prepare('SELECT * FROM server_launch_previews WHERE id=?').get(input.previewId) as
|
||||
{ id: string; recipe: string; server: string; port: number; expires_at: string; deployment_id: string | null; blueprint_name: string | null } | undefined : undefined;
|
||||
if (input.previewId && !preview) throw new Error('Review a fresh launch preview.');
|
||||
if (preview?.deployment_id) return { ok: true as const, name: preview.server, port: preview.port, deploymentId: preview.deployment_id };
|
||||
if (preview && preview.expires_at <= new Date().toISOString()) throw new Error('Launch preview expired.');
|
||||
const saved = preview ? JSON.parse(preview.recipe) as NativeRecipe : undefined;
|
||||
const game = input.game ?? 't5';
|
||||
if (input.gameMode !== 'zombie' || (saved && (saved.game !== game || saved.mapSlug !== input.mapSlug))) throw new Error('Selection changed. Review a fresh launch preview.');
|
||||
const selection = nativeSelection((saved?.native ?? input) as unknown as Record<string, unknown>, game);
|
||||
const recipe: NativeRecipe = { ...captureRecipe(game, input.mapSlug), native: selection, image: selection.image! };
|
||||
const lobby = nextLobbyNumber(`${game}_zombie_${input.mapSlug}`);
|
||||
if (lobby === null) throw new Error('All lobby IDs are reserved.');
|
||||
const name = `${game}_zombie_${input.mapSlug}_${lobby}`;
|
||||
const port = nextNativePort(selection);
|
||||
if (!port) throw new Error('No available host port.');
|
||||
if ((input.expectedName && input.expectedName !== name) || (preview && (preview.server !== name || preview.port !== port))) throw new Error('Available lobby or host port changed. Review a fresh preview.');
|
||||
const key = input.plutoKey?.trim() || getServerKey(name);
|
||||
if (!key) throw new Error(`Save a Plutonium server key for ${name}.`);
|
||||
saveServerKey(name, key);
|
||||
const instanceId = randomUUID();
|
||||
reserveHostPort(selection.hostId, instanceId, { port, bindAddress: selection.bindAddress, protocol: 'udp' });
|
||||
reserveHostPort(selection.hostId, instanceId, { port, bindAddress: selection.bindAddress, protocol: 'tcp' });
|
||||
db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,bind_address,port,public_address,public_port,created_at)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?)`).run(instanceId, name, selection.hostId, selection.libraryId, selection.image, selection.bindAddress, port,
|
||||
selection.publicAddress ?? null, selection.publicPort ?? null, new Date().toISOString());
|
||||
const id = beginDeployment(name, 'create', input.createdBy);
|
||||
db().prepare("UPDATE server_deployments SET state='queued' WHERE id=?").run(id);
|
||||
db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,?,?,?,?)')
|
||||
.run(id, name, port, 'zombie', recipe.mapSlug, recipe.expectedMap);
|
||||
db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,blueprint_name) VALUES(?,?,?)').run(id, JSON.stringify(recipe), preview?.blueprint_name ?? null);
|
||||
if (preview) db().prepare('UPDATE server_launch_previews SET deployment_id=? WHERE id=?').run(id, preview.id);
|
||||
deploymentStage(id, 'Queued for native host deployment', { phase: 'startup-queue', recipeStatus: 'queued' });
|
||||
return { ok: true as const, name, port, deploymentId: id };
|
||||
})();
|
||||
} catch (error) { return { ok: false, kind: 'bad-request', message: (error as Error).message }; }
|
||||
}
|
||||
|
||||
export function bindNativeOperation(id: string, server: string): void {
|
||||
const instance = nativeInstance(server);
|
||||
if (instance) db().prepare('INSERT OR IGNORE INTO native_provisioning(deployment_id,instance_id,generation) VALUES(?,?,?)').run(id, instance.id, instance.generation);
|
||||
}
|
||||
|
||||
/** Cancelling an unsubmitted launch releases only its local reservation. */
|
||||
export function cancelQueuedNativeCreation(server: string, triggeredBy: string): string | null {
|
||||
const instance = nativeInstance(server);
|
||||
if (!instance || getServer(server)) return null;
|
||||
return db().transaction(() => {
|
||||
const creation = db().prepare(`SELECT d.id FROM server_deployments d JOIN native_provisioning n ON n.deployment_id=d.id
|
||||
WHERE n.instance_id=? AND d.action='create' AND d.state='queued' AND n.operation_id IS NULL`).get(instance.id) as { id: string } | undefined;
|
||||
if (!creation) return null;
|
||||
const id = beginDeployment(server, 'destroy', triggeredBy);
|
||||
const now = new Date().toISOString();
|
||||
db().prepare("UPDATE server_deployments SET state='cancelled',finished_at=?,stage='Queued native creation cancelled',server_deleted_at=? WHERE id=?")
|
||||
.run(now, now, creation.id);
|
||||
db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(now, instance.id);
|
||||
releaseHostPorts(instance.id);
|
||||
deploymentStage(id, 'Cancelled queued creation; no agent mutation was submitted');
|
||||
finishDeployment(id, { ok: true });
|
||||
return id;
|
||||
})();
|
||||
}
|
||||
|
||||
export function queueNativeRebuild(name: string, triggeredBy: string): CreateServerResult {
|
||||
try {
|
||||
const instance = nativeInstance(name)!; const server = getServer(name);
|
||||
const blocker = activeBetaRemovalBlocker(name);
|
||||
if (blocker) throw new Error(blocker);
|
||||
if (!server?.map_slug || !getServerKey(name)) throw new Error('A saved map and server key are required before rebuilding.');
|
||||
requireReadyLibrary(instance.host_id, instance.library_id);
|
||||
const recipe: NativeRecipe = { ...captureRecipe(server.game as 't5' | 't6', server.map_slug), image: instance.image,
|
||||
native: { hostId: instance.host_id, libraryId: instance.library_id, image: instance.image, bindAddress: instance.bind_address } };
|
||||
return db().transaction(() => {
|
||||
const id = beginDeployment(name, 'rebuild', triggeredBy);
|
||||
db().prepare("UPDATE server_deployments SET state='waiting' WHERE id=?").run(id);
|
||||
db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,?,?,?,?)')
|
||||
.run(id, name, instance.port, 'zombie', recipe.mapSlug, recipe.expectedMap);
|
||||
db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe) VALUES(?,?)').run(id, JSON.stringify(recipe));
|
||||
deploymentStage(id, 'Waiting to rebuild when empty', { phase: 'eligibility', recipeStatus: 'queued' });
|
||||
return { ok: true as const, name, port: instance.port, deploymentId: id };
|
||||
})();
|
||||
} catch (error) { return { ok: false, kind: 'bad-request', message: (error as Error).message }; }
|
||||
}
|
||||
|
||||
function filesFor(job: Job): FilePlan[] {
|
||||
const rows = db().prepare(`SELECT f.* FROM staged_file_changes f JOIN provisioning_sources s
|
||||
ON s.kind=f.source_kind AND s.source_id=f.source_id AND s.server=f.server
|
||||
WHERE s.operation_id=? AND f.server=? AND f.state='pending' ORDER BY f.id`).all(job.id, job.server) as StagedFileChange[];
|
||||
return rows.map(row => ({ id: row.id, path: posix.join(relDestForKind(row.kind, job.server.startsWith('t6_') ? 't6' : 't5'), row.rel_path),
|
||||
contentBase64: row.content?.toString('base64') ?? null, sha256: row.content ? sha256Buffer(row.content) : null,
|
||||
checkBefore: !!row.check_before, expectedBeforeSha: row.expected_before_sha }));
|
||||
}
|
||||
|
||||
/** Agent receipts are the authoritative before images; do not read a controller mount. */
|
||||
function applyReceipt(job: Job, result: Record<string, unknown>) {
|
||||
if (job.applied_at || !job.files_json) return;
|
||||
const plans = JSON.parse(job.files_json) as FilePlan[];
|
||||
const before = result.before as { id?: number; path: string; contentBase64: string | null }[] | undefined;
|
||||
db().transaction(() => {
|
||||
for (const plan of plans) {
|
||||
const snapshot = before?.find(item => item.id === plan.id || item.path === plan.path);
|
||||
if (!snapshot) throw new Error('Agent activation receipt is missing a before image.');
|
||||
const row = db().prepare('SELECT * FROM staged_file_changes WHERE id=?').get(plan.id) as StagedFileChange;
|
||||
if (!row || row.state === 'applied') continue;
|
||||
const bytes = snapshot.contentBase64 === null ? null : Buffer.from(snapshot.contentBase64, 'base64');
|
||||
const prior = getServerScript(job.server, row.rel_path);
|
||||
if (row.source_kind === 'beta') {
|
||||
if (bytes !== null) db().prepare(`INSERT OR IGNORE INTO beta_file_snapshots(beta_id,server,rel_path,prior_content,prior_sha256,prior_size) VALUES(?,?,?,?,?,?)`)
|
||||
.run(row.source_id, job.server, row.rel_path, bytes, sha256Buffer(bytes), bytes.length);
|
||||
db().prepare(`UPDATE beta_server_scripts SET had_original=?,prior_version_id=? WHERE beta_id=? AND server=?
|
||||
AND script_id IN (SELECT id FROM scripts WHERE target_rel_path=? AND kind=?)`).run(bytes === null ? 0 : 1, prior?.script_version_id ?? null, row.source_id, job.server, row.rel_path, row.kind);
|
||||
db().prepare('INSERT OR IGNORE INTO beta_file_changes(beta_id,server,rel_path) VALUES(?,?,?)').run(row.source_id, job.server, row.rel_path);
|
||||
db().prepare('UPDATE beta_servers SET deployed_at=?,updated_at=? WHERE beta_id=? AND server=?').run(new Date().toISOString(), new Date().toISOString(), row.source_id, job.server);
|
||||
}
|
||||
if (row.deployment_file_id && !row.preserve_before_image) db().prepare('UPDATE deployment_files SET verb=?,content_before=?,sha256_before=? WHERE id=?')
|
||||
.run(row.content === null ? 'removed' : bytes === null ? 'added' : 'updated', bytes, bytes === null ? null : sha256Buffer(bytes), row.deployment_file_id);
|
||||
if (row.content === null) removeScript(job.server, row.rel_path);
|
||||
else recordScript(job.server, row.rel_path, row.script_version_id && db().prepare('SELECT 1 FROM script_versions WHERE id=?').get(row.script_version_id) ? row.script_version_id : null,
|
||||
sha256Buffer(row.content), row.content.length, row.inventory_source);
|
||||
db().prepare("UPDATE staged_file_changes SET state='applied',applied_at=? WHERE id=?").run(new Date().toISOString(), row.id);
|
||||
}
|
||||
db().prepare('UPDATE native_provisioning SET applied_at=? WHERE deployment_id=?').run(new Date().toISOString(), job.id);
|
||||
})();
|
||||
}
|
||||
|
||||
function createPayload(instance: NativeInstance, recipe: LaunchRecipe) {
|
||||
const key = getServerKey(instance.server);
|
||||
if (!key) throw new Error('The saved server key was removed.');
|
||||
return { name: instance.server, game: recipe.game, libraryId: instance.library_id, image: instance.image, port: instance.port, bindAddress: instance.bind_address,
|
||||
rconPassword: config.rcon.password || randomBytes(32).toString('hex'),
|
||||
env: { PLUTAINER_SERVER_NAME: instance.server, PLUTAINER_PORT: String(instance.port), PLUTAINER_GAME: recipe.game === 't6' ? 't6zm' : 't5sp',
|
||||
PLUTAINER_CONFIG_FILE: recipe.configFile, PLUTAINER_MAP_ROTATE: 'false', PLUTAINER_EXTRA_ARGS: `+exec electric_cherry_map.cfg +set tws_server_id ${instance.server} +map_rotate`, PLUTO_SERVER_KEY: key },
|
||||
files: recipe.files.map(file => ({ path: posix.join(relDestForKind(file.kind, recipe.game), file.path), contentBase64: file.content, sha256: file.sha256 })) };
|
||||
}
|
||||
|
||||
function hold(job: Job, message: string) { deploymentStage(job.id, message, { blocker: { code: 'native-host', message, automatic: true } }); }
|
||||
function complete(job: Job) {
|
||||
db().transaction(() => {
|
||||
db().prepare('UPDATE native_provisioning SET verified_at=? WHERE deployment_id=?').run(new Date().toISOString(), job.id);
|
||||
db().prepare('UPDATE restart_rollout SET healthy_at=?,error=NULL WHERE operation_id=?').run(new Date().toISOString(), job.id);
|
||||
finishDeployment(job.id, { ok: true });
|
||||
})();
|
||||
}
|
||||
|
||||
export async function processNativeProvisioning(eligibility: (operation: { id: string; server: string; action: string; state: string; stage: string }) => string | null): Promise<void> {
|
||||
// Draining prevents admission immediately and retires existing lobbies only through the normal safety queue.
|
||||
for (const host of listHosts().filter(host => host.mode === 'draining' && host.connected)) {
|
||||
const instances = db().prepare('SELECT * FROM native_instances WHERE host_id=? AND retired_at IS NULL').all(host.id) as NativeInstance[];
|
||||
for (const instance of instances) {
|
||||
if (!getServer(instance.server) || activeBetaRemovalBlocker(instance.server)) continue;
|
||||
if (db().prepare("SELECT 1 FROM server_deployments WHERE server=? AND server_deleted_at IS NULL AND state IN ('queued','waiting','running','awaiting-health')").get(instance.server)) continue;
|
||||
const id = beginDeployment(instance.server, 'destroy', 'host-drain');
|
||||
db().prepare("UPDATE server_deployments SET state='waiting' WHERE id=?").run(id);
|
||||
deploymentStage(id, 'Host draining; waiting for a fresh empty lobby');
|
||||
}
|
||||
}
|
||||
const jobs = db().prepare(`SELECT d.*,n.instance_id,n.generation,n.stage_operation_id,n.operation_id,n.files_json,n.applied_at
|
||||
FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
|
||||
WHERE d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') ORDER BY d.started_at,d.rowid`).all() as Job[];
|
||||
for (const job of jobs) {
|
||||
try { await processJob(job, eligibility); }
|
||||
catch (error) { hold(job, (error as Error).message); }
|
||||
}
|
||||
}
|
||||
|
||||
async function processJob(job: Job, eligibility: (job: Job) => string | null) {
|
||||
let instance = nativeInstance(job.server);
|
||||
if (!instance || instance.id !== job.instance_id || (instance.generation !== job.generation && !(job.action === 'rebuild' && instance.generation === job.generation + 1))) {
|
||||
finishDeployment(job.id, { ok: false, message: 'Original instance was retired or replaced; no command was sent.' }); return;
|
||||
}
|
||||
const host = getHost(instance.host_id);
|
||||
if (!host?.connected || host.mode === 'disabled') {
|
||||
db().prepare("UPDATE servers SET container_state='unknown',last_players=NULL,last_roster=NULL WHERE name=?").run(job.server);
|
||||
hold(job, 'Host is unavailable; existing server state is unknown. Waiting for reconnection.'); return;
|
||||
}
|
||||
if (job.operation_id) {
|
||||
const operation = getHostOperation(job.operation_id);
|
||||
if (!operation || operation.state === 'unknown') { hold(job, 'Agent result is unknown. Reconciling the original operation; no replacement or repeated command will be sent.'); return; }
|
||||
if (operation.state === 'failed') {
|
||||
// The agent reports failed only when no mutation began. Ambiguous
|
||||
// post-mutation failures remain unknown and retain their capacity.
|
||||
db().transaction(() => {
|
||||
db().prepare('DELETE FROM restart_rollout WHERE operation_id=?').run(job.id);
|
||||
const files = JSON.parse(job.files_json ?? '[]') as FilePlan[];
|
||||
for (const file of files) db().prepare("UPDATE staged_file_changes SET state='pending' WHERE id=? AND state='applying'").run(file.id);
|
||||
finishDeployment(job.id, { ok: false, message: operation.error ?? 'Agent operation failed before mutation.' });
|
||||
})();
|
||||
return;
|
||||
}
|
||||
if (operation.state !== 'completed') {
|
||||
const blocked = operation.result as { message?: string } | null;
|
||||
hold(job, blocked?.message ?? `Agent operation ${operation.state}`); return;
|
||||
}
|
||||
if (job.action === 'destroy') {
|
||||
db().transaction(() => {
|
||||
deploymentStage(job.id, REMOVING_SERVER, { checkpoint: { key: 'delete-deployment', label: 'Agent confirmed instance and container removal' } });
|
||||
db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(new Date().toISOString(), instance!.id);
|
||||
releaseHostPorts(instance!.id);
|
||||
db().prepare('DELETE FROM servers WHERE name=?').run(job.server);
|
||||
complete(job);
|
||||
})();
|
||||
return;
|
||||
}
|
||||
if (job.action === 'rebuild' && instance.generation === job.generation) {
|
||||
db().prepare('UPDATE native_instances SET generation=? WHERE id=? AND generation=?').run(job.generation + 1, instance.id, job.generation);
|
||||
instance = nativeInstance(job.server)!;
|
||||
}
|
||||
applyReceipt(job, (operation.result ?? {}) as Record<string, unknown>);
|
||||
if (job.state !== 'awaiting-health') {
|
||||
db().prepare("UPDATE server_deployments SET state='awaiting-health',error=NULL,finished_at=NULL WHERE id=?").run(job.id);
|
||||
recordProgress(job.id, { command: { state: 'accepted', message: `Agent confirmed ${job.action}; verifying the resulting instance.` }, phase: 'verify-container' });
|
||||
}
|
||||
const health = await nativeRequest<{ healthy: boolean; state: string }>(instance, 'instance.inspect');
|
||||
if (job.action === 'stop') {
|
||||
if (!['stopped','exited','created','dead'].includes(health.state)) { hold(job, 'Waiting for the agent to verify a stopped container.'); return; }
|
||||
db().prepare("UPDATE servers SET container_state=?,last_players=NULL,last_roster=NULL WHERE name=?").run(health.state, job.server);
|
||||
complete(job); return;
|
||||
}
|
||||
if (!health.healthy) { hold(job, `Waiting for container readiness: ${health.state}`); return; }
|
||||
const status = await probeServerStatus(job.server, operation.updatedAt);
|
||||
if (!status.ok) { hold(job, `Waiting for game status: ${status.reason}`); return; }
|
||||
if (job.action === 'create' || job.action === 'rebuild') {
|
||||
const recipe = recipeForDeployment(job.id)!.recipe;
|
||||
if (status.map !== recipe.expectedMap) { hold(job, `Waiting for selected map ${recipe.expectedMap}; observed ${status.map ?? 'unknown'}.`); return; }
|
||||
await checkRelease();
|
||||
const revision = await probeVersion(job.server); const target = freshRelease();
|
||||
if (revision === null || target === null) { hold(job, 'Waiting to verify the Plutonium runtime revision.'); return; }
|
||||
if (revision < target) { finishDeployment(job.id, { ok: false, message: `Running Plutonium r${revision} is older than production r${target}.` }); return; }
|
||||
}
|
||||
db().prepare("UPDATE servers SET container_state='running',container_checked_at=? WHERE name=?").run(new Date().toISOString(), job.server);
|
||||
recordProgress(job.id, { blocker: null, checks: [{ key: 'container', label: 'Container readiness', source: 'agent', result: 'pass', actual: health.state },
|
||||
{ key: 'game', label: 'Game status', source: status.source, result: 'pass', actual: status.players }] });
|
||||
complete(job); return;
|
||||
}
|
||||
const reason = eligibility(job);
|
||||
if (reason) { hold(job, reason); return; }
|
||||
if (job.action === 'destroy' || job.action === 'rebuild') {
|
||||
const beta = activeBetaRemovalBlocker(job.server); if (beta) { hold(job, beta); return; }
|
||||
}
|
||||
const files = job.files_json ? JSON.parse(job.files_json) as FilePlan[] : filesFor(job);
|
||||
if (files.length && !job.stage_operation_id) {
|
||||
const stage = queueHostOperation(instance.host_id, 'files.stage', { stageId: job.id, files }, { instanceId: instance.id, generation: instance.generation, idempotencyKey: `${job.id}:stage` });
|
||||
db().prepare('UPDATE native_provisioning SET stage_operation_id=?,files_json=? WHERE deployment_id=?').run(stage.id, JSON.stringify(files), job.id);
|
||||
hold(job, 'Staging and verifying files on the host.'); return;
|
||||
}
|
||||
if (job.stage_operation_id) {
|
||||
const stage = getHostOperation(job.stage_operation_id);
|
||||
if (stage?.state === 'failed') { finishDeployment(job.id, { ok: false, message: stage.error ?? 'File staging failed.' }); return; }
|
||||
if (stage?.state !== 'completed') { hold(job, 'Waiting for verified remote file staging.'); return; }
|
||||
if (files.some(file => (db().prepare('SELECT state FROM staged_file_changes WHERE id=?').get(file.id) as { state: string } | undefined)?.state !== 'pending')) {
|
||||
finishDeployment(job.id, { ok: false, message: 'Staged file request was cancelled or changed before activation.' }); return;
|
||||
}
|
||||
}
|
||||
const settings = fleetSettings();
|
||||
const count = (actions: string[]) => (db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id != ? AND server_deleted_at IS NULL
|
||||
AND action IN (${actions.map(() => '?').join(',')}) AND state IN ('running','awaiting-health')`).get(job.id, ...actions) as { n: number }).n;
|
||||
if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) >= settings.createConcurrency) { hold(job, 'Waiting for startup capacity.'); return; }
|
||||
if (job.action === 'destroy' && count(['destroy']) >= settings.destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; }
|
||||
if (job.action === 'create') requireReadyHost(instance.host_id, { image: instance.image, requiredBytes: 1024 ** 3 });
|
||||
if (job.action === 'create' || job.action === 'rebuild') requireReadyLibrary(instance.host_id, instance.library_id);
|
||||
if (job.action !== 'create' && job.action !== 'start') {
|
||||
// Fast controller check keeps occupied lobbies out of the operation slots.
|
||||
// The agent repeats this immediately before its local mutation.
|
||||
const health = await nativeRequest<{ state: string; healthy?: boolean }>(instance, 'instance.inspect');
|
||||
if (!['stopped','exited','created','dead','missing'].includes(health.state)) {
|
||||
const occupancy = await probePlayerEvidence(job.server);
|
||||
const recoveredWatchdog = health.healthy && occupancy.players !== null && db().prepare(`SELECT 1 FROM provisioning_sources s
|
||||
JOIN watchdog_incidents i ON i.id=s.source_id JOIN server_deployments d ON d.id=s.operation_id
|
||||
WHERE s.operation_id=? AND s.kind='watchdog' AND i.kind='health' AND d.triggered_by LIKE 'watchdog:%'`).get(job.id);
|
||||
if (recoveredWatchdog && occupancy.players !== null) { resolveRecoveredHealthOperation(job.id, occupancy.players, occupancy.source); return; }
|
||||
if (occupancy.players !== 0) { hold(job, occupancy.players !== null ? `Waiting for ${occupancy.players} player(s) to leave.` : 'Waiting for a fresh local player count.'); return; }
|
||||
}
|
||||
}
|
||||
const current = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined;
|
||||
if (!current || !['queued','waiting','interrupted'].includes(current.state) || eligibility(job)) return;
|
||||
db().transaction(() => {
|
||||
const live = nativeInstance(job.server);
|
||||
if (live?.id !== instance!.id || live.generation !== instance!.generation) throw new Error('Instance changed during preflight.');
|
||||
if (job.action === 'destroy' || job.action === 'rebuild') {
|
||||
const beta = activeBetaRemovalBlocker(job.server);
|
||||
if (beta) { hold(job, beta); return; }
|
||||
}
|
||||
if (JSON.stringify(filesFor(job)) !== JSON.stringify(files)) {
|
||||
finishDeployment(job.id, { ok: false, message: 'File requests changed during preflight. Review and submit a fresh activation.' }); return;
|
||||
}
|
||||
if (job.action === 'start' || job.action === 'restart' || job.action === 'rebuild') {
|
||||
db().prepare(`INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES(?,?,?) ON CONFLICT(server)
|
||||
DO UPDATE SET requested_at=excluded.requested_at,operation_id=excluded.operation_id,healthy_at=NULL,error=NULL`).run(job.server, new Date().toISOString(), job.id);
|
||||
}
|
||||
let payload: Record<string, unknown> = files.length ? { stageId: job.id } : {};
|
||||
if (job.action === 'create' || job.action === 'rebuild') {
|
||||
const prior = recipeForDeployment(job.id)!.recipe as NativeRecipe;
|
||||
const recipe: NativeRecipe = { ...captureRecipe(prior.game, prior.mapSlug), native: prior.native, image: instance!.image };
|
||||
payload = createPayload(instance!, recipe);
|
||||
if (job.action === 'rebuild') payload.nextGeneration = instance!.generation + 1;
|
||||
db().prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?').run(JSON.stringify(recipe), job.id);
|
||||
if (job.action === 'create') db().prepare(`INSERT INTO servers(name,komodo_type,komodo_name,game,map,game_mode,map_slug,host_dir,config_file,game_id,image,game_port,created_by,container_state)
|
||||
VALUES(?,'deployment','',?,?,'zombie',?,?,?,?,?,?,'native','starting')`).run(job.server, recipe.game, recipe.expectedMap, recipe.mapSlug, serverDir(job.server),
|
||||
recipe.configFile, recipe.game === 't6' ? 't6zm' : 't5sp', recipe.image, instance!.port);
|
||||
for (const file of recipe.files) recordScript(job.server, file.path, file.versionId, file.sha256, file.size, 'native-create');
|
||||
}
|
||||
const operation = queueHostOperation(instance!.host_id, `instance.${job.action === 'destroy' ? 'remove' : job.action}`, payload,
|
||||
{ instanceId: instance!.id, generation: instance!.generation, idempotencyKey: job.id });
|
||||
db().prepare('UPDATE native_provisioning SET operation_id=? WHERE deployment_id=?').run(operation.id, job.id);
|
||||
for (const file of files) db().prepare("UPDATE staged_file_changes SET state='applying' WHERE id=? AND state='pending'").run(file.id);
|
||||
db().prepare("UPDATE server_deployments SET state='running',submitted_at=? WHERE id=?").run(new Date().toISOString(), job.id);
|
||||
deploymentStage(job.id, `Host agent ${job.action} queued`, { phase: 'container-action', blocker: null,
|
||||
command: { state: 'attempted', message: `Durable agent operation ${operation.id}; reconciliation always uses this ID.` } });
|
||||
})();
|
||||
}
|
||||
Reference in New Issue
Block a user