Keep native collection launches responsive

This commit is contained in:
2026-10-05 10:48:38 -04:00
parent dbd2940d29
commit 1405114a14
26 changed files with 998 additions and 123 deletions
+4 -3
View File
@@ -151,9 +151,10 @@ npm run build
``` ```
Builds regenerate the downloadable agent, standalone updater and checksum manifest, Builds regenerate the downloadable agent, standalone updater and checksum manifest,
and bundle the metrics/leaderboard workers under `build/reports`. `npm run dev` and bundle the metrics/leaderboard workers under `build/reports` and native launch
and `npm test` compile those workers automatically. After editing a report preparation under `build/launch`. `npm run dev` and `npm test` compile those workers
implementation during development, restart `npm run dev` to rebuild its worker. automatically. After editing a report or launch-preparation implementation during
development, restart `npm run dev` to rebuild its worker.
Reports use separate read-only SQLite connections and bounded revision-aware caches; Reports use separate read-only SQLite connections and bounded revision-aware caches;
metrics snapshots can be reused for up to 30 seconds when their inputs are unchanged. metrics snapshots can be reused for up to 30 seconds when their inputs are unchanged.
+4
View File
@@ -167,6 +167,10 @@ This applies to collection collections, single-server launches, and **Launch aga
Previews and queued jobs store file metadata without copying script or plugin Previews and queued jobs store file metadata without copying script or plugin
contents for every lobby. Provisioning captures the bytes when each job starts. contents for every lobby. Provisioning captures the bytes when each job starts.
Native launch preparation reads files and encodes command payloads in a worker
thread. The controller rechecks capacity, promotions, credentials, and safety
before submitting those commands. Agent delivery streams the stored payloads in
small chunks, while progress polling reads metadata without loading file bytes.
The files selected at provisioning, their hashes and version metadata, composed The files selected at provisioning, their hashes and version metadata, composed
configuration, selected map, and image reference are recorded in SQLite for configuration, selected map, and image reference are recorded in SQLite for
+5 -4
View File
@@ -6,7 +6,7 @@
"type": "module", "type": "module",
"scripts": { "scripts": {
"dev": "vite dev", "dev": "vite dev",
"build": "vite build && npm run build:reports -- --outDir build/reports", "build": "vite build && npm run build:reports -- --outDir build/reports && npm run build:launch -- --outDir build/launch",
"preview": "vite preview", "preview": "vite preview",
"check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json", "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
"test": "vitest run", "test": "vitest run",
@@ -17,9 +17,10 @@
"prebuild": "npm run agent:package", "prebuild": "npm run agent:package",
"test:agent": "python3 -m unittest discover -s agent/tests -v", "test:agent": "python3 -m unittest discover -s agent/tests -v",
"build:reports": "vite build --config vite.reports.config.ts", "build:reports": "vite build --config vite.reports.config.ts",
"predev": "npm run build:reports", "build:launch": "vite build --config vite.launch.config.ts",
"pretest": "npm run build:reports", "predev": "npm run build:reports && npm run build:launch",
"pretest:watch": "npm run build:reports" "pretest": "npm run build:reports && npm run build:launch",
"pretest:watch": "npm run build:reports && npm run build:launch"
}, },
"dependencies": { "dependencies": {
"@fontsource/alfa-slab-one": "^5.3.0", "@fontsource/alfa-slab-one": "^5.3.0",
+2 -2
View File
@@ -1,6 +1,6 @@
import { randomUUID } from 'node:crypto'; import { randomUUID } from 'node:crypto';
import { db } from '../db/index.js'; import { db } from '../db/index.js';
import { getHost, queueHostOperation, waitHostOperation } from '../hosts/index.js'; import { getHost, queueHostOperation, waitHostOperationStatus } from '../hosts/index.js';
import { trackActivity } from '../pipeline/activity.js'; import { trackActivity } from '../pipeline/activity.js';
import { latestGameSubmission } from '../fleet/lifecycle-evidence.js'; import { latestGameSubmission } from '../fleet/lifecycle-evidence.js';
@@ -43,7 +43,7 @@ async function executeRequest<T>(instance: NativeInstance, kind: string, payload
instanceId: instance.id, generation: instance.generation, idempotencyKey: key ?? randomUUID() instanceId: instance.id, generation: instance.generation, idempotencyKey: key ?? randomUUID()
}); });
const result = await trackActivity(`agent:${operation.id}`, `Waiting for ${kind} on ${instance.server}`, const result = await trackActivity(`agent:${operation.id}`, `Waiting for ${kind} on ${instance.server}`,
() => waitHostOperation(operation.id, 25_000)); () => waitHostOperationStatus(operation.id, 25_000));
if (result.state !== 'completed') throw new Error(result.error || `Agent operation ${result.id} is ${result.state}; its original ID is retained.`); if (result.state !== 'completed') throw new Error(result.error || `Agent operation ${result.id} is ${result.state}; its original ID is retained.`);
// Do not apply replies from a retired generation to a replacement server. // Do not apply replies from a retired generation to a replacement server.
const current = nativeInstance(instance.server); const current = nativeInstance(instance.server);
+3
View File
@@ -345,6 +345,9 @@ export function db(): Database.Database {
{ name: 'mode', ddl: "ALTER TABLE server_launch_collections ADD COLUMN mode TEXT NOT NULL DEFAULT 'fill'" }, { name: 'mode', ddl: "ALTER TABLE server_launch_collections ADD COLUMN mode TEXT NOT NULL DEFAULT 'fill'" },
{ name: 'retained_servers', ddl: "ALTER TABLE server_launch_collections ADD COLUMN retained_servers TEXT NOT NULL DEFAULT '[]'" } { name: 'retained_servers', ddl: "ALTER TABLE server_launch_collections ADD COLUMN retained_servers TEXT NOT NULL DEFAULT '[]'" }
]); ]);
migrateTable(handle, 'server_launch_recipes', [
{ name: 'recipe_summary', ddl: 'ALTER TABLE server_launch_recipes ADD COLUMN recipe_summary TEXT' }
]);
migrateServerHistory(handle); migrateServerHistory(handle);
migrateTable(handle, 'server_collection_previews', [ migrateTable(handle, 'server_collection_previews', [
{ name: 'mode', ddl: "ALTER TABLE server_collection_previews ADD COLUMN mode TEXT NOT NULL DEFAULT 'fill'" }, { name: 'mode', ddl: "ALTER TABLE server_collection_previews ADD COLUMN mode TEXT NOT NULL DEFAULT 'fill'" },
+1
View File
@@ -522,6 +522,7 @@ CREATE TABLE IF NOT EXISTS server_launch_previews (
CREATE TABLE IF NOT EXISTS server_launch_recipes ( CREATE TABLE IF NOT EXISTS server_launch_recipes (
deployment_id TEXT PRIMARY KEY REFERENCES server_deployments(id) ON DELETE CASCADE, deployment_id TEXT PRIMARY KEY REFERENCES server_deployments(id) ON DELETE CASCADE,
recipe TEXT NOT NULL, recipe TEXT NOT NULL,
recipe_summary TEXT,
blueprint_name TEXT blueprint_name TEXT
); );
+21 -2
View File
@@ -109,10 +109,29 @@ function buildRecipe(game: ZombieGame, mapSlug: string, includeContent: boolean)
configFile: configFileForGame(game), configSource, files }; configFile: configFileForGame(game), configSource, files };
} }
export function summarizeRecipe(recipe: LaunchRecipePlan): LaunchRecipeSummary { export function summarizeRecipe(recipe: LaunchRecipePlan, fingerprint = sha256Buffer(Buffer.from(JSON.stringify(recipe)))): LaunchRecipeSummary {
const { format: _format, files, ...summary } = recipe; const { format: _format, files, ...summary } = recipe;
return { ...summary, files: files.map(({ content: _content, ...file }) => file), return { ...summary, files: files.map(({ content: _content, ...file }) => file),
fingerprint: sha256Buffer(Buffer.from(JSON.stringify(recipe))) }; fingerprint };
}
/** Keep the small detail response beside the immutable attempted file bytes. */
export function recipeStorage(recipe: LaunchRecipePlan): { recipe: string; recipeSummary: string } {
const encoded = JSON.stringify(recipe);
return { recipe: encoded, recipeSummary: JSON.stringify(summarizeRecipe(recipe, sha256Buffer(Buffer.from(encoded)))) };
}
export function recipeSummaryForDeployment(id: string): { recipe: LaunchRecipeSummary; blueprintName: string | null } | null {
const row = db().prepare('SELECT recipe_summary, blueprint_name FROM server_launch_recipes WHERE deployment_id = ?')
.get(id) as { recipe_summary: string | null; blueprint_name: string | null } | undefined;
if (!row) return null;
if (row.recipe_summary !== null) return { recipe: JSON.parse(row.recipe_summary), blueprintName: row.blueprint_name };
// Old snapshots remain readable. Populate on first access, without making
// controller startup scan and decode every retained plugin in its history.
const launch = recipeForDeployment(id)!;
const summary = summarizeRecipe(launch.recipe);
db().prepare('UPDATE server_launch_recipes SET recipe_summary = ? WHERE deployment_id = ?').run(JSON.stringify(summary), id);
return { recipe: summary, blueprintName: row.blueprint_name };
} }
export function recipeForDeployment(id: string): { recipe: LaunchRecipePlan; blueprintName: string | null } | null { export function recipeForDeployment(id: string): { recipe: LaunchRecipePlan; blueprintName: string | null } | null {
+62
View File
@@ -0,0 +1,62 @@
import { setImmediate } from 'node:timers/promises';
import { getHostOperationPayload, type OperationDeliveryRow } from './index.js';
// Even non-ASCII content stays below 64 KiB per encoded chunk.
const CHUNK_CHARACTERS = 16 * 1024;
const CHUNK_BYTES = 64 * 1024;
const encoder = new TextEncoder();
function* jsonParts(operations: OperationDeliveryRow[], serverTime: string): Generator<string | Buffer> {
yield '{"operations":[';
for (const [index, operation] of operations.entries()) {
if (index) yield ',';
const metadata = JSON.stringify({
id: operation.id, hostId: operation.host_id, kind: operation.kind,
instanceId: operation.instance_id, generation: operation.generation, state: operation.state,
error: operation.error, createdAt: operation.created_at, updatedAt: operation.updated_at
});
yield metadata.slice(0, -1) + ',"payload":';
// Load only the immutable payload currently being sent, after the
// stream has yielded its metadata and had a chance to serve health work.
yield getHostOperationPayload(operation.host_id, operation.id);
yield ',"result":';
yield operation.result ?? 'null';
yield '}';
}
yield '],"serverTime":' + JSON.stringify(serverTime) + '}';
}
function* encodedChunks(operations: OperationDeliveryRow[], serverTime: string): Generator<Uint8Array> {
for (const part of jsonParts(operations, serverTime)) {
if (Buffer.isBuffer(part)) {
for (let start = 0; start < part.length; start += CHUNK_BYTES) yield part.subarray(start, start + CHUNK_BYTES);
continue;
}
for (let start = 0; start < part.length;) {
let end = Math.min(start + CHUNK_CHARACTERS, part.length);
// A UTF-16 surrogate pair must be encoded together, including when
// user-authored configuration text falls on a chunk boundary.
const last = part.charCodeAt(end - 1);
if (end < part.length && last >= 0xd800 && last <= 0xdbff) end--;
yield encoder.encode(part.slice(start, end));
start = end;
}
}
}
/** Bound encoding work and yield between chunks so health requests keep flowing. */
export function hostOperationsResponse(operations: OperationDeliveryRow[], serverTime = new Date().toISOString()): Response {
const chunks = encodedChunks(operations, serverTime);
let cancelled = false;
const stream = new ReadableStream<Uint8Array>({
async pull(controller) {
await setImmediate();
if (cancelled) return;
const next = chunks.next();
if (next.done) controller.close();
else controller.enqueue(next.value);
},
cancel() { cancelled = true; chunks.return(undefined); }
});
return new Response(stream, { headers: { 'Content-Type': 'application/json; charset=utf-8', 'Cache-Control': 'no-store' } });
}
+80 -23
View File
@@ -30,7 +30,7 @@ function decommissionRow(hostId: string): DecommissionRow | undefined {
} }
function presentDecommission(row: DecommissionRow | undefined): HostDecommission | null { function presentDecommission(row: DecommissionRow | undefined): HostDecommission | null {
return row ? { operationId: row.operation_id, stage: row.stage, requestedAt: row.requested_at, updatedAt: row.updated_at, return row ? { operationId: row.operation_id, stage: row.stage, requestedAt: row.requested_at, updatedAt: row.updated_at,
error: row.error, retryable: row.stage === 'failed' && getHostOperation(row.operation_id)?.state === 'failed' } : null; error: row.error, retryable: row.stage === 'failed' && getHostOperationStatus(row.operation_id)?.state === 'failed' } : null;
} }
function hostFromRow(row: HostRow): Host { function hostFromRow(row: HostRow): Host {
const inventory: HostInventory = JSON.parse(row.inventory); const inventory: HostInventory = JSON.parse(row.inventory);
@@ -137,7 +137,7 @@ export function receiveDecommissionReceipt(operationId: string, token: string, i
if (!row || !timingSafeEqual(Buffer.from(row.token_hash, 'hex'), Buffer.from(digest(token), 'hex'))) throw new HostDeletionError('A valid cleanup receipt credential is required.', 401); if (!row || !timingSafeEqual(Buffer.from(row.token_hash, 'hex'), Buffer.from(digest(token), 'hex'))) throw new HostDeletionError('A valid cleanup receipt credential is required.', 401);
if (!['cleaned', 'completed', 'failed'].includes(input.stage as string)) throw new HostDeletionError('Invalid cleanup receipt stage.', 400); if (!['cleaned', 'completed', 'failed'].includes(input.stage as string)) throw new HostDeletionError('Invalid cleanup receipt stage.', 400);
if (row.stage === 'completed') return { acknowledged: true as const }; if (row.stage === 'completed') return { acknowledged: true as const };
const operation = getHostOperation(operationId)!; const operation = getHostOperationStatus(operationId)!;
if (!['accepted','running','unknown'].includes(operation.state)) throw new HostDeletionError('Cleanup has not been accepted by the host.', 409); if (!['accepted','running','unknown'].includes(operation.state)) throw new HostDeletionError('Cleanup has not been accepted by the host.', 409);
const timestamp = now(); const timestamp = now();
if (input.stage === 'failed') { if (input.stage === 'failed') {
@@ -314,41 +314,82 @@ export function requireReadyHost(hostId: string, options: { libraryId?: string;
return host; return host;
} }
type OperationRow = { id: string; host_id: string; kind: string; instance_id: string | null; generation: number | null; payload: string; state: HostOperationState; result: string | null; error: string | null; created_at: string; updated_at: string }; export type OperationRow = { id: string; host_id: string; kind: string; instance_id: string | null; generation: number | null; payload: string; state: HostOperationState; result: string | null; error: string | null; created_at: string; updated_at: string };
function operationFromRow(row: OperationRow): HostOperation { return { id: row.id, hostId: row.host_id, kind: row.kind, instanceId: row.instance_id, generation: row.generation, payload: JSON.parse(row.payload), state: row.state, result: row.result === null ? null : JSON.parse(row.result), error: row.error, createdAt: row.created_at, updatedAt: row.updated_at }; } export type OperationDeliveryRow = Omit<OperationRow, 'payload'>;
export function getHostOperation(id: string): HostOperation | null { const row = database().prepare('SELECT * FROM ec_host_operations WHERE id=?').get(id) as OperationRow | undefined; return row ? operationFromRow(row) : null; } export type HostOperationStatus = Omit<HostOperation, 'payload'>;
export function listHostOperations(hostId: string, limit = 100, summary = false): HostOperation[] { return (database().prepare(`SELECT ${summary ? `id,host_id,kind,instance_id,generation,state,error,created_at,updated_at,'{}' AS payload,NULL AS result` : '*'} FROM ec_host_operations WHERE host_id=? ORDER BY created_at DESC,rowid DESC LIMIT ?`).all(hostId, Math.min(500, Math.max(1, limit))) as OperationRow[]).map(operationFromRow); } const operationStatusColumns = 'o.id,o.host_id,o.kind,o.instance_id,o.generation,o.state,o.result,o.error,o.created_at,o.updated_at';
const operationPayloadSql = 'COALESCE((SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload)';
function operationStatusFromRow(row: Omit<OperationRow, 'payload'>): HostOperationStatus { return { id: row.id, hostId: row.host_id, kind: row.kind, instanceId: row.instance_id, generation: row.generation, state: row.state, result: row.result === null ? null : JSON.parse(row.result), error: row.error, createdAt: row.created_at, updatedAt: row.updated_at }; }
function operationFromRow(row: OperationRow): HostOperation { return { ...operationStatusFromRow(row), payload: JSON.parse(row.payload) }; }
export function getHostOperation(id: string): HostOperation | null {
const row = database().prepare(`SELECT ${operationStatusColumns},${operationPayloadSql} AS payload FROM ec_host_operations o WHERE o.id=?`)
.get(id) as OperationRow | undefined;
return row ? operationFromRow(row) : null;
}
/** Polling and receipts need the outcome, never another copy of launch files. */
export function getHostOperationStatus(id: string): HostOperationStatus | null {
const row = database().prepare('SELECT id,host_id,kind,instance_id,generation,state,result,error,created_at,updated_at FROM ec_host_operations WHERE id=?')
.get(id) as Omit<OperationRow, 'payload'> | undefined;
return row ? operationStatusFromRow(row) : null;
}
export function listHostOperations(hostId: string, limit = 100, summary = false): HostOperation[] {
const columns = summary ? `o.id,o.host_id,o.kind,o.instance_id,o.generation,o.state,o.error,o.created_at,o.updated_at,'{}' AS payload,NULL AS result`
: `${operationStatusColumns},${operationPayloadSql} AS payload`;
return (database().prepare(`SELECT ${columns} FROM ec_host_operations o WHERE o.host_id=? ORDER BY o.created_at DESC,o.rowid DESC LIMIT ?`)
.all(hostId, Math.min(500, Math.max(1, limit))) as OperationRow[]).map(operationFromRow);
}
/** Heartbeats can tell an executing create to stop at its next safe command boundary. */ /** Heartbeats can tell an executing create to stop at its next safe command boundary. */
export function pendingCreationCancellations(hostId: string): string[] { export function pendingCreationCancellations(hostId: string): string[] {
return (database().prepare(`SELECT json_extract(payload,'$.createOperationId') AS id FROM ec_host_operations return (database().prepare(`SELECT json_extract(payload,'$.createOperationId') AS id FROM ec_host_operations
WHERE host_id=? AND kind='instance.cancel-create' AND state IN ('pending','accepted','running','unknown')`).all(hostId) as { id: string }[]).map(row => row.id); WHERE host_id=? AND kind='instance.cancel-create' AND state IN ('pending','accepted','running','unknown')`).all(hostId) as { id: string }[]).map(row => row.id);
} }
export function queueHostOperation(hostId: string, kind: string, payload: Record<string, unknown>, options: { instanceId?: string; generation?: number; idempotencyKey?: string; id?: string } = {}): HostOperation { type QueueHostOperationOptions = { instanceId?: string; generation?: number; idempotencyKey?: string; id?: string };
export function queueHostOperation(hostId: string, kind: string, payload: Record<string, unknown>, options: QueueHostOperationOptions = {}): HostOperation {
if (!payload || typeof payload !== 'object' || Array.isArray(payload)) throw new Error('Operation payload must be an object.');
const operation = enqueueHostOperation(hostId, kind, JSON.stringify(payload), options);
return getHostOperation(operation.id)!;
}
/** Keep immutable launch bytes separate so receipts and delivery do not rewrite large payloads. */
export function queueHostOperationSerialized(hostId: string, kind: string, payload: Buffer, options: QueueHostOperationOptions = {}, routingPayload: Record<string, unknown> = {}): HostOperationStatus {
if (!Buffer.isBuffer(payload) || payload[0] !== 123 || payload[payload.length - 1] !== 125) throw new Error('Serialized operation payload must be a JSON object.');
if (!routingPayload || typeof routingPayload !== 'object' || Array.isArray(routingPayload)) throw new Error('Operation routing payload must be an object.');
return enqueueHostOperation(hostId, kind, payload, options, JSON.stringify(routingPayload));
}
function enqueueHostOperation(hostId: string, kind: string, payload: string | Buffer, options: QueueHostOperationOptions, routingPayload?: string): HostOperationStatus {
const host = requireHost(hostId); const host = requireHost(hostId);
if (host.decommission) throw new Error('Host cleanup is in progress; new operations are blocked.'); if (host.decommission) throw new Error('Host cleanup is in progress; new operations are blocked.');
if (kind === 'host.decommission') throw new Error('Use the host deletion workflow to request cleanup.'); if (kind === 'host.decommission') throw new Error('Use the host deletion workflow to request cleanup.');
if (!/^[a-z][a-z0-9_.-]{0,79}$/.test(kind)) throw new Error('Invalid operation kind.'); if (!/^[a-z][a-z0-9_.-]{0,79}$/.test(kind)) throw new Error('Invalid operation kind.');
if (!payload || typeof payload !== 'object' || Array.isArray(payload)) throw new Error('Operation payload must be an object.');
if (options.generation !== undefined && (!Number.isSafeInteger(options.generation) || options.generation < 1)) throw new Error('Instance generation must be a positive integer.'); if (options.generation !== undefined && (!Number.isSafeInteger(options.generation) || options.generation < 1)) throw new Error('Instance generation must be a positive integer.');
const operation = database().transaction(() => { const operation = database().transaction(() => {
if (options.idempotencyKey) { if (options.idempotencyKey) {
const existing = database().prepare('SELECT * FROM ec_host_operations WHERE host_id=? AND idempotency_key=?').get(hostId, options.idempotencyKey) as OperationRow | undefined; const existing = database().prepare(`SELECT o.id,o.kind,o.instance_id,o.generation,${operationPayloadSql}=CAST(? AS TEXT) AS same_payload
FROM ec_host_operations o WHERE o.host_id=? AND o.idempotency_key=?`).get(payload, hostId, options.idempotencyKey) as
{ id: string; kind: string; instance_id: string | null; generation: number | null; same_payload: number } | undefined;
if (existing) { if (existing) {
if (existing.kind !== kind || existing.instance_id !== (options.instanceId ?? null) || existing.generation !== (options.generation ?? null) || existing.payload !== JSON.stringify(payload)) throw new Error('Operation idempotency key is already associated with a different request.'); if (existing.kind !== kind || existing.instance_id !== (options.instanceId ?? null) || existing.generation !== (options.generation ?? null) || !existing.same_payload) throw new Error('Operation idempotency key is already associated with a different request.');
return operationFromRow(existing); return getHostOperationStatus(existing.id)!;
} }
} }
if (['instance.create','create'].includes(kind)) requireReadyHost(hostId); if (['instance.create','create'].includes(kind)) requireReadyHost(hostId);
if (host.mode === 'disabled' && kind !== 'instance.cancel-create') throw new Error('Host is disabled. Enable it before requesting operations.'); if (host.mode === 'disabled' && kind !== 'instance.cancel-create') throw new Error('Host is disabled. Enable it before requesting operations.');
const id = options.id ?? randomUUID(); const timestamp = now(); const id = options.id ?? randomUUID(); const timestamp = now();
database().prepare('INSERT INTO ec_host_operations(id,host_id,kind,instance_id,generation,idempotency_key,payload,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?)').run(id, hostId, kind, options.instanceId ?? null, options.generation ?? null, options.idempotencyKey ?? null, JSON.stringify(payload), timestamp, timestamp); database().prepare('INSERT INTO ec_host_operations(id,host_id,kind,instance_id,generation,idempotency_key,payload,created_at,updated_at) VALUES(?,?,?,?,?,?,CAST(? AS TEXT),?,?)').run(id, hostId, kind, options.instanceId ?? null, options.generation ?? null, options.idempotencyKey ?? null, routingPayload ?? payload, timestamp, timestamp);
if (routingPayload !== undefined) database().prepare('INSERT INTO ec_host_operation_payloads(operation_id,payload) VALUES(?,CAST(? AS TEXT))').run(id, payload);
database().prepare('INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,?,?)').run(id, 'pending', timestamp); database().prepare('INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,?,?)').run(id, 'pending', timestamp);
return getHostOperation(id)!; return getHostOperationStatus(id)!;
})(); })();
queueMicrotask(() => workEvents.emit(hostId)); queueMicrotask(() => workEvents.emit(hostId));
return operation; return operation;
} }
export function pendingHostOperations(hostId: string, lane: 'all' | 'read' | 'health' | 'write' = 'all'): HostOperation[] { export function pendingHostOperations(hostId: string, lane: 'all' | 'read' | 'health' | 'write' = 'all'): HostOperation[] {
return pendingHostOperationRows(hostId, lane).map(operationFromRow);
}
type OperationLane = 'all' | 'read' | 'health' | 'write';
export function pendingHostOperationRows(hostId: string, lane?: OperationLane, includePayload?: true): OperationRow[];
export function pendingHostOperationRows(hostId: string, lane: OperationLane, includePayload: false): OperationDeliveryRow[];
/** Delivery snapshots retain status/results while large immutable payloads stay in SQLite. */
export function pendingHostOperationRows(hostId: string, lane: OperationLane = 'all', includePayload = true): OperationDeliveryRow[] {
const host = requireHost(hostId); const host = requireHost(hostId);
if (host.mode === 'disabled' || !host.connected) return []; if (host.mode === 'disabled' || !host.connected) return [];
// Older agents hold activity.lock across the entire delivered batch, including // Older agents hold activity.lock across the entire delivered batch, including
@@ -361,8 +402,8 @@ export function pendingHostOperations(hostId: string, lane: 'all' | 'read' | 'he
// Accepted/running/unknown commands keep the same immutable ID. The agent // Accepted/running/unknown commands keep the same immutable ID. The agent
// consults its durable journal before acknowledging or executing a replay. // consults its durable journal before acknowledging or executing a replay.
const readOnly = `(kind IN ('host.inspect','instance.inspect','files.read','files.list','files.stat','logs.read') const readOnly = `(kind IN ('host.inspect','instance.inspect','files.read','files.list','files.stat','logs.read')
OR (kind='rcon.execute' AND json_extract(payload,'$.command') IN ('status','getstatus','version')))`; OR (kind='rcon.execute' AND json_extract(${operationPayloadSql},'$.command') IN ('status','getstatus','version')))`;
const healthOnly = `(kind='instance.inspect' OR (kind='rcon.execute' AND json_extract(payload,'$.command') IN ('status','getstatus','version')))`; const healthOnly = `(kind='instance.inspect' OR (kind='rcon.execute' AND json_extract(${operationPayloadSql},'$.command') IN ('status','getstatus','version')))`;
const healthLane = host.inventory.capabilities?.includes('operations.health-lane'); const healthLane = host.inventory.capabilities?.includes('operations.health-lane');
const laneFilter = lane === 'all' ? '' : lane === 'health' ? ` AND ${healthOnly}` const laneFilter = lane === 'all' ? '' : lane === 'health' ? ` AND ${healthOnly}`
: ` AND ${lane === 'write' ? 'NOT ' : ''}${readOnly}${lane === 'read' && healthLane ? ` AND NOT ${healthOnly}` : ''}`; : ` AND ${lane === 'write' ? 'NOT ' : ''}${readOnly}${lane === 'read' && healthLane ? ` AND NOT ${healthOnly}` : ''}`;
@@ -370,23 +411,31 @@ export function pendingHostOperations(hostId: string, lane: 'all' | 'read' | 'he
// in small batches to amortize delivery without handing out a long backlog. // in small batches to amortize delivery without handing out a long backlog.
const limit = healthLane ? lane === 'health' ? 1 : lane === 'read' ? 4 : 25 : 25; const limit = healthLane ? lane === 'health' ? 1 : lane === 'read' ? 4 : 25 : 25;
return database().transaction(() => { return database().transaction(() => {
const rows = database().prepare(`SELECT * FROM ec_host_operations o WHERE host_id=? AND state IN ('pending','accepted','running','unknown') const columns = includePayload ? `${operationStatusColumns},${operationPayloadSql} AS payload` : operationStatusColumns;
const rows = database().prepare(`SELECT ${columns} FROM ec_host_operations o WHERE host_id=? AND state IN ('pending','accepted','running','unknown')
${laneFilter} ${laneFilter}
AND (? OR state != 'pending' OR kind NOT IN ('instance.create','create')) AND (? OR state != 'pending' OR kind NOT IN ('instance.create','create'))
AND (kind!='instance.cancel-create' OR ?) AND (kind!='instance.cancel-create' OR ?)
AND NOT EXISTS (SELECT 1 FROM ec_host_operations cancellation WHERE cancellation.host_id=o.host_id AND NOT EXISTS (SELECT 1 FROM ec_host_operations cancellation WHERE cancellation.host_id=o.host_id
AND cancellation.kind='instance.cancel-create' AND json_extract(cancellation.payload,'$.createOperationId')=o.id) AND cancellation.kind='instance.cancel-create' AND json_extract(cancellation.payload,'$.createOperationId')=o.id)
ORDER BY CASE WHEN kind='instance.cancel-create' THEN 0 ELSE 1 END,COALESCE(last_delivered_at,''),created_at,rowid LIMIT ${limit}`) ORDER BY CASE WHEN kind='instance.cancel-create' THEN 0 ELSE 1 END,COALESCE(last_delivered_at,''),created_at,rowid LIMIT ${limit}`)
.all(hostId, Number(host.readiness === 'ready'), Number(host.inventory.capabilities?.includes('instance.cancel-create') ?? false)) as OperationRow[]; .all(hostId, Number(host.readiness === 'ready'), Number(host.inventory.capabilities?.includes('instance.cancel-create') ?? false)) as OperationDeliveryRow[];
const deliveredAt = now(); const deliveredAt = now();
for (const operation of rows) database().prepare('UPDATE ec_host_operations SET last_delivered_at=? WHERE id=?').run(deliveredAt, operation.id); for (const operation of rows) database().prepare('UPDATE ec_host_operations SET last_delivered_at=? WHERE id=?').run(deliveredAt, operation.id);
return rows.map(operationFromRow); return rows;
})(); })();
} }
export function reportHostOperation(hostId: string, id: string, input: { state: unknown; result?: unknown; error?: unknown }): HostOperation { /** Payloads never change after enqueueing; fetch one delivery's UTF-8 bytes without decoding. */
export function getHostOperationPayload(hostId: string, id: string): Buffer {
const row = database().prepare(`SELECT CAST(${operationPayloadSql} AS BLOB) AS payload FROM ec_host_operations o WHERE o.id=? AND o.host_id=?`)
.get(id, hostId) as { payload: Buffer } | undefined;
if (!row) throw new Error('Host operation was not found for this host.');
return row.payload;
}
export function reportHostOperationStatus(hostId: string, id: string, input: { state: unknown; result?: unknown; error?: unknown }): HostOperationStatus {
const operation = database().transaction(() => { const operation = database().transaction(() => {
requireHost(hostId); requireHost(hostId);
const operation = getHostOperation(id); const operation = getHostOperationStatus(id);
if (!operation || operation.hostId !== hostId) throw new Error('Operation was not found for this host.'); if (!operation || operation.hostId !== hostId) throw new Error('Operation was not found for this host.');
if (operation.kind === 'host.decommission' && input.state === 'completed') throw new Error('Host removal requires the cleanup worker completion receipt.'); if (operation.kind === 'host.decommission' && input.state === 'completed') throw new Error('Host removal requires the cleanup worker completion receipt.');
if (!['accepted','running','completed','failed','unknown'].includes(input.state as string)) throw new Error('Invalid operation state.'); if (!['accepted','running','completed','failed','unknown'].includes(input.state as string)) throw new Error('Invalid operation state.');
@@ -407,21 +456,29 @@ export function reportHostOperation(hostId: string, id: string, input: { state:
recordAgentReceipt(id, state, now()); recordAgentReceipt(id, state, now());
} }
} }
return getHostOperation(id)!; return getHostOperationStatus(id)!;
})(); })();
queueMicrotask(() => workEvents.emit(`operation:${id}`)); queueMicrotask(() => workEvents.emit(`operation:${id}`));
return operation; return operation;
} }
export async function waitHostOperation(id: string, timeoutMs = 30_000): Promise<HostOperation> { export function reportHostOperation(hostId: string, id: string, input: { state: unknown; result?: unknown; error?: unknown }): HostOperation {
reportHostOperationStatus(hostId, id, input);
return getHostOperation(id)!;
}
export async function waitHostOperationStatus(id: string, timeoutMs = 30_000): Promise<HostOperationStatus> {
const deadline = Date.now() + timeoutMs; const deadline = Date.now() + timeoutMs;
while (true) { while (true) {
const operation = getHostOperation(id); const operation = getHostOperationStatus(id);
if (!operation) throw new Error('Host operation was not found.'); if (!operation) throw new Error('Host operation was not found.');
if (['completed','failed','unknown'].includes(operation.state)) return operation; if (['completed','failed','unknown'].includes(operation.state)) return operation;
if (Date.now() >= deadline) throw new Error(`Host operation ${id} is awaiting acknowledgement; reconcile this operation ID before retrying.`); if (Date.now() >= deadline) throw new Error(`Host operation ${id} is awaiting acknowledgement; reconcile this operation ID before retrying.`);
await waitForHostWork(`operation:${id}`, new AbortController().signal, Math.min(1000, Math.max(1, deadline - Date.now()))); await waitForHostWork(`operation:${id}`, new AbortController().signal, Math.min(1000, Math.max(1, deadline - Date.now())));
} }
} }
export async function waitHostOperation(id: string, timeoutMs = 30_000): Promise<HostOperation> {
await waitHostOperationStatus(id, timeoutMs);
return getHostOperation(id)!;
}
export function hostPortAvailable(hostId: string, candidate: number, bindAddress?: string, protocol: 'udp' | 'tcp' = 'udp', instanceId?: string): boolean { export function hostPortAvailable(hostId: string, candidate: number, bindAddress?: string, protocol: 'udp' | 'tcp' = 'udp', instanceId?: string): boolean {
const host = requireHost(hostId); port(candidate); const bind = bindAddress ?? host.bindAddress; const host = requireHost(hostId); port(candidate); const bind = bindAddress ?? host.bindAddress;
+4
View File
@@ -33,6 +33,10 @@ export function ensureHostSchema(handle: Database.Database): void {
-- history. The queue index cannot narrow those lookups beyond host_id, -- history. The queue index cannot narrow those lookups beyond host_id,
-- causing a history scan for every queued command on Node's main thread. -- causing a history scan for every queued command on Node's main thread.
CREATE INDEX IF NOT EXISTS ec_host_operations_kind ON ec_host_operations(host_id, kind, created_at); CREATE INDEX IF NOT EXISTS ec_host_operations_kind ON ec_host_operations(host_id, kind, created_at);
CREATE TABLE IF NOT EXISTS ec_host_operation_payloads (
operation_id TEXT PRIMARY KEY REFERENCES ec_host_operations(id) ON DELETE CASCADE,
payload TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS ec_host_operation_events ( CREATE TABLE IF NOT EXISTS ec_host_operation_events (
id INTEGER PRIMARY KEY AUTOINCREMENT, operation_id TEXT NOT NULL REFERENCES ec_host_operations(id), id INTEGER PRIMARY KEY AUTOINCREMENT, operation_id TEXT NOT NULL REFERENCES ec_host_operations(id),
state TEXT NOT NULL, created_at TEXT NOT NULL, error TEXT state TEXT NOT NULL, created_at TEXT NOT NULL, error TEXT
+3 -3
View File
@@ -1,6 +1,6 @@
import { randomUUID } from 'node:crypto'; import { randomUUID } from 'node:crypto';
import { db } from '../db/index.js'; import { db } from '../db/index.js';
import { recipeForDeployment, summarizeRecipe } from '../fleet/launch-recipes.js'; import { recipeSummaryForDeployment } from '../fleet/launch-recipes.js';
import { actionLabels, type ProvisioningAction } from '../../provisioning.js'; import { actionLabels, type ProvisioningAction } from '../../provisioning.js';
import type { ProvisioningCapacity, ProvisioningDeployment, ProvisioningDetail, ProvisioningSource } from '../../provisioning-detail.js'; import type { ProvisioningCapacity, ProvisioningDeployment, ProvisioningDetail, ProvisioningSource } from '../../provisioning-detail.js';
import { fleetSettings } from '../settings.js'; import { fleetSettings } from '../settings.js';
@@ -111,7 +111,7 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
const { observation_json: _snapshot, phase_key: _phase, phase_started_at: _phaseStarted, observed_at: _observed, ...deployment } = row; const { observation_json: _snapshot, phase_key: _phase, phase_started_at: _phaseStarted, observed_at: _observed, ...deployment } = row;
const sources = (db().prepare('SELECT kind, source_id, server FROM provisioning_sources WHERE operation_id = ? ORDER BY kind, source_id') const sources = (db().prepare('SELECT kind, source_id, server FROM provisioning_sources WHERE operation_id = ? ORDER BY kind, source_id')
.all(id) as Omit<ProvisioningSource, 'label' | 'href'>[]).map(sourceDetails); .all(id) as Omit<ProvisioningSource, 'label' | 'href'>[]).map(sourceDetails);
const launch = recipeForDeployment(id); const launch = recipeSummaryForDeployment(id);
const collection = db().prepare(`SELECT c.id, c.name FROM ( const collection = db().prepare(`SELECT c.id, c.name FROM (
SELECT collection_id, deployment_id FROM server_collection_members SELECT collection_id, deployment_id FROM server_collection_members
UNION ALL SELECT collection_id, deployment_id FROM server_collection_removals) m UNION ALL SELECT collection_id, deployment_id FROM server_collection_removals) m
@@ -120,7 +120,7 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
canCancelCreation: canCancelCreation(id), cancellation: cancellation ?? null, canCancelCreation: canCancelCreation(id), cancellation: cancellation ?? null,
workerActivity: workerActivity(), events: deploymentEvents(id, options.events === 'recent'), workerActivity: workerActivity(), events: deploymentEvents(id, options.events === 'recent'),
activity: { ...activitySummary(id), failureExcerpt: latestFailureEvent(id) }, sources, activity: { ...activitySummary(id), failureExcerpt: latestFailureEvent(id) }, sources,
recipe: launch ? summarizeRecipe(launch.recipe) : null, blueprintName: launch?.blueprintName ?? null, recipe: launch?.recipe ?? null, blueprintName: launch?.blueprintName ?? null,
canResumeVerification: canResumeVerification(id), canResumeCleanup: canResumeCleanup(id), canResumeVerification: canResumeVerification(id), canResumeCleanup: canResumeCleanup(id),
canResumeRebuild: canResumeRebuild(id), canCancelRemoval: canCancelRemoval(id), collection: collection ?? null, canResumeRebuild: canResumeRebuild(id), canCancelRemoval: canCancelRemoval(id), collection: collection ?? null,
progress: readProgress(row), capacity: capacityFor(deployment, generatedAt), progress: readProgress(row), capacity: capacityFor(deployment, generatedAt),
+11 -7
View File
@@ -4,7 +4,7 @@ import { probePlayerEvidence, probeServerStatus } from '../fleet/status-evidence
import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent, REMOVING_SERVER, resumeCleanup, resumeVerification, claimRemoval } from './history.js'; import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent, REMOVING_SERVER, resumeCleanup, resumeVerification, claimRemoval } from './history.js';
import { randomUUID } from 'node:crypto'; import { randomUUID } from 'node:crypto';
import type { LaunchPreview } from '$lib/deployment-plan.js'; import type { LaunchPreview } from '$lib/deployment-plan.js';
import { blueprintRecipe, captureRecipe, captureRecipePlan, recipeForDeployment, summarizeRecipe, type LaunchRecipe, type LaunchRecipePlan } from '../fleet/launch-recipes.js'; import { blueprintRecipe, captureRecipe, captureRecipePlan, recipeForDeployment, recipeStorage, summarizeRecipe, type LaunchRecipe, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
import { mkdir, chmod, rm, writeFile } from 'node:fs/promises'; import { mkdir, chmod, rm, writeFile } from 'node:fs/promises';
import { dirname, join } from 'node:path'; import { dirname, join } from 'node:path';
import { config, destOfKind, serverDir, gameOfServer, configFileForGame } from '../config.js'; import { config, destOfKind, serverDir, gameOfServer, configFileForGame } from '../config.js';
@@ -240,9 +240,10 @@ export function queueCreateServer(input: CreateServerInput, releasingServers: st
if (!prepared.ok) return prepared; if (!prepared.ok) return prepared;
// A preview records choices, not version pins. Promotions remain authoritative. // A preview records choices, not version pins. Promotions remain authoritative.
const currentRecipe = captureRecipePlan(input.game ?? 't5', prepared.mapEntry.slug); const currentRecipe = captureRecipePlan(input.game ?? 't5', prepared.mapEntry.slug);
const storedRecipe = recipeStorage(currentRecipe);
const id = beginDeployment(prepared.name, 'create', input.createdBy); const id = beginDeployment(prepared.name, 'create', input.createdBy);
db().prepare('INSERT INTO server_launch_recipes (deployment_id, recipe, blueprint_name) VALUES (?, ?, ?)') db().prepare('INSERT INTO server_launch_recipes (deployment_id, recipe, recipe_summary, blueprint_name) VALUES (?, ?, ?, ?)')
.run(id, JSON.stringify(currentRecipe), preview?.blueprint_name ?? null); .run(id, storedRecipe.recipe, storedRecipe.recipeSummary, preview?.blueprint_name ?? null);
if (preview) db().prepare('UPDATE server_launch_previews SET deployment_id = ? WHERE id = ?').run(id, preview.id); if (preview) db().prepare('UPDATE server_launch_previews SET deployment_id = ? WHERE id = ?').run(id, preview.id);
db().prepare("UPDATE server_deployments SET state = 'queued' WHERE id = ?").run(id); db().prepare("UPDATE server_deployments SET state = 'queued' WHERE id = ?").run(id);
db().prepare('INSERT INTO server_creation_jobs (deployment_id, server, port, game_mode, map_slug, expected_map) VALUES (?, ?, ?, ?, ?, ?)') db().prepare('INSERT INTO server_creation_jobs (deployment_id, server, port, game_mode, map_slug, expected_map) VALUES (?, ?, ?, ?, ?, ?)')
@@ -275,7 +276,9 @@ export function queueRebuildServer(name: string, triggeredBy = 'ui'): CreateServ
db().prepare('INSERT INTO server_creation_jobs (deployment_id, server, port, game_mode, map_slug, expected_map) VALUES (?, ?, ?, ?, ?, ?)') db().prepare('INSERT INTO server_creation_jobs (deployment_id, server, port, game_mode, map_slug, expected_map) VALUES (?, ?, ?, ?, ?, ?)')
.run(id, name, server.game_port, 'zombie', map.slug, map.canonical); .run(id, name, server.game_port, 'zombie', map.slug, map.canonical);
db().prepare('INSERT INTO server_rebuild_jobs (deployment_id) VALUES (?)').run(id); db().prepare('INSERT INTO server_rebuild_jobs (deployment_id) VALUES (?)').run(id);
db().prepare('INSERT INTO server_launch_recipes (deployment_id, recipe) VALUES (?, ?)').run(id, JSON.stringify(recipe)); const storedRecipe = recipeStorage(recipe);
db().prepare('INSERT INTO server_launch_recipes (deployment_id, recipe, recipe_summary) VALUES (?, ?, ?)')
.run(id, storedRecipe.recipe, storedRecipe.recipeSummary);
deploymentStage(id, 'Waiting to rebuild the empty server', { phase: 'eligibility', recipeStatus: 'queued', observedAt: null, deploymentStage(id, 'Waiting to rebuild the empty server', { phase: 'eligibility', recipeStatus: 'queued', observedAt: null,
blocker: { code: 'eligibility', message: 'Waiting for safe rebuild eligibility checks.', automatic: true } }); blocker: { code: 'eligibility', message: 'Waiting for safe rebuild eligibility checks.', automatic: true } });
return { ok: true as const, name, port: server.game_port!, deploymentId: id }; return { ok: true as const, name, port: server.game_port!, deploymentId: id };
@@ -399,10 +402,11 @@ async function createServerUnlocked(input: CreateServerInput, prepared: Prepared
// Resolve once more at provisioning so promotions made while queued also apply. // Resolve once more at provisioning so promotions made while queued also apply.
// Persist the exact attempted files for history, never as input to future launches. // Persist the exact attempted files for history, never as input to future launches.
const recipe = captureRecipe(game, mapEntry.slug); const recipe = captureRecipe(game, mapEntry.slug);
const storedRecipe = recipeStorage(recipe);
db().transaction(() => { db().transaction(() => {
db().prepare(`INSERT INTO server_launch_recipes (deployment_id, recipe) VALUES (?, ?) db().prepare(`INSERT INTO server_launch_recipes (deployment_id, recipe, recipe_summary) VALUES (?, ?, ?)
ON CONFLICT(deployment_id) DO UPDATE SET recipe = excluded.recipe`) ON CONFLICT(deployment_id) DO UPDATE SET recipe = excluded.recipe, recipe_summary = excluded.recipe_summary`)
.run(operation, JSON.stringify(recipe)); .run(operation, storedRecipe.recipe, storedRecipe.recipeSummary);
deploymentStage(operation, 'Preparing server files', { phase: 'prepare', recipeStatus: 'attempted', blocker: null, checks: [], deploymentStage(operation, 'Preparing server files', { phase: 'prepare', recipeStatus: 'attempted', blocker: null, checks: [],
checkpoint: { key: 'startup-queue', label: 'Admitted to startup' }, checkpoint: { key: 'startup-queue', label: 'Admitted to startup' },
event: { kind: 'recipe', message: 'Promoted files selected for this creation attempt.', details: { capturedAt: recipe.capturedAt } } }); event: { kind: 'recipe', message: 'Promoted files selected for this creation attempt.', details: { capturedAt: recipe.capturedAt } } });
@@ -0,0 +1,71 @@
import { Worker } from 'node:worker_threads';
import { resolve } from 'node:path';
import { config } from '../config.js';
import { db } from '../db/index.js';
import type { NativeInstance } from '../backends/native.js';
import type { LaunchRecipePlan } from '../fleet/launch-recipes.js';
export interface NativeLaunchInput {
game: 't5' | 't6';
mapSlug: string;
native?: LaunchRecipePlan['native'];
instance: NativeInstance;
serverKey: string;
rconPassword: string;
rebuild: boolean;
}
export interface PreparedNativeLaunch {
recipe: LaunchRecipePlan;
recipeSummary: string;
recipeJson: Uint8Array;
payloadJson: Uint8Array;
}
let worker: Worker | null = null;
let sequence = 0;
const pending = new Map<number, {
resolve: (value: PreparedNativeLaunch) => void;
reject: (error: Error) => void;
timer: NodeJS.Timeout;
}>();
function stop(error: Error, current = worker): Promise<number> | undefined {
if (current !== worker) return;
worker = null;
for (const request of pending.values()) { clearTimeout(request.timer); request.reject(error); }
pending.clear();
return current?.terminate();
}
/** The reader and JSON encoder run off the controller's request thread. */
export function prepareNativeLaunch(input: NativeLaunchInput): Promise<PreparedNativeLaunch> {
db(); // Migrations must finish before the independent read-only connection opens.
if (!worker) {
const current = worker = new Worker(resolve(process.env.NODE_ENV === 'production'
? 'build/launch/worker.mjs' : '.svelte-kit/launch/worker.mjs'), {
workerData: { dbPath: resolve(config.dbPath) }
});
current.on('message', ({ id, value, error }: { id: number; value: PreparedNativeLaunch; error?: string }) => {
if (current !== worker) return;
const request = pending.get(id);
if (!request) return;
pending.delete(id); clearTimeout(request.timer);
if (error) request.reject(new Error(error)); else request.resolve(value);
if (!pending.size) current.unref();
});
current.on('error', error => { void stop(error, current); });
current.on('exit', () => { void stop(new Error('Launch preparation worker stopped; preparation will retry.'), current); });
}
worker.ref();
const id = ++sequence;
return new Promise((resolve, reject) => {
const timer = setTimeout(() => { void stop(new Error('Launch preparation timed out; preparation will retry.')); }, 60_000);
pending.set(id, { resolve, reject, timer });
worker!.postMessage({ id, input });
});
}
export async function closeLaunchPreparation(): Promise<void> {
await stop(new Error('Launch preparation stopped.'));
}
@@ -0,0 +1,38 @@
import { parentPort } from 'node:worker_threads';
import { posix } from 'node:path';
import { db } from '../db/index.js';
import { relDestForKind } from '../config.js';
import { captureRecipe, summarizeRecipe } from '../fleet/launch-recipes.js';
import { sha256Buffer } from '../fleet/files.js';
import type { NativeLaunchInput } from './launch-preparation.js';
parentPort!.on('message', ({ id, input }: { id: number; input: NativeLaunchInput }) => {
try {
// All file versions and composed configuration belong to one snapshot.
const recipe = db().transaction(() => ({ ...captureRecipe(input.game, input.mapSlug),
native: input.native, image: input.instance.image }))();
const instance = input.instance;
const payload = {
name: instance.server, game: recipe.game, libraryId: instance.library_id, image: instance.image,
port: instance.port, bindAddress: instance.bind_address, rconPassword: input.rconPassword,
env: { PLUTAINER_SERVER_NAME: instance.server, PLUTAINER_PORT: String(instance.port),
PLUTAINER_GAME: recipe.game === 't6' ? 't6zm' : 't5sp', PLUTAINER_CONFIG_FILE: recipe.configFile,
PLUTAINER_MAP_ROTATE: 'false',
PLUTAINER_EXTRA_ARGS: `+exec electric_cherry_map.cfg +set tws_server_id ${instance.server} +map_rotate`,
PLUTO_SERVER_KEY: input.serverKey },
files: recipe.files.map(file => ({ path: posix.join(relDestForKind(file.kind, recipe.game), file.path),
contentBase64: file.content, sha256: file.sha256 })),
...(input.rebuild ? { nextGeneration: instance.generation + 1 } : {})
};
const encode = (value: unknown) => new TextEncoder().encode(JSON.stringify(value));
const recipeJson = encode(recipe);
const payloadJson = encode(payload);
const plan = { ...recipe, files: recipe.files.map(({ content: _content, ...file }) => file) };
const recipeSummary = JSON.stringify(summarizeRecipe(plan, sha256Buffer(Buffer.from(recipeJson.buffer))));
// Transfer ownership of the encoded bytes; do not clone giant strings or objects.
parentPort!.postMessage({ id, value: { recipe: plan, recipeSummary, recipeJson, payloadJson } },
[recipeJson.buffer, payloadJson.buffer]);
} catch (error) {
parentPort!.postMessage({ id, error: error instanceof Error ? error.message : 'Launch preparation failed.' });
}
});
+92 -39
View File
@@ -1,13 +1,15 @@
import { isIP } from 'node:net'; import { isIP } from 'node:net';
import { setImmediate as yieldToController } from 'node:timers/promises';
import { CapacityError, choosePlacement, availableHostPort, type PlacementOptions } from '../hosts/capacity.js'; import { CapacityError, choosePlacement, availableHostPort, type PlacementOptions } from '../hosts/capacity.js';
import { randomBytes, randomUUID } from 'node:crypto'; import { randomBytes, randomUUID } from 'node:crypto';
import { posix } from 'node:path'; import { posix } from 'node:path';
import { db } from '../db/index.js'; import { db } from '../db/index.js';
import { config, relDestForKind, serverDir } from '../config.js'; import { config, relDestForKind, serverDir } from '../config.js';
import { nativeInstance, nativeRequest, type NativeInstance } from '../backends/native.js'; import { nativeInstance, nativeRequest, type NativeInstance } from '../backends/native.js';
import { getHost, getHostOperation, listHosts, queueHostOperation, releaseHostPorts, reserveHostPort } from '../hosts/index.js'; import { getHost, getHostOperationStatus, listHosts, queueHostOperation, queueHostOperationSerialized, releaseHostPorts, reserveHostPort } from '../hosts/index.js';
import { newestReadyLibraryId, assertReadyLibrary } from '../game-files/index.js'; import { newestReadyLibraryId, assertReadyLibrary } from '../game-files/index.js';
import { blueprintRecipe, captureRecipe, captureRecipePlan, recipeForDeployment, summarizeRecipe, type LaunchRecipe, type LaunchRecipePlan } from '../fleet/launch-recipes.js'; import { blueprintRecipe, captureRecipePlan, recipeForDeployment, recipeStorage, summarizeRecipe, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
import { prepareNativeLaunch, type PreparedNativeLaunch } from './launch-preparation.js';
import { getServerKey, saveServerKey } from '../fleet/server-keys.js'; import { getServerKey, saveServerKey } from '../fleet/server-keys.js';
import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js'; import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
import { nextLobbyNumber, type CreateServerInput, type CreateServerResult } from './instances.js'; import { nextLobbyNumber, type CreateServerInput, type CreateServerResult } from './instances.js';
@@ -125,7 +127,9 @@ export function queueNativeCreation(input: CreateServerInput): CreateServerResul
db().prepare("UPDATE server_deployments SET state='queued' WHERE id=?").run(id); db().prepare("UPDATE server_deployments SET state='queued' WHERE id=?").run(id);
db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,?,?,?,?)') db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,?,?,?,?)')
.run(id, name, port, 'zombie', recipe.mapSlug, recipe.expectedMap); .run(id, name, port, 'zombie', recipe.mapSlug, recipe.expectedMap);
db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,blueprint_name) VALUES(?,?,?)').run(id, JSON.stringify(recipe), preview?.blueprint_name ?? null); const stored = recipeStorage(recipe);
db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary,blueprint_name) VALUES(?,?,?,?)')
.run(id, stored.recipe, stored.recipeSummary, preview?.blueprint_name ?? null);
if (preview) db().prepare('UPDATE server_launch_previews SET deployment_id=?,port=? WHERE id=?').run(id, port, preview.id); if (preview) db().prepare('UPDATE server_launch_previews SET deployment_id=?,port=? WHERE id=?').run(id, port, preview.id);
deploymentStage(id, 'Waiting for provisioning evaluation', { phase: 'eligibility', recipeStatus: 'queued' }); deploymentStage(id, 'Waiting for provisioning evaluation', { phase: 'eligibility', recipeStatus: 'queued' });
return { ok: true as const, name, port, deploymentId: id }; return { ok: true as const, name, port, deploymentId: id };
@@ -164,7 +168,8 @@ export function queueNativeRebuild(name: string, triggeredBy: string): CreateSer
db().prepare("UPDATE server_deployments SET state='waiting' WHERE id=?").run(id); db().prepare("UPDATE server_deployments SET state='waiting' WHERE id=?").run(id);
db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,?,?,?,?)') db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,?,?,?,?)')
.run(id, name, instance.port, 'zombie', recipe.mapSlug, recipe.expectedMap); .run(id, name, instance.port, 'zombie', recipe.mapSlug, recipe.expectedMap);
db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe) VALUES(?,?)').run(id, JSON.stringify(recipe)); const stored = recipeStorage(recipe);
db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary) VALUES(?,?,?)').run(id, stored.recipe, stored.recipeSummary);
deploymentStage(id, 'Waiting to rebuild when empty', { phase: 'eligibility', recipeStatus: 'queued' }); deploymentStage(id, 'Waiting to rebuild when empty', { phase: 'eligibility', recipeStatus: 'queued' });
return { ok: true as const, name, port: instance.port, deploymentId: id }; return { ok: true as const, name, port: instance.port, deploymentId: id };
})(); })();
@@ -212,16 +217,6 @@ function applyReceipt(job: Job, result: Record<string, unknown>) {
})(); })();
} }
function createPayload(instance: NativeInstance, recipe: LaunchRecipe) {
const key = getServerKey(instance.server);
if (!key) throw new Error('The saved server key was removed.');
return { name: instance.server, game: recipe.game, libraryId: instance.library_id, image: instance.image, port: instance.port, bindAddress: instance.bind_address,
rconPassword: config.rcon.password || randomBytes(32).toString('hex'),
env: { PLUTAINER_SERVER_NAME: instance.server, PLUTAINER_PORT: String(instance.port), PLUTAINER_GAME: recipe.game === 't6' ? 't6zm' : 't5sp',
PLUTAINER_CONFIG_FILE: recipe.configFile, PLUTAINER_MAP_ROTATE: 'false', PLUTAINER_EXTRA_ARGS: `+exec electric_cherry_map.cfg +set tws_server_id ${instance.server} +map_rotate`, PLUTO_SERVER_KEY: key },
files: recipe.files.map(file => ({ path: posix.join(relDestForKind(file.kind, recipe.game), file.path), contentBase64: file.content, sha256: file.sha256 })) };
}
function hold(job: Job, message: string) { deploymentStage(job.id, message, { blocker: { code: 'native-host', message, automatic: true } }); } function hold(job: Job, message: string) { deploymentStage(job.id, message, { blocker: { code: 'native-host', message, automatic: true } }); }
function stillActive(job: Job): boolean { function stillActive(job: Job): boolean {
return !!db().prepare(`SELECT 1 FROM server_deployments WHERE id=? AND server_deleted_at IS NULL return !!db().prepare(`SELECT 1 FROM server_deployments WHERE id=? AND server_deleted_at IS NULL
@@ -267,6 +262,24 @@ export async function processNativeProvisioning(eligibility: (operation: { id: s
} }
const activeJobs = new Map<string, Promise<void>>(); const activeJobs = new Map<string, Promise<void>>();
// Preparation owns startup capacity until its command is committed or discarded.
const preparingJobs = new Set<string>();
function otherPreparations(id: string): number { return preparingJobs.size - Number(preparingJobs.has(id)); }
let launchCommit: Promise<void> = Promise.resolve();
function commitPreparedLaunch(commit: () => void): Promise<void> {
// Worker replies can arrive together. Persist one launch per event-loop turn
// so a batch of large SQLite writes cannot hold off requests and health timers.
const next = launchCommit.then(async () => { await yieldToController(); commit(); });
launchCommit = next.catch(() => {});
return next;
}
function sameLaunchFiles(first: LaunchRecipePlan, second: LaunchRecipePlan): boolean {
const selected = (recipe: LaunchRecipePlan) => ({ game: recipe.game, mapSlug: recipe.mapSlug,
expectedMap: recipe.expectedMap, configFile: recipe.configFile, configSource: recipe.configSource, files: recipe.files });
return JSON.stringify(selected(first)) === JSON.stringify(selected(second));
}
/** Reservations can move only while no remote work has ever been submitted. */ /** Reservations can move only while no remote work has ever been submitted. */
function placeQueuedCreation(job: Job, instance: NativeInstance): NativeInstance { function placeQueuedCreation(job: Job, instance: NativeInstance): NativeInstance {
@@ -285,7 +298,8 @@ function placeQueuedCreation(job: Job, instance: NativeInstance): NativeInstance
.run(host.id, bind, port, network.publicAddress ?? host.publicAddress, network.publicPort ?? null, instance.id); .run(host.id, bind, port, network.publicAddress ?? host.publicAddress, network.publicPort ?? null, instance.id);
db().prepare('UPDATE server_creation_jobs SET port=? WHERE deployment_id=?').run(port, job.id); db().prepare('UPDATE server_creation_jobs SET port=? WHERE deployment_id=?').run(port, job.id);
recipe.native = { ...prior, hostId: host.id, bindAddress: bind, publicAddress: network.publicAddress ?? host.publicAddress, publicPort: network.publicPort }; recipe.native = { ...prior, hostId: host.id, bindAddress: bind, publicAddress: network.publicAddress ?? host.publicAddress, publicPort: network.publicPort };
db().prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?').run(JSON.stringify(recipe), job.id); const stored = recipeStorage(recipe);
db().prepare('UPDATE server_launch_recipes SET recipe=?,recipe_summary=? WHERE deployment_id=?').run(stored.recipe, stored.recipeSummary, job.id);
db().prepare('UPDATE server_launch_previews SET port=? WHERE deployment_id=?').run(port, job.id); db().prepare('UPDATE server_launch_previews SET port=? WHERE deployment_id=?').run(port, job.id);
deploymentStage(job.id, `Placed on ${host.name} after CPU, memory, history and port checks`, { phase: 'eligibility' }); deploymentStage(job.id, `Placed on ${host.name} after CPU, memory, history and port checks`, { phase: 'eligibility' });
return nativeInstance(job.server)!; return nativeInstance(job.server)!;
@@ -302,7 +316,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
const settings = fleetSettings(); const settings = fleetSettings();
const active = db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL const active = db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL
AND action IN ('create','rebuild') AND state IN ('running','awaiting-health')`).get(job.id) as { n: number }; AND action IN ('create','rebuild') AND state IN ('running','awaiting-health')`).get(job.id) as { n: number };
if (active.n >= settings.createConcurrency) { hold(job, 'Waiting for startup capacity.'); return; } if (active.n + otherPreparations(job.id) >= settings.createConcurrency) { hold(job, 'Waiting for startup capacity.'); return; }
instance = placeQueuedCreation(job, instance); instance = placeQueuedCreation(job, instance);
} }
const host = getHost(instance.host_id); const host = getHost(instance.host_id);
@@ -311,7 +325,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
hold(job, 'Host is unavailable; existing server state is unknown. Waiting for reconnection.'); return; hold(job, 'Host is unavailable; existing server state is unknown. Waiting for reconnection.'); return;
} }
if (job.operation_id) { if (job.operation_id) {
const operation = getHostOperation(job.operation_id); const operation = getHostOperationStatus(job.operation_id);
if (operation?.kind === 'instance.cancel-create' && !['completed', 'failed'].includes(operation.state) && !host.inventory.capabilities?.includes('instance.cancel-create')) { if (operation?.kind === 'instance.cancel-create' && !['completed', 'failed'].includes(operation.state) && !host.inventory.capabilities?.includes('instance.cancel-create')) {
hold(job, hostAgentUpdatesEnabled(host.inventory) hold(job, hostAgentUpdatesEnabled(host.inventory)
? 'Waiting for the host agent update. New command delivery is paused so its current batch can finish and the automatic updater can run. Cleanup resumes after the updated agent connects.' ? 'Waiting for the host agent update. New command delivery is paused so its current batch can finish and the automatic updater can run. Cleanup resumes after the updated agent connects.'
@@ -381,7 +395,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
if (!status.ok) { hold(job, `Waiting for game status: ${status.reason}`); return; } if (!status.ok) { hold(job, `Waiting for game status: ${status.reason}`); return; }
if (job.action === 'create' || job.action === 'rebuild') { if (job.action === 'create' || job.action === 'rebuild') {
deploymentStage(job.id, 'Checking selected engine map', { phase: 'verify-map', blocker: null }); deploymentStage(job.id, 'Checking selected engine map', { phase: 'verify-map', blocker: null });
const recipe = recipeForDeployment(job.id)!.recipe; const recipe = db().prepare('SELECT expected_map AS expectedMap FROM server_creation_jobs WHERE deployment_id=?')
.get(job.id) as { expectedMap: string };
check({ key: 'map', label: 'Selected engine map verified', source: status.source, check({ key: 'map', label: 'Selected engine map verified', source: status.source,
result: status.map === recipe.expectedMap ? 'pass' : 'waiting', expected: recipe.expectedMap, actual: status.map }, 'verify-map'); result: status.map === recipe.expectedMap ? 'pass' : 'waiting', expected: recipe.expectedMap, actual: status.map }, 'verify-map');
if (status.map !== recipe.expectedMap) { hold(job, `Waiting for selected map ${recipe.expectedMap}; observed ${status.map ?? 'unknown'}.`); return; } if (status.map !== recipe.expectedMap) { hold(job, `Waiting for selected map ${recipe.expectedMap}; observed ${status.map ?? 'unknown'}.`); return; }
@@ -416,7 +431,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
hold(job, 'Staging and verifying files on the host.'); return; hold(job, 'Staging and verifying files on the host.'); return;
} }
if (job.stage_operation_id) { if (job.stage_operation_id) {
const stage = getHostOperation(job.stage_operation_id); const stage = getHostOperationStatus(job.stage_operation_id);
if (stage?.state === 'failed') { finishDeployment(job.id, { ok: false, message: stage.error ?? 'File staging failed.' }); return; } if (stage?.state === 'failed') { finishDeployment(job.id, { ok: false, message: stage.error ?? 'File staging failed.' }); return; }
if (stage?.state !== 'completed') { hold(job, 'Waiting for verified remote file staging.'); return; } if (stage?.state !== 'completed') { hold(job, 'Waiting for verified remote file staging.'); return; }
if (files.some(file => (db().prepare('SELECT state FROM staged_file_changes WHERE id=?').get(file.id) as { state: string } | undefined)?.state !== 'pending')) { if (files.some(file => (db().prepare('SELECT state FROM staged_file_changes WHERE id=?').get(file.id) as { state: string } | undefined)?.state !== 'pending')) {
@@ -426,7 +441,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
const settings = fleetSettings(); const settings = fleetSettings();
const count = (actions: string[]) => (db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id != ? AND server_deleted_at IS NULL const count = (actions: string[]) => (db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id != ? AND server_deleted_at IS NULL
AND action IN (${actions.map(() => '?').join(',')}) AND state IN ('running','awaiting-health')`).get(job.id, ...actions) as { n: number }).n; AND action IN (${actions.map(() => '?').join(',')}) AND state IN ('running','awaiting-health')`).get(job.id, ...actions) as { n: number }).n;
if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) >= settings.createConcurrency) { if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) + otherPreparations(job.id) >= settings.createConcurrency) {
deploymentStage(job.id, 'Waiting for startup capacity.', { phase: 'startup-queue', deploymentStage(job.id, 'Waiting for startup capacity.', { phase: 'startup-queue',
blocker: { code: 'startup-capacity', message: 'Waiting for startup capacity.', automatic: true } }); return; blocker: { code: 'startup-capacity', message: 'Waiting for startup capacity.', automatic: true } }); return;
} }
@@ -435,27 +450,51 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
if (job.action === 'create') choosePlacement({ hostId: instance.host_id, libraryId: instance.library_id, image: instance.image, if (job.action === 'create') choosePlacement({ hostId: instance.host_id, libraryId: instance.library_id, image: instance.image,
bindAddress: instance.bind_address, excludeInstanceId: instance.id, thorough: true, dispatch: true }); bindAddress: instance.bind_address, excludeInstanceId: instance.id, thorough: true, dispatch: true });
if (job.action === 'create' || job.action === 'rebuild') assertReadyLibrary(instance.host_id, instance.library_id); if (job.action === 'create' || job.action === 'rebuild') assertReadyLibrary(instance.host_id, instance.library_id);
if (job.action !== 'create' && job.action !== 'start') { const checkOccupancy = async (): Promise<boolean> => {
// Fast controller check keeps occupied lobbies out of the operation slots. // Fast controller check keeps occupied lobbies out of the operation slots.
// The agent repeats this immediately before its local mutation. // The agent repeats this immediately before its local mutation.
deploymentStage(job.id, 'Checking container and player occupancy before submission', { phase: 'eligibility', blocker: null }); deploymentStage(job.id, 'Checking container and player occupancy before submission', { phase: 'eligibility', blocker: null });
const health = await nativeRequest<{ state: string; healthy?: boolean }>(instance, 'instance.inspect'); const health = await nativeRequest<{ state: string; healthy?: boolean }>(instance, 'instance.inspect');
if (!stillActive(job)) return; if (!stillActive(job)) return false;
if (!['stopped','exited','created','dead','missing'].includes(health.state)) { if (!['stopped','exited','created','dead','missing'].includes(health.state)) {
const occupancy = await probePlayerEvidence(job.server); const occupancy = await probePlayerEvidence(job.server);
if (!stillActive(job)) return; if (!stillActive(job)) return false;
const recoveredWatchdog = health.healthy && occupancy.players !== null && db().prepare(`SELECT 1 FROM provisioning_sources s const recoveredWatchdog = health.healthy && occupancy.players !== null && db().prepare(`SELECT 1 FROM provisioning_sources s
JOIN watchdog_incidents i ON i.id=s.source_id JOIN server_deployments d ON d.id=s.operation_id JOIN watchdog_incidents i ON i.id=s.source_id JOIN server_deployments d ON d.id=s.operation_id
WHERE s.operation_id=? AND s.kind='watchdog' AND i.kind='health' AND d.triggered_by LIKE 'watchdog:%'`).get(job.id); WHERE s.operation_id=? AND s.kind='watchdog' AND i.kind='health' AND d.triggered_by LIKE 'watchdog:%'`).get(job.id);
if (recoveredWatchdog && occupancy.players !== null) { resolveRecoveredHealthOperation(job.id, occupancy.players, occupancy.source); return; } if (recoveredWatchdog && occupancy.players !== null) { resolveRecoveredHealthOperation(job.id, occupancy.players, occupancy.source); return false; }
if (occupancy.players !== 0) { hold(job, occupancy.players !== null ? `Waiting for ${occupancy.players} player(s) to leave.` : 'Waiting for a fresh local player count.'); return; } if (occupancy.players !== 0) { hold(job, occupancy.players !== null ? `Waiting for ${occupancy.players} player(s) to leave.` : 'Waiting for a fresh local player count.'); return false; }
}
} }
return true;
};
if (job.action !== 'create' && job.action !== 'start' && !await checkOccupancy()) return;
const current = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined; const current = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined;
if (!current || !['queued','waiting','interrupted'].includes(current.state) || eligibility(job)) return; if (!current || !['queued','waiting','interrupted'].includes(current.state) || eligibility(job)) return;
db().transaction(() => { let prepared: PreparedNativeLaunch | undefined;
let preparedKey: string | null = null;
try {
if (job.action === 'create' || job.action === 'rebuild') {
if (count(['create','rebuild']) + otherPreparations(job.id) >= fleetSettings().createConcurrency) {
hold(job, 'Waiting for startup capacity.'); return;
}
preparingJobs.add(job.id);
preparedKey = getServerKey(job.server) ?? null;
if (!preparedKey) throw new Error('The saved server key was removed.');
const prior = recipeForDeployment(job.id)!.recipe as NativeRecipe;
deploymentStage(job.id, 'Preparing launch files for the host agent', { phase: 'prepare',
checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } });
prepared = await prepareNativeLaunch({ game: prior.game, mapSlug: prior.mapSlug, native: prior.native,
instance, serverKey: preparedKey, rconPassword: config.rcon.password || randomBytes(32).toString('hex'),
rebuild: job.action === 'rebuild' });
if (job.action === 'rebuild' && !await checkOccupancy()) return;
}
const commit = db().transaction(() => {
const state = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined;
if (!state || !['queued','waiting','interrupted'].includes(state.state) || eligibility(job)) return;
const live = nativeInstance(job.server); const live = nativeInstance(job.server);
if (live?.id !== instance!.id || live.generation !== instance!.generation) throw new Error('Instance changed during preflight.'); if (live?.id !== instance!.id || live.generation !== instance!.generation) throw new Error('Instance changed during preflight.');
const liveHost = getHost(instance!.host_id);
if (!liveHost?.connected || liveHost.mode === 'disabled') { hold(job, 'Host is unavailable; waiting for reconnection.'); return; }
if (job.action === 'destroy' || job.action === 'rebuild') { if (job.action === 'destroy' || job.action === 'rebuild') {
const beta = activeBetaRemovalBlocker(job.server); const beta = activeBetaRemovalBlocker(job.server);
if (beta) { hold(job, beta); return; } if (beta) { hold(job, beta); return; }
@@ -464,36 +503,50 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
finishDeployment(job.id, { ok: false, message: 'File requests changed during preflight. Review and submit a fresh activation.' }); return; finishDeployment(job.id, { ok: false, message: 'File requests changed during preflight. Review and submit a fresh activation.' }); return;
} }
// Async occupancy checks can overlap another job claiming the final slot. // Async occupancy checks can overlap another job claiming the final slot.
if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) >= settings.createConcurrency) { if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) + otherPreparations(job.id) >= fleetSettings().createConcurrency) {
deploymentStage(job.id, 'Waiting for startup capacity.', { phase: 'startup-queue', blocker: { code: 'startup-capacity', message: 'Waiting for startup capacity.', automatic: true } }); return; deploymentStage(job.id, 'Waiting for startup capacity.', { phase: 'startup-queue', blocker: { code: 'startup-capacity', message: 'Waiting for startup capacity.', automatic: true } }); return;
} }
if (job.action === 'destroy' && count(['destroy']) >= settings.destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; } if (job.action === 'destroy' && count(['destroy']) >= fleetSettings().destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; }
if (prepared) {
if (getServerKey(job.server) !== preparedKey || !sameLaunchFiles(prepared.recipe,
captureRecipePlan(prepared.recipe.game, prepared.recipe.mapSlug))) {
hold(job, 'Launch files or credentials changed during preparation; preparing the current selection next pass.'); return;
}
assertReadyLibrary(instance!.host_id, instance!.library_id);
if (job.action === 'create') choosePlacement({ hostId: instance!.host_id, libraryId: instance!.library_id, image: instance!.image,
bindAddress: instance!.bind_address, excludeInstanceId: instance!.id, thorough: true, dispatch: true });
}
if (job.action === 'start' || job.action === 'restart' || job.action === 'rebuild') { if (job.action === 'start' || job.action === 'restart' || job.action === 'rebuild') {
db().prepare(`INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES(?,?,?) ON CONFLICT(server) db().prepare(`INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES(?,?,?) ON CONFLICT(server)
DO UPDATE SET requested_at=excluded.requested_at,operation_id=excluded.operation_id,healthy_at=NULL,error=NULL`).run(job.server, new Date().toISOString(), job.id); DO UPDATE SET requested_at=excluded.requested_at,operation_id=excluded.operation_id,healthy_at=NULL,error=NULL`).run(job.server, new Date().toISOString(), job.id);
} }
recordProgress(job.id, { blocker: null, checkpoint: { key: 'eligibility', label: 'Host eligibility and safety checks passed' } }); recordProgress(job.id, { blocker: null, checkpoint: { key: 'eligibility', label: 'Host eligibility and safety checks passed' } });
let payload: Record<string, unknown> = files.length ? { stageId: job.id } : {}; const payload: Record<string, unknown> = files.length ? { stageId: job.id } : {};
if (job.action === 'create' || job.action === 'rebuild') { if (prepared) {
deploymentStage(job.id, 'Preparing launch files for the host agent', { phase: 'prepare', recipeStatus: 'attempted', deploymentStage(job.id, 'Preparing launch files for the host agent', { phase: 'prepare', recipeStatus: 'attempted',
checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } }); checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } });
const prior = recipeForDeployment(job.id)!.recipe as NativeRecipe; const recipe = prepared.recipe;
const recipe = { ...captureRecipe(prior.game, prior.mapSlug), native: prior.native, image: instance!.image }; db().prepare('UPDATE server_launch_recipes SET recipe=CAST(? AS TEXT),recipe_summary=? WHERE deployment_id=?')
payload = createPayload(instance!, recipe); .run(Buffer.from(prepared.recipeJson.buffer, prepared.recipeJson.byteOffset, prepared.recipeJson.byteLength), prepared.recipeSummary, job.id);
if (job.action === 'rebuild') payload.nextGeneration = instance!.generation + 1;
db().prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?').run(JSON.stringify(recipe), job.id);
if (job.action === 'create') db().prepare(`INSERT INTO servers(name,komodo_type,komodo_name,game,map,game_mode,map_slug,host_dir,config_file,game_id,image,game_port,created_by,container_state) if (job.action === 'create') db().prepare(`INSERT INTO servers(name,komodo_type,komodo_name,game,map,game_mode,map_slug,host_dir,config_file,game_id,image,game_port,created_by,container_state)
VALUES(?,'deployment','',?,?,'zombie',?,?,?,?,?,?,'native','starting')`).run(job.server, recipe.game, recipe.expectedMap, recipe.mapSlug, serverDir(job.server), VALUES(?,'deployment','',?,?,'zombie',?,?,?,?,?,?,'native','starting')`).run(job.server, recipe.game, recipe.expectedMap, recipe.mapSlug, serverDir(job.server),
recipe.configFile, recipe.game === 't6' ? 't6zm' : 't5sp', recipe.image, instance!.port); recipe.configFile, recipe.game === 't6' ? 't6zm' : 't5sp', recipe.image, instance!.port);
for (const file of recipe.files) recordScript(job.server, file.path, file.versionId, file.sha256, file.size, 'native-create'); for (const file of recipe.files) recordScript(job.server, file.path, file.versionId, file.sha256, file.size, 'native-create');
recordProgress(job.id, { checkpoint: { key: 'prepare', label: 'Launch files selected and command payload prepared' } }); recordProgress(job.id, { checkpoint: { key: 'prepare', label: 'Launch files selected and command payload prepared' } });
} }
const operation = queueHostOperation(instance!.host_id, `instance.${job.action === 'destroy' ? 'remove' : job.action}`, payload, const operationOptions = { instanceId: instance!.id, generation: instance!.generation, idempotencyKey: job.id };
{ instanceId: instance!.id, generation: instance!.generation, idempotencyKey: job.id }); const kind = `instance.${job.action === 'destroy' ? 'remove' : job.action}`;
const operation = prepared ? queueHostOperationSerialized(instance!.host_id, kind,
Buffer.from(prepared.payloadJson.buffer, prepared.payloadJson.byteOffset, prepared.payloadJson.byteLength), operationOptions,
{ name: instance!.server, game: prepared.recipe.game, libraryId: instance!.library_id })
: queueHostOperation(instance!.host_id, kind, payload, operationOptions);
db().prepare('UPDATE native_provisioning SET operation_id=? WHERE deployment_id=?').run(operation.id, job.id); db().prepare('UPDATE native_provisioning SET operation_id=? WHERE deployment_id=?').run(operation.id, job.id);
for (const file of files) db().prepare("UPDATE staged_file_changes SET state='applying' WHERE id=? AND state='pending'").run(file.id); for (const file of files) db().prepare("UPDATE staged_file_changes SET state='applying' WHERE id=? AND state='pending'").run(file.id);
db().prepare("UPDATE server_deployments SET state='running',submitted_at=? WHERE id=?").run(new Date().toISOString(), job.id); db().prepare("UPDATE server_deployments SET state='running',submitted_at=? WHERE id=?").run(new Date().toISOString(), job.id);
deploymentStage(job.id, `Waiting for host agent to acknowledge ${job.action}`, { phase: 'container-action', blocker: null, deploymentStage(job.id, `Waiting for host agent to acknowledge ${job.action}`, { phase: 'container-action', blocker: null,
command: { state: 'attempted', message: `Durable agent operation ${operation.id}; reconciliation always uses this ID.` } }); command: { state: 'attempted', message: `Durable agent operation ${operation.id}; reconciliation always uses this ID.` } });
})(); });
if (prepared) await commitPreparedLaunch(commit);
else commit();
} finally { preparingJobs.delete(job.id); }
} }
+5 -5
View File
@@ -1,17 +1,17 @@
import { json } from '@sveltejs/kit'; import { pendingHostOperationRows, waitForHostWork } from '$lib/server/hosts/index.js';
import { pendingHostOperations, waitForHostWork } from '$lib/server/hosts/index.js';
import { authenticateAgent } from '$lib/server/hosts/http.js'; import { authenticateAgent } from '$lib/server/hosts/http.js';
import { reconcileHostPreparation } from '$lib/server/hosts/preparation.js'; import { reconcileHostPreparation } from '$lib/server/hosts/preparation.js';
import { hostOperationsResponse } from '$lib/server/hosts/delivery.js';
export async function GET({ request }: { request: Request }) { export async function GET({ request }: { request: Request }) {
const host = authenticateAgent(request); const host = authenticateAgent(request);
reconcileHostPreparation(host.id); reconcileHostPreparation(host.id);
const params = new URL(request.url).searchParams; const params = new URL(request.url).searchParams;
const lane = params.get('lane') === 'health' ? 'health' : params.get('lane') === 'read' ? 'read' : params.get('lane') === 'write' ? 'write' : 'all'; const lane = params.get('lane') === 'health' ? 'health' : params.get('lane') === 'read' ? 'read' : params.get('lane') === 'write' ? 'write' : 'all';
let operations = pendingHostOperations(host.id, lane); let operations = pendingHostOperationRows(host.id, lane, false);
if (!operations.length && params.get('wait') === '20') { if (!operations.length && params.get('wait') === '20') {
await waitForHostWork(host.id, request.signal); await waitForHostWork(host.id, request.signal);
authenticateAgent(request); // Revocation still applies after waiting. authenticateAgent(request); // Revocation still applies after waiting.
operations = pendingHostOperations(host.id, lane); operations = pendingHostOperationRows(host.id, lane, false);
} }
return json({ operations, serverTime: new Date().toISOString() }, { headers: { 'Cache-Control': 'no-store' } }); return hostOperationsResponse(operations);
} }
@@ -1,12 +1,12 @@
import { json } from '@sveltejs/kit'; import { json } from '@sveltejs/kit';
import { reportHostOperation } from '$lib/server/hosts/index.js'; import { reportHostOperationStatus } from '$lib/server/hosts/index.js';
import { authenticateAgent, hostApiError, objectBody } from '$lib/server/hosts/http.js'; import { authenticateAgent, hostApiError, objectBody } from '$lib/server/hosts/http.js';
import { kickProvisioning } from '$lib/server/pipeline/worker.js'; import { kickProvisioning } from '$lib/server/pipeline/worker.js';
export async function POST({ request, params }: { request: Request; params: { id: string } }) { export async function POST({ request, params }: { request: Request; params: { id: string } }) {
const host = authenticateAgent(request); const host = authenticateAgent(request);
try { try {
const body = await objectBody(request, 128 * 1024 * 1024); const body = await objectBody(request, 128 * 1024 * 1024);
const operation = reportHostOperation(host.id, params.id, { state: body.state, result: body.result, error: body.error }); const operation = reportHostOperationStatus(host.id, params.id, { state: body.state, result: body.result, error: body.error });
if (['completed', 'failed', 'unknown'].includes(operation.state) if (['completed', 'failed', 'unknown'].includes(operation.state)
&& (operation.kind.startsWith('instance.') && operation.kind !== 'instance.inspect' || operation.kind === 'files.stage')) { && (operation.kind.startsWith('instance.') && operation.kind !== 'instance.inspect' || operation.kind === 'files.stage')) {
void kickProvisioning().catch(error => console.error('[agent] provisioning wake failed:', error)); void kickProvisioning().catch(error => console.error('[agent] provisioning wake failed:', error));
+2
View File
@@ -49,6 +49,8 @@ export async function newFixture(servers: string[] = ['t5_zombie_theater_1']): P
try { try {
const { closeReports } = await import('../../src/lib/server/reports/index.js'); const { closeReports } = await import('../../src/lib/server/reports/index.js');
await closeReports(); await closeReports();
const { closeLaunchPreparation } = await import('../../src/lib/server/provisioning/launch-preparation.js');
await closeLaunchPreparation();
const mod = (await import('../../src/lib/server/db/index.js')) as { const mod = (await import('../../src/lib/server/db/index.js')) as {
closeDb: () => void; closeDb: () => void;
}; };
+142
View File
@@ -0,0 +1,142 @@
import { setImmediate } from 'node:timers/promises';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
vi.mock('../../src/lib/server/hosts/preparation.js', () => ({ reconcileHostPreparation: vi.fn() }));
let fixture: Fixture;
const inventory = {
protocolVersion: 1, dockerRunning: true, directoriesWritable: true, storageAvailable: 100 * 1024 ** 3,
runtimeImages: ['plutainer:test'], libraries: [{ id: 'test-library', state: 'ready' }],
capabilities: ['instance.cancel-create', 'operations.health-lane']
};
beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
afterEach(async () => { vi.restoreAllMocks(); vi.useRealTimers(); await fixture.cleanup(); });
async function enrolled() {
const hosts = await import('../../src/lib/server/hosts/index.js');
const host = hosts.createHost({ name: 'Delivery host' });
const credentials = hosts.enrollHost(hosts.createEnrollment(host.id).token, inventory);
const { GET } = await import('../../src/routes/api/agent/operations/+server.js');
const request = (query = '') => new Request(`https://ec.test/api/agent/operations${query}`, {
headers: { authorization: `Bearer ${credentials.credential}` }
});
return { ...hosts, host, GET, request };
}
describe('agent operation delivery', () => {
it('streams large persisted launch files without parsing payloads and yields to controller work', async () => {
const api = await enrolled();
const payload = {
// Place a Unicode character across the first payload byte chunk.
value: 'x'.repeat(64 * 1024 - '{"value":"'.length - 1) + '🧟 café "quoted"\n',
files: [{ path: 'plugins/runtime.dll', contentBase64: 'A'.repeat(8 * 1024 ** 2) }]
};
const operation = api.queueHostOperation(api.host.id, 'instance.create', payload,
{ instanceId: 'instance-one', generation: 2 });
const result = { phase: 'prepared', description: '世界', details: [null, true, 42] };
api.reportHostOperation(api.host.id, operation.id, { state: 'running', result });
const expected = api.getHostOperation(operation.id)!;
const serialized = JSON.stringify(payload);
const originalParse = JSON.parse;
const parse = vi.spyOn(JSON, 'parse').mockImplementation((value, reviver) => {
if (value === serialized) throw new Error('Delivery must not deserialize launch files.');
return originalParse(value, reviver);
});
const response = await api.GET({ request: api.request('?lane=write') });
expect(response.headers.get('cache-control')).toBe('no-store');
expect(response.headers.get('content-type')).toContain('application/json');
let complete = false;
let interleaved = false;
const controllerWork = setImmediate().then(() => { interleaved = !complete; });
const chunks: Uint8Array[] = [];
const reader = response.body!.getReader();
while (true) {
const next = await reader.read();
if (next.done) break;
expect(next.value.byteLength).toBeLessThanOrEqual(64 * 1024);
chunks.push(next.value);
}
complete = true;
await controllerWork;
expect(interleaved).toBe(true);
expect(chunks.length).toBeGreaterThan(100);
parse.mockRestore();
const body = JSON.parse(Buffer.concat(chunks).toString('utf8'));
expect(body).toEqual({ operations: [expected], serverTime: expect.any(String) });
expect(Number.isFinite(Date.parse(body.serverTime))).toBe(true);
});
it('selects without payloads, loads one payload at a time, and retains the selected receipt snapshot', async () => {
const api = await enrolled();
const payload = Buffer.from(JSON.stringify({ file: 'A'.repeat(512 * 1024) }));
api.queueHostOperationSerialized(api.host.id, 'instance.create', payload);
const second = api.queueHostOperationSerialized(api.host.id, 'instance.create', payload);
api.reportHostOperationStatus(api.host.id, second.id, { state: 'accepted', result: { phase: 'selected' } });
const { db } = await import('../../src/lib/server/db/index.js');
const prepare = vi.spyOn(db(), 'prepare');
const payloadReads = () => prepare.mock.calls.filter(([sql]) => sql.startsWith('SELECT CAST(') && sql.includes(' AS BLOB) AS payload')).length;
const response = await api.GET({ request: api.request('?lane=write') });
const selection = prepare.mock.calls.find(([sql]) => sql.includes('FROM ec_host_operations o'))![0];
const projection = selection.split(' FROM ')[0];
expect(projection).not.toContain('payload');
expect(projection).not.toContain('*');
expect(payloadReads()).toBe(0);
// A receipt arriving during a streamed response must not mix a newer
// result with the state and timestamps chosen for this delivery.
api.reportHostOperationStatus(api.host.id, second.id, { state: 'running', result: { phase: 'newer' } });
const reader = response.body!.getReader();
const first = await reader.read();
expect(first.done).toBe(false);
expect(payloadReads()).toBe(0);
const chunks: Uint8Array[] = [first.value!];
let checkedNextPayload = false;
while (true) {
const next = await reader.read();
if (next.done) break;
if (payloadReads() === 2 && !checkedNextPayload) {
expect(Buffer.concat(chunks).includes(payload)).toBe(true);
checkedNextPayload = true;
}
chunks.push(next.value);
}
expect(payloadReads()).toBe(2);
expect(checkedNextPayload).toBe(true);
const body = JSON.parse(Buffer.concat(chunks).toString('utf8'));
expect(body.operations).toHaveLength(2);
expect(body.operations[1]).toMatchObject({ id: second.id, state: 'accepted', result: { phase: 'selected' } });
expect(body.operations[0].payload).toEqual(JSON.parse(payload.toString('utf8')));
expect(body.operations[1].payload).toEqual(body.operations[0].payload);
const other = api.createHost({ name: 'Other host' });
expect(() => api.getHostOperationPayload(other.id, second.id)).toThrow('not found for this host');
});
it('retains health lanes, delivery rotation, and cancellation suppression through the streamed route', async () => {
const api = await enrolled();
const operations = Array.from({ length: 27 }, (_, index) => api.queueHostOperation(api.host.id, 'instance.restart', { index }));
const inspect = api.queueHostOperation(api.host.id, 'instance.inspect', {});
const first = await (await api.GET({ request: api.request('?lane=write') })).json();
expect(first.operations.map((operation: { id: string }) => operation.id)).toEqual(operations.slice(0, 25).map(operation => operation.id));
const second = await (await api.GET({ request: api.request('?lane=write') })).json();
expect(second.operations.slice(0, 2).map((operation: { id: string }) => operation.id)).toEqual(operations.slice(25).map(operation => operation.id));
const health = await (await api.GET({ request: api.request('?lane=health') })).json();
expect(health.operations).toEqual([inspect]);
const create = api.queueHostOperation(api.host.id, 'instance.create', {});
const cancellation = api.queueHostOperation(api.host.id, 'instance.cancel-create', { createOperationId: create.id });
const cancelled = await (await api.GET({ request: api.request('?lane=write') })).json();
expect(cancelled.operations[0].id).toBe(cancellation.id);
expect(cancelled.operations.some((operation: { id: string }) => operation.id === create.id)).toBe(false);
});
it('rechecks credentials after waking a long poll before sending work', async () => {
const api = await enrolled();
const waiting = api.GET({ request: api.request('?lane=health&wait=20') });
api.revokeCredentials(api.host.id);
api.queueHostOperation(api.host.id, 'instance.inspect', {});
await expect(waiting).rejects.toMatchObject({ status: 401 });
});
});
@@ -125,10 +125,11 @@ describe('matching a native collection', () => {
await new Promise<void>(resolve => setImmediate(resolve)); await new Promise<void>(resolve => setImmediate(resolve));
await processProvisioning(); await processProvisioning();
const dispatched = db().prepare(`SELECT n.deployment_id,o.payload FROM native_provisioning n const dispatched = db().prepare(`SELECT n.deployment_id,o.id FROM native_provisioning n
JOIN ec_host_operations o ON o.id=n.operation_id WHERE o.kind='instance.create'`).all() as { deployment_id: string; payload: string }[]; JOIN ec_host_operations o ON o.id=n.operation_id WHERE o.kind='instance.create'`).all() as { deployment_id: string; id: string }[];
expect(dispatched).toHaveLength(1); expect(dispatched).toHaveLength(1);
const payload = JSON.parse(dispatched[0].payload) as { files: { path: string; contentBase64: string; sha256: string }[] }; const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
const payload = getHostOperation(dispatched[0].id)!.payload as { files: { path: string; contentBase64: string; sha256: string }[] };
expect(payload.files.find(file => file.path.endsWith('/fixture.dll'))).toMatchObject({ expect(payload.files.find(file => file.path.endsWith('/fixture.dll'))).toMatchObject({
sha256: version.sha256, contentBase64: content.toString('base64') sha256: version.sha256, contentBase64: content.toString('base64')
}); });
@@ -0,0 +1,46 @@
import Database from 'better-sqlite3';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import schema from '../../src/lib/server/db/schema.sql?raw';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import type { LaunchRecipe } from '../../src/lib/server/fleet/launch-recipes.js';
let fixture: Fixture;
beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
afterEach(async () => { vi.restoreAllMocks(); await fixture.cleanup(); });
describe('persisted launch recipe summaries', () => {
it('lazily upgrades legacy snapshots once without changing fingerprints or retained bytes', async () => {
const recipe: LaunchRecipe = {
format: 1, game: 't5', mapSlug: 'theater', mapName: 'Kino der Toten', expectedMap: 'zombie_theater',
capturedAt: '2025-01-01T00:00:00.000Z', image: 'plutainer:test', configFile: 'dedicated_sp.cfg', configSource: 'Defaults',
files: [{ name: 'Old plugin', kind: 'plugin', path: 'legacy.dll', versionId: 1, version: 1,
sha256: 'a'.repeat(64), size: 2 * 1024 ** 2, content: Buffer.alloc(2 * 1024 ** 2, 42).toString('base64') }]
};
const legacy = new Database(fixture.dbPath);
const encoded = JSON.stringify(recipe, null, 2);
try {
legacy.exec(schema.replace(' recipe_summary TEXT,\n', ''));
legacy.exec(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,triggered_by)
VALUES('legacy','retired-server','create','done','Complete','2025-01-01','fixture');`);
legacy.prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,blueprint_name) VALUES(?,?,?)')
.run('legacy', encoded, 'Old collection');
} finally { legacy.close(); }
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
const { recipeSummaryForDeployment, recipeForDeployment, summarizeRecipe } = await import('../../src/lib/server/fleet/launch-recipes.js');
const expected = { recipe: summarizeRecipe(recipe), blueprintName: 'Old collection' };
expect(db().prepare('SELECT recipe_summary FROM server_launch_recipes').get()).toEqual({ recipe_summary: null });
expect(recipeSummaryForDeployment('legacy')).toEqual(expected);
closeDb();
const parse = JSON.parse;
const parsing = vi.spyOn(JSON, 'parse').mockImplementation((value, reviver) => {
if (value.length > 1024 * 1024) throw new Error('Legacy recipe was decoded again.');
return parse(value, reviver);
});
try {
expect(recipeSummaryForDeployment('legacy')).toEqual(expected);
expect(recipeSummaryForDeployment('missing')).toBeNull();
} finally { parsing.mockRestore(); }
expect(recipeForDeployment('legacy')).toEqual({ recipe, blueprintName: 'Old collection' });
expect(db().prepare('SELECT recipe FROM server_launch_recipes').get()).toEqual({ recipe: encoded });
});
});
+31
View File
@@ -46,6 +46,37 @@ async function seedArtifacts() {
} }
describe('launch recipes and previews', () => { describe('launch recipes and previews', () => {
it('keeps detail polling independent of retained plugin bytes after launching', async () => {
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { db, closeDb } = await import('../../src/lib/server/db/index.js');
const plugin = createScript({ name: 'Large utility plugin', kind: 'plugin', scopeType: 'game', scopeValue: 't5',
targetRelPath: 'large.dll', content: Buffer.alloc(8 * 1024 ** 2, 42) });
db().prepare('UPDATE scripts SET current_version_id = ? WHERE id = ?').run(plugin.version.id, plugin.script.id);
const { queueCreateServer, processCreations } = await import('../../src/lib/server/fleet/create.js');
const { getDeployment } = await import('../../src/lib/server/fleet/deployments.js');
const { recipeForDeployment, summarizeRecipe } = await import('../../src/lib/server/fleet/launch-recipes.js');
const queued = queueCreateServer(input);
if (!queued.ok) throw new Error(queued.message);
const queuedSummary = getDeployment(queued.deploymentId)!.recipe!;
await processCreations();
const full = recipeForDeployment(queued.deploymentId)!.recipe;
const expected = summarizeRecipe(full);
expect(expected.fingerprint).not.toBe(queuedSummary.fingerprint);
expect(full.files.find(file => file.path === 'large.dll')?.content).toBe(plugin.version.content.toString('base64'));
closeDb();
const parse = JSON.parse;
const parsing = vi.spyOn(JSON, 'parse').mockImplementation((value, reviver) => {
if (value.length > 1024 * 1024) throw new Error('Detail polling parsed plugin bytes.');
return parse(value, reviver);
});
try {
for (let read = 0; read < 3; read++) expect(getDeployment(queued.deploymentId)?.recipe).toEqual(expected);
const row = db().prepare('SELECT length(recipe_summary) AS bytes FROM server_launch_recipes WHERE deployment_id = ?')
.get(queued.deploymentId) as { bytes: number };
expect(row.bytes).toBeLessThan(10_000);
} finally { parsing.mockRestore(); }
});
it('previews without reserving capacity, saving keys, exposing bytes, or calling Komodo', async () => { it('previews without reserving capacity, saving keys, exposing bytes, or calling Komodo', async () => {
await seedArtifacts(); await seedArtifacts();
const { previewCreation } = await import('../../src/lib/server/fleet/create.js'); const { previewCreation } = await import('../../src/lib/server/fleet/create.js');
+63 -1
View File
@@ -15,6 +15,68 @@ async function enrolled() {
} }
describe('native host enrollment and scoped authentication', () => { describe('native host enrollment and scoped authentication', () => {
it('queues serialized launch files and processes receipts without hydrating plugin payloads', async () => {
const api = await enrolled();
const { db } = await import('../../src/lib/server/db/index.js');
const payload = { files: [{ path: 'plugins/fixture.dll', contentBase64: Buffer.alloc(8 * 1024 ** 2, 42).toString('base64') }] };
const bytes = Buffer.from(JSON.stringify(payload));
const options = { instanceId: 'immutable-instance', generation: 1, idempotencyKey: 'large-launch' };
const parse = JSON.parse;
const parsing = vi.spyOn(JSON, 'parse').mockImplementation((value, reviver) => {
if (value.length > 1024 * 1024) throw new Error('Launch payload was unnecessarily parsed.');
return parse(value, reviver);
});
let id: string;
try {
const operation = api.queueHostOperationSerialized(api.host.id, 'instance.create', bytes, options);
id = operation.id;
expect(operation).not.toHaveProperty('payload');
expect(api.queueHostOperationSerialized(api.host.id, 'instance.create', bytes, options).id).toBe(id);
expect(() => api.queueHostOperationSerialized(api.host.id, 'instance.create', Buffer.from('{}'), options)).toThrow('different request');
const stored = db().prepare('SELECT typeof(payload) AS kind,count(*) AS count FROM ec_host_operations WHERE idempotency_key=?').get(options.idempotencyKey);
expect(stored).toEqual({ kind: 'text', count: 1 });
const waiting = api.waitHostOperationStatus(id);
expect(api.reportHostOperationStatus(api.host.id, id, { state: 'accepted' }).state).toBe('accepted');
expect(api.reportHostOperationStatus(api.host.id, id, { state: 'running' }).state).toBe('running');
api.reportHostOperationStatus(api.host.id, id, { state: 'completed', result: { healthy: true } });
expect(await waiting).toMatchObject({ id, state: 'completed', result: { healthy: true } });
expect(api.reportHostOperationStatus(api.host.id, id, { state: 'running' }).state).toBe('completed');
expect(api.getHostOperationStatus(id)).not.toHaveProperty('payload');
const other = api.createHost({ name: 'Other host' });
expect(() => api.reportHostOperationStatus(other.id, id, { state: 'completed' })).toThrow('this host');
} finally { parsing.mockRestore(); }
// Full-data APIs retain their original contract for explicit readers.
expect(api.getHostOperation(id!)?.payload).toEqual(payload);
expect(api.reportHostOperation(api.host.id, id!, { state: 'running' }).payload).toEqual(payload);
expect((await api.waitHostOperation(id!)).payload).toEqual(payload);
});
it('keeps serialized launch routing rows small while full-data APIs hydrate separate immutable bytes', async () => {
const api = await enrolled();
const { db } = await import('../../src/lib/server/db/index.js');
const routing = { libraryId: 'test-library', name: 't5_zombie_theater_1', game: 't5' };
const payload = { ...routing, files: [{ contentBase64: 'A'.repeat(512 * 1024) }] };
const bytes = Buffer.from(JSON.stringify(payload));
const launch = api.queueHostOperationSerialized(api.host.id, 'instance.create', bytes, {}, routing);
const inline = api.queueHostOperation(api.host.id, 'instance.inspect', { name: 'inline-legacy' });
expect(api.getHostOperation(launch.id)?.payload).toEqual(payload);
expect(api.listHostOperations(api.host.id).find(operation => operation.id === launch.id)?.payload).toEqual(payload);
expect(api.pendingHostOperations(api.host.id, 'write').map(operation => operation.payload)).toEqual([payload]);
expect(api.pendingHostOperations(api.host.id, 'read')).toEqual([inline]);
expect(api.pendingHostOperationRows(api.host.id, 'write', false)[0]).not.toHaveProperty('payload');
api.reportHostOperationStatus(api.host.id, launch.id, { state: 'accepted' });
api.reportHostOperationStatus(api.host.id, launch.id, { state: 'running', result: { phase: 'starting' } });
const stored = db().prepare(`SELECT payload,last_delivered_at FROM ec_host_operations WHERE id=?`).get(launch.id) as
{ payload: string; last_delivered_at: string | null };
expect(stored.payload).toBe(JSON.stringify(routing));
expect(stored.last_delivered_at).not.toBeNull();
expect(db().prepare('SELECT length(CAST(payload AS BLOB)) AS bytes FROM ec_host_operation_payloads WHERE operation_id=?').get(launch.id))
.toEqual({ bytes: bytes.length });
expect(api.getHostOperationPayload(api.host.id, launch.id)).toEqual(bytes);
expect(api.getHostOperationPayload(api.host.id, inline.id).toString('utf8')).toBe(JSON.stringify(inline.payload));
expect(db().prepare('SELECT 1 FROM ec_host_operation_payloads WHERE operation_id=?').get(inline.id)).toBeUndefined();
});
it('narrows cancellation lookups by kind instead of scanning completed health history', async () => { it('narrows cancellation lookups by kind instead of scanning completed health history', async () => {
const api = await enrolled(); const api = await enrolled();
api.heartbeat(api.host.id, { ...inventory, capabilities: ['instance.cancel-create', 'operations.health-lane'] }); api.heartbeat(api.host.id, { ...inventory, capabilities: ['instance.cancel-create', 'operations.health-lane'] });
@@ -29,7 +91,7 @@ describe('native host enrollment and scoped authentication', () => {
const inspect = api.queueHostOperation(api.host.id, 'instance.inspect', {}); const inspect = api.queueHostOperation(api.host.id, 'instance.inspect', {});
const prepare = vi.spyOn(handle, 'prepare'); const prepare = vi.spyOn(handle, 'prepare');
expect(api.pendingHostOperations(api.host.id, 'health').map(op => op.id)).toEqual([inspect.id]); expect(api.pendingHostOperations(api.host.id, 'health').map(op => op.id)).toEqual([inspect.id]);
const query = prepare.mock.calls.map(([sql]) => sql).find(sql => sql.includes('SELECT * FROM ec_host_operations o'))!; const query = prepare.mock.calls.map(([sql]) => sql).find(sql => sql.includes('FROM ec_host_operations o WHERE host_id=?'))!;
prepare.mockRestore(); prepare.mockRestore();
// Assert the actual poll's access path, avoiding hardware-dependent timing thresholds. // Assert the actual poll's access path, avoiding hardware-dependent timing thresholds.
const plan = handle.prepare(`EXPLAIN QUERY PLAN ${query}`).all(api.host.id, 1, 1) as { detail: string }[]; const plan = handle.prepare(`EXPLAIN QUERY PLAN ${query}`).all(api.host.id, 1, 1) as { detail: string }[];
@@ -0,0 +1,227 @@
import { join } from 'node:path';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
vi.mock('../../src/lib/server/integrations/plutonium.js', () => ({
checkRelease: vi.fn(async () => {}), freshRelease: () => 5354, probeVersion: vi.fn(async () => 5354)
}));
vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) }));
let fixture: Fixture;
let hostId: string;
let libraryId: string;
const image = 'example.test/plutainer:test';
const releases: (() => void)[] = [];
const passes: Promise<void>[] = [];
beforeEach(async () => {
vi.resetModules(); vi.clearAllMocks();
fixture = await newFixture([]);
vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries'));
vi.stubEnv('PLUTAINER_IMAGE', image); vi.stubEnv('RCON_PASSWORD', 'test-password');
vi.stubEnv('KOMODO_ADDRESS', ''); vi.stubEnv('KOMODO_API_KEY', ''); vi.stubEnv('KOMODO_API_SECRET', '');
const { createHost, createEnrollment, enrollHost, heartbeat } = await import('../../src/lib/server/hosts/index.js');
hostId = createHost({ name: 'Preparation test host', portMin: 28960, portMax: 28980 }).id;
const inventory = { cpuCount: 32, cpuUsagePercent: 0, memoryTotal: 64 * 1024 ** 3,
memoryAvailable: 60 * 1024 ** 3, protocolVersion: 1, dockerRunning: true,
directoriesWritable: true, storageAvailable: 1_000_000_000_000, runtimeImages: [image] };
enrollHost(createEnrollment(hostId).token, inventory);
const { createLibraryImport, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js');
const { db } = await import('../../src/lib/server/db/index.js');
libraryId = createLibraryImport({ hostId, game: 't5' }).id;
db().prepare("UPDATE game_file_versions SET state='ready',manifest='[]',manifest_sha256=?,size_bytes=1,published_at=? WHERE id=?")
.run('a'.repeat(64), new Date().toISOString(), libraryId);
updateHostLibraryCaches(hostId, [{ libraryVersionId: libraryId, status: 'ready', manifestSha256: 'a'.repeat(64) }]);
heartbeat(hostId, { ...inventory, libraries: [{ id: libraryId, state: 'ready' }] });
});
afterEach(async () => {
for (const release of releases.splice(0)) release();
await Promise.allSettled(passes.splice(0));
vi.restoreAllMocks();
await fixture.cleanup();
vi.unstubAllEnvs();
});
async function queue() {
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
const result = queueNativeCreation({ game: 't5', gameMode: 'zombie', mapSlug: 'theater',
hostId, libraryId, image, plutoKey: 'saved-key' });
if (!result.ok) throw new Error(result.message);
return result;
}
async function runPass() {
const { processNativeProvisioning } = await import('../../src/lib/server/provisioning/native.js');
const pass = processNativeProvisioning(() => null);
passes.push(pass);
await pass;
}
async function operationFor(deploymentId: string) {
const { db } = await import('../../src/lib/server/db/index.js');
return db().prepare(`SELECT o.id FROM native_provisioning n JOIN ec_host_operations o ON o.id=n.operation_id
WHERE n.deployment_id=?`).get(deploymentId) as { id: string } | undefined;
}
/** Pause after real bytes are captured so mutations exercise the final admission checks. */
async function pausePreparation() {
const preparation = await import('../../src/lib/server/provisioning/launch-preparation.js');
const prepare = preparation.prepareNativeLaunch;
let started!: () => void;
let release!: () => void;
const ready = new Promise<void>(resolve => { started = resolve; });
const gate = new Promise<void>(resolve => { release = resolve; });
releases.push(release);
const spy = vi.spyOn(preparation, 'prepareNativeLaunch').mockImplementationOnce(async input => {
const result = await prepare(input);
started();
await gate;
return result;
});
return { ready, release, spy };
}
describe('asynchronous native launch preparation', () => {
it('keeps controller timers and launch details responsive while preparing multiple large plugins', async () => {
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
saveFleetSettings({ ...fleetSettings(), createConcurrency: 4 });
const { script, version } = createScript({ name: 'Large launch plugin', kind: 'plugin', targetRelPath: 'fixture.dll',
scopeType: 'game', scopeValue: 't5', content: Buffer.alloc(8 * 1024 ** 2, 42) });
db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(version.id, script.id);
const jobs = await Promise.all(Array.from({ length: 4 }, queue));
const hosts = await import('../../src/lib/server/hosts/index.js');
const queueCommand = hosts.queueHostOperationSerialized;
let yielded = true;
let uninterruptedCommits = 0;
const committing = vi.spyOn(hosts, 'queueHostOperationSerialized').mockImplementation((...args) => {
if (!yielded) uninterruptedCommits++;
yielded = false;
setImmediate(() => { yielded = true; });
return queueCommand(...args);
});
const parse = JSON.parse;
const stringify = JSON.stringify;
const parsing = vi.spyOn(JSON, 'parse').mockImplementation((raw, reviver) => {
if (typeof raw === 'string' && raw.length > 1_000_000) throw new Error('Large file payload parsed on the controller thread');
return parse(raw, reviver);
});
const encoding = vi.spyOn(JSON, 'stringify').mockImplementation((...args: Parameters<typeof JSON.stringify>) => {
const encoded = stringify(...args);
if (encoded && encoded.length > 1_000_000) throw new Error('Large file payload serialized on the controller thread');
return encoded;
});
let ticks = 0;
const timer = setInterval(() => { ticks++; }, 1);
try {
const pass = runPass();
await new Promise<void>(resolve => setTimeout(resolve, 0));
for (const job of jobs) expect(getDeployment(job.deploymentId)?.recipe?.files)
.toContainEqual(expect.objectContaining({ path: 'fixture.dll', size: 8 * 1024 ** 2 }));
await pass;
expect(ticks).toBeGreaterThan(1);
expect(committing).toHaveBeenCalledTimes(4);
expect(uninterruptedCommits).toBe(0);
for (const job of jobs) {
expect(await operationFor(job.deploymentId)).toBeDefined();
expect(getDeployment(job.deploymentId)?.recipe?.files)
.toContainEqual(expect.objectContaining({ path: 'fixture.dll', sha256: version.sha256 }));
}
} finally {
clearInterval(timer); parsing.mockRestore(); encoding.mockRestore(); committing.mockRestore();
}
}, 30_000);
it('does not resurrect an unsubmitted creation cancelled while its files are prepared', async () => {
const job = await queue();
const paused = await pausePreparation();
const pass = runPass();
await paused.ready;
const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
expect(cancelCreation(job.deploymentId)).toEqual(expect.any(String));
paused.release();
await pass;
const { db } = await import('../../src/lib/server/db/index.js');
expect(await operationFor(job.deploymentId)).toBeUndefined();
expect(db().prepare('SELECT name FROM servers WHERE name=?').get(job.name)).toBeUndefined();
expect(db().prepare('SELECT instance_id FROM ec_host_ports').all()).toEqual([]);
expect(db().prepare('SELECT state FROM server_deployments WHERE id=?').get(job.deploymentId)).toEqual({ state: 'cancelled' });
});
it('discards prepared bytes when the promoted version changes before dispatch', async () => {
const { createScript, uploadNewVersion } = await import('../../src/lib/server/fleet/scripts.js');
const { db } = await import('../../src/lib/server/db/index.js');
const { script, version } = createScript({ name: 'Launch plugin', kind: 'plugin', targetRelPath: 'fixture.dll',
scopeType: 'game', scopeValue: 't5', content: Buffer.from('before promotion') });
db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(version.id, script.id);
const job = await queue();
const paused = await pausePreparation();
const pass = runPass();
await paused.ready;
const content = Buffer.from('newly promoted plugin');
const latest = uploadNewVersion(script.id, content);
db().prepare('UPDATE scripts SET current_version_id=? WHERE id=?').run(latest.id, script.id);
paused.release();
await pass;
expect(await operationFor(job.deploymentId)).toBeUndefined();
await runPass();
const operation = await operationFor(job.deploymentId);
expect(operation).toBeDefined();
const { getHostOperation } = await import('../../src/lib/server/hosts/index.js');
const payload = getHostOperation(operation!.id)!.payload as { files: { contentBase64: string; sha256: string }[] };
expect(payload.files).toContainEqual(expect.objectContaining({ contentBase64: content.toString('base64'), sha256: latest.sha256 }));
});
it('prepares only one launch while the single startup slot is reserved', async () => {
const first = await queue();
const second = await queue();
const paused = await pausePreparation();
const pass = runPass();
await paused.ready;
expect(paused.spy).toHaveBeenCalledTimes(1);
const { processNativeProvisioning } = await import('../../src/lib/server/provisioning/native.js');
await processNativeProvisioning(() => null, { wait: false });
expect(paused.spy).toHaveBeenCalledTimes(1);
paused.release();
await pass;
expect(await operationFor(first.deploymentId)).toBeDefined();
expect(await operationFor(second.deploymentId)).toBeUndefined();
});
it('keeps a prepared launch queued if its host is disabled before dispatch', async () => {
const job = await queue();
const paused = await pausePreparation();
const pass = runPass();
await paused.ready;
const { updateHost } = await import('../../src/lib/server/hosts/index.js');
updateHost(hostId, { mode: 'disabled' });
paused.release();
await pass;
expect(await operationFor(job.deploymentId)).toBeUndefined();
updateHost(hostId, { mode: 'active' });
await runPass();
expect(await operationFor(job.deploymentId)).toBeDefined();
});
it('rechecks a reduced startup limit before submitting prepared files', async () => {
const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 });
const first = await queue();
await runPass();
expect(await operationFor(first.deploymentId)).toBeDefined();
const second = await queue();
const paused = await pausePreparation();
const pass = runPass();
await paused.ready;
saveFleetSettings({ ...fleetSettings(), createConcurrency: 1 });
paused.release();
await pass;
expect(await operationFor(second.deploymentId)).toBeUndefined();
saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 });
await runPass();
expect(await operationFor(second.deploymentId)).toBeDefined();
});
});
+30
View File
@@ -246,7 +246,9 @@ describe('native instance provisioning', () => {
try { try {
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js'); const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
expect(getDeployment(first.deploymentId)!.progress?.phase).toBe('verify-container'); expect(getDeployment(first.deploymentId)!.progress?.phase).toBe('verify-container');
await vi.waitFor(async () => {
expect((await operationFor(second.deploymentId))?.kind).toBe('instance.create'); expect((await operationFor(second.deploymentId))?.kind).toBe('instance.create');
}, { timeout: 5_000 });
} finally { } finally {
release({ healthy: true, state: 'running' } as never); release({ healthy: true, state: 'running' } as never);
await pending; await pending;
@@ -268,7 +270,9 @@ describe('native instance provisioning', () => {
try { try {
const second = await queue(); const second = await queue();
await processProvisioning({ wait: false }); await processProvisioning({ wait: false });
await vi.waitFor(async () => {
expect((await operationFor(second.deploymentId))?.kind).toBe('instance.create'); expect((await operationFor(second.deploymentId))?.kind).toBe('instance.create');
}, { timeout: 5_000 });
expect(vi.mocked(nativeRequest)).toHaveBeenCalledTimes(1); expect(vi.mocked(nativeRequest)).toHaveBeenCalledTimes(1);
} finally { } finally {
release({ healthy: true, state: 'running' } as never); release({ healthy: true, state: 'running' } as never);
@@ -401,6 +405,32 @@ describe('native instance provisioning', () => {
await advance(); await advance();
expect(nativeInstance(job.name)?.generation).toBe(2); expect(nativeInstance(job.name)?.generation).toBe(2);
}); });
it('holds a rebuild when players join while its launch files are being prepared', async () => {
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
const rebuild = queueNativeRebuild(job.name, 'test'); if (!rebuild.ok) throw new Error(rebuild.message);
const preparation = await import('../../src/lib/server/provisioning/launch-preparation.js');
const prepare = preparation.prepareNativeLaunch;
let started!: () => void;
let release!: () => void;
const ready = new Promise<void>(resolve => { started = resolve; });
const gate = new Promise<void>(resolve => { release = resolve; });
const spy = vi.spyOn(preparation, 'prepareNativeLaunch').mockImplementationOnce(async input => {
const result = await prepare(input);
started(); await gate;
return result;
});
const pending = advance();
try {
await ready;
remote.players = 2;
release(); await pending;
expect(await operationFor(rebuild.deploymentId)).toBeNull();
remote.players = 0;
await advance();
expect((await operationFor(rebuild.deploymentId))?.kind).toBe('instance.rebuild');
} finally { release(); await pending; spy.mockRestore(); }
});
it('finishes native rebuild verification without waiting for its own recovery slot to be released', async () => { it('finishes native rebuild verification without waiting for its own recovery slot to be released', async () => {
const job = await queue(); await advance(); await acknowledge(job.deploymentId); const job = await queue(); await advance(); await acknowledge(job.deploymentId);
const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js'); const { queueNativeRebuild } = await import('../../src/lib/server/provisioning/native.js');
+16
View File
@@ -0,0 +1,16 @@
import { defineConfig } from 'vite';
import { resolve } from 'node:path';
// Reuse launch composition with an independent read-only database connection.
export default defineConfig({
plugins: [{ name: 'launch-database', enforce: 'pre', resolveId(source) {
if (source.endsWith('/db/index.js')) return resolve('src/lib/server/reports/database.ts');
} }],
build: {
ssr: true, outDir: '.svelte-kit/launch', emptyOutDir: true,
rollupOptions: {
input: 'src/lib/server/provisioning/launch-worker.ts', external: ['better-sqlite3'],
output: { entryFileNames: 'worker.mjs' }
}
}
});