josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Keep native collection launches responsive

Browse Source
This commit is contained in:
josh
2026-10-05 10:48:38 -04:00
parent dbd2940d29
commit 1405114a14
26 changed files with 998 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/backends/native.ts
+2 -2
View File
@@ -1,6 +1,6 @@
import { randomUUID } from 'node:crypto';
import { db } from '../db/index.js';
import { getHost, queueHostOperation, waitHostOperation } from '../hosts/index.js';
import { getHost, queueHostOperation, waitHostOperationStatus } from '../hosts/index.js';
import { trackActivity } from '../pipeline/activity.js';
import { latestGameSubmission } from '../fleet/lifecycle-evidence.js';
@@ -43,7 +43,7 @@ async function executeRequest<T>(instance: NativeInstance, kind: string, payload
instanceId: instance.id, generation: instance.generation, idempotencyKey: key ?? randomUUID()
});
const result = await trackActivity(`agent:${operation.id}`, `Waiting for ${kind} on ${instance.server}`,
() => waitHostOperation(operation.id, 25_000));
() => waitHostOperationStatus(operation.id, 25_000));
if (result.state !== 'completed') throw new Error(result.error || `Agent operation ${result.id} is ${result.state}; its original ID is retained.`);
// Do not apply replies from a retired generation to a replacement server.
const current = nativeInstance(instance.server);
src/lib/server/db/index.ts
+3
View File
@@ -345,6 +345,9 @@ export function db(): Database.Database {
{ name: 'mode', ddl: "ALTER TABLE server_launch_collections ADD COLUMN mode TEXT NOT NULL DEFAULT 'fill'" },
{ name: 'retained_servers', ddl: "ALTER TABLE server_launch_collections ADD COLUMN retained_servers TEXT NOT NULL DEFAULT '[]'" }
]);
migrateTable(handle, 'server_launch_recipes', [
{ name: 'recipe_summary', ddl: 'ALTER TABLE server_launch_recipes ADD COLUMN recipe_summary TEXT' }
]);
migrateServerHistory(handle);
migrateTable(handle, 'server_collection_previews', [
{ name: 'mode', ddl: "ALTER TABLE server_collection_previews ADD COLUMN mode TEXT NOT NULL DEFAULT 'fill'" },
src/lib/server/db/schema.sql
+1
View File
@@ -522,6 +522,7 @@ CREATE TABLE IF NOT EXISTS server_launch_previews (
CREATE TABLE IF NOT EXISTS server_launch_recipes (
deployment_id TEXT PRIMARY KEY REFERENCES server_deployments(id) ON DELETE CASCADE,
recipe TEXT NOT NULL,
recipe_summary TEXT,
blueprint_name TEXT
);
src/lib/server/fleet/launch-recipes.ts
+21 -2
View File
@@ -109,10 +109,29 @@ function buildRecipe(game: ZombieGame, mapSlug: string, includeContent: boolean)
configFile: configFileForGame(game), configSource, files };
}
export function summarizeRecipe(recipe: LaunchRecipePlan): LaunchRecipeSummary {
export function summarizeRecipe(recipe: LaunchRecipePlan, fingerprint = sha256Buffer(Buffer.from(JSON.stringify(recipe)))): LaunchRecipeSummary {
const { format: _format, files, ...summary } = recipe;
return { ...summary, files: files.map(({ content: _content, ...file }) => file),
fingerprint: sha256Buffer(Buffer.from(JSON.stringify(recipe))) };
fingerprint };
}
/** Keep the small detail response beside the immutable attempted file bytes. */
export function recipeStorage(recipe: LaunchRecipePlan): { recipe: string; recipeSummary: string } {
const encoded = JSON.stringify(recipe);
return { recipe: encoded, recipeSummary: JSON.stringify(summarizeRecipe(recipe, sha256Buffer(Buffer.from(encoded)))) };
}
export function recipeSummaryForDeployment(id: string): { recipe: LaunchRecipeSummary; blueprintName: string | null } | null {
const row = db().prepare('SELECT recipe_summary, blueprint_name FROM server_launch_recipes WHERE deployment_id = ?')
.get(id) as { recipe_summary: string | null; blueprint_name: string | null } | undefined;
if (!row) return null;
if (row.recipe_summary !== null) return { recipe: JSON.parse(row.recipe_summary), blueprintName: row.blueprint_name };
// Old snapshots remain readable. Populate on first access, without making
// controller startup scan and decode every retained plugin in its history.
const launch = recipeForDeployment(id)!;
const summary = summarizeRecipe(launch.recipe);
db().prepare('UPDATE server_launch_recipes SET recipe_summary = ? WHERE deployment_id = ?').run(JSON.stringify(summary), id);
return { recipe: summary, blueprintName: row.blueprint_name };
}
export function recipeForDeployment(id: string): { recipe: LaunchRecipePlan; blueprintName: string | null } | null {
src/lib/server/hosts/delivery.ts
+62
View File
@@ -0,0 +1,62 @@
import { setImmediate } from 'node:timers/promises';
import { getHostOperationPayload, type OperationDeliveryRow } from './index.js';
// Even non-ASCII content stays below 64 KiB per encoded chunk.
const CHUNK_CHARACTERS = 16 * 1024;
const CHUNK_BYTES = 64 * 1024;
const encoder = new TextEncoder();
function* jsonParts(operations: OperationDeliveryRow[], serverTime: string): Generator<string | Buffer> {
yield '{"operations":[';
for (const [index, operation] of operations.entries()) {
if (index) yield ',';
const metadata = JSON.stringify({
id: operation.id, hostId: operation.host_id, kind: operation.kind,
instanceId: operation.instance_id, generation: operation.generation, state: operation.state,
error: operation.error, createdAt: operation.created_at, updatedAt: operation.updated_at
});
yield metadata.slice(0, -1) + ',"payload":';
// Load only the immutable payload currently being sent, after the
// stream has yielded its metadata and had a chance to serve health work.
yield getHostOperationPayload(operation.host_id, operation.id);
yield ',"result":';
yield operation.result ?? 'null';
yield '}';
}
yield '],"serverTime":' + JSON.stringify(serverTime) + '}';
}
function* encodedChunks(operations: OperationDeliveryRow[], serverTime: string): Generator<Uint8Array> {
for (const part of jsonParts(operations, serverTime)) {
if (Buffer.isBuffer(part)) {
for (let start = 0; start < part.length; start += CHUNK_BYTES) yield part.subarray(start, start + CHUNK_BYTES);
continue;
}
for (let start = 0; start < part.length;) {
let end = Math.min(start + CHUNK_CHARACTERS, part.length);
// A UTF-16 surrogate pair must be encoded together, including when
// user-authored configuration text falls on a chunk boundary.
const last = part.charCodeAt(end - 1);
if (end < part.length && last >= 0xd800 && last <= 0xdbff) end--;
yield encoder.encode(part.slice(start, end));
start = end;
}
}
}
/** Bound encoding work and yield between chunks so health requests keep flowing. */
export function hostOperationsResponse(operations: OperationDeliveryRow[], serverTime = new Date().toISOString()): Response {
const chunks = encodedChunks(operations, serverTime);
let cancelled = false;
const stream = new ReadableStream<Uint8Array>({
async pull(controller) {
await setImmediate();
if (cancelled) return;
const next = chunks.next();
if (next.done) controller.close();
else controller.enqueue(next.value);
},
cancel() { cancelled = true; chunks.return(undefined); }
});
return new Response(stream, { headers: { 'Content-Type': 'application/json; charset=utf-8', 'Cache-Control': 'no-store' } });
}
src/lib/server/hosts/index.ts
+80 -23
View File
@@ -30,7 +30,7 @@ function decommissionRow(hostId: string): DecommissionRow | undefined {
}
function presentDecommission(row: DecommissionRow | undefined): HostDecommission | null {
return row ? { operationId: row.operation_id, stage: row.stage, requestedAt: row.requested_at, updatedAt: row.updated_at,
error: row.error, retryable: row.stage === 'failed' && getHostOperation(row.operation_id)?.state === 'failed' } : null;
error: row.error, retryable: row.stage === 'failed' && getHostOperationStatus(row.operation_id)?.state === 'failed' } : null;
}
function hostFromRow(row: HostRow): Host {
const inventory: HostInventory = JSON.parse(row.inventory);
@@ -137,7 +137,7 @@ export function receiveDecommissionReceipt(operationId: string, token: string, i
if (!row || !timingSafeEqual(Buffer.from(row.token_hash, 'hex'), Buffer.from(digest(token), 'hex'))) throw new HostDeletionError('A valid cleanup receipt credential is required.', 401);
if (!['cleaned', 'completed', 'failed'].includes(input.stage as string)) throw new HostDeletionError('Invalid cleanup receipt stage.', 400);
if (row.stage === 'completed') return { acknowledged: true as const };
const operation = getHostOperation(operationId)!;
const operation = getHostOperationStatus(operationId)!;
if (!['accepted','running','unknown'].includes(operation.state)) throw new HostDeletionError('Cleanup has not been accepted by the host.', 409);
const timestamp = now();
if (input.stage === 'failed') {
@@ -314,41 +314,82 @@ export function requireReadyHost(hostId: string, options: { libraryId?: string;
return host;
}
type OperationRow = { id: string; host_id: string; kind: string; instance_id: string | null; generation: number | null; payload: string; state: HostOperationState; result: string | null; error: string | null; created_at: string; updated_at: string };
function operationFromRow(row: OperationRow): HostOperation { return { id: row.id, hostId: row.host_id, kind: row.kind, instanceId: row.instance_id, generation: row.generation, payload: JSON.parse(row.payload), state: row.state, result: row.result === null ? null : JSON.parse(row.result), error: row.error, createdAt: row.created_at, updatedAt: row.updated_at }; }
export function getHostOperation(id: string): HostOperation | null { const row = database().prepare('SELECT * FROM ec_host_operations WHERE id=?').get(id) as OperationRow | undefined; return row ? operationFromRow(row) : null; }
export function listHostOperations(hostId: string, limit = 100, summary = false): HostOperation[] { return (database().prepare(`SELECT ${summary ? `id,host_id,kind,instance_id,generation,state,error,created_at,updated_at,'{}' AS payload,NULL AS result` : '*'} FROM ec_host_operations WHERE host_id=? ORDER BY created_at DESC,rowid DESC LIMIT ?`).all(hostId, Math.min(500, Math.max(1, limit))) as OperationRow[]).map(operationFromRow); }
export type OperationRow = { id: string; host_id: string; kind: string; instance_id: string | null; generation: number | null; payload: string; state: HostOperationState; result: string | null; error: string | null; created_at: string; updated_at: string };
export type OperationDeliveryRow = Omit<OperationRow, 'payload'>;
export type HostOperationStatus = Omit<HostOperation, 'payload'>;
const operationStatusColumns = 'o.id,o.host_id,o.kind,o.instance_id,o.generation,o.state,o.result,o.error,o.created_at,o.updated_at';
const operationPayloadSql = 'COALESCE((SELECT p.payload FROM ec_host_operation_payloads p WHERE p.operation_id=o.id),o.payload)';
function operationStatusFromRow(row: Omit<OperationRow, 'payload'>): HostOperationStatus { return { id: row.id, hostId: row.host_id, kind: row.kind, instanceId: row.instance_id, generation: row.generation, state: row.state, result: row.result === null ? null : JSON.parse(row.result), error: row.error, createdAt: row.created_at, updatedAt: row.updated_at }; }
function operationFromRow(row: OperationRow): HostOperation { return { ...operationStatusFromRow(row), payload: JSON.parse(row.payload) }; }
export function getHostOperation(id: string): HostOperation | null {
const row = database().prepare(`SELECT ${operationStatusColumns},${operationPayloadSql} AS payload FROM ec_host_operations o WHERE o.id=?`)
.get(id) as OperationRow | undefined;
return row ? operationFromRow(row) : null;
}
/** Polling and receipts need the outcome, never another copy of launch files. */
export function getHostOperationStatus(id: string): HostOperationStatus | null {
const row = database().prepare('SELECT id,host_id,kind,instance_id,generation,state,result,error,created_at,updated_at FROM ec_host_operations WHERE id=?')
.get(id) as Omit<OperationRow, 'payload'> | undefined;
return row ? operationStatusFromRow(row) : null;
}
export function listHostOperations(hostId: string, limit = 100, summary = false): HostOperation[] {
const columns = summary ? `o.id,o.host_id,o.kind,o.instance_id,o.generation,o.state,o.error,o.created_at,o.updated_at,'{}' AS payload,NULL AS result`
: `${operationStatusColumns},${operationPayloadSql} AS payload`;
return (database().prepare(`SELECT ${columns} FROM ec_host_operations o WHERE o.host_id=? ORDER BY o.created_at DESC,o.rowid DESC LIMIT ?`)
.all(hostId, Math.min(500, Math.max(1, limit))) as OperationRow[]).map(operationFromRow);
}
/** Heartbeats can tell an executing create to stop at its next safe command boundary. */
export function pendingCreationCancellations(hostId: string): string[] {
return (database().prepare(`SELECT json_extract(payload,'$.createOperationId') AS id FROM ec_host_operations
WHERE host_id=? AND kind='instance.cancel-create' AND state IN ('pending','accepted','running','unknown')`).all(hostId) as { id: string }[]).map(row => row.id);
}
export function queueHostOperation(hostId: string, kind: string, payload: Record<string, unknown>, options: { instanceId?: string; generation?: number; idempotencyKey?: string; id?: string } = {}): HostOperation {
type QueueHostOperationOptions = { instanceId?: string; generation?: number; idempotencyKey?: string; id?: string };
export function queueHostOperation(hostId: string, kind: string, payload: Record<string, unknown>, options: QueueHostOperationOptions = {}): HostOperation {
if (!payload || typeof payload !== 'object' || Array.isArray(payload)) throw new Error('Operation payload must be an object.');
const operation = enqueueHostOperation(hostId, kind, JSON.stringify(payload), options);
return getHostOperation(operation.id)!;
}
/** Keep immutable launch bytes separate so receipts and delivery do not rewrite large payloads. */
export function queueHostOperationSerialized(hostId: string, kind: string, payload: Buffer, options: QueueHostOperationOptions = {}, routingPayload: Record<string, unknown> = {}): HostOperationStatus {
if (!Buffer.isBuffer(payload) || payload[0] !== 123 || payload[payload.length - 1] !== 125) throw new Error('Serialized operation payload must be a JSON object.');
if (!routingPayload || typeof routingPayload !== 'object' || Array.isArray(routingPayload)) throw new Error('Operation routing payload must be an object.');
return enqueueHostOperation(hostId, kind, payload, options, JSON.stringify(routingPayload));
}
function enqueueHostOperation(hostId: string, kind: string, payload: string | Buffer, options: QueueHostOperationOptions, routingPayload?: string): HostOperationStatus {
const host = requireHost(hostId);
if (host.decommission) throw new Error('Host cleanup is in progress; new operations are blocked.');
if (kind === 'host.decommission') throw new Error('Use the host deletion workflow to request cleanup.');
if (!/^[a-z][a-z0-9_.-]{0,79}$/.test(kind)) throw new Error('Invalid operation kind.');
if (!payload || typeof payload !== 'object' || Array.isArray(payload)) throw new Error('Operation payload must be an object.');
if (options.generation !== undefined && (!Number.isSafeInteger(options.generation) || options.generation < 1)) throw new Error('Instance generation must be a positive integer.');
const operation = database().transaction(() => {
if (options.idempotencyKey) {
const existing = database().prepare('SELECT * FROM ec_host_operations WHERE host_id=? AND idempotency_key=?').get(hostId, options.idempotencyKey) as OperationRow | undefined;
const existing = database().prepare(`SELECT o.id,o.kind,o.instance_id,o.generation,${operationPayloadSql}=CAST(? AS TEXT) AS same_payload
FROM ec_host_operations o WHERE o.host_id=? AND o.idempotency_key=?`).get(payload, hostId, options.idempotencyKey) as
{ id: string; kind: string; instance_id: string | null; generation: number | null; same_payload: number } | undefined;
if (existing) {
if (existing.kind !== kind || existing.instance_id !== (options.instanceId ?? null) || existing.generation !== (options.generation ?? null) || existing.payload !== JSON.stringify(payload)) throw new Error('Operation idempotency key is already associated with a different request.');
return operationFromRow(existing);
if (existing.kind !== kind || existing.instance_id !== (options.instanceId ?? null) || existing.generation !== (options.generation ?? null) || !existing.same_payload) throw new Error('Operation idempotency key is already associated with a different request.');
return getHostOperationStatus(existing.id)!;
}
}
if (['instance.create','create'].includes(kind)) requireReadyHost(hostId);
if (host.mode === 'disabled' && kind !== 'instance.cancel-create') throw new Error('Host is disabled. Enable it before requesting operations.');
const id = options.id ?? randomUUID(); const timestamp = now();
database().prepare('INSERT INTO ec_host_operations(id,host_id,kind,instance_id,generation,idempotency_key,payload,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?)').run(id, hostId, kind, options.instanceId ?? null, options.generation ?? null, options.idempotencyKey ?? null, JSON.stringify(payload), timestamp, timestamp);
database().prepare('INSERT INTO ec_host_operations(id,host_id,kind,instance_id,generation,idempotency_key,payload,created_at,updated_at) VALUES(?,?,?,?,?,?,CAST(? AS TEXT),?,?)').run(id, hostId, kind, options.instanceId ?? null, options.generation ?? null, options.idempotencyKey ?? null, routingPayload ?? payload, timestamp, timestamp);
if (routingPayload !== undefined) database().prepare('INSERT INTO ec_host_operation_payloads(operation_id,payload) VALUES(?,CAST(? AS TEXT))').run(id, payload);
database().prepare('INSERT INTO ec_host_operation_events(operation_id,state,created_at) VALUES(?,?,?)').run(id, 'pending', timestamp);
return getHostOperation(id)!;
return getHostOperationStatus(id)!;
})();
queueMicrotask(() => workEvents.emit(hostId));
return operation;
}
export function pendingHostOperations(hostId: string, lane: 'all' | 'read' | 'health' | 'write' = 'all'): HostOperation[] {
return pendingHostOperationRows(hostId, lane).map(operationFromRow);
}
type OperationLane = 'all' | 'read' | 'health' | 'write';
export function pendingHostOperationRows(hostId: string, lane?: OperationLane, includePayload?: true): OperationRow[];
export function pendingHostOperationRows(hostId: string, lane: OperationLane, includePayload: false): OperationDeliveryRow[];
/** Delivery snapshots retain status/results while large immutable payloads stay in SQLite. */
export function pendingHostOperationRows(hostId: string, lane: OperationLane = 'all', includePayload = true): OperationDeliveryRow[] {
const host = requireHost(hostId);
if (host.mode === 'disabled' || !host.connected) return [];
// Older agents hold activity.lock across the entire delivered batch, including
@@ -361,8 +402,8 @@ export function pendingHostOperations(hostId: string, lane: 'all' | 'read' | 'he
// Accepted/running/unknown commands keep the same immutable ID. The agent
// consults its durable journal before acknowledging or executing a replay.
const readOnly = `(kind IN ('host.inspect','instance.inspect','files.read','files.list','files.stat','logs.read')
OR (kind='rcon.execute' AND json_extract(payload,'$.command') IN ('status','getstatus','version')))`;
const healthOnly = `(kind='instance.inspect' OR (kind='rcon.execute' AND json_extract(payload,'$.command') IN ('status','getstatus','version')))`;
OR (kind='rcon.execute' AND json_extract(${operationPayloadSql},'$.command') IN ('status','getstatus','version')))`;
const healthOnly = `(kind='instance.inspect' OR (kind='rcon.execute' AND json_extract(${operationPayloadSql},'$.command') IN ('status','getstatus','version')))`;
const healthLane = host.inventory.capabilities?.includes('operations.health-lane');
const laneFilter = lane === 'all' ? '' : lane === 'health' ? ` AND ${healthOnly}`
: ` AND ${lane === 'write' ? 'NOT ' : ''}${readOnly}${lane === 'read' && healthLane ? ` AND NOT ${healthOnly}` : ''}`;
@@ -370,23 +411,31 @@ export function pendingHostOperations(hostId: string, lane: 'all' | 'read' | 'he
// in small batches to amortize delivery without handing out a long backlog.
const limit = healthLane ? lane === 'health' ? 1 : lane === 'read' ? 4 : 25 : 25;
return database().transaction(() => {
const rows = database().prepare(`SELECT * FROM ec_host_operations o WHERE host_id=? AND state IN ('pending','accepted','running','unknown')
const columns = includePayload ? `${operationStatusColumns},${operationPayloadSql} AS payload` : operationStatusColumns;
const rows = database().prepare(`SELECT ${columns} FROM ec_host_operations o WHERE host_id=? AND state IN ('pending','accepted','running','unknown')
${laneFilter}
AND (? OR state != 'pending' OR kind NOT IN ('instance.create','create'))
AND (kind!='instance.cancel-create' OR ?)
AND NOT EXISTS (SELECT 1 FROM ec_host_operations cancellation WHERE cancellation.host_id=o.host_id
AND cancellation.kind='instance.cancel-create' AND json_extract(cancellation.payload,'$.createOperationId')=o.id)
ORDER BY CASE WHEN kind='instance.cancel-create' THEN 0 ELSE 1 END,COALESCE(last_delivered_at,''),created_at,rowid LIMIT ${limit}`)
.all(hostId, Number(host.readiness === 'ready'), Number(host.inventory.capabilities?.includes('instance.cancel-create') ?? false)) as OperationRow[];
.all(hostId, Number(host.readiness === 'ready'), Number(host.inventory.capabilities?.includes('instance.cancel-create') ?? false)) as OperationDeliveryRow[];
const deliveredAt = now();
for (const operation of rows) database().prepare('UPDATE ec_host_operations SET last_delivered_at=? WHERE id=?').run(deliveredAt, operation.id);
return rows.map(operationFromRow);
return rows;
})();
}
export function reportHostOperation(hostId: string, id: string, input: { state: unknown; result?: unknown; error?: unknown }): HostOperation {
/** Payloads never change after enqueueing; fetch one delivery's UTF-8 bytes without decoding. */
export function getHostOperationPayload(hostId: string, id: string): Buffer {
const row = database().prepare(`SELECT CAST(${operationPayloadSql} AS BLOB) AS payload FROM ec_host_operations o WHERE o.id=? AND o.host_id=?`)
.get(id, hostId) as { payload: Buffer } | undefined;
if (!row) throw new Error('Host operation was not found for this host.');
return row.payload;
}
export function reportHostOperationStatus(hostId: string, id: string, input: { state: unknown; result?: unknown; error?: unknown }): HostOperationStatus {
const operation = database().transaction(() => {
requireHost(hostId);
const operation = getHostOperation(id);
const operation = getHostOperationStatus(id);
if (!operation || operation.hostId !== hostId) throw new Error('Operation was not found for this host.');
if (operation.kind === 'host.decommission' && input.state === 'completed') throw new Error('Host removal requires the cleanup worker completion receipt.');
if (!['accepted','running','completed','failed','unknown'].includes(input.state as string)) throw new Error('Invalid operation state.');
@@ -407,21 +456,29 @@ export function reportHostOperation(hostId: string, id: string, input: { state:
recordAgentReceipt(id, state, now());
}
}
return getHostOperation(id)!;
return getHostOperationStatus(id)!;
})();
queueMicrotask(() => workEvents.emit(`operation:${id}`));
return operation;
}
export async function waitHostOperation(id: string, timeoutMs = 30_000): Promise<HostOperation> {
export function reportHostOperation(hostId: string, id: string, input: { state: unknown; result?: unknown; error?: unknown }): HostOperation {
reportHostOperationStatus(hostId, id, input);
return getHostOperation(id)!;
}
export async function waitHostOperationStatus(id: string, timeoutMs = 30_000): Promise<HostOperationStatus> {
const deadline = Date.now() + timeoutMs;
while (true) {
const operation = getHostOperation(id);
const operation = getHostOperationStatus(id);
if (!operation) throw new Error('Host operation was not found.');
if (['completed','failed','unknown'].includes(operation.state)) return operation;
if (Date.now() >= deadline) throw new Error(`Host operation ${id} is awaiting acknowledgement; reconcile this operation ID before retrying.`);
await waitForHostWork(`operation:${id}`, new AbortController().signal, Math.min(1000, Math.max(1, deadline - Date.now())));
}
}
export async function waitHostOperation(id: string, timeoutMs = 30_000): Promise<HostOperation> {
await waitHostOperationStatus(id, timeoutMs);
return getHostOperation(id)!;
}
export function hostPortAvailable(hostId: string, candidate: number, bindAddress?: string, protocol: 'udp' | 'tcp' = 'udp', instanceId?: string): boolean {
const host = requireHost(hostId); port(candidate); const bind = bindAddress ?? host.bindAddress;
src/lib/server/hosts/schema.ts
+4
View File
@@ -33,6 +33,10 @@ export function ensureHostSchema(handle: Database.Database): void {
-- history. The queue index cannot narrow those lookups beyond host_id,
-- causing a history scan for every queued command on Node's main thread.
CREATE INDEX IF NOT EXISTS ec_host_operations_kind ON ec_host_operations(host_id, kind, created_at);
CREATE TABLE IF NOT EXISTS ec_host_operation_payloads (
operation_id TEXT PRIMARY KEY REFERENCES ec_host_operations(id) ON DELETE CASCADE,
payload TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS ec_host_operation_events (
id INTEGER PRIMARY KEY AUTOINCREMENT, operation_id TEXT NOT NULL REFERENCES ec_host_operations(id),
state TEXT NOT NULL, created_at TEXT NOT NULL, error TEXT
src/lib/server/provisioning/history.ts
+3 -3
View File
@@ -1,6 +1,6 @@
import { randomUUID } from 'node:crypto';
import { db } from '../db/index.js';
import { recipeForDeployment, summarizeRecipe } from '../fleet/launch-recipes.js';
import { recipeSummaryForDeployment } from '../fleet/launch-recipes.js';
import { actionLabels, type ProvisioningAction } from '../../provisioning.js';
import type { ProvisioningCapacity, ProvisioningDeployment, ProvisioningDetail, ProvisioningSource } from '../../provisioning-detail.js';
import { fleetSettings } from '../settings.js';
@@ -111,7 +111,7 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
const { observation_json: _snapshot, phase_key: _phase, phase_started_at: _phaseStarted, observed_at: _observed, ...deployment } = row;
const sources = (db().prepare('SELECT kind, source_id, server FROM provisioning_sources WHERE operation_id = ? ORDER BY kind, source_id')
.all(id) as Omit<ProvisioningSource, 'label' | 'href'>[]).map(sourceDetails);
const launch = recipeForDeployment(id);
const launch = recipeSummaryForDeployment(id);
const collection = db().prepare(`SELECT c.id, c.name FROM (
SELECT collection_id, deployment_id FROM server_collection_members
UNION ALL SELECT collection_id, deployment_id FROM server_collection_removals) m
@@ -120,7 +120,7 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
canCancelCreation: canCancelCreation(id), cancellation: cancellation ?? null,
workerActivity: workerActivity(), events: deploymentEvents(id, options.events === 'recent'),
activity: { ...activitySummary(id), failureExcerpt: latestFailureEvent(id) }, sources,
recipe: launch ? summarizeRecipe(launch.recipe) : null, blueprintName: launch?.blueprintName ?? null,
recipe: launch?.recipe ?? null, blueprintName: launch?.blueprintName ?? null,
canResumeVerification: canResumeVerification(id), canResumeCleanup: canResumeCleanup(id),
canResumeRebuild: canResumeRebuild(id), canCancelRemoval: canCancelRemoval(id), collection: collection ?? null,
progress: readProgress(row), capacity: capacityFor(deployment, generatedAt),
src/lib/server/provisioning/instances.ts
+11 -7
View File
@@ -4,7 +4,7 @@ import { probePlayerEvidence, probeServerStatus } from '../fleet/status-evidence
import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent, REMOVING_SERVER, resumeCleanup, resumeVerification, claimRemoval } from './history.js';
import { randomUUID } from 'node:crypto';
import type { LaunchPreview } from '$lib/deployment-plan.js';
import { blueprintRecipe, captureRecipe, captureRecipePlan, recipeForDeployment, summarizeRecipe, type LaunchRecipe, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
import { blueprintRecipe, captureRecipe, captureRecipePlan, recipeForDeployment, recipeStorage, summarizeRecipe, type LaunchRecipe, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
import { mkdir, chmod, rm, writeFile } from 'node:fs/promises';
import { dirname, join } from 'node:path';
import { config, destOfKind, serverDir, gameOfServer, configFileForGame } from '../config.js';
@@ -240,9 +240,10 @@ export function queueCreateServer(input: CreateServerInput, releasingServers: st
if (!prepared.ok) return prepared;
// A preview records choices, not version pins. Promotions remain authoritative.
const currentRecipe = captureRecipePlan(input.game ?? 't5', prepared.mapEntry.slug);
const storedRecipe = recipeStorage(currentRecipe);
const id = beginDeployment(prepared.name, 'create', input.createdBy);
db().prepare('INSERT INTO server_launch_recipes (deployment_id, recipe, blueprint_name) VALUES (?, ?, ?)')
.run(id, JSON.stringify(currentRecipe), preview?.blueprint_name ?? null);
db().prepare('INSERT INTO server_launch_recipes (deployment_id, recipe, recipe_summary, blueprint_name) VALUES (?, ?, ?, ?)')
.run(id, storedRecipe.recipe, storedRecipe.recipeSummary, preview?.blueprint_name ?? null);
if (preview) db().prepare('UPDATE server_launch_previews SET deployment_id = ? WHERE id = ?').run(id, preview.id);
db().prepare("UPDATE server_deployments SET state = 'queued' WHERE id = ?").run(id);
db().prepare('INSERT INTO server_creation_jobs (deployment_id, server, port, game_mode, map_slug, expected_map) VALUES (?, ?, ?, ?, ?, ?)')
@@ -275,7 +276,9 @@ export function queueRebuildServer(name: string, triggeredBy = 'ui'): CreateServ
db().prepare('INSERT INTO server_creation_jobs (deployment_id, server, port, game_mode, map_slug, expected_map) VALUES (?, ?, ?, ?, ?, ?)')
.run(id, name, server.game_port, 'zombie', map.slug, map.canonical);
db().prepare('INSERT INTO server_rebuild_jobs (deployment_id) VALUES (?)').run(id);
db().prepare('INSERT INTO server_launch_recipes (deployment_id, recipe) VALUES (?, ?)').run(id, JSON.stringify(recipe));
const storedRecipe = recipeStorage(recipe);
db().prepare('INSERT INTO server_launch_recipes (deployment_id, recipe, recipe_summary) VALUES (?, ?, ?)')
.run(id, storedRecipe.recipe, storedRecipe.recipeSummary);
deploymentStage(id, 'Waiting to rebuild the empty server', { phase: 'eligibility', recipeStatus: 'queued', observedAt: null,
blocker: { code: 'eligibility', message: 'Waiting for safe rebuild eligibility checks.', automatic: true } });
return { ok: true as const, name, port: server.game_port!, deploymentId: id };
@@ -399,10 +402,11 @@ async function createServerUnlocked(input: CreateServerInput, prepared: Prepared
// Resolve once more at provisioning so promotions made while queued also apply.
// Persist the exact attempted files for history, never as input to future launches.
const recipe = captureRecipe(game, mapEntry.slug);
const storedRecipe = recipeStorage(recipe);
db().transaction(() => {
db().prepare(`INSERT INTO server_launch_recipes (deployment_id, recipe) VALUES (?, ?)
ON CONFLICT(deployment_id) DO UPDATE SET recipe = excluded.recipe`)
.run(operation, JSON.stringify(recipe));
db().prepare(`INSERT INTO server_launch_recipes (deployment_id, recipe, recipe_summary) VALUES (?, ?, ?)
ON CONFLICT(deployment_id) DO UPDATE SET recipe = excluded.recipe, recipe_summary = excluded.recipe_summary`)
.run(operation, storedRecipe.recipe, storedRecipe.recipeSummary);
deploymentStage(operation, 'Preparing server files', { phase: 'prepare', recipeStatus: 'attempted', blocker: null, checks: [],
checkpoint: { key: 'startup-queue', label: 'Admitted to startup' },
event: { kind: 'recipe', message: 'Promoted files selected for this creation attempt.', details: { capturedAt: recipe.capturedAt } } });
src/lib/server/provisioning/launch-preparation.ts
+71
View File
@@ -0,0 +1,71 @@
import { Worker } from 'node:worker_threads';
import { resolve } from 'node:path';
import { config } from '../config.js';
import { db } from '../db/index.js';
import type { NativeInstance } from '../backends/native.js';
import type { LaunchRecipePlan } from '../fleet/launch-recipes.js';
export interface NativeLaunchInput {
game: 't5' | 't6';
mapSlug: string;
native?: LaunchRecipePlan['native'];
instance: NativeInstance;
serverKey: string;
rconPassword: string;
rebuild: boolean;
}
export interface PreparedNativeLaunch {
recipe: LaunchRecipePlan;
recipeSummary: string;
recipeJson: Uint8Array;
payloadJson: Uint8Array;
}
let worker: Worker | null = null;
let sequence = 0;
const pending = new Map<number, {
resolve: (value: PreparedNativeLaunch) => void;
reject: (error: Error) => void;
timer: NodeJS.Timeout;
}>();
function stop(error: Error, current = worker): Promise<number> | undefined {
if (current !== worker) return;
worker = null;
for (const request of pending.values()) { clearTimeout(request.timer); request.reject(error); }
pending.clear();
return current?.terminate();
}
/** The reader and JSON encoder run off the controller's request thread. */
export function prepareNativeLaunch(input: NativeLaunchInput): Promise<PreparedNativeLaunch> {
db(); // Migrations must finish before the independent read-only connection opens.
if (!worker) {
const current = worker = new Worker(resolve(process.env.NODE_ENV === 'production'
? 'build/launch/worker.mjs' : '.svelte-kit/launch/worker.mjs'), {
workerData: { dbPath: resolve(config.dbPath) }
});
current.on('message', ({ id, value, error }: { id: number; value: PreparedNativeLaunch; error?: string }) => {
if (current !== worker) return;
const request = pending.get(id);
if (!request) return;
pending.delete(id); clearTimeout(request.timer);
if (error) request.reject(new Error(error)); else request.resolve(value);
if (!pending.size) current.unref();
});
current.on('error', error => { void stop(error, current); });
current.on('exit', () => { void stop(new Error('Launch preparation worker stopped; preparation will retry.'), current); });
}
worker.ref();
const id = ++sequence;
return new Promise((resolve, reject) => {
const timer = setTimeout(() => { void stop(new Error('Launch preparation timed out; preparation will retry.')); }, 60_000);
pending.set(id, { resolve, reject, timer });
worker!.postMessage({ id, input });
});
}
export async function closeLaunchPreparation(): Promise<void> {
await stop(new Error('Launch preparation stopped.'));
}
src/lib/server/provisioning/launch-worker.ts
+38
View File
@@ -0,0 +1,38 @@
import { parentPort } from 'node:worker_threads';
import { posix } from 'node:path';
import { db } from '../db/index.js';
import { relDestForKind } from '../config.js';
import { captureRecipe, summarizeRecipe } from '../fleet/launch-recipes.js';
import { sha256Buffer } from '../fleet/files.js';
import type { NativeLaunchInput } from './launch-preparation.js';
parentPort!.on('message', ({ id, input }: { id: number; input: NativeLaunchInput }) => {
try {
// All file versions and composed configuration belong to one snapshot.
const recipe = db().transaction(() => ({ ...captureRecipe(input.game, input.mapSlug),
native: input.native, image: input.instance.image }))();
const instance = input.instance;
const payload = {
name: instance.server, game: recipe.game, libraryId: instance.library_id, image: instance.image,
port: instance.port, bindAddress: instance.bind_address, rconPassword: input.rconPassword,
env: { PLUTAINER_SERVER_NAME: instance.server, PLUTAINER_PORT: String(instance.port),
PLUTAINER_GAME: recipe.game === 't6' ? 't6zm' : 't5sp', PLUTAINER_CONFIG_FILE: recipe.configFile,
PLUTAINER_MAP_ROTATE: 'false',
PLUTAINER_EXTRA_ARGS: `+exec electric_cherry_map.cfg +set tws_server_id ${instance.server} +map_rotate`,
PLUTO_SERVER_KEY: input.serverKey },
files: recipe.files.map(file => ({ path: posix.join(relDestForKind(file.kind, recipe.game), file.path),
contentBase64: file.content, sha256: file.sha256 })),
...(input.rebuild ? { nextGeneration: instance.generation + 1 } : {})
};
const encode = (value: unknown) => new TextEncoder().encode(JSON.stringify(value));
const recipeJson = encode(recipe);
const payloadJson = encode(payload);
const plan = { ...recipe, files: recipe.files.map(({ content: _content, ...file }) => file) };
const recipeSummary = JSON.stringify(summarizeRecipe(plan, sha256Buffer(Buffer.from(recipeJson.buffer))));
// Transfer ownership of the encoded bytes; do not clone giant strings or objects.
parentPort!.postMessage({ id, value: { recipe: plan, recipeSummary, recipeJson, payloadJson } },
[recipeJson.buffer, payloadJson.buffer]);
} catch (error) {
parentPort!.postMessage({ id, error: error instanceof Error ? error.message : 'Launch preparation failed.' });
}
});
src/lib/server/provisioning/native.ts
+119 -66
View File
@@ -1,13 +1,15 @@
import { isIP } from 'node:net';
import { setImmediate as yieldToController } from 'node:timers/promises';
import { CapacityError, choosePlacement, availableHostPort, type PlacementOptions } from '../hosts/capacity.js';
import { randomBytes, randomUUID } from 'node:crypto';
import { posix } from 'node:path';
import { db } from '../db/index.js';
import { config, relDestForKind, serverDir } from '../config.js';
import { nativeInstance, nativeRequest, type NativeInstance } from '../backends/native.js';
import { getHost, getHostOperation, listHosts, queueHostOperation, releaseHostPorts, reserveHostPort } from '../hosts/index.js';
import { getHost, getHostOperationStatus, listHosts, queueHostOperation, queueHostOperationSerialized, releaseHostPorts, reserveHostPort } from '../hosts/index.js';
import { newestReadyLibraryId, assertReadyLibrary } from '../game-files/index.js';
import { blueprintRecipe, captureRecipe, captureRecipePlan, recipeForDeployment, summarizeRecipe, type LaunchRecipe, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
import { blueprintRecipe, captureRecipePlan, recipeForDeployment, recipeStorage, summarizeRecipe, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
import { prepareNativeLaunch, type PreparedNativeLaunch } from './launch-preparation.js';
import { getServerKey, saveServerKey } from '../fleet/server-keys.js';
import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
import { nextLobbyNumber, type CreateServerInput, type CreateServerResult } from './instances.js';
@@ -125,7 +127,9 @@ export function queueNativeCreation(input: CreateServerInput): CreateServerResul
db().prepare("UPDATE server_deployments SET state='queued' WHERE id=?").run(id);
db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,?,?,?,?)')
.run(id, name, port, 'zombie', recipe.mapSlug, recipe.expectedMap);
db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,blueprint_name) VALUES(?,?,?)').run(id, JSON.stringify(recipe), preview?.blueprint_name ?? null);
const stored = recipeStorage(recipe);
db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary,blueprint_name) VALUES(?,?,?,?)')
.run(id, stored.recipe, stored.recipeSummary, preview?.blueprint_name ?? null);
if (preview) db().prepare('UPDATE server_launch_previews SET deployment_id=?,port=? WHERE id=?').run(id, port, preview.id);
deploymentStage(id, 'Waiting for provisioning evaluation', { phase: 'eligibility', recipeStatus: 'queued' });
return { ok: true as const, name, port, deploymentId: id };
@@ -164,7 +168,8 @@ export function queueNativeRebuild(name: string, triggeredBy: string): CreateSer
db().prepare("UPDATE server_deployments SET state='waiting' WHERE id=?").run(id);
db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,?,?,?,?)')
.run(id, name, instance.port, 'zombie', recipe.mapSlug, recipe.expectedMap);
db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe) VALUES(?,?)').run(id, JSON.stringify(recipe));
const stored = recipeStorage(recipe);
db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary) VALUES(?,?,?)').run(id, stored.recipe, stored.recipeSummary);
deploymentStage(id, 'Waiting to rebuild when empty', { phase: 'eligibility', recipeStatus: 'queued' });
return { ok: true as const, name, port: instance.port, deploymentId: id };
})();
@@ -212,16 +217,6 @@ function applyReceipt(job: Job, result: Record<string, unknown>) {
})();
}
function createPayload(instance: NativeInstance, recipe: LaunchRecipe) {
const key = getServerKey(instance.server);
if (!key) throw new Error('The saved server key was removed.');
return { name: instance.server, game: recipe.game, libraryId: instance.library_id, image: instance.image, port: instance.port, bindAddress: instance.bind_address,
rconPassword: config.rcon.password || randomBytes(32).toString('hex'),
env: { PLUTAINER_SERVER_NAME: instance.server, PLUTAINER_PORT: String(instance.port), PLUTAINER_GAME: recipe.game === 't6' ? 't6zm' : 't5sp',
PLUTAINER_CONFIG_FILE: recipe.configFile, PLUTAINER_MAP_ROTATE: 'false', PLUTAINER_EXTRA_ARGS: `+exec electric_cherry_map.cfg +set tws_server_id ${instance.server} +map_rotate`, PLUTO_SERVER_KEY: key },
files: recipe.files.map(file => ({ path: posix.join(relDestForKind(file.kind, recipe.game), file.path), contentBase64: file.content, sha256: file.sha256 })) };
}
function hold(job: Job, message: string) { deploymentStage(job.id, message, { blocker: { code: 'native-host', message, automatic: true } }); }
function stillActive(job: Job): boolean {
return !!db().prepare(`SELECT 1 FROM server_deployments WHERE id=? AND server_deleted_at IS NULL
@@ -267,6 +262,24 @@ export async function processNativeProvisioning(eligibility: (operation: { id: s
}
const activeJobs = new Map<string, Promise<void>>();
// Preparation owns startup capacity until its command is committed or discarded.
const preparingJobs = new Set<string>();
function otherPreparations(id: string): number { return preparingJobs.size - Number(preparingJobs.has(id)); }
let launchCommit: Promise<void> = Promise.resolve();
function commitPreparedLaunch(commit: () => void): Promise<void> {
// Worker replies can arrive together. Persist one launch per event-loop turn
// so a batch of large SQLite writes cannot hold off requests and health timers.
const next = launchCommit.then(async () => { await yieldToController(); commit(); });
launchCommit = next.catch(() => {});
return next;
}
function sameLaunchFiles(first: LaunchRecipePlan, second: LaunchRecipePlan): boolean {
const selected = (recipe: LaunchRecipePlan) => ({ game: recipe.game, mapSlug: recipe.mapSlug,
expectedMap: recipe.expectedMap, configFile: recipe.configFile, configSource: recipe.configSource, files: recipe.files });
return JSON.stringify(selected(first)) === JSON.stringify(selected(second));
}
/** Reservations can move only while no remote work has ever been submitted. */
function placeQueuedCreation(job: Job, instance: NativeInstance): NativeInstance {
@@ -285,7 +298,8 @@ function placeQueuedCreation(job: Job, instance: NativeInstance): NativeInstance
.run(host.id, bind, port, network.publicAddress ?? host.publicAddress, network.publicPort ?? null, instance.id);
db().prepare('UPDATE server_creation_jobs SET port=? WHERE deployment_id=?').run(port, job.id);
recipe.native = { ...prior, hostId: host.id, bindAddress: bind, publicAddress: network.publicAddress ?? host.publicAddress, publicPort: network.publicPort };
db().prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?').run(JSON.stringify(recipe), job.id);
const stored = recipeStorage(recipe);
db().prepare('UPDATE server_launch_recipes SET recipe=?,recipe_summary=? WHERE deployment_id=?').run(stored.recipe, stored.recipeSummary, job.id);
db().prepare('UPDATE server_launch_previews SET port=? WHERE deployment_id=?').run(port, job.id);
deploymentStage(job.id, `Placed on ${host.name} after CPU, memory, history and port checks`, { phase: 'eligibility' });
return nativeInstance(job.server)!;
@@ -302,7 +316,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
const settings = fleetSettings();
const active = db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL
AND action IN ('create','rebuild') AND state IN ('running','awaiting-health')`).get(job.id) as { n: number };
if (active.n >= settings.createConcurrency) { hold(job, 'Waiting for startup capacity.'); return; }
if (active.n + otherPreparations(job.id) >= settings.createConcurrency) { hold(job, 'Waiting for startup capacity.'); return; }
instance = placeQueuedCreation(job, instance);
}
const host = getHost(instance.host_id);
@@ -311,7 +325,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
hold(job, 'Host is unavailable; existing server state is unknown. Waiting for reconnection.'); return;
}
if (job.operation_id) {
const operation = getHostOperation(job.operation_id);
const operation = getHostOperationStatus(job.operation_id);
if (operation?.kind === 'instance.cancel-create' && !['completed', 'failed'].includes(operation.state) && !host.inventory.capabilities?.includes('instance.cancel-create')) {
hold(job, hostAgentUpdatesEnabled(host.inventory)
? 'Waiting for the host agent update. New command delivery is paused so its current batch can finish and the automatic updater can run. Cleanup resumes after the updated agent connects.'
@@ -381,7 +395,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
if (!status.ok) { hold(job, `Waiting for game status: ${status.reason}`); return; }
if (job.action === 'create' || job.action === 'rebuild') {
deploymentStage(job.id, 'Checking selected engine map', { phase: 'verify-map', blocker: null });
const recipe = recipeForDeployment(job.id)!.recipe;
const recipe = db().prepare('SELECT expected_map AS expectedMap FROM server_creation_jobs WHERE deployment_id=?')
.get(job.id) as { expectedMap: string };
check({ key: 'map', label: 'Selected engine map verified', source: status.source,
result: status.map === recipe.expectedMap ? 'pass' : 'waiting', expected: recipe.expectedMap, actual: status.map }, 'verify-map');
if (status.map !== recipe.expectedMap) { hold(job, `Waiting for selected map ${recipe.expectedMap}; observed ${status.map ?? 'unknown'}.`); return; }
@@ -416,7 +431,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
hold(job, 'Staging and verifying files on the host.'); return;
}
if (job.stage_operation_id) {
const stage = getHostOperation(job.stage_operation_id);
const stage = getHostOperationStatus(job.stage_operation_id);
if (stage?.state === 'failed') { finishDeployment(job.id, { ok: false, message: stage.error ?? 'File staging failed.' }); return; }
if (stage?.state !== 'completed') { hold(job, 'Waiting for verified remote file staging.'); return; }
if (files.some(file => (db().prepare('SELECT state FROM staged_file_changes WHERE id=?').get(file.id) as { state: string } | undefined)?.state !== 'pending')) {
@@ -426,7 +441,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
const settings = fleetSettings();
const count = (actions: string[]) => (db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id != ? AND server_deleted_at IS NULL
AND action IN (${actions.map(() => '?').join(',')}) AND state IN ('running','awaiting-health')`).get(job.id, ...actions) as { n: number }).n;
if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) >= settings.createConcurrency) {
if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) + otherPreparations(job.id) >= settings.createConcurrency) {
deploymentStage(job.id, 'Waiting for startup capacity.', { phase: 'startup-queue',
blocker: { code: 'startup-capacity', message: 'Waiting for startup capacity.', automatic: true } }); return;
}
@@ -435,65 +450,103 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
if (job.action === 'create') choosePlacement({ hostId: instance.host_id, libraryId: instance.library_id, image: instance.image,
bindAddress: instance.bind_address, excludeInstanceId: instance.id, thorough: true, dispatch: true });
if (job.action === 'create' || job.action === 'rebuild') assertReadyLibrary(instance.host_id, instance.library_id);
if (job.action !== 'create' && job.action !== 'start') {
const checkOccupancy = async (): Promise<boolean> => {
// Fast controller check keeps occupied lobbies out of the operation slots.
// The agent repeats this immediately before its local mutation.
deploymentStage(job.id, 'Checking container and player occupancy before submission', { phase: 'eligibility', blocker: null });
const health = await nativeRequest<{ state: string; healthy?: boolean }>(instance, 'instance.inspect');
if (!stillActive(job)) return;
if (!stillActive(job)) return false;
if (!['stopped','exited','created','dead','missing'].includes(health.state)) {
const occupancy = await probePlayerEvidence(job.server);
if (!stillActive(job)) return;
if (!stillActive(job)) return false;
const recoveredWatchdog = health.healthy && occupancy.players !== null && db().prepare(`SELECT 1 FROM provisioning_sources s
JOIN watchdog_incidents i ON i.id=s.source_id JOIN server_deployments d ON d.id=s.operation_id
WHERE s.operation_id=? AND s.kind='watchdog' AND i.kind='health' AND d.triggered_by LIKE 'watchdog:%'`).get(job.id);
if (recoveredWatchdog && occupancy.players !== null) { resolveRecoveredHealthOperation(job.id, occupancy.players, occupancy.source); return; }
if (occupancy.players !== 0) { hold(job, occupancy.players !== null ? `Waiting for ${occupancy.players} player(s) to leave.` : 'Waiting for a fresh local player count.'); return; }
if (recoveredWatchdog && occupancy.players !== null) { resolveRecoveredHealthOperation(job.id, occupancy.players, occupancy.source); return false; }
if (occupancy.players !== 0) { hold(job, occupancy.players !== null ? `Waiting for ${occupancy.players} player(s) to leave.` : 'Waiting for a fresh local player count.'); return false; }
}
}
return true;
};
if (job.action !== 'create' && job.action !== 'start' && !await checkOccupancy()) return;
const current = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined;
if (!current || !['queued','waiting','interrupted'].includes(current.state) || eligibility(job)) return;
db().transaction(() => {
const live = nativeInstance(job.server);
if (live?.id !== instance!.id || live.generation !== instance!.generation) throw new Error('Instance changed during preflight.');
if (job.action === 'destroy' || job.action === 'rebuild') {
const beta = activeBetaRemovalBlocker(job.server);
if (beta) { hold(job, beta); return; }
}
if (JSON.stringify(filesFor(job)) !== JSON.stringify(files)) {
finishDeployment(job.id, { ok: false, message: 'File requests changed during preflight. Review and submit a fresh activation.' }); return;
}
// Async occupancy checks can overlap another job claiming the final slot.
if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) >= settings.createConcurrency) {
deploymentStage(job.id, 'Waiting for startup capacity.', { phase: 'startup-queue', blocker: { code: 'startup-capacity', message: 'Waiting for startup capacity.', automatic: true } }); return;
}
if (job.action === 'destroy' && count(['destroy']) >= settings.destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; }
if (job.action === 'start' || job.action === 'restart' || job.action === 'rebuild') {
db().prepare(`INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES(?,?,?) ON CONFLICT(server)
DO UPDATE SET requested_at=excluded.requested_at,operation_id=excluded.operation_id,healthy_at=NULL,error=NULL`).run(job.server, new Date().toISOString(), job.id);
}
recordProgress(job.id, { blocker: null, checkpoint: { key: 'eligibility', label: 'Host eligibility and safety checks passed' } });
let payload: Record<string, unknown> = files.length ? { stageId: job.id } : {};
let prepared: PreparedNativeLaunch | undefined;
let preparedKey: string | null = null;
try {
if (job.action === 'create' || job.action === 'rebuild') {
deploymentStage(job.id, 'Preparing launch files for the host agent', { phase: 'prepare', recipeStatus: 'attempted',
checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } });
if (count(['create','rebuild']) + otherPreparations(job.id) >= fleetSettings().createConcurrency) {
hold(job, 'Waiting for startup capacity.'); return;
}
preparingJobs.add(job.id);
preparedKey = getServerKey(job.server) ?? null;
if (!preparedKey) throw new Error('The saved server key was removed.');
const prior = recipeForDeployment(job.id)!.recipe as NativeRecipe;
const recipe = { ...captureRecipe(prior.game, prior.mapSlug), native: prior.native, image: instance!.image };
payload = createPayload(instance!, recipe);
if (job.action === 'rebuild') payload.nextGeneration = instance!.generation + 1;
db().prepare('UPDATE server_launch_recipes SET recipe=? WHERE deployment_id=?').run(JSON.stringify(recipe), job.id);
if (job.action === 'create') db().prepare(`INSERT INTO servers(name,komodo_type,komodo_name,game,map,game_mode,map_slug,host_dir,config_file,game_id,image,game_port,created_by,container_state)
VALUES(?,'deployment','',?,?,'zombie',?,?,?,?,?,?,'native','starting')`).run(job.server, recipe.game, recipe.expectedMap, recipe.mapSlug, serverDir(job.server),
recipe.configFile, recipe.game === 't6' ? 't6zm' : 't5sp', recipe.image, instance!.port);
for (const file of recipe.files) recordScript(job.server, file.path, file.versionId, file.sha256, file.size, 'native-create');
recordProgress(job.id, { checkpoint: { key: 'prepare', label: 'Launch files selected and command payload prepared' } });
deploymentStage(job.id, 'Preparing launch files for the host agent', { phase: 'prepare',
checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } });
prepared = await prepareNativeLaunch({ game: prior.game, mapSlug: prior.mapSlug, native: prior.native,
instance, serverKey: preparedKey, rconPassword: config.rcon.password || randomBytes(32).toString('hex'),
rebuild: job.action === 'rebuild' });
if (job.action === 'rebuild' && !await checkOccupancy()) return;
}
const operation = queueHostOperation(instance!.host_id, `instance.${job.action === 'destroy' ? 'remove' : job.action}`, payload,
{ instanceId: instance!.id, generation: instance!.generation, idempotencyKey: job.id });
db().prepare('UPDATE native_provisioning SET operation_id=? WHERE deployment_id=?').run(operation.id, job.id);
for (const file of files) db().prepare("UPDATE staged_file_changes SET state='applying' WHERE id=? AND state='pending'").run(file.id);
db().prepare("UPDATE server_deployments SET state='running',submitted_at=? WHERE id=?").run(new Date().toISOString(), job.id);
deploymentStage(job.id, `Waiting for host agent to acknowledge ${job.action}`, { phase: 'container-action', blocker: null,
command: { state: 'attempted', message: `Durable agent operation ${operation.id}; reconciliation always uses this ID.` } });
})();
const commit = db().transaction(() => {
const state = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined;
if (!state || !['queued','waiting','interrupted'].includes(state.state) || eligibility(job)) return;
const live = nativeInstance(job.server);
if (live?.id !== instance!.id || live.generation !== instance!.generation) throw new Error('Instance changed during preflight.');
const liveHost = getHost(instance!.host_id);
if (!liveHost?.connected || liveHost.mode === 'disabled') { hold(job, 'Host is unavailable; waiting for reconnection.'); return; }
if (job.action === 'destroy' || job.action === 'rebuild') {
const beta = activeBetaRemovalBlocker(job.server);
if (beta) { hold(job, beta); return; }
}
if (JSON.stringify(filesFor(job)) !== JSON.stringify(files)) {
finishDeployment(job.id, { ok: false, message: 'File requests changed during preflight. Review and submit a fresh activation.' }); return;
}
// Async occupancy checks can overlap another job claiming the final slot.
if ((job.action === 'create' || job.action === 'rebuild') && count(['create','rebuild']) + otherPreparations(job.id) >= fleetSettings().createConcurrency) {
deploymentStage(job.id, 'Waiting for startup capacity.', { phase: 'startup-queue', blocker: { code: 'startup-capacity', message: 'Waiting for startup capacity.', automatic: true } }); return;
}
if (job.action === 'destroy' && count(['destroy']) >= fleetSettings().destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; }
if (prepared) {
if (getServerKey(job.server) !== preparedKey || !sameLaunchFiles(prepared.recipe,
captureRecipePlan(prepared.recipe.game, prepared.recipe.mapSlug))) {
hold(job, 'Launch files or credentials changed during preparation; preparing the current selection next pass.'); return;
}
assertReadyLibrary(instance!.host_id, instance!.library_id);
if (job.action === 'create') choosePlacement({ hostId: instance!.host_id, libraryId: instance!.library_id, image: instance!.image,
bindAddress: instance!.bind_address, excludeInstanceId: instance!.id, thorough: true, dispatch: true });
}
if (job.action === 'start' || job.action === 'restart' || job.action === 'rebuild') {
db().prepare(`INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES(?,?,?) ON CONFLICT(server)
DO UPDATE SET requested_at=excluded.requested_at,operation_id=excluded.operation_id,healthy_at=NULL,error=NULL`).run(job.server, new Date().toISOString(), job.id);
}
recordProgress(job.id, { blocker: null, checkpoint: { key: 'eligibility', label: 'Host eligibility and safety checks passed' } });
const payload: Record<string, unknown> = files.length ? { stageId: job.id } : {};
if (prepared) {
deploymentStage(job.id, 'Preparing launch files for the host agent', { phase: 'prepare', recipeStatus: 'attempted',
checkpoint: { key: 'startup-queue', label: 'Startup capacity available' } });
const recipe = prepared.recipe;
db().prepare('UPDATE server_launch_recipes SET recipe=CAST(? AS TEXT),recipe_summary=? WHERE deployment_id=?')
.run(Buffer.from(prepared.recipeJson.buffer, prepared.recipeJson.byteOffset, prepared.recipeJson.byteLength), prepared.recipeSummary, job.id);
if (job.action === 'create') db().prepare(`INSERT INTO servers(name,komodo_type,komodo_name,game,map,game_mode,map_slug,host_dir,config_file,game_id,image,game_port,created_by,container_state)
VALUES(?,'deployment','',?,?,'zombie',?,?,?,?,?,?,'native','starting')`).run(job.server, recipe.game, recipe.expectedMap, recipe.mapSlug, serverDir(job.server),
recipe.configFile, recipe.game === 't6' ? 't6zm' : 't5sp', recipe.image, instance!.port);
for (const file of recipe.files) recordScript(job.server, file.path, file.versionId, file.sha256, file.size, 'native-create');
recordProgress(job.id, { checkpoint: { key: 'prepare', label: 'Launch files selected and command payload prepared' } });
}
const operationOptions = { instanceId: instance!.id, generation: instance!.generation, idempotencyKey: job.id };
const kind = `instance.${job.action === 'destroy' ? 'remove' : job.action}`;
const operation = prepared ? queueHostOperationSerialized(instance!.host_id, kind,
Buffer.from(prepared.payloadJson.buffer, prepared.payloadJson.byteOffset, prepared.payloadJson.byteLength), operationOptions,
{ name: instance!.server, game: prepared.recipe.game, libraryId: instance!.library_id })
: queueHostOperation(instance!.host_id, kind, payload, operationOptions);
db().prepare('UPDATE native_provisioning SET operation_id=? WHERE deployment_id=?').run(operation.id, job.id);
for (const file of files) db().prepare("UPDATE staged_file_changes SET state='applying' WHERE id=? AND state='pending'").run(file.id);
db().prepare("UPDATE server_deployments SET state='running',submitted_at=? WHERE id=?").run(new Date().toISOString(), job.id);
deploymentStage(job.id, `Waiting for host agent to acknowledge ${job.action}`, { phase: 'container-action', blocker: null,
command: { state: 'attempted', message: `Durable agent operation ${operation.id}; reconciliation always uses this ID.` } });
});
if (prepared) await commitPreparedLaunch(commit);
else commit();
} finally { preparingJobs.delete(job.id); }
}
src/routes/api/agent/operations/+server.ts
+5 -5
View File
@@ -1,17 +1,17 @@
import { json } from '@sveltejs/kit';
import { pendingHostOperations, waitForHostWork } from '$lib/server/hosts/index.js';
import { pendingHostOperationRows, waitForHostWork } from '$lib/server/hosts/index.js';
import { authenticateAgent } from '$lib/server/hosts/http.js';
import { reconcileHostPreparation } from '$lib/server/hosts/preparation.js';
import { hostOperationsResponse } from '$lib/server/hosts/delivery.js';
export async function GET({ request }: { request: Request }) {
const host = authenticateAgent(request);
reconcileHostPreparation(host.id);
const params = new URL(request.url).searchParams;
const lane = params.get('lane') === 'health' ? 'health' : params.get('lane') === 'read' ? 'read' : params.get('lane') === 'write' ? 'write' : 'all';
let operations = pendingHostOperations(host.id, lane);
let operations = pendingHostOperationRows(host.id, lane, false);
if (!operations.length && params.get('wait') === '20') {
await waitForHostWork(host.id, request.signal);
authenticateAgent(request); // Revocation still applies after waiting.
operations = pendingHostOperations(host.id, lane);
operations = pendingHostOperationRows(host.id, lane, false);
}
return json({ operations, serverTime: new Date().toISOString() }, { headers: { 'Cache-Control': 'no-store' } });
return hostOperationsResponse(operations);
}
src/routes/api/agent/operations/[id]/+server.ts
+2 -2
View File
@@ -1,12 +1,12 @@
import { json } from '@sveltejs/kit';
import { reportHostOperation } from '$lib/server/hosts/index.js';
import { reportHostOperationStatus } from '$lib/server/hosts/index.js';
import { authenticateAgent, hostApiError, objectBody } from '$lib/server/hosts/http.js';
import { kickProvisioning } from '$lib/server/pipeline/worker.js';
export async function POST({ request, params }: { request: Request; params: { id: string } }) {
const host = authenticateAgent(request);
try {
const body = await objectBody(request, 128 * 1024 * 1024);
const operation = reportHostOperation(host.id, params.id, { state: body.state, result: body.result, error: body.error });
const operation = reportHostOperationStatus(host.id, params.id, { state: body.state, result: body.result, error: body.error });
if (['completed', 'failed', 'unknown'].includes(operation.state)
&& (operation.kind.startsWith('instance.') && operation.kind !== 'instance.inspect' || operation.kind === 'files.stage')) {
void kickProvisioning().catch(error => console.error('[agent] provisioning wake failed:', error));