josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Queue native collection launches before async placement

Browse Source
This commit is contained in:
josh
2026-10-05 11:35:50 -04:00
parent 1405114a14
commit 7c76851252
34 changed files with 1324 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/provisioning/cancel-creation.ts
+15 -8
View File
@@ -6,9 +6,13 @@ import { recordProgress } from './progress.js';
export function canCancelCreation(id: string): boolean {
return !!db().prepare(`SELECT 1 FROM server_deployments d
JOIN native_provisioning n ON n.deployment_id=d.id
JOIN native_instances i ON i.id=n.instance_id AND i.generation=n.generation AND i.retired_at IS NULL
LEFT JOIN native_provisioning n ON n.deployment_id=d.id
LEFT JOIN native_instances i ON i.id=n.instance_id AND i.generation=n.generation AND i.retired_at IS NULL
WHERE d.id=? AND d.action='create' AND d.server_deleted_at IS NULL
AND (i.id IS NOT NULL OR (n.deployment_id IS NULL
AND EXISTS (SELECT 1 FROM native_creation_intents intent WHERE intent.deployment_id=d.id)
AND NOT EXISTS (SELECT 1 FROM native_instances replacement WHERE replacement.server=d.server AND replacement.retired_at IS NULL)
AND NOT EXISTS (SELECT 1 FROM servers replacement WHERE replacement.name=d.server AND replacement.destroyed_at IS NULL)))
AND d.state IN ('queued','waiting','running','awaiting-health','interrupted','failed')
AND NOT EXISTS (SELECT 1 FROM server_deployments other WHERE other.server=d.server
AND other.server_deleted_at IS NULL AND other.rowid>d.rowid AND other.state!='cancelled'
@@ -23,9 +27,9 @@ export function cancelCreation(id: string, triggeredBy = 'ui'): string | null {
if (prior) return prior.cleanup_id;
if (!canCancelCreation(id)) return null;
const job = db().prepare(`SELECT d.server,n.operation_id,n.stage_operation_id FROM server_deployments d
JOIN native_provisioning n ON n.deployment_id=d.id WHERE d.id=?`).get(id) as
LEFT JOIN native_provisioning n ON n.deployment_id=d.id WHERE d.id=?`).get(id) as
{ server: string; operation_id: string | null; stage_operation_id: string | null };
const instance = nativeInstance(job.server)!;
const instance = nativeInstance(job.server);
// Replace a removal still waiting on occupancy, including one whose asynchronous probe is outstanding.
for (const removal of db().prepare("SELECT id FROM server_deployments WHERE server=? AND action='destroy' AND state='waiting' AND server_deleted_at IS NULL").all(job.server) as { id: string }[]) {
cancelRemoval(removal.id);
@@ -37,13 +41,16 @@ export function cancelCreation(id: string, triggeredBy = 'ui'): string | null {
event: { kind: 'cancelled', message: 'Creation cancelled. Its partial instance will be removed without a player-count check.', details: { cleanupId } } });
if (!job.operation_id && !job.stage_operation_id) {
// No remote mutation was ever queued. The reservation is the only resource to remove.
db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(new Date().toISOString(), instance.id);
releaseHostPorts(instance.id);
if (instance) {
db().prepare('UPDATE native_instances SET retired_at=? WHERE id=?').run(new Date().toISOString(), instance.id);
releaseHostPorts(instance.id);
}
deploymentStage(cleanupId, 'Unsubmitted creation cancelled; reservations released', { phase: 'cleanup',
checkpoint: { key: 'cleanup', label: 'No host command submitted; local reservations released' } });
finishDeployment(cleanupId, { ok: true });
finishCreationCancellation(cleanupId);
} else {
if (!instance) throw new Error('The original native instance is unavailable.');
const operation = queueHostOperation(instance.host_id, 'instance.cancel-create', { createOperationId: job.operation_id ?? id },
{ instanceId: instance.id, generation: instance.generation, idempotencyKey: cleanupId });
db().prepare('UPDATE native_provisioning SET operation_id=? WHERE deployment_id=?').run(operation.id, cleanupId);
@@ -57,12 +64,12 @@ export function cancelCreation(id: string, triggeredBy = 'ui'): string | null {
export function finishCreationCancellation(cleanupId: string): void {
const cancellation = db().prepare(`SELECT c.creation_id,n.operation_id FROM native_creation_cancellations c
JOIN native_provisioning n ON n.deployment_id=c.creation_id WHERE c.cleanup_id=?`).get(cleanupId) as
LEFT JOIN native_provisioning n ON n.deployment_id=c.creation_id WHERE c.cleanup_id=?`).get(cleanupId) as
{ creation_id: string; operation_id: string | null } | undefined;
if (!cancellation) return;
const now = new Date().toISOString();
db().prepare('UPDATE server_deployments SET server_deleted_at=? WHERE id IN (?,?)').run(now, cancellation.creation_id, cleanupId);
deploymentStage(cancellation.creation_id, 'Creation cancelled; partial instance removed', { blocker: null,
deploymentStage(cancellation.creation_id, 'Creation cancelled; cleanup completed', { blocker: null,
checkpoint: { key: 'cleanup', phase: 'cleanup', label: 'Cancellation cleanup confirmed; instance and reservations removed' } });
// The original command may have lost its receipt. Cleanup proves that its effects are now gone.
if (cancellation.operation_id) {
src/lib/server/provisioning/history.ts
+5 -3
View File
@@ -9,6 +9,7 @@ import { readProgress, recordProgress, type ProgressUpdate } from './progress.js
import { bindNativeOperation } from './native.js';
import { workerActivity } from '../pipeline/activity.js';
import { canCancelCreation } from './cancel-creation.js';
import { nativeDeployment } from '../backends/native.js';
export function beginDeployment(server: string, action: ProvisioningAction, triggeredBy = 'ui'): string {
const id = randomUUID();
@@ -103,7 +104,7 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
(ProvisioningDeployment & { observation_json: string | null; phase_key: string | null; phase_started_at: string | null; observed_at: string | null }) | undefined;
if (!row) return null;
const generatedAt = new Date().toISOString();
const native = db().prepare('SELECT instance_id FROM native_provisioning WHERE deployment_id=?').get(id);
const native = nativeDeployment(id);
const agentOperation = native ? db().prepare(`SELECT o.id, o.host_id AS hostId, o.state, o.updated_at AS updatedAt, o.created_at AS createdAt, o.error
FROM native_provisioning n JOIN ec_host_operations o ON o.id=n.operation_id WHERE n.deployment_id=?`).get(id) as ProvisioningDetail['agentOperation'] : null;
const cancellation = db().prepare(`SELECT c.cleanup_id AS cleanupId,d.state,d.stage,d.error FROM native_creation_cancellations c
@@ -254,10 +255,11 @@ export function listDeployments() {
export function interruptDeployments(): void {
db().transaction(() => {
const running = db().prepare("SELECT id FROM server_deployments WHERE state = 'running' AND id NOT IN (SELECT deployment_id FROM native_provisioning)").all() as { id: string }[];
const native = 'SELECT deployment_id FROM native_provisioning UNION ALL SELECT deployment_id FROM native_creation_intents';
const running = db().prepare(`SELECT id FROM server_deployments WHERE state = 'running' AND id NOT IN (${native})`).all() as { id: string }[];
const finishedAt = new Date().toISOString();
const error = 'The application restarted before this operation finished. Check the server before retrying.';
db().prepare(`UPDATE server_deployments SET state = 'interrupted', finished_at = ?, error = ? WHERE state = 'running' AND id NOT IN (SELECT deployment_id FROM native_provisioning)`).run(finishedAt, error);
db().prepare(`UPDATE server_deployments SET state = 'interrupted', finished_at = ?, error = ? WHERE state = 'running' AND id NOT IN (${native})`).run(finishedAt, error);
for (const { id } of running) recordProgress(id, { blocker: null, event: { kind: 'outcome',
message: 'Interrupted by application restart. Check the server before retrying.', details: { state: 'interrupted', finishedAt, error } } });
})();
src/lib/server/provisioning/instances.ts
+1 -1
View File
@@ -44,7 +44,7 @@ export type CreateServerFailure =
| 'db'; // shouldn't happen, but wraps INSERT failures
export type CreateServerResult =
| { ok: true; name: string; port: number; deploymentId: string }
| { ok: true; name: string; port: number | null; deploymentId: string }
| { ok: false; kind: CreateServerFailure; message: string };
export type DestroyServerResult =
src/lib/server/provisioning/launch-preparation.ts
+12 -6
View File
@@ -4,6 +4,7 @@ import { config } from '../config.js';
import { db } from '../db/index.js';
import type { NativeInstance } from '../backends/native.js';
import type { LaunchRecipePlan } from '../fleet/launch-recipes.js';
import type { PlacementOptions } from '../hosts/capacity.js';
export interface NativeLaunchInput {
game: 't5' | 't6';
@@ -25,7 +26,7 @@ export interface PreparedNativeLaunch {
let worker: Worker | null = null;
let sequence = 0;
const pending = new Map<number, {
resolve: (value: PreparedNativeLaunch) => void;
resolve: (value: unknown) => void;
reject: (error: Error) => void;
timer: NodeJS.Timeout;
}>();
@@ -39,14 +40,14 @@ function stop(error: Error, current = worker): Promise<number> | undefined {
}
/** The reader and JSON encoder run off the controller's request thread. */
export function prepareNativeLaunch(input: NativeLaunchInput): Promise<PreparedNativeLaunch> {
function run<T>(kind: 'launch' | 'placement', input: NativeLaunchInput | PlacementOptions): Promise<T> {
db(); // Migrations must finish before the independent read-only connection opens.
if (!worker) {
const current = worker = new Worker(resolve(process.env.NODE_ENV === 'production'
? 'build/launch/worker.mjs' : '.svelte-kit/launch/worker.mjs'), {
workerData: { dbPath: resolve(config.dbPath) }
});
current.on('message', ({ id, value, error }: { id: number; value: PreparedNativeLaunch; error?: string }) => {
current.on('message', ({ id, value, error }: { id: number; value: unknown; error?: string }) => {
if (current !== worker) return;
const request = pending.get(id);
if (!request) return;
@@ -59,13 +60,18 @@ export function prepareNativeLaunch(input: NativeLaunchInput): Promise<PreparedN
}
worker.ref();
const id = ++sequence;
return new Promise((resolve, reject) => {
return new Promise<T>((resolve, reject) => {
const timer = setTimeout(() => { void stop(new Error('Launch preparation timed out; preparation will retry.')); }, 60_000);
pending.set(id, { resolve, reject, timer });
worker!.postMessage({ id, input });
pending.set(id, { resolve: value => resolve(value as T), reject, timer });
worker!.postMessage({ id, kind, input });
});
}
export function prepareNativeLaunch(input: NativeLaunchInput): Promise<PreparedNativeLaunch> { return run('launch', input); }
export function evaluateNativePlacement(input: PlacementOptions): Promise<{ hostId: string; lastHeartbeat: string | null }> {
return run('placement', input);
}
export async function closeLaunchPreparation(): Promise<void> {
await stop(new Error('Launch preparation stopped.'));
}
src/lib/server/provisioning/launch-worker.ts
+9 -1
View File
@@ -5,9 +5,17 @@ import { relDestForKind } from '../config.js';
import { captureRecipe, summarizeRecipe } from '../fleet/launch-recipes.js';
import { sha256Buffer } from '../fleet/files.js';
import type { NativeLaunchInput } from './launch-preparation.js';
import { choosePlacement, type PlacementOptions } from '../hosts/capacity.js';
parentPort!.on('message', ({ id, input }: { id: number; input: NativeLaunchInput }) => {
parentPort!.on('message', (message: { id: number; kind: 'launch'; input: NativeLaunchInput } | { id: number; kind: 'placement'; input: PlacementOptions }) => {
const { id } = message;
try {
if (message.kind === 'placement') {
const decision = db().transaction(() => choosePlacement(message.input))();
parentPort!.postMessage({ id, value: { hostId: decision.host.id, lastHeartbeat: decision.host.lastHeartbeat } });
return;
}
const input = message.input;
// All file versions and composed configuration belong to one snapshot.
const recipe = db().transaction(() => ({ ...captureRecipe(input.game, input.mapSlug),
native: input.native, image: input.instance.image }))();
src/lib/server/provisioning/native.ts
+113 -57
View File
@@ -1,15 +1,15 @@
import { isIP } from 'node:net';
import { setImmediate as yieldToController } from 'node:timers/promises';
import { CapacityError, choosePlacement, availableHostPort, type PlacementOptions } from '../hosts/capacity.js';
import { choosePlacement, availableHostPort, type PlacementOptions } from '../hosts/capacity.js';
import { randomBytes, randomUUID } from 'node:crypto';
import { posix } from 'node:path';
import { db } from '../db/index.js';
import { config, relDestForKind, serverDir } from '../config.js';
import { nativeInstance, nativeRequest, type NativeInstance } from '../backends/native.js';
import { nativeInstance, nativeRequest, unplacedNativeCreation, type NativeInstance } from '../backends/native.js';
import { getHost, getHostOperationStatus, listHosts, queueHostOperation, queueHostOperationSerialized, releaseHostPorts, reserveHostPort } from '../hosts/index.js';
import { newestReadyLibraryId, assertReadyLibrary } from '../game-files/index.js';
import { blueprintRecipe, captureRecipePlan, recipeForDeployment, recipeStorage, summarizeRecipe, type LaunchRecipePlan } from '../fleet/launch-recipes.js';
import { prepareNativeLaunch, type PreparedNativeLaunch } from './launch-preparation.js';
import { prepareNativeLaunch, evaluateNativePlacement, type PreparedNativeLaunch } from './launch-preparation.js';
import { getServerKey, saveServerKey } from '../fleet/server-keys.js';
import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
import { nextLobbyNumber, type CreateServerInput, type CreateServerResult } from './instances.js';
@@ -30,11 +30,12 @@ import { cancelCreation, finishCreationCancellation } from './cancel-creation.js
export type NativeSelection = import('../../hosts.js').NativePlacement;
type NativeRecipe = LaunchRecipePlan & { native?: NativeSelection };
interface Job { id: string; server: string; action: ProvisioningAction; state: string; submitted_at: string | null; stage: string;
interface Job { selection?: string | null; id: string; server: string; action: ProvisioningAction; state: string; submitted_at: string | null; stage: string;
instance_id: string; generation: number; stage_operation_id: string | null; operation_id: string | null; files_json: string | null; applied_at: string | null }
interface FilePlan { id: number; path: string; contentBase64: string | null; sha256: string | null; checkBefore: boolean; expectedBeforeSha: string | null }
export function nativeSelection(input: Record<string, unknown>, game: string, options: Pick<PlacementOptions, 'planned'> = {}): NativeSelection {
/** Validate a launch request without selecting a host or consuming capacity. */
export function nativeLaunchSelection(input: Record<string, unknown>, game: string): NativeSelection {
if (input.hostId !== undefined && typeof input.hostId !== 'string') throw new Error('Invalid host.');
const libraryId = newestReadyLibraryId(game);
if (!libraryId) throw new Error(`Upload ${game.toUpperCase()} game files before launching a server.`);
@@ -44,16 +45,29 @@ export function nativeSelection(input: Record<string, unknown>, game: string, op
const image = input.image === undefined || input.image === '' ? config.komodo.defaultImage : input.image;
if (typeof image !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9._/:@-]{0,499}$/.test(image)) throw new Error('Invalid runtime image.');
const automatic = input.automatic === true || !input.hostId;
if (!automatic) {
const host = getHost(input.hostId as string);
if (!host) throw new Error('The selected host was not found.');
if (host.decommission) throw new Error('The selected host is being removed. Choose another host.');
}
const rawNetwork = input.automatic === true ? input.network ?? {} : input;
if (!rawNetwork || typeof rawNetwork !== 'object' || Array.isArray(rawNetwork)) throw new Error('Invalid network overrides.');
const network = rawNetwork as NativeSelection;
const bindAddress = network.bindAddress || undefined;
if (bindAddress !== undefined && (typeof bindAddress !== 'string' || !isIP(bindAddress))) throw new Error('Invalid bind address.');
const { host } = choosePlacement({ hostId: automatic ? undefined : input.hostId as string, libraryId, image, bindAddress, ...options });
if (network.publicAddress !== undefined && (typeof network.publicAddress !== 'string' || network.publicAddress.length > 253)) throw new Error('Invalid public address.');
if (network.publicPort !== undefined && (!Number.isInteger(network.publicPort) || Number(network.publicPort) < 1 || Number(network.publicPort) > 65535)) throw new Error('Invalid public port.');
return { hostId: host.id, libraryId, image, bindAddress: bindAddress ?? host.bindAddress, publicAddress: network.publicAddress ?? host.publicAddress,
publicPort: network.publicPort, automatic, network: { bindAddress, publicAddress: network.publicAddress, publicPort: network.publicPort } };
return { hostId: automatic ? undefined : input.hostId as string, libraryId, image, automatic,
bindAddress, publicAddress: network.publicAddress, publicPort: network.publicPort,
network: { bindAddress, publicAddress: network.publicAddress, publicPort: network.publicPort } };
}
export function nativeSelection(input: Record<string, unknown>, game: string, options: Pick<PlacementOptions, 'planned'> = {}): NativeSelection {
const selection = nativeLaunchSelection(input, game);
const { host } = choosePlacement({ hostId: selection.hostId, libraryId: selection.libraryId,
image: selection.image!, bindAddress: selection.bindAddress, ...options });
return { ...selection, hostId: host.id, bindAddress: selection.bindAddress ?? host.bindAddress,
publicAddress: selection.publicAddress ?? host.publicAddress };
}
export function previewNativeCreation(input: Record<string, unknown>): LaunchPreview {
@@ -79,7 +93,7 @@ export function previewNativeCreation(input: Record<string, unknown>): LaunchPre
db().prepare('INSERT INTO server_launch_previews(id,expires_at,server,port,recipe,blueprint_name) VALUES(?,?,?,?,?,?)')
.run(id, expiresAt, name, port, JSON.stringify(recipe), blueprintName);
const checks: LaunchPreview['checks'] = [
{ label: 'Host capacity', state: 'pass', detail: selection.automatic ? 'Capacity available. The least loaded eligible host is selected when provisioning starts.' : `${getHost(selection.hostId)!.name}: capacity available.` },
{ label: 'Host capacity', state: 'pass', detail: selection.automatic ? 'Capacity available. The least loaded eligible host is selected when provisioning starts.' : `${getHost(selection.hostId!)!.name}: capacity available.` },
{ label: 'Game files', state: 'pass', detail: `The latest ${recipe.game.toUpperCase()} game files are verified on an eligible host.` },
{ label: 'Lobby', state: name ? 'pass' : 'blocked', detail: name ?? 'All lobby IDs for this map are reserved.' },
{ label: 'Host port', state: port ? 'pass' : 'blocked', detail: port ? selection.automatic ? 'UDP/TCP capacity available; port assigned at placement.' : `${selection.bindAddress}:${port} / UDP and TCP` : 'No available UDP/TCP port on this host.' },
@@ -90,7 +104,7 @@ export function previewNativeCreation(input: Record<string, unknown>): LaunchPre
}
export function nextNativePort(selection: NativeSelection, taken: number[] = []): number | null {
const host = getHost(selection.hostId);
const host = selection.hostId ? getHost(selection.hostId) : null;
return host ? availableHostPort(host, { bindAddress: selection.bindAddress,
planned: taken.map(port => ({ hostId: host.id, port })) }) : null;
}
@@ -99,42 +113,51 @@ export function queueNativeCreation(input: CreateServerInput): CreateServerResul
try {
return db().transaction(() => {
const preview = input.previewId ? db().prepare('SELECT * FROM server_launch_previews WHERE id=?').get(input.previewId) as
{ id: string; recipe: string; server: string; port: number; expires_at: string; deployment_id: string | null; blueprint_name: string | null } | undefined : undefined;
{ id: string; recipe: string; server: string; port: number | null; expires_at: string; deployment_id: string | null; blueprint_name: string | null } | undefined : undefined;
if (input.previewId && !preview) throw new Error('Review a fresh launch preview.');
if (preview?.deployment_id) return { ok: true as const, name: preview.server, port: preview.port, deploymentId: preview.deployment_id };
if (preview && preview.expires_at <= new Date().toISOString()) throw new Error('Launch preview expired.');
const saved = preview ? JSON.parse(preview.recipe) as NativeRecipe : undefined;
const game = input.game ?? 't5';
if (input.gameMode !== 'zombie' || (saved && (saved.game !== game || saved.mapSlug !== input.mapSlug))) throw new Error('Selection changed. Review a fresh launch preview.');
const selection = nativeSelection((saved?.native ?? input) as unknown as Record<string, unknown>, game);
const recipe: NativeRecipe = { ...captureRecipePlan(game, input.mapSlug), native: selection, image: selection.image! };
const selection = nativeLaunchSelection((saved?.native ?? input) as unknown as Record<string, unknown>, game);
const lobby = nextLobbyNumber(`${game}_zombie_${input.mapSlug}`);
if (lobby === null) throw new Error('All lobby IDs are reserved.');
const name = `${game}_zombie_${input.mapSlug}_${lobby}`;
const port = nextNativePort(selection);
if (!port) throw new Error('No available host port.');
if ((input.expectedName && input.expectedName !== name) || (preview && (preview.server !== name || (!selection.automatic && preview.port !== port)))) throw new Error('Available lobby or host port changed. Review a fresh preview.');
const key = input.plutoKey?.trim() || getServerKey(name);
if (!key) throw new Error(`Save a Plutonium server key for ${name}.`);
saveServerKey(name, key);
const instanceId = randomUUID();
reserveHostPort(selection.hostId, instanceId, { port, bindAddress: selection.bindAddress, protocol: 'udp' });
reserveHostPort(selection.hostId, instanceId, { port, bindAddress: selection.bindAddress, protocol: 'tcp' });
db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,bind_address,port,public_address,public_port,created_at)
VALUES(?,?,?,?,?,?,?,?,?,?)`).run(instanceId, name, selection.hostId, selection.libraryId, selection.image, selection.bindAddress, port,
selection.publicAddress ?? null, selection.publicPort ?? null, new Date().toISOString());
const id = beginDeployment(name, 'create', input.createdBy);
if ((input.expectedName && input.expectedName !== name) || (preview && preview.server !== name)) throw new Error('Available lobby changed. Review a fresh preview.');
const result = queueNativeCreationIntent({ server: name, game, mapSlug: input.mapSlug, selection,
plutoKey: input.plutoKey, createdBy: input.createdBy, blueprintName: preview?.blueprint_name,
recipe: saved ?? captureRecipePlan(game, input.mapSlug) });
if (result.ok && preview) db().prepare('UPDATE server_launch_previews SET deployment_id=?,port=NULL WHERE id=?').run(result.deploymentId, preview.id);
return result;
})();
} catch (error) { return { ok: false, kind: 'bad-request', message: (error as Error).message }; }
}
/** Persist the reviewed logical identity; all host/port/resource decisions belong to provisioning. */
export function queueNativeCreationIntent(input: { server: string; game: 't5' | 't6'; mapSlug: string; selection: NativeSelection;
plutoKey?: string; createdBy?: string; blueprintName?: string | null; recipe?: LaunchRecipePlan }): CreateServerResult {
try {
return db().transaction(() => {
const expected = `${input.game}_zombie_${input.mapSlug}_${nextLobbyNumber(`${input.game}_zombie_${input.mapSlug}`)}`;
if (input.server !== expected || nativeInstance(input.server)) throw new Error('Available lobby changed. Review a fresh preview.');
const key = input.plutoKey?.trim() || getServerKey(input.server);
if (!key) throw new Error(`Save a Plutonium server key for ${input.server}.`);
saveServerKey(input.server, key);
const recipe: NativeRecipe = { ...(input.recipe ?? captureRecipePlan(input.game, input.mapSlug)),
native: input.selection, image: input.selection.image! };
const id = beginDeployment(input.server, 'create', input.createdBy ?? 'ui');
db().prepare("UPDATE server_deployments SET state='queued' WHERE id=?").run(id);
db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,?,?,?,?)')
.run(id, name, port, 'zombie', recipe.mapSlug, recipe.expectedMap);
db().prepare('INSERT INTO server_creation_jobs(deployment_id,server,port,game_mode,map_slug,expected_map) VALUES(?,?,NULL,?,?,?)')
.run(id, input.server, 'zombie', input.mapSlug, recipe.expectedMap);
db().prepare('INSERT INTO native_creation_intents(deployment_id,selection) VALUES(?,?)').run(id, JSON.stringify(input.selection));
const stored = recipeStorage(recipe);
db().prepare('INSERT INTO server_launch_recipes(deployment_id,recipe,recipe_summary,blueprint_name) VALUES(?,?,?,?)')
.run(id, stored.recipe, stored.recipeSummary, preview?.blueprint_name ?? null);
if (preview) db().prepare('UPDATE server_launch_previews SET deployment_id=?,port=? WHERE id=?').run(id, port, preview.id);
deploymentStage(id, 'Waiting for provisioning evaluation', { phase: 'eligibility', recipeStatus: 'queued' });
return { ok: true as const, name, port, deploymentId: id };
.run(id, stored.recipe, stored.recipeSummary, input.blueprintName ?? null);
deploymentStage(id, 'Awaiting placement', { phase: 'eligibility', recipeStatus: 'queued' });
return { ok: true as const, name: input.server, port: null, deploymentId: id };
})();
} catch (error) { return { ok: false, kind: error instanceof CapacityError ? 'conflict' : 'bad-request', message: (error as Error).message }; }
} catch (error) { return { ok: false, kind: 'bad-request', message: (error as Error).message }; }
}
export function bindNativeOperation(id: string, server: string): void {
@@ -144,8 +167,11 @@ export function bindNativeOperation(id: string, server: string): void {
/** Cancelling an unsubmitted launch releases only its local reservation. */
export function cancelQueuedNativeCreation(server: string, triggeredBy: string): string | null {
if (getServer(server)) return null;
const unplaced = unplacedNativeCreation(server);
if (unplaced) return cancelCreation(unplaced, triggeredBy);
const instance = nativeInstance(server);
if (!instance || getServer(server)) return null;
if (!instance) return null;
return db().transaction(() => {
const creation = db().prepare(`SELECT d.id FROM server_deployments d JOIN native_provisioning n ON n.deployment_id=d.id
WHERE n.instance_id=? AND d.action='create' AND d.state='queued' AND n.operation_id IS NULL`).get(instance.id) as { id: string } | undefined;
@@ -243,9 +269,10 @@ export async function processNativeProvisioning(eligibility: (operation: { id: s
deploymentStage(id, 'Host draining; waiting for a fresh empty lobby');
}
}
const jobs = db().prepare(`SELECT d.*,n.instance_id,n.generation,n.stage_operation_id,n.operation_id,n.files_json,n.applied_at
FROM native_provisioning n JOIN server_deployments d ON d.id=n.deployment_id
WHERE d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') ORDER BY d.started_at,d.rowid`).all() as Job[];
const jobs = db().prepare(`SELECT d.*,n.instance_id,n.generation,n.stage_operation_id,n.operation_id,n.files_json,n.applied_at,q.selection
FROM server_deployments d LEFT JOIN native_provisioning n ON n.deployment_id=d.id
LEFT JOIN native_creation_intents q ON q.deployment_id=d.id
WHERE (n.deployment_id IS NOT NULL OR q.deployment_id IS NOT NULL) AND d.server_deleted_at IS NULL AND d.state IN ('queued','waiting','running','awaiting-health','interrupted') ORDER BY d.started_at,d.rowid`).all() as Job[];
// One host's observation must not prevent another job being admitted.
// Per-job ownership also covers overlapping direct calls and worker passes.
const work = jobs.map(job => {
@@ -254,7 +281,7 @@ export async function processNativeProvisioning(eligibility: (operation: { id: s
const work = trackActivity(`provisioning:${job.id}`, `Evaluating ${job.action} for ${job.server}`,
() => processJob(job, eligibility))
.catch(error => { if (stillActive(job)) hold(job, (error as Error).message); })
.finally(() => { activeJobs.delete(job.id); });
.finally(() => { activeJobs.delete(job.id); preparingJobs.delete(job.id); });
activeJobs.set(job.id, work);
return work;
});
@@ -281,27 +308,47 @@ function sameLaunchFiles(first: LaunchRecipePlan, second: LaunchRecipePlan): boo
return JSON.stringify(selected(first)) === JSON.stringify(selected(second));
}
/** Reservations can move only while no remote work has ever been submitted. */
function placeQueuedCreation(job: Job, instance: NativeInstance): NativeInstance {
/** Capacity history and host ranking run in the background reader, one job at a time. */
async function placeQueuedCreation(job: Job, instance: NativeInstance | null): Promise<NativeInstance | null> {
const recipe = recipeForDeployment(job.id)!.recipe as NativeRecipe;
const prior: NativeSelection = job.selection ? JSON.parse(job.selection) : recipe.native!;
const network = prior.automatic ? prior.network ?? {} : prior;
const options: PlacementOptions = { hostId: prior.automatic ? undefined : prior.hostId,
libraryId: prior.libraryId, image: prior.image ?? recipe.image, bindAddress: network.bindAddress,
thorough: true, dispatch: true, excludeInstanceId: instance?.id };
const decision = await evaluateNativePlacement(options);
return db().transaction(() => {
const recipe = recipeForDeployment(job.id)!.recipe as NativeRecipe;
const prior = recipe.native!;
const network = prior.automatic ? prior.network ?? {} : prior;
const { host, port } = choosePlacement({ hostId: prior.automatic ? undefined : instance.host_id,
libraryId: instance.library_id, image: instance.image, bindAddress: network.bindAddress,
thorough: true, dispatch: true, excludeInstanceId: instance.id });
const state = db().prepare('SELECT state FROM server_deployments WHERE id=? AND server_deleted_at IS NULL').get(job.id) as { state: string } | undefined;
if (state?.state !== 'queued') return null;
const live = nativeInstance(job.server);
if (instance ? live?.id !== instance.id || live.generation !== instance.generation : !!live || !!getServer(job.server)) return null;
const active = (db().prepare(`SELECT count(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL
AND action IN ('create','rebuild') AND state IN ('running','awaiting-health')`).get(job.id) as { n: number }).n;
if (active + otherPreparations(job.id) >= fleetSettings().createConcurrency) { hold(job, 'Waiting for startup capacity.'); return null; }
if (getHost(decision.hostId)?.lastHeartbeat !== decision.lastHeartbeat) { hold(job, 'Host measurements changed; waiting for another placement check.'); return null; }
// A short current-state check protects against reservations made while the
// reader ranked hosts. It never scans resource history or contacts agents.
const { host, port } = choosePlacement({ ...options, hostId: decision.hostId, thorough: false });
const bind = network.bindAddress ?? host.bindAddress;
releaseHostPorts(instance.id);
reserveHostPort(host.id, instance.id, { port, bindAddress: bind, protocol: 'udp' });
reserveHostPort(host.id, instance.id, { port, bindAddress: bind, protocol: 'tcp' });
db().prepare(`UPDATE native_instances SET host_id=?,bind_address=?,port=?,public_address=?,public_port=? WHERE id=?`)
.run(host.id, bind, port, network.publicAddress ?? host.publicAddress, network.publicPort ?? null, instance.id);
const instanceId = instance?.id ?? randomUUID();
if (instance) releaseHostPorts(instanceId);
reserveHostPort(host.id, instanceId, { port, bindAddress: bind, protocol: 'udp' });
reserveHostPort(host.id, instanceId, { port, bindAddress: bind, protocol: 'tcp' });
if (instance) db().prepare(`UPDATE native_instances SET host_id=?,bind_address=?,port=?,public_address=?,public_port=? WHERE id=?`)
.run(host.id, bind, port, network.publicAddress ?? host.publicAddress, network.publicPort ?? null, instanceId);
else {
db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,bind_address,port,public_address,public_port,created_at)
VALUES(?,?,?,?,?,?,?,?,?,?)`).run(instanceId, job.server, host.id, prior.libraryId, options.image, bind, port,
network.publicAddress ?? host.publicAddress, network.publicPort ?? null, new Date().toISOString());
db().prepare('INSERT INTO native_provisioning(deployment_id,instance_id,generation) VALUES(?,?,1)').run(job.id, instanceId);
}
db().prepare('UPDATE server_creation_jobs SET port=? WHERE deployment_id=?').run(port, job.id);
recipe.native = { ...prior, hostId: host.id, bindAddress: bind, publicAddress: network.publicAddress ?? host.publicAddress, publicPort: network.publicPort };
const stored = recipeStorage(recipe);
db().prepare('UPDATE server_launch_recipes SET recipe=?,recipe_summary=? WHERE deployment_id=?').run(stored.recipe, stored.recipeSummary, job.id);
db().prepare('UPDATE server_launch_previews SET port=? WHERE deployment_id=?').run(port, job.id);
deploymentStage(job.id, `Placed on ${host.name} after CPU, memory, history and port checks`, { phase: 'eligibility' });
preparingJobs.add(job.id);
return nativeInstance(job.server)!;
})();
}
@@ -309,15 +356,24 @@ function placeQueuedCreation(job: Job, instance: NativeInstance): NativeInstance
async function processJob(job: Job, eligibility: (job: Job) => string | null) {
if (!stillActive(job)) return;
let instance = nativeInstance(job.server);
if (!instance || instance.id !== job.instance_id || (instance.generation !== job.generation && !(job.action === 'rebuild' && instance.generation === job.generation + 1))) {
finishDeployment(job.id, { ok: false, message: 'Original instance was retired or replaced; no command was sent.' }); return;
}
if (job.action === 'create' && !job.operation_id && !job.stage_operation_id && !job.submitted_at) {
// A resumed bound job owns its original identity. An unplaced intent
// cannot adopt an instance or registered server that appeared later.
if (job.instance_id ? !instance || instance.id !== job.instance_id || instance.generation !== job.generation
: !!instance || !!getServer(job.server)) {
finishDeployment(job.id, { ok: false, message: 'Original instance was retired or replaced; no command was sent.' }); return;
}
const settings = fleetSettings();
const active = db().prepare(`SELECT COUNT(*) AS n FROM server_deployments WHERE id!=? AND server_deleted_at IS NULL
AND action IN ('create','rebuild') AND state IN ('running','awaiting-health')`).get(job.id) as { n: number };
if (active.n + otherPreparations(job.id) >= settings.createConcurrency) { hold(job, 'Waiting for startup capacity.'); return; }
instance = placeQueuedCreation(job, instance);
if (eligibility(job)) return;
instance = await placeQueuedCreation(job, instance);
if (!instance) return;
job.instance_id = instance.id; job.generation = instance.generation;
}
if (!instance || instance.id !== job.instance_id || (instance.generation !== job.generation && !(job.action === 'rebuild' && instance.generation === job.generation + 1))) {
finishDeployment(job.id, { ok: false, message: 'Original instance was retired or replaced; no command was sent.' }); return;
}
const host = getHost(instance.host_id);
if (!host?.connected || host.mode === 'disabled') {
@@ -448,7 +504,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
if (job.action === 'destroy' && count(['destroy']) >= settings.destroyConcurrency) { hold(job, 'Waiting for destruction capacity.'); return; }
deploymentStage(job.id, 'Checking host eligibility', { phase: 'eligibility', blocker: null });
if (job.action === 'create') choosePlacement({ hostId: instance.host_id, libraryId: instance.library_id, image: instance.image,
bindAddress: instance.bind_address, excludeInstanceId: instance.id, thorough: true, dispatch: true });
bindAddress: instance.bind_address, excludeInstanceId: instance.id, thorough: false, dispatch: true });
if (job.action === 'create' || job.action === 'rebuild') assertReadyLibrary(instance.host_id, instance.library_id);
const checkOccupancy = async (): Promise<boolean> => {
// Fast controller check keeps occupied lobbies out of the operation slots.
@@ -514,7 +570,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
}
assertReadyLibrary(instance!.host_id, instance!.library_id);
if (job.action === 'create') choosePlacement({ hostId: instance!.host_id, libraryId: instance!.library_id, image: instance!.image,
bindAddress: instance!.bind_address, excludeInstanceId: instance!.id, thorough: true, dispatch: true });
bindAddress: instance!.bind_address, excludeInstanceId: instance!.id, thorough: false, dispatch: true });
}
if (job.action === 'start' || job.action === 'restart' || job.action === 'rebuild') {
db().prepare(`INSERT INTO restart_rollout(server,requested_at,operation_id) VALUES(?,?,?) ON CONFLICT(server)
src/lib/server/provisioning/requests.ts
+2 -2
View File
@@ -7,7 +7,7 @@ import { queueRebuildServer } from './instances.js';
import { restartRequestInFlight } from './restarts.js';
import { runtimeActions, type RuntimeAction } from '../../provisioning.js';
import { recordProgress } from './progress.js';
import { nativeInstance } from '../backends/native.js';
import { nativeInstance, unplacedNativeCreation } from '../backends/native.js';
export interface LifecycleOperation {
id: string; server: string; action: string; state: string; stage: string;
@@ -137,7 +137,7 @@ export function releaseRestartRequest(server: string, source: RestartSource): vo
export function requestLifecycle(server: string, action: RuntimeAction) {
if (!runtimeActions.includes(action)) return { ok: false as const, status: 400, error: 'Choose start, restart, rebuild, or stop.' };
if (!nativeInstance(server) && (!komodoConfigured() || !rconConfigured())) return { ok: false as const, status: 503, error: 'Configure a host and RCON before provisioning.' };
if (!nativeInstance(server) && !unplacedNativeCreation(server) && (!komodoConfigured() || !rconConfigured())) return { ok: false as const, status: 503, error: 'Configure a host and RCON before provisioning.' };
return db().transaction(() => {
const blocker = action === 'rebuild' ? activeBetaRemovalBlocker(server) : null;
if (blocker) return { ok: false as const, status: 409, error: blocker };