josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Remove Komodo backend in favor of native hosts

Browse Source
This commit is contained in:
josh
2026-10-05 18:07:07 -04:00
parent 701066aaf8
commit 4b0f341d64
116 changed files with 557 additions and 8132 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/provisioning-detail.ts
+4 -4
View File
@@ -1,12 +1,12 @@
import type { ProvisioningAction } from './provisioning.js';
import type { LaunchRecipeSummary } from './deployment-plan.js';
export const provisioningPhases = ['requested', 'eligibility', 'startup-queue', 'prepare', 'create-deployment',
'container-action', 'remove-container', 'delete-deployment', 'cleanup', 'verify-container', 'verify-game',
export const provisioningPhases = ['requested', 'eligibility', 'startup-queue', 'prepare',
'container-action', 'cleanup', 'verify-container', 'verify-game',
'verify-map', 'verify-plutonium', 'verify-stopped'] as const;
export type ProvisioningPhase = typeof provisioningPhases[number];
export type EvidenceResult = 'pass' | 'waiting' | 'fail' | 'unknown';
export type EvidenceSource = 'komodo' | 'agent' | 'rcon' | 'telemetry' | 'combined' | 'manifest' | null;
export type EvidenceSource = 'agent' | 'rcon' | 'telemetry' | 'combined' | 'manifest' | null;
export interface ProvisioningCheck {
key: 'container' | 'game' | 'map' | 'plutonium-release' | 'plutonium-version' | 'occupancy' | 'stopped';
@@ -98,7 +98,7 @@ export interface ProvisioningDetail {
canCancel?: boolean;
canCancelCreation?: boolean;
cancellation?: { cleanupId: string; state: string; stage: string; error: string | null } | null;
backend?: 'native' | 'komodo';
backend?: 'native';
workerActivity?: WorkerActivity[];
agentOperation?: { id: string; hostId: string; state: string; updatedAt: string; createdAt: string; error: string | null } | null;
deployment: ProvisioningDeployment;
src/lib/server/config.ts
+1 -29
View File
@@ -33,31 +33,7 @@ export const config = {
betaArchiveDir: req('BETA_ARCHIVE_DIR', '/root/zombies/beta-archive'),
dbPath: req('DB_PATH', './data/cherry.db'),
httpTimeoutMs: num('HTTP_TIMEOUT_MS', 10_000),
komodo: {
address: env.KOMODO_ADDRESS ?? '',
apiKey: env.KOMODO_API_KEY ?? '',
apiSecret: env.KOMODO_API_SECRET ?? '',
// Komodo server (host) new deployments land on. Required for the
// deploy flow — Komodo needs to know which machine to schedule the
// container onto. Look up the exact name in Komodo's UI (Servers page).
targetServer: env.KOMODO_TARGET_SERVER ?? '',
// Default image for freshly-provisioned T5 and T6 zombie servers.
// Overridable per-deploy in the UI later if we ever need to.
defaultImage: env.PLUTAINER_IMAGE ?? 'ghcr.io/ayymoss/plutainer:latest',
defaultNetwork: env.PLUTAINER_NETWORK ?? 'komodo_default',
// Named docker volume that holds the read-only game files layer.
gameFilesVolume: env.PLUTAINER_GAMEFILES_VOLUME ?? 't5_game_files',
t6GameFilesVolume: env.PLUTAINER_T6_GAMEFILES_VOLUME ?? 't6_game_files'
},
// Shared across every newly-provisioned server. Kept in sync with the
// fleet's RCON_PASSWORD (rcon.password) so freshly-created servers respond
// to the same rcon secret the sweep already uses.
// Port range Electric Cherry auto-allocates from when creating a new
// deployment. Update this when the host's firewall opens more.
portRange: {
min: num('SERVER_PORT_MIN', 28960),
max: num('SERVER_PORT_MAX', 28980)
},
defaultRuntimeImage: env.PLUTAINER_IMAGE ?? 'ghcr.io/ayymoss/plutainer:latest',
rcon: {
// Host every RCON packet is sent to. Defaults to the Docker
// host-gateway alias so the electric-cherry container can reach
@@ -70,10 +46,6 @@ export const config = {
}
} as const;
export function komodoConfigured(): boolean {
return !!(config.komodo.address && config.komodo.apiKey && config.komodo.apiSecret);
}
export function rconConfigured(): boolean {
return !!config.rcon.password;
}
src/lib/server/db/index.ts
+9 -2
View File
@@ -24,7 +24,6 @@ const SERVERS_COLUMNS: Array<{ name: string; ddl: string }> = [
{ name: 'desired_state', ddl: "ALTER TABLE servers ADD COLUMN desired_state TEXT NOT NULL DEFAULT 'running' CHECK (desired_state IN ('running','stopped'))" },
// Base columns that shipped in the very first schema stay here for parity
// with fresh installs — no ALTER is issued if they're already present.
{ name: 'komodo_server', ddl: 'ALTER TABLE servers ADD COLUMN komodo_server TEXT' },
{ name: 'game_port', ddl: 'ALTER TABLE servers ADD COLUMN game_port INTEGER' },
{ name: 'rcon_host', ddl: 'ALTER TABLE servers ADD COLUMN rcon_host TEXT' },
{ name: 'last_players', ddl: 'ALTER TABLE servers ADD COLUMN last_players INTEGER' },
@@ -65,6 +64,14 @@ function migrateServers(handle: Database.Database): void {
handle.exec(col.ddl);
}
}
// Retire external-backend registry fields, including required fields that
// would otherwise prevent native inserts into an upgraded database.
const retired = new Set(['komodo_type', 'komodo_name', 'komodo_server']);
handle.transaction(() => {
for (const column of rows) {
if (retired.has(column.name)) handle.exec(`ALTER TABLE servers DROP COLUMN "${column.name.replaceAll('"', '""')}"`);
}
})();
}
// Additive columns on tables the script model owns. Same pattern as
@@ -286,7 +293,7 @@ function migrateWatchdogConstraints(handle: Database.Database): void {
function backfillDesiredState(handle: Database.Database): void {
if (handle.prepare("SELECT 1 FROM settings WHERE key = 'watchdog_desired_state_v1'").get()) return;
handle.transaction(() => {
// A failed stop may have reached Komodo. Preserve that explicit intent.
// A failed stop may have reached the host agent. Preserve that explicit intent.
handle.exec(`UPDATE servers SET desired_state = 'stopped' WHERE 'stop' = (
SELECT action FROM server_deployments d WHERE d.server = servers.name AND d.state != 'cancelled'
AND d.action IN ('create','start','restart','rebuild','stop')
src/lib/server/db/schema.sql
+2 -5
View File
@@ -63,13 +63,10 @@ CREATE INDEX IF NOT EXISTS idx_leaderboard_player_matches ON leaderboard_match_p
CREATE TABLE IF NOT EXISTS servers (
name TEXT PRIMARY KEY,
komodo_type TEXT NOT NULL DEFAULT 'deployment', -- deployment|stack|container
komodo_name TEXT NOT NULL,
komodo_server TEXT,
container_checked_at TEXT,
container_state TEXT, -- last observed Komodo runtime state
container_state TEXT, -- last observed host-agent runtime state
desired_state TEXT NOT NULL DEFAULT 'running' CHECK (desired_state IN ('running','stopped')),
game_port INTEGER, -- UDP RCON port; lazy-filled from Komodo
game_port INTEGER, -- UDP game port assigned on the host
rcon_host TEXT, -- optional per-server override of RCON_HOST
game TEXT, -- 't5' etc.; static per-server (no map rotation)
map TEXT, -- 'zombie_theater' etc.; static per-server
src/lib/server/fleet/beta.ts
+1 -1
View File
@@ -376,7 +376,7 @@ export async function openBeta(input: OpenBetaInput): Promise<OpenBetaResult> {
const removal = db().prepare(`SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL
AND action IN ('destroy', 'rebuild') AND (state IN ('waiting', 'queued', 'running', 'awaiting-health')
OR (state IN ('failed', 'interrupted') AND stage IN (
'Stopping and removing container', 'Deleting Komodo deployment', 'Removing server files and registry')))
'Stopping and removing container', 'Removing server files and registry')))
LIMIT 1`).get(name);
if (removal) throw new BetaError(`${name}: cancel or finish the pending destruction or rebuild before opening a beta.`, 'conflict');
serverRows.push({ name, game: row.game, map: row.map });
src/lib/server/fleet/collections.ts
+18 -47
View File
@@ -2,16 +2,14 @@ import { fleetSettings } from '../settings.js';
import { randomUUID } from 'node:crypto';
import type { CollectionDetail, CollectionMode, CollectionPreview, CollectionSummary } from '$lib/deployment-plan.js';
import { db } from '../db/index.js';
import { config, komodoConfigured, rconConfigured } from '../config.js';
import { blueprintEntries, summarizeRecipe, type BlueprintEntry } from './launch-recipes.js';
import { creationQueue, destroyServer, nextAvailablePort, queueCreateServer } from '../provisioning/index.js';
import { creationQueue, destroyServer } from '../provisioning/index.js';
import { deploymentStage, finishDeployment, recordDeploymentEvent } from './deployments.js';
import { housePartyTargets } from './house-party.js';
import { serverObservation } from './status.js';
import { activeBetaRemovalBlocker, getServer } from './servers.js';
import { parseServerName, zombieMapBySlug, zombieMaps } from './map-aliases.js';
import { nativeLaunchSelection, queueNativeCreationIntent } from '../provisioning/native.js';
import { listHosts } from '../hosts/index.js';
import { nativeDeployment, nativeInstance } from '../backends/native.js';
import { launchAvailability } from '../hosts/availability.js';
@@ -71,7 +69,7 @@ function collectionPlan(entries: BlueprintEntry[], mode: CollectionMode, include
return { counts, retainedServers, removals };
}
function allocate(entries: BlueprintEntry[], releasingServers: string[] = [], counts = coverage(entries)): CollectionPreview['servers'] {
function allocate(entries: BlueprintEntry[], counts = coverage(entries)): CollectionPreview['servers'] {
const occupied = db().prepare(`SELECT name FROM servers UNION
SELECT j.server AS name FROM server_creation_jobs j JOIN server_deployments d ON d.id=j.deployment_id
WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued','running','awaiting-health')
@@ -85,16 +83,14 @@ function allocate(entries: BlueprintEntry[], releasingServers: string[] = [], co
taken.add(Number(match[2])); lobbies.set(match[1], taken);
}
const savedKeys = new Set((db().prepare("SELECT server_id FROM server_keys WHERE length(trim(pluto_key))>0").all() as { server_id: string }[]).map(row => row.server_id));
const ports: number[] = [];
return entries.flatMap((entry, entryIndex) => Array.from({ length: counts[entryIndex].missing }, () => {
const base = `${entry.recipe.game}_zombie_${entry.recipe.mapSlug}`;
const taken = lobbies.get(base) ?? new Set<number>();
const lobby = Array.from({ length: 20 }, (_, index) => index + 1).find(number => !taken.has(number)) ?? null;
const name = lobby === null ? null : `${base}_${lobby}`;
const native = entry.recipe.native;
const port = native ? null : nextAvailablePort(ports, releasingServers);
const port = null;
if (lobby !== null) { taken.add(lobby); lobbies.set(base, taken); }
if (port !== null) ports.push(port);
return { entryIndex, name, port, ...(native ? { native } : {}), hasSavedKey: !!name && savedKeys.has(name) };
}));
}
@@ -102,13 +98,8 @@ function allocate(entries: BlueprintEntry[], releasingServers: string[] = [], co
function blockersFor(servers: CollectionPreview['servers'], entries: BlueprintEntry[] = []): string[] {
const blockers: string[] = [];
if (!servers.length) return blockers;
const legacy = servers.filter(server => !server.native && !entries[server.entryIndex]?.recipe.native);
if (legacy.length) {
if (!komodoConfigured() || !config.komodo.targetServer) blockers.push('Select and prepare a host before launching.');
if (!rconConfigured()) blockers.push('Configure RCON_PASSWORD so every new server can be verified.');
}
if (servers.some(server => !server.native && !entries[server.entryIndex]?.recipe.native)) blockers.push('Select and prepare a host before launching.');
if (servers.some(server => !server.name)) blockers.push('There are not enough free lobbies for one or more maps (20 per map). Reduce their counts or free capacity.');
if (legacy.some(server => server.port === null)) blockers.push(`There are not enough free game ports in ${config.portRange.min}–${config.portRange.max} for the whole collection.`);
return blockers;
}
@@ -120,7 +111,7 @@ function removalBlockersFor(removals: CollectionPreview['removals']): string[] {
AND action='create' AND state IN ('queued','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(name) as { id: string } | undefined;
return !!creation && nativeDeployment(creation.id);
};
if (removals.some(server => !native(server.name)) && (!komodoConfigured() || !rconConfigured())) blockers.push('Configure the legacy backend before removing its servers.');
if (removals.some(server => !native(server.name))) blockers.push('Every server to remove must have a native host instance.');
for (const server of removals) {
const blocker = activeBetaRemovalBlocker(server.name);
if (blocker) blockers.push(blocker);
@@ -133,15 +124,13 @@ export function previewCollection(blueprintId: unknown, mode: unknown = 'fill',
if (mode !== 'fill' && mode !== 'exact') throw new Error('Choose fill or exact collection mode.');
return db().transaction((): CollectionPreview => {
const { entries, blueprintName } = blueprintEntries(blueprintId);
if (selection?.hostId !== undefined || !config.komodo.address || listHosts().length > 0) {
const libraries = selection?.libraryIds as Record<string, unknown> | undefined;
for (const entry of entries) {
entry.recipe.native = nativeLaunchSelection({ ...selection, libraryId: libraries?.[entry.recipe.game] ?? selection?.libraryId }, entry.recipe.game);
entry.recipe.image = entry.recipe.native.image!;
}
const libraries = selection?.libraryIds as Record<string, unknown> | undefined;
for (const entry of entries) {
entry.recipe.native = nativeLaunchSelection({ ...selection, libraryId: libraries?.[entry.recipe.game] ?? selection?.libraryId }, entry.recipe.game);
entry.recipe.image = entry.recipe.native.image!;
}
const { counts, retainedServers, removals } = collectionPlan(entries, mode);
const servers = allocate(entries, removals.map(server => server.name), counts);
const servers = allocate(entries, counts);
const blockers = [...blockersFor(servers, entries), ...removalBlockersFor(removals)];
const selections = servers.flatMap(server => server.native ? [server.native] : []);
const availability = selections.length ? launchAvailability(selections) : {};
@@ -176,16 +165,12 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
throw new Error('The fleet changed. Nothing was queued. Refresh the preview and review the servers to keep and remove.');
}
const { counts, retainedServers, removals } = collectionPlan(entries, preview.mode, false);
const releasingServers = removals.map(server => server.name);
const nativeReview = reviewed.every(server => server.native || entries[server.entryIndex]?.recipe.native);
// Native Apply accepts the reviewed logical identities. Capacity and
// physical ports can change while this request waits in the queue.
const servers = nativeReview ? reviewed : allocate(entries, releasingServers, counts);
const servers = reviewed;
const validatedSelections = new Set<string>();
for (const server of servers) {
const recipe = entries[server.entryIndex].recipe;
const selection = server.native ?? recipe.native;
if (!selection) continue;
if (!selection) throw new Error('Select a native host before launching. Refresh the preview.');
const identity = JSON.stringify([recipe.game, selection]);
if (validatedSelections.has(identity)) continue;
// Recheck saved configuration once per selection without allocating
@@ -195,9 +180,8 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
}
const blockers = [...blockersFor(servers, entries), ...removalBlockersFor(removals)];
if (blockers.length) throw new Error(blockers.join(' '));
if (entries.some((_entry, index) => counts[index].missing !== reviewed.filter(server => server.entryIndex === index).length)
|| servers.length !== reviewed.length || servers.some((server, i) => server.entryIndex !== reviewed[i]?.entryIndex || server.name !== reviewed[i]?.name || (!server.native && server.port !== reviewed[i]?.port))) {
throw new Error('Fleet counts, lobby or port availability changed. Nothing was queued. Refresh the preview and review each server key.');
if (entries.some((_entry, index) => counts[index].missing !== reviewed.filter(server => server.entryIndex === index).length)) {
throw new Error('Fleet counts changed. Nothing was queued. Refresh the preview and review each server key.');
}
const expected = new Set(servers.map(server => server.name));
for (const [name, value] of Object.entries(keys)) {
@@ -207,10 +191,7 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
const collectionId = randomUUID();
db().prepare('INSERT INTO server_launch_collections (id, name, created_at, mode, retained_servers) VALUES (?, ?, ?, ?, ?)')
.run(collectionId, preview.blueprint_name, new Date().toISOString(), preview.mode, JSON.stringify(retainedServers));
const removalJobs = removals.map((server, position) => {
const ports = (db().prepare(`SELECT game_port AS port FROM servers WHERE name = ? UNION
SELECT j.port FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
WHERE j.server = ? AND d.server_deleted_at IS NULL AND d.state IN ('queued', 'running', 'awaiting-health')`).all(server.name, server.name) as { port: number | null }[]).map(row => row.port);
removals.forEach((server, position) => {
const queued = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'create' AND state = 'queued'").get(server.name) as { id: string } | undefined;
let id: string;
if (queued && !getServer(server.name)) {
@@ -230,27 +211,17 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
id = result.deploymentId;
}
db().prepare('INSERT INTO server_collection_removals (collection_id, deployment_id, position) VALUES (?, ?, ?)').run(collectionId, id, position);
return { id, ports };
});
servers.forEach((server, index) => {
const recipe = { ...entries[server.entryIndex].recipe, ...(server.native ? { native: server.native } : {}) };
let result;
if (recipe.native) result = queueNativeCreationIntent({ server: server.name!, game: recipe.game, mapSlug: recipe.mapSlug,
if (!recipe.native) throw new Error('Select a native host before launching. Refresh the preview.');
const result = queueNativeCreationIntent({ server: server.name!, game: recipe.game, mapSlug: recipe.mapSlug,
selection: recipe.native, recipe, plutoKey: keys[server.name!] as string | undefined, createdBy: 'ui', blueprintName: preview.blueprint_name });
else {
const childPreviewId = randomUUID();
db().prepare('INSERT INTO server_launch_previews (id, expires_at, server, port, recipe, blueprint_name) VALUES (?, ?, ?, ?, ?, ?)')
.run(childPreviewId, preview.expires_at, server.name, server.port, JSON.stringify(recipe), preview.blueprint_name);
result = queueCreateServer({ game: recipe.game, gameMode: 'zombie', mapSlug: recipe.mapSlug,
expectedName: server.name!, previewId: childPreviewId, plutoKey: keys[server.name!] as string | undefined, createdBy: 'ui' }, releasingServers);
}
// Throw so the outer transaction also reverses all earlier keys and children.
if (!result.ok) throw new Error(`${server.name}: ${result.message} Nothing was queued.`);
db().prepare('INSERT INTO server_collection_members (collection_id, deployment_id, position) VALUES (?, ?, ?)')
.run(collectionId, result.deploymentId, index);
for (const removal of recipe.native ? [] : removalJobs.filter(removal => removal.ports.includes(server.port))) {
db().prepare('INSERT INTO server_creation_dependencies (deployment_id, removal_id) VALUES (?, ?)').run(result.deploymentId, removal.id);
}
});
db().prepare('UPDATE server_collection_previews SET collection_id = ? WHERE id = ?').run(collectionId, preview.id);
return { ok: true as const, collectionId };
src/lib/server/fleet/health-observations.ts
+6 -13
View File
@@ -3,7 +3,7 @@ import { config } from '../config.js';
import { db } from '../db/index.js';
import { getHost, queueHostOperation, waitHostOperationStatus } from '../hosts/index.js';
import { nativeInstance, type NativeInstance } from '../backends/native.js';
import { komodoInspectHealthy } from '../integrations/komodo.js';
import { nativeHealth } from '../backends/native.js';
import { rconStatusResult, agentRconStatusResult, type AgentRconReply } from '../integrations/rcon.js';
import { withOperationSource } from '../hosts/operation-context.js';
import { getServer } from './servers.js';
@@ -39,7 +39,6 @@ function identity(server: string, probe: Probe): Identity {
return {
key: JSON.stringify([row.created_at, row.desired_state, latestGameSubmission(server), lifecycle,
native ? [native.host_id, native.id, native.generation] : null,
native ? null : [row.komodo_type, row.komodo_name, row.komodo_server, config.komodo, config.httpTimeoutMs],
probe === 'rcon' ? [row.rcon_host, config.rcon] : null]),
port: probe === 'rcon' ? native?.port ?? row.game_port : null,
checkedAt: probe === 'rcon' ? row.last_checked_at : row.container_checked_at
@@ -77,13 +76,7 @@ async function observe<T extends { ok: boolean }>(server: string, fresh: boolean
while (entries.size > MAX_SERVERS) entries.delete(entries.keys().next().value!);
const pending = Promise.resolve().then(() => request(server)).then(result => {
const latest = identity(server, probe);
// Legacy RCON may discover and save its first port during this request.
// Accept that enrichment only when all other identity fields agree.
const endpoint = 'endpoint' in result ? result.endpoint : null;
const learnedPort = probe === 'rcon' && !current.port && result.ok
&& endpoint !== null && typeof endpoint === 'object' && 'port' in endpoint && endpoint.port === latest.port;
if (current.key !== latest.key || (current.port !== latest.port && !learnedPort)
|| newerEvidence(latest, entry.startedAt)) {
if (!sameIdentity(current, latest) || newerEvidence(latest, entry.startedAt)) {
throw new MonitoringChangedError('Server changed during monitoring; the previous observation was discarded.');
}
const observation = { result, observedAt: new Date().toISOString() };
@@ -100,12 +93,12 @@ async function observe<T extends { ok: boolean }>(server: string, fresh: boolean
return pending;
}
const inspections = new Map<string, Entry<Awaited<ReturnType<typeof komodoInspectHealthy>>>>();
const inspections = new Map<string, Entry<Awaited<ReturnType<typeof nativeHealth>>>>();
const rcon = new Map<string, Entry<Awaited<ReturnType<typeof rconStatusResult>>>>();
interface MonitoringStatus {
identity: string;
container: Observation<Awaited<ReturnType<typeof komodoInspectHealthy>>>;
container: Observation<Awaited<ReturnType<typeof nativeHealth>>>;
game?: Observation<Awaited<ReturnType<typeof rconStatusResult>>>;
}
// Keep failed samples too, independently of the successful-probe cache. Reading
@@ -160,7 +153,7 @@ export function monitoringStatus(server: string): MonitoringStatus | null {
}
/** Keep the actual sample time: a later heartbeat must not renew old evidence. */
export function heartbeatInspection(server: string): Observation<Awaited<ReturnType<typeof komodoInspectHealthy>>> | null {
export function heartbeatInspection(server: string): Observation<Awaited<ReturnType<typeof nativeHealth>>> | null {
const row = getServer(server);
if (!row?.native_instance_id || !row.host_id) return null;
const host = getHost(row.host_id);
@@ -242,7 +235,7 @@ export function inspectForMonitoring(server: string, fresh = false) {
if (fresh) statuses.delete(server);
const heartbeat = !fresh && heartbeatInspection(server);
if (heartbeat) return Promise.resolve(heartbeat);
return observe(server, fresh, 'inspect', inspections, komodoInspectHealthy);
return observe(server, fresh, 'inspect', inspections, nativeHealth);
}
export function rconForMonitoring(server: string, fresh = false) {
if (fresh) statuses.delete(server);
src/lib/server/fleet/launch-recipes.ts
+1 -1
View File
@@ -105,7 +105,7 @@ function buildRecipe(game: ZombieGame, mapSlug: string, includeContent: boolean)
}
addConfig('Selected map', MAP_CONFIG_FILE, getServerMapConfig(game, map.canonical, map.slug)!);
return { format: 1, game, mapSlug, mapName: map.friendly, expectedMap: map.canonical,
capturedAt: new Date().toISOString(), image: config.komodo.defaultImage,
capturedAt: new Date().toISOString(), image: config.defaultRuntimeImage,
configFile: configFileForGame(game), configSource, files };
}
src/lib/server/fleet/servers.ts
+3 -11
View File
@@ -74,9 +74,6 @@ export interface ServerRow {
instance_generation?: number | null;
game_library_id?: string | null;
name: string;
komodo_type: 'deployment' | 'stack' | 'container';
komodo_name: string;
komodo_server: string | null;
container_state: string | null;
container_checked_at: string | null;
desired_state: 'running' | 'stopped';
@@ -112,12 +109,10 @@ export interface ServerRow {
export async function seedServerRegistry(): Promise<number> {
// Explicit legacy import only; never run as part of application startup.
const rows = await serverDirectories();
// Under the new naming scheme Komodo entities match EC server names 1:1
// (both underscore-separated), so komodo_name = name — no transformation.
const insert = db().prepare(
`INSERT OR IGNORE INTO servers
(name, komodo_type, komodo_name, game, map, game_mode, map_slug, host_dir)
VALUES (?, 'deployment', ?, ?, ?, ?, ?, ?)`
(name, game, map, game_mode, map_slug, host_dir)
VALUES (?, ?, ?, ?, ?, ?)`
);
// Backfill existing rows where derived columns were never set. game_mode
// and map_slug are new; older rows can still surface them from the name.
@@ -139,7 +134,7 @@ export async function seedServerRegistry(): Promise<number> {
const gameMode = parsed?.gameMode ?? null;
const mapSlug = parsed?.mapSlug ?? null;
const hostDir = serverDir(name);
const res = insert.run(name, name, game, map, gameMode, mapSlug, hostDir);
const res = insert.run(name, game, map, gameMode, mapSlug, hostDir);
if (res.changes > 0) {
n++;
} else {
@@ -185,9 +180,6 @@ export function setServerField(
export function setServerFields(name: string, fields: Partial<Record<keyof ServerRow, string | number | null>>): void {
const allowed: Array<keyof ServerRow> = [
'komodo_type',
'komodo_name',
'komodo_server',
'container_state',
'container_checked_at',
'game_port',
src/lib/server/fleet/snapshot.ts
+3 -5
View File
@@ -1,7 +1,7 @@
import { db } from '../db/index.js';
import { getServer, listServers } from './servers.js';
import { observationReader, serverObservation } from './status.js';
import { config, komodoConfigured, rconConfigured } from '../config.js';
import { config, rconConfigured } from '../config.js';
import { runningVersions } from '../integrations/plutonium.js';
import { listHosts } from '../hosts/index.js';
import type { NativeInstance } from '../backends/native.js';
@@ -33,8 +33,7 @@ export function fleetSnapshot(serverName?: string) {
host: s.host_id ? hostsById.get(s.host_id) ?? null : null,
network: instance ? { bindAddress: instance.bind_address, localPort: instance.port, publicAddress: instance.public_address, publicPort: instance.public_port } : null,
plutonium: versions.get(s.name),
// Derived endpoint — same logic as endpointFor() but without asking
// Komodo. If we don't have a port yet, endpoint is null.
// The host agent owns the game connection.
endpoint: instance ? { host: `agent:${instance.host_id}`, port: instance.port } : s.game_port ? { host: s.rcon_host ?? rconHost, port: s.game_port } : null,
held: holds.get(s.name) ?? []
};
@@ -44,11 +43,10 @@ export function fleetSnapshot(serverName?: string) {
serverNow: Date.now(),
reservedNames: serverName === undefined ? (db().prepare("SELECT j.server FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')").all() as { server: string }[]).map(row => row.server) : [],
integrations: {
native: hosts.length > 0 || !komodoConfigured(),
native: true,
hostsReady: hosts.filter(host => host.readiness === 'ready').length,
hostsTotal: hosts.length,
rcon: rconConfigured(),
komodo: komodoConfigured(),
rconTimeoutMs: config.rcon.timeoutMs,
defaultRconHost: rconHost
}
src/lib/server/fleet/status-evidence.ts
+1 -1
View File
@@ -59,7 +59,7 @@ export async function probePlayerEvidence(server: string): Promise<PlayerEvidenc
return { players: null, source: null, observedAt: new Date().toISOString() };
}
/** Komodo readiness remains a caller prerequisite. Old-process telemetry is excluded. */
/** Host agent readiness remains a caller prerequisite. Old-process telemetry is excluded. */
export async function probeServerStatus(server: string, after?: string | null): Promise<
{ ok: true; players: number; map: string | null; source: 'rcon' | 'telemetry'; observedAt: string; receivedAt?: string } | { ok: false; reason: string; observedAt: string }
> {
src/lib/server/fleet/status.ts
+1 -2
View File
@@ -7,7 +7,6 @@ import { telemetryReport, latestGameSubmission } from './status-evidence.js';
import { latestGameSubmissions } from './lifecycle-evidence.js';
import { db } from '../db/index.js';
import { parseGameEvent, type GameTelemetry } from '../../telemetry.js';
import { restartRequestInFlight } from '../provisioning/restarts.js';
interface Evidence { provisioning: boolean; settling: boolean; telemetry: GameTelemetry | null; submission: string | null }
@@ -33,7 +32,7 @@ export function observationReader() {
const operation = operations.get(server.name), restart = restarts.get(server.name);
return serverObservation(server, {
provisioning: !!operation && ['create','rebuild','start'].includes(operation.action) && ['queued','running','awaiting-health'].includes(operation.state),
settling: restartRequestInFlight(server.name) || !!(restart && !restart.healthy_at && Date.now() - Date.parse(restart.requested_at) < 15_000),
settling: !!(restart && !restart.healthy_at && Date.now() - Date.parse(restart.requested_at) < 15_000),
telemetry: reports.get(server.name) ?? null, submission: submissions.get(server.name) ?? null
});
};
src/lib/server/hosts/capacity-overview.ts
+1 -1
View File
@@ -32,7 +32,7 @@ export async function readCapacityCurrent(query: CapacityQuery): Promise<Capacit
if (host.decommission) reasons.push('Host is being removed.');
if (!host.credentialActive && host.enrolled) reasons.push('Host credentials are inactive.');
if (host.readiness !== 'ready') reasons.push(...host.readinessReasons);
if (!host.inventory.runtimeImages?.includes(config.komodo.defaultImage)) reasons.push('The default runtime is not ready.');
if (!host.inventory.runtimeImages?.includes(config.defaultRuntimeImage)) reasons.push('The default runtime is not ready.');
if (!requestedGames.some(game => games.includes(game))) reasons.push('Latest game files for this selection are not ready.');
if ((host.inventory.storageAvailable ?? 0) < host.minimumFreeBytes + 1024 ** 3) reasons.push('Storage is below the launch reserve.');
const eligible = connected && host.readiness === 'ready' && reasons.length === 0;
src/lib/server/hosts/preparation.ts
+1 -1
View File
@@ -88,7 +88,7 @@ function targets(host: Host): Target[] {
&& operation.sequence > enabling.operation_floor && operation.state === 'completed'
&& Date.parse(host.lastHeartbeat ?? '') > Date.parse(operation.updated_at);
const selections = queuedSelections(host.id);
const images = new Set([config.komodo.defaultImage, ...selections.flatMap(selection => selection.image ? [selection.image] : [])]);
const images = new Set([config.defaultRuntimeImage, ...selections.flatMap(selection => selection.image ? [selection.image] : [])]);
const result: Target[] = [...images].map(image => {
const operation = latestOperation(host.id, 'runtime.pull', 'image', image);
const runtimeReady = host.inventory.runtimeImages?.includes(image) && verifiedThisEnable(operation);
src/lib/server/integrations/komodo.ts
-832
View File
@@ -1,832 +0,0 @@
import { config, komodoConfigured } from '../config.js';
import { getServer } from '../fleet/servers.js';
import { nativeHealth, nativeInstance } from '../backends/native.js';
// Ports `komodo_restart` in push_scripts.sh:622. Three shapes depending on
// the tracked komodo_type: deployment | container | stack.
export type KomodoRestartResult =
| { ok: true }
| { ok: false; kind: 'unconfigured' | 'no-name' | 'http-error' | 'unreachable'; message: string };
async function komodoCall(path: string, body: unknown): Promise<Response> {
const url = `${config.komodo.address.replace(/\/+$/, '')}/${path}`;
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), config.httpTimeoutMs);
try {
return await fetch(url, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'X-Api-Key': config.komodo.apiKey,
'X-Api-Secret': config.komodo.apiSecret
},
body: JSON.stringify(body),
signal: controller.signal
});
} finally {
clearTimeout(timer);
}
}
export async function komodoRestart(serverName: string): Promise<KomodoRestartResult> {
if (!komodoConfigured()) {
return { ok: false, kind: 'unconfigured', message: 'komodo not configured' };
}
const row = getServer(serverName);
if (!row?.komodo_name) return { ok: false, kind: 'no-name', message: 'no komodo_name set for server' };
const type = row.komodo_type ?? 'deployment';
let path: string;
let body: Record<string, string>;
switch (type) {
case 'stack':
path = 'execute/RestartStack';
body = { stack: row.komodo_name };
break;
case 'container':
if (!row.komodo_server) return { ok: false, kind: 'no-name', message: 'container type needs komodo_server' };
path = 'execute/RestartContainer';
body = { server: row.komodo_server, container: row.komodo_name };
break;
case 'deployment':
default:
path = 'execute/RestartDeployment';
body = { deployment: row.komodo_name };
break;
}
const url = `${config.komodo.address.replace(/\/+$/, '')}/${path}`;
let res: Response;
try {
res = await komodoCall(path, body);
} catch (err) {
return { ok: false, kind: 'unreachable', message: describeFetchError(err, url) };
}
if (!res.ok) {
return { ok: false, kind: 'http-error', message: `HTTP ${res.status}` };
}
// The script also checks the response body: if it's an object with
// .success === false, treat as failure. If .success is absent, trust the
// HTTP status (line 646).
try {
const json = await res.json();
if (json && typeof json === 'object' && 'success' in json && json.success === false) {
return { ok: false, kind: 'http-error', message: 'komodo returned success=false' };
}
} catch {
// Non-JSON body is fine — HTTP 2xx already indicates success.
}
return { ok: true };
}
// ---------- Port resolution ----------
export type KomodoPortFailure =
| 'unconfigured'
| 'no-name'
| 'unreachable' // network / DNS / timeout
| 'http-error' // non-2xx
| 'unparseable' // 2xx but body wasn't JSON
| 'no-udp-in-response'; // parsed but no UDP port found
export interface KomodoPortSuccess {
ok: true;
port: number;
snippet: string;
url: string;
}
export interface KomodoPortFailureResult {
ok: false;
reason: KomodoPortFailure;
detail: string;
httpStatus?: number;
snippet?: string;
// URL we tried (or would have tried) — surfaced in the UI so operators
// can `curl` it from the container to verify reachability.
url?: string;
}
export type KomodoPortResult = KomodoPortSuccess | KomodoPortFailureResult;
const SNIPPET_LIMIT = 4000;
function truncate(s: string): string {
if (s.length <= SNIPPET_LIMIT) return s;
return s.slice(0, SNIPPET_LIMIT) + `\n… (${s.length - SNIPPET_LIMIT} more bytes)`;
}
/**
* Turn a Node/Undici fetch failure into something an operator can act on.
* The default `.message` for an aborted fetch is just "This operation was
* aborted" — useless. For network failures Node hides the real cause under
* `.cause` (e.g. `ECONNREFUSED`, `ENOTFOUND`, `EHOSTUNREACH`). Unwrap both
* and always include the URL we tried.
*/
function describeFetchError(err: unknown, url: string): string {
const e = err as (Error & { name?: string; cause?: unknown }) | undefined;
if (e?.name === 'AbortError') {
return `timed out after ${config.httpTimeoutMs}ms dialing ${url}`;
}
const cause = e?.cause as (Error & { code?: string }) | undefined;
if (cause && (cause.code || cause.message)) {
const code = cause.code ? `${cause.code}: ` : '';
return `${code}${cause.message ?? 'network error'} (dialing ${url})`;
}
return `${e?.message ?? String(err)} (dialing ${url})`;
}
function safeJsonStringify(value: unknown): string {
try {
return JSON.stringify(value, null, 2);
} catch {
return String(value);
}
}
/**
* Detailed variant of `komodoResolveGamePort` — returns a discriminated
* union so callers can distinguish "network died" from "auth failed" from
* "we parsed the JSON but couldn't find a UDP port in a shape we recognize".
*
* On success, the response snippet is included so the caller can render it
* as "here's where we found the port". On no-udp-in-response, the snippet
* is the only way we'll know what shape Komodo actually emitted — surface
* it prominently in the UI so we can extend `extractUdpPort` for that shape.
*/
export async function komodoResolvePortDetailed(serverName: string): Promise<KomodoPortResult> {
if (!komodoConfigured()) {
return { ok: false, reason: 'unconfigured', detail: 'KOMODO_ADDRESS/KEY/SECRET are not all set' };
}
const row = getServer(serverName);
if (!row?.komodo_name) {
return { ok: false, reason: 'no-name', detail: `server has no komodo_name: ${serverName}` };
}
const type = row.komodo_type ?? 'deployment';
let path: string;
let body: Record<string, string>;
switch (type) {
case 'stack':
path = 'read/GetStack';
body = { stack: row.komodo_name };
break;
case 'container':
if (!row.komodo_server) {
return { ok: false, reason: 'no-name', detail: 'container type needs komodo_server' };
}
path = 'read/InspectContainer';
body = { server: row.komodo_server, container: row.komodo_name };
break;
case 'deployment':
default:
path = 'read/GetDeployment';
body = { deployment: row.komodo_name };
break;
}
const url = `${config.komodo.address.replace(/\/+$/, '')}/${path}`;
let res: Response;
try {
res = await komodoCall(path, body);
} catch (err) {
return { ok: false, reason: 'unreachable', detail: describeFetchError(err, url), url };
}
// Read the body up front — we want the snippet even on non-2xx so the
// UI can show what the error page actually said.
const rawText = await res.text().catch(() => '');
if (!res.ok) {
return {
ok: false,
reason: 'http-error',
detail: `HTTP ${res.status} ${res.statusText || ''}`.trim(),
httpStatus: res.status,
snippet: truncate(rawText),
url
};
}
let json: unknown;
try {
json = JSON.parse(rawText);
} catch (err) {
return {
ok: false,
reason: 'unparseable',
detail: `response body is not JSON: ${(err as Error).message}`,
httpStatus: res.status,
snippet: truncate(rawText),
url
};
}
const port = extractUdpPort(json);
const snippet = truncate(safeJsonStringify(json));
if (port === null) {
return {
ok: false,
reason: 'no-udp-in-response',
detail: 'parsed the response but no UDP port matched any known shape',
httpStatus: res.status,
snippet,
url
};
}
return { ok: true, port, snippet, url };
}
/**
* Query Komodo's read API for a server's UDP game port. Called lazily by
* the RCON client and cached into `servers.game_port` on success. Thin shim
* over `komodoResolvePortDetailed` that swallows diagnostics — use the
* detailed variant when you care why it failed.
*/
export async function komodoResolveGamePort(serverName: string): Promise<number | null> {
const result = await komodoResolvePortDetailed(serverName);
return result.ok ? result.port : null;
}
// ---------- Health probe ----------
export type KomodoHealthResult =
| { ok: true; state: string }
| { ok: false; reason: string; state?: string; snippet?: string };
// Komodo's container summary has { state: "running", container: { status:
// "Up 35 seconds (health: starting)" } }. Inspect responses instead have
// State.Health.Status (or snake/lowercase equivalents). Runtime health wins
// over a coarse running flag. Unknown/configuration strings are not readiness.
export function extractContainerState(payload: unknown): string | null {
const seen = new Set<unknown>();
const known = new Set(['running','healthy','starting','restarting','unhealthy','stopped','exited','created','dead','paused','removing','notdeployed','unknown']);
const stateOf = (value: unknown): string | null => {
if (typeof value !== 'string') return null;
const state = value.toLowerCase().trim();
if (known.has(state)) return state;
if (/^up\b/.test(state)) {
const health = state.match(/\((?:health:\s*)?(starting|unhealthy|healthy)\)/)?.[1];
return health && health !== 'healthy' ? health : 'running';
}
if (/^exited\b/.test(state)) return 'exited';
if (/^restarting\b/.test(state)) return 'restarting';
return null;
};
const walk = (value: unknown): string | null => {
if (!value || typeof value !== 'object' || seen.has(value)) return null;
seen.add(value);
if (Array.isArray(value)) {
const states = value.map(walk).filter((state): state is string => !!state);
return states.find(state => !['running','healthy'].includes(state)) ?? states[0] ?? null;
}
const object = value as Record<string, unknown>;
const rawState = object.State ?? object.state;
const state = stateOf(rawState) ?? stateOf(object.container_state ?? object.containerState ?? object.deployment_state ?? object.deploymentState);
// A stopped process is never ready just because its previous health check passed.
if (state && !['running','healthy','unknown'].includes(state)) return state;
if (rawState && typeof rawState === 'object') {
const nested = walk(rawState);
if (nested) return nested;
}
for (const key of ['container','Container','containers','Containers']) {
if (object[key] === null && state === 'running') return 'unknown';
const nested = walk(object[key]);
if (nested) return nested;
}
const status = stateOf(object.Status ?? object.status ?? object.docker_status);
if (status && !['running','healthy'].includes(status)) return status;
const health = object.Health ?? object.health;
const healthState = stateOf(typeof health === 'object' && health ?
(health as Record<string, unknown>).Status ?? (health as Record<string, unknown>).status : health);
if (healthState && healthState !== 'healthy') return healthState;
if (state || status) return state ?? status;
if (object.Running === true || object.running === true) return healthState ?? 'running';
for (const [key, item] of Object.entries(object)) {
if (['config','Config','labels','Labels'].includes(key)) continue;
const nested = walk(item);
if (nested) return nested;
}
return null;
};
return walk(payload);
}
const HEALTHY_STATES = new Set(['running', 'healthy', 'ok', 'started', 'up']);
async function fetchJsonWithSnippet(
path: string,
body: Record<string, string>
): Promise<{ ok: true; json: unknown; snippet: string } | { ok: false; reason: string; snippet?: string; unsupportedEndpoint?: boolean }> {
const url = `${config.komodo.address.replace(/\/+$/, '')}/${path}`;
let res: Response;
try {
res = await komodoCall(path, body);
} catch (err) {
return { ok: false, reason: describeFetchError(err, url) };
}
const text = await res.text().catch(() => '');
if (!res.ok) {
const unsupportedEndpoint = res.status === 404 || (
[400, 405, 422, 500, 501].includes(res.status)
&& /\b(?:unknown|unsupported|unrecognized)\s+(?:method|endpoint|variant)\b|\bmethod not found\b/i.test(text)
);
return { ok: false, reason: `HTTP ${res.status}`, snippet: truncate(text), unsupportedEndpoint };
}
let json: unknown;
try {
json = JSON.parse(text);
} catch (err) {
return { ok: false, reason: `unparseable json: ${(err as Error).message}`, snippet: truncate(text) };
}
return { ok: true, json, snippet: truncate(safeJsonStringify(json)) };
}
/**
* Ask Komodo whether the server's container is currently running. Used by the
* sweep during the health phase (post-restart) and by the beta runner. For
* deployment-type servers we hit the container-inspect endpoint first because
* GetDeployment returns config, not runtime state — falling back only when
* the container endpoint isn't available.
*/
export async function komodoInspectHealthy(serverName: string): Promise<KomodoHealthResult> {
if (getServer(serverName)?.native_instance_id) return nativeHealth(serverName);
if (!komodoConfigured()) return { ok: false, reason: 'komodo not configured' };
const row = getServer(serverName);
if (!row?.komodo_name) return { ok: false, reason: 'no komodo_name for server' };
const type = row.komodo_type ?? 'deployment';
const attempts: Array<{ path: string; body: Record<string, string> }> = [];
switch (type) {
case 'stack':
attempts.push({ path: 'read/GetStack', body: { stack: row.komodo_name } });
break;
case 'container':
if (!row.komodo_server) return { ok: false, reason: 'container type needs komodo_server' };
attempts.push({
path: 'read/InspectContainer',
body: { server: row.komodo_server, container: row.komodo_name }
});
break;
case 'deployment':
default:
// Container-specific endpoints first — they return docker-inspect
// shape with State.Status. GetDeployment is the last-resort
// config-only fallback.
attempts.push({ path: 'read/GetDeploymentContainer', body: { deployment: row.komodo_name } });
attempts.push({ path: 'read/InspectDeploymentContainer', body: { deployment: row.komodo_name } });
attempts.push({ path: 'read/GetDeploymentStatus', body: { deployment: row.komodo_name } });
attempts.push({ path: 'read/GetDeployment', body: { deployment: row.komodo_name } });
break;
}
let lastSnippet: string | undefined;
for (const { path, body } of attempts) {
const r = await fetchJsonWithSnippet(path, body);
if (!r.ok) {
// Alternate endpoints handle API-version differences. Retrying an
// outage against every endpoint delays all other server probes.
if (!r.unsupportedEndpoint) return { ok: false, reason: r.reason, snippet: r.snippet };
lastSnippet = r.snippet ?? lastSnippet;
continue;
}
const state = extractContainerState(r.json);
if (state) {
const healthy = HEALTHY_STATES.has(state.toLowerCase());
if (!healthy) return { ok: false, state, reason: `container state: ${state}`, snippet: r.snippet };
return { ok: true, state };
}
lastSnippet = r.snippet;
}
return {
ok: false,
reason: 'no container state found in any known Komodo response shape',
snippet: lastSnippet
};
}
// ---------- Write / Execute (create + destroy deployments) ----------
export type KomodoWriteResult =
| { ok: true }
| { ok: false; kind: 'unconfigured' | 'http-error' | 'unreachable'; message: string; url?: string; httpStatus?: number };
/**
* Env var pairs get flattened into a single newline-delimited string, matching
* Komodo's DeploymentConfig.environment shape.
*/
export interface DeploymentEnvVar {
variable: string;
value: string;
}
export interface CreateDeploymentSpec {
name: string; // Komodo deployment name (== EC server name)
server: string; // Komodo server (host) id or name
image: string; // 'ghcr.io/ayymoss/plutainer:latest'
network?: string; // 'komodo_default'
ports: string[]; // ['28960:28960/udp', '28960:28960/tcp']
volumes: string[]; // ['/host/path:/container/path', 'namedvol:/container/path:ro']
environment: DeploymentEnvVar[];
}
async function postAndClassify(path: string, body: unknown): Promise<KomodoWriteResult> {
const url = `${config.komodo.address.replace(/\/+$/, '')}/${path}`;
let res: Response;
try {
res = await komodoCall(path, body);
} catch (err) {
return { ok: false, kind: 'unreachable', message: describeFetchError(err, url), url };
}
if (!res.ok) {
const snippet = await res.text().catch(() => '');
return {
ok: false,
kind: 'http-error',
httpStatus: res.status,
message: `HTTP ${res.status}${snippet ? ` — ${truncate(snippet)}` : ''}`,
url
};
}
// Some Komodo endpoints return `{ success: false }` with a 200. Treat that
// as a failure so callers can surface the real problem.
try {
const json = await res.json();
if (json && typeof json === 'object' && 'success' in json && json.success === false) {
const msg = typeof (json as { message?: unknown }).message === 'string'
? (json as { message: string }).message
: 'komodo returned success=false';
return { ok: false, kind: 'http-error', message: msg, url, httpStatus: res.status };
}
} catch {
// Non-JSON body — trust the 2xx.
}
return { ok: true };
}
/**
* Create a new deployment entity in Komodo. Doesn't start the container —
* caller must follow up with `komodoDeploy(name)` to actually run it.
*
* The config shape here targets Komodo's DeploymentConfig: newline-delimited
* ports/volumes strings and an environment array with `{ variable, value }`
* entries. If Komodo rejects the payload, the error surfaces in the http-error
* message so we can iterate on the shape without guessing.
*/
export async function komodoCreateDeployment(spec: CreateDeploymentSpec): Promise<KomodoWriteResult> {
if (!komodoConfigured()) {
return { ok: false, kind: 'unconfigured', message: 'komodo not configured' };
}
const body = {
name: spec.name,
config: {
server_id: spec.server,
image: { type: 'Image', params: { image: spec.image } },
network: spec.network ?? 'komodo_default',
ports: spec.ports.join('\n'),
volumes: spec.volumes.join('\n'),
environment: spec.environment,
restart: 'unless-stopped'
}
};
return postAndClassify('write/CreateDeployment', body);
}
/** Start (or restart) the container backing a Komodo deployment. */
export async function komodoDeploy(deployment: string): Promise<KomodoWriteResult> {
if (!komodoConfigured()) {
return { ok: false, kind: 'unconfigured', message: 'komodo not configured' };
}
return postAndClassify('execute/Deploy', { deployment });
}
/** Runtime controls use the saved Komodo resource, never the EC display name. */
export async function komodoRuntimeAction(server: string, action: 'start' | 'stop'): Promise<KomodoRestartResult> {
if (!komodoConfigured()) return { ok: false, kind: 'unconfigured', message: 'komodo not configured' };
const row = getServer(server);
if (!row?.komodo_name) return { ok: false, kind: 'no-name', message: 'no komodo_name set for server' };
const type = row.komodo_type ?? 'deployment';
const verb = action === 'start' ? 'Start' : 'Stop';
if (type === 'container') {
if (!row.komodo_server) return { ok: false, kind: 'no-name', message: 'container type needs komodo_server' };
return postAndClassify(`execute/${verb}Container`, { server: row.komodo_server, container: row.komodo_name });
}
if (type === 'stack') return postAndClassify(`execute/${verb}Stack`, { stack: row.komodo_name });
return postAndClassify(`execute/${verb}Deployment`, { deployment: row.komodo_name });
}
/** Stop + remove the container backing a Komodo deployment. Entity remains. */
export async function komodoDestroyDeployment(deployment: string): Promise<KomodoWriteResult> {
if (!komodoConfigured()) {
return { ok: false, kind: 'unconfigured', message: 'komodo not configured' };
}
return postAndClassify('execute/DestroyDeployment', { deployment });
}
/** Remove the deployment entity from Komodo entirely. */
export async function komodoDeleteDeployment(deployment: string): Promise<KomodoWriteResult> {
if (!komodoConfigured()) {
return { ok: false, kind: 'unconfigured', message: 'komodo not configured' };
}
// Komodo's DeleteDeployment historically accepts `{ id }` where id may be
// the name; supply both to be forgiving.
return postAndClassify('write/DeleteDeployment', { id: deployment, name: deployment });
}
// ---------- Tags ----------
export type KomodoTagResult =
| { ok: true; id: string }
| { ok: false; message: string };
// Accept API string IDs and raw Mongo Extended JSON: { _id: { $oid: "..." } }.
function extractTagId(t: unknown): string | null {
if (!t || typeof t !== 'object') return null;
const o = t as { id?: unknown; _id?: unknown };
if (typeof o.id === 'string' && o.id.length > 0) return o.id;
if (typeof o._id === 'string' && o._id.length > 0) return o._id;
if (o._id && typeof o._id === 'object') {
const oid = (o._id as { $oid?: unknown }).$oid;
if (typeof oid === 'string' && oid.length > 0) return oid;
}
return null;
}
// ListTags response shape varies by Komodo version: bare array `[...]`, or
// wrapped under one of a few common keys. Peel back the common wrappers.
function unwrapTagList(body: unknown): unknown[] {
if (Array.isArray(body)) return body;
if (body && typeof body === 'object') {
for (const key of ['tags', 'data', 'results', 'items']) {
const val = (body as Record<string, unknown>)[key];
if (Array.isArray(val)) return val;
}
}
return [];
}
/**
* Fetch the tag list from Komodo. Two request styles are attempted in order:
*
* 1. `POST /read/ListTags` with an empty body — the per-endpoint style
* already used by the rest of this file (see execute/RestartDeployment).
* 2. `POST /read` with `{type: "ListTags", params: {}}` — the tagged-union
* style used by mbecker20/komodo's canonical API. If your Komodo build
* wires up only the tagged-union path, style #1 returns something we
* can't parse (empty array or non-tag payload) and we fall through.
*
* The winning parsed list is returned. The raw response bodies get logged
* only when both attempts fail to yield a match, so a fresh Komodo API shape
* is diagnosable without spamming stderr on the happy path.
*/
async function fetchTagsWithFallback(): Promise<{ tags: unknown[]; rawBodies: string[] }> {
const attempts: Array<{ path: string; body: unknown }> = [
{ path: 'read/ListTags', body: {} },
{ path: 'read', body: { type: 'ListTags', params: {} } }
];
const rawBodies: string[] = [];
for (const attempt of attempts) {
let res: Response;
try {
res = await komodoCall(attempt.path, attempt.body);
} catch (err) {
rawBodies.push(`${attempt.path}: network error — ${(err as Error).message}`);
continue;
}
if (!res.ok) {
const snippet = await res.text().catch(() => '');
rawBodies.push(`${attempt.path}: HTTP ${res.status}${snippet ? ` — ${truncate(snippet)}` : ''}`);
continue;
}
const raw = await res.text().catch(() => '');
rawBodies.push(`${attempt.path}: ${truncate(raw)}`);
let parsed: unknown;
try {
parsed = JSON.parse(raw);
} catch {
continue;
}
const tags = unwrapTagList(parsed);
if (tags.length > 0) return { tags, rawBodies };
}
return { tags: [], rawBodies };
}
async function findTagByName(name: string): Promise<KomodoTagResult | null> {
const { tags, rawBodies } = await fetchTagsWithFallback();
for (const t of tags) {
if (!t || typeof t !== 'object') continue;
const tagName = (t as { name?: unknown }).name;
if (typeof tagName !== 'string' || tagName !== name) continue;
const id = extractTagId(t);
if (id) return { ok: true, id };
}
// Miss — dump the raw bodies so the next iteration knows the real shape.
console.warn(
`[komodoEnsureTag] no matchable tag "${name}" in any ListTags variant. Bodies:\n ` +
rawBodies.join('\n ')
);
return null;
}
/**
* Resolve a tag by name, creating it if it doesn't exist. Returns the tag's
* Komodo id (Mongo ObjectId string) for use in UpdateResourceMeta.
*
* Best-effort: callers should treat failure as "skip tagging", not "roll back
* the whole operation". Tags are cosmetic organization inside Komodo.
*/
export async function komodoEnsureTag(name: string): Promise<KomodoTagResult> {
if (!komodoConfigured()) return { ok: false, message: 'komodo not configured' };
// 1. Look for an existing tag by name.
const existing = await findTagByName(name);
if (existing) return existing;
// 2. Not found — create it.
let createRes: Response;
try {
createRes = await komodoCall('write/CreateTag', { name });
} catch (err) {
return { ok: false, message: `CreateTag ${name}: ${(err as Error).message}` };
}
if (createRes.ok) {
const body = await createRes.json().catch(() => null);
const id = extractTagId(body);
if (id) return { ok: true, id };
// 200 but shape we don't recognize — fall back to re-list rather than
// fail, since the tag definitely exists now.
const fresh = await findTagByName(name);
if (fresh) return fresh;
console.warn(`[komodoEnsureTag] CreateTag ${name}: unexpected 2xx body, re-list also missed`, body);
return { ok: false, message: `CreateTag ${name}: unexpected response shape` };
}
// Non-2xx. Duplicate-key 500 means the tag exists under a different
// ListTags projection than we probed — re-list and try to find it. If
// that still misses, surface the create error.
const snippet = await createRes.text().catch(() => '');
const looksLikeDuplicate =
createRes.status === 500 &&
(/duplicate key/i.test(snippet) || /E11000/i.test(snippet) || /dup key/i.test(snippet));
if (looksLikeDuplicate) {
const fresh = await findTagByName(name);
if (fresh) return fresh;
return {
ok: false,
message: `CreateTag ${name}: duplicate reported but re-list missed it. Body: ${truncate(snippet)}`
};
}
return {
ok: false,
message: `CreateTag ${name}: HTTP ${createRes.status}${snippet ? ` — ${truncate(snippet)}` : ''}`
};
}
/**
* Attach tags to a Komodo resource. `tagIds` is the array of tag Mongo IDs
* from `komodoEnsureTag`. Replaces the resource's existing tag set — pass the
* full desired list, not a delta.
*
* Komodo's target.id accepts either the true id or the resource name for
* Deployments, so we pass the name here to save a lookup.
*/
export async function komodoUpdateTagsOnResource(
targetType: 'Deployment' | 'Stack' | 'Server' | 'Repo' | 'Build' | 'Procedure',
targetName: string,
tagIds: string[]
): Promise<KomodoWriteResult> {
if (!komodoConfigured()) {
return { ok: false, kind: 'unconfigured', message: 'komodo not configured' };
}
// UpdateResourceMeta owns tags; omit the other optional metadata fields
// so the deployment's description and template flag remain unchanged.
return postAndClassify('write/UpdateResourceMeta', {
target: { type: targetType, id: targetName },
tags: tagIds
});
}
// ---------- Extractor ----------
// Property names Komodo/Docker use for the collection of port mappings.
const PORT_ARRAY_KEYS = new Set(['ports', 'Ports', 'port_bindings', 'PortBindings']);
// Value field names for the host-facing port on an object entry.
const HOST_PORT_KEYS = ['published', 'PublishedPort', 'public_port', 'PublicPort', 'HostPort', 'hostPort', 'host_port'];
interface PortsCandidate {
arr: unknown[];
/** True when the array was reached via a `ports`-flavored key — preferred. */
preferred: boolean;
}
/**
* Walks a Komodo response looking for a UDP port. Handles the shapes we've
* seen in the wild:
*
* docker-compose object: [{ protocol: "udp", published: 4976, target: 28960 }]
* docker-compose caps: [{ Protocol: "udp", PublishedPort: 4976 }]
* docker-run string: ["4976:28960/udp", "3074:3074/tcp"]
* host-network string: ["28962/udp"]
* docker-inspect object: { "28960/udp": [{ "HostIp": "0.0.0.0", "HostPort": "28962" }] }
*
* Arrays reached via a `ports`/`Ports`/`port_bindings`/`PortBindings` key
* are preferred over arrays discovered elsewhere in the tree, so an
* unrelated array (e.g. tags, labels) can't shadow a real port list.
*/
export function extractUdpPort(payload: unknown): number | null {
const candidates: PortsCandidate[] = [];
// Objects whose keys encode `<container-port>/<proto>` — a la docker inspect.
const containerKeyed: Record<string, unknown>[] = [];
const seen = new Set<unknown>();
const walk = (v: unknown, viaPortsKey: boolean): void => {
if (v === null || typeof v !== 'object' || seen.has(v)) return;
seen.add(v);
if (Array.isArray(v)) {
for (const item of v) walk(item, false);
candidates.push({ arr: v, preferred: viaPortsKey });
return;
}
const obj = v as Record<string, unknown>;
// Detect docker-inspect container-keyed objects: any key like `28960/udp`.
for (const k of Object.keys(obj)) {
if (/^\d+\/(udp|tcp)$/i.test(k)) {
containerKeyed.push(obj);
break;
}
}
for (const [key, val] of Object.entries(obj)) {
// Komodo's GetDeployment renders `config.ports` as a single
// newline-delimited string ("28963:28963/udp\n28963:28963/tcp").
// Synthesize an array from it and treat as a preferred candidate.
if (PORT_ARRAY_KEYS.has(key) && typeof val === 'string' && val.trim()) {
const parts = val
.split(/[\r\n,;]+/)
.map((s) => s.trim())
.filter(Boolean);
if (parts.length > 0) {
candidates.push({ arr: parts, preferred: true });
}
}
walk(val, PORT_ARRAY_KEYS.has(key));
}
};
walk(payload, false);
// Sort so preferred `ports`-keyed arrays win over accidental hits.
candidates.sort((a, b) => (a.preferred === b.preferred ? 0 : a.preferred ? -1 : 1));
// Try object-entry shape first — most Komodo/Docker responses use this.
for (const { arr } of candidates) {
for (const entry of arr) {
if (entry && typeof entry === 'object' && !Array.isArray(entry)) {
const e = entry as Record<string, unknown>;
const proto = String(e.protocol ?? e.Protocol ?? e.type ?? e.Type ?? '').toLowerCase();
if (proto !== 'udp') continue;
for (const field of HOST_PORT_KEYS) {
const raw = e[field];
if (raw === undefined || raw === null) continue;
const num = typeof raw === 'string' ? Number.parseInt(raw, 10) : Number(raw);
if (Number.isFinite(num) && num > 0) return num;
}
}
}
}
// Then string shapes: "host:container/udp" or "container/udp" (host-net).
for (const { arr } of candidates) {
for (const entry of arr) {
if (typeof entry !== 'string') continue;
const hostMapped = /^(\d+):\d+\/udp$/i.exec(entry);
if (hostMapped) return Number.parseInt(hostMapped[1], 10);
const hostNet = /^(\d+)\/udp$/i.exec(entry);
if (hostNet) return Number.parseInt(hostNet[1], 10);
}
}
// Finally, docker-inspect container-keyed shape:
// { "28960/udp": [{ HostIp, HostPort }] }
for (const obj of containerKeyed) {
for (const [key, val] of Object.entries(obj)) {
if (!/^\d+\/udp$/i.test(key)) continue;
if (!Array.isArray(val)) continue;
for (const binding of val) {
if (!binding || typeof binding !== 'object') continue;
const b = binding as Record<string, unknown>;
for (const field of HOST_PORT_KEYS) {
const raw = b[field];
if (raw === undefined || raw === null) continue;
const num = typeof raw === 'string' ? Number.parseInt(raw, 10) : Number(raw);
if (Number.isFinite(num) && num > 0) return num;
}
}
}
}
return null;
}
src/lib/server/integrations/netProbe.ts
-93
View File
@@ -1,93 +0,0 @@
import { createConnection } from 'node:net';
import { lookup } from 'node:dns/promises';
export type TcpProbeOutcome =
| 'connected'
| 'refused'
| 'timeout'
| 'dns-fail'
| 'unreachable'
| 'error';
export interface TcpProbeResult {
label: string;
host: string;
port: number;
resolvedIp: string | null;
outcome: TcpProbeOutcome;
elapsedMs: number;
detail?: string;
}
const DEFAULT_TIMEOUT_MS = 3000;
/**
* Best-effort DNS resolution. Returns null on failure so the caller can
* distinguish "DNS failed" from "connect failed".
*/
async function resolve(host: string): Promise<{ ip: string | null; error: string | null }> {
try {
const r = await lookup(host);
return { ip: r.address, error: null };
} catch (err) {
const e = err as NodeJS.ErrnoException;
return { ip: null, error: `${e.code ?? 'UNKNOWN'}: ${e.message}` };
}
}
/**
* Attempt a raw TCP connect to `host:port` with an explicit timeout, and
* classify the result. Purely diagnostic — no bytes are sent after the
* handshake completes. Isolates network reachability from every HTTP-layer
* concern (auth, TLS, redirects, wrong path).
*/
export async function tcpProbe(
label: string,
host: string,
port: number,
timeoutMs: number = DEFAULT_TIMEOUT_MS
): Promise<TcpProbeResult> {
const started = Date.now();
const { ip, error: dnsError } = await resolve(host);
if (!ip) {
return {
label,
host,
port,
resolvedIp: null,
outcome: 'dns-fail',
elapsedMs: Date.now() - started,
detail: dnsError ?? 'DNS lookup returned no address'
};
}
return new Promise<TcpProbeResult>((resolve) => {
const sock = createConnection({ host: ip, port, timeout: timeoutMs });
let settled = false;
const finish = (outcome: TcpProbeOutcome, detail?: string): void => {
if (settled) return;
settled = true;
try { sock.destroy(); } catch { /* already destroyed */ }
resolve({ label, host, port, resolvedIp: ip, outcome, elapsedMs: Date.now() - started, detail });
};
sock.once('connect', () => finish('connected'));
sock.once('timeout', () => finish('timeout', `no SYN-ACK within ${timeoutMs}ms`));
sock.once('error', (err) => {
const code = (err as NodeJS.ErrnoException).code ?? '';
const msg = (err as Error).message ?? String(err);
if (code === 'ECONNREFUSED') return finish('refused', code);
if (code === 'EHOSTUNREACH' || code === 'ENETUNREACH') return finish('unreachable', `${code}: ${msg}`);
return finish('error', `${code || 'unknown'}: ${msg}`);
});
});
}
export async function tcpProbeAll(
trials: Array<{ label: string; host: string; port: number }>,
timeoutMs: number = DEFAULT_TIMEOUT_MS
): Promise<TcpProbeResult[]> {
// Run in parallel — they don't share sockets and the whole point is to
// finish fast even when some probes hit the timeout.
return Promise.all(trials.map((t) => tcpProbe(t.label, t.host, t.port, timeoutMs)));
}
src/lib/server/integrations/plutonium.ts
+2 -2
View File
@@ -5,7 +5,7 @@ import { hasActiveProvisioning } from '../fleet/provisioning.js';
import { latestGameSubmission } from '../fleet/status-evidence.js';
import { latestGameSubmissions } from '../fleet/lifecycle-evidence.js';
import { rconPlutoniumVersion } from './rcon.js';
import { komodoInspectHealthy } from './komodo.js';
import { nativeHealth } from '../backends/native.js';
import { restartRecoveryPending } from '../provisioning/index.js';
import { parsePlutoniumManifest, PLUTONIUM_MANIFEST_URL } from '../../plutonium.js';
import type { PlutoniumRelease, PlutoniumVersion } from '../../plutonium.js';
@@ -60,7 +60,7 @@ export async function probeVersion(server: string, options: { maintenance?: bool
const epoch = latestGameSubmission(server);
const recovering = recoveryBlocked();
const health = !recovering && original.desired_state !== 'stopped'
? await komodoInspectHealthy(server).catch(() => ({ ok: false })) : { ok: false };
? await nativeHealth(server).catch(() => ({ ok: false })) : { ok: false };
if (suppressed()) return null;
let result = !health.ok ? { revision: null, error: 'Waiting for a healthy running server.' }
: await rconPlutoniumVersion(server).catch(() => ({ revision: null, error: 'Version check failed.' }));
src/lib/server/integrations/rcon.ts
+7 -33
View File
@@ -1,9 +1,8 @@
import { createSocket } from 'node:dgram';
import type { Socket } from 'node:dgram';
import { config, komodoConfigured, rconConfigured } from '../config.js';
import { getServer, setServerField } from '../fleet/servers.js';
import { config, rconConfigured } from '../config.js';
import { getServer } from '../fleet/servers.js';
import { latestGameSubmission } from '../fleet/lifecycle-evidence.js';
import { komodoResolveGamePort } from './komodo.js';
import { parsePlutoniumRevision } from '../../plutonium.js';
import { nativeInstance, nativeRequest } from '../backends/native.js';
@@ -81,27 +80,15 @@ async function sendRcon(host: string, port: number, command: string): Promise<Se
interface EndpointResolution {
endpoint: RconEndpoint | null;
// Present when endpoint is null — tells the caller why we couldn't dial.
reason?: 'unknown-server' | 'no-port-no-komodo' | 'no-port-komodo-empty';
reason?: 'unknown-server' | 'no-port';
}
/**
* Resolve the RCON endpoint (host + port) for a server, filling
* `servers.game_port` from Komodo on first miss.
*/
/** Resolve a saved RCON endpoint without remote resource discovery. */
async function endpointFor(server: string): Promise<EndpointResolution> {
const row = getServer(server);
if (!row) return { endpoint: null, reason: 'unknown-server' };
let port = row.game_port;
if (!port) {
if (!komodoConfigured()) {
return { endpoint: null, reason: 'no-port-no-komodo' };
}
port = await komodoResolveGamePort(server);
if (port) setServerField(server, 'game_port', port);
}
if (!port) return { endpoint: null, reason: 'no-port-komodo-empty' };
return { endpoint: { host: row.rcon_host ?? config.rcon.host, port } };
if (!row.game_port) return { endpoint: null, reason: 'no-port' };
return { endpoint: { host: row.rcon_host ?? config.rcon.host, port: row.game_port } };
}
function stripPrefix(buf: Buffer): string | null {
@@ -421,20 +408,7 @@ export async function rconStatusResult(server: string): Promise<RconStatusResult
if (res.reason === 'unknown-server') {
return { ok: false, endpoint: null, reason: 'no-port', detail: `unknown server: ${server}` };
}
if (res.reason === 'no-port-no-komodo') {
return {
ok: false,
endpoint: null,
reason: 'no-port',
detail: 'no game_port set and Komodo is not configured'
};
}
return {
ok: false,
endpoint: null,
reason: 'no-port',
detail: 'no game_port — Komodo did not return one for this server'
};
return { ok: false, endpoint: null, reason: 'no-port', detail: 'No game port is registered for this server.' };
}
const ep = res.endpoint;
src/lib/server/maintenance/recovery.ts
+4 -5
View File
@@ -1,6 +1,5 @@
import { randomUUID } from 'node:crypto';
import { db } from '../db/index.js';
import { komodoConfigured } from '../config.js';
import { nativeInstance } from '../backends/native.js';
import { getServer, listServers } from '../fleet/servers.js';
import { hasActiveProvisioning } from '../fleet/provisioning.js';
@@ -100,7 +99,7 @@ export function resolveRecoveredHealthOperation(operationId: string, players: nu
db().prepare('UPDATE server_deployments SET checked_at = ?, players = ? WHERE id = ?').run(now, players, operationId);
recordProgress(operationId, { blocker: null, observedAt: now,
recovery: { state: 'released', at: now, operationId, message },
checks: [{ key: 'container', label: 'Container readiness', source: 'komodo', result: 'pass', checkedAt: now },
checks: [{ key: 'container', label: 'Container readiness', source: 'agent', result: 'pass', checkedAt: now },
{ key: 'game', label: 'Game status', source, result: 'pass', actual: players, checkedAt: now }] });
if (operation.state === 'awaiting-health') {
deploymentStage(operationId, message);
@@ -275,7 +274,7 @@ async function scanServer(server: string, forceScan: boolean): Promise<void> {
if (observation.identity !== monitoringIdentity(server)) return;
if (!scanSuppressed(server, 'health') && observation.healthy) {
const current = active(server, 'health');
if (current) transition(current.id, 'resolved', 'Fresh game status and Komodo health verified');
if (current) transition(current.id, 'resolved', 'Fresh game status and host agent health verified');
} else if (!scanSuppressed(server, 'health')) {
const problem = detect(server, 'health', observation.reason, observation);
const startup = ['starting','restarting'].includes(observation.containerState ?? '');
@@ -334,8 +333,8 @@ async function run(forceScan: boolean): Promise<void> {
const lastScan = setting('watchdog_last_scan');
if (!forceScan && lastScan && Date.now() - Date.parse(lastScan) < WATCHDOG_SCAN_MS) return;
setSetting('watchdog_last_scan', iso());
if (!komodoConfigured() && !listServers().some(server => nativeInstance(server.name))) {
setSetting('watchdog_scan_error', 'Configure Komodo before automatic recovery. RCON or fresh telemetry verifies game status.'); return;
if (!listServers().some(server => nativeInstance(server.name))) {
setSetting('watchdog_scan_error', 'Register a native host instance before automatic recovery. RCON or fresh telemetry verifies game status.'); return;
}
const errors: string[] = [];
const servers = listServers();
src/lib/server/pipeline/restart-rollout.ts
+1 -1
View File
@@ -1 +1 @@
export { restartSettling, rolloutWaitReason, refreshRestartRollout, rolloutHealth } from '../provisioning/restarts.js';
export { restartSettling, rolloutWaitReason, rolloutHealth } from '../provisioning/restarts.js';
src/lib/server/pipeline/worker.ts
+2 -2
View File
@@ -100,8 +100,8 @@ function queueStatus(srv: ServerRow, probe?: () => ReturnType<typeof rconForMoni
const server = srv.name;
const pending = pendingStatus.get(server);
if (pending) return pending;
// Legacy servers without a recorded host share the configured host's pool.
const host = srv.host_id ? `native:${srv.host_id}` : `komodo:${srv.komodo_server || config.komodo.targetServer}`;
// Host queues keep slow instances from consuming other hosts' slots.
const host = srv.host_id ? `native:${srv.host_id}` : `unregistered:${srv.name}`;
let queue = statusQueues.get(host);
if (!queue) { queue = { active: 0, queued: [] }; statusQueues.set(host, queue); }
let finish!: (capacity: number | null) => void;
src/lib/server/provisioning/history.ts
+5 -14
View File
@@ -121,7 +121,7 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
SELECT collection_id, deployment_id FROM server_collection_members
UNION ALL SELECT collection_id, deployment_id FROM server_collection_removals) m
JOIN server_launch_collections c ON c.id = m.collection_id WHERE m.deployment_id = ? ORDER BY c.created_at, c.id LIMIT 1`).get(id) as { id: string; name: string } | undefined;
return { deployment, backend: native ? 'native' as const : 'komodo' as const, agentOperation: agentOperation ?? null,
return { deployment, backend: 'native' as const, agentOperation: agentOperation ?? null,
canCancel: canCancelProvisioningJob(id), canCancelCreation: canCancelCreation(id), cancellation: cancellation ?? null,
workerActivity: workerActivity(), events: deploymentEvents(id, options.events === 'recent'),
activity: { ...activitySummary(id), failureExcerpt: latestFailureEvent(id) }, sources,
@@ -133,7 +133,7 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
})();
}
// This stage is written only after Komodo has confirmed deployment deletion.
// This stage is written only after the host agent has confirmed instance removal.
export const REMOVING_SERVER = 'Removing server files and registry';
export function canCancelRemoval(id: string): boolean {
@@ -154,15 +154,6 @@ export function cancelRemoval(id: string): boolean {
})();
}
export function claimRemoval(id: string): boolean {
return db().prepare(`UPDATE server_deployments SET state = 'running'
WHERE id = ? AND action = 'destroy' AND state = 'waiting'
AND (server_deleted_at IS NULL OR (stage = ?
AND NOT EXISTS (SELECT 1 FROM servers s WHERE s.name = server_deployments.server)
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = server_deployments.server
AND newer.rowid > server_deployments.rowid)))`).run(id, REMOVING_SERVER).changes > 0;
}
export function canResumeCleanup(id: string): boolean {
return !!db().prepare(`SELECT d.id FROM server_deployments d
WHERE d.id = ? AND (d.server_deleted_at IS NULL OR NOT EXISTS (SELECT 1 FROM servers s WHERE s.name = d.server))
@@ -177,7 +168,7 @@ export function resumeCleanup(id: string): boolean {
preserveOutcome(id);
db().prepare("UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?").run(id);
recordProgress(id, { phase: 'cleanup', blocker: null, checks: [], observedAt: null,
event: { kind: 'resumed', message: 'Resumed local cleanup after confirmed Komodo deletion' } });
event: { kind: 'resumed', message: 'Resumed local cleanup after confirmed instance removal' } });
return true;
})();
}
@@ -207,7 +198,7 @@ export function resumeRebuild(id: string): boolean {
preserveOutcome(id);
db().prepare("UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?").run(id);
recordProgress(id, { phase: 'cleanup', blocker: null, checks: [], observedAt: null,
event: { kind: 'resumed', message: 'Resumed rebuild cleanup after confirmed Komodo deletion' } });
event: { kind: 'resumed', message: 'Resumed rebuild cleanup after confirmed instance removal' } });
return true;
})();
}
@@ -218,7 +209,7 @@ export function resumeVerification(id: string): boolean {
if (!canResumeVerification(id)) return false;
preserveOutcome(id);
db().prepare("UPDATE server_deployments SET state = 'awaiting-health', finished_at = NULL, error = NULL WHERE id = ?").run(id);
deploymentStage(id, 'Waiting for Komodo health', { phase: 'verify-container', blocker: null, checks: [], observedAt: null,
deploymentStage(id, 'Waiting for host agent health', { phase: 'verify-container', blocker: null, checks: [], observedAt: null,
event: { kind: 'resumed', message: 'Resumed verification of the existing server' } });
return true;
})();
src/lib/server/provisioning/index.ts
+1 -1
View File
@@ -2,7 +2,7 @@
// Execution is driven exclusively by the provisioning worker.
export { requestLifecycle, requestRestart, requestWatchdogLifecycle, escalateWatchdogRestart, provisioningOperation, cancelProvisioning, releaseRestartRequest } from './requests.js';
export type { LifecycleOperation, RestartSource } from './requests.js';
export { queueCreateServer, destroyServer, previewCreation, creationQueue, nextAvailablePort, nextLobbyNumber } from './instances.js';
export { queueCreateServer, destroyServer, previewCreation, creationQueue, nextLobbyNumber } from './instances.js';
export { getDeployment, listDeployments, resumeVerification, resumeCleanup, resumeRebuild, cancelRemoval } from './history.js';
export { deploymentActivity } from './activity.js';
export { restartSettling, restartRecoveryPending } from './restarts.js';
src/lib/server/provisioning/instances.ts
+9 -956
View File
File diff suppressed because it is too large Load Diff
src/lib/server/provisioning/native.ts
+3 -3
View File
@@ -45,7 +45,7 @@ export function nativeLaunchSelection(input: Record<string, unknown>, game: stri
if (input.libraryId !== undefined && input.libraryId !== '' && input.libraryId !== libraryId) {
throw new Error(`The current ${game.toUpperCase()} game files changed. Refresh the launch preview.`);
}
const image = input.image === undefined || input.image === '' ? config.komodo.defaultImage : input.image;
const image = input.image === undefined || input.image === '' ? config.defaultRuntimeImage : input.image;
if (typeof image !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9._/:@-]{0,499}$/.test(image)) throw new Error('Invalid runtime image.');
const automatic = input.automatic === true || !input.hostId;
if (!automatic) {
@@ -673,8 +673,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
const recipe = prepared.recipe;
db().prepare('UPDATE server_launch_recipes SET recipe=CAST(? AS TEXT),recipe_summary=? WHERE deployment_id=?')
.run(Buffer.from(prepared.recipeJson.buffer, prepared.recipeJson.byteOffset, prepared.recipeJson.byteLength), prepared.recipeSummary, job.id);
if (job.action === 'create') db().prepare(`INSERT INTO servers(name,komodo_type,komodo_name,game,map,game_mode,map_slug,host_dir,config_file,game_id,image,game_port,created_by,container_state)
VALUES(?,'deployment','',?,?,'zombie',?,?,?,?,?,?,'native','starting')`).run(job.server, recipe.game, recipe.expectedMap, recipe.mapSlug, serverDir(job.server),
if (job.action === 'create') db().prepare(`INSERT INTO servers(name,game,map,game_mode,map_slug,host_dir,config_file,game_id,image,game_port,created_by,container_state)
VALUES(?,?,?,'zombie',?,?,?,?,?,?,'native','starting')`).run(job.server, recipe.game, recipe.expectedMap, recipe.mapSlug, serverDir(job.server),
recipe.configFile, recipe.game === 't6' ? 't6zm' : 't5sp', recipe.image, instance!.port);
for (const file of recipe.files) recordScript(job.server, file.path, file.versionId, file.sha256, file.size, 'native-create');
recordProgress(job.id, { checkpoint: { key: 'prepare', label: 'Launch files selected and command payload prepared' } });
src/lib/server/provisioning/progress.ts
+1 -1
View File
@@ -37,7 +37,7 @@ export function readProgress(row: { observation_json?: string | null }): Provisi
|| (v.blocker.label !== undefined && !nullableString(v.blocker.label)))) return null;
if (v.checks.length > 10 || !v.checks.every(c => object(c) && ['container', 'game', 'map', 'plutonium-release', 'plutonium-version', 'occupancy', 'stopped'].includes(String(c.key))
&& typeof c.label === 'string' && ['pass', 'waiting', 'fail', 'unknown'].includes(String(c.result))
&& (c.source === null || ['komodo', 'agent', 'rcon', 'telemetry', 'combined', 'manifest'].includes(String(c.source)))
&& (c.source === null || ['agent', 'rcon', 'telemetry', 'combined', 'manifest'].includes(String(c.source)))
&& typeof c.checkedAt === 'string' && nullableString(c.receivedAt) && scalar(c.expected) && scalar(c.actual) && nullableString(c.message))) return null;
if (v.checkpoints.length > 100 || !v.checkpoints.every(c => object(c) && typeof c.key === 'string' && typeof c.label === 'string'
&& typeof c.at === 'string' && (c.phase === null || phase(c.phase)))) return null;
src/lib/server/provisioning/requests.ts
+4 -22
View File
@@ -1,10 +1,8 @@
import { maintenanceAllowed } from '../maintenance/policies.js';
import { db } from '../db/index.js';
import { komodoConfigured, rconConfigured } from '../config.js';
import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent } from './history.js';
import { queueRebuildServer } from './instances.js';
import { restartRequestInFlight } from './restarts.js';
import { runtimeActions, type RuntimeAction } from '../../provisioning.js';
import { recordProgress } from './progress.js';
import { nativeInstance, unplacedNativeCreation } from '../backends/native.js';
@@ -51,22 +49,6 @@ export function requestRestart(server: string, source: RestartSource): Lifecycle
.get(server, row.restarted_at, row.restarted_at) as { id: string } | undefined)?.id;
if (id) db().prepare('UPDATE server_deployments SET submitted_at = COALESCE(submitted_at, ?) WHERE id = ?').run(row.restarted_at, id);
}
// Only not-yet-submitted campaign changes can share an activation. A later
// file deployment must never be credited to an already running restart.
// Restorations need their own activation. An older campaign can have
// stale file preconditions after a beta; its failure must not strand
// the rollback that would restore the server.
const rollback = source.kind === 'campaign' && !!db().prepare("SELECT 1 FROM deployments WHERE id = ? AND action = 'rollback'").get(source.id);
if (source.kind === 'campaign' && !row.restarted_at && !rollback && !nativeInstance(server)) {
id = (db().prepare(`SELECT d.id FROM server_deployments d WHERE d.server = ? AND d.server_deleted_at IS NULL AND d.action = 'restart'
AND d.state = 'waiting' AND d.submitted_at IS NULL
AND NOT EXISTS (SELECT 1 FROM native_provisioning n WHERE n.deployment_id=d.id AND n.stage_operation_id IS NOT NULL)
AND EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'campaign')
AND NOT EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind != 'campaign')
AND NOT EXISTS (SELECT 1 FROM provisioning_sources s JOIN deployments c ON c.id = s.source_id
WHERE s.operation_id = d.id AND s.kind = 'campaign' AND c.action = 'rollback')
ORDER BY d.rowid LIMIT 1`).get(server) as { id: string } | undefined)?.id;
}
if (!id) {
id = beginDeployment(server, 'restart', `${source.kind}:${source.id}`);
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
@@ -138,7 +120,7 @@ export function releaseRestartRequest(server: string, source: RestartSource): vo
export function requestLifecycle(server: string, action: RuntimeAction) {
if (!runtimeActions.includes(action)) return { ok: false as const, status: 400, error: 'Choose start, restart, rebuild, or stop.' };
if (!nativeInstance(server) && !unplacedNativeCreation(server) && (!komodoConfigured() || !rconConfigured())) return { ok: false as const, status: 503, error: 'Configure a host and RCON before provisioning.' };
if (!nativeInstance(server) && !unplacedNativeCreation(server)) return { ok: false as const, status: 503, error: 'Server has no native host instance.' };
return db().transaction(() => {
const blocker = action === 'rebuild' ? activeBetaRemovalBlocker(server) : null;
if (blocker) return { ok: false as const, status: 409, error: blocker };
@@ -187,7 +169,7 @@ function watchdogRequestGuard(server: string, incidentId: string): Extract<Lifec
const row = getServer(server);
if (!row || row.destroyed_at) return { ok: false, status: 404, error: 'Server not found.' };
if (row.desired_state === 'stopped') return { ok: false, status: 409, error: 'This server is intentionally stopped.' };
if (!nativeInstance(server) && !komodoConfigured()) return { ok: false, status: 503, error: 'Configure a host before provisioning.' };
if (!nativeInstance(server)) return { ok: false, status: 503, error: 'Configure a host before provisioning.' };
return null;
}
@@ -237,7 +219,7 @@ export function requestWatchdogLifecycle(server: string, action: 'restart' | 're
JOIN watchdog_incidents i ON i.id = a.incident_id
WHERE r.server = ? AND r.healthy_at IS NULL AND d.state IN ('failed','interrupted','cancelled')
AND a.cycle < i.cycle`).get(incidentId, server) as { operation_id: string } | undefined;
if (retained && !restartRequestInFlight(server)) {
if (retained) {
const transferred = db().prepare('UPDATE restart_rollout SET operation_id = ? WHERE server = ? AND operation_id = ? AND healthy_at IS NULL')
.run(next.deploymentId, server, retained.operation_id);
if (transferred.changes) {
@@ -275,7 +257,7 @@ export function escalateWatchdogRestart(incidentId: string, operationId: string,
const slot = db().prepare('SELECT operation_id, requested_at, healthy_at FROM restart_rollout WHERE server = ?').get(operation.server) as
{ operation_id: string | null; requested_at: string; healthy_at: string | null } | undefined;
const anchor = Date.parse(slot?.requested_at ?? operation.submitted_at ?? operation.started_at);
if (restartRequestInFlight(operation.server) || (slot && (slot.operation_id !== operationId || slot.healthy_at))
if ((slot && (slot.operation_id !== operationId || slot.healthy_at))
|| !Number.isFinite(anchor) || Date.now() - anchor < Math.max(300_000, recoveryTimeoutMs)) {
return { ok: false as const, status: 409, error: 'Waiting for the submitted restart to finish its recovery window.' };
}
src/lib/server/provisioning/restarts.ts
+1 -174
View File
@@ -1,20 +1,9 @@
import { fleetSettings } from '../settings.js';
import { db } from '../db/index.js';
import { provisioningCancelled } from './cancellation.js';
import { komodoInspectHealthy, komodoRestart, komodoRuntimeAction } from '../integrations/komodo.js';
import { probeServerStatus } from '../fleet/status-evidence.js';
import { deploymentStage, finishDeployment } from './history.js';
import { applyStagedFilesForOperation } from '../fleet/staged-files.js';
import type { RuntimeAction } from '../../provisioning.js';
import type { ProvisioningBlocker } from '../../provisioning-detail.js';
import { recordProgress, type ProgressUpdate } from './progress.js';
import { nativeDeployment } from '../backends/native.js';
// Komodo queues restart work. Don't mistake the still-running old container
// for a completed restart when an API kick immediately follows the request.
const HEALTH_POLL_MS = 15_000;
const requestsInFlight = new Map<string, string>();
export const restartRequestInFlight = (server: string) => requestsInFlight.has(server);
interface RestartSlot {
server: string;
@@ -37,7 +26,7 @@ export function restartRecoveryPending(server: string): boolean {
/** The queued restart may still expose the previous healthy container. */
export function restartSettling(server: string): boolean {
const slot = slotFor(server);
return requestsInFlight.has(server) || !!(slot && !slot.healthy_at
return !!(slot && !slot.healthy_at
&& Date.now() - Date.parse(slot.requested_at) < HEALTH_POLL_MS);
}
@@ -46,7 +35,6 @@ export function rolloutWaitReason(server: string, operationId = '', action: Runt
}
export function rolloutBlocker(server: string, operationId = '', action: RuntimeAction = 'restart'): ProvisioningBlocker | null {
if (requestsInFlight.has(server)) return { code: 'operation-conflict', message: 'Waiting for the previous command submission to finish', automatic: true };
const conflict = db().prepare(`SELECT id FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL
AND ((action IN ('start','stop','rebuild') AND state IN ('running','awaiting-health'))
OR (action = 'destroy' AND state IN ('waiting','running')))`).get(server, operationId) as { id: string } | undefined;
@@ -67,167 +55,6 @@ export function rolloutBlocker(server: string, operationId = '', action: Runtime
? { code: 'restart-capacity', message: `Queued to ${action}; waiting for another server to recover`, automatic: true } : null;
}
export type RolloutRestartResult =
| { kind: 'started' }
| { kind: 'queued'; message: string }
| { kind: 'failed'; message: string };
/** Reserve capacity before any async I/O; callers still check for players first. */
export async function restartInRollout(server: string, options: { operationId: string; action: 'start' | 'restart' }): Promise<RolloutRestartResult> {
const operationId = options.operationId;
const waitReason = db().transaction(() => {
if (provisioningCancelled(operationId)) return 'Operation was cancelled';
if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return 'Original server was deleted';
const reason = rolloutWaitReason(server, operationId, options.action);
if (reason) return reason;
db().prepare(`INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)
ON CONFLICT(server) DO UPDATE SET requested_at = excluded.requested_at,
checked_at = NULL, healthy_at = NULL, players = NULL, error = NULL, operation_id = excluded.operation_id`)
.run(server, new Date().toISOString(), operationId);
recordProgress(operationId, { blocker: null, recovery: { state: 'held', message: 'Restart recovery capacity reserved.', operationId } });
return null;
}).immediate();
if (waitReason) return { kind: 'queued', message: waitReason };
// Every gate, including rollout capacity, has cleared. No async work may
// intervene between applying these files and submitting the restart.
try {
applyStagedFilesForOperation(operationId, server);
} catch (err) {
const message = `File activation failed: ${(err as Error).message}`;
db().transaction(() => {
db().prepare('DELETE FROM restart_rollout WHERE server = ? AND operation_id = ?').run(server, operationId);
recordProgress(operationId, { recovery: { state: 'released', message: 'Capacity released before any remote command: file activation failed.', operationId } });
finishDeployment(operationId, { ok: false, message });
})();
return { kind: 'failed', message };
}
db().transaction(() => {
db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), operationId);
deploymentStage(operationId, options.action === 'start' ? 'Starting container' : 'Restarting container', {
phase: 'container-action', blocker: null, checks: [], checkpoint: { key: 'eligibility', label: 'Eligibility checks passed' },
command: { state: 'attempted', message: `${options.action === 'start' ? 'Start' : 'Restart'} submission attempted.` }
});
})();
requestsInFlight.set(server, operationId);
db().prepare(`UPDATE servers SET container_state = 'restarting', last_error = NULL, rcon_failures = 0, last_reply_snippet = NULL,
game_started_at = NULL, game_ended_at = NULL WHERE name = ?`).run(server);
try {
const result = options.action === 'start' ? await komodoRuntimeAction(server, 'start') : await komodoRestart(server);
if (provisioningCancelled(operationId)) return { kind: 'failed', message: 'Operation was cancelled' };
if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return { kind: 'failed', message: 'Original server was deleted' };
if (result.ok) {
db().transaction(() => {
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ? AND server_deleted_at IS NULL").run(operationId);
deploymentStage(operationId!, 'Waiting for container and game health (RCON or fresh telemetry)', {
phase: 'verify-container', command: { state: 'accepted', message: 'Komodo accepted the request; recovery still requires verification.' },
checkpoint: { key: 'container-action', label: 'Container command accepted' },
blocker: { code: 'recovery-settling', message: 'Waiting for the recovery settling interval before checking health.', automatic: true }
});
})();
return { kind: 'started' };
}
// These failures cannot have queued a remote restart. Other failures
// (especially a timeout) may have: retain capacity until recovery.
db().transaction(() => {
if (result.kind === 'no-name' || result.kind === 'unconfigured') {
db().prepare('DELETE FROM restart_rollout WHERE server = ? AND operation_id = ?').run(server, operationId);
recordProgress(operationId, { command: { state: 'rejected', message: 'The command could not be submitted: ' + result.message },
recovery: { state: 'released', message: 'Capacity released because no remote command could be submitted.', operationId } });
} else {
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ? AND operation_id = ?').run(`komodo: ${result.message}`, server, operationId);
recordProgress(operationId, { command: { state: 'uncertain', message: 'The command may have reached Komodo: ' + result.message } });
}
finishDeployment(operationId!, { ok: false, message: `komodo: ${result.message}` });
})();
return { kind: 'failed', message: `komodo: ${result.message}` };
} catch (err) {
if (provisioningCancelled(operationId)) return { kind: 'failed', message: 'Operation was cancelled' };
if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return { kind: 'failed', message: 'Original server was deleted' };
const message = `komodo: ${(err as Error).message}`;
db().transaction(() => {
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ? AND operation_id = ?').run(message, server, operationId);
recordProgress(operationId, { command: { state: 'uncertain', message: 'The command may have reached Komodo: ' + (err as Error).message } });
finishDeployment(operationId!, { ok: false, message });
})();
return { kind: 'failed', message };
} finally {
// Start the grace period at completion, including slow/failed requests.
db().prepare('UPDATE restart_rollout SET requested_at = ? WHERE server = ? AND operation_id = ?')
.run(new Date().toISOString(), server, operationId);
if (requestsInFlight.get(server) === operationId) requestsInFlight.delete(server);
}
}
let refreshInFlight: Promise<void> | null = null;
/** Poll all occupied slots, even if their originating beta was already closed. */
export function refreshRestartRollout(): Promise<void> {
if (refreshInFlight) return refreshInFlight;
refreshInFlight = refresh().finally(() => { refreshInFlight = null; });
return refreshInFlight;
}
async function refresh(): Promise<void> {
const slots = db().prepare('SELECT * FROM restart_rollout WHERE healthy_at IS NULL ORDER BY requested_at, server')
.all() as RestartSlot[];
for (const slot of slots) {
if (slot.operation_id && nativeDeployment(slot.operation_id)) continue;
if (requestsInFlight.has(slot.server)) continue;
// Recovery capacity transferred to a rebuild remains reserved until
// teardown. Its health is verified by the creation worker, never here.
if (slot.operation_id && db().prepare(`SELECT 1 FROM server_deployments d WHERE d.id = ? AND d.server_deleted_at IS NULL AND (
(d.action = 'rebuild' AND d.state IN ('waiting','queued','running','awaiting-health'))
OR (d.action = 'restart' AND d.state = 'waiting' AND d.submitted_at IS NULL AND EXISTS (
SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'watchdog')))`)
.get(slot.operation_id)) continue;
if (Date.now() - Date.parse(slot.checked_at ?? slot.requested_at) < HEALTH_POLL_MS) continue;
let error: string | null = null;
let players: number | null = null;
const checks: NonNullable<ProgressUpdate['checks']> = [];
try {
const health = await komodoInspectHealthy(slot.server);
checks.push({ key: 'container', label: 'Container readiness', result: health.ok ? 'pass' : 'waiting', source: 'komodo', actual: health.state ?? null,
checkedAt: new Date().toISOString(), message: health.ok ? null : health.reason });
if (!health.ok) {
error = `unhealthy: ${health.reason}${health.snippet ? ` snippet=${health.snippet.slice(0, 400)}` : ''}`;
} else {
const status = await probeServerStatus(slot.server, slot.requested_at);
checks.push({ key: 'game', label: 'Game status', result: status.ok ? 'pass' : 'waiting', source: status.ok ? status.source : null,
checkedAt: status.observedAt, receivedAt: status.ok ? status.receivedAt : null, actual: status.ok ? status.players : null,
message: status.ok ? null : status.reason });
if (!status.ok) error = `rcon not answering and no fresh telemetry: ${status.reason}`;
else players = status.players;
}
} catch (err) {
error = `health check: ${(err as Error).message}`;
checks.push({ key: checks.length ? 'game' : 'container', label: checks.length ? 'Game status' : 'Container readiness', result: 'unknown', source: null, message: error });
}
const now = new Date().toISOString();
db().transaction(() => {
const checked = db().prepare(`UPDATE restart_rollout SET checked_at = ?, healthy_at = ?, players = ?, error = ?
WHERE server = ? AND requested_at = ? AND operation_id IS ? AND healthy_at IS NULL`)
.run(now, error ? null : now, players, error, slot.server, slot.requested_at, slot.operation_id);
if (!checked.changes) return;
if (slot.operation_id) recordProgress(slot.operation_id, {
phase: checks.some(check => check.key === 'game') ? 'verify-game' : 'verify-container', checks, observedAt: now,
blocker: error ? { code: 'recovery-health', message: error, automatic: true } : null,
recovery: { state: error ? 'held' : 'released', at: now, operationId: slot.operation_id,
message: error ? 'Restart capacity remains reserved while recovery is checked.' : 'Container and game recovery verified; restart capacity released.' },
checkpoints: error ? [] : [{ key: 'verify-container', label: 'Container readiness verified', at: now }, { key: 'verify-game', label: 'Game status verified', at: now }]
});
if (slot.operation_id && db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'awaiting-health'").get(slot.operation_id)) {
db().prepare('UPDATE server_deployments SET checked_at = ?, players = ? WHERE id = ?').run(now, players, slot.operation_id);
if (error) deploymentStage(slot.operation_id, 'Waiting for health: ' + error);
else {
deploymentStage(slot.operation_id, 'Verified container and game health (RCON or fresh telemetry)');
finishDeployment(slot.operation_id, { ok: true });
}
}
})();
}
}
export function rolloutHealth(server: string): { ok: true; players: number } | { ok: false; reason: string } {
const slot = slotFor(server);
if (slot?.healthy_at) return { ok: true, players: slot.players ?? 0 };
src/lib/server/provisioning/worker.ts
+5 -170
View File
@@ -1,24 +1,11 @@
import { maintenanceAllowed } from '../maintenance/policies.js';
import { resolveRecoveredHealthOperation } from '../maintenance/recovery.js';
import { db } from '../db/index.js';
import { provisioningCancelled } from './cancellation.js';
import { komodoConfigured, rconConfigured } from '../config.js';
import { getServer } from '../fleet/servers.js';
import { deploymentStage, finishDeployment, REMOVING_SERVER } from './history.js';
import { processCreations, processDestructions, rebuildServer } from './instances.js';
import { komodoInspectHealthy, komodoRuntimeAction } from '../integrations/komodo.js';
import { latestGameSubmission, probePlayerEvidence } from '../fleet/status-evidence.js';
import { refreshRestartRollout, restartInRollout, rolloutBlocker } from './restarts.js';
import { rolloutBlocker } from './restarts.js';
import { cancelProvisioning } from './requests.js';
import type { RuntimeAction } from '../../provisioning.js';
import type { ProvisioningBlocker } from '../../provisioning-detail.js';
import { recordProgress } from './progress.js';
import { nativeInstance, nativeDeployment } from '../backends/native.js';
import { processNativeProvisioning } from './native.js';
import { trackActivity } from '../pipeline/activity.js';
const stoppedStates = new Set(['stopped', 'exited', 'created', 'dead']);
export const isStopped = (state?: string) => stoppedStates.has(state?.toLowerCase() ?? '');
interface Source { kind: 'campaign' | 'beta' | 'maintenance' | 'watchdog'; source_id: string }
const sourcesFor = (id: string) => db().prepare('SELECT kind, source_id FROM provisioning_sources WHERE operation_id = ?').all(id) as Source[];
@@ -92,20 +79,13 @@ function blocker(server: string, id: string, action: RuntimeAction): Provisionin
}
interface Operation { id: string; server: string; action: RuntimeAction; state: string; stage: string }
let ticking: Promise<void> | null = null;
let awaitedPass: Promise<void> | null = null;
export function processProvisioning(options: { wait?: boolean } = {}): Promise<void> {
if (options.wait !== false && awaitedPass) return awaitedPass;
// Native jobs own their individual flights. A slow observation must not
// prevent the next scheduler pass admitting newly arrived native work.
const native = nativePass(options);
if (!ticking) ticking = trackActivity('legacy-provisioning', 'Processing legacy provisioning', run)
.finally(() => { ticking = null; });
if (options.wait === false) {
void ticking.catch(error => console.error('[worker] legacy provisioning error:', error));
return native;
}
awaitedPass = Promise.all([native, ticking]).then(() => {}).finally(() => { awaitedPass = null; });
// Each job owns its flight, so slow observations do not block new work.
const pass = nativePass(options);
if (options.wait === false) return pass;
awaitedPass = pass.finally(() => { awaitedPass = null; });
return awaitedPass;
}
@@ -126,148 +106,3 @@ function nativePass(options: { wait?: boolean }): Promise<void> {
return blocker(operation.server, operation.id, typed.action)?.message ?? null;
}, options);
}
async function run(): Promise<void> {
await processCreations();
await processDestructions();
await refreshRestartRollout();
const operations = db().prepare(`SELECT id, server, action, state, stage FROM server_deployments
WHERE server_deleted_at IS NULL AND action IN ('start','restart','stop','rebuild') AND (state = 'waiting' OR (action = 'stop' AND state = 'awaiting-health'))
ORDER BY started_at, rowid`).all() as Operation[];
for (const operation of operations) {
if (nativeDeployment(operation.id)) continue;
try { await processOperation(operation); }
catch (err) {
if (provisioningCancelled(operation.id)) continue;
// Observation failures are retried; remote mutations are never replayed.
deploymentStage(operation.id, 'Waiting for verification: ' + (err as Error).message, {
checks: [], blocker: { code: 'observation-unavailable', message: (err as Error).message, automatic: true }
});
}
}
}
async function processOperation(operation: Operation): Promise<void> {
const { id, server, action } = operation;
if (provisioningCancelled(id)) return;
// An explicit cleanup resume follows confirmed remote deletion. Complete
// that durable checkpoint even if the incident now needs attention or the
// Watchdog is paused; never strand the reserved replacement identity.
if (action === 'rebuild' && operation.stage === REMOVING_SERVER) { await rebuildServer(id); return; }
if (operation.state === 'waiting') {
const heldBy: Source[] = [];
const eligibility = sourceEligibility(operation, source => heldBy.push(source));
if (eligibility === 'cancelled') { cancelProvisioning(id, 'Requester cancelled or replaced this operation'); return; }
if (eligibility === 'held') {
const source = heldBy[0];
const href = source ? source.kind === 'campaign' ? `/campaigns/${encodeURIComponent(source.source_id)}`
: source.kind === 'beta' ? `/betas/${encodeURIComponent(source.source_id)}`
: source.kind === 'watchdog' ? `/maintenance/${encodeURIComponent(source.source_id)}`
: `/maintenance?view=activity#job-${encodeURIComponent(source.source_id)}` : null;
deploymentStage(id, 'Waiting for the requester to resume', { phase: 'eligibility', checks: [],
blocker: { code: 'requester-held', message: 'Waiting for the requester to resume', automatic: false, href, label: 'Requesting work' } });
return;
}
}
if (!getServer(server)) { finishDeployment(id, { ok: false, message: 'Server is no longer registered.' }); return; }
const watchdog = sourcesFor(id).some(source => source.kind === 'watchdog');
if (!komodoConfigured() || (!rconConfigured() && !watchdog)) { deploymentStage(id, 'Waiting for Komodo and RCON configuration', {
phase: 'eligibility', checks: [], blocker: { code: 'configuration', message: 'Waiting for Komodo and RCON configuration', automatic: false, href: '/settings', label: 'Integration settings' }
}); return; }
if (operation.state === 'awaiting-health') {
const health = await komodoInspectHealthy(server);
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'awaiting-health'").get(id)) return;
recordProgress(id, { phase: 'verify-stopped', checks: [{ key: 'stopped', label: 'Container stopped', source: 'komodo',
result: isStopped(health.state) ? 'pass' : health.state ? 'waiting' : 'unknown', actual: health.state ?? null,
message: health.ok ? null : health.reason }], blocker: isStopped(health.state) ? null
: { code: 'stopped-state', message: 'Waiting for an explicit stopped container state.', automatic: true } });
if (isStopped(health.state)) {
db().transaction(() => {
db().prepare("UPDATE servers SET container_state = ?, last_players = NULL, last_roster = NULL, game_started_at = NULL, game_ended_at = NULL WHERE name = ?").run(health.state, server);
deploymentStage(id, 'Verified container stopped', { checkpoint: { key: 'verify-stopped', label: 'Stopped container state verified' } });
finishDeployment(id, { ok: true });
})();
} else deploymentStage(id, 'Waiting for container to stop' + (!health.ok ? ': ' + health.reason : ''));
return;
}
const reason = blocker(server, id, action);
if (reason) { deploymentStage(id, reason.message, { phase: 'eligibility', checks: [], blocker: reason }); return; }
const healthRecovery = watchdog && !!db().prepare(`SELECT 1 FROM provisioning_sources s
JOIN watchdog_incidents i ON i.id = s.source_id JOIN server_deployments d ON d.id = s.operation_id
WHERE s.operation_id = ? AND s.kind = 'watchdog' AND i.kind = 'health' AND d.triggered_by LIKE 'watchdog:%'`).get(id);
const epoch = latestGameSubmission(server);
const health = await komodoInspectHealthy(server);
if (provisioningCancelled(id)) return;
if (healthRecovery && epoch !== latestGameSubmission(server)) return;
if (healthRecovery && !health.ok && !['stopped','exited','created','dead','unhealthy','starting','restarting'].includes(health.state?.toLowerCase() ?? '')) {
deploymentStage(id, 'Waiting for current container health before automatic recovery', { phase: 'eligibility', checks: [],
blocker: { code: 'observation-unavailable', message: health.reason, automatic: true } });
return;
}
if (action !== 'start' && !isStopped(health.state)) {
const evidence = await probePlayerEvidence(server);
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return;
const players = evidence.players;
if (healthRecovery && epoch !== latestGameSubmission(server)) return;
if (healthRecovery && health.ok && players !== null) {
// The fresh occupancy check also proves that the game recovered.
// Empty is not a reason to execute an obsolete health restart.
resolveRecoveredHealthOperation(id, players, evidence.source);
return;
}
recordProgress(id, { phase: 'eligibility', checks: [{ key: 'occupancy', label: 'Player occupancy', result: players === 0 ? 'pass' : players === null ? 'unknown' : 'waiting',
source: evidence.source, checkedAt: evidence.observedAt, receivedAt: evidence.receivedAt, expected: 0, actual: players }],
blocker: players === 0 ? null : { code: players === null ? 'players-unknown' : 'players-present',
message: players === null ? 'Waiting for a fresh player count' : `Waiting for ${players} player(s) to leave`, automatic: true } });
db().prepare('UPDATE server_deployments SET players = ?, checked_at = ? WHERE id = ?').run(players, new Date().toISOString(), id);
if (players !== 0) {
deploymentStage(id, players === null ? 'Waiting for a fresh player count' : `Waiting for ${players} player(s) to leave`);
return;
}
} else if (action !== 'start') {
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return;
recordProgress(id, { phase: 'eligibility', blocker: null, checks: [{ key: 'stopped', label: 'Stopped container permits disruption',
source: 'komodo', result: 'pass', actual: health.state ?? null }] });
}
// Requests can arrive during the probes. Recheck before claiming the action.
const latestReason = blocker(server, id, action);
if (latestReason) { deploymentStage(id, latestReason.message, { phase: 'eligibility', blocker: latestReason }); return; }
if (sourceEligibility(operation) !== 'ready') return;
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return;
if (action === 'rebuild') { await rebuildServer(id); return; }
const claim = db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").run(id);
if (!claim.changes) return;
if (action === 'start' || action === 'restart') {
const result = await restartInRollout(server, { operationId: id, action });
if (provisioningCancelled(id)) return;
if (result.kind === 'queued') {
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
deploymentStage(id, result.message);
}
return;
}
db().transaction(() => {
db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), id);
deploymentStage(id, 'Stopping container', { phase: 'container-action', blocker: null, checks: [],
checkpoint: { key: 'eligibility', label: 'Eligibility checks passed' }, command: { state: 'attempted', message: 'Stop submission attempted.' } });
})();
// Persist intent first; a lost reply must not issue another stop later.
try {
const result = await komodoRuntimeAction(server, 'stop');
if (provisioningCancelled(id)) return;
if (!result.ok) {
recordProgress(id, { command: { state: result.kind === 'no-name' || result.kind === 'unconfigured' ? 'rejected' : 'uncertain', message: result.message } });
finishDeployment(id, { ok: false, message: result.message }); return;
}
db().transaction(() => {
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
deploymentStage(id, 'Waiting for container to stop', { phase: 'verify-stopped',
checkpoint: { key: 'container-action', label: 'Stop command accepted' },
command: { state: 'accepted', message: 'Komodo accepted the stop request; stopped state still requires verification.' } });
})();
} catch (err) {
if (provisioningCancelled(id)) return;
recordProgress(id, { command: { state: 'uncertain', message: (err as Error).message } });
finishDeployment(id, { ok: false, message: (err as Error).message });
}
}
src/lib/ui/components/ProvisioningProgress.svelte
+1 -1
View File
@@ -9,7 +9,7 @@
const progress = $derived(detail.progress);
let expanded = $state<Record<string, boolean>>({});
const evidenceLabels = { pass: 'Passed', waiting: 'Waiting', fail: 'Failed', unknown: 'Unknown' };
const sources = { komodo: 'Komodo', agent: 'Host agent', rcon: 'RCON', telemetry: 'Telemetry', combined: 'RCON and telemetry', manifest: 'Plutonium manifest' };
const sources = { agent: 'Host agent', rcon: 'RCON', telemetry: 'Telemetry', combined: 'RCON and telemetry', manifest: 'Plutonium manifest' };
</script>
<section class="execution" aria-labelledby="execution-title">
src/lib/ui/komodoError.ts
-27
View File
@@ -1,27 +0,0 @@
export type KomodoPortFailure =
| 'unconfigured'
| 'no-name'
| 'unreachable'
| 'http-error'
| 'unparseable'
| 'no-udp-in-response';
/** Short fix hint per failure reason, shown under the Komodo probe result. */
export function komodoFixHint(reason: KomodoPortFailure, httpStatus?: number): string {
switch (reason) {
case 'unconfigured':
return 'Set KOMODO_ADDRESS, KOMODO_API_KEY, and KOMODO_API_SECRET in the container env.';
case 'no-name':
return 'This server row has no komodo_name — set it via SQL or via the fleet config.';
case 'unreachable':
return 'Network, DNS, or timeout. Check that KOMODO_ADDRESS is reachable from this container.';
case 'http-error':
if (httpStatus === 401 || httpStatus === 403) return 'Auth rejected — verify KOMODO_API_KEY and KOMODO_API_SECRET.';
if (httpStatus === 404) return 'Endpoint or deployment not found — check komodo_name matches the Komodo UI exactly.';
return `Komodo returned ${httpStatus ?? 'an error'}. Check the endpoint path and credentials.`;
case 'unparseable':
return 'Response was not JSON — is KOMODO_ADDRESS pointing at the Core API (not the Periphery)?';
case 'no-udp-in-response':
return 'Parsed the response but no UDP port matched our known shapes. See the snippet below and extend extractUdpPort.';
}
}
src/lib/ui/provisioning-presentation.ts
+15 -24
View File
@@ -8,35 +8,27 @@ export interface ProgressGroup { key: string; label: string; steps: ProgressStep
const phaseLabels: Record<ProvisioningPhase, string> = {
requested: 'Request recorded', eligibility: 'Check eligibility and capacity', 'startup-queue': 'Wait for a startup slot',
prepare: 'Prepare server files', 'create-deployment': 'Create Komodo deployment', 'container-action': 'Start container',
'remove-container': 'Stop and remove container', 'delete-deployment': 'Delete Komodo deployment', cleanup: 'Remove files and registration',
prepare: 'Prepare server files', 'container-action': 'Start container',
cleanup: 'Remove files and registration',
'verify-container': 'Confirm container readiness', 'verify-game': 'Verify game status', 'verify-map': 'Verify selected engine map',
'verify-plutonium': 'Verify Plutonium revision',
'verify-stopped': 'Confirm container is stopped'
};
type GroupDefinition = { key: string; label: string; phases: ProvisioningPhase[] };
const request: GroupDefinition = { key: 'request', label: 'Request and eligibility', phases: ['requested', 'eligibility'] };
const creation: GroupDefinition = { key: 'create', label: 'Create and start', phases: ['startup-queue', 'prepare', 'create-deployment', 'container-action'] };
const removal: GroupDefinition = { key: 'remove', label: 'Remove the instance', phases: ['remove-container', 'delete-deployment', 'cleanup'] };
const readiness: GroupDefinition = { key: 'verify', label: 'Verify readiness', phases: ['verify-container', 'verify-game', 'verify-map', 'verify-plutonium'] };
function definitions(action: ProvisioningAction, native = false): GroupDefinition[] {
if (native) {
const command: GroupDefinition = { key: 'command', label: `${actionLabels[action]} on host agent`, phases: ['container-action'] };
if (action === 'create' || action === 'rebuild') return [request,
{ key: 'create', label: action === 'rebuild' ? 'Rebuild on host agent' : 'Create and start on host agent', phases: ['startup-queue', 'prepare', 'container-action'] }, readiness];
if (action === 'destroy') return [request, command, { key: 'cleanup', label: 'Retire instance', phases: ['cleanup'] }];
}
if (action === 'create') return [{ ...request, phases: ['requested'] }, creation, readiness];
if (action === 'rebuild') return [request, removal, { ...creation, label: 'Create a fresh instance' }, readiness];
if (action === 'destroy') return [request, removal];
return [request, { key: 'command', label: `${actionLabels[action]} the existing container`, phases: ['container-action'] },
{ ...readiness, phases: action === 'stop' ? ['verify-stopped'] : ['verify-container', 'verify-game'] }];
function definitions(action: ProvisioningAction): GroupDefinition[] {
const command: GroupDefinition = { key: 'command', label: `${actionLabels[action]} on host agent`, phases: ['container-action'] };
if (action === 'create' || action === 'rebuild') return [request,
{ key: 'create', label: action === 'rebuild' ? 'Rebuild on host agent' : 'Create and start on host agent', phases: ['startup-queue', 'prepare', 'container-action'] }, readiness];
if (action === 'destroy') return [request, command, { key: 'cleanup', label: 'Retire instance', phases: ['cleanup'] }];
return [request, command, { ...readiness, phases: action === 'stop' ? ['verify-stopped'] : ['verify-container', 'verify-game'] }];
}
export function provisioningGroups(detail: ProvisioningDetail): ProgressGroup[] {
const progress = detail.progress;
if (!progress) return [];
const groups = definitions(detail.deployment.action, detail.backend === 'native');
const groups = definitions(detail.deployment.action);
const phases = groups.flatMap(group => group.phases);
const current = progress.phase ? phases.indexOf(progress.phase) : -1;
const active = activeProvisioningStates.includes(detail.deployment.state);
@@ -54,11 +46,10 @@ export function provisioningGroups(detail: ProvisioningDetail): ProgressGroup[]
: isCurrent && active ? progress.blocker || ['waiting', 'queued'].includes(detail.deployment.state) ? 'waiting' : 'current'
: current >= 0 && index >= current && detail.deployment.state === 'cancelled' ? 'cancelled'
: active && current >= 0 && index > current ? 'pending' : 'unknown';
const label = detail.backend === 'native' && key === 'startup-queue' ? 'Admit to startup capacity'
: detail.backend === 'native' && key === 'prepare' ? 'Prepare launch payload'
: detail.backend === 'native' && key === 'container-action' ? `${actionLabels[detail.deployment.action]} instance through host agent`
: key === 'container-action' && ['start', 'restart', 'stop'].includes(detail.deployment.action)
? `${actionLabels[detail.deployment.action]} existing container` : phaseLabels[key];
const label = key === 'startup-queue' ? 'Admit to startup capacity'
: key === 'prepare' ? 'Prepare launch payload'
: key === 'container-action' ? `${actionLabels[detail.deployment.action]} instance through host agent`
: phaseLabels[key];
return { key, label, state, at: check?.checkedAt ?? checkpoint?.at ?? (isCurrent ? progress.phaseStartedAt : null) };
});
const states = steps.map(step => step.state);
@@ -168,7 +159,7 @@ export function isProvisioningDetail(value: unknown, id: string): value is Provi
if (value.cancellation !== undefined && value.cancellation !== null && !(record(value.cancellation)
&& text(value.cancellation.cleanupId) && text(value.cancellation.state) && text(value.cancellation.stage)
&& nullableText(value.cancellation.error))) return false;
if (value.backend !== undefined && !['native', 'komodo'].includes(value.backend as string)) return false;
if (value.backend !== undefined && !['native'].includes(value.backend as string)) return false;
if (value.workerActivity !== undefined && (!Array.isArray(value.workerActivity) || !value.workerActivity.every(item =>
record(item) && text(item.key) && text(item.label) && timestamp(item.startedAt)))) return false;
if (value.agentOperation !== undefined && value.agentOperation !== null && !(record(value.agentOperation)
@@ -212,7 +203,7 @@ export function isProvisioningDetail(value: unknown, id: string): value is Provi
if (!Array.isArray(progress.checks) || !progress.checks.every(check => record(check)
&& ['container', 'game', 'map', 'plutonium-release', 'plutonium-version', 'occupancy', 'stopped'].includes(check.key as string)
&& ['pass', 'waiting', 'fail', 'unknown'].includes(check.result as string)
&& [null, 'komodo', 'agent', 'rcon', 'telemetry', 'combined', 'manifest'].includes(check.source as null | string)
&& [null, 'agent', 'rcon', 'telemetry', 'combined', 'manifest'].includes(check.source as null | string)
&& text(check.label) && timestamp(check.checkedAt) && nullableTime(check.receivedAt)
&& scalar(check.expected) && scalar(check.actual) && nullableText(check.message))) return false;
if (!Array.isArray(progress.checkpoints) || !progress.checkpoints.every(checkpoint => record(checkpoint)
src/lib/ui/rconError.ts
+1 -1
View File
@@ -30,7 +30,7 @@ export function rconFixHint(reason: RconFailure): string {
case 'no-config':
return 'Set RCON_PASSWORD in the container env and restart.';
case 'no-port':
return 'Set game_port on the server row, or configure Komodo so it can resolve the port.';
return 'Check the host agent and the game port assigned to this server.';
case 'send-error':
return 'DNS or route problem — check the RCON host is reachable from this container.';
case 'timeout':
src/lib/ui/stores/fleet.ts
+1 -4
View File
@@ -13,7 +13,7 @@ export function createFleetStore(serverName?: string) {
});
const fleetError = writable('');
const fleetRefreshing = writable(false);
const fleetIntegrations = writable<{ rcon: boolean; komodo: boolean; native?: boolean; hostsReady?: number; hostsTotal?: number; rconTimeoutMs: number; defaultRconHost: string } | null>(null);
const fleetIntegrations = writable<{ rcon: boolean; native?: boolean; hostsReady?: number; hostsTotal?: number; rconTimeoutMs: number; defaultRconHost: string } | null>(null);
const observed = new Map<string, { server: FleetServer; status: unknown; match: unknown; report: unknown; phase: string; value: FleetServer }>();
const freshnessPhase = derived([fleet, statusByServer, telemetryByServer, uiNow], ([snapshot, live, reports, now]) => {
@@ -63,7 +63,6 @@ export function createFleetStore(serverName?: string) {
return next;
});
let snapshotServerNow = 0;
function hydrateFleet(snapshot: { servers: FleetServer[]; operations: FleetOperation[]; integrations: NonNullable<ReturnType<typeof getIntegrations>>; serverNow: number }) {
if (!browser || snapshot.serverNow < snapshotServerNow) return;
@@ -74,8 +73,6 @@ export function createFleetStore(serverName?: string) {
}
function getIntegrations() { return get(fleetIntegrations); }
let inFlight: Promise<void> | null = null;
let refreshRequested = false;
function refreshFleet(afterCurrent = false): Promise<void> {
src/routes/+page.svelte
-2
View File
@@ -165,8 +165,6 @@
</div>{/if}
{#if integrations?.native && !integrations.hostsReady}
<div class="setup-note"><span><strong>Prepare a game host</strong> · {integrations.hostsTotal ? 'Hosts are connected or enrolled but need setup before new deployments.' : 'Add a host, prepare its runtime, and download game files.'}</span><a href="/hosts">Review hosts →</a></div>
{:else if $page.data.user?.role === 'admin' && $fleetIntegrations && !integrations.native && (!integrations.komodo || !integrations.rcon)}
<div class="setup-note"><span><strong>Finish system setup</strong> · {!integrations.komodo ? 'Komodo' : ''}{!integrations.komodo && !integrations.rcon ? ' and ' : ''}{!integrations.rcon ? 'RCON' : ''} not configured.</span><a href="/settings">Review setup →</a></div>
{/if}
{#if problems.length || attentionOperations.length || starting || provisioning}
src/routes/api/hosts/+server.ts
+1 -1
View File
@@ -4,7 +4,7 @@ import { hostApiError, installerCommand, objectBody, requireManagementOrigin } f
import { presentHostPreparation } from '$lib/server/hosts/preparation.js';
import { config } from '$lib/server/config.js';
export function GET({ request }: { request: Request }) { requireManagementOrigin(request); return json({ hosts: listHosts().map(presentHostPreparation), defaultRuntimeImage: config.komodo.defaultImage }); }
export function GET({ request }: { request: Request }) { requireManagementOrigin(request); return json({ hosts: listHosts().map(presentHostPreparation), defaultRuntimeImage: config.defaultRuntimeImage }); }
export async function POST({ request }: { request: Request }) {
requireManagementOrigin(request);
try {
src/routes/api/hosts/[id]/+server.ts
+1 -1
View File
@@ -10,7 +10,7 @@ import { kickProvisioning } from '$lib/server/pipeline/worker.js';
type Event = { request: Request; params: { id: string } };
export function GET({ request, params }: Event) {
requireManagementOrigin(request); const host = getHost(params.id); if (!host) error(404, 'Host not found.');
return json({ host: presentHostPreparation(host), instances: hostInstances(host.id), serverNow: Date.now(), calibration: capacityCalibration(host.id), defaultRuntimeImage: config.komodo.defaultImage, deletion: hostDeletionStatus(host.id) });
return json({ host: presentHostPreparation(host), instances: hostInstances(host.id), serverNow: Date.now(), calibration: capacityCalibration(host.id), defaultRuntimeImage: config.defaultRuntimeImage, deletion: hostDeletionStatus(host.id) });
}
export async function PATCH({ request, params }: Event) {
requireManagementOrigin(request);
src/routes/api/servers/[name]/diagnose-network/+server.ts
-77
View File
@@ -1,77 +0,0 @@
import { json } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { getServer } from '$lib/server/fleet/servers.js';
import { config } from '$lib/server/config.js';
import { tcpProbeAll } from '$lib/server/integrations/netProbe.js';
/**
* Multi-target TCP reachability probe. Fires parallel connect attempts at a
* fixed matrix of candidate hosts + the configured KOMODO_ADDRESS host, so
* we can see at a glance which network paths reach anything and where
* packets die (DNS, routing, or refused-at-port).
*/
export const POST: RequestHandler = async ({ params }) => {
const name = params.name;
if (!name) return json({ ok: false, error: 'missing server name' }, { status: 400 });
const row = getServer(name);
if (!row) return json({ ok: false, error: `unknown server: ${name}` }, { status: 404 });
// Parse the configured Komodo URL to extract host+port.
let komodoHost: string | null = null;
let komodoPort: number | null = null;
if (config.komodo.address) {
try {
const u = new URL(config.komodo.address);
komodoHost = u.hostname;
komodoPort = u.port ? Number.parseInt(u.port, 10) : u.protocol === 'https:' ? 443 : 80;
} catch {
/* leave null */
}
}
const probePort = komodoPort ?? 9120;
// Build the trial matrix. Every trial hits the same target port (the
// Komodo port) via a different network path, so we can see which paths
// reach the same service.
const trials: Array<{ label: string; host: string; port: number }> = [
{ label: `127.0.0.1:${probePort} (container loopback)`, host: '127.0.0.1', port: probePort },
{ label: `host.docker.internal:${probePort} (bridge gateway alias)`, host: 'host.docker.internal', port: probePort },
{ label: `172.17.0.1:${probePort} (default bridge gateway)`, host: '172.17.0.1', port: probePort }
];
if (komodoHost && !trials.some((t) => t.host === komodoHost)) {
trials.push({
label: `${komodoHost}:${probePort} (KOMODO_ADDRESS)`,
host: komodoHost,
port: probePort
});
}
// RCON's default host is worth testing too — even though RCON is UDP,
// if TCP to the same host also fails, it's a routing/firewall issue,
// not a UDP-specific one.
const rconHost = row.rcon_host ?? config.rcon.host;
if (rconHost && !trials.some((t) => t.host === rconHost)) {
trials.push({
label: `${rconHost}:${probePort} (RCON_HOST)`,
host: rconHost,
port: probePort
});
}
const results = await tcpProbeAll(trials, config.httpTimeoutMs);
return json({
ok: true,
config: {
komodoAddress: config.komodo.address || null,
komodoHost,
komodoPort,
rconHost,
probeTimeoutMs: config.httpTimeoutMs
},
results
});
};
src/routes/api/servers/[name]/probe-komodo/+server.ts
-27
View File
@@ -1,27 +0,0 @@
import { json } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { komodoResolvePortDetailed } from '$lib/server/integrations/komodo.js';
import { getServer, setServerField } from '$lib/server/fleet/servers.js';
/**
* Trigger a fresh Komodo GetDeployment lookup for a specific server and
* return the structured result — including a snippet of the response so
* the UI can render "here's where we found the port" (or "here's what
* Komodo returned; extend extractUdpPort for this shape").
*
* On success, the resolved port is written to servers.game_port so the
* next RCON attempt uses it without waiting for another sweep tick.
*/
export const POST: RequestHandler = async ({ params }) => {
const name = params.name;
if (!name) return json({ ok: false, reason: 'no-name', detail: 'missing server name' }, { status: 400 });
const row = getServer(name);
if (!row) return json({ ok: false, reason: 'no-name', detail: `unknown server: ${name}` }, { status: 404 });
const result = await komodoResolvePortDetailed(name);
if (result.ok) {
setServerField(name, 'game_port', result.port);
}
return json(result);
};
src/routes/api/system-status/+server.ts
+13 -42
View File
@@ -1,59 +1,30 @@
import { json } from '@sveltejs/kit';
import { config, komodoConfigured, rconConfigured } from '$lib/server/config.js';
import { listServers } from '$lib/server/fleet/servers.js';
import { listServerKeys } from '$lib/server/fleet/server-keys.js';
import { serverObservation } from '$lib/server/fleet/status.js';
import { freshStatusTimestamp } from '$lib/server-freshness.js';
import { tcpProbe } from '$lib/server/integrations/netProbe.js';
import type { SetupCheck, SystemStatus } from '$lib/system-status.js';
import { listHosts } from '$lib/server/hosts/index.js';
import { listLibraries } from '$lib/server/game-files/index.js';
export async function GET({ url }: { url: URL }) {
export async function GET() {
const servers = listServers();
const readyServers = servers.filter(server => serverObservation(server).health === 'online').length;
const rconServers = servers.filter(server => !server.last_error && freshStatusTimestamp(server.last_seen_at, Date.now()) && server.container_state === 'running').length;
const savedKeys = listServerKeys().length;
const hosts = listHosts();
if (hosts.length || !komodoConfigured()) {
const readyHosts = hosts.filter(host => host.readiness === 'ready');
const connected = hosts.filter(host => host.connected);
const versions = listLibraries().flatMap(game => game.versions.filter(version => version.id === game.latestVersionId));
const cached = versions.filter(version => version.caches.some(cache => cache.status === 'ready' && readyHosts.some(host => host.id === cache.hostId)));
const runtimes = connected.filter(host => host.inventory.runtimeImages?.length);
const checks: SetupCheck[] = [
{ id: 'hosts', title: 'Host agents', state: readyHosts.length ? 'ready' : hosts.length ? 'attention' : 'unconfigured', detail: `${connected.length} connected, ${readyHosts.length} deployment-ready, ${hosts.length} enrolled or awaiting enrollment.`, href: '/hosts', action: hosts.length ? 'Review hosts' : 'Add host' },
{ id: 'runtime', title: 'Plutainer runtime', state: runtimes.length ? 'ready' : 'attention', detail: runtimes.length ? `${runtimes.length} connected hosts report cached runtime images.` : 'Hosts prepare the configured Plutainer runtime automatically.', href: '/hosts', action: 'View host setup' },
{ id: 'game-files', title: 'T5 / T6 game files', state: cached.length ? 'ready' : 'attention', detail: `${versions.length} games have uploaded files; ${cached.length} have their latest files verified on a ready host.`, href: '/game-files', action: 'Manage game files' },
{ id: 'rcon', title: 'Agent-local game connection', state: rconServers ? 'ready' : 'unverified', detail: rconServers ? `${rconServers} servers have fresh RCON responses.` : 'The host agent checks RCON locally after deployment. Controller access to game ports is not required.', href: servers.length ? '/' : '/hosts', action: servers.length ? 'Review fleet' : 'Review hosts' },
{ id: 'keys', title: 'Saved server keys', state: savedKeys ? 'ready' : 'attention', detail: `${savedKeys} saved ${savedKeys === 1 ? 'key' : 'keys'}. Validity is checked with Plutonium at startup.`, href: '/server-keys', action: 'Manage keys' },
{ id: 'first-server', title: 'First server ready', state: readyServers ? 'ready' : 'attention', detail: readyServers ? `${readyServers} of ${servers.length} servers are responding.` : servers.length ? 'Registered servers are awaiting fresh healthy reports. Inspect their host and diagnostics.' : 'Choose a host to launch your first server with the latest game files.', href: servers.length ? '/' : '/provisioning', action: servers.length ? 'Review fleet' : 'Create a server' }
];
return json({ checkedAt: new Date().toISOString(), checks, readyServers, totalServers: servers.length, savedKeys } satisfies SystemStatus, { headers: { 'cache-control': 'no-store' } });
}
const missing = [!config.komodo.address && 'KOMODO_ADDRESS', !config.komodo.apiKey && 'KOMODO_API_KEY', !config.komodo.apiSecret && 'KOMODO_API_SECRET'].filter(Boolean);
const connection: SetupCheck = { id: 'komodo', title: 'Komodo connection', state: komodoConfigured() ? 'unverified' : 'unconfigured', detail: komodoConfigured() ? 'Credentials configured. Run connection check to verify host reachability.' : `Set ${missing.join(', ')} in the app environment.`, href: '#setup-guide', action: 'Setup instructions' };
if (komodoConfigured() && url.searchParams.get('probe') === '1') {
try {
const address = new URL(config.komodo.address);
if (!['http:', 'https:'].includes(address.protocol)) throw new Error('invalid address');
const result = await Promise.race([
tcpProbe('Komodo', address.hostname, Number(address.port || (address.protocol === 'https:' ? 443 : 80)), 3000),
new Promise<null>(resolve => setTimeout(() => resolve(null), 4000))
]);
const reachable = result?.outcome === 'connected';
connection.state = reachable ? 'ready' : 'attention';
connection.detail = reachable ? 'Host reachable. This network check does not validate API credentials; deployment previews check launch prerequisites.' : 'Host unreachable from Electric Cherry. Check the address, DNS, port, and network access.';
} catch { connection.state = 'attention'; connection.detail = 'Komodo address is invalid or could not be reached. Check KOMODO_ADDRESS.'; }
}
const readyHosts = hosts.filter(host => host.readiness === 'ready');
const connected = hosts.filter(host => host.connected);
const versions = listLibraries().flatMap(game => game.versions.filter(version => version.id === game.latestVersionId));
const cached = versions.filter(version => version.caches.some(cache => cache.status === 'ready' && readyHosts.some(host => host.id === cache.hostId)));
const runtimes = connected.filter(host => host.inventory.runtimeImages?.length);
const checks: SetupCheck[] = [
connection,
{ id: 'target', title: 'Deployment host', state: config.komodo.targetServer ? 'ready' : 'unconfigured', detail: config.komodo.targetServer ? `New servers target ${config.komodo.targetServer}.` : 'Set KOMODO_TARGET_SERVER to the exact host name shown in Komodo.', href: '#setup-guide', action: 'Setup instructions' },
{ id: 'rcon', title: 'Game server connection', state: !rconConfigured() ? 'unconfigured' : rconServers ? 'ready' : 'unverified', detail: !rconConfigured() ? 'Set RCON_PASSWORD to the password used by your game servers.' : rconServers ? `${rconServers} servers have fresh successful RCON reports.` : 'RCON password configured. A running server is needed to verify a response.', href: servers.length ? '/' : '#setup-guide', action: servers.length ? 'Review fleet' : 'Setup instructions' },
{ id: 'game-files', title: 'Game files on the host', state: 'unverified', detail: 'T5 and T6 game-file volumes must be populated on the Komodo host. Their contents cannot be verified from this app.', href: '#game-files', action: 'Game-file instructions' },
{ id: 'keys', title: 'Saved server keys', state: savedKeys ? 'ready' : 'attention', detail: `${savedKeys} saved ${savedKeys === 1 ? 'key' : 'keys'}. Key presence does not confirm validity with Plutonium.`, href: '/server-keys', action: savedKeys ? 'Manage keys' : 'Add a key' },
{ id: 'first-server', title: 'First server ready', state: readyServers ? 'ready' : 'attention', detail: readyServers ? `${readyServers} of ${servers.length} servers are responding.` : servers.length ? 'Servers are registered but none has a fresh healthy report. Open diagnostics to investigate.' : 'Create your first server after completing the setup checks.', href: servers.length ? '/' : '/provisioning', action: servers.length ? 'Review fleet' : 'Create a server' }
{ id: 'hosts', title: 'Host agents', state: readyHosts.length ? 'ready' : hosts.length ? 'attention' : 'unconfigured', detail: `${connected.length} connected, ${readyHosts.length} deployment-ready, ${hosts.length} enrolled or awaiting enrollment.`, href: '/hosts', action: hosts.length ? 'Review hosts' : 'Add host' },
{ id: 'runtime', title: 'Plutainer runtime', state: runtimes.length ? 'ready' : 'attention', detail: runtimes.length ? `${runtimes.length} connected hosts report cached runtime images.` : 'Hosts prepare the configured Plutainer runtime automatically.', href: '/hosts', action: 'View host setup' },
{ id: 'game-files', title: 'T5 / T6 game files', state: cached.length ? 'ready' : 'attention', detail: `${versions.length} games have uploaded files; ${cached.length} have their latest files verified on a ready host.`, href: '/game-files', action: 'Manage game files' },
{ id: 'rcon', title: 'Agent-local game connection', state: rconServers ? 'ready' : 'unverified', detail: rconServers ? `${rconServers} servers have fresh RCON responses.` : 'The host agent checks RCON locally after deployment. Controller access to game ports is not required.', href: servers.length ? '/' : '/hosts', action: servers.length ? 'Review fleet' : 'Review hosts' },
{ id: 'keys', title: 'Saved server keys', state: savedKeys ? 'ready' : 'attention', detail: `${savedKeys} saved ${savedKeys === 1 ? 'key' : 'keys'}. Validity is checked with Plutonium at startup.`, href: '/server-keys', action: 'Manage keys' },
{ id: 'first-server', title: 'First server ready', state: readyServers ? 'ready' : 'attention', detail: readyServers ? `${readyServers} of ${servers.length} servers are responding.` : servers.length ? 'Registered servers are awaiting fresh healthy reports. Inspect their host and diagnostics.' : 'Choose a host to launch your first server with the latest game files.', href: servers.length ? '/' : '/provisioning', action: servers.length ? 'Review fleet' : 'Create a server' }
];
const body: SystemStatus = { checkedAt: new Date().toISOString(), checks, readyServers, totalServers: servers.length, savedKeys };
return json(body, { headers: { 'cache-control': 'no-store' } });
return json({ checkedAt: new Date().toISOString(), checks, readyServers, totalServers: servers.length, savedKeys } satisfies SystemStatus, { headers: { 'cache-control': 'no-store' } });
}
src/routes/servers/[name]/+page.svelte
+6 -222
View File
@@ -29,7 +29,6 @@
import ChatPanel from '$ui/components/ChatPanel.svelte';
import { gametypeLabel } from '$ui/gametype.js';
import { decodeRconError, rconFixHint } from '$ui/rconError.js';
import { komodoFixHint, type KomodoPortFailure } from '$ui/komodoError.js';
import Card from '$ui/components/primitives/Card.svelte';
import Badge from '$ui/components/primitives/Badge.svelte';
@@ -41,29 +40,6 @@
import Breadcrumbs from '$ui/components/primitives/Breadcrumbs.svelte';
import EmptyState from '$ui/components/primitives/EmptyState.svelte';
interface KomodoProbeResult {
ok: boolean;
port?: number;
reason?: KomodoPortFailure;
detail?: string;
httpStatus?: number;
snippet?: string;
url?: string;
}
type NetOutcome = 'connected' | 'refused' | 'timeout' | 'dns-fail' | 'unreachable' | 'error';
interface NetTrial {
label: string; host: string; port: number;
resolvedIp: string | null; outcome: NetOutcome; elapsedMs: number; detail?: string;
}
interface NetDiagnosis {
ok: boolean;
config?: {
komodoAddress: string | null; komodoHost: string | null; komodoPort: number | null;
rconHost: string; probeTimeoutMs: number;
};
results?: NetTrial[];
error?: string;
}
interface RconProbeResult {
ok: boolean;
endpoint?: { host: string; port: number } | null;
@@ -82,7 +58,6 @@
native_instance_id?: string | null; host_id?: string | null; instance_generation?: number | null; game_library_id?: string | null;
host?: import('$lib/hosts.js').Host | null;
network?: { bindAddress: string; localPort: number; publicAddress: string | null; publicPort: number | null } | null;
komodo_type: string; komodo_name: string; komodo_server: string | null;
game_port: number | null; rcon_host: string | null;
game: string | null; map: string | null;
last_players: number | null; last_checked_at: string | null;
@@ -98,9 +73,6 @@
relPath: string; kind?: string; scriptVersionId: number | null;
sha256: string; size: number; updatedAt: string; source: string | null;
}
interface Integrations {
rcon: boolean; komodo: boolean; rconTimeoutMs: number; defaultRconHost: string;
}
interface DeploymentRow {
id: string; action: string; started_at: string; finished_at: string | null;
servers: Array<{ server: string; restart_state: string; players: number | null; error: string | null; files_changed: number }>;
@@ -108,7 +80,7 @@
let name = $derived($page.params.name ?? '');
const detail = $derived(createFleetStore(name));
const { fleet, fleetServers, fleetError, fleetIntegrations, refreshFleet } = $derived(detail);
const { fleet, fleetServers, fleetError, refreshFleet } = $derived(detail);
$effect(() => {
detail.hydrateFleet(data.serverSnapshot);
return detail.startFleetMonitor();
@@ -117,13 +89,8 @@
const visibleFleet = $derived($fleet.receivedAt ? $fleetServers : data.serverSnapshot.servers);
let server = $derived((visibleFleet.find(s => s.name === name) as Server | undefined) ?? null);
const native = $derived(!!server?.native_instance_id);
let integrations = $derived($fleetIntegrations ?? data.serverSnapshot.integrations);
let history = $state<DeploymentRow[]>([]);
let inventory = $state<InventoryEntry[]>([]);
let komodoProbe = $state<KomodoProbeResult | null>(null);
let probing = $state(false);
let netDiagnosis = $state<NetDiagnosis | null>(null);
let diagnosing = $state(false);
let rconProbe = $state<RconProbeResult | null>(null);
let probingRcon = $state(false);
let destroyOpen = $state(false);
@@ -171,7 +138,7 @@
tabLoading = tab === 'history' || tab === 'inventory';
return pollVisible(signal => loadTab(currentName, tab, signal), 10000);
});
$effect(() => { name; komodoProbe = null; rconProbe = null; netDiagnosis = null; });
$effect(() => { name; rconProbe = null; });
const fresh = $derived(server ? freshServer(server, $uiNow) : false);
let effectivePlayers = $derived(fresh ? server?.last_players ?? null : null);
@@ -214,29 +181,6 @@
return `${ep.host}:${ep.port}`;
}
async function probeKomodo(): Promise<void> {
if (probing || !name) return;
probing = true; komodoProbe = null;
try {
const res = await fetch(`/api/servers/${encodeURIComponent(name)}/probe-komodo`, { method: 'POST' });
komodoProbe = (await res.json()) as KomodoProbeResult;
if (komodoProbe.ok && server) {
await refreshFleet();
}
} catch (err) {
komodoProbe = { ok: false, reason: 'unreachable', detail: (err as Error).message };
} finally { probing = false; }
}
async function diagnoseNetwork(): Promise<void> {
if (diagnosing || !name) return;
diagnosing = true; netDiagnosis = null;
try {
const res = await fetch(`/api/servers/${encodeURIComponent(name)}/diagnose-network`, { method: 'POST' });
netDiagnosis = (await res.json()) as NetDiagnosis;
} catch (err) {
netDiagnosis = { ok: false, error: (err as Error).message };
} finally { diagnosing = false; }
}
async function probeRcon(): Promise<void> {
if (probingRcon || !name) return;
probingRcon = true; rconProbe = null;
@@ -337,8 +281,8 @@
{#if $fleetError}<div class="ui-error" role="alert">{$fleetError}<Button size="sm" onclick={() => refreshFleet(true)}>Retry</Button></div>{/if}
{#if hasDiagnosticIssue}
<section class="connection-issue" aria-label="Server connection issue">
<div><strong>{effectiveHealth === 'stale' ? 'Server report is outdated' : 'Server is not responding'}</strong><p>Player count is unknown. {effectiveSeen ? `Last successful report ${ago(effectiveSeen, $uiNow)}.` : 'No successful report has been received.'} {native && !server.host?.connected ? 'The host agent is offline; the game may still be running.' : !native && integrations && !integrations.rcon ? 'RCON is not configured.' : ''}</p></div>
<div class="issue-actions"><Button size="sm" onclick={() => changeTab('diagnostics')}>Diagnose connection</Button><Button size="sm" variant="ghost" onclick={() => changeTab('logs')}>Open logs</Button>{#if native && server.host_id}<Button size="sm" href={`/hosts/${server.host_id}`}>Check host</Button>{:else if integrations && (!integrations.rcon || !integrations.komodo)}<Button size="sm" href="/settings">Check system setup</Button>{/if}</div>
<div><strong>{effectiveHealth === 'stale' ? 'Server report is outdated' : 'Server is not responding'}</strong><p>Player count is unknown. {effectiveSeen ? `Last successful report ${ago(effectiveSeen, $uiNow)}.` : 'No successful report has been received.'} {native && !server.host?.connected ? 'The host agent is offline; the game may still be running.' : ''}</p></div>
<div class="issue-actions"><Button size="sm" onclick={() => changeTab('diagnostics')}>Diagnose connection</Button><Button size="sm" variant="ghost" onclick={() => changeTab('logs')}>Open logs</Button>{#if native && server.host_id}<Button size="sm" href={`/hosts/${server.host_id}`}>Check host</Button>{:else}<Button size="sm" href="/settings">Check system setup</Button>{/if}</div>
</section>
{/if}
<div class="tabs-row">
@@ -410,13 +354,6 @@
<p class="sub">Current player count: {server.status_source === 'telemetry' ? 'game telemetry' : server.status_source === 'rcon' ? 'RCON' : 'unavailable'}. RCON probes below show that connection's separate results.</p>
{#if native}
<div class="diag-grid"><div class="kv"><span class="k">Game connection</span><span class="v">RCON runs locally through the host agent</span></div><div class="kv"><span class="k">Host agent</span><span class="v {server.host?.connected ? 'ok' : 'bad'}">{server.host?.connectionState ?? 'unknown'}</span></div><div class="kv"><span class="k">Last heartbeat</span><span class="v">{server.host?.lastHeartbeat ? ago(server.host.lastHeartbeat, $uiNow) : 'Not received'}</span></div></div>
{:else if integrations}
<div class="diag-grid">
<div class="kv"><span class="k">Endpoint</span><span class="v mono">{endpointLabel(effectiveEndpoint)}</span></div>
<div class="kv"><span class="k">RCON password</span><span class="v mono {integrations.rcon ? 'ok' : 'bad'}">{integrations.rcon ? 'configured' : 'missing'}</span></div>
<div class="kv"><span class="k">Komodo</span><span class="v mono {integrations.komodo ? 'ok' : 'bad'}">{integrations.komodo ? 'configured' : 'missing'}</span></div>
<div class="kv"><span class="k">Timeout</span><span class="v mono">{integrations.rconTimeoutMs} ms</span></div>
</div>
{/if}
{#if effectiveReason}
@@ -445,14 +382,7 @@
{native ? 'Probe RCON through agent' : 'Probe RCON'}
</Button>
{/if}
{#if !native && !effectiveEndpoint && integrations?.komodo}
<Button variant="secondary" size="sm" onclick={probeKomodo} disabled={probing} loading={probing}>
Probe Komodo
</Button>
{/if}
{#if !native}<Button variant="secondary" size="sm" onclick={diagnoseNetwork} disabled={diagnosing} loading={diagnosing}>
Diagnose network
</Button>{:else if server.host_id}<Button variant="secondary" size="sm" href={`/hosts/${server.host_id}`}>Inspect host</Button>{/if}
{#if server.host_id}<Button variant="secondary" size="sm" href={`/hosts/${server.host_id}`}>Inspect host</Button>{/if}
</div>
{#if rconProbe?.ok}
@@ -466,91 +396,12 @@
</div>
{/if}
{#if !native && netDiagnosis}
{#if netDiagnosis.ok && netDiagnosis.results}
<!-- svelte-ignore a11y_no_noninteractive_tabindex (Keyboard users need to scroll the table.) -->
<div class="table-scroll" role="region" aria-label="Network diagnostics" tabindex="0">
<table class="net-table">
<thead>
<tr>
<th>Trial</th><th>Resolved</th><th>Outcome</th>
<th class="num">Elapsed</th><th>Detail</th>
</tr>
</thead>
<tbody>
{#each netDiagnosis.results as r (r.label)}
<tr>
<td class="mono">{r.label}</td>
<td class="mono">{r.resolvedIp ?? '—'}</td>
<td>
<Badge
variant={r.outcome === 'connected' ? 'online' :
r.outcome === 'refused' ? 'warning' :
'danger'}
uppercase
>{r.outcome}</Badge>
</td>
<td class="mono num">{r.elapsedMs} ms</td>
<td class="mono detail-cell">{r.detail ?? ''}</td>
</tr>
{/each}
</tbody>
</table>
</div>
{#if netDiagnosis.config}
<div class="net-meta">
KOMODO_ADDRESS = <code>{netDiagnosis.config.komodoAddress ?? '(unset)'}</code>
&nbsp;·&nbsp; RCON_HOST = <code>{netDiagnosis.config.rconHost}</code>
&nbsp;·&nbsp; probe timeout = <code>{netDiagnosis.config.probeTimeoutMs} ms</code>
</div>
{/if}
{:else}
<div class="alert alert-danger">
<Badge variant="danger" uppercase>Error</Badge>
<span class="alert-detail">{netDiagnosis.error ?? 'network diagnostic failed'}</span>
</div>
{/if}
{/if}
{#if !native && komodoProbe}
{#if komodoProbe.ok}
<div class="alert alert-ok">
<Badge variant="online" uppercase>Resolved</Badge>
<span class="alert-detail">
Port <strong>{komodoProbe.port}</strong> cached — RCON will retry on the next tick.
</span>
</div>
{#if komodoProbe.snippet}
<details class="snippet-details">
<summary>Show response snippet</summary>
<pre class="snippet">{komodoProbe.snippet}</pre>
</details>
{/if}
{:else}
<div class="alert alert-danger">
<Badge variant="danger" uppercase>{komodoProbe.reason}</Badge>
<span class="alert-detail">
{komodoProbe.detail}{#if komodoProbe.httpStatus} · HTTP {komodoProbe.httpStatus}{/if}
</span>
</div>
{#if komodoProbe.url}
<div class="url-row"><span class="k">Tried</span><code>{komodoProbe.url}</code></div>
{/if}
{#if komodoProbe.reason}
<div class="hint">{komodoFixHint(komodoProbe.reason, komodoProbe.httpStatus)}</div>
{/if}
{#if komodoProbe.snippet}
<pre class="snippet">{komodoProbe.snippet}</pre>
{/if}
{/if}
{/if}
<div class="panel-head divider">
<h2>Configuration</h2>
</div>
<div class="diag-grid">
{#if native}<div class="kv"><span class="k">Host</span><span class="v"><a href={`/hosts/${server.host_id}`}>{server.host?.name ?? server.host_id}</a></span></div><div class="kv"><span class="k">Instance</span><span class="v mono">{server.native_instance_id}</span></div><div class="kv"><span class="k">Generation</span><span class="v mono">{server.instance_generation}</span></div><div class="kv"><span class="k">Game-file version</span><span class="v mono"><a href="/game-files">{server.game_library_id}</a></span></div>{#if server.network}<div class="kv"><span class="k">Local UDP binding</span><span class="v mono">{server.network.bindAddress}:{server.network.localPort}</span></div><div class="kv"><span class="k">Public / NAT endpoint</span><span class="v mono">{server.network.publicAddress || server.host?.publicAddress || server.host?.hostname || 'Not configured'}:{server.network.publicPort ?? server.network.localPort}</span></div>{/if}
{:else}<div class="kv"><span class="k">Komodo</span><span class="v mono">{server.komodo_type} · {server.komodo_name}</span></div><div class="kv"><span class="k">RCON override</span><span class="v mono">{server.rcon_host ?? '(default)'}</span></div>{/if}
{/if}
<div class="kv"><span class="k">Game port</span><span class="v mono">{server.game_port ?? '(unresolved)'}</span></div>
<div class="kv"><span class="k">Game</span><span class="v mono">{server.game ?? '(unset)'}</span></div>
<div class="kv"><span class="k">Map</span><span class="v mono">{server.map ?? '(unset)'}</span></div>
@@ -777,19 +628,6 @@
margin-top: var(--space-4);
}
.url-row {
display: flex;
gap: var(--space-2);
margin-top: var(--space-2);
padding: var(--space-2) var(--space-3);
background: var(--surface-sunken);
border-radius: var(--radius-md);
font-family: var(--font-mono);
font-size: var(--text-xs);
}
.url-row .k { color: var(--text-tertiary); }
.url-row code { color: var(--text-primary); word-break: break-all; }
/* Snippet details */
.snippet-details {
margin-top: var(--space-3);
@@ -818,60 +656,6 @@
word-break: break-all;
}
/* Network table */
.table-scroll { max-width: 100%; overflow-x: auto; }
.table-scroll:focus-visible { box-shadow: var(--focus-ring); }
.net-table {
min-width: 600px;
width: 100%;
margin-top: var(--space-4);
border-collapse: separate;
border-spacing: 0;
background: var(--surface-sunken);
border: 1px solid var(--border-subtle);
border-radius: var(--radius-md);
overflow: hidden;
font-size: var(--text-sm);
}
.net-table thead th {
text-align: left;
padding: var(--space-3) var(--space-4);
font-family: var(--font-body);
font-size: var(--text-xs);
font-weight: var(--weight-semibold);
text-transform: uppercase;
letter-spacing: var(--tracking-label);
color: var(--text-tertiary);
border-bottom: 1px solid var(--border-subtle);
}
.net-table tbody td {
padding: var(--space-3) var(--space-4);
border-bottom: 1px solid var(--border-subtle);
color: var(--text-primary);
vertical-align: top;
}
.net-table tbody tr:last-child td { border-bottom: none; }
.net-table .num { text-align: right; }
.net-table .detail-cell {
color: var(--text-tertiary);
font-size: var(--text-xs);
word-break: break-word;
max-width: 320px;
}
.net-meta {
margin-top: var(--space-3);
font-family: var(--font-mono);
font-size: var(--text-xs);
color: var(--text-tertiary);
}
.net-meta code {
color: var(--text-primary);
background: var(--surface-sunken);
padding: 2px 6px;
border-radius: var(--radius-sm);
}
/* Inventory */
.inv-list {
display: flex;