Remove Komodo backend in favor of native hosts
This commit is contained in:
@@ -33,31 +33,7 @@ export const config = {
|
||||
betaArchiveDir: req('BETA_ARCHIVE_DIR', '/root/zombies/beta-archive'),
|
||||
dbPath: req('DB_PATH', './data/cherry.db'),
|
||||
httpTimeoutMs: num('HTTP_TIMEOUT_MS', 10_000),
|
||||
komodo: {
|
||||
address: env.KOMODO_ADDRESS ?? '',
|
||||
apiKey: env.KOMODO_API_KEY ?? '',
|
||||
apiSecret: env.KOMODO_API_SECRET ?? '',
|
||||
// Komodo server (host) new deployments land on. Required for the
|
||||
// deploy flow — Komodo needs to know which machine to schedule the
|
||||
// container onto. Look up the exact name in Komodo's UI (Servers page).
|
||||
targetServer: env.KOMODO_TARGET_SERVER ?? '',
|
||||
// Default image for freshly-provisioned T5 and T6 zombie servers.
|
||||
// Overridable per-deploy in the UI later if we ever need to.
|
||||
defaultImage: env.PLUTAINER_IMAGE ?? 'ghcr.io/ayymoss/plutainer:latest',
|
||||
defaultNetwork: env.PLUTAINER_NETWORK ?? 'komodo_default',
|
||||
// Named docker volume that holds the read-only game files layer.
|
||||
gameFilesVolume: env.PLUTAINER_GAMEFILES_VOLUME ?? 't5_game_files',
|
||||
t6GameFilesVolume: env.PLUTAINER_T6_GAMEFILES_VOLUME ?? 't6_game_files'
|
||||
},
|
||||
// Shared across every newly-provisioned server. Kept in sync with the
|
||||
// fleet's RCON_PASSWORD (rcon.password) so freshly-created servers respond
|
||||
// to the same rcon secret the sweep already uses.
|
||||
// Port range Electric Cherry auto-allocates from when creating a new
|
||||
// deployment. Update this when the host's firewall opens more.
|
||||
portRange: {
|
||||
min: num('SERVER_PORT_MIN', 28960),
|
||||
max: num('SERVER_PORT_MAX', 28980)
|
||||
},
|
||||
defaultRuntimeImage: env.PLUTAINER_IMAGE ?? 'ghcr.io/ayymoss/plutainer:latest',
|
||||
rcon: {
|
||||
// Host every RCON packet is sent to. Defaults to the Docker
|
||||
// host-gateway alias so the electric-cherry container can reach
|
||||
@@ -70,10 +46,6 @@ export const config = {
|
||||
}
|
||||
} as const;
|
||||
|
||||
export function komodoConfigured(): boolean {
|
||||
return !!(config.komodo.address && config.komodo.apiKey && config.komodo.apiSecret);
|
||||
}
|
||||
|
||||
export function rconConfigured(): boolean {
|
||||
return !!config.rcon.password;
|
||||
}
|
||||
|
||||
@@ -24,7 +24,6 @@ const SERVERS_COLUMNS: Array<{ name: string; ddl: string }> = [
|
||||
{ name: 'desired_state', ddl: "ALTER TABLE servers ADD COLUMN desired_state TEXT NOT NULL DEFAULT 'running' CHECK (desired_state IN ('running','stopped'))" },
|
||||
// Base columns that shipped in the very first schema stay here for parity
|
||||
// with fresh installs — no ALTER is issued if they're already present.
|
||||
{ name: 'komodo_server', ddl: 'ALTER TABLE servers ADD COLUMN komodo_server TEXT' },
|
||||
{ name: 'game_port', ddl: 'ALTER TABLE servers ADD COLUMN game_port INTEGER' },
|
||||
{ name: 'rcon_host', ddl: 'ALTER TABLE servers ADD COLUMN rcon_host TEXT' },
|
||||
{ name: 'last_players', ddl: 'ALTER TABLE servers ADD COLUMN last_players INTEGER' },
|
||||
@@ -65,6 +64,14 @@ function migrateServers(handle: Database.Database): void {
|
||||
handle.exec(col.ddl);
|
||||
}
|
||||
}
|
||||
// Retire external-backend registry fields, including required fields that
|
||||
// would otherwise prevent native inserts into an upgraded database.
|
||||
const retired = new Set(['komodo_type', 'komodo_name', 'komodo_server']);
|
||||
handle.transaction(() => {
|
||||
for (const column of rows) {
|
||||
if (retired.has(column.name)) handle.exec(`ALTER TABLE servers DROP COLUMN "${column.name.replaceAll('"', '""')}"`);
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
// Additive columns on tables the script model owns. Same pattern as
|
||||
@@ -286,7 +293,7 @@ function migrateWatchdogConstraints(handle: Database.Database): void {
|
||||
function backfillDesiredState(handle: Database.Database): void {
|
||||
if (handle.prepare("SELECT 1 FROM settings WHERE key = 'watchdog_desired_state_v1'").get()) return;
|
||||
handle.transaction(() => {
|
||||
// A failed stop may have reached Komodo. Preserve that explicit intent.
|
||||
// A failed stop may have reached the host agent. Preserve that explicit intent.
|
||||
handle.exec(`UPDATE servers SET desired_state = 'stopped' WHERE 'stop' = (
|
||||
SELECT action FROM server_deployments d WHERE d.server = servers.name AND d.state != 'cancelled'
|
||||
AND d.action IN ('create','start','restart','rebuild','stop')
|
||||
|
||||
@@ -63,13 +63,10 @@ CREATE INDEX IF NOT EXISTS idx_leaderboard_player_matches ON leaderboard_match_p
|
||||
|
||||
CREATE TABLE IF NOT EXISTS servers (
|
||||
name TEXT PRIMARY KEY,
|
||||
komodo_type TEXT NOT NULL DEFAULT 'deployment', -- deployment|stack|container
|
||||
komodo_name TEXT NOT NULL,
|
||||
komodo_server TEXT,
|
||||
container_checked_at TEXT,
|
||||
container_state TEXT, -- last observed Komodo runtime state
|
||||
container_state TEXT, -- last observed host-agent runtime state
|
||||
desired_state TEXT NOT NULL DEFAULT 'running' CHECK (desired_state IN ('running','stopped')),
|
||||
game_port INTEGER, -- UDP RCON port; lazy-filled from Komodo
|
||||
game_port INTEGER, -- UDP game port assigned on the host
|
||||
rcon_host TEXT, -- optional per-server override of RCON_HOST
|
||||
game TEXT, -- 't5' etc.; static per-server (no map rotation)
|
||||
map TEXT, -- 'zombie_theater' etc.; static per-server
|
||||
|
||||
@@ -376,7 +376,7 @@ export async function openBeta(input: OpenBetaInput): Promise<OpenBetaResult> {
|
||||
const removal = db().prepare(`SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL
|
||||
AND action IN ('destroy', 'rebuild') AND (state IN ('waiting', 'queued', 'running', 'awaiting-health')
|
||||
OR (state IN ('failed', 'interrupted') AND stage IN (
|
||||
'Stopping and removing container', 'Deleting Komodo deployment', 'Removing server files and registry')))
|
||||
'Stopping and removing container', 'Removing server files and registry')))
|
||||
LIMIT 1`).get(name);
|
||||
if (removal) throw new BetaError(`${name}: cancel or finish the pending destruction or rebuild before opening a beta.`, 'conflict');
|
||||
serverRows.push({ name, game: row.game, map: row.map });
|
||||
|
||||
@@ -2,16 +2,14 @@ import { fleetSettings } from '../settings.js';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import type { CollectionDetail, CollectionMode, CollectionPreview, CollectionSummary } from '$lib/deployment-plan.js';
|
||||
import { db } from '../db/index.js';
|
||||
import { config, komodoConfigured, rconConfigured } from '../config.js';
|
||||
import { blueprintEntries, summarizeRecipe, type BlueprintEntry } from './launch-recipes.js';
|
||||
import { creationQueue, destroyServer, nextAvailablePort, queueCreateServer } from '../provisioning/index.js';
|
||||
import { creationQueue, destroyServer } from '../provisioning/index.js';
|
||||
import { deploymentStage, finishDeployment, recordDeploymentEvent } from './deployments.js';
|
||||
import { housePartyTargets } from './house-party.js';
|
||||
import { serverObservation } from './status.js';
|
||||
import { activeBetaRemovalBlocker, getServer } from './servers.js';
|
||||
import { parseServerName, zombieMapBySlug, zombieMaps } from './map-aliases.js';
|
||||
import { nativeLaunchSelection, queueNativeCreationIntent } from '../provisioning/native.js';
|
||||
import { listHosts } from '../hosts/index.js';
|
||||
import { nativeDeployment, nativeInstance } from '../backends/native.js';
|
||||
import { launchAvailability } from '../hosts/availability.js';
|
||||
|
||||
@@ -71,7 +69,7 @@ function collectionPlan(entries: BlueprintEntry[], mode: CollectionMode, include
|
||||
return { counts, retainedServers, removals };
|
||||
}
|
||||
|
||||
function allocate(entries: BlueprintEntry[], releasingServers: string[] = [], counts = coverage(entries)): CollectionPreview['servers'] {
|
||||
function allocate(entries: BlueprintEntry[], counts = coverage(entries)): CollectionPreview['servers'] {
|
||||
const occupied = db().prepare(`SELECT name FROM servers UNION
|
||||
SELECT j.server AS name FROM server_creation_jobs j JOIN server_deployments d ON d.id=j.deployment_id
|
||||
WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued','running','awaiting-health')
|
||||
@@ -85,16 +83,14 @@ function allocate(entries: BlueprintEntry[], releasingServers: string[] = [], co
|
||||
taken.add(Number(match[2])); lobbies.set(match[1], taken);
|
||||
}
|
||||
const savedKeys = new Set((db().prepare("SELECT server_id FROM server_keys WHERE length(trim(pluto_key))>0").all() as { server_id: string }[]).map(row => row.server_id));
|
||||
const ports: number[] = [];
|
||||
return entries.flatMap((entry, entryIndex) => Array.from({ length: counts[entryIndex].missing }, () => {
|
||||
const base = `${entry.recipe.game}_zombie_${entry.recipe.mapSlug}`;
|
||||
const taken = lobbies.get(base) ?? new Set<number>();
|
||||
const lobby = Array.from({ length: 20 }, (_, index) => index + 1).find(number => !taken.has(number)) ?? null;
|
||||
const name = lobby === null ? null : `${base}_${lobby}`;
|
||||
const native = entry.recipe.native;
|
||||
const port = native ? null : nextAvailablePort(ports, releasingServers);
|
||||
const port = null;
|
||||
if (lobby !== null) { taken.add(lobby); lobbies.set(base, taken); }
|
||||
if (port !== null) ports.push(port);
|
||||
return { entryIndex, name, port, ...(native ? { native } : {}), hasSavedKey: !!name && savedKeys.has(name) };
|
||||
}));
|
||||
}
|
||||
@@ -102,13 +98,8 @@ function allocate(entries: BlueprintEntry[], releasingServers: string[] = [], co
|
||||
function blockersFor(servers: CollectionPreview['servers'], entries: BlueprintEntry[] = []): string[] {
|
||||
const blockers: string[] = [];
|
||||
if (!servers.length) return blockers;
|
||||
const legacy = servers.filter(server => !server.native && !entries[server.entryIndex]?.recipe.native);
|
||||
if (legacy.length) {
|
||||
if (!komodoConfigured() || !config.komodo.targetServer) blockers.push('Select and prepare a host before launching.');
|
||||
if (!rconConfigured()) blockers.push('Configure RCON_PASSWORD so every new server can be verified.');
|
||||
}
|
||||
if (servers.some(server => !server.native && !entries[server.entryIndex]?.recipe.native)) blockers.push('Select and prepare a host before launching.');
|
||||
if (servers.some(server => !server.name)) blockers.push('There are not enough free lobbies for one or more maps (20 per map). Reduce their counts or free capacity.');
|
||||
if (legacy.some(server => server.port === null)) blockers.push(`There are not enough free game ports in ${config.portRange.min}–${config.portRange.max} for the whole collection.`);
|
||||
return blockers;
|
||||
}
|
||||
|
||||
@@ -120,7 +111,7 @@ function removalBlockersFor(removals: CollectionPreview['removals']): string[] {
|
||||
AND action='create' AND state IN ('queued','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(name) as { id: string } | undefined;
|
||||
return !!creation && nativeDeployment(creation.id);
|
||||
};
|
||||
if (removals.some(server => !native(server.name)) && (!komodoConfigured() || !rconConfigured())) blockers.push('Configure the legacy backend before removing its servers.');
|
||||
if (removals.some(server => !native(server.name))) blockers.push('Every server to remove must have a native host instance.');
|
||||
for (const server of removals) {
|
||||
const blocker = activeBetaRemovalBlocker(server.name);
|
||||
if (blocker) blockers.push(blocker);
|
||||
@@ -133,15 +124,13 @@ export function previewCollection(blueprintId: unknown, mode: unknown = 'fill',
|
||||
if (mode !== 'fill' && mode !== 'exact') throw new Error('Choose fill or exact collection mode.');
|
||||
return db().transaction((): CollectionPreview => {
|
||||
const { entries, blueprintName } = blueprintEntries(blueprintId);
|
||||
if (selection?.hostId !== undefined || !config.komodo.address || listHosts().length > 0) {
|
||||
const libraries = selection?.libraryIds as Record<string, unknown> | undefined;
|
||||
for (const entry of entries) {
|
||||
entry.recipe.native = nativeLaunchSelection({ ...selection, libraryId: libraries?.[entry.recipe.game] ?? selection?.libraryId }, entry.recipe.game);
|
||||
entry.recipe.image = entry.recipe.native.image!;
|
||||
}
|
||||
const libraries = selection?.libraryIds as Record<string, unknown> | undefined;
|
||||
for (const entry of entries) {
|
||||
entry.recipe.native = nativeLaunchSelection({ ...selection, libraryId: libraries?.[entry.recipe.game] ?? selection?.libraryId }, entry.recipe.game);
|
||||
entry.recipe.image = entry.recipe.native.image!;
|
||||
}
|
||||
const { counts, retainedServers, removals } = collectionPlan(entries, mode);
|
||||
const servers = allocate(entries, removals.map(server => server.name), counts);
|
||||
const servers = allocate(entries, counts);
|
||||
const blockers = [...blockersFor(servers, entries), ...removalBlockersFor(removals)];
|
||||
const selections = servers.flatMap(server => server.native ? [server.native] : []);
|
||||
const availability = selections.length ? launchAvailability(selections) : {};
|
||||
@@ -176,16 +165,12 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
|
||||
throw new Error('The fleet changed. Nothing was queued. Refresh the preview and review the servers to keep and remove.');
|
||||
}
|
||||
const { counts, retainedServers, removals } = collectionPlan(entries, preview.mode, false);
|
||||
const releasingServers = removals.map(server => server.name);
|
||||
const nativeReview = reviewed.every(server => server.native || entries[server.entryIndex]?.recipe.native);
|
||||
// Native Apply accepts the reviewed logical identities. Capacity and
|
||||
// physical ports can change while this request waits in the queue.
|
||||
const servers = nativeReview ? reviewed : allocate(entries, releasingServers, counts);
|
||||
const servers = reviewed;
|
||||
const validatedSelections = new Set<string>();
|
||||
for (const server of servers) {
|
||||
const recipe = entries[server.entryIndex].recipe;
|
||||
const selection = server.native ?? recipe.native;
|
||||
if (!selection) continue;
|
||||
if (!selection) throw new Error('Select a native host before launching. Refresh the preview.');
|
||||
const identity = JSON.stringify([recipe.game, selection]);
|
||||
if (validatedSelections.has(identity)) continue;
|
||||
// Recheck saved configuration once per selection without allocating
|
||||
@@ -195,9 +180,8 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
|
||||
}
|
||||
const blockers = [...blockersFor(servers, entries), ...removalBlockersFor(removals)];
|
||||
if (blockers.length) throw new Error(blockers.join(' '));
|
||||
if (entries.some((_entry, index) => counts[index].missing !== reviewed.filter(server => server.entryIndex === index).length)
|
||||
|| servers.length !== reviewed.length || servers.some((server, i) => server.entryIndex !== reviewed[i]?.entryIndex || server.name !== reviewed[i]?.name || (!server.native && server.port !== reviewed[i]?.port))) {
|
||||
throw new Error('Fleet counts, lobby or port availability changed. Nothing was queued. Refresh the preview and review each server key.');
|
||||
if (entries.some((_entry, index) => counts[index].missing !== reviewed.filter(server => server.entryIndex === index).length)) {
|
||||
throw new Error('Fleet counts changed. Nothing was queued. Refresh the preview and review each server key.');
|
||||
}
|
||||
const expected = new Set(servers.map(server => server.name));
|
||||
for (const [name, value] of Object.entries(keys)) {
|
||||
@@ -207,10 +191,7 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
|
||||
const collectionId = randomUUID();
|
||||
db().prepare('INSERT INTO server_launch_collections (id, name, created_at, mode, retained_servers) VALUES (?, ?, ?, ?, ?)')
|
||||
.run(collectionId, preview.blueprint_name, new Date().toISOString(), preview.mode, JSON.stringify(retainedServers));
|
||||
const removalJobs = removals.map((server, position) => {
|
||||
const ports = (db().prepare(`SELECT game_port AS port FROM servers WHERE name = ? UNION
|
||||
SELECT j.port FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
|
||||
WHERE j.server = ? AND d.server_deleted_at IS NULL AND d.state IN ('queued', 'running', 'awaiting-health')`).all(server.name, server.name) as { port: number | null }[]).map(row => row.port);
|
||||
removals.forEach((server, position) => {
|
||||
const queued = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'create' AND state = 'queued'").get(server.name) as { id: string } | undefined;
|
||||
let id: string;
|
||||
if (queued && !getServer(server.name)) {
|
||||
@@ -230,27 +211,17 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
|
||||
id = result.deploymentId;
|
||||
}
|
||||
db().prepare('INSERT INTO server_collection_removals (collection_id, deployment_id, position) VALUES (?, ?, ?)').run(collectionId, id, position);
|
||||
return { id, ports };
|
||||
});
|
||||
servers.forEach((server, index) => {
|
||||
const recipe = { ...entries[server.entryIndex].recipe, ...(server.native ? { native: server.native } : {}) };
|
||||
let result;
|
||||
if (recipe.native) result = queueNativeCreationIntent({ server: server.name!, game: recipe.game, mapSlug: recipe.mapSlug,
|
||||
if (!recipe.native) throw new Error('Select a native host before launching. Refresh the preview.');
|
||||
const result = queueNativeCreationIntent({ server: server.name!, game: recipe.game, mapSlug: recipe.mapSlug,
|
||||
selection: recipe.native, recipe, plutoKey: keys[server.name!] as string | undefined, createdBy: 'ui', blueprintName: preview.blueprint_name });
|
||||
else {
|
||||
const childPreviewId = randomUUID();
|
||||
db().prepare('INSERT INTO server_launch_previews (id, expires_at, server, port, recipe, blueprint_name) VALUES (?, ?, ?, ?, ?, ?)')
|
||||
.run(childPreviewId, preview.expires_at, server.name, server.port, JSON.stringify(recipe), preview.blueprint_name);
|
||||
result = queueCreateServer({ game: recipe.game, gameMode: 'zombie', mapSlug: recipe.mapSlug,
|
||||
expectedName: server.name!, previewId: childPreviewId, plutoKey: keys[server.name!] as string | undefined, createdBy: 'ui' }, releasingServers);
|
||||
}
|
||||
// Throw so the outer transaction also reverses all earlier keys and children.
|
||||
if (!result.ok) throw new Error(`${server.name}: ${result.message} Nothing was queued.`);
|
||||
db().prepare('INSERT INTO server_collection_members (collection_id, deployment_id, position) VALUES (?, ?, ?)')
|
||||
.run(collectionId, result.deploymentId, index);
|
||||
for (const removal of recipe.native ? [] : removalJobs.filter(removal => removal.ports.includes(server.port))) {
|
||||
db().prepare('INSERT INTO server_creation_dependencies (deployment_id, removal_id) VALUES (?, ?)').run(result.deploymentId, removal.id);
|
||||
}
|
||||
|
||||
});
|
||||
db().prepare('UPDATE server_collection_previews SET collection_id = ? WHERE id = ?').run(collectionId, preview.id);
|
||||
return { ok: true as const, collectionId };
|
||||
|
||||
@@ -3,7 +3,7 @@ import { config } from '../config.js';
|
||||
import { db } from '../db/index.js';
|
||||
import { getHost, queueHostOperation, waitHostOperationStatus } from '../hosts/index.js';
|
||||
import { nativeInstance, type NativeInstance } from '../backends/native.js';
|
||||
import { komodoInspectHealthy } from '../integrations/komodo.js';
|
||||
import { nativeHealth } from '../backends/native.js';
|
||||
import { rconStatusResult, agentRconStatusResult, type AgentRconReply } from '../integrations/rcon.js';
|
||||
import { withOperationSource } from '../hosts/operation-context.js';
|
||||
import { getServer } from './servers.js';
|
||||
@@ -39,7 +39,6 @@ function identity(server: string, probe: Probe): Identity {
|
||||
return {
|
||||
key: JSON.stringify([row.created_at, row.desired_state, latestGameSubmission(server), lifecycle,
|
||||
native ? [native.host_id, native.id, native.generation] : null,
|
||||
native ? null : [row.komodo_type, row.komodo_name, row.komodo_server, config.komodo, config.httpTimeoutMs],
|
||||
probe === 'rcon' ? [row.rcon_host, config.rcon] : null]),
|
||||
port: probe === 'rcon' ? native?.port ?? row.game_port : null,
|
||||
checkedAt: probe === 'rcon' ? row.last_checked_at : row.container_checked_at
|
||||
@@ -77,13 +76,7 @@ async function observe<T extends { ok: boolean }>(server: string, fresh: boolean
|
||||
while (entries.size > MAX_SERVERS) entries.delete(entries.keys().next().value!);
|
||||
const pending = Promise.resolve().then(() => request(server)).then(result => {
|
||||
const latest = identity(server, probe);
|
||||
// Legacy RCON may discover and save its first port during this request.
|
||||
// Accept that enrichment only when all other identity fields agree.
|
||||
const endpoint = 'endpoint' in result ? result.endpoint : null;
|
||||
const learnedPort = probe === 'rcon' && !current.port && result.ok
|
||||
&& endpoint !== null && typeof endpoint === 'object' && 'port' in endpoint && endpoint.port === latest.port;
|
||||
if (current.key !== latest.key || (current.port !== latest.port && !learnedPort)
|
||||
|| newerEvidence(latest, entry.startedAt)) {
|
||||
if (!sameIdentity(current, latest) || newerEvidence(latest, entry.startedAt)) {
|
||||
throw new MonitoringChangedError('Server changed during monitoring; the previous observation was discarded.');
|
||||
}
|
||||
const observation = { result, observedAt: new Date().toISOString() };
|
||||
@@ -100,12 +93,12 @@ async function observe<T extends { ok: boolean }>(server: string, fresh: boolean
|
||||
return pending;
|
||||
}
|
||||
|
||||
const inspections = new Map<string, Entry<Awaited<ReturnType<typeof komodoInspectHealthy>>>>();
|
||||
const inspections = new Map<string, Entry<Awaited<ReturnType<typeof nativeHealth>>>>();
|
||||
const rcon = new Map<string, Entry<Awaited<ReturnType<typeof rconStatusResult>>>>();
|
||||
|
||||
interface MonitoringStatus {
|
||||
identity: string;
|
||||
container: Observation<Awaited<ReturnType<typeof komodoInspectHealthy>>>;
|
||||
container: Observation<Awaited<ReturnType<typeof nativeHealth>>>;
|
||||
game?: Observation<Awaited<ReturnType<typeof rconStatusResult>>>;
|
||||
}
|
||||
// Keep failed samples too, independently of the successful-probe cache. Reading
|
||||
@@ -160,7 +153,7 @@ export function monitoringStatus(server: string): MonitoringStatus | null {
|
||||
}
|
||||
|
||||
/** Keep the actual sample time: a later heartbeat must not renew old evidence. */
|
||||
export function heartbeatInspection(server: string): Observation<Awaited<ReturnType<typeof komodoInspectHealthy>>> | null {
|
||||
export function heartbeatInspection(server: string): Observation<Awaited<ReturnType<typeof nativeHealth>>> | null {
|
||||
const row = getServer(server);
|
||||
if (!row?.native_instance_id || !row.host_id) return null;
|
||||
const host = getHost(row.host_id);
|
||||
@@ -242,7 +235,7 @@ export function inspectForMonitoring(server: string, fresh = false) {
|
||||
if (fresh) statuses.delete(server);
|
||||
const heartbeat = !fresh && heartbeatInspection(server);
|
||||
if (heartbeat) return Promise.resolve(heartbeat);
|
||||
return observe(server, fresh, 'inspect', inspections, komodoInspectHealthy);
|
||||
return observe(server, fresh, 'inspect', inspections, nativeHealth);
|
||||
}
|
||||
export function rconForMonitoring(server: string, fresh = false) {
|
||||
if (fresh) statuses.delete(server);
|
||||
|
||||
@@ -105,7 +105,7 @@ function buildRecipe(game: ZombieGame, mapSlug: string, includeContent: boolean)
|
||||
}
|
||||
addConfig('Selected map', MAP_CONFIG_FILE, getServerMapConfig(game, map.canonical, map.slug)!);
|
||||
return { format: 1, game, mapSlug, mapName: map.friendly, expectedMap: map.canonical,
|
||||
capturedAt: new Date().toISOString(), image: config.komodo.defaultImage,
|
||||
capturedAt: new Date().toISOString(), image: config.defaultRuntimeImage,
|
||||
configFile: configFileForGame(game), configSource, files };
|
||||
}
|
||||
|
||||
|
||||
@@ -74,9 +74,6 @@ export interface ServerRow {
|
||||
instance_generation?: number | null;
|
||||
game_library_id?: string | null;
|
||||
name: string;
|
||||
komodo_type: 'deployment' | 'stack' | 'container';
|
||||
komodo_name: string;
|
||||
komodo_server: string | null;
|
||||
container_state: string | null;
|
||||
container_checked_at: string | null;
|
||||
desired_state: 'running' | 'stopped';
|
||||
@@ -112,12 +109,10 @@ export interface ServerRow {
|
||||
export async function seedServerRegistry(): Promise<number> {
|
||||
// Explicit legacy import only; never run as part of application startup.
|
||||
const rows = await serverDirectories();
|
||||
// Under the new naming scheme Komodo entities match EC server names 1:1
|
||||
// (both underscore-separated), so komodo_name = name — no transformation.
|
||||
const insert = db().prepare(
|
||||
`INSERT OR IGNORE INTO servers
|
||||
(name, komodo_type, komodo_name, game, map, game_mode, map_slug, host_dir)
|
||||
VALUES (?, 'deployment', ?, ?, ?, ?, ?, ?)`
|
||||
(name, game, map, game_mode, map_slug, host_dir)
|
||||
VALUES (?, ?, ?, ?, ?, ?)`
|
||||
);
|
||||
// Backfill existing rows where derived columns were never set. game_mode
|
||||
// and map_slug are new; older rows can still surface them from the name.
|
||||
@@ -139,7 +134,7 @@ export async function seedServerRegistry(): Promise<number> {
|
||||
const gameMode = parsed?.gameMode ?? null;
|
||||
const mapSlug = parsed?.mapSlug ?? null;
|
||||
const hostDir = serverDir(name);
|
||||
const res = insert.run(name, name, game, map, gameMode, mapSlug, hostDir);
|
||||
const res = insert.run(name, game, map, gameMode, mapSlug, hostDir);
|
||||
if (res.changes > 0) {
|
||||
n++;
|
||||
} else {
|
||||
@@ -185,9 +180,6 @@ export function setServerField(
|
||||
|
||||
export function setServerFields(name: string, fields: Partial<Record<keyof ServerRow, string | number | null>>): void {
|
||||
const allowed: Array<keyof ServerRow> = [
|
||||
'komodo_type',
|
||||
'komodo_name',
|
||||
'komodo_server',
|
||||
'container_state',
|
||||
'container_checked_at',
|
||||
'game_port',
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { db } from '../db/index.js';
|
||||
import { getServer, listServers } from './servers.js';
|
||||
import { observationReader, serverObservation } from './status.js';
|
||||
import { config, komodoConfigured, rconConfigured } from '../config.js';
|
||||
import { config, rconConfigured } from '../config.js';
|
||||
import { runningVersions } from '../integrations/plutonium.js';
|
||||
import { listHosts } from '../hosts/index.js';
|
||||
import type { NativeInstance } from '../backends/native.js';
|
||||
@@ -33,8 +33,7 @@ export function fleetSnapshot(serverName?: string) {
|
||||
host: s.host_id ? hostsById.get(s.host_id) ?? null : null,
|
||||
network: instance ? { bindAddress: instance.bind_address, localPort: instance.port, publicAddress: instance.public_address, publicPort: instance.public_port } : null,
|
||||
plutonium: versions.get(s.name),
|
||||
// Derived endpoint — same logic as endpointFor() but without asking
|
||||
// Komodo. If we don't have a port yet, endpoint is null.
|
||||
// The host agent owns the game connection.
|
||||
endpoint: instance ? { host: `agent:${instance.host_id}`, port: instance.port } : s.game_port ? { host: s.rcon_host ?? rconHost, port: s.game_port } : null,
|
||||
held: holds.get(s.name) ?? []
|
||||
};
|
||||
@@ -44,11 +43,10 @@ export function fleetSnapshot(serverName?: string) {
|
||||
serverNow: Date.now(),
|
||||
reservedNames: serverName === undefined ? (db().prepare("SELECT j.server FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')").all() as { server: string }[]).map(row => row.server) : [],
|
||||
integrations: {
|
||||
native: hosts.length > 0 || !komodoConfigured(),
|
||||
native: true,
|
||||
hostsReady: hosts.filter(host => host.readiness === 'ready').length,
|
||||
hostsTotal: hosts.length,
|
||||
rcon: rconConfigured(),
|
||||
komodo: komodoConfigured(),
|
||||
rconTimeoutMs: config.rcon.timeoutMs,
|
||||
defaultRconHost: rconHost
|
||||
}
|
||||
|
||||
@@ -59,7 +59,7 @@ export async function probePlayerEvidence(server: string): Promise<PlayerEvidenc
|
||||
return { players: null, source: null, observedAt: new Date().toISOString() };
|
||||
}
|
||||
|
||||
/** Komodo readiness remains a caller prerequisite. Old-process telemetry is excluded. */
|
||||
/** Host agent readiness remains a caller prerequisite. Old-process telemetry is excluded. */
|
||||
export async function probeServerStatus(server: string, after?: string | null): Promise<
|
||||
{ ok: true; players: number; map: string | null; source: 'rcon' | 'telemetry'; observedAt: string; receivedAt?: string } | { ok: false; reason: string; observedAt: string }
|
||||
> {
|
||||
|
||||
@@ -7,7 +7,6 @@ import { telemetryReport, latestGameSubmission } from './status-evidence.js';
|
||||
import { latestGameSubmissions } from './lifecycle-evidence.js';
|
||||
import { db } from '../db/index.js';
|
||||
import { parseGameEvent, type GameTelemetry } from '../../telemetry.js';
|
||||
import { restartRequestInFlight } from '../provisioning/restarts.js';
|
||||
|
||||
interface Evidence { provisioning: boolean; settling: boolean; telemetry: GameTelemetry | null; submission: string | null }
|
||||
|
||||
@@ -33,7 +32,7 @@ export function observationReader() {
|
||||
const operation = operations.get(server.name), restart = restarts.get(server.name);
|
||||
return serverObservation(server, {
|
||||
provisioning: !!operation && ['create','rebuild','start'].includes(operation.action) && ['queued','running','awaiting-health'].includes(operation.state),
|
||||
settling: restartRequestInFlight(server.name) || !!(restart && !restart.healthy_at && Date.now() - Date.parse(restart.requested_at) < 15_000),
|
||||
settling: !!(restart && !restart.healthy_at && Date.now() - Date.parse(restart.requested_at) < 15_000),
|
||||
telemetry: reports.get(server.name) ?? null, submission: submissions.get(server.name) ?? null
|
||||
});
|
||||
};
|
||||
|
||||
@@ -32,7 +32,7 @@ export async function readCapacityCurrent(query: CapacityQuery): Promise<Capacit
|
||||
if (host.decommission) reasons.push('Host is being removed.');
|
||||
if (!host.credentialActive && host.enrolled) reasons.push('Host credentials are inactive.');
|
||||
if (host.readiness !== 'ready') reasons.push(...host.readinessReasons);
|
||||
if (!host.inventory.runtimeImages?.includes(config.komodo.defaultImage)) reasons.push('The default runtime is not ready.');
|
||||
if (!host.inventory.runtimeImages?.includes(config.defaultRuntimeImage)) reasons.push('The default runtime is not ready.');
|
||||
if (!requestedGames.some(game => games.includes(game))) reasons.push('Latest game files for this selection are not ready.');
|
||||
if ((host.inventory.storageAvailable ?? 0) < host.minimumFreeBytes + 1024 ** 3) reasons.push('Storage is below the launch reserve.');
|
||||
const eligible = connected && host.readiness === 'ready' && reasons.length === 0;
|
||||
|
||||
@@ -88,7 +88,7 @@ function targets(host: Host): Target[] {
|
||||
&& operation.sequence > enabling.operation_floor && operation.state === 'completed'
|
||||
&& Date.parse(host.lastHeartbeat ?? '') > Date.parse(operation.updated_at);
|
||||
const selections = queuedSelections(host.id);
|
||||
const images = new Set([config.komodo.defaultImage, ...selections.flatMap(selection => selection.image ? [selection.image] : [])]);
|
||||
const images = new Set([config.defaultRuntimeImage, ...selections.flatMap(selection => selection.image ? [selection.image] : [])]);
|
||||
const result: Target[] = [...images].map(image => {
|
||||
const operation = latestOperation(host.id, 'runtime.pull', 'image', image);
|
||||
const runtimeReady = host.inventory.runtimeImages?.includes(image) && verifiedThisEnable(operation);
|
||||
|
||||
@@ -1,832 +0,0 @@
|
||||
import { config, komodoConfigured } from '../config.js';
|
||||
import { getServer } from '../fleet/servers.js';
|
||||
import { nativeHealth, nativeInstance } from '../backends/native.js';
|
||||
|
||||
// Ports `komodo_restart` in push_scripts.sh:622. Three shapes depending on
|
||||
// the tracked komodo_type: deployment | container | stack.
|
||||
|
||||
export type KomodoRestartResult =
|
||||
| { ok: true }
|
||||
| { ok: false; kind: 'unconfigured' | 'no-name' | 'http-error' | 'unreachable'; message: string };
|
||||
|
||||
async function komodoCall(path: string, body: unknown): Promise<Response> {
|
||||
const url = `${config.komodo.address.replace(/\/+$/, '')}/${path}`;
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), config.httpTimeoutMs);
|
||||
try {
|
||||
return await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'X-Api-Key': config.komodo.apiKey,
|
||||
'X-Api-Secret': config.komodo.apiSecret
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
signal: controller.signal
|
||||
});
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
export async function komodoRestart(serverName: string): Promise<KomodoRestartResult> {
|
||||
if (!komodoConfigured()) {
|
||||
return { ok: false, kind: 'unconfigured', message: 'komodo not configured' };
|
||||
}
|
||||
const row = getServer(serverName);
|
||||
if (!row?.komodo_name) return { ok: false, kind: 'no-name', message: 'no komodo_name set for server' };
|
||||
|
||||
const type = row.komodo_type ?? 'deployment';
|
||||
let path: string;
|
||||
let body: Record<string, string>;
|
||||
|
||||
switch (type) {
|
||||
case 'stack':
|
||||
path = 'execute/RestartStack';
|
||||
body = { stack: row.komodo_name };
|
||||
break;
|
||||
case 'container':
|
||||
if (!row.komodo_server) return { ok: false, kind: 'no-name', message: 'container type needs komodo_server' };
|
||||
path = 'execute/RestartContainer';
|
||||
body = { server: row.komodo_server, container: row.komodo_name };
|
||||
break;
|
||||
case 'deployment':
|
||||
default:
|
||||
path = 'execute/RestartDeployment';
|
||||
body = { deployment: row.komodo_name };
|
||||
break;
|
||||
}
|
||||
|
||||
const url = `${config.komodo.address.replace(/\/+$/, '')}/${path}`;
|
||||
|
||||
let res: Response;
|
||||
try {
|
||||
res = await komodoCall(path, body);
|
||||
} catch (err) {
|
||||
return { ok: false, kind: 'unreachable', message: describeFetchError(err, url) };
|
||||
}
|
||||
if (!res.ok) {
|
||||
return { ok: false, kind: 'http-error', message: `HTTP ${res.status}` };
|
||||
}
|
||||
|
||||
// The script also checks the response body: if it's an object with
|
||||
// .success === false, treat as failure. If .success is absent, trust the
|
||||
// HTTP status (line 646).
|
||||
try {
|
||||
const json = await res.json();
|
||||
if (json && typeof json === 'object' && 'success' in json && json.success === false) {
|
||||
return { ok: false, kind: 'http-error', message: 'komodo returned success=false' };
|
||||
}
|
||||
} catch {
|
||||
// Non-JSON body is fine — HTTP 2xx already indicates success.
|
||||
}
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
// ---------- Port resolution ----------
|
||||
|
||||
export type KomodoPortFailure =
|
||||
| 'unconfigured'
|
||||
| 'no-name'
|
||||
| 'unreachable' // network / DNS / timeout
|
||||
| 'http-error' // non-2xx
|
||||
| 'unparseable' // 2xx but body wasn't JSON
|
||||
| 'no-udp-in-response'; // parsed but no UDP port found
|
||||
|
||||
export interface KomodoPortSuccess {
|
||||
ok: true;
|
||||
port: number;
|
||||
snippet: string;
|
||||
url: string;
|
||||
}
|
||||
export interface KomodoPortFailureResult {
|
||||
ok: false;
|
||||
reason: KomodoPortFailure;
|
||||
detail: string;
|
||||
httpStatus?: number;
|
||||
snippet?: string;
|
||||
// URL we tried (or would have tried) — surfaced in the UI so operators
|
||||
// can `curl` it from the container to verify reachability.
|
||||
url?: string;
|
||||
}
|
||||
export type KomodoPortResult = KomodoPortSuccess | KomodoPortFailureResult;
|
||||
|
||||
const SNIPPET_LIMIT = 4000;
|
||||
|
||||
function truncate(s: string): string {
|
||||
if (s.length <= SNIPPET_LIMIT) return s;
|
||||
return s.slice(0, SNIPPET_LIMIT) + `\n… (${s.length - SNIPPET_LIMIT} more bytes)`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Turn a Node/Undici fetch failure into something an operator can act on.
|
||||
* The default `.message` for an aborted fetch is just "This operation was
|
||||
* aborted" — useless. For network failures Node hides the real cause under
|
||||
* `.cause` (e.g. `ECONNREFUSED`, `ENOTFOUND`, `EHOSTUNREACH`). Unwrap both
|
||||
* and always include the URL we tried.
|
||||
*/
|
||||
function describeFetchError(err: unknown, url: string): string {
|
||||
const e = err as (Error & { name?: string; cause?: unknown }) | undefined;
|
||||
if (e?.name === 'AbortError') {
|
||||
return `timed out after ${config.httpTimeoutMs}ms dialing ${url}`;
|
||||
}
|
||||
const cause = e?.cause as (Error & { code?: string }) | undefined;
|
||||
if (cause && (cause.code || cause.message)) {
|
||||
const code = cause.code ? `${cause.code}: ` : '';
|
||||
return `${code}${cause.message ?? 'network error'} (dialing ${url})`;
|
||||
}
|
||||
return `${e?.message ?? String(err)} (dialing ${url})`;
|
||||
}
|
||||
|
||||
function safeJsonStringify(value: unknown): string {
|
||||
try {
|
||||
return JSON.stringify(value, null, 2);
|
||||
} catch {
|
||||
return String(value);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detailed variant of `komodoResolveGamePort` — returns a discriminated
|
||||
* union so callers can distinguish "network died" from "auth failed" from
|
||||
* "we parsed the JSON but couldn't find a UDP port in a shape we recognize".
|
||||
*
|
||||
* On success, the response snippet is included so the caller can render it
|
||||
* as "here's where we found the port". On no-udp-in-response, the snippet
|
||||
* is the only way we'll know what shape Komodo actually emitted — surface
|
||||
* it prominently in the UI so we can extend `extractUdpPort` for that shape.
|
||||
*/
|
||||
export async function komodoResolvePortDetailed(serverName: string): Promise<KomodoPortResult> {
|
||||
if (!komodoConfigured()) {
|
||||
return { ok: false, reason: 'unconfigured', detail: 'KOMODO_ADDRESS/KEY/SECRET are not all set' };
|
||||
}
|
||||
const row = getServer(serverName);
|
||||
if (!row?.komodo_name) {
|
||||
return { ok: false, reason: 'no-name', detail: `server has no komodo_name: ${serverName}` };
|
||||
}
|
||||
|
||||
const type = row.komodo_type ?? 'deployment';
|
||||
let path: string;
|
||||
let body: Record<string, string>;
|
||||
|
||||
switch (type) {
|
||||
case 'stack':
|
||||
path = 'read/GetStack';
|
||||
body = { stack: row.komodo_name };
|
||||
break;
|
||||
case 'container':
|
||||
if (!row.komodo_server) {
|
||||
return { ok: false, reason: 'no-name', detail: 'container type needs komodo_server' };
|
||||
}
|
||||
path = 'read/InspectContainer';
|
||||
body = { server: row.komodo_server, container: row.komodo_name };
|
||||
break;
|
||||
case 'deployment':
|
||||
default:
|
||||
path = 'read/GetDeployment';
|
||||
body = { deployment: row.komodo_name };
|
||||
break;
|
||||
}
|
||||
|
||||
const url = `${config.komodo.address.replace(/\/+$/, '')}/${path}`;
|
||||
|
||||
let res: Response;
|
||||
try {
|
||||
res = await komodoCall(path, body);
|
||||
} catch (err) {
|
||||
return { ok: false, reason: 'unreachable', detail: describeFetchError(err, url), url };
|
||||
}
|
||||
|
||||
// Read the body up front — we want the snippet even on non-2xx so the
|
||||
// UI can show what the error page actually said.
|
||||
const rawText = await res.text().catch(() => '');
|
||||
|
||||
if (!res.ok) {
|
||||
return {
|
||||
ok: false,
|
||||
reason: 'http-error',
|
||||
detail: `HTTP ${res.status} ${res.statusText || ''}`.trim(),
|
||||
httpStatus: res.status,
|
||||
snippet: truncate(rawText),
|
||||
url
|
||||
};
|
||||
}
|
||||
|
||||
let json: unknown;
|
||||
try {
|
||||
json = JSON.parse(rawText);
|
||||
} catch (err) {
|
||||
return {
|
||||
ok: false,
|
||||
reason: 'unparseable',
|
||||
detail: `response body is not JSON: ${(err as Error).message}`,
|
||||
httpStatus: res.status,
|
||||
snippet: truncate(rawText),
|
||||
url
|
||||
};
|
||||
}
|
||||
|
||||
const port = extractUdpPort(json);
|
||||
const snippet = truncate(safeJsonStringify(json));
|
||||
if (port === null) {
|
||||
return {
|
||||
ok: false,
|
||||
reason: 'no-udp-in-response',
|
||||
detail: 'parsed the response but no UDP port matched any known shape',
|
||||
httpStatus: res.status,
|
||||
snippet,
|
||||
url
|
||||
};
|
||||
}
|
||||
return { ok: true, port, snippet, url };
|
||||
}
|
||||
|
||||
/**
|
||||
* Query Komodo's read API for a server's UDP game port. Called lazily by
|
||||
* the RCON client and cached into `servers.game_port` on success. Thin shim
|
||||
* over `komodoResolvePortDetailed` that swallows diagnostics — use the
|
||||
* detailed variant when you care why it failed.
|
||||
*/
|
||||
export async function komodoResolveGamePort(serverName: string): Promise<number | null> {
|
||||
const result = await komodoResolvePortDetailed(serverName);
|
||||
return result.ok ? result.port : null;
|
||||
}
|
||||
|
||||
// ---------- Health probe ----------
|
||||
|
||||
export type KomodoHealthResult =
|
||||
| { ok: true; state: string }
|
||||
| { ok: false; reason: string; state?: string; snippet?: string };
|
||||
|
||||
// Komodo's container summary has { state: "running", container: { status:
|
||||
// "Up 35 seconds (health: starting)" } }. Inspect responses instead have
|
||||
// State.Health.Status (or snake/lowercase equivalents). Runtime health wins
|
||||
// over a coarse running flag. Unknown/configuration strings are not readiness.
|
||||
export function extractContainerState(payload: unknown): string | null {
|
||||
const seen = new Set<unknown>();
|
||||
const known = new Set(['running','healthy','starting','restarting','unhealthy','stopped','exited','created','dead','paused','removing','notdeployed','unknown']);
|
||||
const stateOf = (value: unknown): string | null => {
|
||||
if (typeof value !== 'string') return null;
|
||||
const state = value.toLowerCase().trim();
|
||||
if (known.has(state)) return state;
|
||||
if (/^up\b/.test(state)) {
|
||||
const health = state.match(/\((?:health:\s*)?(starting|unhealthy|healthy)\)/)?.[1];
|
||||
return health && health !== 'healthy' ? health : 'running';
|
||||
}
|
||||
if (/^exited\b/.test(state)) return 'exited';
|
||||
if (/^restarting\b/.test(state)) return 'restarting';
|
||||
return null;
|
||||
};
|
||||
const walk = (value: unknown): string | null => {
|
||||
if (!value || typeof value !== 'object' || seen.has(value)) return null;
|
||||
seen.add(value);
|
||||
if (Array.isArray(value)) {
|
||||
const states = value.map(walk).filter((state): state is string => !!state);
|
||||
return states.find(state => !['running','healthy'].includes(state)) ?? states[0] ?? null;
|
||||
}
|
||||
const object = value as Record<string, unknown>;
|
||||
const rawState = object.State ?? object.state;
|
||||
const state = stateOf(rawState) ?? stateOf(object.container_state ?? object.containerState ?? object.deployment_state ?? object.deploymentState);
|
||||
// A stopped process is never ready just because its previous health check passed.
|
||||
if (state && !['running','healthy','unknown'].includes(state)) return state;
|
||||
if (rawState && typeof rawState === 'object') {
|
||||
const nested = walk(rawState);
|
||||
if (nested) return nested;
|
||||
}
|
||||
for (const key of ['container','Container','containers','Containers']) {
|
||||
if (object[key] === null && state === 'running') return 'unknown';
|
||||
const nested = walk(object[key]);
|
||||
if (nested) return nested;
|
||||
}
|
||||
const status = stateOf(object.Status ?? object.status ?? object.docker_status);
|
||||
if (status && !['running','healthy'].includes(status)) return status;
|
||||
const health = object.Health ?? object.health;
|
||||
const healthState = stateOf(typeof health === 'object' && health ?
|
||||
(health as Record<string, unknown>).Status ?? (health as Record<string, unknown>).status : health);
|
||||
if (healthState && healthState !== 'healthy') return healthState;
|
||||
if (state || status) return state ?? status;
|
||||
if (object.Running === true || object.running === true) return healthState ?? 'running';
|
||||
for (const [key, item] of Object.entries(object)) {
|
||||
if (['config','Config','labels','Labels'].includes(key)) continue;
|
||||
const nested = walk(item);
|
||||
if (nested) return nested;
|
||||
}
|
||||
return null;
|
||||
};
|
||||
return walk(payload);
|
||||
}
|
||||
|
||||
const HEALTHY_STATES = new Set(['running', 'healthy', 'ok', 'started', 'up']);
|
||||
|
||||
async function fetchJsonWithSnippet(
|
||||
path: string,
|
||||
body: Record<string, string>
|
||||
): Promise<{ ok: true; json: unknown; snippet: string } | { ok: false; reason: string; snippet?: string; unsupportedEndpoint?: boolean }> {
|
||||
const url = `${config.komodo.address.replace(/\/+$/, '')}/${path}`;
|
||||
let res: Response;
|
||||
try {
|
||||
res = await komodoCall(path, body);
|
||||
} catch (err) {
|
||||
return { ok: false, reason: describeFetchError(err, url) };
|
||||
}
|
||||
const text = await res.text().catch(() => '');
|
||||
if (!res.ok) {
|
||||
const unsupportedEndpoint = res.status === 404 || (
|
||||
[400, 405, 422, 500, 501].includes(res.status)
|
||||
&& /\b(?:unknown|unsupported|unrecognized)\s+(?:method|endpoint|variant)\b|\bmethod not found\b/i.test(text)
|
||||
);
|
||||
return { ok: false, reason: `HTTP ${res.status}`, snippet: truncate(text), unsupportedEndpoint };
|
||||
}
|
||||
let json: unknown;
|
||||
try {
|
||||
json = JSON.parse(text);
|
||||
} catch (err) {
|
||||
return { ok: false, reason: `unparseable json: ${(err as Error).message}`, snippet: truncate(text) };
|
||||
}
|
||||
return { ok: true, json, snippet: truncate(safeJsonStringify(json)) };
|
||||
}
|
||||
|
||||
/**
|
||||
* Ask Komodo whether the server's container is currently running. Used by the
|
||||
* sweep during the health phase (post-restart) and by the beta runner. For
|
||||
* deployment-type servers we hit the container-inspect endpoint first because
|
||||
* GetDeployment returns config, not runtime state — falling back only when
|
||||
* the container endpoint isn't available.
|
||||
*/
|
||||
export async function komodoInspectHealthy(serverName: string): Promise<KomodoHealthResult> {
|
||||
if (getServer(serverName)?.native_instance_id) return nativeHealth(serverName);
|
||||
if (!komodoConfigured()) return { ok: false, reason: 'komodo not configured' };
|
||||
const row = getServer(serverName);
|
||||
if (!row?.komodo_name) return { ok: false, reason: 'no komodo_name for server' };
|
||||
|
||||
const type = row.komodo_type ?? 'deployment';
|
||||
const attempts: Array<{ path: string; body: Record<string, string> }> = [];
|
||||
switch (type) {
|
||||
case 'stack':
|
||||
attempts.push({ path: 'read/GetStack', body: { stack: row.komodo_name } });
|
||||
break;
|
||||
case 'container':
|
||||
if (!row.komodo_server) return { ok: false, reason: 'container type needs komodo_server' };
|
||||
attempts.push({
|
||||
path: 'read/InspectContainer',
|
||||
body: { server: row.komodo_server, container: row.komodo_name }
|
||||
});
|
||||
break;
|
||||
case 'deployment':
|
||||
default:
|
||||
// Container-specific endpoints first — they return docker-inspect
|
||||
// shape with State.Status. GetDeployment is the last-resort
|
||||
// config-only fallback.
|
||||
attempts.push({ path: 'read/GetDeploymentContainer', body: { deployment: row.komodo_name } });
|
||||
attempts.push({ path: 'read/InspectDeploymentContainer', body: { deployment: row.komodo_name } });
|
||||
attempts.push({ path: 'read/GetDeploymentStatus', body: { deployment: row.komodo_name } });
|
||||
attempts.push({ path: 'read/GetDeployment', body: { deployment: row.komodo_name } });
|
||||
break;
|
||||
}
|
||||
|
||||
let lastSnippet: string | undefined;
|
||||
for (const { path, body } of attempts) {
|
||||
const r = await fetchJsonWithSnippet(path, body);
|
||||
if (!r.ok) {
|
||||
// Alternate endpoints handle API-version differences. Retrying an
|
||||
// outage against every endpoint delays all other server probes.
|
||||
if (!r.unsupportedEndpoint) return { ok: false, reason: r.reason, snippet: r.snippet };
|
||||
lastSnippet = r.snippet ?? lastSnippet;
|
||||
continue;
|
||||
}
|
||||
const state = extractContainerState(r.json);
|
||||
if (state) {
|
||||
const healthy = HEALTHY_STATES.has(state.toLowerCase());
|
||||
if (!healthy) return { ok: false, state, reason: `container state: ${state}`, snippet: r.snippet };
|
||||
return { ok: true, state };
|
||||
}
|
||||
lastSnippet = r.snippet;
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
reason: 'no container state found in any known Komodo response shape',
|
||||
snippet: lastSnippet
|
||||
};
|
||||
}
|
||||
|
||||
// ---------- Write / Execute (create + destroy deployments) ----------
|
||||
|
||||
export type KomodoWriteResult =
|
||||
| { ok: true }
|
||||
| { ok: false; kind: 'unconfigured' | 'http-error' | 'unreachable'; message: string; url?: string; httpStatus?: number };
|
||||
|
||||
/**
|
||||
* Env var pairs get flattened into a single newline-delimited string, matching
|
||||
* Komodo's DeploymentConfig.environment shape.
|
||||
*/
|
||||
export interface DeploymentEnvVar {
|
||||
variable: string;
|
||||
value: string;
|
||||
}
|
||||
|
||||
export interface CreateDeploymentSpec {
|
||||
name: string; // Komodo deployment name (== EC server name)
|
||||
server: string; // Komodo server (host) id or name
|
||||
image: string; // 'ghcr.io/ayymoss/plutainer:latest'
|
||||
network?: string; // 'komodo_default'
|
||||
ports: string[]; // ['28960:28960/udp', '28960:28960/tcp']
|
||||
volumes: string[]; // ['/host/path:/container/path', 'namedvol:/container/path:ro']
|
||||
environment: DeploymentEnvVar[];
|
||||
}
|
||||
|
||||
async function postAndClassify(path: string, body: unknown): Promise<KomodoWriteResult> {
|
||||
const url = `${config.komodo.address.replace(/\/+$/, '')}/${path}`;
|
||||
let res: Response;
|
||||
try {
|
||||
res = await komodoCall(path, body);
|
||||
} catch (err) {
|
||||
return { ok: false, kind: 'unreachable', message: describeFetchError(err, url), url };
|
||||
}
|
||||
if (!res.ok) {
|
||||
const snippet = await res.text().catch(() => '');
|
||||
return {
|
||||
ok: false,
|
||||
kind: 'http-error',
|
||||
httpStatus: res.status,
|
||||
message: `HTTP ${res.status}${snippet ? ` — ${truncate(snippet)}` : ''}`,
|
||||
url
|
||||
};
|
||||
}
|
||||
// Some Komodo endpoints return `{ success: false }` with a 200. Treat that
|
||||
// as a failure so callers can surface the real problem.
|
||||
try {
|
||||
const json = await res.json();
|
||||
if (json && typeof json === 'object' && 'success' in json && json.success === false) {
|
||||
const msg = typeof (json as { message?: unknown }).message === 'string'
|
||||
? (json as { message: string }).message
|
||||
: 'komodo returned success=false';
|
||||
return { ok: false, kind: 'http-error', message: msg, url, httpStatus: res.status };
|
||||
}
|
||||
} catch {
|
||||
// Non-JSON body — trust the 2xx.
|
||||
}
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new deployment entity in Komodo. Doesn't start the container —
|
||||
* caller must follow up with `komodoDeploy(name)` to actually run it.
|
||||
*
|
||||
* The config shape here targets Komodo's DeploymentConfig: newline-delimited
|
||||
* ports/volumes strings and an environment array with `{ variable, value }`
|
||||
* entries. If Komodo rejects the payload, the error surfaces in the http-error
|
||||
* message so we can iterate on the shape without guessing.
|
||||
*/
|
||||
export async function komodoCreateDeployment(spec: CreateDeploymentSpec): Promise<KomodoWriteResult> {
|
||||
if (!komodoConfigured()) {
|
||||
return { ok: false, kind: 'unconfigured', message: 'komodo not configured' };
|
||||
}
|
||||
const body = {
|
||||
name: spec.name,
|
||||
config: {
|
||||
server_id: spec.server,
|
||||
image: { type: 'Image', params: { image: spec.image } },
|
||||
network: spec.network ?? 'komodo_default',
|
||||
ports: spec.ports.join('\n'),
|
||||
volumes: spec.volumes.join('\n'),
|
||||
environment: spec.environment,
|
||||
restart: 'unless-stopped'
|
||||
}
|
||||
};
|
||||
return postAndClassify('write/CreateDeployment', body);
|
||||
}
|
||||
|
||||
/** Start (or restart) the container backing a Komodo deployment. */
|
||||
export async function komodoDeploy(deployment: string): Promise<KomodoWriteResult> {
|
||||
if (!komodoConfigured()) {
|
||||
return { ok: false, kind: 'unconfigured', message: 'komodo not configured' };
|
||||
}
|
||||
return postAndClassify('execute/Deploy', { deployment });
|
||||
}
|
||||
|
||||
/** Runtime controls use the saved Komodo resource, never the EC display name. */
|
||||
export async function komodoRuntimeAction(server: string, action: 'start' | 'stop'): Promise<KomodoRestartResult> {
|
||||
if (!komodoConfigured()) return { ok: false, kind: 'unconfigured', message: 'komodo not configured' };
|
||||
const row = getServer(server);
|
||||
if (!row?.komodo_name) return { ok: false, kind: 'no-name', message: 'no komodo_name set for server' };
|
||||
const type = row.komodo_type ?? 'deployment';
|
||||
const verb = action === 'start' ? 'Start' : 'Stop';
|
||||
if (type === 'container') {
|
||||
if (!row.komodo_server) return { ok: false, kind: 'no-name', message: 'container type needs komodo_server' };
|
||||
return postAndClassify(`execute/${verb}Container`, { server: row.komodo_server, container: row.komodo_name });
|
||||
}
|
||||
if (type === 'stack') return postAndClassify(`execute/${verb}Stack`, { stack: row.komodo_name });
|
||||
return postAndClassify(`execute/${verb}Deployment`, { deployment: row.komodo_name });
|
||||
}
|
||||
|
||||
/** Stop + remove the container backing a Komodo deployment. Entity remains. */
|
||||
export async function komodoDestroyDeployment(deployment: string): Promise<KomodoWriteResult> {
|
||||
if (!komodoConfigured()) {
|
||||
return { ok: false, kind: 'unconfigured', message: 'komodo not configured' };
|
||||
}
|
||||
return postAndClassify('execute/DestroyDeployment', { deployment });
|
||||
}
|
||||
|
||||
/** Remove the deployment entity from Komodo entirely. */
|
||||
export async function komodoDeleteDeployment(deployment: string): Promise<KomodoWriteResult> {
|
||||
if (!komodoConfigured()) {
|
||||
return { ok: false, kind: 'unconfigured', message: 'komodo not configured' };
|
||||
}
|
||||
// Komodo's DeleteDeployment historically accepts `{ id }` where id may be
|
||||
// the name; supply both to be forgiving.
|
||||
return postAndClassify('write/DeleteDeployment', { id: deployment, name: deployment });
|
||||
}
|
||||
|
||||
// ---------- Tags ----------
|
||||
|
||||
export type KomodoTagResult =
|
||||
| { ok: true; id: string }
|
||||
| { ok: false; message: string };
|
||||
|
||||
// Accept API string IDs and raw Mongo Extended JSON: { _id: { $oid: "..." } }.
|
||||
function extractTagId(t: unknown): string | null {
|
||||
if (!t || typeof t !== 'object') return null;
|
||||
const o = t as { id?: unknown; _id?: unknown };
|
||||
if (typeof o.id === 'string' && o.id.length > 0) return o.id;
|
||||
if (typeof o._id === 'string' && o._id.length > 0) return o._id;
|
||||
if (o._id && typeof o._id === 'object') {
|
||||
const oid = (o._id as { $oid?: unknown }).$oid;
|
||||
if (typeof oid === 'string' && oid.length > 0) return oid;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ListTags response shape varies by Komodo version: bare array `[...]`, or
|
||||
// wrapped under one of a few common keys. Peel back the common wrappers.
|
||||
function unwrapTagList(body: unknown): unknown[] {
|
||||
if (Array.isArray(body)) return body;
|
||||
if (body && typeof body === 'object') {
|
||||
for (const key of ['tags', 'data', 'results', 'items']) {
|
||||
const val = (body as Record<string, unknown>)[key];
|
||||
if (Array.isArray(val)) return val;
|
||||
}
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the tag list from Komodo. Two request styles are attempted in order:
|
||||
*
|
||||
* 1. `POST /read/ListTags` with an empty body — the per-endpoint style
|
||||
* already used by the rest of this file (see execute/RestartDeployment).
|
||||
* 2. `POST /read` with `{type: "ListTags", params: {}}` — the tagged-union
|
||||
* style used by mbecker20/komodo's canonical API. If your Komodo build
|
||||
* wires up only the tagged-union path, style #1 returns something we
|
||||
* can't parse (empty array or non-tag payload) and we fall through.
|
||||
*
|
||||
* The winning parsed list is returned. The raw response bodies get logged
|
||||
* only when both attempts fail to yield a match, so a fresh Komodo API shape
|
||||
* is diagnosable without spamming stderr on the happy path.
|
||||
*/
|
||||
async function fetchTagsWithFallback(): Promise<{ tags: unknown[]; rawBodies: string[] }> {
|
||||
const attempts: Array<{ path: string; body: unknown }> = [
|
||||
{ path: 'read/ListTags', body: {} },
|
||||
{ path: 'read', body: { type: 'ListTags', params: {} } }
|
||||
];
|
||||
const rawBodies: string[] = [];
|
||||
for (const attempt of attempts) {
|
||||
let res: Response;
|
||||
try {
|
||||
res = await komodoCall(attempt.path, attempt.body);
|
||||
} catch (err) {
|
||||
rawBodies.push(`${attempt.path}: network error — ${(err as Error).message}`);
|
||||
continue;
|
||||
}
|
||||
if (!res.ok) {
|
||||
const snippet = await res.text().catch(() => '');
|
||||
rawBodies.push(`${attempt.path}: HTTP ${res.status}${snippet ? ` — ${truncate(snippet)}` : ''}`);
|
||||
continue;
|
||||
}
|
||||
const raw = await res.text().catch(() => '');
|
||||
rawBodies.push(`${attempt.path}: ${truncate(raw)}`);
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(raw);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const tags = unwrapTagList(parsed);
|
||||
if (tags.length > 0) return { tags, rawBodies };
|
||||
}
|
||||
return { tags: [], rawBodies };
|
||||
}
|
||||
|
||||
async function findTagByName(name: string): Promise<KomodoTagResult | null> {
|
||||
const { tags, rawBodies } = await fetchTagsWithFallback();
|
||||
for (const t of tags) {
|
||||
if (!t || typeof t !== 'object') continue;
|
||||
const tagName = (t as { name?: unknown }).name;
|
||||
if (typeof tagName !== 'string' || tagName !== name) continue;
|
||||
const id = extractTagId(t);
|
||||
if (id) return { ok: true, id };
|
||||
}
|
||||
// Miss — dump the raw bodies so the next iteration knows the real shape.
|
||||
console.warn(
|
||||
`[komodoEnsureTag] no matchable tag "${name}" in any ListTags variant. Bodies:\n ` +
|
||||
rawBodies.join('\n ')
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a tag by name, creating it if it doesn't exist. Returns the tag's
|
||||
* Komodo id (Mongo ObjectId string) for use in UpdateResourceMeta.
|
||||
*
|
||||
* Best-effort: callers should treat failure as "skip tagging", not "roll back
|
||||
* the whole operation". Tags are cosmetic organization inside Komodo.
|
||||
*/
|
||||
export async function komodoEnsureTag(name: string): Promise<KomodoTagResult> {
|
||||
if (!komodoConfigured()) return { ok: false, message: 'komodo not configured' };
|
||||
|
||||
// 1. Look for an existing tag by name.
|
||||
const existing = await findTagByName(name);
|
||||
if (existing) return existing;
|
||||
|
||||
// 2. Not found — create it.
|
||||
let createRes: Response;
|
||||
try {
|
||||
createRes = await komodoCall('write/CreateTag', { name });
|
||||
} catch (err) {
|
||||
return { ok: false, message: `CreateTag ${name}: ${(err as Error).message}` };
|
||||
}
|
||||
if (createRes.ok) {
|
||||
const body = await createRes.json().catch(() => null);
|
||||
const id = extractTagId(body);
|
||||
if (id) return { ok: true, id };
|
||||
// 200 but shape we don't recognize — fall back to re-list rather than
|
||||
// fail, since the tag definitely exists now.
|
||||
const fresh = await findTagByName(name);
|
||||
if (fresh) return fresh;
|
||||
console.warn(`[komodoEnsureTag] CreateTag ${name}: unexpected 2xx body, re-list also missed`, body);
|
||||
return { ok: false, message: `CreateTag ${name}: unexpected response shape` };
|
||||
}
|
||||
|
||||
// Non-2xx. Duplicate-key 500 means the tag exists under a different
|
||||
// ListTags projection than we probed — re-list and try to find it. If
|
||||
// that still misses, surface the create error.
|
||||
const snippet = await createRes.text().catch(() => '');
|
||||
const looksLikeDuplicate =
|
||||
createRes.status === 500 &&
|
||||
(/duplicate key/i.test(snippet) || /E11000/i.test(snippet) || /dup key/i.test(snippet));
|
||||
if (looksLikeDuplicate) {
|
||||
const fresh = await findTagByName(name);
|
||||
if (fresh) return fresh;
|
||||
return {
|
||||
ok: false,
|
||||
message: `CreateTag ${name}: duplicate reported but re-list missed it. Body: ${truncate(snippet)}`
|
||||
};
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
message: `CreateTag ${name}: HTTP ${createRes.status}${snippet ? ` — ${truncate(snippet)}` : ''}`
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Attach tags to a Komodo resource. `tagIds` is the array of tag Mongo IDs
|
||||
* from `komodoEnsureTag`. Replaces the resource's existing tag set — pass the
|
||||
* full desired list, not a delta.
|
||||
*
|
||||
* Komodo's target.id accepts either the true id or the resource name for
|
||||
* Deployments, so we pass the name here to save a lookup.
|
||||
*/
|
||||
export async function komodoUpdateTagsOnResource(
|
||||
targetType: 'Deployment' | 'Stack' | 'Server' | 'Repo' | 'Build' | 'Procedure',
|
||||
targetName: string,
|
||||
tagIds: string[]
|
||||
): Promise<KomodoWriteResult> {
|
||||
if (!komodoConfigured()) {
|
||||
return { ok: false, kind: 'unconfigured', message: 'komodo not configured' };
|
||||
}
|
||||
// UpdateResourceMeta owns tags; omit the other optional metadata fields
|
||||
// so the deployment's description and template flag remain unchanged.
|
||||
return postAndClassify('write/UpdateResourceMeta', {
|
||||
target: { type: targetType, id: targetName },
|
||||
tags: tagIds
|
||||
});
|
||||
}
|
||||
|
||||
// ---------- Extractor ----------
|
||||
|
||||
// Property names Komodo/Docker use for the collection of port mappings.
|
||||
const PORT_ARRAY_KEYS = new Set(['ports', 'Ports', 'port_bindings', 'PortBindings']);
|
||||
// Value field names for the host-facing port on an object entry.
|
||||
const HOST_PORT_KEYS = ['published', 'PublishedPort', 'public_port', 'PublicPort', 'HostPort', 'hostPort', 'host_port'];
|
||||
|
||||
interface PortsCandidate {
|
||||
arr: unknown[];
|
||||
/** True when the array was reached via a `ports`-flavored key — preferred. */
|
||||
preferred: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Walks a Komodo response looking for a UDP port. Handles the shapes we've
|
||||
* seen in the wild:
|
||||
*
|
||||
* docker-compose object: [{ protocol: "udp", published: 4976, target: 28960 }]
|
||||
* docker-compose caps: [{ Protocol: "udp", PublishedPort: 4976 }]
|
||||
* docker-run string: ["4976:28960/udp", "3074:3074/tcp"]
|
||||
* host-network string: ["28962/udp"]
|
||||
* docker-inspect object: { "28960/udp": [{ "HostIp": "0.0.0.0", "HostPort": "28962" }] }
|
||||
*
|
||||
* Arrays reached via a `ports`/`Ports`/`port_bindings`/`PortBindings` key
|
||||
* are preferred over arrays discovered elsewhere in the tree, so an
|
||||
* unrelated array (e.g. tags, labels) can't shadow a real port list.
|
||||
*/
|
||||
export function extractUdpPort(payload: unknown): number | null {
|
||||
const candidates: PortsCandidate[] = [];
|
||||
// Objects whose keys encode `<container-port>/<proto>` — a la docker inspect.
|
||||
const containerKeyed: Record<string, unknown>[] = [];
|
||||
const seen = new Set<unknown>();
|
||||
|
||||
const walk = (v: unknown, viaPortsKey: boolean): void => {
|
||||
if (v === null || typeof v !== 'object' || seen.has(v)) return;
|
||||
seen.add(v);
|
||||
if (Array.isArray(v)) {
|
||||
for (const item of v) walk(item, false);
|
||||
candidates.push({ arr: v, preferred: viaPortsKey });
|
||||
return;
|
||||
}
|
||||
const obj = v as Record<string, unknown>;
|
||||
// Detect docker-inspect container-keyed objects: any key like `28960/udp`.
|
||||
for (const k of Object.keys(obj)) {
|
||||
if (/^\d+\/(udp|tcp)$/i.test(k)) {
|
||||
containerKeyed.push(obj);
|
||||
break;
|
||||
}
|
||||
}
|
||||
for (const [key, val] of Object.entries(obj)) {
|
||||
// Komodo's GetDeployment renders `config.ports` as a single
|
||||
// newline-delimited string ("28963:28963/udp\n28963:28963/tcp").
|
||||
// Synthesize an array from it and treat as a preferred candidate.
|
||||
if (PORT_ARRAY_KEYS.has(key) && typeof val === 'string' && val.trim()) {
|
||||
const parts = val
|
||||
.split(/[\r\n,;]+/)
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean);
|
||||
if (parts.length > 0) {
|
||||
candidates.push({ arr: parts, preferred: true });
|
||||
}
|
||||
}
|
||||
walk(val, PORT_ARRAY_KEYS.has(key));
|
||||
}
|
||||
};
|
||||
walk(payload, false);
|
||||
|
||||
// Sort so preferred `ports`-keyed arrays win over accidental hits.
|
||||
candidates.sort((a, b) => (a.preferred === b.preferred ? 0 : a.preferred ? -1 : 1));
|
||||
|
||||
// Try object-entry shape first — most Komodo/Docker responses use this.
|
||||
for (const { arr } of candidates) {
|
||||
for (const entry of arr) {
|
||||
if (entry && typeof entry === 'object' && !Array.isArray(entry)) {
|
||||
const e = entry as Record<string, unknown>;
|
||||
const proto = String(e.protocol ?? e.Protocol ?? e.type ?? e.Type ?? '').toLowerCase();
|
||||
if (proto !== 'udp') continue;
|
||||
for (const field of HOST_PORT_KEYS) {
|
||||
const raw = e[field];
|
||||
if (raw === undefined || raw === null) continue;
|
||||
const num = typeof raw === 'string' ? Number.parseInt(raw, 10) : Number(raw);
|
||||
if (Number.isFinite(num) && num > 0) return num;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Then string shapes: "host:container/udp" or "container/udp" (host-net).
|
||||
for (const { arr } of candidates) {
|
||||
for (const entry of arr) {
|
||||
if (typeof entry !== 'string') continue;
|
||||
const hostMapped = /^(\d+):\d+\/udp$/i.exec(entry);
|
||||
if (hostMapped) return Number.parseInt(hostMapped[1], 10);
|
||||
const hostNet = /^(\d+)\/udp$/i.exec(entry);
|
||||
if (hostNet) return Number.parseInt(hostNet[1], 10);
|
||||
}
|
||||
}
|
||||
|
||||
// Finally, docker-inspect container-keyed shape:
|
||||
// { "28960/udp": [{ HostIp, HostPort }] }
|
||||
for (const obj of containerKeyed) {
|
||||
for (const [key, val] of Object.entries(obj)) {
|
||||
if (!/^\d+\/udp$/i.test(key)) continue;
|
||||
if (!Array.isArray(val)) continue;
|
||||
for (const binding of val) {
|
||||
if (!binding || typeof binding !== 'object') continue;
|
||||
const b = binding as Record<string, unknown>;
|
||||
for (const field of HOST_PORT_KEYS) {
|
||||
const raw = b[field];
|
||||
if (raw === undefined || raw === null) continue;
|
||||
const num = typeof raw === 'string' ? Number.parseInt(raw, 10) : Number(raw);
|
||||
if (Number.isFinite(num) && num > 0) return num;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
@@ -1,93 +0,0 @@
|
||||
import { createConnection } from 'node:net';
|
||||
import { lookup } from 'node:dns/promises';
|
||||
|
||||
export type TcpProbeOutcome =
|
||||
| 'connected'
|
||||
| 'refused'
|
||||
| 'timeout'
|
||||
| 'dns-fail'
|
||||
| 'unreachable'
|
||||
| 'error';
|
||||
|
||||
export interface TcpProbeResult {
|
||||
label: string;
|
||||
host: string;
|
||||
port: number;
|
||||
resolvedIp: string | null;
|
||||
outcome: TcpProbeOutcome;
|
||||
elapsedMs: number;
|
||||
detail?: string;
|
||||
}
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 3000;
|
||||
|
||||
/**
|
||||
* Best-effort DNS resolution. Returns null on failure so the caller can
|
||||
* distinguish "DNS failed" from "connect failed".
|
||||
*/
|
||||
async function resolve(host: string): Promise<{ ip: string | null; error: string | null }> {
|
||||
try {
|
||||
const r = await lookup(host);
|
||||
return { ip: r.address, error: null };
|
||||
} catch (err) {
|
||||
const e = err as NodeJS.ErrnoException;
|
||||
return { ip: null, error: `${e.code ?? 'UNKNOWN'}: ${e.message}` };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempt a raw TCP connect to `host:port` with an explicit timeout, and
|
||||
* classify the result. Purely diagnostic — no bytes are sent after the
|
||||
* handshake completes. Isolates network reachability from every HTTP-layer
|
||||
* concern (auth, TLS, redirects, wrong path).
|
||||
*/
|
||||
export async function tcpProbe(
|
||||
label: string,
|
||||
host: string,
|
||||
port: number,
|
||||
timeoutMs: number = DEFAULT_TIMEOUT_MS
|
||||
): Promise<TcpProbeResult> {
|
||||
const started = Date.now();
|
||||
const { ip, error: dnsError } = await resolve(host);
|
||||
|
||||
if (!ip) {
|
||||
return {
|
||||
label,
|
||||
host,
|
||||
port,
|
||||
resolvedIp: null,
|
||||
outcome: 'dns-fail',
|
||||
elapsedMs: Date.now() - started,
|
||||
detail: dnsError ?? 'DNS lookup returned no address'
|
||||
};
|
||||
}
|
||||
|
||||
return new Promise<TcpProbeResult>((resolve) => {
|
||||
const sock = createConnection({ host: ip, port, timeout: timeoutMs });
|
||||
let settled = false;
|
||||
const finish = (outcome: TcpProbeOutcome, detail?: string): void => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
try { sock.destroy(); } catch { /* already destroyed */ }
|
||||
resolve({ label, host, port, resolvedIp: ip, outcome, elapsedMs: Date.now() - started, detail });
|
||||
};
|
||||
sock.once('connect', () => finish('connected'));
|
||||
sock.once('timeout', () => finish('timeout', `no SYN-ACK within ${timeoutMs}ms`));
|
||||
sock.once('error', (err) => {
|
||||
const code = (err as NodeJS.ErrnoException).code ?? '';
|
||||
const msg = (err as Error).message ?? String(err);
|
||||
if (code === 'ECONNREFUSED') return finish('refused', code);
|
||||
if (code === 'EHOSTUNREACH' || code === 'ENETUNREACH') return finish('unreachable', `${code}: ${msg}`);
|
||||
return finish('error', `${code || 'unknown'}: ${msg}`);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export async function tcpProbeAll(
|
||||
trials: Array<{ label: string; host: string; port: number }>,
|
||||
timeoutMs: number = DEFAULT_TIMEOUT_MS
|
||||
): Promise<TcpProbeResult[]> {
|
||||
// Run in parallel — they don't share sockets and the whole point is to
|
||||
// finish fast even when some probes hit the timeout.
|
||||
return Promise.all(trials.map((t) => tcpProbe(t.label, t.host, t.port, timeoutMs)));
|
||||
}
|
||||
@@ -5,7 +5,7 @@ import { hasActiveProvisioning } from '../fleet/provisioning.js';
|
||||
import { latestGameSubmission } from '../fleet/status-evidence.js';
|
||||
import { latestGameSubmissions } from '../fleet/lifecycle-evidence.js';
|
||||
import { rconPlutoniumVersion } from './rcon.js';
|
||||
import { komodoInspectHealthy } from './komodo.js';
|
||||
import { nativeHealth } from '../backends/native.js';
|
||||
import { restartRecoveryPending } from '../provisioning/index.js';
|
||||
import { parsePlutoniumManifest, PLUTONIUM_MANIFEST_URL } from '../../plutonium.js';
|
||||
import type { PlutoniumRelease, PlutoniumVersion } from '../../plutonium.js';
|
||||
@@ -60,7 +60,7 @@ export async function probeVersion(server: string, options: { maintenance?: bool
|
||||
const epoch = latestGameSubmission(server);
|
||||
const recovering = recoveryBlocked();
|
||||
const health = !recovering && original.desired_state !== 'stopped'
|
||||
? await komodoInspectHealthy(server).catch(() => ({ ok: false })) : { ok: false };
|
||||
? await nativeHealth(server).catch(() => ({ ok: false })) : { ok: false };
|
||||
if (suppressed()) return null;
|
||||
let result = !health.ok ? { revision: null, error: 'Waiting for a healthy running server.' }
|
||||
: await rconPlutoniumVersion(server).catch(() => ({ revision: null, error: 'Version check failed.' }));
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
import { createSocket } from 'node:dgram';
|
||||
import type { Socket } from 'node:dgram';
|
||||
import { config, komodoConfigured, rconConfigured } from '../config.js';
|
||||
import { getServer, setServerField } from '../fleet/servers.js';
|
||||
import { config, rconConfigured } from '../config.js';
|
||||
import { getServer } from '../fleet/servers.js';
|
||||
import { latestGameSubmission } from '../fleet/lifecycle-evidence.js';
|
||||
import { komodoResolveGamePort } from './komodo.js';
|
||||
import { parsePlutoniumRevision } from '../../plutonium.js';
|
||||
import { nativeInstance, nativeRequest } from '../backends/native.js';
|
||||
|
||||
@@ -81,27 +80,15 @@ async function sendRcon(host: string, port: number, command: string): Promise<Se
|
||||
|
||||
interface EndpointResolution {
|
||||
endpoint: RconEndpoint | null;
|
||||
// Present when endpoint is null — tells the caller why we couldn't dial.
|
||||
reason?: 'unknown-server' | 'no-port-no-komodo' | 'no-port-komodo-empty';
|
||||
reason?: 'unknown-server' | 'no-port';
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the RCON endpoint (host + port) for a server, filling
|
||||
* `servers.game_port` from Komodo on first miss.
|
||||
*/
|
||||
/** Resolve a saved RCON endpoint without remote resource discovery. */
|
||||
async function endpointFor(server: string): Promise<EndpointResolution> {
|
||||
const row = getServer(server);
|
||||
if (!row) return { endpoint: null, reason: 'unknown-server' };
|
||||
let port = row.game_port;
|
||||
if (!port) {
|
||||
if (!komodoConfigured()) {
|
||||
return { endpoint: null, reason: 'no-port-no-komodo' };
|
||||
}
|
||||
port = await komodoResolveGamePort(server);
|
||||
if (port) setServerField(server, 'game_port', port);
|
||||
}
|
||||
if (!port) return { endpoint: null, reason: 'no-port-komodo-empty' };
|
||||
return { endpoint: { host: row.rcon_host ?? config.rcon.host, port } };
|
||||
if (!row.game_port) return { endpoint: null, reason: 'no-port' };
|
||||
return { endpoint: { host: row.rcon_host ?? config.rcon.host, port: row.game_port } };
|
||||
}
|
||||
|
||||
function stripPrefix(buf: Buffer): string | null {
|
||||
@@ -421,20 +408,7 @@ export async function rconStatusResult(server: string): Promise<RconStatusResult
|
||||
if (res.reason === 'unknown-server') {
|
||||
return { ok: false, endpoint: null, reason: 'no-port', detail: `unknown server: ${server}` };
|
||||
}
|
||||
if (res.reason === 'no-port-no-komodo') {
|
||||
return {
|
||||
ok: false,
|
||||
endpoint: null,
|
||||
reason: 'no-port',
|
||||
detail: 'no game_port set and Komodo is not configured'
|
||||
};
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
endpoint: null,
|
||||
reason: 'no-port',
|
||||
detail: 'no game_port — Komodo did not return one for this server'
|
||||
};
|
||||
return { ok: false, endpoint: null, reason: 'no-port', detail: 'No game port is registered for this server.' };
|
||||
}
|
||||
|
||||
const ep = res.endpoint;
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { db } from '../db/index.js';
|
||||
import { komodoConfigured } from '../config.js';
|
||||
import { nativeInstance } from '../backends/native.js';
|
||||
import { getServer, listServers } from '../fleet/servers.js';
|
||||
import { hasActiveProvisioning } from '../fleet/provisioning.js';
|
||||
@@ -100,7 +99,7 @@ export function resolveRecoveredHealthOperation(operationId: string, players: nu
|
||||
db().prepare('UPDATE server_deployments SET checked_at = ?, players = ? WHERE id = ?').run(now, players, operationId);
|
||||
recordProgress(operationId, { blocker: null, observedAt: now,
|
||||
recovery: { state: 'released', at: now, operationId, message },
|
||||
checks: [{ key: 'container', label: 'Container readiness', source: 'komodo', result: 'pass', checkedAt: now },
|
||||
checks: [{ key: 'container', label: 'Container readiness', source: 'agent', result: 'pass', checkedAt: now },
|
||||
{ key: 'game', label: 'Game status', source, result: 'pass', actual: players, checkedAt: now }] });
|
||||
if (operation.state === 'awaiting-health') {
|
||||
deploymentStage(operationId, message);
|
||||
@@ -275,7 +274,7 @@ async function scanServer(server: string, forceScan: boolean): Promise<void> {
|
||||
if (observation.identity !== monitoringIdentity(server)) return;
|
||||
if (!scanSuppressed(server, 'health') && observation.healthy) {
|
||||
const current = active(server, 'health');
|
||||
if (current) transition(current.id, 'resolved', 'Fresh game status and Komodo health verified');
|
||||
if (current) transition(current.id, 'resolved', 'Fresh game status and host agent health verified');
|
||||
} else if (!scanSuppressed(server, 'health')) {
|
||||
const problem = detect(server, 'health', observation.reason, observation);
|
||||
const startup = ['starting','restarting'].includes(observation.containerState ?? '');
|
||||
@@ -334,8 +333,8 @@ async function run(forceScan: boolean): Promise<void> {
|
||||
const lastScan = setting('watchdog_last_scan');
|
||||
if (!forceScan && lastScan && Date.now() - Date.parse(lastScan) < WATCHDOG_SCAN_MS) return;
|
||||
setSetting('watchdog_last_scan', iso());
|
||||
if (!komodoConfigured() && !listServers().some(server => nativeInstance(server.name))) {
|
||||
setSetting('watchdog_scan_error', 'Configure Komodo before automatic recovery. RCON or fresh telemetry verifies game status.'); return;
|
||||
if (!listServers().some(server => nativeInstance(server.name))) {
|
||||
setSetting('watchdog_scan_error', 'Register a native host instance before automatic recovery. RCON or fresh telemetry verifies game status.'); return;
|
||||
}
|
||||
const errors: string[] = [];
|
||||
const servers = listServers();
|
||||
|
||||
@@ -1 +1 @@
|
||||
export { restartSettling, rolloutWaitReason, refreshRestartRollout, rolloutHealth } from '../provisioning/restarts.js';
|
||||
export { restartSettling, rolloutWaitReason, rolloutHealth } from '../provisioning/restarts.js';
|
||||
|
||||
@@ -100,8 +100,8 @@ function queueStatus(srv: ServerRow, probe?: () => ReturnType<typeof rconForMoni
|
||||
const server = srv.name;
|
||||
const pending = pendingStatus.get(server);
|
||||
if (pending) return pending;
|
||||
// Legacy servers without a recorded host share the configured host's pool.
|
||||
const host = srv.host_id ? `native:${srv.host_id}` : `komodo:${srv.komodo_server || config.komodo.targetServer}`;
|
||||
// Host queues keep slow instances from consuming other hosts' slots.
|
||||
const host = srv.host_id ? `native:${srv.host_id}` : `unregistered:${srv.name}`;
|
||||
let queue = statusQueues.get(host);
|
||||
if (!queue) { queue = { active: 0, queued: [] }; statusQueues.set(host, queue); }
|
||||
let finish!: (capacity: number | null) => void;
|
||||
|
||||
@@ -121,7 +121,7 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
|
||||
SELECT collection_id, deployment_id FROM server_collection_members
|
||||
UNION ALL SELECT collection_id, deployment_id FROM server_collection_removals) m
|
||||
JOIN server_launch_collections c ON c.id = m.collection_id WHERE m.deployment_id = ? ORDER BY c.created_at, c.id LIMIT 1`).get(id) as { id: string; name: string } | undefined;
|
||||
return { deployment, backend: native ? 'native' as const : 'komodo' as const, agentOperation: agentOperation ?? null,
|
||||
return { deployment, backend: 'native' as const, agentOperation: agentOperation ?? null,
|
||||
canCancel: canCancelProvisioningJob(id), canCancelCreation: canCancelCreation(id), cancellation: cancellation ?? null,
|
||||
workerActivity: workerActivity(), events: deploymentEvents(id, options.events === 'recent'),
|
||||
activity: { ...activitySummary(id), failureExcerpt: latestFailureEvent(id) }, sources,
|
||||
@@ -133,7 +133,7 @@ export function getDeployment(id: string, options: { events?: 'all' | 'recent' }
|
||||
})();
|
||||
}
|
||||
|
||||
// This stage is written only after Komodo has confirmed deployment deletion.
|
||||
// This stage is written only after the host agent has confirmed instance removal.
|
||||
export const REMOVING_SERVER = 'Removing server files and registry';
|
||||
|
||||
export function canCancelRemoval(id: string): boolean {
|
||||
@@ -154,15 +154,6 @@ export function cancelRemoval(id: string): boolean {
|
||||
})();
|
||||
}
|
||||
|
||||
export function claimRemoval(id: string): boolean {
|
||||
return db().prepare(`UPDATE server_deployments SET state = 'running'
|
||||
WHERE id = ? AND action = 'destroy' AND state = 'waiting'
|
||||
AND (server_deleted_at IS NULL OR (stage = ?
|
||||
AND NOT EXISTS (SELECT 1 FROM servers s WHERE s.name = server_deployments.server)
|
||||
AND NOT EXISTS (SELECT 1 FROM server_deployments newer WHERE newer.server = server_deployments.server
|
||||
AND newer.rowid > server_deployments.rowid)))`).run(id, REMOVING_SERVER).changes > 0;
|
||||
}
|
||||
|
||||
export function canResumeCleanup(id: string): boolean {
|
||||
return !!db().prepare(`SELECT d.id FROM server_deployments d
|
||||
WHERE d.id = ? AND (d.server_deleted_at IS NULL OR NOT EXISTS (SELECT 1 FROM servers s WHERE s.name = d.server))
|
||||
@@ -177,7 +168,7 @@ export function resumeCleanup(id: string): boolean {
|
||||
preserveOutcome(id);
|
||||
db().prepare("UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?").run(id);
|
||||
recordProgress(id, { phase: 'cleanup', blocker: null, checks: [], observedAt: null,
|
||||
event: { kind: 'resumed', message: 'Resumed local cleanup after confirmed Komodo deletion' } });
|
||||
event: { kind: 'resumed', message: 'Resumed local cleanup after confirmed instance removal' } });
|
||||
return true;
|
||||
})();
|
||||
}
|
||||
@@ -207,7 +198,7 @@ export function resumeRebuild(id: string): boolean {
|
||||
preserveOutcome(id);
|
||||
db().prepare("UPDATE server_deployments SET state = 'waiting', finished_at = NULL, error = NULL WHERE id = ?").run(id);
|
||||
recordProgress(id, { phase: 'cleanup', blocker: null, checks: [], observedAt: null,
|
||||
event: { kind: 'resumed', message: 'Resumed rebuild cleanup after confirmed Komodo deletion' } });
|
||||
event: { kind: 'resumed', message: 'Resumed rebuild cleanup after confirmed instance removal' } });
|
||||
return true;
|
||||
})();
|
||||
}
|
||||
@@ -218,7 +209,7 @@ export function resumeVerification(id: string): boolean {
|
||||
if (!canResumeVerification(id)) return false;
|
||||
preserveOutcome(id);
|
||||
db().prepare("UPDATE server_deployments SET state = 'awaiting-health', finished_at = NULL, error = NULL WHERE id = ?").run(id);
|
||||
deploymentStage(id, 'Waiting for Komodo health', { phase: 'verify-container', blocker: null, checks: [], observedAt: null,
|
||||
deploymentStage(id, 'Waiting for host agent health', { phase: 'verify-container', blocker: null, checks: [], observedAt: null,
|
||||
event: { kind: 'resumed', message: 'Resumed verification of the existing server' } });
|
||||
return true;
|
||||
})();
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// Execution is driven exclusively by the provisioning worker.
|
||||
export { requestLifecycle, requestRestart, requestWatchdogLifecycle, escalateWatchdogRestart, provisioningOperation, cancelProvisioning, releaseRestartRequest } from './requests.js';
|
||||
export type { LifecycleOperation, RestartSource } from './requests.js';
|
||||
export { queueCreateServer, destroyServer, previewCreation, creationQueue, nextAvailablePort, nextLobbyNumber } from './instances.js';
|
||||
export { queueCreateServer, destroyServer, previewCreation, creationQueue, nextLobbyNumber } from './instances.js';
|
||||
export { getDeployment, listDeployments, resumeVerification, resumeCleanup, resumeRebuild, cancelRemoval } from './history.js';
|
||||
export { deploymentActivity } from './activity.js';
|
||||
export { restartSettling, restartRecoveryPending } from './restarts.js';
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -45,7 +45,7 @@ export function nativeLaunchSelection(input: Record<string, unknown>, game: stri
|
||||
if (input.libraryId !== undefined && input.libraryId !== '' && input.libraryId !== libraryId) {
|
||||
throw new Error(`The current ${game.toUpperCase()} game files changed. Refresh the launch preview.`);
|
||||
}
|
||||
const image = input.image === undefined || input.image === '' ? config.komodo.defaultImage : input.image;
|
||||
const image = input.image === undefined || input.image === '' ? config.defaultRuntimeImage : input.image;
|
||||
if (typeof image !== 'string' || !/^[a-zA-Z0-9][a-zA-Z0-9._/:@-]{0,499}$/.test(image)) throw new Error('Invalid runtime image.');
|
||||
const automatic = input.automatic === true || !input.hostId;
|
||||
if (!automatic) {
|
||||
@@ -673,8 +673,8 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) {
|
||||
const recipe = prepared.recipe;
|
||||
db().prepare('UPDATE server_launch_recipes SET recipe=CAST(? AS TEXT),recipe_summary=? WHERE deployment_id=?')
|
||||
.run(Buffer.from(prepared.recipeJson.buffer, prepared.recipeJson.byteOffset, prepared.recipeJson.byteLength), prepared.recipeSummary, job.id);
|
||||
if (job.action === 'create') db().prepare(`INSERT INTO servers(name,komodo_type,komodo_name,game,map,game_mode,map_slug,host_dir,config_file,game_id,image,game_port,created_by,container_state)
|
||||
VALUES(?,'deployment','',?,?,'zombie',?,?,?,?,?,?,'native','starting')`).run(job.server, recipe.game, recipe.expectedMap, recipe.mapSlug, serverDir(job.server),
|
||||
if (job.action === 'create') db().prepare(`INSERT INTO servers(name,game,map,game_mode,map_slug,host_dir,config_file,game_id,image,game_port,created_by,container_state)
|
||||
VALUES(?,?,?,'zombie',?,?,?,?,?,?,'native','starting')`).run(job.server, recipe.game, recipe.expectedMap, recipe.mapSlug, serverDir(job.server),
|
||||
recipe.configFile, recipe.game === 't6' ? 't6zm' : 't5sp', recipe.image, instance!.port);
|
||||
for (const file of recipe.files) recordScript(job.server, file.path, file.versionId, file.sha256, file.size, 'native-create');
|
||||
recordProgress(job.id, { checkpoint: { key: 'prepare', label: 'Launch files selected and command payload prepared' } });
|
||||
|
||||
@@ -37,7 +37,7 @@ export function readProgress(row: { observation_json?: string | null }): Provisi
|
||||
|| (v.blocker.label !== undefined && !nullableString(v.blocker.label)))) return null;
|
||||
if (v.checks.length > 10 || !v.checks.every(c => object(c) && ['container', 'game', 'map', 'plutonium-release', 'plutonium-version', 'occupancy', 'stopped'].includes(String(c.key))
|
||||
&& typeof c.label === 'string' && ['pass', 'waiting', 'fail', 'unknown'].includes(String(c.result))
|
||||
&& (c.source === null || ['komodo', 'agent', 'rcon', 'telemetry', 'combined', 'manifest'].includes(String(c.source)))
|
||||
&& (c.source === null || ['agent', 'rcon', 'telemetry', 'combined', 'manifest'].includes(String(c.source)))
|
||||
&& typeof c.checkedAt === 'string' && nullableString(c.receivedAt) && scalar(c.expected) && scalar(c.actual) && nullableString(c.message))) return null;
|
||||
if (v.checkpoints.length > 100 || !v.checkpoints.every(c => object(c) && typeof c.key === 'string' && typeof c.label === 'string'
|
||||
&& typeof c.at === 'string' && (c.phase === null || phase(c.phase)))) return null;
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
import { maintenanceAllowed } from '../maintenance/policies.js';
|
||||
import { db } from '../db/index.js';
|
||||
import { komodoConfigured, rconConfigured } from '../config.js';
|
||||
import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js';
|
||||
import { beginDeployment, deploymentStage, finishDeployment, recordDeploymentEvent } from './history.js';
|
||||
import { queueRebuildServer } from './instances.js';
|
||||
import { restartRequestInFlight } from './restarts.js';
|
||||
import { runtimeActions, type RuntimeAction } from '../../provisioning.js';
|
||||
import { recordProgress } from './progress.js';
|
||||
import { nativeInstance, unplacedNativeCreation } from '../backends/native.js';
|
||||
@@ -51,22 +49,6 @@ export function requestRestart(server: string, source: RestartSource): Lifecycle
|
||||
.get(server, row.restarted_at, row.restarted_at) as { id: string } | undefined)?.id;
|
||||
if (id) db().prepare('UPDATE server_deployments SET submitted_at = COALESCE(submitted_at, ?) WHERE id = ?').run(row.restarted_at, id);
|
||||
}
|
||||
// Only not-yet-submitted campaign changes can share an activation. A later
|
||||
// file deployment must never be credited to an already running restart.
|
||||
// Restorations need their own activation. An older campaign can have
|
||||
// stale file preconditions after a beta; its failure must not strand
|
||||
// the rollback that would restore the server.
|
||||
const rollback = source.kind === 'campaign' && !!db().prepare("SELECT 1 FROM deployments WHERE id = ? AND action = 'rollback'").get(source.id);
|
||||
if (source.kind === 'campaign' && !row.restarted_at && !rollback && !nativeInstance(server)) {
|
||||
id = (db().prepare(`SELECT d.id FROM server_deployments d WHERE d.server = ? AND d.server_deleted_at IS NULL AND d.action = 'restart'
|
||||
AND d.state = 'waiting' AND d.submitted_at IS NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM native_provisioning n WHERE n.deployment_id=d.id AND n.stage_operation_id IS NOT NULL)
|
||||
AND EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'campaign')
|
||||
AND NOT EXISTS (SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind != 'campaign')
|
||||
AND NOT EXISTS (SELECT 1 FROM provisioning_sources s JOIN deployments c ON c.id = s.source_id
|
||||
WHERE s.operation_id = d.id AND s.kind = 'campaign' AND c.action = 'rollback')
|
||||
ORDER BY d.rowid LIMIT 1`).get(server) as { id: string } | undefined)?.id;
|
||||
}
|
||||
if (!id) {
|
||||
id = beginDeployment(server, 'restart', `${source.kind}:${source.id}`);
|
||||
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
|
||||
@@ -138,7 +120,7 @@ export function releaseRestartRequest(server: string, source: RestartSource): vo
|
||||
|
||||
export function requestLifecycle(server: string, action: RuntimeAction) {
|
||||
if (!runtimeActions.includes(action)) return { ok: false as const, status: 400, error: 'Choose start, restart, rebuild, or stop.' };
|
||||
if (!nativeInstance(server) && !unplacedNativeCreation(server) && (!komodoConfigured() || !rconConfigured())) return { ok: false as const, status: 503, error: 'Configure a host and RCON before provisioning.' };
|
||||
if (!nativeInstance(server) && !unplacedNativeCreation(server)) return { ok: false as const, status: 503, error: 'Server has no native host instance.' };
|
||||
return db().transaction(() => {
|
||||
const blocker = action === 'rebuild' ? activeBetaRemovalBlocker(server) : null;
|
||||
if (blocker) return { ok: false as const, status: 409, error: blocker };
|
||||
@@ -187,7 +169,7 @@ function watchdogRequestGuard(server: string, incidentId: string): Extract<Lifec
|
||||
const row = getServer(server);
|
||||
if (!row || row.destroyed_at) return { ok: false, status: 404, error: 'Server not found.' };
|
||||
if (row.desired_state === 'stopped') return { ok: false, status: 409, error: 'This server is intentionally stopped.' };
|
||||
if (!nativeInstance(server) && !komodoConfigured()) return { ok: false, status: 503, error: 'Configure a host before provisioning.' };
|
||||
if (!nativeInstance(server)) return { ok: false, status: 503, error: 'Configure a host before provisioning.' };
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -237,7 +219,7 @@ export function requestWatchdogLifecycle(server: string, action: 'restart' | 're
|
||||
JOIN watchdog_incidents i ON i.id = a.incident_id
|
||||
WHERE r.server = ? AND r.healthy_at IS NULL AND d.state IN ('failed','interrupted','cancelled')
|
||||
AND a.cycle < i.cycle`).get(incidentId, server) as { operation_id: string } | undefined;
|
||||
if (retained && !restartRequestInFlight(server)) {
|
||||
if (retained) {
|
||||
const transferred = db().prepare('UPDATE restart_rollout SET operation_id = ? WHERE server = ? AND operation_id = ? AND healthy_at IS NULL')
|
||||
.run(next.deploymentId, server, retained.operation_id);
|
||||
if (transferred.changes) {
|
||||
@@ -275,7 +257,7 @@ export function escalateWatchdogRestart(incidentId: string, operationId: string,
|
||||
const slot = db().prepare('SELECT operation_id, requested_at, healthy_at FROM restart_rollout WHERE server = ?').get(operation.server) as
|
||||
{ operation_id: string | null; requested_at: string; healthy_at: string | null } | undefined;
|
||||
const anchor = Date.parse(slot?.requested_at ?? operation.submitted_at ?? operation.started_at);
|
||||
if (restartRequestInFlight(operation.server) || (slot && (slot.operation_id !== operationId || slot.healthy_at))
|
||||
if ((slot && (slot.operation_id !== operationId || slot.healthy_at))
|
||||
|| !Number.isFinite(anchor) || Date.now() - anchor < Math.max(300_000, recoveryTimeoutMs)) {
|
||||
return { ok: false as const, status: 409, error: 'Waiting for the submitted restart to finish its recovery window.' };
|
||||
}
|
||||
|
||||
@@ -1,20 +1,9 @@
|
||||
import { fleetSettings } from '../settings.js';
|
||||
import { db } from '../db/index.js';
|
||||
import { provisioningCancelled } from './cancellation.js';
|
||||
import { komodoInspectHealthy, komodoRestart, komodoRuntimeAction } from '../integrations/komodo.js';
|
||||
import { probeServerStatus } from '../fleet/status-evidence.js';
|
||||
import { deploymentStage, finishDeployment } from './history.js';
|
||||
import { applyStagedFilesForOperation } from '../fleet/staged-files.js';
|
||||
import type { RuntimeAction } from '../../provisioning.js';
|
||||
import type { ProvisioningBlocker } from '../../provisioning-detail.js';
|
||||
import { recordProgress, type ProgressUpdate } from './progress.js';
|
||||
import { nativeDeployment } from '../backends/native.js';
|
||||
|
||||
// Komodo queues restart work. Don't mistake the still-running old container
|
||||
// for a completed restart when an API kick immediately follows the request.
|
||||
const HEALTH_POLL_MS = 15_000;
|
||||
const requestsInFlight = new Map<string, string>();
|
||||
export const restartRequestInFlight = (server: string) => requestsInFlight.has(server);
|
||||
|
||||
interface RestartSlot {
|
||||
server: string;
|
||||
@@ -37,7 +26,7 @@ export function restartRecoveryPending(server: string): boolean {
|
||||
/** The queued restart may still expose the previous healthy container. */
|
||||
export function restartSettling(server: string): boolean {
|
||||
const slot = slotFor(server);
|
||||
return requestsInFlight.has(server) || !!(slot && !slot.healthy_at
|
||||
return !!(slot && !slot.healthy_at
|
||||
&& Date.now() - Date.parse(slot.requested_at) < HEALTH_POLL_MS);
|
||||
}
|
||||
|
||||
@@ -46,7 +35,6 @@ export function rolloutWaitReason(server: string, operationId = '', action: Runt
|
||||
}
|
||||
|
||||
export function rolloutBlocker(server: string, operationId = '', action: RuntimeAction = 'restart'): ProvisioningBlocker | null {
|
||||
if (requestsInFlight.has(server)) return { code: 'operation-conflict', message: 'Waiting for the previous command submission to finish', automatic: true };
|
||||
const conflict = db().prepare(`SELECT id FROM server_deployments WHERE server = ? AND id != ? AND server_deleted_at IS NULL
|
||||
AND ((action IN ('start','stop','rebuild') AND state IN ('running','awaiting-health'))
|
||||
OR (action = 'destroy' AND state IN ('waiting','running')))`).get(server, operationId) as { id: string } | undefined;
|
||||
@@ -67,167 +55,6 @@ export function rolloutBlocker(server: string, operationId = '', action: Runtime
|
||||
? { code: 'restart-capacity', message: `Queued to ${action}; waiting for another server to recover`, automatic: true } : null;
|
||||
}
|
||||
|
||||
export type RolloutRestartResult =
|
||||
| { kind: 'started' }
|
||||
| { kind: 'queued'; message: string }
|
||||
| { kind: 'failed'; message: string };
|
||||
|
||||
/** Reserve capacity before any async I/O; callers still check for players first. */
|
||||
export async function restartInRollout(server: string, options: { operationId: string; action: 'start' | 'restart' }): Promise<RolloutRestartResult> {
|
||||
const operationId = options.operationId;
|
||||
const waitReason = db().transaction(() => {
|
||||
if (provisioningCancelled(operationId)) return 'Operation was cancelled';
|
||||
if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return 'Original server was deleted';
|
||||
const reason = rolloutWaitReason(server, operationId, options.action);
|
||||
if (reason) return reason;
|
||||
db().prepare(`INSERT INTO restart_rollout (server, requested_at, operation_id) VALUES (?, ?, ?)
|
||||
ON CONFLICT(server) DO UPDATE SET requested_at = excluded.requested_at,
|
||||
checked_at = NULL, healthy_at = NULL, players = NULL, error = NULL, operation_id = excluded.operation_id`)
|
||||
.run(server, new Date().toISOString(), operationId);
|
||||
recordProgress(operationId, { blocker: null, recovery: { state: 'held', message: 'Restart recovery capacity reserved.', operationId } });
|
||||
return null;
|
||||
}).immediate();
|
||||
if (waitReason) return { kind: 'queued', message: waitReason };
|
||||
|
||||
// Every gate, including rollout capacity, has cleared. No async work may
|
||||
// intervene between applying these files and submitting the restart.
|
||||
try {
|
||||
applyStagedFilesForOperation(operationId, server);
|
||||
} catch (err) {
|
||||
const message = `File activation failed: ${(err as Error).message}`;
|
||||
db().transaction(() => {
|
||||
db().prepare('DELETE FROM restart_rollout WHERE server = ? AND operation_id = ?').run(server, operationId);
|
||||
recordProgress(operationId, { recovery: { state: 'released', message: 'Capacity released before any remote command: file activation failed.', operationId } });
|
||||
finishDeployment(operationId, { ok: false, message });
|
||||
})();
|
||||
return { kind: 'failed', message };
|
||||
}
|
||||
db().transaction(() => {
|
||||
db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), operationId);
|
||||
deploymentStage(operationId, options.action === 'start' ? 'Starting container' : 'Restarting container', {
|
||||
phase: 'container-action', blocker: null, checks: [], checkpoint: { key: 'eligibility', label: 'Eligibility checks passed' },
|
||||
command: { state: 'attempted', message: `${options.action === 'start' ? 'Start' : 'Restart'} submission attempted.` }
|
||||
});
|
||||
})();
|
||||
requestsInFlight.set(server, operationId);
|
||||
db().prepare(`UPDATE servers SET container_state = 'restarting', last_error = NULL, rcon_failures = 0, last_reply_snippet = NULL,
|
||||
game_started_at = NULL, game_ended_at = NULL WHERE name = ?`).run(server);
|
||||
try {
|
||||
const result = options.action === 'start' ? await komodoRuntimeAction(server, 'start') : await komodoRestart(server);
|
||||
if (provisioningCancelled(operationId)) return { kind: 'failed', message: 'Operation was cancelled' };
|
||||
if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return { kind: 'failed', message: 'Original server was deleted' };
|
||||
if (result.ok) {
|
||||
db().transaction(() => {
|
||||
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ? AND server_deleted_at IS NULL").run(operationId);
|
||||
deploymentStage(operationId!, 'Waiting for container and game health (RCON or fresh telemetry)', {
|
||||
phase: 'verify-container', command: { state: 'accepted', message: 'Komodo accepted the request; recovery still requires verification.' },
|
||||
checkpoint: { key: 'container-action', label: 'Container command accepted' },
|
||||
blocker: { code: 'recovery-settling', message: 'Waiting for the recovery settling interval before checking health.', automatic: true }
|
||||
});
|
||||
})();
|
||||
return { kind: 'started' };
|
||||
}
|
||||
// These failures cannot have queued a remote restart. Other failures
|
||||
// (especially a timeout) may have: retain capacity until recovery.
|
||||
db().transaction(() => {
|
||||
if (result.kind === 'no-name' || result.kind === 'unconfigured') {
|
||||
db().prepare('DELETE FROM restart_rollout WHERE server = ? AND operation_id = ?').run(server, operationId);
|
||||
recordProgress(operationId, { command: { state: 'rejected', message: 'The command could not be submitted: ' + result.message },
|
||||
recovery: { state: 'released', message: 'Capacity released because no remote command could be submitted.', operationId } });
|
||||
} else {
|
||||
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ? AND operation_id = ?').run(`komodo: ${result.message}`, server, operationId);
|
||||
recordProgress(operationId, { command: { state: 'uncertain', message: 'The command may have reached Komodo: ' + result.message } });
|
||||
}
|
||||
finishDeployment(operationId!, { ok: false, message: `komodo: ${result.message}` });
|
||||
})();
|
||||
return { kind: 'failed', message: `komodo: ${result.message}` };
|
||||
} catch (err) {
|
||||
if (provisioningCancelled(operationId)) return { kind: 'failed', message: 'Operation was cancelled' };
|
||||
if (!db().prepare('SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL').get(operationId)) return { kind: 'failed', message: 'Original server was deleted' };
|
||||
const message = `komodo: ${(err as Error).message}`;
|
||||
db().transaction(() => {
|
||||
db().prepare('UPDATE restart_rollout SET error = ? WHERE server = ? AND operation_id = ?').run(message, server, operationId);
|
||||
recordProgress(operationId, { command: { state: 'uncertain', message: 'The command may have reached Komodo: ' + (err as Error).message } });
|
||||
finishDeployment(operationId!, { ok: false, message });
|
||||
})();
|
||||
return { kind: 'failed', message };
|
||||
} finally {
|
||||
// Start the grace period at completion, including slow/failed requests.
|
||||
db().prepare('UPDATE restart_rollout SET requested_at = ? WHERE server = ? AND operation_id = ?')
|
||||
.run(new Date().toISOString(), server, operationId);
|
||||
if (requestsInFlight.get(server) === operationId) requestsInFlight.delete(server);
|
||||
}
|
||||
}
|
||||
|
||||
let refreshInFlight: Promise<void> | null = null;
|
||||
|
||||
/** Poll all occupied slots, even if their originating beta was already closed. */
|
||||
export function refreshRestartRollout(): Promise<void> {
|
||||
if (refreshInFlight) return refreshInFlight;
|
||||
refreshInFlight = refresh().finally(() => { refreshInFlight = null; });
|
||||
return refreshInFlight;
|
||||
}
|
||||
|
||||
async function refresh(): Promise<void> {
|
||||
const slots = db().prepare('SELECT * FROM restart_rollout WHERE healthy_at IS NULL ORDER BY requested_at, server')
|
||||
.all() as RestartSlot[];
|
||||
for (const slot of slots) {
|
||||
if (slot.operation_id && nativeDeployment(slot.operation_id)) continue;
|
||||
if (requestsInFlight.has(slot.server)) continue;
|
||||
// Recovery capacity transferred to a rebuild remains reserved until
|
||||
// teardown. Its health is verified by the creation worker, never here.
|
||||
if (slot.operation_id && db().prepare(`SELECT 1 FROM server_deployments d WHERE d.id = ? AND d.server_deleted_at IS NULL AND (
|
||||
(d.action = 'rebuild' AND d.state IN ('waiting','queued','running','awaiting-health'))
|
||||
OR (d.action = 'restart' AND d.state = 'waiting' AND d.submitted_at IS NULL AND EXISTS (
|
||||
SELECT 1 FROM provisioning_sources s WHERE s.operation_id = d.id AND s.kind = 'watchdog')))`)
|
||||
.get(slot.operation_id)) continue;
|
||||
if (Date.now() - Date.parse(slot.checked_at ?? slot.requested_at) < HEALTH_POLL_MS) continue;
|
||||
let error: string | null = null;
|
||||
let players: number | null = null;
|
||||
const checks: NonNullable<ProgressUpdate['checks']> = [];
|
||||
try {
|
||||
const health = await komodoInspectHealthy(slot.server);
|
||||
checks.push({ key: 'container', label: 'Container readiness', result: health.ok ? 'pass' : 'waiting', source: 'komodo', actual: health.state ?? null,
|
||||
checkedAt: new Date().toISOString(), message: health.ok ? null : health.reason });
|
||||
if (!health.ok) {
|
||||
error = `unhealthy: ${health.reason}${health.snippet ? ` snippet=${health.snippet.slice(0, 400)}` : ''}`;
|
||||
} else {
|
||||
const status = await probeServerStatus(slot.server, slot.requested_at);
|
||||
checks.push({ key: 'game', label: 'Game status', result: status.ok ? 'pass' : 'waiting', source: status.ok ? status.source : null,
|
||||
checkedAt: status.observedAt, receivedAt: status.ok ? status.receivedAt : null, actual: status.ok ? status.players : null,
|
||||
message: status.ok ? null : status.reason });
|
||||
if (!status.ok) error = `rcon not answering and no fresh telemetry: ${status.reason}`;
|
||||
else players = status.players;
|
||||
}
|
||||
} catch (err) {
|
||||
error = `health check: ${(err as Error).message}`;
|
||||
checks.push({ key: checks.length ? 'game' : 'container', label: checks.length ? 'Game status' : 'Container readiness', result: 'unknown', source: null, message: error });
|
||||
}
|
||||
const now = new Date().toISOString();
|
||||
db().transaction(() => {
|
||||
const checked = db().prepare(`UPDATE restart_rollout SET checked_at = ?, healthy_at = ?, players = ?, error = ?
|
||||
WHERE server = ? AND requested_at = ? AND operation_id IS ? AND healthy_at IS NULL`)
|
||||
.run(now, error ? null : now, players, error, slot.server, slot.requested_at, slot.operation_id);
|
||||
if (!checked.changes) return;
|
||||
if (slot.operation_id) recordProgress(slot.operation_id, {
|
||||
phase: checks.some(check => check.key === 'game') ? 'verify-game' : 'verify-container', checks, observedAt: now,
|
||||
blocker: error ? { code: 'recovery-health', message: error, automatic: true } : null,
|
||||
recovery: { state: error ? 'held' : 'released', at: now, operationId: slot.operation_id,
|
||||
message: error ? 'Restart capacity remains reserved while recovery is checked.' : 'Container and game recovery verified; restart capacity released.' },
|
||||
checkpoints: error ? [] : [{ key: 'verify-container', label: 'Container readiness verified', at: now }, { key: 'verify-game', label: 'Game status verified', at: now }]
|
||||
});
|
||||
if (slot.operation_id && db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'awaiting-health'").get(slot.operation_id)) {
|
||||
db().prepare('UPDATE server_deployments SET checked_at = ?, players = ? WHERE id = ?').run(now, players, slot.operation_id);
|
||||
if (error) deploymentStage(slot.operation_id, 'Waiting for health: ' + error);
|
||||
else {
|
||||
deploymentStage(slot.operation_id, 'Verified container and game health (RCON or fresh telemetry)');
|
||||
finishDeployment(slot.operation_id, { ok: true });
|
||||
}
|
||||
}
|
||||
})();
|
||||
}
|
||||
}
|
||||
|
||||
export function rolloutHealth(server: string): { ok: true; players: number } | { ok: false; reason: string } {
|
||||
const slot = slotFor(server);
|
||||
if (slot?.healthy_at) return { ok: true, players: slot.players ?? 0 };
|
||||
|
||||
@@ -1,24 +1,11 @@
|
||||
import { maintenanceAllowed } from '../maintenance/policies.js';
|
||||
import { resolveRecoveredHealthOperation } from '../maintenance/recovery.js';
|
||||
import { db } from '../db/index.js';
|
||||
import { provisioningCancelled } from './cancellation.js';
|
||||
import { komodoConfigured, rconConfigured } from '../config.js';
|
||||
import { getServer } from '../fleet/servers.js';
|
||||
import { deploymentStage, finishDeployment, REMOVING_SERVER } from './history.js';
|
||||
import { processCreations, processDestructions, rebuildServer } from './instances.js';
|
||||
import { komodoInspectHealthy, komodoRuntimeAction } from '../integrations/komodo.js';
|
||||
import { latestGameSubmission, probePlayerEvidence } from '../fleet/status-evidence.js';
|
||||
import { refreshRestartRollout, restartInRollout, rolloutBlocker } from './restarts.js';
|
||||
import { rolloutBlocker } from './restarts.js';
|
||||
import { cancelProvisioning } from './requests.js';
|
||||
import type { RuntimeAction } from '../../provisioning.js';
|
||||
import type { ProvisioningBlocker } from '../../provisioning-detail.js';
|
||||
import { recordProgress } from './progress.js';
|
||||
import { nativeInstance, nativeDeployment } from '../backends/native.js';
|
||||
import { processNativeProvisioning } from './native.js';
|
||||
import { trackActivity } from '../pipeline/activity.js';
|
||||
|
||||
const stoppedStates = new Set(['stopped', 'exited', 'created', 'dead']);
|
||||
export const isStopped = (state?: string) => stoppedStates.has(state?.toLowerCase() ?? '');
|
||||
|
||||
interface Source { kind: 'campaign' | 'beta' | 'maintenance' | 'watchdog'; source_id: string }
|
||||
const sourcesFor = (id: string) => db().prepare('SELECT kind, source_id FROM provisioning_sources WHERE operation_id = ?').all(id) as Source[];
|
||||
@@ -92,20 +79,13 @@ function blocker(server: string, id: string, action: RuntimeAction): Provisionin
|
||||
}
|
||||
|
||||
interface Operation { id: string; server: string; action: RuntimeAction; state: string; stage: string }
|
||||
let ticking: Promise<void> | null = null;
|
||||
let awaitedPass: Promise<void> | null = null;
|
||||
export function processProvisioning(options: { wait?: boolean } = {}): Promise<void> {
|
||||
if (options.wait !== false && awaitedPass) return awaitedPass;
|
||||
// Native jobs own their individual flights. A slow observation must not
|
||||
// prevent the next scheduler pass admitting newly arrived native work.
|
||||
const native = nativePass(options);
|
||||
if (!ticking) ticking = trackActivity('legacy-provisioning', 'Processing legacy provisioning', run)
|
||||
.finally(() => { ticking = null; });
|
||||
if (options.wait === false) {
|
||||
void ticking.catch(error => console.error('[worker] legacy provisioning error:', error));
|
||||
return native;
|
||||
}
|
||||
awaitedPass = Promise.all([native, ticking]).then(() => {}).finally(() => { awaitedPass = null; });
|
||||
// Each job owns its flight, so slow observations do not block new work.
|
||||
const pass = nativePass(options);
|
||||
if (options.wait === false) return pass;
|
||||
awaitedPass = pass.finally(() => { awaitedPass = null; });
|
||||
return awaitedPass;
|
||||
}
|
||||
|
||||
@@ -126,148 +106,3 @@ function nativePass(options: { wait?: boolean }): Promise<void> {
|
||||
return blocker(operation.server, operation.id, typed.action)?.message ?? null;
|
||||
}, options);
|
||||
}
|
||||
|
||||
async function run(): Promise<void> {
|
||||
await processCreations();
|
||||
await processDestructions();
|
||||
await refreshRestartRollout();
|
||||
const operations = db().prepare(`SELECT id, server, action, state, stage FROM server_deployments
|
||||
WHERE server_deleted_at IS NULL AND action IN ('start','restart','stop','rebuild') AND (state = 'waiting' OR (action = 'stop' AND state = 'awaiting-health'))
|
||||
ORDER BY started_at, rowid`).all() as Operation[];
|
||||
for (const operation of operations) {
|
||||
if (nativeDeployment(operation.id)) continue;
|
||||
try { await processOperation(operation); }
|
||||
catch (err) {
|
||||
if (provisioningCancelled(operation.id)) continue;
|
||||
// Observation failures are retried; remote mutations are never replayed.
|
||||
deploymentStage(operation.id, 'Waiting for verification: ' + (err as Error).message, {
|
||||
checks: [], blocker: { code: 'observation-unavailable', message: (err as Error).message, automatic: true }
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function processOperation(operation: Operation): Promise<void> {
|
||||
const { id, server, action } = operation;
|
||||
if (provisioningCancelled(id)) return;
|
||||
// An explicit cleanup resume follows confirmed remote deletion. Complete
|
||||
// that durable checkpoint even if the incident now needs attention or the
|
||||
// Watchdog is paused; never strand the reserved replacement identity.
|
||||
if (action === 'rebuild' && operation.stage === REMOVING_SERVER) { await rebuildServer(id); return; }
|
||||
if (operation.state === 'waiting') {
|
||||
const heldBy: Source[] = [];
|
||||
const eligibility = sourceEligibility(operation, source => heldBy.push(source));
|
||||
if (eligibility === 'cancelled') { cancelProvisioning(id, 'Requester cancelled or replaced this operation'); return; }
|
||||
if (eligibility === 'held') {
|
||||
const source = heldBy[0];
|
||||
const href = source ? source.kind === 'campaign' ? `/campaigns/${encodeURIComponent(source.source_id)}`
|
||||
: source.kind === 'beta' ? `/betas/${encodeURIComponent(source.source_id)}`
|
||||
: source.kind === 'watchdog' ? `/maintenance/${encodeURIComponent(source.source_id)}`
|
||||
: `/maintenance?view=activity#job-${encodeURIComponent(source.source_id)}` : null;
|
||||
deploymentStage(id, 'Waiting for the requester to resume', { phase: 'eligibility', checks: [],
|
||||
blocker: { code: 'requester-held', message: 'Waiting for the requester to resume', automatic: false, href, label: 'Requesting work' } });
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (!getServer(server)) { finishDeployment(id, { ok: false, message: 'Server is no longer registered.' }); return; }
|
||||
const watchdog = sourcesFor(id).some(source => source.kind === 'watchdog');
|
||||
if (!komodoConfigured() || (!rconConfigured() && !watchdog)) { deploymentStage(id, 'Waiting for Komodo and RCON configuration', {
|
||||
phase: 'eligibility', checks: [], blocker: { code: 'configuration', message: 'Waiting for Komodo and RCON configuration', automatic: false, href: '/settings', label: 'Integration settings' }
|
||||
}); return; }
|
||||
if (operation.state === 'awaiting-health') {
|
||||
const health = await komodoInspectHealthy(server);
|
||||
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'awaiting-health'").get(id)) return;
|
||||
recordProgress(id, { phase: 'verify-stopped', checks: [{ key: 'stopped', label: 'Container stopped', source: 'komodo',
|
||||
result: isStopped(health.state) ? 'pass' : health.state ? 'waiting' : 'unknown', actual: health.state ?? null,
|
||||
message: health.ok ? null : health.reason }], blocker: isStopped(health.state) ? null
|
||||
: { code: 'stopped-state', message: 'Waiting for an explicit stopped container state.', automatic: true } });
|
||||
if (isStopped(health.state)) {
|
||||
db().transaction(() => {
|
||||
db().prepare("UPDATE servers SET container_state = ?, last_players = NULL, last_roster = NULL, game_started_at = NULL, game_ended_at = NULL WHERE name = ?").run(health.state, server);
|
||||
deploymentStage(id, 'Verified container stopped', { checkpoint: { key: 'verify-stopped', label: 'Stopped container state verified' } });
|
||||
finishDeployment(id, { ok: true });
|
||||
})();
|
||||
} else deploymentStage(id, 'Waiting for container to stop' + (!health.ok ? ': ' + health.reason : ''));
|
||||
return;
|
||||
}
|
||||
const reason = blocker(server, id, action);
|
||||
if (reason) { deploymentStage(id, reason.message, { phase: 'eligibility', checks: [], blocker: reason }); return; }
|
||||
const healthRecovery = watchdog && !!db().prepare(`SELECT 1 FROM provisioning_sources s
|
||||
JOIN watchdog_incidents i ON i.id = s.source_id JOIN server_deployments d ON d.id = s.operation_id
|
||||
WHERE s.operation_id = ? AND s.kind = 'watchdog' AND i.kind = 'health' AND d.triggered_by LIKE 'watchdog:%'`).get(id);
|
||||
const epoch = latestGameSubmission(server);
|
||||
const health = await komodoInspectHealthy(server);
|
||||
if (provisioningCancelled(id)) return;
|
||||
if (healthRecovery && epoch !== latestGameSubmission(server)) return;
|
||||
if (healthRecovery && !health.ok && !['stopped','exited','created','dead','unhealthy','starting','restarting'].includes(health.state?.toLowerCase() ?? '')) {
|
||||
deploymentStage(id, 'Waiting for current container health before automatic recovery', { phase: 'eligibility', checks: [],
|
||||
blocker: { code: 'observation-unavailable', message: health.reason, automatic: true } });
|
||||
return;
|
||||
}
|
||||
if (action !== 'start' && !isStopped(health.state)) {
|
||||
const evidence = await probePlayerEvidence(server);
|
||||
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return;
|
||||
const players = evidence.players;
|
||||
if (healthRecovery && epoch !== latestGameSubmission(server)) return;
|
||||
if (healthRecovery && health.ok && players !== null) {
|
||||
// The fresh occupancy check also proves that the game recovered.
|
||||
// Empty is not a reason to execute an obsolete health restart.
|
||||
resolveRecoveredHealthOperation(id, players, evidence.source);
|
||||
return;
|
||||
}
|
||||
recordProgress(id, { phase: 'eligibility', checks: [{ key: 'occupancy', label: 'Player occupancy', result: players === 0 ? 'pass' : players === null ? 'unknown' : 'waiting',
|
||||
source: evidence.source, checkedAt: evidence.observedAt, receivedAt: evidence.receivedAt, expected: 0, actual: players }],
|
||||
blocker: players === 0 ? null : { code: players === null ? 'players-unknown' : 'players-present',
|
||||
message: players === null ? 'Waiting for a fresh player count' : `Waiting for ${players} player(s) to leave`, automatic: true } });
|
||||
db().prepare('UPDATE server_deployments SET players = ?, checked_at = ? WHERE id = ?').run(players, new Date().toISOString(), id);
|
||||
if (players !== 0) {
|
||||
deploymentStage(id, players === null ? 'Waiting for a fresh player count' : `Waiting for ${players} player(s) to leave`);
|
||||
return;
|
||||
}
|
||||
} else if (action !== 'start') {
|
||||
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return;
|
||||
recordProgress(id, { phase: 'eligibility', blocker: null, checks: [{ key: 'stopped', label: 'Stopped container permits disruption',
|
||||
source: 'komodo', result: 'pass', actual: health.state ?? null }] });
|
||||
}
|
||||
// Requests can arrive during the probes. Recheck before claiming the action.
|
||||
const latestReason = blocker(server, id, action);
|
||||
if (latestReason) { deploymentStage(id, latestReason.message, { phase: 'eligibility', blocker: latestReason }); return; }
|
||||
if (sourceEligibility(operation) !== 'ready') return;
|
||||
if (!db().prepare("SELECT 1 FROM server_deployments WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").get(id)) return;
|
||||
if (action === 'rebuild') { await rebuildServer(id); return; }
|
||||
const claim = db().prepare("UPDATE server_deployments SET state = 'running' WHERE id = ? AND server_deleted_at IS NULL AND state = 'waiting'").run(id);
|
||||
if (!claim.changes) return;
|
||||
if (action === 'start' || action === 'restart') {
|
||||
const result = await restartInRollout(server, { operationId: id, action });
|
||||
if (provisioningCancelled(id)) return;
|
||||
if (result.kind === 'queued') {
|
||||
db().prepare("UPDATE server_deployments SET state = 'waiting' WHERE id = ?").run(id);
|
||||
deploymentStage(id, result.message);
|
||||
}
|
||||
return;
|
||||
}
|
||||
db().transaction(() => {
|
||||
db().prepare('UPDATE server_deployments SET submitted_at = ? WHERE id = ?').run(new Date().toISOString(), id);
|
||||
deploymentStage(id, 'Stopping container', { phase: 'container-action', blocker: null, checks: [],
|
||||
checkpoint: { key: 'eligibility', label: 'Eligibility checks passed' }, command: { state: 'attempted', message: 'Stop submission attempted.' } });
|
||||
})();
|
||||
// Persist intent first; a lost reply must not issue another stop later.
|
||||
try {
|
||||
const result = await komodoRuntimeAction(server, 'stop');
|
||||
if (provisioningCancelled(id)) return;
|
||||
if (!result.ok) {
|
||||
recordProgress(id, { command: { state: result.kind === 'no-name' || result.kind === 'unconfigured' ? 'rejected' : 'uncertain', message: result.message } });
|
||||
finishDeployment(id, { ok: false, message: result.message }); return;
|
||||
}
|
||||
db().transaction(() => {
|
||||
db().prepare("UPDATE server_deployments SET state = 'awaiting-health' WHERE id = ?").run(id);
|
||||
deploymentStage(id, 'Waiting for container to stop', { phase: 'verify-stopped',
|
||||
checkpoint: { key: 'container-action', label: 'Stop command accepted' },
|
||||
command: { state: 'accepted', message: 'Komodo accepted the stop request; stopped state still requires verification.' } });
|
||||
})();
|
||||
} catch (err) {
|
||||
if (provisioningCancelled(id)) return;
|
||||
recordProgress(id, { command: { state: 'uncertain', message: (err as Error).message } });
|
||||
finishDeployment(id, { ok: false, message: (err as Error).message });
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user