josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Remove Komodo backend in favor of native hosts

Browse Source
This commit is contained in:
josh
2026-10-05 18:07:07 -04:00
parent 701066aaf8
commit 4b0f341d64
116 changed files with 557 additions and 8132 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/fleet/beta.ts
+1 -1
View File
@@ -376,7 +376,7 @@ export async function openBeta(input: OpenBetaInput): Promise<OpenBetaResult> {
const removal = db().prepare(`SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL
AND action IN ('destroy', 'rebuild') AND (state IN ('waiting', 'queued', 'running', 'awaiting-health')
OR (state IN ('failed', 'interrupted') AND stage IN (
'Stopping and removing container', 'Deleting Komodo deployment', 'Removing server files and registry')))
'Stopping and removing container', 'Removing server files and registry')))
LIMIT 1`).get(name);
if (removal) throw new BetaError(`${name}: cancel or finish the pending destruction or rebuild before opening a beta.`, 'conflict');
serverRows.push({ name, game: row.game, map: row.map });
src/lib/server/fleet/collections.ts
+18 -47
View File
@@ -2,16 +2,14 @@ import { fleetSettings } from '../settings.js';
import { randomUUID } from 'node:crypto';
import type { CollectionDetail, CollectionMode, CollectionPreview, CollectionSummary } from '$lib/deployment-plan.js';
import { db } from '../db/index.js';
import { config, komodoConfigured, rconConfigured } from '../config.js';
import { blueprintEntries, summarizeRecipe, type BlueprintEntry } from './launch-recipes.js';
import { creationQueue, destroyServer, nextAvailablePort, queueCreateServer } from '../provisioning/index.js';
import { creationQueue, destroyServer } from '../provisioning/index.js';
import { deploymentStage, finishDeployment, recordDeploymentEvent } from './deployments.js';
import { housePartyTargets } from './house-party.js';
import { serverObservation } from './status.js';
import { activeBetaRemovalBlocker, getServer } from './servers.js';
import { parseServerName, zombieMapBySlug, zombieMaps } from './map-aliases.js';
import { nativeLaunchSelection, queueNativeCreationIntent } from '../provisioning/native.js';
import { listHosts } from '../hosts/index.js';
import { nativeDeployment, nativeInstance } from '../backends/native.js';
import { launchAvailability } from '../hosts/availability.js';
@@ -71,7 +69,7 @@ function collectionPlan(entries: BlueprintEntry[], mode: CollectionMode, include
return { counts, retainedServers, removals };
}
function allocate(entries: BlueprintEntry[], releasingServers: string[] = [], counts = coverage(entries)): CollectionPreview['servers'] {
function allocate(entries: BlueprintEntry[], counts = coverage(entries)): CollectionPreview['servers'] {
const occupied = db().prepare(`SELECT name FROM servers UNION
SELECT j.server AS name FROM server_creation_jobs j JOIN server_deployments d ON d.id=j.deployment_id
WHERE d.server_deleted_at IS NULL AND (d.state IN ('queued','running','awaiting-health')
@@ -85,16 +83,14 @@ function allocate(entries: BlueprintEntry[], releasingServers: string[] = [], co
taken.add(Number(match[2])); lobbies.set(match[1], taken);
}
const savedKeys = new Set((db().prepare("SELECT server_id FROM server_keys WHERE length(trim(pluto_key))>0").all() as { server_id: string }[]).map(row => row.server_id));
const ports: number[] = [];
return entries.flatMap((entry, entryIndex) => Array.from({ length: counts[entryIndex].missing }, () => {
const base = `${entry.recipe.game}_zombie_${entry.recipe.mapSlug}`;
const taken = lobbies.get(base) ?? new Set<number>();
const lobby = Array.from({ length: 20 }, (_, index) => index + 1).find(number => !taken.has(number)) ?? null;
const name = lobby === null ? null : `${base}_${lobby}`;
const native = entry.recipe.native;
const port = native ? null : nextAvailablePort(ports, releasingServers);
const port = null;
if (lobby !== null) { taken.add(lobby); lobbies.set(base, taken); }
if (port !== null) ports.push(port);
return { entryIndex, name, port, ...(native ? { native } : {}), hasSavedKey: !!name && savedKeys.has(name) };
}));
}
@@ -102,13 +98,8 @@ function allocate(entries: BlueprintEntry[], releasingServers: string[] = [], co
function blockersFor(servers: CollectionPreview['servers'], entries: BlueprintEntry[] = []): string[] {
const blockers: string[] = [];
if (!servers.length) return blockers;
const legacy = servers.filter(server => !server.native && !entries[server.entryIndex]?.recipe.native);
if (legacy.length) {
if (!komodoConfigured() || !config.komodo.targetServer) blockers.push('Select and prepare a host before launching.');
if (!rconConfigured()) blockers.push('Configure RCON_PASSWORD so every new server can be verified.');
}
if (servers.some(server => !server.native && !entries[server.entryIndex]?.recipe.native)) blockers.push('Select and prepare a host before launching.');
if (servers.some(server => !server.name)) blockers.push('There are not enough free lobbies for one or more maps (20 per map). Reduce their counts or free capacity.');
if (legacy.some(server => server.port === null)) blockers.push(`There are not enough free game ports in ${config.portRange.min}–${config.portRange.max} for the whole collection.`);
return blockers;
}
@@ -120,7 +111,7 @@ function removalBlockersFor(removals: CollectionPreview['removals']): string[] {
AND action='create' AND state IN ('queued','running','awaiting-health') ORDER BY rowid DESC LIMIT 1`).get(name) as { id: string } | undefined;
return !!creation && nativeDeployment(creation.id);
};
if (removals.some(server => !native(server.name)) && (!komodoConfigured() || !rconConfigured())) blockers.push('Configure the legacy backend before removing its servers.');
if (removals.some(server => !native(server.name))) blockers.push('Every server to remove must have a native host instance.');
for (const server of removals) {
const blocker = activeBetaRemovalBlocker(server.name);
if (blocker) blockers.push(blocker);
@@ -133,15 +124,13 @@ export function previewCollection(blueprintId: unknown, mode: unknown = 'fill',
if (mode !== 'fill' && mode !== 'exact') throw new Error('Choose fill or exact collection mode.');
return db().transaction((): CollectionPreview => {
const { entries, blueprintName } = blueprintEntries(blueprintId);
if (selection?.hostId !== undefined || !config.komodo.address || listHosts().length > 0) {
const libraries = selection?.libraryIds as Record<string, unknown> | undefined;
for (const entry of entries) {
entry.recipe.native = nativeLaunchSelection({ ...selection, libraryId: libraries?.[entry.recipe.game] ?? selection?.libraryId }, entry.recipe.game);
entry.recipe.image = entry.recipe.native.image!;
}
const libraries = selection?.libraryIds as Record<string, unknown> | undefined;
for (const entry of entries) {
entry.recipe.native = nativeLaunchSelection({ ...selection, libraryId: libraries?.[entry.recipe.game] ?? selection?.libraryId }, entry.recipe.game);
entry.recipe.image = entry.recipe.native.image!;
}
const { counts, retainedServers, removals } = collectionPlan(entries, mode);
const servers = allocate(entries, removals.map(server => server.name), counts);
const servers = allocate(entries, counts);
const blockers = [...blockersFor(servers, entries), ...removalBlockersFor(removals)];
const selections = servers.flatMap(server => server.native ? [server.native] : []);
const availability = selections.length ? launchAvailability(selections) : {};
@@ -176,16 +165,12 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
throw new Error('The fleet changed. Nothing was queued. Refresh the preview and review the servers to keep and remove.');
}
const { counts, retainedServers, removals } = collectionPlan(entries, preview.mode, false);
const releasingServers = removals.map(server => server.name);
const nativeReview = reviewed.every(server => server.native || entries[server.entryIndex]?.recipe.native);
// Native Apply accepts the reviewed logical identities. Capacity and
// physical ports can change while this request waits in the queue.
const servers = nativeReview ? reviewed : allocate(entries, releasingServers, counts);
const servers = reviewed;
const validatedSelections = new Set<string>();
for (const server of servers) {
const recipe = entries[server.entryIndex].recipe;
const selection = server.native ?? recipe.native;
if (!selection) continue;
if (!selection) throw new Error('Select a native host before launching. Refresh the preview.');
const identity = JSON.stringify([recipe.game, selection]);
if (validatedSelections.has(identity)) continue;
// Recheck saved configuration once per selection without allocating
@@ -195,9 +180,8 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
}
const blockers = [...blockersFor(servers, entries), ...removalBlockersFor(removals)];
if (blockers.length) throw new Error(blockers.join(' '));
if (entries.some((_entry, index) => counts[index].missing !== reviewed.filter(server => server.entryIndex === index).length)
|| servers.length !== reviewed.length || servers.some((server, i) => server.entryIndex !== reviewed[i]?.entryIndex || server.name !== reviewed[i]?.name || (!server.native && server.port !== reviewed[i]?.port))) {
throw new Error('Fleet counts, lobby or port availability changed. Nothing was queued. Refresh the preview and review each server key.');
if (entries.some((_entry, index) => counts[index].missing !== reviewed.filter(server => server.entryIndex === index).length)) {
throw new Error('Fleet counts changed. Nothing was queued. Refresh the preview and review each server key.');
}
const expected = new Set(servers.map(server => server.name));
for (const [name, value] of Object.entries(keys)) {
@@ -207,10 +191,7 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
const collectionId = randomUUID();
db().prepare('INSERT INTO server_launch_collections (id, name, created_at, mode, retained_servers) VALUES (?, ?, ?, ?, ?)')
.run(collectionId, preview.blueprint_name, new Date().toISOString(), preview.mode, JSON.stringify(retainedServers));
const removalJobs = removals.map((server, position) => {
const ports = (db().prepare(`SELECT game_port AS port FROM servers WHERE name = ? UNION
SELECT j.port FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id
WHERE j.server = ? AND d.server_deleted_at IS NULL AND d.state IN ('queued', 'running', 'awaiting-health')`).all(server.name, server.name) as { port: number | null }[]).map(row => row.port);
removals.forEach((server, position) => {
const queued = db().prepare("SELECT id FROM server_deployments WHERE server = ? AND server_deleted_at IS NULL AND action = 'create' AND state = 'queued'").get(server.name) as { id: string } | undefined;
let id: string;
if (queued && !getServer(server.name)) {
@@ -230,27 +211,17 @@ export function queueCollection(input: { previewId?: unknown; keys?: unknown }):
id = result.deploymentId;
}
db().prepare('INSERT INTO server_collection_removals (collection_id, deployment_id, position) VALUES (?, ?, ?)').run(collectionId, id, position);
return { id, ports };
});
servers.forEach((server, index) => {
const recipe = { ...entries[server.entryIndex].recipe, ...(server.native ? { native: server.native } : {}) };
let result;
if (recipe.native) result = queueNativeCreationIntent({ server: server.name!, game: recipe.game, mapSlug: recipe.mapSlug,
if (!recipe.native) throw new Error('Select a native host before launching. Refresh the preview.');
const result = queueNativeCreationIntent({ server: server.name!, game: recipe.game, mapSlug: recipe.mapSlug,
selection: recipe.native, recipe, plutoKey: keys[server.name!] as string | undefined, createdBy: 'ui', blueprintName: preview.blueprint_name });
else {
const childPreviewId = randomUUID();
db().prepare('INSERT INTO server_launch_previews (id, expires_at, server, port, recipe, blueprint_name) VALUES (?, ?, ?, ?, ?, ?)')
.run(childPreviewId, preview.expires_at, server.name, server.port, JSON.stringify(recipe), preview.blueprint_name);
result = queueCreateServer({ game: recipe.game, gameMode: 'zombie', mapSlug: recipe.mapSlug,
expectedName: server.name!, previewId: childPreviewId, plutoKey: keys[server.name!] as string | undefined, createdBy: 'ui' }, releasingServers);
}
// Throw so the outer transaction also reverses all earlier keys and children.
if (!result.ok) throw new Error(`${server.name}: ${result.message} Nothing was queued.`);
db().prepare('INSERT INTO server_collection_members (collection_id, deployment_id, position) VALUES (?, ?, ?)')
.run(collectionId, result.deploymentId, index);
for (const removal of recipe.native ? [] : removalJobs.filter(removal => removal.ports.includes(server.port))) {
db().prepare('INSERT INTO server_creation_dependencies (deployment_id, removal_id) VALUES (?, ?)').run(result.deploymentId, removal.id);
}
});
db().prepare('UPDATE server_collection_previews SET collection_id = ? WHERE id = ?').run(collectionId, preview.id);
return { ok: true as const, collectionId };
src/lib/server/fleet/health-observations.ts
+6 -13
View File
@@ -3,7 +3,7 @@ import { config } from '../config.js';
import { db } from '../db/index.js';
import { getHost, queueHostOperation, waitHostOperationStatus } from '../hosts/index.js';
import { nativeInstance, type NativeInstance } from '../backends/native.js';
import { komodoInspectHealthy } from '../integrations/komodo.js';
import { nativeHealth } from '../backends/native.js';
import { rconStatusResult, agentRconStatusResult, type AgentRconReply } from '../integrations/rcon.js';
import { withOperationSource } from '../hosts/operation-context.js';
import { getServer } from './servers.js';
@@ -39,7 +39,6 @@ function identity(server: string, probe: Probe): Identity {
return {
key: JSON.stringify([row.created_at, row.desired_state, latestGameSubmission(server), lifecycle,
native ? [native.host_id, native.id, native.generation] : null,
native ? null : [row.komodo_type, row.komodo_name, row.komodo_server, config.komodo, config.httpTimeoutMs],
probe === 'rcon' ? [row.rcon_host, config.rcon] : null]),
port: probe === 'rcon' ? native?.port ?? row.game_port : null,
checkedAt: probe === 'rcon' ? row.last_checked_at : row.container_checked_at
@@ -77,13 +76,7 @@ async function observe<T extends { ok: boolean }>(server: string, fresh: boolean
while (entries.size > MAX_SERVERS) entries.delete(entries.keys().next().value!);
const pending = Promise.resolve().then(() => request(server)).then(result => {
const latest = identity(server, probe);
// Legacy RCON may discover and save its first port during this request.
// Accept that enrichment only when all other identity fields agree.
const endpoint = 'endpoint' in result ? result.endpoint : null;
const learnedPort = probe === 'rcon' && !current.port && result.ok
&& endpoint !== null && typeof endpoint === 'object' && 'port' in endpoint && endpoint.port === latest.port;
if (current.key !== latest.key || (current.port !== latest.port && !learnedPort)
|| newerEvidence(latest, entry.startedAt)) {
if (!sameIdentity(current, latest) || newerEvidence(latest, entry.startedAt)) {
throw new MonitoringChangedError('Server changed during monitoring; the previous observation was discarded.');
}
const observation = { result, observedAt: new Date().toISOString() };
@@ -100,12 +93,12 @@ async function observe<T extends { ok: boolean }>(server: string, fresh: boolean
return pending;
}
const inspections = new Map<string, Entry<Awaited<ReturnType<typeof komodoInspectHealthy>>>>();
const inspections = new Map<string, Entry<Awaited<ReturnType<typeof nativeHealth>>>>();
const rcon = new Map<string, Entry<Awaited<ReturnType<typeof rconStatusResult>>>>();
interface MonitoringStatus {
identity: string;
container: Observation<Awaited<ReturnType<typeof komodoInspectHealthy>>>;
container: Observation<Awaited<ReturnType<typeof nativeHealth>>>;
game?: Observation<Awaited<ReturnType<typeof rconStatusResult>>>;
}
// Keep failed samples too, independently of the successful-probe cache. Reading
@@ -160,7 +153,7 @@ export function monitoringStatus(server: string): MonitoringStatus | null {
}
/** Keep the actual sample time: a later heartbeat must not renew old evidence. */
export function heartbeatInspection(server: string): Observation<Awaited<ReturnType<typeof komodoInspectHealthy>>> | null {
export function heartbeatInspection(server: string): Observation<Awaited<ReturnType<typeof nativeHealth>>> | null {
const row = getServer(server);
if (!row?.native_instance_id || !row.host_id) return null;
const host = getHost(row.host_id);
@@ -242,7 +235,7 @@ export function inspectForMonitoring(server: string, fresh = false) {
if (fresh) statuses.delete(server);
const heartbeat = !fresh && heartbeatInspection(server);
if (heartbeat) return Promise.resolve(heartbeat);
return observe(server, fresh, 'inspect', inspections, komodoInspectHealthy);
return observe(server, fresh, 'inspect', inspections, nativeHealth);
}
export function rconForMonitoring(server: string, fresh = false) {
if (fresh) statuses.delete(server);
src/lib/server/fleet/launch-recipes.ts
+1 -1
View File
@@ -105,7 +105,7 @@ function buildRecipe(game: ZombieGame, mapSlug: string, includeContent: boolean)
}
addConfig('Selected map', MAP_CONFIG_FILE, getServerMapConfig(game, map.canonical, map.slug)!);
return { format: 1, game, mapSlug, mapName: map.friendly, expectedMap: map.canonical,
capturedAt: new Date().toISOString(), image: config.komodo.defaultImage,
capturedAt: new Date().toISOString(), image: config.defaultRuntimeImage,
configFile: configFileForGame(game), configSource, files };
}
src/lib/server/fleet/servers.ts
+3 -11
View File
@@ -74,9 +74,6 @@ export interface ServerRow {
instance_generation?: number | null;
game_library_id?: string | null;
name: string;
komodo_type: 'deployment' | 'stack' | 'container';
komodo_name: string;
komodo_server: string | null;
container_state: string | null;
container_checked_at: string | null;
desired_state: 'running' | 'stopped';
@@ -112,12 +109,10 @@ export interface ServerRow {
export async function seedServerRegistry(): Promise<number> {
// Explicit legacy import only; never run as part of application startup.
const rows = await serverDirectories();
// Under the new naming scheme Komodo entities match EC server names 1:1
// (both underscore-separated), so komodo_name = name — no transformation.
const insert = db().prepare(
`INSERT OR IGNORE INTO servers
(name, komodo_type, komodo_name, game, map, game_mode, map_slug, host_dir)
VALUES (?, 'deployment', ?, ?, ?, ?, ?, ?)`
(name, game, map, game_mode, map_slug, host_dir)
VALUES (?, ?, ?, ?, ?, ?)`
);
// Backfill existing rows where derived columns were never set. game_mode
// and map_slug are new; older rows can still surface them from the name.
@@ -139,7 +134,7 @@ export async function seedServerRegistry(): Promise<number> {
const gameMode = parsed?.gameMode ?? null;
const mapSlug = parsed?.mapSlug ?? null;
const hostDir = serverDir(name);
const res = insert.run(name, name, game, map, gameMode, mapSlug, hostDir);
const res = insert.run(name, game, map, gameMode, mapSlug, hostDir);
if (res.changes > 0) {
n++;
} else {
@@ -185,9 +180,6 @@ export function setServerField(
export function setServerFields(name: string, fields: Partial<Record<keyof ServerRow, string | number | null>>): void {
const allowed: Array<keyof ServerRow> = [
'komodo_type',
'komodo_name',
'komodo_server',
'container_state',
'container_checked_at',
'game_port',
src/lib/server/fleet/snapshot.ts
+3 -5
View File
@@ -1,7 +1,7 @@
import { db } from '../db/index.js';
import { getServer, listServers } from './servers.js';
import { observationReader, serverObservation } from './status.js';
import { config, komodoConfigured, rconConfigured } from '../config.js';
import { config, rconConfigured } from '../config.js';
import { runningVersions } from '../integrations/plutonium.js';
import { listHosts } from '../hosts/index.js';
import type { NativeInstance } from '../backends/native.js';
@@ -33,8 +33,7 @@ export function fleetSnapshot(serverName?: string) {
host: s.host_id ? hostsById.get(s.host_id) ?? null : null,
network: instance ? { bindAddress: instance.bind_address, localPort: instance.port, publicAddress: instance.public_address, publicPort: instance.public_port } : null,
plutonium: versions.get(s.name),
// Derived endpoint — same logic as endpointFor() but without asking
// Komodo. If we don't have a port yet, endpoint is null.
// The host agent owns the game connection.
endpoint: instance ? { host: `agent:${instance.host_id}`, port: instance.port } : s.game_port ? { host: s.rcon_host ?? rconHost, port: s.game_port } : null,
held: holds.get(s.name) ?? []
};
@@ -44,11 +43,10 @@ export function fleetSnapshot(serverName?: string) {
serverNow: Date.now(),
reservedNames: serverName === undefined ? (db().prepare("SELECT j.server FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')").all() as { server: string }[]).map(row => row.server) : [],
integrations: {
native: hosts.length > 0 || !komodoConfigured(),
native: true,
hostsReady: hosts.filter(host => host.readiness === 'ready').length,
hostsTotal: hosts.length,
rcon: rconConfigured(),
komodo: komodoConfigured(),
rconTimeoutMs: config.rcon.timeoutMs,
defaultRconHost: rconHost
}
src/lib/server/fleet/status-evidence.ts
+1 -1
View File
@@ -59,7 +59,7 @@ export async function probePlayerEvidence(server: string): Promise<PlayerEvidenc
return { players: null, source: null, observedAt: new Date().toISOString() };
}
/** Komodo readiness remains a caller prerequisite. Old-process telemetry is excluded. */
/** Host agent readiness remains a caller prerequisite. Old-process telemetry is excluded. */
export async function probeServerStatus(server: string, after?: string | null): Promise<
{ ok: true; players: number; map: string | null; source: 'rcon' | 'telemetry'; observedAt: string; receivedAt?: string } | { ok: false; reason: string; observedAt: string }
> {
src/lib/server/fleet/status.ts
+1 -2
View File
@@ -7,7 +7,6 @@ import { telemetryReport, latestGameSubmission } from './status-evidence.js';
import { latestGameSubmissions } from './lifecycle-evidence.js';
import { db } from '../db/index.js';
import { parseGameEvent, type GameTelemetry } from '../../telemetry.js';
import { restartRequestInFlight } from '../provisioning/restarts.js';
interface Evidence { provisioning: boolean; settling: boolean; telemetry: GameTelemetry | null; submission: string | null }
@@ -33,7 +32,7 @@ export function observationReader() {
const operation = operations.get(server.name), restart = restarts.get(server.name);
return serverObservation(server, {
provisioning: !!operation && ['create','rebuild','start'].includes(operation.action) && ['queued','running','awaiting-health'].includes(operation.state),
settling: restartRequestInFlight(server.name) || !!(restart && !restart.healthy_at && Date.now() - Date.parse(restart.requested_at) < 15_000),
settling: !!(restart && !restart.healthy_at && Date.now() - Date.parse(restart.requested_at) < 15_000),
telemetry: reports.get(server.name) ?? null, submission: submissions.get(server.name) ?? null
});
};