josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Use database fleet registry and gate RCON polling on container readiness

Browse Source
This commit is contained in:
josh
2026-09-17 17:28:38 -04:00
parent 3b1c9bf3d9
commit 885abbdffe
19 changed files with 208 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/db/index.ts
+1
View File
@@ -12,6 +12,7 @@ let _db: Database.Database | null = null;
// IF NOT EXISTS in schema.sql is a no-op on existing tables, so it can't
// backfill columns; this can.
const SERVERS_COLUMNS: Array<{ name: string; ddl: string }> = [
{ name: 'container_state', ddl: 'ALTER TABLE servers ADD COLUMN container_state TEXT' },
// Base columns that shipped in the very first schema stay here for parity
// with fresh installs — no ALTER is issued if they're already present.
{ name: 'komodo_server', ddl: 'ALTER TABLE servers ADD COLUMN komodo_server TEXT' },
src/lib/server/db/schema.sql
+1
View File
@@ -11,6 +11,7 @@ CREATE TABLE IF NOT EXISTS servers (
komodo_type TEXT NOT NULL DEFAULT 'deployment', -- deployment|stack|container
komodo_name TEXT NOT NULL,
komodo_server TEXT,
container_state TEXT, -- last observed Komodo runtime state
game_port INTEGER, -- UDP RCON port; lazy-filled from Komodo
rcon_host TEXT, -- optional per-server override of RCON_HOST
game TEXT, -- 't5' etc.; static per-server (no map rotation)
src/lib/server/fleet/create.ts
+2 -2
View File
@@ -271,11 +271,11 @@ async function createServerUnlocked(input: CreateServerInput, prepared: Prepared
`INSERT INTO servers
(name, komodo_type, komodo_name, game, map,
game_mode, map_slug, host_dir, config_file, game_id, image,
game_port, created_by)
game_port, created_by, container_state)
VALUES
(?, 'deployment', ?, 't5', ?,
?, ?, ?, ?, ?, ?,
?, ?)`
?, ?, 'starting')`
).run(
name, name, mapEntry.canonical,
input.gameMode, mapEntry.slug, hostDir,
src/lib/server/fleet/drift.ts
+2 -2
View File
@@ -14,7 +14,7 @@ import {
import { getBaselineForScope, getVersion } from './scripts.js';
import { getPromotedServerConfig, getServerMapConfig, MAP_CONFIG_FILE } from './server-config.js';
import { T5_ZOMBIE_MAPS } from './map-aliases.js';
import { allServers, getServer, listServers } from './servers.js';
import { serverDirectories, getServer, listServers } from './servers.js';
import { promotedArtifacts } from './promoted-artifacts.js';
// Destinations drift scans, in a stable order so the UI shows scripts first.
@@ -228,7 +228,7 @@ function composeExpectedCfg(
export async function scanFleetDrift(): Promise<DriftEntry[]> {
const servers = listServers().map((s) => s.name);
const registered = new Set(servers);
const allDirs = new Set(await allServers());
const allDirs = new Set(await serverDirectories());
// Only touch server dirs that are both registered AND present on disk.
const eligible = servers.filter((n) => registered.has(n) && allDirs.has(n));
const out: DriftEntry[] = [];
src/lib/server/fleet/health.ts
+4 -2
View File
@@ -1,4 +1,4 @@
export type Health = 'online' | 'stale' | 'offline';
export type Health = 'online' | 'stale' | 'offline' | 'starting';
const ONLINE_MAX_MS = 30_000;
const STALE_MAX_MS = 120_000;
@@ -9,7 +9,9 @@ const STALE_MAX_MS = 120_000;
* successful reply is >120s old. `stale` if the last reply is 30-120s old
* (RCON glitchy). `online` if fresh.
*/
export function classifyHealth(lastSeenAt: string | null, lastError: string | null): Health {
export function classifyHealth(lastSeenAt: string | null, lastError: string | null, containerState?: string | null): Health {
if (containerState === 'starting' || containerState === 'restarting') return 'starting';
if (containerState && containerState !== 'running') return 'offline';
if (!lastSeenAt) return 'offline';
const age = Date.now() - new Date(lastSeenAt).getTime();
if (!Number.isFinite(age) || age < 0) return 'offline';
src/lib/server/fleet/servers.ts
+11 -3
View File
@@ -8,7 +8,7 @@ import { parseServerName } from './map-aliases.js';
* All server folders under SERVERS_ROOT. Used by seedServerRegistry to backfill
* the servers table for any on-disk server directory that isn't yet tracked.
*/
export async function allServers(): Promise<string[]> {
export async function serverDirectories(): Promise<string[]> {
let entries;
try {
entries = await readdir(config.serversRoot, { withFileTypes: true });
@@ -21,6 +21,11 @@ export async function allServers(): Promise<string[]> {
.sort();
}
/** Fleet membership is owned by the database, independent of mounted files. */
export async function allServers(): Promise<string[]> {
return listServers().map((server) => server.name);
}
/**
* Match a server name against a list of glob-like patterns (supports `*` and
* `?` and comma-separated lists).
@@ -64,6 +69,7 @@ export interface ServerRow {
komodo_type: 'deployment' | 'stack' | 'container';
komodo_name: string;
komodo_server: string | null;
container_state: string | null;
game_port: number | null;
rcon_host: string | null;
game: string | null;
@@ -93,7 +99,8 @@ export interface ServerRow {
}
export async function seedServerRegistry(): Promise<number> {
const rows = await allServers();
// Explicit legacy import only; never run as part of application startup.
const rows = await serverDirectories();
// Under the new naming scheme Komodo entities match EC server names 1:1
// (both underscore-separated), so komodo_name = name — no transformation.
const insert = db().prepare(
@@ -143,7 +150,7 @@ export function getServer(name: string): ServerRow | undefined {
}
export function listServers(): ServerRow[] {
return db().prepare('SELECT * FROM servers ORDER BY name').all() as ServerRow[];
return db().prepare('SELECT * FROM servers WHERE destroyed_at IS NULL ORDER BY name').all() as ServerRow[];
}
export function setServerField(
@@ -155,6 +162,7 @@ export function setServerField(
'komodo_type',
'komodo_name',
'komodo_server',
'container_state',
'game_port',
'rcon_host',
'game',
src/lib/server/integrations/komodo.ts
+14 -2
View File
@@ -255,7 +255,7 @@ export async function komodoResolveGamePort(serverName: string): Promise<number
export type KomodoHealthResult =
| { ok: true; state: string }
| { ok: false; reason: string; snippet?: string };
| { ok: false; reason: string; state?: string; snippet?: string };
// Shape-agnostic scan for the container's running-state. Walks the entire
// payload looking for anything that resembles a state field. Covered shapes:
@@ -290,6 +290,18 @@ export function extractContainerState(payload: unknown): string | null {
}
const obj = v as Record<string, unknown>;
// Komodo summaries expose health alongside a flat running state.
const flatState = obj['state'] ?? obj['State'] ?? obj['status'] ?? obj['Status'];
const flatHealth = obj['health'] ?? obj['Health'];
const healthStatus = typeof flatHealth === 'string' ? flatHealth
: flatHealth && typeof flatHealth === 'object'
? ((flatHealth as Record<string, unknown>)['status'] ?? (flatHealth as Record<string, unknown>)['Status']) : null;
if (typeof flatState === 'string' && flatState.toLowerCase() === 'running'
&& typeof healthStatus === 'string' && healthStatus.toLowerCase() !== 'healthy') {
found = healthStatus;
return;
}
// Docker inspect nested: { State: { Status, Running, Health: { Status } } }
const stateObj = obj['State'];
if (stateObj && typeof stateObj === 'object') {
@@ -401,7 +413,7 @@ export async function komodoInspectHealthy(serverName: string): Promise<KomodoHe
const state = extractContainerState(r.json);
if (state) {
const healthy = HEALTHY_STATES.has(state.toLowerCase());
if (!healthy) return { ok: false, reason: `container state: ${state}`, snippet: r.snippet };
if (!healthy) return { ok: false, state, reason: `container state: ${state}`, snippet: r.snippet };
return { ok: true, state };
}
lastSnippet = r.snippet;
src/lib/server/pipeline/restart-rollout.ts
+8
View File
@@ -21,6 +21,13 @@ function slotFor(server: string): RestartSlot | undefined {
return db().prepare('SELECT * FROM restart_rollout WHERE server = ?').get(server) as RestartSlot | undefined;
}
/** The queued restart may still expose the previous healthy container. */
export function restartSettling(server: string): boolean {
const slot = slotFor(server);
return requestsInFlight.has(server) || !!(slot && !slot.healthy_at
&& Date.now() - Date.parse(slot.requested_at) < HEALTH_POLL_MS);
}
export function rolloutWaitReason(server: string): string | null {
const slot = slotFor(server);
if (slot?.healthy_at === null) {
@@ -49,6 +56,7 @@ export async function restartInRollout(server: string): Promise<RolloutRestartRe
if (waitReason) return { kind: 'queued', message: waitReason };
requestsInFlight.add(server);
db().prepare("UPDATE servers SET container_state = 'restarting', last_error = NULL, last_reply_snippet = NULL WHERE name = ?").run(server);
try {
const result = await komodoRestart(server);
if (result.ok) return { kind: 'started' };
src/lib/server/pipeline/worker.ts
+34 -4
View File
@@ -5,7 +5,9 @@ import { emit } from './bus.js';
import type { ServerStatusEntry } from './bus.js';
import { rconStatusResult } from '../integrations/rcon.js';
import type { RconFailure } from '../integrations/rcon.js';
import { listServers, seedServerRegistry, setServerField } from '../fleet/servers.js';
import { listServers, setServerField } from '../fleet/servers.js';
import { komodoInspectHealthy } from '../integrations/komodo.js';
import { restartSettling } from './restart-rollout.js';
import { classifyHealth } from '../fleet/health.js';
import { scanMatchState } from '../fleet/match-log.js';
@@ -108,7 +110,15 @@ function encodeError(reason: RconFailure, detail: string): string {
return `${reason}:${detail}`;
}
async function fanoutStatus(): Promise<void> {
let statusInFlight: Promise<void> | null = null;
export function fanoutStatus(): Promise<void> {
if (statusInFlight) return statusInFlight;
statusInFlight = refreshStatus().finally(() => { statusInFlight = null; });
return statusInFlight;
}
async function refreshStatus(): Promise<void> {
const servers = listServers();
if (servers.length === 0) return;
@@ -120,6 +130,28 @@ async function fanoutStatus(): Promise<void> {
const i = cursor++;
if (i >= servers.length) return;
const srv = servers[i];
const container = restartSettling(srv.name)
? { ok: false as const, state: 'restarting', reason: 'Restart requested' }
: await komodoInspectHealthy(srv.name).catch((err) => ({
ok: false as const, reason: (err as Error).message
}));
const containerState = container.ok ? 'running'
: ('state' in container && typeof container.state === 'string' ? container.state.toLowerCase() : 'unknown');
setServerField(srv.name, 'container_state', containerState);
if (!container.ok) {
// Container readiness precedes RCON readiness. Old replies and
// failures describe the previous process, not this startup.
setServerField(srv.name, 'last_error', null);
setServerField(srv.name, 'last_reply_snippet', null);
entries[i] = {
server: srv.name, players: null, map: srv.map, gametype: null,
roster: null, health: classifyHealth(null, null, containerState),
lastSeenAt: srv.last_seen_at, lastError: null, endpoint: null,
reason: null, replySnippet: null, activeSince: null,
inactiveSince: null, gameStartedAt: null, gameEndedAt: null
};
continue;
}
const result = await rconStatusResult(srv.name).catch((err) => ({
ok: false as const,
endpoint: null,
@@ -231,8 +263,6 @@ export function startWorker(): void {
if (started) return;
started = true;
seedServerRegistry().catch((err) => console.error('[worker] seed error:', err));
void tick();
void fanoutStatus();
tickTimer = setInterval(tick, TICK_MS);