Use database fleet registry and gate RCON polling on container readiness

This commit is contained in:
2026-09-17 17:28:38 -04:00
parent 3b1c9bf3d9
commit 885abbdffe
19 changed files with 208 additions and 41 deletions
+11
View File
@@ -2,6 +2,17 @@
Web app that manages a fleet of T5 Plutonium servers running under Komodo. Web app that manages a fleet of T5 Plutonium servers running under Komodo.
Fleet membership lives in SQLite. EC registers servers during creation and removes
them during destruction; startup does not discover or import server folders.
Existing database registrations are preserved, and missing mounts do not remove
servers from the fleet. The legacy `seedServerRegistry` helper is an explicit
import utility only.
Status refreshes check Komodo container readiness before polling RCON. Starting
or restarting containers show **Starting**, without old RCON errors or player
data. Stopped, unhealthy, or unverifiable containers are not polled. Containers
without a Docker health check use their running state as the readiness signal.
## Stack ## Stack
- SvelteKit (adapter-node) + TypeScript - SvelteKit (adapter-node) + TypeScript
+1
View File
@@ -12,6 +12,7 @@ let _db: Database.Database | null = null;
// IF NOT EXISTS in schema.sql is a no-op on existing tables, so it can't // IF NOT EXISTS in schema.sql is a no-op on existing tables, so it can't
// backfill columns; this can. // backfill columns; this can.
const SERVERS_COLUMNS: Array<{ name: string; ddl: string }> = [ const SERVERS_COLUMNS: Array<{ name: string; ddl: string }> = [
{ name: 'container_state', ddl: 'ALTER TABLE servers ADD COLUMN container_state TEXT' },
// Base columns that shipped in the very first schema stay here for parity // Base columns that shipped in the very first schema stay here for parity
// with fresh installs — no ALTER is issued if they're already present. // with fresh installs — no ALTER is issued if they're already present.
{ name: 'komodo_server', ddl: 'ALTER TABLE servers ADD COLUMN komodo_server TEXT' }, { name: 'komodo_server', ddl: 'ALTER TABLE servers ADD COLUMN komodo_server TEXT' },
+1
View File
@@ -11,6 +11,7 @@ CREATE TABLE IF NOT EXISTS servers (
komodo_type TEXT NOT NULL DEFAULT 'deployment', -- deployment|stack|container komodo_type TEXT NOT NULL DEFAULT 'deployment', -- deployment|stack|container
komodo_name TEXT NOT NULL, komodo_name TEXT NOT NULL,
komodo_server TEXT, komodo_server TEXT,
container_state TEXT, -- last observed Komodo runtime state
game_port INTEGER, -- UDP RCON port; lazy-filled from Komodo game_port INTEGER, -- UDP RCON port; lazy-filled from Komodo
rcon_host TEXT, -- optional per-server override of RCON_HOST rcon_host TEXT, -- optional per-server override of RCON_HOST
game TEXT, -- 't5' etc.; static per-server (no map rotation) game TEXT, -- 't5' etc.; static per-server (no map rotation)
+2 -2
View File
@@ -271,11 +271,11 @@ async function createServerUnlocked(input: CreateServerInput, prepared: Prepared
`INSERT INTO servers `INSERT INTO servers
(name, komodo_type, komodo_name, game, map, (name, komodo_type, komodo_name, game, map,
game_mode, map_slug, host_dir, config_file, game_id, image, game_mode, map_slug, host_dir, config_file, game_id, image,
game_port, created_by) game_port, created_by, container_state)
VALUES VALUES
(?, 'deployment', ?, 't5', ?, (?, 'deployment', ?, 't5', ?,
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?)` ?, ?, 'starting')`
).run( ).run(
name, name, mapEntry.canonical, name, name, mapEntry.canonical,
input.gameMode, mapEntry.slug, hostDir, input.gameMode, mapEntry.slug, hostDir,
+2 -2
View File
@@ -14,7 +14,7 @@ import {
import { getBaselineForScope, getVersion } from './scripts.js'; import { getBaselineForScope, getVersion } from './scripts.js';
import { getPromotedServerConfig, getServerMapConfig, MAP_CONFIG_FILE } from './server-config.js'; import { getPromotedServerConfig, getServerMapConfig, MAP_CONFIG_FILE } from './server-config.js';
import { T5_ZOMBIE_MAPS } from './map-aliases.js'; import { T5_ZOMBIE_MAPS } from './map-aliases.js';
import { allServers, getServer, listServers } from './servers.js'; import { serverDirectories, getServer, listServers } from './servers.js';
import { promotedArtifacts } from './promoted-artifacts.js'; import { promotedArtifacts } from './promoted-artifacts.js';
// Destinations drift scans, in a stable order so the UI shows scripts first. // Destinations drift scans, in a stable order so the UI shows scripts first.
@@ -228,7 +228,7 @@ function composeExpectedCfg(
export async function scanFleetDrift(): Promise<DriftEntry[]> { export async function scanFleetDrift(): Promise<DriftEntry[]> {
const servers = listServers().map((s) => s.name); const servers = listServers().map((s) => s.name);
const registered = new Set(servers); const registered = new Set(servers);
const allDirs = new Set(await allServers()); const allDirs = new Set(await serverDirectories());
// Only touch server dirs that are both registered AND present on disk. // Only touch server dirs that are both registered AND present on disk.
const eligible = servers.filter((n) => registered.has(n) && allDirs.has(n)); const eligible = servers.filter((n) => registered.has(n) && allDirs.has(n));
const out: DriftEntry[] = []; const out: DriftEntry[] = [];
+4 -2
View File
@@ -1,4 +1,4 @@
export type Health = 'online' | 'stale' | 'offline'; export type Health = 'online' | 'stale' | 'offline' | 'starting';
const ONLINE_MAX_MS = 30_000; const ONLINE_MAX_MS = 30_000;
const STALE_MAX_MS = 120_000; const STALE_MAX_MS = 120_000;
@@ -9,7 +9,9 @@ const STALE_MAX_MS = 120_000;
* successful reply is >120s old. `stale` if the last reply is 30-120s old * successful reply is >120s old. `stale` if the last reply is 30-120s old
* (RCON glitchy). `online` if fresh. * (RCON glitchy). `online` if fresh.
*/ */
export function classifyHealth(lastSeenAt: string | null, lastError: string | null): Health { export function classifyHealth(lastSeenAt: string | null, lastError: string | null, containerState?: string | null): Health {
if (containerState === 'starting' || containerState === 'restarting') return 'starting';
if (containerState && containerState !== 'running') return 'offline';
if (!lastSeenAt) return 'offline'; if (!lastSeenAt) return 'offline';
const age = Date.now() - new Date(lastSeenAt).getTime(); const age = Date.now() - new Date(lastSeenAt).getTime();
if (!Number.isFinite(age) || age < 0) return 'offline'; if (!Number.isFinite(age) || age < 0) return 'offline';
+11 -3
View File
@@ -8,7 +8,7 @@ import { parseServerName } from './map-aliases.js';
* All server folders under SERVERS_ROOT. Used by seedServerRegistry to backfill * All server folders under SERVERS_ROOT. Used by seedServerRegistry to backfill
* the servers table for any on-disk server directory that isn't yet tracked. * the servers table for any on-disk server directory that isn't yet tracked.
*/ */
export async function allServers(): Promise<string[]> { export async function serverDirectories(): Promise<string[]> {
let entries; let entries;
try { try {
entries = await readdir(config.serversRoot, { withFileTypes: true }); entries = await readdir(config.serversRoot, { withFileTypes: true });
@@ -21,6 +21,11 @@ export async function allServers(): Promise<string[]> {
.sort(); .sort();
} }
/** Fleet membership is owned by the database, independent of mounted files. */
export async function allServers(): Promise<string[]> {
return listServers().map((server) => server.name);
}
/** /**
* Match a server name against a list of glob-like patterns (supports `*` and * Match a server name against a list of glob-like patterns (supports `*` and
* `?` and comma-separated lists). * `?` and comma-separated lists).
@@ -64,6 +69,7 @@ export interface ServerRow {
komodo_type: 'deployment' | 'stack' | 'container'; komodo_type: 'deployment' | 'stack' | 'container';
komodo_name: string; komodo_name: string;
komodo_server: string | null; komodo_server: string | null;
container_state: string | null;
game_port: number | null; game_port: number | null;
rcon_host: string | null; rcon_host: string | null;
game: string | null; game: string | null;
@@ -93,7 +99,8 @@ export interface ServerRow {
} }
export async function seedServerRegistry(): Promise<number> { export async function seedServerRegistry(): Promise<number> {
const rows = await allServers(); // Explicit legacy import only; never run as part of application startup.
const rows = await serverDirectories();
// Under the new naming scheme Komodo entities match EC server names 1:1 // Under the new naming scheme Komodo entities match EC server names 1:1
// (both underscore-separated), so komodo_name = name — no transformation. // (both underscore-separated), so komodo_name = name — no transformation.
const insert = db().prepare( const insert = db().prepare(
@@ -143,7 +150,7 @@ export function getServer(name: string): ServerRow | undefined {
} }
export function listServers(): ServerRow[] { export function listServers(): ServerRow[] {
return db().prepare('SELECT * FROM servers ORDER BY name').all() as ServerRow[]; return db().prepare('SELECT * FROM servers WHERE destroyed_at IS NULL ORDER BY name').all() as ServerRow[];
} }
export function setServerField( export function setServerField(
@@ -155,6 +162,7 @@ export function setServerField(
'komodo_type', 'komodo_type',
'komodo_name', 'komodo_name',
'komodo_server', 'komodo_server',
'container_state',
'game_port', 'game_port',
'rcon_host', 'rcon_host',
'game', 'game',
+14 -2
View File
@@ -255,7 +255,7 @@ export async function komodoResolveGamePort(serverName: string): Promise<number
export type KomodoHealthResult = export type KomodoHealthResult =
| { ok: true; state: string } | { ok: true; state: string }
| { ok: false; reason: string; snippet?: string }; | { ok: false; reason: string; state?: string; snippet?: string };
// Shape-agnostic scan for the container's running-state. Walks the entire // Shape-agnostic scan for the container's running-state. Walks the entire
// payload looking for anything that resembles a state field. Covered shapes: // payload looking for anything that resembles a state field. Covered shapes:
@@ -290,6 +290,18 @@ export function extractContainerState(payload: unknown): string | null {
} }
const obj = v as Record<string, unknown>; const obj = v as Record<string, unknown>;
// Komodo summaries expose health alongside a flat running state.
const flatState = obj['state'] ?? obj['State'] ?? obj['status'] ?? obj['Status'];
const flatHealth = obj['health'] ?? obj['Health'];
const healthStatus = typeof flatHealth === 'string' ? flatHealth
: flatHealth && typeof flatHealth === 'object'
? ((flatHealth as Record<string, unknown>)['status'] ?? (flatHealth as Record<string, unknown>)['Status']) : null;
if (typeof flatState === 'string' && flatState.toLowerCase() === 'running'
&& typeof healthStatus === 'string' && healthStatus.toLowerCase() !== 'healthy') {
found = healthStatus;
return;
}
// Docker inspect nested: { State: { Status, Running, Health: { Status } } } // Docker inspect nested: { State: { Status, Running, Health: { Status } } }
const stateObj = obj['State']; const stateObj = obj['State'];
if (stateObj && typeof stateObj === 'object') { if (stateObj && typeof stateObj === 'object') {
@@ -401,7 +413,7 @@ export async function komodoInspectHealthy(serverName: string): Promise<KomodoHe
const state = extractContainerState(r.json); const state = extractContainerState(r.json);
if (state) { if (state) {
const healthy = HEALTHY_STATES.has(state.toLowerCase()); const healthy = HEALTHY_STATES.has(state.toLowerCase());
if (!healthy) return { ok: false, reason: `container state: ${state}`, snippet: r.snippet }; if (!healthy) return { ok: false, state, reason: `container state: ${state}`, snippet: r.snippet };
return { ok: true, state }; return { ok: true, state };
} }
lastSnippet = r.snippet; lastSnippet = r.snippet;
@@ -21,6 +21,13 @@ function slotFor(server: string): RestartSlot | undefined {
return db().prepare('SELECT * FROM restart_rollout WHERE server = ?').get(server) as RestartSlot | undefined; return db().prepare('SELECT * FROM restart_rollout WHERE server = ?').get(server) as RestartSlot | undefined;
} }
/** The queued restart may still expose the previous healthy container. */
export function restartSettling(server: string): boolean {
const slot = slotFor(server);
return requestsInFlight.has(server) || !!(slot && !slot.healthy_at
&& Date.now() - Date.parse(slot.requested_at) < HEALTH_POLL_MS);
}
export function rolloutWaitReason(server: string): string | null { export function rolloutWaitReason(server: string): string | null {
const slot = slotFor(server); const slot = slotFor(server);
if (slot?.healthy_at === null) { if (slot?.healthy_at === null) {
@@ -49,6 +56,7 @@ export async function restartInRollout(server: string): Promise<RolloutRestartRe
if (waitReason) return { kind: 'queued', message: waitReason }; if (waitReason) return { kind: 'queued', message: waitReason };
requestsInFlight.add(server); requestsInFlight.add(server);
db().prepare("UPDATE servers SET container_state = 'restarting', last_error = NULL, last_reply_snippet = NULL WHERE name = ?").run(server);
try { try {
const result = await komodoRestart(server); const result = await komodoRestart(server);
if (result.ok) return { kind: 'started' }; if (result.ok) return { kind: 'started' };
+34 -4
View File
@@ -5,7 +5,9 @@ import { emit } from './bus.js';
import type { ServerStatusEntry } from './bus.js'; import type { ServerStatusEntry } from './bus.js';
import { rconStatusResult } from '../integrations/rcon.js'; import { rconStatusResult } from '../integrations/rcon.js';
import type { RconFailure } from '../integrations/rcon.js'; import type { RconFailure } from '../integrations/rcon.js';
import { listServers, seedServerRegistry, setServerField } from '../fleet/servers.js'; import { listServers, setServerField } from '../fleet/servers.js';
import { komodoInspectHealthy } from '../integrations/komodo.js';
import { restartSettling } from './restart-rollout.js';
import { classifyHealth } from '../fleet/health.js'; import { classifyHealth } from '../fleet/health.js';
import { scanMatchState } from '../fleet/match-log.js'; import { scanMatchState } from '../fleet/match-log.js';
@@ -108,7 +110,15 @@ function encodeError(reason: RconFailure, detail: string): string {
return `${reason}:${detail}`; return `${reason}:${detail}`;
} }
async function fanoutStatus(): Promise<void> { let statusInFlight: Promise<void> | null = null;
export function fanoutStatus(): Promise<void> {
if (statusInFlight) return statusInFlight;
statusInFlight = refreshStatus().finally(() => { statusInFlight = null; });
return statusInFlight;
}
async function refreshStatus(): Promise<void> {
const servers = listServers(); const servers = listServers();
if (servers.length === 0) return; if (servers.length === 0) return;
@@ -120,6 +130,28 @@ async function fanoutStatus(): Promise<void> {
const i = cursor++; const i = cursor++;
if (i >= servers.length) return; if (i >= servers.length) return;
const srv = servers[i]; const srv = servers[i];
const container = restartSettling(srv.name)
? { ok: false as const, state: 'restarting', reason: 'Restart requested' }
: await komodoInspectHealthy(srv.name).catch((err) => ({
ok: false as const, reason: (err as Error).message
}));
const containerState = container.ok ? 'running'
: ('state' in container && typeof container.state === 'string' ? container.state.toLowerCase() : 'unknown');
setServerField(srv.name, 'container_state', containerState);
if (!container.ok) {
// Container readiness precedes RCON readiness. Old replies and
// failures describe the previous process, not this startup.
setServerField(srv.name, 'last_error', null);
setServerField(srv.name, 'last_reply_snippet', null);
entries[i] = {
server: srv.name, players: null, map: srv.map, gametype: null,
roster: null, health: classifyHealth(null, null, containerState),
lastSeenAt: srv.last_seen_at, lastError: null, endpoint: null,
reason: null, replySnippet: null, activeSince: null,
inactiveSince: null, gameStartedAt: null, gameEndedAt: null
};
continue;
}
const result = await rconStatusResult(srv.name).catch((err) => ({ const result = await rconStatusResult(srv.name).catch((err) => ({
ok: false as const, ok: false as const,
endpoint: null, endpoint: null,
@@ -231,8 +263,6 @@ export function startWorker(): void {
if (started) return; if (started) return;
started = true; started = true;
seedServerRegistry().catch((err) => console.error('[worker] seed error:', err));
void tick(); void tick();
void fanoutStatus(); void fanoutStatus();
tickTimer = setInterval(tick, TICK_MS); tickTimer = setInterval(tick, TICK_MS);
+3 -1
View File
@@ -23,7 +23,9 @@
} }
</script> </script>
{#if health === 'offline'} {#if health === 'starting'}
<div class="state empty">Server starting</div>
{:else if health === 'offline'}
<div class="state offline">server unreachable</div> <div class="state offline">server unreachable</div>
{:else if !roster || roster.length === 0} {:else if !roster || roster.length === 0}
<div class="state empty">lobby is empty</div> <div class="state empty">lobby is empty</div>
+6 -4
View File
@@ -63,7 +63,7 @@
.sort((a, b) => b.score - a.score) .sort((a, b) => b.score - a.score)
.slice(0, 3) .slice(0, 3)
); );
let showRoster = $derived(health !== 'offline'); let showRoster = $derived(health === 'online' || health === 'stale');
let mapLabel = $derived(map ?? '—'); let mapLabel = $derived(map ?? '—');
let gtLabel = $derived(gametypeLabel(gametype)); let gtLabel = $derived(gametypeLabel(gametype));
@@ -72,7 +72,7 @@
let gameInProgress = $derived(!!gameStartedAt && !gameEndedAt && !!activeSince); let gameInProgress = $derived(!!gameStartedAt && !gameEndedAt && !!activeSince);
let gameElapsed = $derived(gameInProgress ? elapsedSince(gameStartedAt, $uiNow) : null); let gameElapsed = $derived(gameInProgress ? elapsedSince(gameStartedAt, $uiNow) : null);
let showTiming = $derived( let showTiming = $derived(
health !== 'offline' && (activeElapsed !== null || inactiveElapsed !== null || gameElapsed !== null) showRoster && (activeElapsed !== null || inactiveElapsed !== null || gameElapsed !== null)
); );
// Shared-element transition to /servers/{name} — Chromium only, others // Shared-element transition to /servers/{name} — Chromium only, others
@@ -97,7 +97,9 @@
<StatusPip {health} /> <StatusPip {health} />
<strong class="name">{name}</strong> <strong class="name">{name}</strong>
</div> </div>
{#if health === 'offline'} {#if health === 'starting'}
<Badge variant="info" uppercase>Starting</Badge>
{:else if health === 'offline'}
<Badge variant="danger" uppercase>offline</Badge> <Badge variant="danger" uppercase>offline</Badge>
{:else} {:else}
<div class="players" aria-live="polite"> <div class="players" aria-live="polite">
@@ -119,7 +121,7 @@
{/if} {/if}
</div> </div>
<RoundTelemetry {name} /> {#if showRoster}<RoundTelemetry {name} />{/if}
{#if showTiming} {#if showTiming}
<div class="timing"> <div class="timing">
+1 -1
View File
@@ -26,7 +26,7 @@
background: var(--spark-500); background: var(--spark-500);
animation: electric-pulse 1.6s ease-in-out infinite; animation: electric-pulse 1.6s ease-in-out infinite;
} }
.dot-stale { .dot-stale, .dot-starting {
background: var(--amber-500); background: var(--amber-500);
box-shadow: var(--amber-glow); box-shadow: var(--amber-glow);
} }
@@ -11,13 +11,14 @@
let toneClass = $derived( let toneClass = $derived(
health === 'online' ? 't-online' : health === 'online' ? 't-online' :
health === 'stale' ? 't-warning' : health === 'stale' || health === 'starting' ? 't-warning' :
't-danger' 't-danger'
); );
let announceLabel = $derived( let announceLabel = $derived(
label ?? ( label ?? (
health === 'online' ? 'online' : health === 'online' ? 'online' :
health === 'starting' ? 'starting' :
health === 'stale' ? 'stale' : health === 'stale' ? 'stale' :
'offline' 'offline'
) )
+1 -1
View File
@@ -12,7 +12,7 @@ export interface RosterPlayer {
name: string; name: string;
} }
export type Health = 'online' | 'stale' | 'offline'; export type Health = 'online' | 'stale' | 'offline' | 'starting';
export type RconFailure = 'no-config' | 'no-port' | 'send-error' | 'timeout' | 'bad-reply' | 'no-players'; export type RconFailure = 'no-config' | 'no-port' | 'send-error' | 'timeout' | 'bad-reply' | 'no-players';
export interface RconEndpoint { export interface RconEndpoint {
+14 -5
View File
@@ -19,16 +19,25 @@ function safeJson<T>(raw: string | null): T | null {
export async function GET() { export async function GET() {
const rconHost = config.rcon.host; const rconHost = config.rcon.host;
const servers = listServers().map((s) => ({ const servers = listServers().map((s) => {
const ready = !s.container_state || s.container_state === 'running';
return {
...s, ...s,
roster: safeJson<RconPlayer[]>(s.last_roster), last_players: ready ? s.last_players : null,
health: classifyHealth(s.last_seen_at, s.last_error), last_error: ready ? s.last_error : null,
replySnippet: s.last_reply_snippet, active_since: ready ? s.active_since : null,
inactive_since: ready ? s.inactive_since : null,
game_started_at: ready ? s.game_started_at : null,
game_ended_at: ready ? s.game_ended_at : null,
roster: ready ? safeJson<RconPlayer[]>(s.last_roster) : null,
health: classifyHealth(s.last_seen_at, s.last_error, s.container_state),
replySnippet: ready ? s.last_reply_snippet : null,
// Derived endpoint — same logic as endpointFor() but without asking // Derived endpoint — same logic as endpointFor() but without asking
// Komodo. If we don't have a port yet, endpoint is null. // Komodo. If we don't have a port yet, endpoint is null.
endpoint: s.game_port ? { host: s.rcon_host ?? rconHost, port: s.game_port } : null, endpoint: s.game_port ? { host: s.rcon_host ?? rconHost, port: s.game_port } : null,
held: [...heldByBeta(s.name).entries()].map(([target, betaId]) => ({ target, betaId })) held: [...heldByBeta(s.name).entries()].map(([target, betaId]) => ({ target, betaId }))
})); };
});
return json({ return json({
servers, servers,
reservedNames: (db().prepare("SELECT j.server FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')").all() as { server: string }[]).map(row => row.server), reservedNames: (db().prepare("SELECT j.server FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')").all() as { server: string }[]).map(row => row.server),
+11 -8
View File
@@ -126,7 +126,7 @@
let effectiveRoster = $derived(live?.roster ?? server?.roster ?? null); let effectiveRoster = $derived(live?.roster ?? server?.roster ?? null);
let effectiveHealth = $derived<Health>(live?.health ?? server?.health ?? 'offline'); let effectiveHealth = $derived<Health>(live?.health ?? server?.health ?? 'offline');
let effectiveSeen = $derived(live?.lastSeenAt ?? server?.last_seen_at ?? null); let effectiveSeen = $derived(live?.lastSeenAt ?? server?.last_seen_at ?? null);
let effectiveErr = $derived(live?.lastError ?? server?.last_error ?? null); let effectiveErr = $derived(live ? live.lastError : server?.last_error ?? null);
let effectiveEndpoint = $derived<RconEndpoint | null>(live?.endpoint ?? server?.endpoint ?? null); let effectiveEndpoint = $derived<RconEndpoint | null>(live?.endpoint ?? server?.endpoint ?? null);
let liveReason = $derived<RconFailure | null>(live?.reason ?? null); let liveReason = $derived<RconFailure | null>(live?.reason ?? null);
let decoded = $derived(decodeRconError(effectiveErr)); let decoded = $derived(decodeRconError(effectiveErr));
@@ -139,12 +139,12 @@
let effectiveInactiveSince = $derived<string | null>(live?.inactiveSince ?? server?.inactive_since ?? null); let effectiveInactiveSince = $derived<string | null>(live?.inactiveSince ?? server?.inactive_since ?? null);
let effectiveGameStartedAt = $derived<string | null>(live?.gameStartedAt ?? server?.game_started_at ?? null); let effectiveGameStartedAt = $derived<string | null>(live?.gameStartedAt ?? server?.game_started_at ?? null);
let effectiveGameEndedAt = $derived<string | null>(live?.gameEndedAt ?? server?.game_ended_at ?? null); let effectiveGameEndedAt = $derived<string | null>(live?.gameEndedAt ?? server?.game_ended_at ?? null);
let activeElapsed = $derived(effectiveActiveSince ? elapsedSince(effectiveActiveSince, $uiNow) : null); let activeElapsed = $derived(effectiveHealth !== 'starting' && effectiveActiveSince ? elapsedSince(effectiveActiveSince, $uiNow) : null);
let inactiveElapsed = $derived(effectiveInactiveSince ? elapsedSince(effectiveInactiveSince, $uiNow) : null); let inactiveElapsed = $derived(effectiveHealth !== 'starting' && effectiveInactiveSince ? elapsedSince(effectiveInactiveSince, $uiNow) : null);
let gameInProgress = $derived(!!effectiveGameStartedAt && !effectiveGameEndedAt && !!effectiveActiveSince); let gameInProgress = $derived(!!effectiveGameStartedAt && !effectiveGameEndedAt && !!effectiveActiveSince);
let gameElapsed = $derived(gameInProgress ? elapsedSince(effectiveGameStartedAt, $uiNow) : null); let gameElapsed = $derived(effectiveHealth !== 'starting' && gameInProgress ? elapsedSince(effectiveGameStartedAt, $uiNow) : null);
let hasDiagnosticIssue = $derived(effectiveHealth !== 'online'); let hasDiagnosticIssue = $derived(effectiveHealth !== 'online' && effectiveHealth !== 'starting');
// Auto-open the Diagnostics tab if there's a problem the operator should see. // Auto-open the Diagnostics tab if there's a problem the operator should see.
$effect(() => { $effect(() => {
@@ -237,13 +237,14 @@
<StatusPip health={effectiveHealth} size="md" /> <StatusPip health={effectiveHealth} size="md" />
<span class="health-label"> <span class="health-label">
{effectiveHealth === 'online' ? 'Online' : {effectiveHealth === 'online' ? 'Online' :
effectiveHealth === 'starting' ? 'Starting' :
effectiveHealth === 'stale' ? 'Stale' : 'Offline'} effectiveHealth === 'stale' ? 'Stale' : 'Offline'}
</span> </span>
</div> </div>
<div class="metric-cell"> <div class="metric-cell">
<Stat <Stat
label="Players" label="Players"
value={effectivePlayers ?? 0} value={effectiveHealth === 'starting' ? '—' : effectivePlayers ?? 0}
size="xl" size="xl"
tone={(effectivePlayers ?? 0) > 0 ? 'online' : 'primary'} tone={(effectivePlayers ?? 0) > 0 ? 'online' : 'primary'}
tween tween
@@ -276,7 +277,7 @@
{/if} {/if}
</div> </div>
<RoundTelemetry {name} /> {#if effectiveHealth !== 'starting'}<RoundTelemetry {name} />{/if}
</Card> </Card>
@@ -327,7 +328,9 @@
<Card padding="lg"> <Card padding="lg">
<div class="panel-head"> <div class="panel-head">
<h2>RCON diagnostic</h2> <h2>RCON diagnostic</h2>
{#if hasDiagnosticIssue} {#if effectiveHealth === 'starting'}
<Badge variant="info" uppercase>Waiting for container</Badge>
{:else if hasDiagnosticIssue}
<Badge variant="warning" uppercase>Issue</Badge> <Badge variant="warning" uppercase>Issue</Badge>
{:else} {:else}
<Badge variant="online" uppercase>Healthy</Badge> <Badge variant="online" uppercase>Healthy</Badge>
+2
View File
@@ -4,6 +4,8 @@ import { extractUdpPort, extractContainerState } from '../../src/lib/server/inte
describe('container readiness', () => { describe('container readiness', () => {
it.each(['starting', 'unhealthy'])('does not accept a running container whose health is %s', (health) => { it.each(['starting', 'unhealthy'])('does not accept a running container whose health is %s', (health) => {
expect(extractContainerState({ State: { Status: 'running', Running: true, Health: { Status: health } } })).toBe(health); expect(extractContainerState({ State: { Status: 'running', Running: true, Health: { Status: health } } })).toBe(health);
expect(extractContainerState({ state: 'running', health })).toBe(health);
expect(extractContainerState({ state: 'running', health: { status: health } })).toBe(health);
}); });
it('accepts healthy running containers and preserves stopped states', () => { it('accepts healthy running containers and preserves stopped states', () => {
expect(extractContainerState({ State: { Status: 'running', Running: true, Health: { Status: 'healthy' } } })).toBe('running'); expect(extractContainerState({ State: { Status: 'running', Running: true, Health: { Status: 'healthy' } } })).toBe('running');
+75
View File
@@ -0,0 +1,75 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
komodoInspectHealthy: vi.fn(), komodoRestart: vi.fn()
}));
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
rconStatusResult: vi.fn(), rconGetPlayerCount: vi.fn()
}));
let fixture: Fixture;
beforeEach(async () => {
vi.resetModules();
vi.clearAllMocks();
fixture = await newFixture(['unregistered-folder']);
});
afterEach(async () => { await fixture.cleanup(); });
async function setup() {
const { db } = await import('../../src/lib/server/db/index.js');
db().prepare("INSERT INTO servers (name, komodo_name, last_error) VALUES ('managed', 'managed', 'timeout:old process')").run();
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
const { fanoutStatus } = await import('../../src/lib/server/pipeline/worker.js');
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
return { db: db(), inspect: vi.mocked(komodoInspectHealthy), rcon: vi.mocked(rconStatusResult), fanoutStatus, getServer };
}
describe('database fleet and container readiness', () => {
it('lists registered servers without folders and ignores unregistered folders and tombstones', async () => {
const { db } = await setup();
const { allServers, listServers } = await import('../../src/lib/server/fleet/servers.js');
expect(await allServers()).toEqual(['managed']);
db.prepare("UPDATE servers SET destroyed_at = '2026-01-01' WHERE name = 'managed'").run();
expect(listServers()).toEqual([]);
});
it('skips startup RCON, clears old errors, and resumes polling only after Komodo is ready', async () => {
const { inspect, rcon, fanoutStatus, getServer } = await setup();
const { GET } = await import('../../src/routes/api/servers/+server.js');
inspect.mockResolvedValue({ ok: false, state: 'starting', reason: 'container state: starting' });
await fanoutStatus();
await fanoutStatus();
expect(rcon).not.toHaveBeenCalled();
expect(getServer('managed')).toMatchObject({ container_state: 'starting', last_error: null });
expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'starting', last_error: null, last_players: null });
inspect.mockResolvedValue({ ok: true, state: 'running' });
rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'no reply' });
await fanoutStatus();
expect(rcon).toHaveBeenCalledTimes(1);
expect(getServer('managed')).toMatchObject({ container_state: 'running', last_error: 'timeout:no reply' });
rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status',
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } });
await fanoutStatus();
expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'online', last_error: null });
});
it.each(['unhealthy', 'exited', undefined])('does not poll RCON with container state %s', async (state) => {
const { inspect, rcon, fanoutStatus } = await setup();
inspect.mockResolvedValue({ ok: false, state, reason: 'not ready' });
await fanoutStatus();
expect(rcon).not.toHaveBeenCalled();
});
it('waits for an EC restart request to settle before trusting the old healthy container', async () => {
const { db, inspect, rcon, fanoutStatus, getServer } = await setup();
db.prepare("INSERT INTO restart_rollout (server, requested_at) VALUES ('managed', ?)").run(new Date().toISOString());
await fanoutStatus();
expect(inspect).not.toHaveBeenCalled();
expect(rcon).not.toHaveBeenCalled();
expect(getServer('managed')?.container_state).toBe('restarting');
});
});