Use database fleet registry and gate RCON polling on container readiness
This commit is contained in:
@@ -2,6 +2,17 @@
|
|||||||
|
|
||||||
Web app that manages a fleet of T5 Plutonium servers running under Komodo.
|
Web app that manages a fleet of T5 Plutonium servers running under Komodo.
|
||||||
|
|
||||||
|
Fleet membership lives in SQLite. EC registers servers during creation and removes
|
||||||
|
them during destruction; startup does not discover or import server folders.
|
||||||
|
Existing database registrations are preserved, and missing mounts do not remove
|
||||||
|
servers from the fleet. The legacy `seedServerRegistry` helper is an explicit
|
||||||
|
import utility only.
|
||||||
|
|
||||||
|
Status refreshes check Komodo container readiness before polling RCON. Starting
|
||||||
|
or restarting containers show **Starting**, without old RCON errors or player
|
||||||
|
data. Stopped, unhealthy, or unverifiable containers are not polled. Containers
|
||||||
|
without a Docker health check use their running state as the readiness signal.
|
||||||
|
|
||||||
## Stack
|
## Stack
|
||||||
|
|
||||||
- SvelteKit (adapter-node) + TypeScript
|
- SvelteKit (adapter-node) + TypeScript
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ let _db: Database.Database | null = null;
|
|||||||
// IF NOT EXISTS in schema.sql is a no-op on existing tables, so it can't
|
// IF NOT EXISTS in schema.sql is a no-op on existing tables, so it can't
|
||||||
// backfill columns; this can.
|
// backfill columns; this can.
|
||||||
const SERVERS_COLUMNS: Array<{ name: string; ddl: string }> = [
|
const SERVERS_COLUMNS: Array<{ name: string; ddl: string }> = [
|
||||||
|
{ name: 'container_state', ddl: 'ALTER TABLE servers ADD COLUMN container_state TEXT' },
|
||||||
// Base columns that shipped in the very first schema stay here for parity
|
// Base columns that shipped in the very first schema stay here for parity
|
||||||
// with fresh installs — no ALTER is issued if they're already present.
|
// with fresh installs — no ALTER is issued if they're already present.
|
||||||
{ name: 'komodo_server', ddl: 'ALTER TABLE servers ADD COLUMN komodo_server TEXT' },
|
{ name: 'komodo_server', ddl: 'ALTER TABLE servers ADD COLUMN komodo_server TEXT' },
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ CREATE TABLE IF NOT EXISTS servers (
|
|||||||
komodo_type TEXT NOT NULL DEFAULT 'deployment', -- deployment|stack|container
|
komodo_type TEXT NOT NULL DEFAULT 'deployment', -- deployment|stack|container
|
||||||
komodo_name TEXT NOT NULL,
|
komodo_name TEXT NOT NULL,
|
||||||
komodo_server TEXT,
|
komodo_server TEXT,
|
||||||
|
container_state TEXT, -- last observed Komodo runtime state
|
||||||
game_port INTEGER, -- UDP RCON port; lazy-filled from Komodo
|
game_port INTEGER, -- UDP RCON port; lazy-filled from Komodo
|
||||||
rcon_host TEXT, -- optional per-server override of RCON_HOST
|
rcon_host TEXT, -- optional per-server override of RCON_HOST
|
||||||
game TEXT, -- 't5' etc.; static per-server (no map rotation)
|
game TEXT, -- 't5' etc.; static per-server (no map rotation)
|
||||||
|
|||||||
@@ -271,11 +271,11 @@ async function createServerUnlocked(input: CreateServerInput, prepared: Prepared
|
|||||||
`INSERT INTO servers
|
`INSERT INTO servers
|
||||||
(name, komodo_type, komodo_name, game, map,
|
(name, komodo_type, komodo_name, game, map,
|
||||||
game_mode, map_slug, host_dir, config_file, game_id, image,
|
game_mode, map_slug, host_dir, config_file, game_id, image,
|
||||||
game_port, created_by)
|
game_port, created_by, container_state)
|
||||||
VALUES
|
VALUES
|
||||||
(?, 'deployment', ?, 't5', ?,
|
(?, 'deployment', ?, 't5', ?,
|
||||||
?, ?, ?, ?, ?, ?,
|
?, ?, ?, ?, ?, ?,
|
||||||
?, ?)`
|
?, ?, 'starting')`
|
||||||
).run(
|
).run(
|
||||||
name, name, mapEntry.canonical,
|
name, name, mapEntry.canonical,
|
||||||
input.gameMode, mapEntry.slug, hostDir,
|
input.gameMode, mapEntry.slug, hostDir,
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ import {
|
|||||||
import { getBaselineForScope, getVersion } from './scripts.js';
|
import { getBaselineForScope, getVersion } from './scripts.js';
|
||||||
import { getPromotedServerConfig, getServerMapConfig, MAP_CONFIG_FILE } from './server-config.js';
|
import { getPromotedServerConfig, getServerMapConfig, MAP_CONFIG_FILE } from './server-config.js';
|
||||||
import { T5_ZOMBIE_MAPS } from './map-aliases.js';
|
import { T5_ZOMBIE_MAPS } from './map-aliases.js';
|
||||||
import { allServers, getServer, listServers } from './servers.js';
|
import { serverDirectories, getServer, listServers } from './servers.js';
|
||||||
import { promotedArtifacts } from './promoted-artifacts.js';
|
import { promotedArtifacts } from './promoted-artifacts.js';
|
||||||
|
|
||||||
// Destinations drift scans, in a stable order so the UI shows scripts first.
|
// Destinations drift scans, in a stable order so the UI shows scripts first.
|
||||||
@@ -228,7 +228,7 @@ function composeExpectedCfg(
|
|||||||
export async function scanFleetDrift(): Promise<DriftEntry[]> {
|
export async function scanFleetDrift(): Promise<DriftEntry[]> {
|
||||||
const servers = listServers().map((s) => s.name);
|
const servers = listServers().map((s) => s.name);
|
||||||
const registered = new Set(servers);
|
const registered = new Set(servers);
|
||||||
const allDirs = new Set(await allServers());
|
const allDirs = new Set(await serverDirectories());
|
||||||
// Only touch server dirs that are both registered AND present on disk.
|
// Only touch server dirs that are both registered AND present on disk.
|
||||||
const eligible = servers.filter((n) => registered.has(n) && allDirs.has(n));
|
const eligible = servers.filter((n) => registered.has(n) && allDirs.has(n));
|
||||||
const out: DriftEntry[] = [];
|
const out: DriftEntry[] = [];
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
export type Health = 'online' | 'stale' | 'offline';
|
export type Health = 'online' | 'stale' | 'offline' | 'starting';
|
||||||
|
|
||||||
const ONLINE_MAX_MS = 30_000;
|
const ONLINE_MAX_MS = 30_000;
|
||||||
const STALE_MAX_MS = 120_000;
|
const STALE_MAX_MS = 120_000;
|
||||||
@@ -9,7 +9,9 @@ const STALE_MAX_MS = 120_000;
|
|||||||
* successful reply is >120s old. `stale` if the last reply is 30-120s old
|
* successful reply is >120s old. `stale` if the last reply is 30-120s old
|
||||||
* (RCON glitchy). `online` if fresh.
|
* (RCON glitchy). `online` if fresh.
|
||||||
*/
|
*/
|
||||||
export function classifyHealth(lastSeenAt: string | null, lastError: string | null): Health {
|
export function classifyHealth(lastSeenAt: string | null, lastError: string | null, containerState?: string | null): Health {
|
||||||
|
if (containerState === 'starting' || containerState === 'restarting') return 'starting';
|
||||||
|
if (containerState && containerState !== 'running') return 'offline';
|
||||||
if (!lastSeenAt) return 'offline';
|
if (!lastSeenAt) return 'offline';
|
||||||
const age = Date.now() - new Date(lastSeenAt).getTime();
|
const age = Date.now() - new Date(lastSeenAt).getTime();
|
||||||
if (!Number.isFinite(age) || age < 0) return 'offline';
|
if (!Number.isFinite(age) || age < 0) return 'offline';
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import { parseServerName } from './map-aliases.js';
|
|||||||
* All server folders under SERVERS_ROOT. Used by seedServerRegistry to backfill
|
* All server folders under SERVERS_ROOT. Used by seedServerRegistry to backfill
|
||||||
* the servers table for any on-disk server directory that isn't yet tracked.
|
* the servers table for any on-disk server directory that isn't yet tracked.
|
||||||
*/
|
*/
|
||||||
export async function allServers(): Promise<string[]> {
|
export async function serverDirectories(): Promise<string[]> {
|
||||||
let entries;
|
let entries;
|
||||||
try {
|
try {
|
||||||
entries = await readdir(config.serversRoot, { withFileTypes: true });
|
entries = await readdir(config.serversRoot, { withFileTypes: true });
|
||||||
@@ -21,6 +21,11 @@ export async function allServers(): Promise<string[]> {
|
|||||||
.sort();
|
.sort();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Fleet membership is owned by the database, independent of mounted files. */
|
||||||
|
export async function allServers(): Promise<string[]> {
|
||||||
|
return listServers().map((server) => server.name);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Match a server name against a list of glob-like patterns (supports `*` and
|
* Match a server name against a list of glob-like patterns (supports `*` and
|
||||||
* `?` and comma-separated lists).
|
* `?` and comma-separated lists).
|
||||||
@@ -64,6 +69,7 @@ export interface ServerRow {
|
|||||||
komodo_type: 'deployment' | 'stack' | 'container';
|
komodo_type: 'deployment' | 'stack' | 'container';
|
||||||
komodo_name: string;
|
komodo_name: string;
|
||||||
komodo_server: string | null;
|
komodo_server: string | null;
|
||||||
|
container_state: string | null;
|
||||||
game_port: number | null;
|
game_port: number | null;
|
||||||
rcon_host: string | null;
|
rcon_host: string | null;
|
||||||
game: string | null;
|
game: string | null;
|
||||||
@@ -93,7 +99,8 @@ export interface ServerRow {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export async function seedServerRegistry(): Promise<number> {
|
export async function seedServerRegistry(): Promise<number> {
|
||||||
const rows = await allServers();
|
// Explicit legacy import only; never run as part of application startup.
|
||||||
|
const rows = await serverDirectories();
|
||||||
// Under the new naming scheme Komodo entities match EC server names 1:1
|
// Under the new naming scheme Komodo entities match EC server names 1:1
|
||||||
// (both underscore-separated), so komodo_name = name — no transformation.
|
// (both underscore-separated), so komodo_name = name — no transformation.
|
||||||
const insert = db().prepare(
|
const insert = db().prepare(
|
||||||
@@ -143,7 +150,7 @@ export function getServer(name: string): ServerRow | undefined {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export function listServers(): ServerRow[] {
|
export function listServers(): ServerRow[] {
|
||||||
return db().prepare('SELECT * FROM servers ORDER BY name').all() as ServerRow[];
|
return db().prepare('SELECT * FROM servers WHERE destroyed_at IS NULL ORDER BY name').all() as ServerRow[];
|
||||||
}
|
}
|
||||||
|
|
||||||
export function setServerField(
|
export function setServerField(
|
||||||
@@ -155,6 +162,7 @@ export function setServerField(
|
|||||||
'komodo_type',
|
'komodo_type',
|
||||||
'komodo_name',
|
'komodo_name',
|
||||||
'komodo_server',
|
'komodo_server',
|
||||||
|
'container_state',
|
||||||
'game_port',
|
'game_port',
|
||||||
'rcon_host',
|
'rcon_host',
|
||||||
'game',
|
'game',
|
||||||
|
|||||||
@@ -255,7 +255,7 @@ export async function komodoResolveGamePort(serverName: string): Promise<number
|
|||||||
|
|
||||||
export type KomodoHealthResult =
|
export type KomodoHealthResult =
|
||||||
| { ok: true; state: string }
|
| { ok: true; state: string }
|
||||||
| { ok: false; reason: string; snippet?: string };
|
| { ok: false; reason: string; state?: string; snippet?: string };
|
||||||
|
|
||||||
// Shape-agnostic scan for the container's running-state. Walks the entire
|
// Shape-agnostic scan for the container's running-state. Walks the entire
|
||||||
// payload looking for anything that resembles a state field. Covered shapes:
|
// payload looking for anything that resembles a state field. Covered shapes:
|
||||||
@@ -290,6 +290,18 @@ export function extractContainerState(payload: unknown): string | null {
|
|||||||
}
|
}
|
||||||
const obj = v as Record<string, unknown>;
|
const obj = v as Record<string, unknown>;
|
||||||
|
|
||||||
|
// Komodo summaries expose health alongside a flat running state.
|
||||||
|
const flatState = obj['state'] ?? obj['State'] ?? obj['status'] ?? obj['Status'];
|
||||||
|
const flatHealth = obj['health'] ?? obj['Health'];
|
||||||
|
const healthStatus = typeof flatHealth === 'string' ? flatHealth
|
||||||
|
: flatHealth && typeof flatHealth === 'object'
|
||||||
|
? ((flatHealth as Record<string, unknown>)['status'] ?? (flatHealth as Record<string, unknown>)['Status']) : null;
|
||||||
|
if (typeof flatState === 'string' && flatState.toLowerCase() === 'running'
|
||||||
|
&& typeof healthStatus === 'string' && healthStatus.toLowerCase() !== 'healthy') {
|
||||||
|
found = healthStatus;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
// Docker inspect nested: { State: { Status, Running, Health: { Status } } }
|
// Docker inspect nested: { State: { Status, Running, Health: { Status } } }
|
||||||
const stateObj = obj['State'];
|
const stateObj = obj['State'];
|
||||||
if (stateObj && typeof stateObj === 'object') {
|
if (stateObj && typeof stateObj === 'object') {
|
||||||
@@ -401,7 +413,7 @@ export async function komodoInspectHealthy(serverName: string): Promise<KomodoHe
|
|||||||
const state = extractContainerState(r.json);
|
const state = extractContainerState(r.json);
|
||||||
if (state) {
|
if (state) {
|
||||||
const healthy = HEALTHY_STATES.has(state.toLowerCase());
|
const healthy = HEALTHY_STATES.has(state.toLowerCase());
|
||||||
if (!healthy) return { ok: false, reason: `container state: ${state}`, snippet: r.snippet };
|
if (!healthy) return { ok: false, state, reason: `container state: ${state}`, snippet: r.snippet };
|
||||||
return { ok: true, state };
|
return { ok: true, state };
|
||||||
}
|
}
|
||||||
lastSnippet = r.snippet;
|
lastSnippet = r.snippet;
|
||||||
|
|||||||
@@ -21,6 +21,13 @@ function slotFor(server: string): RestartSlot | undefined {
|
|||||||
return db().prepare('SELECT * FROM restart_rollout WHERE server = ?').get(server) as RestartSlot | undefined;
|
return db().prepare('SELECT * FROM restart_rollout WHERE server = ?').get(server) as RestartSlot | undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** The queued restart may still expose the previous healthy container. */
|
||||||
|
export function restartSettling(server: string): boolean {
|
||||||
|
const slot = slotFor(server);
|
||||||
|
return requestsInFlight.has(server) || !!(slot && !slot.healthy_at
|
||||||
|
&& Date.now() - Date.parse(slot.requested_at) < HEALTH_POLL_MS);
|
||||||
|
}
|
||||||
|
|
||||||
export function rolloutWaitReason(server: string): string | null {
|
export function rolloutWaitReason(server: string): string | null {
|
||||||
const slot = slotFor(server);
|
const slot = slotFor(server);
|
||||||
if (slot?.healthy_at === null) {
|
if (slot?.healthy_at === null) {
|
||||||
@@ -49,6 +56,7 @@ export async function restartInRollout(server: string): Promise<RolloutRestartRe
|
|||||||
if (waitReason) return { kind: 'queued', message: waitReason };
|
if (waitReason) return { kind: 'queued', message: waitReason };
|
||||||
|
|
||||||
requestsInFlight.add(server);
|
requestsInFlight.add(server);
|
||||||
|
db().prepare("UPDATE servers SET container_state = 'restarting', last_error = NULL, last_reply_snippet = NULL WHERE name = ?").run(server);
|
||||||
try {
|
try {
|
||||||
const result = await komodoRestart(server);
|
const result = await komodoRestart(server);
|
||||||
if (result.ok) return { kind: 'started' };
|
if (result.ok) return { kind: 'started' };
|
||||||
|
|||||||
@@ -5,7 +5,9 @@ import { emit } from './bus.js';
|
|||||||
import type { ServerStatusEntry } from './bus.js';
|
import type { ServerStatusEntry } from './bus.js';
|
||||||
import { rconStatusResult } from '../integrations/rcon.js';
|
import { rconStatusResult } from '../integrations/rcon.js';
|
||||||
import type { RconFailure } from '../integrations/rcon.js';
|
import type { RconFailure } from '../integrations/rcon.js';
|
||||||
import { listServers, seedServerRegistry, setServerField } from '../fleet/servers.js';
|
import { listServers, setServerField } from '../fleet/servers.js';
|
||||||
|
import { komodoInspectHealthy } from '../integrations/komodo.js';
|
||||||
|
import { restartSettling } from './restart-rollout.js';
|
||||||
import { classifyHealth } from '../fleet/health.js';
|
import { classifyHealth } from '../fleet/health.js';
|
||||||
import { scanMatchState } from '../fleet/match-log.js';
|
import { scanMatchState } from '../fleet/match-log.js';
|
||||||
|
|
||||||
@@ -108,7 +110,15 @@ function encodeError(reason: RconFailure, detail: string): string {
|
|||||||
return `${reason}:${detail}`;
|
return `${reason}:${detail}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fanoutStatus(): Promise<void> {
|
let statusInFlight: Promise<void> | null = null;
|
||||||
|
|
||||||
|
export function fanoutStatus(): Promise<void> {
|
||||||
|
if (statusInFlight) return statusInFlight;
|
||||||
|
statusInFlight = refreshStatus().finally(() => { statusInFlight = null; });
|
||||||
|
return statusInFlight;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function refreshStatus(): Promise<void> {
|
||||||
const servers = listServers();
|
const servers = listServers();
|
||||||
if (servers.length === 0) return;
|
if (servers.length === 0) return;
|
||||||
|
|
||||||
@@ -120,6 +130,28 @@ async function fanoutStatus(): Promise<void> {
|
|||||||
const i = cursor++;
|
const i = cursor++;
|
||||||
if (i >= servers.length) return;
|
if (i >= servers.length) return;
|
||||||
const srv = servers[i];
|
const srv = servers[i];
|
||||||
|
const container = restartSettling(srv.name)
|
||||||
|
? { ok: false as const, state: 'restarting', reason: 'Restart requested' }
|
||||||
|
: await komodoInspectHealthy(srv.name).catch((err) => ({
|
||||||
|
ok: false as const, reason: (err as Error).message
|
||||||
|
}));
|
||||||
|
const containerState = container.ok ? 'running'
|
||||||
|
: ('state' in container && typeof container.state === 'string' ? container.state.toLowerCase() : 'unknown');
|
||||||
|
setServerField(srv.name, 'container_state', containerState);
|
||||||
|
if (!container.ok) {
|
||||||
|
// Container readiness precedes RCON readiness. Old replies and
|
||||||
|
// failures describe the previous process, not this startup.
|
||||||
|
setServerField(srv.name, 'last_error', null);
|
||||||
|
setServerField(srv.name, 'last_reply_snippet', null);
|
||||||
|
entries[i] = {
|
||||||
|
server: srv.name, players: null, map: srv.map, gametype: null,
|
||||||
|
roster: null, health: classifyHealth(null, null, containerState),
|
||||||
|
lastSeenAt: srv.last_seen_at, lastError: null, endpoint: null,
|
||||||
|
reason: null, replySnippet: null, activeSince: null,
|
||||||
|
inactiveSince: null, gameStartedAt: null, gameEndedAt: null
|
||||||
|
};
|
||||||
|
continue;
|
||||||
|
}
|
||||||
const result = await rconStatusResult(srv.name).catch((err) => ({
|
const result = await rconStatusResult(srv.name).catch((err) => ({
|
||||||
ok: false as const,
|
ok: false as const,
|
||||||
endpoint: null,
|
endpoint: null,
|
||||||
@@ -231,8 +263,6 @@ export function startWorker(): void {
|
|||||||
if (started) return;
|
if (started) return;
|
||||||
started = true;
|
started = true;
|
||||||
|
|
||||||
seedServerRegistry().catch((err) => console.error('[worker] seed error:', err));
|
|
||||||
|
|
||||||
void tick();
|
void tick();
|
||||||
void fanoutStatus();
|
void fanoutStatus();
|
||||||
tickTimer = setInterval(tick, TICK_MS);
|
tickTimer = setInterval(tick, TICK_MS);
|
||||||
|
|||||||
@@ -23,7 +23,9 @@
|
|||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
{#if health === 'offline'}
|
{#if health === 'starting'}
|
||||||
|
<div class="state empty">Server starting</div>
|
||||||
|
{:else if health === 'offline'}
|
||||||
<div class="state offline">server unreachable</div>
|
<div class="state offline">server unreachable</div>
|
||||||
{:else if !roster || roster.length === 0}
|
{:else if !roster || roster.length === 0}
|
||||||
<div class="state empty">lobby is empty</div>
|
<div class="state empty">lobby is empty</div>
|
||||||
|
|||||||
@@ -63,7 +63,7 @@
|
|||||||
.sort((a, b) => b.score - a.score)
|
.sort((a, b) => b.score - a.score)
|
||||||
.slice(0, 3)
|
.slice(0, 3)
|
||||||
);
|
);
|
||||||
let showRoster = $derived(health !== 'offline');
|
let showRoster = $derived(health === 'online' || health === 'stale');
|
||||||
let mapLabel = $derived(map ?? '—');
|
let mapLabel = $derived(map ?? '—');
|
||||||
let gtLabel = $derived(gametypeLabel(gametype));
|
let gtLabel = $derived(gametypeLabel(gametype));
|
||||||
|
|
||||||
@@ -72,7 +72,7 @@
|
|||||||
let gameInProgress = $derived(!!gameStartedAt && !gameEndedAt && !!activeSince);
|
let gameInProgress = $derived(!!gameStartedAt && !gameEndedAt && !!activeSince);
|
||||||
let gameElapsed = $derived(gameInProgress ? elapsedSince(gameStartedAt, $uiNow) : null);
|
let gameElapsed = $derived(gameInProgress ? elapsedSince(gameStartedAt, $uiNow) : null);
|
||||||
let showTiming = $derived(
|
let showTiming = $derived(
|
||||||
health !== 'offline' && (activeElapsed !== null || inactiveElapsed !== null || gameElapsed !== null)
|
showRoster && (activeElapsed !== null || inactiveElapsed !== null || gameElapsed !== null)
|
||||||
);
|
);
|
||||||
|
|
||||||
// Shared-element transition to /servers/{name} — Chromium only, others
|
// Shared-element transition to /servers/{name} — Chromium only, others
|
||||||
@@ -97,7 +97,9 @@
|
|||||||
<StatusPip {health} />
|
<StatusPip {health} />
|
||||||
<strong class="name">{name}</strong>
|
<strong class="name">{name}</strong>
|
||||||
</div>
|
</div>
|
||||||
{#if health === 'offline'}
|
{#if health === 'starting'}
|
||||||
|
<Badge variant="info" uppercase>Starting</Badge>
|
||||||
|
{:else if health === 'offline'}
|
||||||
<Badge variant="danger" uppercase>offline</Badge>
|
<Badge variant="danger" uppercase>offline</Badge>
|
||||||
{:else}
|
{:else}
|
||||||
<div class="players" aria-live="polite">
|
<div class="players" aria-live="polite">
|
||||||
@@ -119,7 +121,7 @@
|
|||||||
{/if}
|
{/if}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<RoundTelemetry {name} />
|
{#if showRoster}<RoundTelemetry {name} />{/if}
|
||||||
|
|
||||||
{#if showTiming}
|
{#if showTiming}
|
||||||
<div class="timing">
|
<div class="timing">
|
||||||
|
|||||||
@@ -26,7 +26,7 @@
|
|||||||
background: var(--spark-500);
|
background: var(--spark-500);
|
||||||
animation: electric-pulse 1.6s ease-in-out infinite;
|
animation: electric-pulse 1.6s ease-in-out infinite;
|
||||||
}
|
}
|
||||||
.dot-stale {
|
.dot-stale, .dot-starting {
|
||||||
background: var(--amber-500);
|
background: var(--amber-500);
|
||||||
box-shadow: var(--amber-glow);
|
box-shadow: var(--amber-glow);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,13 +11,14 @@
|
|||||||
|
|
||||||
let toneClass = $derived(
|
let toneClass = $derived(
|
||||||
health === 'online' ? 't-online' :
|
health === 'online' ? 't-online' :
|
||||||
health === 'stale' ? 't-warning' :
|
health === 'stale' || health === 'starting' ? 't-warning' :
|
||||||
't-danger'
|
't-danger'
|
||||||
);
|
);
|
||||||
|
|
||||||
let announceLabel = $derived(
|
let announceLabel = $derived(
|
||||||
label ?? (
|
label ?? (
|
||||||
health === 'online' ? 'online' :
|
health === 'online' ? 'online' :
|
||||||
|
health === 'starting' ? 'starting' :
|
||||||
health === 'stale' ? 'stale' :
|
health === 'stale' ? 'stale' :
|
||||||
'offline'
|
'offline'
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ export interface RosterPlayer {
|
|||||||
name: string;
|
name: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export type Health = 'online' | 'stale' | 'offline';
|
export type Health = 'online' | 'stale' | 'offline' | 'starting';
|
||||||
export type RconFailure = 'no-config' | 'no-port' | 'send-error' | 'timeout' | 'bad-reply' | 'no-players';
|
export type RconFailure = 'no-config' | 'no-port' | 'send-error' | 'timeout' | 'bad-reply' | 'no-players';
|
||||||
|
|
||||||
export interface RconEndpoint {
|
export interface RconEndpoint {
|
||||||
|
|||||||
@@ -19,16 +19,25 @@ function safeJson<T>(raw: string | null): T | null {
|
|||||||
|
|
||||||
export async function GET() {
|
export async function GET() {
|
||||||
const rconHost = config.rcon.host;
|
const rconHost = config.rcon.host;
|
||||||
const servers = listServers().map((s) => ({
|
const servers = listServers().map((s) => {
|
||||||
...s,
|
const ready = !s.container_state || s.container_state === 'running';
|
||||||
roster: safeJson<RconPlayer[]>(s.last_roster),
|
return {
|
||||||
health: classifyHealth(s.last_seen_at, s.last_error),
|
...s,
|
||||||
replySnippet: s.last_reply_snippet,
|
last_players: ready ? s.last_players : null,
|
||||||
// Derived endpoint — same logic as endpointFor() but without asking
|
last_error: ready ? s.last_error : null,
|
||||||
// Komodo. If we don't have a port yet, endpoint is null.
|
active_since: ready ? s.active_since : null,
|
||||||
endpoint: s.game_port ? { host: s.rcon_host ?? rconHost, port: s.game_port } : null,
|
inactive_since: ready ? s.inactive_since : null,
|
||||||
held: [...heldByBeta(s.name).entries()].map(([target, betaId]) => ({ target, betaId }))
|
game_started_at: ready ? s.game_started_at : null,
|
||||||
}));
|
game_ended_at: ready ? s.game_ended_at : null,
|
||||||
|
roster: ready ? safeJson<RconPlayer[]>(s.last_roster) : null,
|
||||||
|
health: classifyHealth(s.last_seen_at, s.last_error, s.container_state),
|
||||||
|
replySnippet: ready ? s.last_reply_snippet : null,
|
||||||
|
// Derived endpoint — same logic as endpointFor() but without asking
|
||||||
|
// Komodo. If we don't have a port yet, endpoint is null.
|
||||||
|
endpoint: s.game_port ? { host: s.rcon_host ?? rconHost, port: s.game_port } : null,
|
||||||
|
held: [...heldByBeta(s.name).entries()].map(([target, betaId]) => ({ target, betaId }))
|
||||||
|
};
|
||||||
|
});
|
||||||
return json({
|
return json({
|
||||||
servers,
|
servers,
|
||||||
reservedNames: (db().prepare("SELECT j.server FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')").all() as { server: string }[]).map(row => row.server),
|
reservedNames: (db().prepare("SELECT j.server FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')").all() as { server: string }[]).map(row => row.server),
|
||||||
|
|||||||
@@ -126,7 +126,7 @@
|
|||||||
let effectiveRoster = $derived(live?.roster ?? server?.roster ?? null);
|
let effectiveRoster = $derived(live?.roster ?? server?.roster ?? null);
|
||||||
let effectiveHealth = $derived<Health>(live?.health ?? server?.health ?? 'offline');
|
let effectiveHealth = $derived<Health>(live?.health ?? server?.health ?? 'offline');
|
||||||
let effectiveSeen = $derived(live?.lastSeenAt ?? server?.last_seen_at ?? null);
|
let effectiveSeen = $derived(live?.lastSeenAt ?? server?.last_seen_at ?? null);
|
||||||
let effectiveErr = $derived(live?.lastError ?? server?.last_error ?? null);
|
let effectiveErr = $derived(live ? live.lastError : server?.last_error ?? null);
|
||||||
let effectiveEndpoint = $derived<RconEndpoint | null>(live?.endpoint ?? server?.endpoint ?? null);
|
let effectiveEndpoint = $derived<RconEndpoint | null>(live?.endpoint ?? server?.endpoint ?? null);
|
||||||
let liveReason = $derived<RconFailure | null>(live?.reason ?? null);
|
let liveReason = $derived<RconFailure | null>(live?.reason ?? null);
|
||||||
let decoded = $derived(decodeRconError(effectiveErr));
|
let decoded = $derived(decodeRconError(effectiveErr));
|
||||||
@@ -139,12 +139,12 @@
|
|||||||
let effectiveInactiveSince = $derived<string | null>(live?.inactiveSince ?? server?.inactive_since ?? null);
|
let effectiveInactiveSince = $derived<string | null>(live?.inactiveSince ?? server?.inactive_since ?? null);
|
||||||
let effectiveGameStartedAt = $derived<string | null>(live?.gameStartedAt ?? server?.game_started_at ?? null);
|
let effectiveGameStartedAt = $derived<string | null>(live?.gameStartedAt ?? server?.game_started_at ?? null);
|
||||||
let effectiveGameEndedAt = $derived<string | null>(live?.gameEndedAt ?? server?.game_ended_at ?? null);
|
let effectiveGameEndedAt = $derived<string | null>(live?.gameEndedAt ?? server?.game_ended_at ?? null);
|
||||||
let activeElapsed = $derived(effectiveActiveSince ? elapsedSince(effectiveActiveSince, $uiNow) : null);
|
let activeElapsed = $derived(effectiveHealth !== 'starting' && effectiveActiveSince ? elapsedSince(effectiveActiveSince, $uiNow) : null);
|
||||||
let inactiveElapsed = $derived(effectiveInactiveSince ? elapsedSince(effectiveInactiveSince, $uiNow) : null);
|
let inactiveElapsed = $derived(effectiveHealth !== 'starting' && effectiveInactiveSince ? elapsedSince(effectiveInactiveSince, $uiNow) : null);
|
||||||
let gameInProgress = $derived(!!effectiveGameStartedAt && !effectiveGameEndedAt && !!effectiveActiveSince);
|
let gameInProgress = $derived(!!effectiveGameStartedAt && !effectiveGameEndedAt && !!effectiveActiveSince);
|
||||||
let gameElapsed = $derived(gameInProgress ? elapsedSince(effectiveGameStartedAt, $uiNow) : null);
|
let gameElapsed = $derived(effectiveHealth !== 'starting' && gameInProgress ? elapsedSince(effectiveGameStartedAt, $uiNow) : null);
|
||||||
|
|
||||||
let hasDiagnosticIssue = $derived(effectiveHealth !== 'online');
|
let hasDiagnosticIssue = $derived(effectiveHealth !== 'online' && effectiveHealth !== 'starting');
|
||||||
|
|
||||||
// Auto-open the Diagnostics tab if there's a problem the operator should see.
|
// Auto-open the Diagnostics tab if there's a problem the operator should see.
|
||||||
$effect(() => {
|
$effect(() => {
|
||||||
@@ -237,13 +237,14 @@
|
|||||||
<StatusPip health={effectiveHealth} size="md" />
|
<StatusPip health={effectiveHealth} size="md" />
|
||||||
<span class="health-label">
|
<span class="health-label">
|
||||||
{effectiveHealth === 'online' ? 'Online' :
|
{effectiveHealth === 'online' ? 'Online' :
|
||||||
|
effectiveHealth === 'starting' ? 'Starting' :
|
||||||
effectiveHealth === 'stale' ? 'Stale' : 'Offline'}
|
effectiveHealth === 'stale' ? 'Stale' : 'Offline'}
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
<div class="metric-cell">
|
<div class="metric-cell">
|
||||||
<Stat
|
<Stat
|
||||||
label="Players"
|
label="Players"
|
||||||
value={effectivePlayers ?? 0}
|
value={effectiveHealth === 'starting' ? '—' : effectivePlayers ?? 0}
|
||||||
size="xl"
|
size="xl"
|
||||||
tone={(effectivePlayers ?? 0) > 0 ? 'online' : 'primary'}
|
tone={(effectivePlayers ?? 0) > 0 ? 'online' : 'primary'}
|
||||||
tween
|
tween
|
||||||
@@ -276,7 +277,7 @@
|
|||||||
{/if}
|
{/if}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<RoundTelemetry {name} />
|
{#if effectiveHealth !== 'starting'}<RoundTelemetry {name} />{/if}
|
||||||
|
|
||||||
</Card>
|
</Card>
|
||||||
|
|
||||||
@@ -327,7 +328,9 @@
|
|||||||
<Card padding="lg">
|
<Card padding="lg">
|
||||||
<div class="panel-head">
|
<div class="panel-head">
|
||||||
<h2>RCON diagnostic</h2>
|
<h2>RCON diagnostic</h2>
|
||||||
{#if hasDiagnosticIssue}
|
{#if effectiveHealth === 'starting'}
|
||||||
|
<Badge variant="info" uppercase>Waiting for container</Badge>
|
||||||
|
{:else if hasDiagnosticIssue}
|
||||||
<Badge variant="warning" uppercase>Issue</Badge>
|
<Badge variant="warning" uppercase>Issue</Badge>
|
||||||
{:else}
|
{:else}
|
||||||
<Badge variant="online" uppercase>Healthy</Badge>
|
<Badge variant="online" uppercase>Healthy</Badge>
|
||||||
|
|||||||
@@ -4,6 +4,8 @@ import { extractUdpPort, extractContainerState } from '../../src/lib/server/inte
|
|||||||
describe('container readiness', () => {
|
describe('container readiness', () => {
|
||||||
it.each(['starting', 'unhealthy'])('does not accept a running container whose health is %s', (health) => {
|
it.each(['starting', 'unhealthy'])('does not accept a running container whose health is %s', (health) => {
|
||||||
expect(extractContainerState({ State: { Status: 'running', Running: true, Health: { Status: health } } })).toBe(health);
|
expect(extractContainerState({ State: { Status: 'running', Running: true, Health: { Status: health } } })).toBe(health);
|
||||||
|
expect(extractContainerState({ state: 'running', health })).toBe(health);
|
||||||
|
expect(extractContainerState({ state: 'running', health: { status: health } })).toBe(health);
|
||||||
});
|
});
|
||||||
it('accepts healthy running containers and preserves stopped states', () => {
|
it('accepts healthy running containers and preserves stopped states', () => {
|
||||||
expect(extractContainerState({ State: { Status: 'running', Running: true, Health: { Status: 'healthy' } } })).toBe('running');
|
expect(extractContainerState({ State: { Status: 'running', Running: true, Health: { Status: 'healthy' } } })).toBe('running');
|
||||||
|
|||||||
@@ -0,0 +1,75 @@
|
|||||||
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||||
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||||
|
|
||||||
|
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
||||||
|
komodoInspectHealthy: vi.fn(), komodoRestart: vi.fn()
|
||||||
|
}));
|
||||||
|
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
||||||
|
rconStatusResult: vi.fn(), rconGetPlayerCount: vi.fn()
|
||||||
|
}));
|
||||||
|
|
||||||
|
let fixture: Fixture;
|
||||||
|
beforeEach(async () => {
|
||||||
|
vi.resetModules();
|
||||||
|
vi.clearAllMocks();
|
||||||
|
fixture = await newFixture(['unregistered-folder']);
|
||||||
|
});
|
||||||
|
afterEach(async () => { await fixture.cleanup(); });
|
||||||
|
|
||||||
|
async function setup() {
|
||||||
|
const { db } = await import('../../src/lib/server/db/index.js');
|
||||||
|
db().prepare("INSERT INTO servers (name, komodo_name, last_error) VALUES ('managed', 'managed', 'timeout:old process')").run();
|
||||||
|
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||||
|
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
||||||
|
const { fanoutStatus } = await import('../../src/lib/server/pipeline/worker.js');
|
||||||
|
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
|
||||||
|
return { db: db(), inspect: vi.mocked(komodoInspectHealthy), rcon: vi.mocked(rconStatusResult), fanoutStatus, getServer };
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('database fleet and container readiness', () => {
|
||||||
|
it('lists registered servers without folders and ignores unregistered folders and tombstones', async () => {
|
||||||
|
const { db } = await setup();
|
||||||
|
const { allServers, listServers } = await import('../../src/lib/server/fleet/servers.js');
|
||||||
|
expect(await allServers()).toEqual(['managed']);
|
||||||
|
db.prepare("UPDATE servers SET destroyed_at = '2026-01-01' WHERE name = 'managed'").run();
|
||||||
|
expect(listServers()).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('skips startup RCON, clears old errors, and resumes polling only after Komodo is ready', async () => {
|
||||||
|
const { inspect, rcon, fanoutStatus, getServer } = await setup();
|
||||||
|
const { GET } = await import('../../src/routes/api/servers/+server.js');
|
||||||
|
inspect.mockResolvedValue({ ok: false, state: 'starting', reason: 'container state: starting' });
|
||||||
|
await fanoutStatus();
|
||||||
|
await fanoutStatus();
|
||||||
|
expect(rcon).not.toHaveBeenCalled();
|
||||||
|
expect(getServer('managed')).toMatchObject({ container_state: 'starting', last_error: null });
|
||||||
|
expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'starting', last_error: null, last_players: null });
|
||||||
|
|
||||||
|
inspect.mockResolvedValue({ ok: true, state: 'running' });
|
||||||
|
rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'no reply' });
|
||||||
|
await fanoutStatus();
|
||||||
|
expect(rcon).toHaveBeenCalledTimes(1);
|
||||||
|
expect(getServer('managed')).toMatchObject({ container_state: 'running', last_error: 'timeout:no reply' });
|
||||||
|
|
||||||
|
rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status',
|
||||||
|
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } });
|
||||||
|
await fanoutStatus();
|
||||||
|
expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'online', last_error: null });
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each(['unhealthy', 'exited', undefined])('does not poll RCON with container state %s', async (state) => {
|
||||||
|
const { inspect, rcon, fanoutStatus } = await setup();
|
||||||
|
inspect.mockResolvedValue({ ok: false, state, reason: 'not ready' });
|
||||||
|
await fanoutStatus();
|
||||||
|
expect(rcon).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('waits for an EC restart request to settle before trusting the old healthy container', async () => {
|
||||||
|
const { db, inspect, rcon, fanoutStatus, getServer } = await setup();
|
||||||
|
db.prepare("INSERT INTO restart_rollout (server, requested_at) VALUES ('managed', ?)").run(new Date().toISOString());
|
||||||
|
await fanoutStatus();
|
||||||
|
expect(inspect).not.toHaveBeenCalled();
|
||||||
|
expect(rcon).not.toHaveBeenCalled();
|
||||||
|
expect(getServer('managed')?.container_state).toBe('restarting');
|
||||||
|
});
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user