Use database fleet registry and gate RCON polling on container readiness

src/lib/server/db/index.ts

@@ -12,6 +12,7 @@ let _db: Database.Database | null = null;
 // IF NOT EXISTS in schema.sql is a no-op on existing tables, so it can't
 // backfill columns; this can.
 const SERVERS_COLUMNS: Array<{ name: string; ddl: string }> = [
+    { name: 'container_state', ddl: 'ALTER TABLE servers ADD COLUMN container_state TEXT' },
     // Base columns that shipped in the very first schema stay here for parity
     // with fresh installs — no ALTER is issued if they're already present.
     { name: 'komodo_server', ddl: 'ALTER TABLE servers ADD COLUMN komodo_server TEXT' },

src/lib/server/db/schema.sql

@@ -11,6 +11,7 @@ CREATE TABLE IF NOT EXISTS servers (
     komodo_type      TEXT NOT NULL DEFAULT 'deployment',  -- deployment|stack|container
     komodo_name      TEXT NOT NULL,
     komodo_server    TEXT,
+    container_state TEXT,                                -- last observed Komodo runtime state
     game_port        INTEGER,                             -- UDP RCON port; lazy-filled from Komodo
     rcon_host        TEXT,                                -- optional per-server override of RCON_HOST
     game             TEXT,                                -- 't5' etc.; static per-server (no map rotation)

src/lib/server/fleet/create.ts

@@ -271,11 +271,11 @@ async function createServerUnlocked(input: CreateServerInput, prepared: Prepared
                 `INSERT INTO servers
                     (name, komodo_type, komodo_name, game, map,
                      game_mode, map_slug, host_dir, config_file, game_id, image,
-                     game_port, created_by)
+                     game_port, created_by, container_state)
                  VALUES
                     (?, 'deployment', ?, 't5', ?,
                      ?, ?, ?, ?, ?, ?,
-                     ?, ?)`
+                     ?, ?, 'starting')`
             ).run(
                 name, name, mapEntry.canonical,
                 input.gameMode, mapEntry.slug, hostDir,

src/lib/server/fleet/drift.ts

@@ -14,7 +14,7 @@ import {
 import { getBaselineForScope, getVersion } from './scripts.js';
 import { getPromotedServerConfig, getServerMapConfig, MAP_CONFIG_FILE } from './server-config.js';
 import { T5_ZOMBIE_MAPS } from './map-aliases.js';
-import { allServers, getServer, listServers } from './servers.js';
+import { serverDirectories, getServer, listServers } from './servers.js';
 import { promotedArtifacts } from './promoted-artifacts.js';
 
 // Destinations drift scans, in a stable order so the UI shows scripts first.
@@ -228,7 +228,7 @@ function composeExpectedCfg(
 export async function scanFleetDrift(): Promise<DriftEntry[]> {
     const servers = listServers().map((s) => s.name);
     const registered = new Set(servers);
-    const allDirs = new Set(await allServers());
+    const allDirs = new Set(await serverDirectories());
     // Only touch server dirs that are both registered AND present on disk.
     const eligible = servers.filter((n) => registered.has(n) && allDirs.has(n));
     const out: DriftEntry[] = [];

src/lib/server/fleet/health.ts

@@ -1,4 +1,4 @@
-export type Health = 'online' | 'stale' | 'offline';
+export type Health = 'online' | 'stale' | 'offline' | 'starting';
 
 const ONLINE_MAX_MS = 30_000;
 const STALE_MAX_MS = 120_000;
@@ -9,7 +9,9 @@ const STALE_MAX_MS = 120_000;
  * successful reply is >120s old. `stale` if the last reply is 30-120s old
  * (RCON glitchy). `online` if fresh.
  */
-export function classifyHealth(lastSeenAt: string | null, lastError: string | null): Health {
+export function classifyHealth(lastSeenAt: string | null, lastError: string | null, containerState?: string | null): Health {
+    if (containerState === 'starting' || containerState === 'restarting') return 'starting';
+    if (containerState && containerState !== 'running') return 'offline';
     if (!lastSeenAt) return 'offline';
     const age = Date.now() - new Date(lastSeenAt).getTime();
     if (!Number.isFinite(age) || age < 0) return 'offline';

src/lib/server/fleet/servers.ts

@@ -8,7 +8,7 @@ import { parseServerName } from './map-aliases.js';
  * All server folders under SERVERS_ROOT. Used by seedServerRegistry to backfill
  * the servers table for any on-disk server directory that isn't yet tracked.
  */
-export async function allServers(): Promise<string[]> {
+export async function serverDirectories(): Promise<string[]> {
     let entries;
     try {
         entries = await readdir(config.serversRoot, { withFileTypes: true });
@@ -21,6 +21,11 @@ export async function allServers(): Promise<string[]> {
         .sort();
 }
 
+/** Fleet membership is owned by the database, independent of mounted files. */
+export async function allServers(): Promise<string[]> {
+    return listServers().map((server) => server.name);
+}
+
 /**
  * Match a server name against a list of glob-like patterns (supports `*` and
  * `?` and comma-separated lists).
@@ -64,6 +69,7 @@ export interface ServerRow {
     komodo_type: 'deployment' | 'stack' | 'container';
     komodo_name: string;
     komodo_server: string | null;
+    container_state: string | null;
     game_port: number | null;
     rcon_host: string | null;
     game: string | null;
@@ -93,7 +99,8 @@ export interface ServerRow {
 }
 
 export async function seedServerRegistry(): Promise<number> {
-    const rows = await allServers();
+    // Explicit legacy import only; never run as part of application startup.
+    const rows = await serverDirectories();
     // Under the new naming scheme Komodo entities match EC server names 1:1
     // (both underscore-separated), so komodo_name = name — no transformation.
     const insert = db().prepare(
@@ -143,7 +150,7 @@ export function getServer(name: string): ServerRow | undefined {
 }
 
 export function listServers(): ServerRow[] {
-    return db().prepare('SELECT * FROM servers ORDER BY name').all() as ServerRow[];
+    return db().prepare('SELECT * FROM servers WHERE destroyed_at IS NULL ORDER BY name').all() as ServerRow[];
 }
 
 export function setServerField(
@@ -155,6 +162,7 @@ export function setServerField(
         'komodo_type',
         'komodo_name',
         'komodo_server',
+        'container_state',
         'game_port',
         'rcon_host',
         'game',

src/lib/server/integrations/komodo.ts

@@ -255,7 +255,7 @@ export async function komodoResolveGamePort(serverName: string): Promise<number
 
 export type KomodoHealthResult =
     | { ok: true; state: string }
-    | { ok: false; reason: string; snippet?: string };
+    | { ok: false; reason: string; state?: string; snippet?: string };
 
 // Shape-agnostic scan for the container's running-state. Walks the entire
 // payload looking for anything that resembles a state field. Covered shapes:
@@ -290,6 +290,18 @@ export function extractContainerState(payload: unknown): string | null {
         }
         const obj = v as Record<string, unknown>;
 
+        // Komodo summaries expose health alongside a flat running state.
+        const flatState = obj['state'] ?? obj['State'] ?? obj['status'] ?? obj['Status'];
+        const flatHealth = obj['health'] ?? obj['Health'];
+        const healthStatus = typeof flatHealth === 'string' ? flatHealth
+            : flatHealth && typeof flatHealth === 'object'
+                ? ((flatHealth as Record<string, unknown>)['status'] ?? (flatHealth as Record<string, unknown>)['Status']) : null;
+        if (typeof flatState === 'string' && flatState.toLowerCase() === 'running'
+            && typeof healthStatus === 'string' && healthStatus.toLowerCase() !== 'healthy') {
+            found = healthStatus;
+            return;
+        }
+
         // Docker inspect nested: { State: { Status, Running, Health: { Status } } }
         const stateObj = obj['State'];
         if (stateObj && typeof stateObj === 'object') {
@@ -401,7 +413,7 @@ export async function komodoInspectHealthy(serverName: string): Promise<KomodoHe
         const state = extractContainerState(r.json);
         if (state) {
             const healthy = HEALTHY_STATES.has(state.toLowerCase());
-            if (!healthy) return { ok: false, reason: `container state: ${state}`, snippet: r.snippet };
+            if (!healthy) return { ok: false, state, reason: `container state: ${state}`, snippet: r.snippet };
             return { ok: true, state };
         }
         lastSnippet = r.snippet;

src/lib/server/pipeline/restart-rollout.ts

@@ -21,6 +21,13 @@ function slotFor(server: string): RestartSlot | undefined {
     return db().prepare('SELECT * FROM restart_rollout WHERE server = ?').get(server) as RestartSlot | undefined;
 }
 
+/** The queued restart may still expose the previous healthy container. */
+export function restartSettling(server: string): boolean {
+    const slot = slotFor(server);
+    return requestsInFlight.has(server) || !!(slot && !slot.healthy_at
+        && Date.now() - Date.parse(slot.requested_at) < HEALTH_POLL_MS);
+}
+
 export function rolloutWaitReason(server: string): string | null {
     const slot = slotFor(server);
     if (slot?.healthy_at === null) {
@@ -49,6 +56,7 @@ export async function restartInRollout(server: string): Promise<RolloutRestartRe
     if (waitReason) return { kind: 'queued', message: waitReason };
 
     requestsInFlight.add(server);
+    db().prepare("UPDATE servers SET container_state = 'restarting', last_error = NULL, last_reply_snippet = NULL WHERE name = ?").run(server);
     try {
         const result = await komodoRestart(server);
         if (result.ok) return { kind: 'started' };

src/lib/server/pipeline/worker.ts

@@ -5,7 +5,9 @@ import { emit } from './bus.js';
 import type { ServerStatusEntry } from './bus.js';
 import { rconStatusResult } from '../integrations/rcon.js';
 import type { RconFailure } from '../integrations/rcon.js';
-import { listServers, seedServerRegistry, setServerField } from '../fleet/servers.js';
+import { listServers, setServerField } from '../fleet/servers.js';
+import { komodoInspectHealthy } from '../integrations/komodo.js';
+import { restartSettling } from './restart-rollout.js';
 import { classifyHealth } from '../fleet/health.js';
 import { scanMatchState } from '../fleet/match-log.js';
 
@@ -108,7 +110,15 @@ function encodeError(reason: RconFailure, detail: string): string {
     return `${reason}:${detail}`;
 }
 
-async function fanoutStatus(): Promise<void> {
+let statusInFlight: Promise<void> | null = null;
+
+export function fanoutStatus(): Promise<void> {
+    if (statusInFlight) return statusInFlight;
+    statusInFlight = refreshStatus().finally(() => { statusInFlight = null; });
+    return statusInFlight;
+}
+
+async function refreshStatus(): Promise<void> {
     const servers = listServers();
     if (servers.length === 0) return;
 
@@ -120,6 +130,28 @@ async function fanoutStatus(): Promise<void> {
             const i = cursor++;
             if (i >= servers.length) return;
             const srv = servers[i];
+            const container = restartSettling(srv.name)
+                ? { ok: false as const, state: 'restarting', reason: 'Restart requested' }
+                : await komodoInspectHealthy(srv.name).catch((err) => ({
+                ok: false as const, reason: (err as Error).message
+            }));
+            const containerState = container.ok ? 'running'
+                : ('state' in container && typeof container.state === 'string' ? container.state.toLowerCase() : 'unknown');
+            setServerField(srv.name, 'container_state', containerState);
+            if (!container.ok) {
+                // Container readiness precedes RCON readiness. Old replies and
+                // failures describe the previous process, not this startup.
+                setServerField(srv.name, 'last_error', null);
+                setServerField(srv.name, 'last_reply_snippet', null);
+                entries[i] = {
+                    server: srv.name, players: null, map: srv.map, gametype: null,
+                    roster: null, health: classifyHealth(null, null, containerState),
+                    lastSeenAt: srv.last_seen_at, lastError: null, endpoint: null,
+                    reason: null, replySnippet: null, activeSince: null,
+                    inactiveSince: null, gameStartedAt: null, gameEndedAt: null
+                };
+                continue;
+            }
             const result = await rconStatusResult(srv.name).catch((err) => ({
                 ok: false as const,
                 endpoint: null,
@@ -231,8 +263,6 @@ export function startWorker(): void {
     if (started) return;
     started = true;
 
-    seedServerRegistry().catch((err) => console.error('[worker] seed error:', err));
-
     void tick();
     void fanoutStatus();
     tickTimer = setInterval(tick, TICK_MS);

src/lib/ui/components/PlayerRoster.svelte

@@ -23,7 +23,9 @@
     }
 </script>
 
-{#if health === 'offline'}
+{#if health === 'starting'}
+    <div class="state empty">Server starting</div>
+{:else if health === 'offline'}
     <div class="state offline">server unreachable</div>
 {:else if !roster || roster.length === 0}
     <div class="state empty">lobby is empty</div>

src/lib/ui/components/ServerCard.svelte

@@ -63,7 +63,7 @@
             .sort((a, b) => b.score - a.score)
             .slice(0, 3)
     );
-    let showRoster = $derived(health !== 'offline');
+    let showRoster = $derived(health === 'online' || health === 'stale');
     let mapLabel = $derived(map ?? '—');
     let gtLabel = $derived(gametypeLabel(gametype));
 
@@ -72,7 +72,7 @@
     let gameInProgress = $derived(!!gameStartedAt && !gameEndedAt && !!activeSince);
     let gameElapsed = $derived(gameInProgress ? elapsedSince(gameStartedAt, $uiNow) : null);
     let showTiming = $derived(
-        health !== 'offline' && (activeElapsed !== null || inactiveElapsed !== null || gameElapsed !== null)
+        showRoster && (activeElapsed !== null || inactiveElapsed !== null || gameElapsed !== null)
     );
 
     // Shared-element transition to /servers/{name} — Chromium only, others
@@ -97,7 +97,9 @@
             <StatusPip {health} />
             <strong class="name">{name}</strong>
         </div>
-        {#if health === 'offline'}
+        {#if health === 'starting'}
+            <Badge variant="info" uppercase>Starting</Badge>
+        {:else if health === 'offline'}
             <Badge variant="danger" uppercase>offline</Badge>
         {:else}
             <div class="players" aria-live="polite">
@@ -119,7 +121,7 @@
         {/if}
     </div>
 
-    <RoundTelemetry {name} />
+    {#if showRoster}<RoundTelemetry {name} />{/if}
 
     {#if showTiming}
         <div class="timing">

src/lib/ui/components/StatusDot.svelte

@@ -26,7 +26,7 @@
         background: var(--spark-500);
         animation: electric-pulse 1.6s ease-in-out infinite;
     }
-    .dot-stale {
+    .dot-stale, .dot-starting {
         background: var(--amber-500);
         box-shadow: var(--amber-glow);
     }

src/lib/ui/components/primitives/StatusPip.svelte

@@ -11,13 +11,14 @@
 
     let toneClass = $derived(
         health === 'online' ? 't-online' :
-        health === 'stale'  ? 't-warning' :
+        health === 'stale' || health === 'starting' ? 't-warning' :
                               't-danger'
     );
 
     let announceLabel = $derived(
         label ?? (
             health === 'online' ? 'online' :
+            health === 'starting' ? 'starting' :
             health === 'stale'  ? 'stale'  :
                                   'offline'
         )

src/lib/ui/stores/servers.ts

@@ -12,7 +12,7 @@ export interface RosterPlayer {
     name: string;
 }
 
-export type Health = 'online' | 'stale' | 'offline';
+export type Health = 'online' | 'stale' | 'offline' | 'starting';
 export type RconFailure = 'no-config' | 'no-port' | 'send-error' | 'timeout' | 'bad-reply' | 'no-players';
 
 export interface RconEndpoint {

src/routes/api/servers/+server.ts

@@ -19,16 +19,25 @@ function safeJson<T>(raw: string | null): T | null {
 
 export async function GET() {
     const rconHost = config.rcon.host;
-    const servers = listServers().map((s) => ({
-        ...s,
-        roster: safeJson<RconPlayer[]>(s.last_roster),
-        health: classifyHealth(s.last_seen_at, s.last_error),
-        replySnippet: s.last_reply_snippet,
-        // Derived endpoint — same logic as endpointFor() but without asking
-        // Komodo. If we don't have a port yet, endpoint is null.
-        endpoint: s.game_port ? { host: s.rcon_host ?? rconHost, port: s.game_port } : null,
-        held: [...heldByBeta(s.name).entries()].map(([target, betaId]) => ({ target, betaId }))
-    }));
+    const servers = listServers().map((s) => {
+        const ready = !s.container_state || s.container_state === 'running';
+        return {
+            ...s,
+            last_players: ready ? s.last_players : null,
+            last_error: ready ? s.last_error : null,
+            active_since: ready ? s.active_since : null,
+            inactive_since: ready ? s.inactive_since : null,
+            game_started_at: ready ? s.game_started_at : null,
+            game_ended_at: ready ? s.game_ended_at : null,
+            roster: ready ? safeJson<RconPlayer[]>(s.last_roster) : null,
+            health: classifyHealth(s.last_seen_at, s.last_error, s.container_state),
+            replySnippet: ready ? s.last_reply_snippet : null,
+            // Derived endpoint — same logic as endpointFor() but without asking
+            // Komodo. If we don't have a port yet, endpoint is null.
+            endpoint: s.game_port ? { host: s.rcon_host ?? rconHost, port: s.game_port } : null,
+            held: [...heldByBeta(s.name).entries()].map(([target, betaId]) => ({ target, betaId }))
+        };
+    });
     return json({
         servers,
         reservedNames: (db().prepare("SELECT j.server FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')").all() as { server: string }[]).map(row => row.server),

src/routes/servers/[name]/+page.svelte

@@ -126,7 +126,7 @@
     let effectiveRoster = $derived(live?.roster ?? server?.roster ?? null);
     let effectiveHealth = $derived<Health>(live?.health ?? server?.health ?? 'offline');
     let effectiveSeen = $derived(live?.lastSeenAt ?? server?.last_seen_at ?? null);
-    let effectiveErr = $derived(live?.lastError ?? server?.last_error ?? null);
+    let effectiveErr = $derived(live ? live.lastError : server?.last_error ?? null);
     let effectiveEndpoint = $derived<RconEndpoint | null>(live?.endpoint ?? server?.endpoint ?? null);
     let liveReason = $derived<RconFailure | null>(live?.reason ?? null);
     let decoded = $derived(decodeRconError(effectiveErr));
@@ -139,12 +139,12 @@
     let effectiveInactiveSince = $derived<string | null>(live?.inactiveSince ?? server?.inactive_since ?? null);
     let effectiveGameStartedAt = $derived<string | null>(live?.gameStartedAt ?? server?.game_started_at ?? null);
     let effectiveGameEndedAt = $derived<string | null>(live?.gameEndedAt ?? server?.game_ended_at ?? null);
-    let activeElapsed = $derived(effectiveActiveSince ? elapsedSince(effectiveActiveSince, $uiNow) : null);
-    let inactiveElapsed = $derived(effectiveInactiveSince ? elapsedSince(effectiveInactiveSince, $uiNow) : null);
+    let activeElapsed = $derived(effectiveHealth !== 'starting' && effectiveActiveSince ? elapsedSince(effectiveActiveSince, $uiNow) : null);
+    let inactiveElapsed = $derived(effectiveHealth !== 'starting' && effectiveInactiveSince ? elapsedSince(effectiveInactiveSince, $uiNow) : null);
     let gameInProgress = $derived(!!effectiveGameStartedAt && !effectiveGameEndedAt && !!effectiveActiveSince);
-    let gameElapsed = $derived(gameInProgress ? elapsedSince(effectiveGameStartedAt, $uiNow) : null);
+    let gameElapsed = $derived(effectiveHealth !== 'starting' && gameInProgress ? elapsedSince(effectiveGameStartedAt, $uiNow) : null);
 
-    let hasDiagnosticIssue = $derived(effectiveHealth !== 'online');
+    let hasDiagnosticIssue = $derived(effectiveHealth !== 'online' && effectiveHealth !== 'starting');
 
     // Auto-open the Diagnostics tab if there's a problem the operator should see.
     $effect(() => {
@@ -237,13 +237,14 @@
                     <StatusPip health={effectiveHealth} size="md" />
                     <span class="health-label">
                         {effectiveHealth === 'online' ? 'Online' :
+                         effectiveHealth === 'starting' ? 'Starting' :
                          effectiveHealth === 'stale'  ? 'Stale'  : 'Offline'}
                     </span>
                 </div>
                 <div class="metric-cell">
                     <Stat
                         label="Players"
-                        value={effectivePlayers ?? 0}
+                        value={effectiveHealth === 'starting' ? '—' : effectivePlayers ?? 0}
                         size="xl"
                         tone={(effectivePlayers ?? 0) > 0 ? 'online' : 'primary'}
                         tween
@@ -276,7 +277,7 @@
                 {/if}
             </div>
 
-            <RoundTelemetry {name} />
+            {#if effectiveHealth !== 'starting'}<RoundTelemetry {name} />{/if}
 
         </Card>
 
@@ -327,7 +328,9 @@
                 <Card padding="lg">
                     <div class="panel-head">
                         <h2>RCON diagnostic</h2>
-                        {#if hasDiagnosticIssue}
+                        {#if effectiveHealth === 'starting'}
+                            <Badge variant="info" uppercase>Waiting for container</Badge>
+                        {:else if hasDiagnosticIssue}
                             <Badge variant="warning" uppercase>Issue</Badge>
                         {:else}
                             <Badge variant="online" uppercase>Healthy</Badge>