From 885abbdffe99a128aafb714fe02eee86cc816369 Mon Sep 17 00:00:00 2001 From: josh Date: Thu, 17 Sep 2026 17:28:38 -0400 Subject: [PATCH] Use database fleet registry and gate RCON polling on container readiness --- README.md | 11 +++ src/lib/server/db/index.ts | 1 + src/lib/server/db/schema.sql | 1 + src/lib/server/fleet/create.ts | 4 +- src/lib/server/fleet/drift.ts | 4 +- src/lib/server/fleet/health.ts | 6 +- src/lib/server/fleet/servers.ts | 14 +++- src/lib/server/integrations/komodo.ts | 16 +++- src/lib/server/pipeline/restart-rollout.ts | 8 ++ src/lib/server/pipeline/worker.ts | 38 +++++++++- src/lib/ui/components/PlayerRoster.svelte | 4 +- src/lib/ui/components/ServerCard.svelte | 10 ++- src/lib/ui/components/StatusDot.svelte | 2 +- .../ui/components/primitives/StatusPip.svelte | 3 +- src/lib/ui/stores/servers.ts | 2 +- src/routes/api/servers/+server.ts | 29 ++++--- src/routes/servers/[name]/+page.svelte | 19 +++-- tests/unit/komodo.test.ts | 2 + tests/unit/server-status.test.ts | 75 +++++++++++++++++++ 19 files changed, 208 insertions(+), 41 deletions(-) create mode 100644 tests/unit/server-status.test.ts diff --git a/README.md b/README.md index 0c4f269..c9ef9ba 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,17 @@ Web app that manages a fleet of T5 Plutonium servers running under Komodo. +Fleet membership lives in SQLite. EC registers servers during creation and removes +them during destruction; startup does not discover or import server folders. +Existing database registrations are preserved, and missing mounts do not remove +servers from the fleet. The legacy `seedServerRegistry` helper is an explicit +import utility only. + +Status refreshes check Komodo container readiness before polling RCON. Starting +or restarting containers show **Starting**, without old RCON errors or player +data. Stopped, unhealthy, or unverifiable containers are not polled. Containers +without a Docker health check use their running state as the readiness signal. + ## Stack - SvelteKit (adapter-node) + TypeScript diff --git a/src/lib/server/db/index.ts b/src/lib/server/db/index.ts index a262d68..fb48a5a 100644 --- a/src/lib/server/db/index.ts +++ b/src/lib/server/db/index.ts @@ -12,6 +12,7 @@ let _db: Database.Database | null = null; // IF NOT EXISTS in schema.sql is a no-op on existing tables, so it can't // backfill columns; this can. const SERVERS_COLUMNS: Array<{ name: string; ddl: string }> = [ + { name: 'container_state', ddl: 'ALTER TABLE servers ADD COLUMN container_state TEXT' }, // Base columns that shipped in the very first schema stay here for parity // with fresh installs — no ALTER is issued if they're already present. { name: 'komodo_server', ddl: 'ALTER TABLE servers ADD COLUMN komodo_server TEXT' }, diff --git a/src/lib/server/db/schema.sql b/src/lib/server/db/schema.sql index 5bc302e..cca9016 100644 --- a/src/lib/server/db/schema.sql +++ b/src/lib/server/db/schema.sql @@ -11,6 +11,7 @@ CREATE TABLE IF NOT EXISTS servers ( komodo_type TEXT NOT NULL DEFAULT 'deployment', -- deployment|stack|container komodo_name TEXT NOT NULL, komodo_server TEXT, + container_state TEXT, -- last observed Komodo runtime state game_port INTEGER, -- UDP RCON port; lazy-filled from Komodo rcon_host TEXT, -- optional per-server override of RCON_HOST game TEXT, -- 't5' etc.; static per-server (no map rotation) diff --git a/src/lib/server/fleet/create.ts b/src/lib/server/fleet/create.ts index c6b9cb6..801c051 100644 --- a/src/lib/server/fleet/create.ts +++ b/src/lib/server/fleet/create.ts @@ -271,11 +271,11 @@ async function createServerUnlocked(input: CreateServerInput, prepared: Prepared `INSERT INTO servers (name, komodo_type, komodo_name, game, map, game_mode, map_slug, host_dir, config_file, game_id, image, - game_port, created_by) + game_port, created_by, container_state) VALUES (?, 'deployment', ?, 't5', ?, ?, ?, ?, ?, ?, ?, - ?, ?)` + ?, ?, 'starting')` ).run( name, name, mapEntry.canonical, input.gameMode, mapEntry.slug, hostDir, diff --git a/src/lib/server/fleet/drift.ts b/src/lib/server/fleet/drift.ts index d8accef..95cc230 100644 --- a/src/lib/server/fleet/drift.ts +++ b/src/lib/server/fleet/drift.ts @@ -14,7 +14,7 @@ import { import { getBaselineForScope, getVersion } from './scripts.js'; import { getPromotedServerConfig, getServerMapConfig, MAP_CONFIG_FILE } from './server-config.js'; import { T5_ZOMBIE_MAPS } from './map-aliases.js'; -import { allServers, getServer, listServers } from './servers.js'; +import { serverDirectories, getServer, listServers } from './servers.js'; import { promotedArtifacts } from './promoted-artifacts.js'; // Destinations drift scans, in a stable order so the UI shows scripts first. @@ -228,7 +228,7 @@ function composeExpectedCfg( export async function scanFleetDrift(): Promise { const servers = listServers().map((s) => s.name); const registered = new Set(servers); - const allDirs = new Set(await allServers()); + const allDirs = new Set(await serverDirectories()); // Only touch server dirs that are both registered AND present on disk. const eligible = servers.filter((n) => registered.has(n) && allDirs.has(n)); const out: DriftEntry[] = []; diff --git a/src/lib/server/fleet/health.ts b/src/lib/server/fleet/health.ts index 0e53cef..96726bf 100644 --- a/src/lib/server/fleet/health.ts +++ b/src/lib/server/fleet/health.ts @@ -1,4 +1,4 @@ -export type Health = 'online' | 'stale' | 'offline'; +export type Health = 'online' | 'stale' | 'offline' | 'starting'; const ONLINE_MAX_MS = 30_000; const STALE_MAX_MS = 120_000; @@ -9,7 +9,9 @@ const STALE_MAX_MS = 120_000; * successful reply is >120s old. `stale` if the last reply is 30-120s old * (RCON glitchy). `online` if fresh. */ -export function classifyHealth(lastSeenAt: string | null, lastError: string | null): Health { +export function classifyHealth(lastSeenAt: string | null, lastError: string | null, containerState?: string | null): Health { + if (containerState === 'starting' || containerState === 'restarting') return 'starting'; + if (containerState && containerState !== 'running') return 'offline'; if (!lastSeenAt) return 'offline'; const age = Date.now() - new Date(lastSeenAt).getTime(); if (!Number.isFinite(age) || age < 0) return 'offline'; diff --git a/src/lib/server/fleet/servers.ts b/src/lib/server/fleet/servers.ts index de0d146..405eb92 100644 --- a/src/lib/server/fleet/servers.ts +++ b/src/lib/server/fleet/servers.ts @@ -8,7 +8,7 @@ import { parseServerName } from './map-aliases.js'; * All server folders under SERVERS_ROOT. Used by seedServerRegistry to backfill * the servers table for any on-disk server directory that isn't yet tracked. */ -export async function allServers(): Promise { +export async function serverDirectories(): Promise { let entries; try { entries = await readdir(config.serversRoot, { withFileTypes: true }); @@ -21,6 +21,11 @@ export async function allServers(): Promise { .sort(); } +/** Fleet membership is owned by the database, independent of mounted files. */ +export async function allServers(): Promise { + return listServers().map((server) => server.name); +} + /** * Match a server name against a list of glob-like patterns (supports `*` and * `?` and comma-separated lists). @@ -64,6 +69,7 @@ export interface ServerRow { komodo_type: 'deployment' | 'stack' | 'container'; komodo_name: string; komodo_server: string | null; + container_state: string | null; game_port: number | null; rcon_host: string | null; game: string | null; @@ -93,7 +99,8 @@ export interface ServerRow { } export async function seedServerRegistry(): Promise { - const rows = await allServers(); + // Explicit legacy import only; never run as part of application startup. + const rows = await serverDirectories(); // Under the new naming scheme Komodo entities match EC server names 1:1 // (both underscore-separated), so komodo_name = name — no transformation. const insert = db().prepare( @@ -143,7 +150,7 @@ export function getServer(name: string): ServerRow | undefined { } export function listServers(): ServerRow[] { - return db().prepare('SELECT * FROM servers ORDER BY name').all() as ServerRow[]; + return db().prepare('SELECT * FROM servers WHERE destroyed_at IS NULL ORDER BY name').all() as ServerRow[]; } export function setServerField( @@ -155,6 +162,7 @@ export function setServerField( 'komodo_type', 'komodo_name', 'komodo_server', + 'container_state', 'game_port', 'rcon_host', 'game', diff --git a/src/lib/server/integrations/komodo.ts b/src/lib/server/integrations/komodo.ts index d888545..35f46a9 100644 --- a/src/lib/server/integrations/komodo.ts +++ b/src/lib/server/integrations/komodo.ts @@ -255,7 +255,7 @@ export async function komodoResolveGamePort(serverName: string): Promise; + // Komodo summaries expose health alongside a flat running state. + const flatState = obj['state'] ?? obj['State'] ?? obj['status'] ?? obj['Status']; + const flatHealth = obj['health'] ?? obj['Health']; + const healthStatus = typeof flatHealth === 'string' ? flatHealth + : flatHealth && typeof flatHealth === 'object' + ? ((flatHealth as Record)['status'] ?? (flatHealth as Record)['Status']) : null; + if (typeof flatState === 'string' && flatState.toLowerCase() === 'running' + && typeof healthStatus === 'string' && healthStatus.toLowerCase() !== 'healthy') { + found = healthStatus; + return; + } + // Docker inspect nested: { State: { Status, Running, Health: { Status } } } const stateObj = obj['State']; if (stateObj && typeof stateObj === 'object') { @@ -401,7 +413,7 @@ export async function komodoInspectHealthy(serverName: string): Promise { +let statusInFlight: Promise | null = null; + +export function fanoutStatus(): Promise { + if (statusInFlight) return statusInFlight; + statusInFlight = refreshStatus().finally(() => { statusInFlight = null; }); + return statusInFlight; +} + +async function refreshStatus(): Promise { const servers = listServers(); if (servers.length === 0) return; @@ -120,6 +130,28 @@ async function fanoutStatus(): Promise { const i = cursor++; if (i >= servers.length) return; const srv = servers[i]; + const container = restartSettling(srv.name) + ? { ok: false as const, state: 'restarting', reason: 'Restart requested' } + : await komodoInspectHealthy(srv.name).catch((err) => ({ + ok: false as const, reason: (err as Error).message + })); + const containerState = container.ok ? 'running' + : ('state' in container && typeof container.state === 'string' ? container.state.toLowerCase() : 'unknown'); + setServerField(srv.name, 'container_state', containerState); + if (!container.ok) { + // Container readiness precedes RCON readiness. Old replies and + // failures describe the previous process, not this startup. + setServerField(srv.name, 'last_error', null); + setServerField(srv.name, 'last_reply_snippet', null); + entries[i] = { + server: srv.name, players: null, map: srv.map, gametype: null, + roster: null, health: classifyHealth(null, null, containerState), + lastSeenAt: srv.last_seen_at, lastError: null, endpoint: null, + reason: null, replySnippet: null, activeSince: null, + inactiveSince: null, gameStartedAt: null, gameEndedAt: null + }; + continue; + } const result = await rconStatusResult(srv.name).catch((err) => ({ ok: false as const, endpoint: null, @@ -231,8 +263,6 @@ export function startWorker(): void { if (started) return; started = true; - seedServerRegistry().catch((err) => console.error('[worker] seed error:', err)); - void tick(); void fanoutStatus(); tickTimer = setInterval(tick, TICK_MS); diff --git a/src/lib/ui/components/PlayerRoster.svelte b/src/lib/ui/components/PlayerRoster.svelte index 08b0bcd..4da0a32 100644 --- a/src/lib/ui/components/PlayerRoster.svelte +++ b/src/lib/ui/components/PlayerRoster.svelte @@ -23,7 +23,9 @@ } -{#if health === 'offline'} +{#if health === 'starting'} +
Server starting
+{:else if health === 'offline'}
server unreachable
{:else if !roster || roster.length === 0}
lobby is empty
diff --git a/src/lib/ui/components/ServerCard.svelte b/src/lib/ui/components/ServerCard.svelte index ceff928..29f59fa 100644 --- a/src/lib/ui/components/ServerCard.svelte +++ b/src/lib/ui/components/ServerCard.svelte @@ -63,7 +63,7 @@ .sort((a, b) => b.score - a.score) .slice(0, 3) ); - let showRoster = $derived(health !== 'offline'); + let showRoster = $derived(health === 'online' || health === 'stale'); let mapLabel = $derived(map ?? '—'); let gtLabel = $derived(gametypeLabel(gametype)); @@ -72,7 +72,7 @@ let gameInProgress = $derived(!!gameStartedAt && !gameEndedAt && !!activeSince); let gameElapsed = $derived(gameInProgress ? elapsedSince(gameStartedAt, $uiNow) : null); let showTiming = $derived( - health !== 'offline' && (activeElapsed !== null || inactiveElapsed !== null || gameElapsed !== null) + showRoster && (activeElapsed !== null || inactiveElapsed !== null || gameElapsed !== null) ); // Shared-element transition to /servers/{name} — Chromium only, others @@ -97,7 +97,9 @@ {name} - {#if health === 'offline'} + {#if health === 'starting'} + Starting + {:else if health === 'offline'} offline {:else}
@@ -119,7 +121,7 @@ {/if}
- + {#if showRoster}{/if} {#if showTiming}
diff --git a/src/lib/ui/components/StatusDot.svelte b/src/lib/ui/components/StatusDot.svelte index 6a4d357..2025338 100644 --- a/src/lib/ui/components/StatusDot.svelte +++ b/src/lib/ui/components/StatusDot.svelte @@ -26,7 +26,7 @@ background: var(--spark-500); animation: electric-pulse 1.6s ease-in-out infinite; } - .dot-stale { + .dot-stale, .dot-starting { background: var(--amber-500); box-shadow: var(--amber-glow); } diff --git a/src/lib/ui/components/primitives/StatusPip.svelte b/src/lib/ui/components/primitives/StatusPip.svelte index c0e484d..2dc65cf 100644 --- a/src/lib/ui/components/primitives/StatusPip.svelte +++ b/src/lib/ui/components/primitives/StatusPip.svelte @@ -11,13 +11,14 @@ let toneClass = $derived( health === 'online' ? 't-online' : - health === 'stale' ? 't-warning' : + health === 'stale' || health === 'starting' ? 't-warning' : 't-danger' ); let announceLabel = $derived( label ?? ( health === 'online' ? 'online' : + health === 'starting' ? 'starting' : health === 'stale' ? 'stale' : 'offline' ) diff --git a/src/lib/ui/stores/servers.ts b/src/lib/ui/stores/servers.ts index aa6682e..69d3dca 100644 --- a/src/lib/ui/stores/servers.ts +++ b/src/lib/ui/stores/servers.ts @@ -12,7 +12,7 @@ export interface RosterPlayer { name: string; } -export type Health = 'online' | 'stale' | 'offline'; +export type Health = 'online' | 'stale' | 'offline' | 'starting'; export type RconFailure = 'no-config' | 'no-port' | 'send-error' | 'timeout' | 'bad-reply' | 'no-players'; export interface RconEndpoint { diff --git a/src/routes/api/servers/+server.ts b/src/routes/api/servers/+server.ts index 050db4a..4be66cb 100644 --- a/src/routes/api/servers/+server.ts +++ b/src/routes/api/servers/+server.ts @@ -19,16 +19,25 @@ function safeJson(raw: string | null): T | null { export async function GET() { const rconHost = config.rcon.host; - const servers = listServers().map((s) => ({ - ...s, - roster: safeJson(s.last_roster), - health: classifyHealth(s.last_seen_at, s.last_error), - replySnippet: s.last_reply_snippet, - // Derived endpoint — same logic as endpointFor() but without asking - // Komodo. If we don't have a port yet, endpoint is null. - endpoint: s.game_port ? { host: s.rcon_host ?? rconHost, port: s.game_port } : null, - held: [...heldByBeta(s.name).entries()].map(([target, betaId]) => ({ target, betaId })) - })); + const servers = listServers().map((s) => { + const ready = !s.container_state || s.container_state === 'running'; + return { + ...s, + last_players: ready ? s.last_players : null, + last_error: ready ? s.last_error : null, + active_since: ready ? s.active_since : null, + inactive_since: ready ? s.inactive_since : null, + game_started_at: ready ? s.game_started_at : null, + game_ended_at: ready ? s.game_ended_at : null, + roster: ready ? safeJson(s.last_roster) : null, + health: classifyHealth(s.last_seen_at, s.last_error, s.container_state), + replySnippet: ready ? s.last_reply_snippet : null, + // Derived endpoint — same logic as endpointFor() but without asking + // Komodo. If we don't have a port yet, endpoint is null. + endpoint: s.game_port ? { host: s.rcon_host ?? rconHost, port: s.game_port } : null, + held: [...heldByBeta(s.name).entries()].map(([target, betaId]) => ({ target, betaId })) + }; + }); return json({ servers, reservedNames: (db().prepare("SELECT j.server FROM server_creation_jobs j JOIN server_deployments d ON d.id = j.deployment_id WHERE d.state IN ('queued', 'running', 'awaiting-health')").all() as { server: string }[]).map(row => row.server), diff --git a/src/routes/servers/[name]/+page.svelte b/src/routes/servers/[name]/+page.svelte index caf9aff..b6522e8 100644 --- a/src/routes/servers/[name]/+page.svelte +++ b/src/routes/servers/[name]/+page.svelte @@ -126,7 +126,7 @@ let effectiveRoster = $derived(live?.roster ?? server?.roster ?? null); let effectiveHealth = $derived(live?.health ?? server?.health ?? 'offline'); let effectiveSeen = $derived(live?.lastSeenAt ?? server?.last_seen_at ?? null); - let effectiveErr = $derived(live?.lastError ?? server?.last_error ?? null); + let effectiveErr = $derived(live ? live.lastError : server?.last_error ?? null); let effectiveEndpoint = $derived(live?.endpoint ?? server?.endpoint ?? null); let liveReason = $derived(live?.reason ?? null); let decoded = $derived(decodeRconError(effectiveErr)); @@ -139,12 +139,12 @@ let effectiveInactiveSince = $derived(live?.inactiveSince ?? server?.inactive_since ?? null); let effectiveGameStartedAt = $derived(live?.gameStartedAt ?? server?.game_started_at ?? null); let effectiveGameEndedAt = $derived(live?.gameEndedAt ?? server?.game_ended_at ?? null); - let activeElapsed = $derived(effectiveActiveSince ? elapsedSince(effectiveActiveSince, $uiNow) : null); - let inactiveElapsed = $derived(effectiveInactiveSince ? elapsedSince(effectiveInactiveSince, $uiNow) : null); + let activeElapsed = $derived(effectiveHealth !== 'starting' && effectiveActiveSince ? elapsedSince(effectiveActiveSince, $uiNow) : null); + let inactiveElapsed = $derived(effectiveHealth !== 'starting' && effectiveInactiveSince ? elapsedSince(effectiveInactiveSince, $uiNow) : null); let gameInProgress = $derived(!!effectiveGameStartedAt && !effectiveGameEndedAt && !!effectiveActiveSince); - let gameElapsed = $derived(gameInProgress ? elapsedSince(effectiveGameStartedAt, $uiNow) : null); + let gameElapsed = $derived(effectiveHealth !== 'starting' && gameInProgress ? elapsedSince(effectiveGameStartedAt, $uiNow) : null); - let hasDiagnosticIssue = $derived(effectiveHealth !== 'online'); + let hasDiagnosticIssue = $derived(effectiveHealth !== 'online' && effectiveHealth !== 'starting'); // Auto-open the Diagnostics tab if there's a problem the operator should see. $effect(() => { @@ -237,13 +237,14 @@ {effectiveHealth === 'online' ? 'Online' : + effectiveHealth === 'starting' ? 'Starting' : effectiveHealth === 'stale' ? 'Stale' : 'Offline'}
0 ? 'online' : 'primary'} tween @@ -276,7 +277,7 @@ {/if}
- + {#if effectiveHealth !== 'starting'}{/if} @@ -327,7 +328,9 @@

RCON diagnostic

- {#if hasDiagnosticIssue} + {#if effectiveHealth === 'starting'} + Waiting for container + {:else if hasDiagnosticIssue} Issue {:else} Healthy diff --git a/tests/unit/komodo.test.ts b/tests/unit/komodo.test.ts index 7422003..531cc06 100644 --- a/tests/unit/komodo.test.ts +++ b/tests/unit/komodo.test.ts @@ -4,6 +4,8 @@ import { extractUdpPort, extractContainerState } from '../../src/lib/server/inte describe('container readiness', () => { it.each(['starting', 'unhealthy'])('does not accept a running container whose health is %s', (health) => { expect(extractContainerState({ State: { Status: 'running', Running: true, Health: { Status: health } } })).toBe(health); + expect(extractContainerState({ state: 'running', health })).toBe(health); + expect(extractContainerState({ state: 'running', health: { status: health } })).toBe(health); }); it('accepts healthy running containers and preserves stopped states', () => { expect(extractContainerState({ State: { Status: 'running', Running: true, Health: { Status: 'healthy' } } })).toBe('running'); diff --git a/tests/unit/server-status.test.ts b/tests/unit/server-status.test.ts new file mode 100644 index 0000000..f920132 --- /dev/null +++ b/tests/unit/server-status.test.ts @@ -0,0 +1,75 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { newFixture, type Fixture } from '../helpers/fixture.js'; + +vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ + komodoInspectHealthy: vi.fn(), komodoRestart: vi.fn() +})); +vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ + rconStatusResult: vi.fn(), rconGetPlayerCount: vi.fn() +})); + +let fixture: Fixture; +beforeEach(async () => { + vi.resetModules(); + vi.clearAllMocks(); + fixture = await newFixture(['unregistered-folder']); +}); +afterEach(async () => { await fixture.cleanup(); }); + +async function setup() { + const { db } = await import('../../src/lib/server/db/index.js'); + db().prepare("INSERT INTO servers (name, komodo_name, last_error) VALUES ('managed', 'managed', 'timeout:old process')").run(); + const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); + const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); + const { fanoutStatus } = await import('../../src/lib/server/pipeline/worker.js'); + const { getServer } = await import('../../src/lib/server/fleet/servers.js'); + return { db: db(), inspect: vi.mocked(komodoInspectHealthy), rcon: vi.mocked(rconStatusResult), fanoutStatus, getServer }; +} + +describe('database fleet and container readiness', () => { + it('lists registered servers without folders and ignores unregistered folders and tombstones', async () => { + const { db } = await setup(); + const { allServers, listServers } = await import('../../src/lib/server/fleet/servers.js'); + expect(await allServers()).toEqual(['managed']); + db.prepare("UPDATE servers SET destroyed_at = '2026-01-01' WHERE name = 'managed'").run(); + expect(listServers()).toEqual([]); + }); + + it('skips startup RCON, clears old errors, and resumes polling only after Komodo is ready', async () => { + const { inspect, rcon, fanoutStatus, getServer } = await setup(); + const { GET } = await import('../../src/routes/api/servers/+server.js'); + inspect.mockResolvedValue({ ok: false, state: 'starting', reason: 'container state: starting' }); + await fanoutStatus(); + await fanoutStatus(); + expect(rcon).not.toHaveBeenCalled(); + expect(getServer('managed')).toMatchObject({ container_state: 'starting', last_error: null }); + expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'starting', last_error: null, last_players: null }); + + inspect.mockResolvedValue({ ok: true, state: 'running' }); + rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'no reply' }); + await fanoutStatus(); + expect(rcon).toHaveBeenCalledTimes(1); + expect(getServer('managed')).toMatchObject({ container_state: 'running', last_error: 'timeout:no reply' }); + + rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status', + status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } }); + await fanoutStatus(); + expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'online', last_error: null }); + }); + + it.each(['unhealthy', 'exited', undefined])('does not poll RCON with container state %s', async (state) => { + const { inspect, rcon, fanoutStatus } = await setup(); + inspect.mockResolvedValue({ ok: false, state, reason: 'not ready' }); + await fanoutStatus(); + expect(rcon).not.toHaveBeenCalled(); + }); + + it('waits for an EC restart request to settle before trusting the old healthy container', async () => { + const { db, inspect, rcon, fanoutStatus, getServer } = await setup(); + db.prepare("INSERT INTO restart_rollout (server, requested_at) VALUES ('managed', ?)").run(new Date().toISOString()); + await fanoutStatus(); + expect(inspect).not.toHaveBeenCalled(); + expect(rcon).not.toHaveBeenCalled(); + expect(getServer('managed')?.container_state).toBe('restarting'); + }); +});