import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { newFixture, type Fixture } from '../helpers/fixture.js'; import { mkdir, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; import { gameEvent, postTelemetry } from '../helpers/telemetry.js'; vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoInspectHealthy: vi.fn(), komodoRestart: vi.fn() })); vi.mock('../../src/lib/server/integrations/rcon.js', () => ({ rconStatusResult: vi.fn(), rconGetPlayerCount: vi.fn() })); let fixture: Fixture; beforeEach(async () => { vi.resetModules(); vi.clearAllMocks(); fixture = await newFixture(['unregistered-folder']); }); afterEach(async () => { vi.useRealTimers(); await fixture.cleanup(); }); async function setup() { const { db } = await import('../../src/lib/server/db/index.js'); db().prepare("INSERT INTO servers (name, komodo_name, last_error) VALUES ('managed', 'managed', 'timeout:old process')").run(); const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js'); const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js'); const { fanoutStatus } = await import('../../src/lib/server/pipeline/worker.js'); const { getServer } = await import('../../src/lib/server/fleet/servers.js'); return { db: db(), inspect: vi.mocked(komodoInspectHealthy), rcon: vi.mocked(rconStatusResult), fanoutStatus, getServer }; } describe('database fleet and container readiness', () => { it('refreshes a ten-server fleet while one check is slow, without duplicating or exceeding four checks', async () => { const { db, inspect, rcon, fanoutStatus, getServer } = await setup(); const { subscribe } = await import('../../src/lib/server/pipeline/bus.js'); vi.useFakeTimers({ toFake: ['Date'] }); const now = Date.parse('2026-10-02T12:00:00Z'); vi.setSystemTime(now); for (let i = 0; i < 9; i++) db.prepare('INSERT INTO servers (name, komodo_name) VALUES (?, ?)').run(`fast-${i}`, `fast-${i}`); let release!: (result: { ok: true; state: string }) => void; let active = 0; let peak = 0; inspect.mockImplementation(async name => { peak = Math.max(peak, ++active); try { await Promise.resolve(); return name === 'managed' ? await new Promise<{ ok: true; state: string }>(resolve => { release = resolve; }) : { ok: true as const, state: 'running' }; } finally { active--; } }); rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status', status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } }); const events: unknown[] = []; const unsubscribe = subscribe(event => events.push(event)); const first = fanoutStatus(); try { await vi.waitFor(() => expect(rcon).toHaveBeenCalledTimes(9)); expect(events).toContainEqual(expect.objectContaining({ type: 'server-status', entries: expect.arrayContaining([ expect.objectContaining({ server: 'fast-8', health: 'online', players: 2 }) ]) })); vi.setSystemTime(now + 30_000); const second = fanoutStatus(); await vi.waitFor(() => expect(Date.parse(getServer('fast-8')?.last_seen_at ?? '')).toBeGreaterThanOrEqual(now + 30_000)); expect(rcon).toHaveBeenCalledTimes(18); expect(inspect.mock.calls.filter(([name]) => name === 'managed')).toHaveLength(1); expect(peak).toBe(4); release({ ok: true, state: 'running' }); await Promise.all([first, second]); expect(rcon).toHaveBeenCalledTimes(19); } finally { release({ ok: true, state: 'running' }); await first; unsubscribe(); } }); it('expires early results before recording a fleet collection delayed by a slow check', async () => { const { db, inspect, rcon, fanoutStatus } = await setup(); vi.useFakeTimers({ toFake: ['Date'] }); const now = Date.parse('2026-10-02T12:00:00Z'); vi.setSystemTime(now); db.prepare("INSERT INTO servers (name, komodo_name) VALUES ('slow', 'slow')").run(); let release!: (result: { ok: true; state: string }) => void; inspect.mockImplementation(async name => name === 'slow' ? await new Promise<{ ok: true; state: string }>(resolve => { release = resolve; }) : { ok: true as const, state: 'running' }); rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status', status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } }); const pending = fanoutStatus(); try { await vi.waitFor(() => expect(rcon).toHaveBeenCalledTimes(1)); vi.setSystemTime(now + 46_000); } finally { release({ ok: true, state: 'running' }); await pending; } expect(db.prepare('SELECT server, players FROM metric_samples ORDER BY server').all()).toEqual([ { server: 'managed', players: null }, { server: 'slow', players: 2 } ]); }); it('keeps a manual probe diagnostic without persisting it across a restart', async () => { const { db, rcon, getServer } = await setup(); const { POST } = await import('../../src/routes/api/servers/[name]/probe-rcon/+server.js'); let release!: (result: Awaited>) => void; rcon.mockImplementation(() => new Promise(resolve => { release = resolve; })); const pending = POST({ params: { name: 'managed' } } as never); db.prepare("INSERT INTO restart_rollout (server, requested_at) VALUES ('managed', ?)").run(new Date().toISOString()); release({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'old status', status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } }); expect(await (await pending).json()).toMatchObject({ ok: true }); expect(getServer('managed')).toMatchObject({ last_seen_at: null, last_error: 'timeout:old process' }); }); it.each(['container', 'rcon'])('ignores an old registration whose %s check finishes after replacement', async phase => { const { db, inspect, rcon, fanoutStatus, getServer } = await setup(); let release!: () => void; const blocked = new Promise(resolve => { release = resolve; }); inspect.mockImplementation(async () => { if (phase === 'container') await blocked; return { ok: true, state: 'running' }; }); rcon.mockImplementation(async () => { if (phase === 'rcon') await blocked; return { ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'old status', status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } }; }); const pending = fanoutStatus(); try { if (phase === 'rcon') await vi.waitFor(() => expect(rcon).toHaveBeenCalled()); db.prepare("UPDATE servers SET created_at = 'replacement', container_state = NULL, last_error = NULL WHERE name = 'managed'").run(); } finally { release(); await pending; } expect(getServer('managed')).toMatchObject({ container_state: null, last_seen_at: null, last_error: null }); }); it.each(['queued', 'running', 'awaiting-health'])('shows provisioning during %s creation and resumes health reporting afterward', async (state) => { const { db, inspect, rcon, fanoutStatus } = await setup(); const { GET } = await import('../../src/routes/api/servers/+server.js'); const { subscribe } = await import('../../src/lib/server/pipeline/bus.js'); db.prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by) VALUES ('creation', 'managed', 'create', ?, 'Creating', ?, 'test')`).run(state, new Date().toISOString()); const snapshot = async () => (await (await GET()).json()).servers[0]; expect(await snapshot()).toMatchObject({ health: 'provisioning', last_error: null, last_players: null }); const events: unknown[] = []; const unsubscribe = subscribe(event => events.push(event)); try { await fanoutStatus(); } finally { unsubscribe(); } expect(inspect).not.toHaveBeenCalled(); expect(rcon).not.toHaveBeenCalled(); expect(events).toContainEqual(expect.objectContaining({ type: 'server-status', entries: [expect.objectContaining({ health: 'provisioning', lastError: null, reason: null, replySnippet: null })] })); db.prepare("UPDATE server_deployments SET state = 'failed' WHERE id = 'creation'").run(); inspect.mockResolvedValue({ ok: true, state: 'running' }); rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'no reply' }); await fanoutStatus(); expect(await snapshot()).toMatchObject({ health: 'offline', last_error: 'timeout:no reply' }); db.prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'creation'").run(); rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status', status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } }); await fanoutStatus(); expect(await snapshot()).toMatchObject({ health: 'online', last_error: null }); }); it('keeps game_end through occupied polls until game_start or an empty lobby', async () => { const { inspect, rcon, fanoutStatus, getServer } = await setup(); inspect.mockResolvedValue({ ok: true, state: 'running' }); const reply = (players: number) => ({ ok: true as const, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status', status: { players, map: 'zombie_theater', gametype: 'zom', roster: [] } }); rcon.mockResolvedValue(reply(1)); await postTelemetry(gameEvent('game_start', 0, 1, 'managed')); await postTelemetry(gameEvent('game_end', 10, 1, 'managed')); const ended = getServer('managed')?.game_ended_at; expect(ended).toEqual(expect.any(String)); await postTelemetry(gameEvent('heartbeat', 10, 1, 'managed')); await fanoutStatus(); expect(getServer('managed')?.game_ended_at).toBe(ended); await postTelemetry(gameEvent('game_start', 0, 1, 'managed')); expect(getServer('managed')?.game_ended_at).toBeNull(); await postTelemetry(gameEvent('game_end', 10, 1, 'managed')); rcon.mockResolvedValue(reply(0)); await fanoutStatus(); expect(getServer('managed')?.game_ended_at).not.toBeNull(); await postTelemetry(gameEvent('heartbeat', 0, 0, 'managed')); await fanoutStatus(); expect(getServer('managed')).toMatchObject({ game_started_at: null, game_ended_at: null }); rcon.mockResolvedValue(reply(1)); await fanoutStatus(); expect(getServer('managed')?.game_ended_at).toBeNull(); }); it('keeps the first occupied report as the active timer when more telemetry arrives before polling', async () => { const { inspect, rcon, fanoutStatus, getServer } = await setup(); const { GET } = await import('../../src/routes/api/servers/+server.js'); const { subscribe } = await import('../../src/lib/server/pipeline/bus.js'); const startedAt = Date.parse('2026-09-27T12:00:00Z'); const at = (seconds: number) => new Date(startedAt + seconds * 1000).toISOString(); vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(startedAt); inspect.mockResolvedValue({ ok: true, state: 'running' }); expect((await postTelemetry(gameEvent('game_start', 0, 1, 'managed'))).status).toBe(200); expect(getServer('managed')).toMatchObject({ active_since: at(0), inactive_since: null, game_started_at: at(0) }); vi.setSystemTime(startedAt + 19_000); const heartbeat = await (await postTelemetry(gameEvent('heartbeat', 2, 1, 'managed'))).json(); expect(heartbeat.telemetry.timing.server).toEqual({ activeSince: at(0), inactiveSince: null, gameStartedAt: at(0), gameEndedAt: null }); vi.setSystemTime(startedAt + 30_000); const events: unknown[] = []; const unsubscribe = subscribe(event => events.push(event)); try { await fanoutStatus(); } finally { unsubscribe(); } expect(rcon).not.toHaveBeenCalled(); expect((await (await GET()).json()).servers[0]).toMatchObject({ active_since: at(0), game_started_at: at(0), last_seen_at: at(19) }); expect(events).toContainEqual(expect.objectContaining({ type: 'server-status', entries: [expect.objectContaining({ activeSince: at(0), gameStartedAt: at(0), lastSeenAt: at(19) })] })); const { closeDb } = await import('../../src/lib/server/db/index.js'); closeDb(); expect((await (await GET()).json()).servers[0]).toMatchObject({ active_since: at(0), game_started_at: at(0) }); }); it('preserves occupancy observed by RCON before game-start telemetry', async () => { const { inspect, rcon, fanoutStatus, getServer } = await setup(); const startedAt = Date.parse('2026-09-27T12:00:00Z'); const at = (seconds: number) => new Date(startedAt + seconds * 1000).toISOString(); vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(startedAt - 20_000); inspect.mockResolvedValue({ ok: true, state: 'running' }); rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status', status: { players: 1, map: 'zombie_theater', gametype: 'zom', roster: [] } }); await fanoutStatus(); expect(getServer('managed')).toMatchObject({ active_since: at(-20), game_started_at: null }); vi.setSystemTime(startedAt); const start = await (await postTelemetry(gameEvent('game_start', 0, 1, 'managed'))).json(); expect(start.telemetry.timing.server).toEqual({ activeSince: at(-20), inactiveSince: null, gameStartedAt: at(0), gameEndedAt: null }); expect(getServer('managed')).toMatchObject({ active_since: at(-20), inactive_since: null, game_started_at: at(0) }); }); it('records an empty lobby and rejoin between status polls without restoring the old match', async () => { const { inspect, fanoutStatus, getServer } = await setup(); const { GET } = await import('../../src/routes/api/servers/+server.js'); const startedAt = Date.parse('2026-09-27T12:00:00Z'); const at = (seconds: number) => new Date(startedAt + seconds * 1000).toISOString(); vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(startedAt); inspect.mockResolvedValue({ ok: true, state: 'running' }); await postTelemetry(gameEvent('game_start', 0, 1, 'managed')); vi.setSystemTime(startedAt + 10_000); await postTelemetry(gameEvent('game_end', 2, 1, 'managed')); vi.setSystemTime(startedAt + 11_000); const empty = await (await postTelemetry(gameEvent('heartbeat', 0, 0, 'managed'))).json(); expect(empty.telemetry.timing.server).toEqual({ activeSince: null, inactiveSince: at(11), gameStartedAt: null, gameEndedAt: null }); expect(getServer('managed')).toMatchObject({ active_since: null, inactive_since: at(11), game_started_at: null, game_ended_at: null }); vi.setSystemTime(startedAt + 15_000); const rejoin = await (await postTelemetry(gameEvent('heartbeat', 1, 1, 'managed'))).json(); expect(rejoin.telemetry.timing.server).toEqual({ activeSince: at(15), inactiveSince: null, gameStartedAt: null, gameEndedAt: null }); expect(getServer('managed')).toMatchObject({ active_since: at(15), inactive_since: null, game_started_at: null, game_ended_at: null }); vi.setSystemTime(startedAt + 30_000); await fanoutStatus(); expect((await (await GET()).json()).servers[0]).toMatchObject({ active_since: at(15), inactive_since: null, game_started_at: null, game_ended_at: null }); }); it('waits for game_start telemetry after log initialization and player joins', async () => { const { inspect, rcon, fanoutStatus, getServer } = await setup(); const { GET } = await import('../../src/routes/api/servers/+server.js'); const logs = join(fixture.t5, 'managed', 'logs'); await mkdir(logs, { recursive: true }); await writeFile(join(logs, 'games_sp.log'), '0:00 InitGame:\n0:01 J;123;0;Player\n'); inspect.mockResolvedValue({ ok: true, state: 'running' }); rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status', status: { players: 1, map: 'zombie_theater', gametype: 'zom', roster: [] } }); await fanoutStatus(); expect(getServer('managed')).toMatchObject({ active_since: expect.any(String), game_started_at: null }); for (const event of ['round_change', 'heartbeat'] as const) { expect((await postTelemetry(gameEvent(event, 1, 1, 'managed'))).status).toBe(200); await fanoutStatus(); expect((await (await GET()).json()).servers[0].game_started_at).toBeNull(); } const response = await postTelemetry(gameEvent('game_start', 0, 1, 'managed')); expect(response.status).toBe(200); const { telemetry } = await response.json(); await fanoutStatus(); expect((await (await GET()).json()).servers[0]).toMatchObject({ game_started_at: telemetry.receivedAt, game_ended_at: null }); }); it('includes the server clock with the health snapshot', async () => { await setup(); const { GET } = await import('../../src/routes/api/servers/+server.js'); const requestedAt = Date.now(); const snapshot = await (await GET()).json(); expect(snapshot.serverNow).toBeGreaterThanOrEqual(requestedAt); expect(snapshot.serverNow).toBeLessThanOrEqual(Date.now()); }); it('uses telemetry consistently in REST, status events, and metrics without polling RCON', async () => { const { db, inspect, rcon, fanoutStatus } = await setup(); const { GET } = await import('../../src/routes/api/servers/+server.js'); const { subscribe } = await import('../../src/lib/server/pipeline/bus.js'); db.prepare("UPDATE servers SET last_players = 4, last_seen_at = ? WHERE name = 'managed'").run(new Date().toISOString()); inspect.mockResolvedValue({ ok: true, state: 'running' }); await postTelemetry(gameEvent('heartbeat', 15, 3, 'managed')); const events: unknown[] = []; const unsubscribe = subscribe(event => events.push(event)); try { await fanoutStatus(); } finally { unsubscribe(); } expect(rcon).not.toHaveBeenCalled(); const server = (await (await GET()).json()).servers[0]; expect(server).toMatchObject({ health: 'online', last_players: 3, status_source: 'telemetry', last_error: null, rcon_status: { last_players: 4, last_error: 'timeout:old process' } }); expect(server.roster).toHaveLength(3); expect(server.roster[0]).toMatchObject({ ping: null, score: 5000 }); expect(events).toContainEqual(expect.objectContaining({ type: 'server-status', entries: [expect.objectContaining({ health: 'online', players: 3, roster: server.roster, lastSeenAt: server.last_seen_at })] })); expect(db.prepare("SELECT players FROM metric_samples WHERE server = 'managed'").get()).toEqual({ players: 3 }); }); it('falls back to RCON when telemetry expires and does not renew old reports', async () => { const { db, inspect, rcon, fanoutStatus } = await setup(); const { GET } = await import('../../src/routes/api/servers/+server.js'); inspect.mockResolvedValue({ ok: true, state: 'running' }); await postTelemetry(gameEvent('heartbeat', 15, 3, 'managed')); db.prepare('UPDATE server_telemetry SET received_at = ?').run(new Date(Date.now() - 46_000).toISOString()); rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status', status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } }); await fanoutStatus(); expect(rcon).toHaveBeenCalledTimes(1); expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'online', last_players: 0, status_source: 'rcon' }); }); it('lists registered servers without folders and ignores unregistered folders and tombstones', async () => { const { db } = await setup(); const { allServers, listServers } = await import('../../src/lib/server/fleet/servers.js'); expect(await allServers()).toEqual(['managed']); db.prepare("UPDATE servers SET destroyed_at = '2026-01-01' WHERE name = 'managed'").run(); expect(listServers()).toEqual([]); }); it('skips startup RCON, clears old errors, and resumes polling only after Komodo is ready', async () => { const { inspect, rcon, fanoutStatus, getServer } = await setup(); const { GET } = await import('../../src/routes/api/servers/+server.js'); inspect.mockResolvedValue({ ok: false, state: 'starting', reason: 'container state: starting' }); await fanoutStatus(); await fanoutStatus(); expect(rcon).not.toHaveBeenCalled(); expect(getServer('managed')).toMatchObject({ container_state: 'starting', last_error: null }); expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'starting', last_error: null, last_players: null }); inspect.mockResolvedValue({ ok: true, state: 'running' }); rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'no reply' }); await fanoutStatus(); expect(rcon).toHaveBeenCalledTimes(1); expect(getServer('managed')).toMatchObject({ container_state: 'running', last_error: 'timeout:no reply' }); rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status', status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } }); await fanoutStatus(); expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'online', last_error: null }); }); it.each(['unhealthy', 'exited', 'stopped', 'missing'])('does not poll RCON with container state %s', async (state) => { const { inspect, rcon, fanoutStatus } = await setup(); inspect.mockResolvedValue({ ok: false, state, reason: 'not ready' }); await postTelemetry(gameEvent('heartbeat', 15, 3, 'managed')); await fanoutStatus(); expect(rcon).not.toHaveBeenCalled(); const { GET } = await import('../../src/routes/api/servers/+server.js'); expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'offline', last_players: null }); }); it.each(['unknown', undefined, 'throw'])('automatically recovers through RCON when inspection returns %s', async state => { const { inspect, rcon, fanoutStatus, getServer } = await setup(); const { GET } = await import('../../src/routes/api/servers/+server.js'); const { subscribe } = await import('../../src/lib/server/pipeline/bus.js'); if (state === 'throw') inspect.mockRejectedValue(new Error('Inspection timed out')); else inspect.mockResolvedValue({ ok: false, state, reason: 'Inspection timed out' }); await postTelemetry(gameEvent('heartbeat', 15, 3, 'managed')); rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status', status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } }); const events: unknown[] = []; const unsubscribe = subscribe(event => events.push(event)); try { await fanoutStatus(); } finally { unsubscribe(); } expect(rcon).toHaveBeenCalledOnce(); expect(getServer('managed')?.container_state).toBe('unknown'); expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'online', last_players: 2, status_source: 'rcon' }); expect(events).toContainEqual(expect.objectContaining({ type: 'server-status', entries: [expect.objectContaining({ health: 'online', players: 2, containerState: 'unknown' })] })); rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'no reply' }); await fanoutStatus(); expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'stale', last_error: 'timeout:no reply' }); }); it('waits for an EC restart request to settle before trusting the old healthy container', async () => { const { db, inspect, rcon, fanoutStatus, getServer } = await setup(); db.prepare("INSERT INTO restart_rollout (server, requested_at) VALUES ('managed', ?)").run(new Date().toISOString()); await fanoutStatus(); expect(inspect).not.toHaveBeenCalled(); expect(rcon).not.toHaveBeenCalled(); expect(getServer('managed')?.container_state).toBe('restarting'); }); });