208 lines
12 KiB
TypeScript
208 lines
12 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
import { mkdir, writeFile } from 'node:fs/promises';
|
|
import { join } from 'node:path';
|
|
import { gameEvent, postTelemetry } from '../helpers/telemetry.js';
|
|
|
|
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({
|
|
komodoInspectHealthy: vi.fn(), komodoRestart: vi.fn()
|
|
}));
|
|
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
|
rconStatusResult: vi.fn(), rconGetPlayerCount: vi.fn()
|
|
}));
|
|
|
|
let fixture: Fixture;
|
|
beforeEach(async () => {
|
|
vi.resetModules();
|
|
vi.clearAllMocks();
|
|
fixture = await newFixture(['unregistered-folder']);
|
|
});
|
|
afterEach(async () => { await fixture.cleanup(); });
|
|
|
|
async function setup() {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db().prepare("INSERT INTO servers (name, komodo_name, last_error) VALUES ('managed', 'managed', 'timeout:old process')").run();
|
|
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
|
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
|
const { fanoutStatus } = await import('../../src/lib/server/pipeline/worker.js');
|
|
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
|
|
return { db: db(), inspect: vi.mocked(komodoInspectHealthy), rcon: vi.mocked(rconStatusResult), fanoutStatus, getServer };
|
|
}
|
|
|
|
describe('database fleet and container readiness', () => {
|
|
it.each(['queued', 'running', 'awaiting-health'])('shows provisioning during %s creation and resumes health reporting afterward', async (state) => {
|
|
const { db, inspect, rcon, fanoutStatus } = await setup();
|
|
const { GET } = await import('../../src/routes/api/servers/+server.js');
|
|
const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
|
|
db.prepare(`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
|
|
VALUES ('creation', 'managed', 'create', ?, 'Creating', ?, 'test')`).run(state, new Date().toISOString());
|
|
const snapshot = async () => (await (await GET()).json()).servers[0];
|
|
expect(await snapshot()).toMatchObject({ health: 'provisioning', last_error: null, last_players: null });
|
|
const events: unknown[] = [];
|
|
const unsubscribe = subscribe(event => events.push(event));
|
|
try { await fanoutStatus(); } finally { unsubscribe(); }
|
|
expect(inspect).not.toHaveBeenCalled();
|
|
expect(rcon).not.toHaveBeenCalled();
|
|
expect(events).toContainEqual(expect.objectContaining({ type: 'server-status', entries: [expect.objectContaining({
|
|
health: 'provisioning', lastError: null, reason: null, replySnippet: null
|
|
})] }));
|
|
db.prepare("UPDATE server_deployments SET state = 'failed' WHERE id = 'creation'").run();
|
|
inspect.mockResolvedValue({ ok: true, state: 'running' });
|
|
rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'no reply' });
|
|
await fanoutStatus();
|
|
expect(await snapshot()).toMatchObject({ health: 'offline', last_error: 'timeout:no reply' });
|
|
db.prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'creation'").run();
|
|
rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status',
|
|
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } });
|
|
await fanoutStatus();
|
|
expect(await snapshot()).toMatchObject({ health: 'online', last_error: null });
|
|
});
|
|
|
|
it('keeps game_end through occupied polls until game_start or an empty lobby', async () => {
|
|
const { inspect, rcon, fanoutStatus, getServer } = await setup();
|
|
inspect.mockResolvedValue({ ok: true, state: 'running' });
|
|
const reply = (players: number) => ({ ok: true as const, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status',
|
|
status: { players, map: 'zombie_theater', gametype: 'zom', roster: [] } });
|
|
rcon.mockResolvedValue(reply(1));
|
|
await postTelemetry(gameEvent('game_start', 0, 1, 'managed'));
|
|
await postTelemetry(gameEvent('game_end', 10, 1, 'managed'));
|
|
const ended = getServer('managed')?.game_ended_at;
|
|
expect(ended).toEqual(expect.any(String));
|
|
await postTelemetry(gameEvent('heartbeat', 10, 1, 'managed'));
|
|
await fanoutStatus();
|
|
expect(getServer('managed')?.game_ended_at).toBe(ended);
|
|
await postTelemetry(gameEvent('game_start', 0, 1, 'managed'));
|
|
expect(getServer('managed')?.game_ended_at).toBeNull();
|
|
await postTelemetry(gameEvent('game_end', 10, 1, 'managed'));
|
|
rcon.mockResolvedValue(reply(0));
|
|
await fanoutStatus();
|
|
expect(getServer('managed')?.game_ended_at).not.toBeNull();
|
|
await postTelemetry(gameEvent('heartbeat', 0, 0, 'managed'));
|
|
await fanoutStatus();
|
|
expect(getServer('managed')).toMatchObject({ game_started_at: null, game_ended_at: null });
|
|
rcon.mockResolvedValue(reply(1));
|
|
await fanoutStatus();
|
|
expect(getServer('managed')?.game_ended_at).toBeNull();
|
|
});
|
|
|
|
it('waits for game_start telemetry after log initialization and player joins', async () => {
|
|
const { inspect, rcon, fanoutStatus, getServer } = await setup();
|
|
const { GET } = await import('../../src/routes/api/servers/+server.js');
|
|
const logs = join(fixture.t5, 'managed', 'logs');
|
|
await mkdir(logs, { recursive: true });
|
|
await writeFile(join(logs, 'games_sp.log'), '0:00 InitGame:\n0:01 J;123;0;Player\n');
|
|
inspect.mockResolvedValue({ ok: true, state: 'running' });
|
|
rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status',
|
|
status: { players: 1, map: 'zombie_theater', gametype: 'zom', roster: [] } });
|
|
await fanoutStatus();
|
|
expect(getServer('managed')).toMatchObject({ active_since: expect.any(String), game_started_at: null });
|
|
|
|
for (const event of ['round_change', 'heartbeat'] as const) {
|
|
expect((await postTelemetry(gameEvent(event, 1, 1, 'managed'))).status).toBe(200);
|
|
await fanoutStatus();
|
|
expect((await (await GET()).json()).servers[0].game_started_at).toBeNull();
|
|
}
|
|
|
|
const response = await postTelemetry(gameEvent('game_start', 0, 1, 'managed'));
|
|
expect(response.status).toBe(200);
|
|
const { telemetry } = await response.json();
|
|
await fanoutStatus();
|
|
expect((await (await GET()).json()).servers[0]).toMatchObject({
|
|
game_started_at: telemetry.receivedAt, game_ended_at: null
|
|
});
|
|
});
|
|
|
|
it('includes the server clock with the health snapshot', async () => {
|
|
await setup();
|
|
const { GET } = await import('../../src/routes/api/servers/+server.js');
|
|
const requestedAt = Date.now();
|
|
const snapshot = await (await GET()).json();
|
|
expect(snapshot.serverNow).toBeGreaterThanOrEqual(requestedAt);
|
|
expect(snapshot.serverNow).toBeLessThanOrEqual(Date.now());
|
|
});
|
|
|
|
it('uses telemetry consistently in REST, status events, and metrics without polling RCON', async () => {
|
|
const { db, inspect, rcon, fanoutStatus } = await setup();
|
|
const { GET } = await import('../../src/routes/api/servers/+server.js');
|
|
const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
|
|
db.prepare("UPDATE servers SET last_players = 4, last_seen_at = ? WHERE name = 'managed'").run(new Date().toISOString());
|
|
inspect.mockResolvedValue({ ok: true, state: 'running' });
|
|
await postTelemetry(gameEvent('heartbeat', 15, 3, 'managed'));
|
|
const events: unknown[] = [];
|
|
const unsubscribe = subscribe(event => events.push(event));
|
|
try { await fanoutStatus(); } finally { unsubscribe(); }
|
|
expect(rcon).not.toHaveBeenCalled();
|
|
const server = (await (await GET()).json()).servers[0];
|
|
expect(server).toMatchObject({ health: 'online', last_players: 3, status_source: 'telemetry', last_error: null,
|
|
rcon_status: { last_players: 4, last_error: 'timeout:old process' } });
|
|
expect(server.roster).toHaveLength(3);
|
|
expect(server.roster[0]).toMatchObject({ ping: null, score: 5000 });
|
|
expect(events).toContainEqual(expect.objectContaining({ type: 'server-status', entries: [expect.objectContaining({
|
|
health: 'online', players: 3, roster: server.roster, lastSeenAt: server.last_seen_at
|
|
})] }));
|
|
expect(db.prepare("SELECT players FROM metric_samples WHERE server = 'managed'").get()).toEqual({ players: 3 });
|
|
});
|
|
|
|
it('falls back to RCON when telemetry expires and does not renew old reports', async () => {
|
|
const { db, inspect, rcon, fanoutStatus } = await setup();
|
|
const { GET } = await import('../../src/routes/api/servers/+server.js');
|
|
inspect.mockResolvedValue({ ok: true, state: 'running' });
|
|
await postTelemetry(gameEvent('heartbeat', 15, 3, 'managed'));
|
|
db.prepare('UPDATE server_telemetry SET received_at = ?').run(new Date(Date.now() - 46_000).toISOString());
|
|
rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status',
|
|
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } });
|
|
await fanoutStatus();
|
|
expect(rcon).toHaveBeenCalledTimes(1);
|
|
expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'online', last_players: 0, status_source: 'rcon' });
|
|
});
|
|
|
|
it('lists registered servers without folders and ignores unregistered folders and tombstones', async () => {
|
|
const { db } = await setup();
|
|
const { allServers, listServers } = await import('../../src/lib/server/fleet/servers.js');
|
|
expect(await allServers()).toEqual(['managed']);
|
|
db.prepare("UPDATE servers SET destroyed_at = '2026-01-01' WHERE name = 'managed'").run();
|
|
expect(listServers()).toEqual([]);
|
|
});
|
|
|
|
it('skips startup RCON, clears old errors, and resumes polling only after Komodo is ready', async () => {
|
|
const { inspect, rcon, fanoutStatus, getServer } = await setup();
|
|
const { GET } = await import('../../src/routes/api/servers/+server.js');
|
|
inspect.mockResolvedValue({ ok: false, state: 'starting', reason: 'container state: starting' });
|
|
await fanoutStatus();
|
|
await fanoutStatus();
|
|
expect(rcon).not.toHaveBeenCalled();
|
|
expect(getServer('managed')).toMatchObject({ container_state: 'starting', last_error: null });
|
|
expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'starting', last_error: null, last_players: null });
|
|
|
|
inspect.mockResolvedValue({ ok: true, state: 'running' });
|
|
rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'no reply' });
|
|
await fanoutStatus();
|
|
expect(rcon).toHaveBeenCalledTimes(1);
|
|
expect(getServer('managed')).toMatchObject({ container_state: 'running', last_error: 'timeout:no reply' });
|
|
|
|
rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status',
|
|
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } });
|
|
await fanoutStatus();
|
|
expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'online', last_error: null });
|
|
});
|
|
|
|
it.each(['unhealthy', 'exited', undefined])('does not poll RCON with container state %s', async (state) => {
|
|
const { inspect, rcon, fanoutStatus } = await setup();
|
|
inspect.mockResolvedValue({ ok: false, state, reason: 'not ready' });
|
|
await postTelemetry(gameEvent('heartbeat', 15, 3, 'managed'));
|
|
await fanoutStatus();
|
|
expect(rcon).not.toHaveBeenCalled();
|
|
const { GET } = await import('../../src/routes/api/servers/+server.js');
|
|
expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'offline', last_players: null });
|
|
});
|
|
|
|
it('waits for an EC restart request to settle before trusting the old healthy container', async () => {
|
|
const { db, inspect, rcon, fanoutStatus, getServer } = await setup();
|
|
db.prepare("INSERT INTO restart_rollout (server, requested_at) VALUES ('managed', ?)").run(new Date().toISOString());
|
|
await fanoutStatus();
|
|
expect(inspect).not.toHaveBeenCalled();
|
|
expect(rcon).not.toHaveBeenCalled();
|
|
expect(getServer('managed')?.container_state).toBe('restarting');
|
|
});
|
|
});
|