1173 lines
50 KiB
TypeScript
1173 lines
50 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
import { mkdir, writeFile } from 'node:fs/promises';
|
|
import { join } from 'node:path';
|
|
import { gameEvent, postTelemetry } from '../helpers/telemetry.js';
|
|
|
|
vi.mock('../../src/lib/server/integrations/rcon.js', () => ({
|
|
rconStatusResult: vi.fn(),
|
|
rconGetPlayerCount: vi.fn()
|
|
}));
|
|
|
|
let fixture: Fixture;
|
|
beforeEach(async () => {
|
|
vi.resetModules();
|
|
vi.clearAllMocks();
|
|
fixture = await newFixture(['unregistered-folder']);
|
|
});
|
|
afterEach(async () => {
|
|
vi.useRealTimers();
|
|
await fixture.cleanup();
|
|
});
|
|
|
|
async function setup() {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
db()
|
|
.prepare("INSERT INTO servers (name, last_error) VALUES ('managed', 'timeout:old process')")
|
|
.run();
|
|
vi.spyOn(await import('../../src/lib/server/backends/native.js'), 'nativeHealth');
|
|
const { nativeHealth } = await import('../../src/lib/server/backends/native.js');
|
|
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
|
|
const { fanoutStatus } = await import('../../src/lib/server/pipeline/worker.js');
|
|
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
|
|
return {
|
|
db: db(),
|
|
inspect: vi.mocked(nativeHealth),
|
|
rcon: vi.mocked(rconStatusResult),
|
|
fanoutStatus,
|
|
getServer
|
|
};
|
|
}
|
|
|
|
describe('database fleet and container readiness', () => {
|
|
it('retains only sanitized failed replies and clears them after a successful monitoring check', async () => {
|
|
const { inspect, rcon, fanoutStatus, getServer } = await setup();
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
const at = Date.parse('2026-10-08T12:00:00Z');
|
|
vi.setSystemTime(at);
|
|
inspect.mockResolvedValue({ ok: true, state: 'running' });
|
|
rcon.mockResolvedValue({
|
|
ok: false,
|
|
endpoint: null,
|
|
reason: 'bad-reply',
|
|
detail: 'unrecognized status',
|
|
rawReply: 'player 192.0.2.123:28960 invalid reply'
|
|
});
|
|
await fanoutStatus();
|
|
expect(getServer('managed')?.last_reply_snippet).toBe('player [redacted] invalid reply');
|
|
vi.setSystemTime(at + 30_000);
|
|
rcon.mockResolvedValue({
|
|
ok: true,
|
|
endpoint: { host: 'agent', port: 1 },
|
|
status: { players: 0, map: null, gametype: null, roster: [] }
|
|
});
|
|
await fanoutStatus();
|
|
expect(getServer('managed')?.last_reply_snippet).toBeNull();
|
|
});
|
|
it('publishes capacity from accepted monitoring samples and clears it on the next failed check', async () => {
|
|
const { inspect, rcon, fanoutStatus } = await setup();
|
|
const { GET } = await import('../../src/routes/api/servers/+server.js');
|
|
const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
const at = Date.parse('2026-10-07T12:00:00Z');
|
|
vi.setSystemTime(at);
|
|
inspect.mockResolvedValue({ ok: true, state: 'running' });
|
|
rcon.mockResolvedValue({
|
|
ok: true,
|
|
endpoint: { host: 'localhost', port: 28960 },
|
|
status: { players: 3, capacity: 4, map: 'zombie_theater', gametype: 'zom', roster: [] }
|
|
});
|
|
const events: unknown[] = [];
|
|
const stop = subscribe((event) => {
|
|
if (event.type === 'server-status') events.push(event);
|
|
});
|
|
try {
|
|
await fanoutStatus();
|
|
expect(events.at(-1)).toMatchObject({
|
|
entries: [{ capacity: 4, capacityObservedAt: new Date(at).toISOString() }]
|
|
});
|
|
expect((await GET().json()).servers[0]).toMatchObject({
|
|
capacity: 4,
|
|
capacity_observed_at: new Date(at).toISOString()
|
|
});
|
|
vi.setSystemTime(at + 30_000);
|
|
rcon.mockResolvedValue({
|
|
ok: false,
|
|
endpoint: null,
|
|
reason: 'timeout',
|
|
detail: 'No reply'
|
|
});
|
|
await fanoutStatus();
|
|
expect(events.at(-1)).toMatchObject({
|
|
entries: [{ capacity: null, capacityObservedAt: null }]
|
|
});
|
|
expect((await GET().json()).servers[0]).toMatchObject({
|
|
capacity: null,
|
|
capacity_observed_at: null
|
|
});
|
|
} finally {
|
|
stop();
|
|
}
|
|
});
|
|
|
|
it('requires two distinct failed monitoring checks, preserves diagnostics, and resets on success', async () => {
|
|
const { inspect, rcon, fanoutStatus, getServer } = await setup();
|
|
const { GET } = await import('../../src/routes/api/servers/+server.js');
|
|
const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
|
|
const snapshot = async () => (await (await GET()).json()).servers[0];
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
const at = Date.parse('2026-10-05T12:00:00Z');
|
|
vi.setSystemTime(at);
|
|
inspect.mockResolvedValue({ ok: true, state: 'running' });
|
|
const success = {
|
|
ok: true as const,
|
|
endpoint: { host: 'localhost', port: 28960 },
|
|
status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] }
|
|
};
|
|
rcon.mockResolvedValue(success);
|
|
await fanoutStatus();
|
|
const events: unknown[] = [];
|
|
const unsubscribe = subscribe((event) => events.push(event));
|
|
try {
|
|
rcon.mockResolvedValue({
|
|
ok: false,
|
|
endpoint: null,
|
|
reason: 'timeout',
|
|
detail: 'no reply'
|
|
});
|
|
vi.setSystemTime(at + 30_000);
|
|
await fanoutStatus();
|
|
expect(getServer('managed')).toMatchObject({
|
|
rcon_failures: 1,
|
|
last_error: 'timeout:no reply'
|
|
});
|
|
expect(await snapshot()).toMatchObject({
|
|
health: 'online',
|
|
last_error: null,
|
|
rcon_status: { last_error: 'timeout:no reply', rcon_failures: 1 }
|
|
});
|
|
expect(events).toContainEqual(
|
|
expect.objectContaining({
|
|
type: 'server-status',
|
|
entries: [
|
|
expect.objectContaining({
|
|
health: 'online',
|
|
lastError: null,
|
|
rconStatus: expect.objectContaining({
|
|
rcon_failures: 1,
|
|
last_error: 'timeout:no reply'
|
|
})
|
|
})
|
|
]
|
|
})
|
|
);
|
|
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
|
closeDb();
|
|
expect(await snapshot()).toMatchObject({
|
|
health: 'online',
|
|
rcon_status: { rcon_failures: 1 }
|
|
});
|
|
// Reprocessing a sample with the same timestamp cannot count it twice.
|
|
await fanoutStatus();
|
|
expect(getServer('managed')?.rcon_failures).toBe(1);
|
|
vi.setSystemTime(at + 60_000);
|
|
await fanoutStatus();
|
|
expect(await snapshot()).toMatchObject({
|
|
health: 'stale',
|
|
last_error: 'timeout:no reply',
|
|
rcon_status: { rcon_failures: 2 }
|
|
});
|
|
rcon.mockResolvedValue(success);
|
|
vi.setSystemTime(at + 90_000);
|
|
await fanoutStatus();
|
|
expect(getServer('managed')).toMatchObject({ rcon_failures: 0, last_error: null });
|
|
rcon.mockResolvedValue({
|
|
ok: false,
|
|
endpoint: null,
|
|
reason: 'send-error',
|
|
detail: 'unreachable'
|
|
});
|
|
vi.setSystemTime(at + 120_000);
|
|
await fanoutStatus();
|
|
expect(await snapshot()).toMatchObject({
|
|
health: 'online',
|
|
rcon_status: { rcon_failures: 1 }
|
|
});
|
|
vi.setSystemTime(at + 180_001);
|
|
expect(await snapshot()).toMatchObject({ health: 'stale' });
|
|
vi.setSystemTime(at + 210_001);
|
|
expect(await snapshot()).toMatchObject({ health: 'offline' });
|
|
} finally {
|
|
unsubscribe();
|
|
}
|
|
});
|
|
|
|
it('adds the failure counter to an existing database without suppressing legacy errors', async () => {
|
|
const { db, getServer } = await setup();
|
|
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
|
db.exec('ALTER TABLE servers DROP COLUMN rcon_failures');
|
|
db.pragma('user_version = 0');
|
|
closeDb();
|
|
expect(getServer('managed')).toMatchObject({
|
|
rcon_failures: 0,
|
|
last_error: 'timeout:old process'
|
|
});
|
|
});
|
|
|
|
it('does not carry a failure streak across a new lifecycle submission', async () => {
|
|
const { db, inspect, rcon, fanoutStatus, getServer } = await setup();
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
const at = Date.parse('2026-10-05T12:00:00Z');
|
|
vi.setSystemTime(at);
|
|
db.prepare('UPDATE servers SET rcon_failures=1,last_checked_at=?').run(
|
|
new Date(at - 30_000).toISOString()
|
|
);
|
|
db.prepare(
|
|
`INSERT INTO server_deployments(id,server,action,state,stage,started_at,submitted_at,triggered_by)
|
|
VALUES('new-start','managed','restart','done','done',?,?,'test')`
|
|
).run(new Date(at - 10_000).toISOString(), new Date(at - 10_000).toISOString());
|
|
inspect.mockResolvedValue({ ok: true, state: 'running' });
|
|
rcon.mockResolvedValue({
|
|
ok: false,
|
|
endpoint: null,
|
|
reason: 'timeout',
|
|
detail: 'new process'
|
|
});
|
|
await fanoutStatus();
|
|
expect(getServer('managed')).toMatchObject({
|
|
rcon_failures: 1,
|
|
last_error: 'timeout:new process'
|
|
});
|
|
});
|
|
|
|
it('shares routine observations without renewing their timestamps and keeps manual probes fresh', async () => {
|
|
const { inspect, rcon, fanoutStatus, getServer } = await setup();
|
|
const { inspectForMonitoring, rconForMonitoring } =
|
|
await import('../../src/lib/server/fleet/health-observations.js');
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
const at = Date.parse('2026-10-05T12:00:00Z');
|
|
vi.setSystemTime(at);
|
|
inspect.mockResolvedValue({ ok: true, state: 'running' });
|
|
rcon.mockResolvedValue({
|
|
ok: true,
|
|
endpoint: { host: 'localhost', port: 28960 },
|
|
status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] }
|
|
});
|
|
await inspectForMonitoring('managed');
|
|
await rconForMonitoring('managed');
|
|
vi.setSystemTime(at + 10_000);
|
|
await fanoutStatus();
|
|
expect(inspect).toHaveBeenCalledTimes(1);
|
|
expect(rcon).toHaveBeenCalledTimes(1);
|
|
expect(getServer('managed')).toMatchObject({
|
|
container_checked_at: new Date(at).toISOString(),
|
|
last_seen_at: new Date(at).toISOString()
|
|
});
|
|
|
|
const { POST } = await import('../../src/routes/api/servers/[name]/probe-rcon/+server.js');
|
|
rcon.mockResolvedValue({
|
|
ok: false,
|
|
endpoint: null,
|
|
reason: 'timeout',
|
|
detail: 'manual failure'
|
|
});
|
|
await POST({ params: { name: 'managed' } } as never);
|
|
expect(rcon).toHaveBeenCalledTimes(2);
|
|
vi.setSystemTime(at + 20_000);
|
|
await fanoutStatus();
|
|
expect(rcon).toHaveBeenCalledTimes(3);
|
|
expect(getServer('managed')).toMatchObject({
|
|
last_error: 'timeout:manual failure',
|
|
last_seen_at: new Date(at).toISOString()
|
|
});
|
|
});
|
|
|
|
it('does not overwrite a newer manual observation with an earlier background reply', async () => {
|
|
const { db, inspect, rcon, fanoutStatus, getServer } = await setup();
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
const at = Date.parse('2026-10-05T12:00:00Z');
|
|
vi.setSystemTime(at);
|
|
inspect.mockResolvedValue({ ok: true, state: 'running' });
|
|
let release!: (result: Awaited<ReturnType<typeof rcon>>) => void;
|
|
rcon.mockImplementationOnce(
|
|
() =>
|
|
new Promise((resolve) => {
|
|
release = resolve;
|
|
})
|
|
);
|
|
const pending = fanoutStatus();
|
|
await vi.waitFor(() => expect(rcon).toHaveBeenCalledTimes(1));
|
|
vi.setSystemTime(at + 10_000);
|
|
db.prepare(
|
|
"UPDATE servers SET last_checked_at=?,last_error='timeout:manual failure' WHERE name='managed'"
|
|
).run(new Date().toISOString());
|
|
release({
|
|
ok: true,
|
|
endpoint: { host: 'localhost', port: 28960 },
|
|
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] }
|
|
});
|
|
await pending;
|
|
expect(getServer('managed')).toMatchObject({
|
|
last_error: 'timeout:manual failure',
|
|
last_seen_at: null,
|
|
last_checked_at: new Date(at + 10_000).toISOString()
|
|
});
|
|
});
|
|
|
|
it('refreshes a ten-server host while one check is slow, without duplicating or exceeding two checks', async () => {
|
|
const { db, inspect, rcon, fanoutStatus, getServer } = await setup();
|
|
const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
const now = Date.parse('2026-10-02T12:00:00Z');
|
|
vi.setSystemTime(now);
|
|
for (let i = 0; i < 9; i++)
|
|
db.prepare('INSERT INTO servers (name) VALUES (?)').run(`fast-${i}`);
|
|
const { prepareNativeHost } = await import('../helpers/native-host.js');
|
|
await prepareNativeHost(['managed', ...Array.from({ length: 9 }, (_, i) => `fast-${i}`)]);
|
|
let release!: (result: { ok: true; state: string }) => void;
|
|
let active = 0;
|
|
let peak = 0;
|
|
inspect.mockImplementation(async (name) => {
|
|
peak = Math.max(peak, ++active);
|
|
try {
|
|
await Promise.resolve();
|
|
return name === 'managed'
|
|
? await new Promise<{ ok: true; state: string }>((resolve) => {
|
|
release = resolve;
|
|
})
|
|
: { ok: true as const, state: 'running' };
|
|
} finally {
|
|
active--;
|
|
}
|
|
});
|
|
rcon.mockResolvedValue({
|
|
ok: true,
|
|
endpoint: { host: 'localhost', port: 28960 },
|
|
status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] }
|
|
});
|
|
const events: unknown[] = [];
|
|
const unsubscribe = subscribe((event) => events.push(event));
|
|
const first = fanoutStatus();
|
|
try {
|
|
await vi.waitFor(() => expect(rcon).toHaveBeenCalledTimes(9));
|
|
expect(events).toContainEqual(
|
|
expect.objectContaining({
|
|
type: 'server-status',
|
|
entries: expect.arrayContaining([
|
|
expect.objectContaining({ server: 'fast-8', health: 'online', players: 2 })
|
|
])
|
|
})
|
|
);
|
|
vi.setSystemTime(now + 30_000);
|
|
const second = fanoutStatus();
|
|
await vi.waitFor(() =>
|
|
expect(Date.parse(getServer('fast-8')?.last_seen_at ?? '')).toBeGreaterThanOrEqual(
|
|
now + 30_000
|
|
)
|
|
);
|
|
expect(rcon).toHaveBeenCalledTimes(18);
|
|
expect(inspect.mock.calls.filter(([name]) => name === 'managed')).toHaveLength(1);
|
|
expect(peak).toBe(2);
|
|
release({ ok: true, state: 'running' });
|
|
await Promise.all([first, second]);
|
|
expect(rcon).toHaveBeenCalledTimes(19);
|
|
} finally {
|
|
release({ ok: true, state: 'running' });
|
|
await first;
|
|
unsubscribe();
|
|
}
|
|
});
|
|
|
|
it('gives each host two slots through inspection and RCON, independently of a stalled host', async () => {
|
|
const { db, inspect, rcon, fanoutStatus, getServer } = await setup();
|
|
db.prepare("DELETE FROM servers WHERE name='managed'").run();
|
|
const hosts = await import('../../src/lib/server/hosts/index.js');
|
|
for (let host = 0; host < 3; host++) {
|
|
const registered = hosts.createHost({ name: `Host ${host}` });
|
|
hosts.enrollHost(hosts.createEnrollment(registered.id).token, { protocolVersion: 1 });
|
|
for (let instance = 0; instance < 4; instance++) {
|
|
const name = `host-${host}-${instance}`;
|
|
db.prepare('INSERT INTO servers (name) VALUES (?)').run(name);
|
|
db.prepare(
|
|
`INSERT INTO native_instances (id,server,host_id,library_id,image,port,created_at)
|
|
VALUES (?, ?, ?, 'library', 'image', ?, ?)`
|
|
).run(name, name, registered.id, 28960 + instance, new Date().toISOString());
|
|
}
|
|
}
|
|
let releaseSlow!: () => void;
|
|
let releaseFast!: () => void;
|
|
const slow = new Promise<void>((resolve) => {
|
|
releaseSlow = resolve;
|
|
});
|
|
const fast = new Promise<void>((resolve) => {
|
|
releaseFast = resolve;
|
|
});
|
|
const active = new Map<string, number>();
|
|
const peak = new Map<string, number>();
|
|
inspect.mockResolvedValue({ ok: true, state: 'running' });
|
|
rcon.mockImplementation(async (name) => {
|
|
const host = name.slice(0, -2);
|
|
active.set(host, (active.get(host) ?? 0) + 1);
|
|
peak.set(host, Math.max(peak.get(host) ?? 0, active.get(host)!));
|
|
try {
|
|
await (host === 'host-0' ? slow : fast);
|
|
if (name === 'host-0-0') throw new Error('simulated failed probe');
|
|
return {
|
|
ok: true,
|
|
endpoint: { host: 'localhost', port: 28960 },
|
|
status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] }
|
|
};
|
|
} finally {
|
|
active.set(host, active.get(host)! - 1);
|
|
}
|
|
});
|
|
const passes = [fanoutStatus()];
|
|
try {
|
|
await vi.waitFor(() => expect(rcon).toHaveBeenCalledTimes(6));
|
|
// A slot covers the whole check: waiting for RCON cannot admit a third inspection.
|
|
expect(inspect).toHaveBeenCalledTimes(6);
|
|
expect([...active.values()]).toEqual([2, 2, 2]);
|
|
passes.push(fanoutStatus());
|
|
releaseFast();
|
|
await vi.waitFor(() => {
|
|
expect(getServer('host-1-3')?.last_players).toBe(2);
|
|
expect(getServer('host-2-3')?.last_players).toBe(2);
|
|
});
|
|
expect(getServer('host-0-2')?.container_checked_at).toBeNull();
|
|
expect(rcon).toHaveBeenCalledTimes(10);
|
|
} finally {
|
|
releaseSlow();
|
|
releaseFast();
|
|
await Promise.all(passes);
|
|
}
|
|
expect([...peak.values()]).toEqual([2, 2, 2]);
|
|
expect(inspect).toHaveBeenCalledTimes(12);
|
|
expect(rcon).toHaveBeenCalledTimes(12);
|
|
expect(getServer('host-0-0')?.last_error).toBe('send-error:simulated failed probe');
|
|
expect(getServer('host-0-3')?.last_players).toBe(2);
|
|
});
|
|
|
|
it('expires early results before recording a fleet collection delayed by a slow check', async () => {
|
|
const { db, inspect, rcon, fanoutStatus } = await setup();
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
const now = Date.parse('2026-10-02T12:00:00Z');
|
|
vi.setSystemTime(now);
|
|
db.prepare("INSERT INTO servers (name) VALUES ('slow')").run();
|
|
let release!: (result: { ok: true; state: string }) => void;
|
|
inspect.mockImplementation(async (name) =>
|
|
name === 'slow'
|
|
? await new Promise<{ ok: true; state: string }>((resolve) => {
|
|
release = resolve;
|
|
})
|
|
: { ok: true as const, state: 'running' }
|
|
);
|
|
rcon.mockResolvedValue({
|
|
ok: true,
|
|
endpoint: { host: 'localhost', port: 28960 },
|
|
status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] }
|
|
});
|
|
const pending = fanoutStatus();
|
|
try {
|
|
await vi.waitFor(() => expect(rcon).toHaveBeenCalledTimes(1));
|
|
vi.setSystemTime(now + 91_000);
|
|
} finally {
|
|
release({ ok: true, state: 'running' });
|
|
await pending;
|
|
}
|
|
expect(
|
|
db.prepare('SELECT server, players FROM metric_samples ORDER BY server').all()
|
|
).toEqual([
|
|
{ server: 'managed', players: null },
|
|
{ server: 'slow', players: 2 }
|
|
]);
|
|
});
|
|
|
|
it('keeps a manual probe diagnostic without persisting it across a restart', async () => {
|
|
const { db, rcon, getServer } = await setup();
|
|
const { POST } = await import('../../src/routes/api/servers/[name]/probe-rcon/+server.js');
|
|
let release!: (result: Awaited<ReturnType<typeof rcon>>) => void;
|
|
rcon.mockImplementation(
|
|
() =>
|
|
new Promise((resolve) => {
|
|
release = resolve;
|
|
})
|
|
);
|
|
const pending = POST({ params: { name: 'managed' } } as never);
|
|
db.prepare("INSERT INTO restart_rollout (server, requested_at) VALUES ('managed', ?)").run(
|
|
new Date().toISOString()
|
|
);
|
|
release({
|
|
ok: true,
|
|
endpoint: { host: 'localhost', port: 28960 },
|
|
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] }
|
|
});
|
|
expect(await (await pending).json()).toMatchObject({ ok: true });
|
|
expect(getServer('managed')).toMatchObject({
|
|
last_seen_at: null,
|
|
last_error: 'timeout:old process'
|
|
});
|
|
});
|
|
|
|
it.each(['container', 'rcon'])(
|
|
'ignores an old registration whose %s check finishes after replacement',
|
|
async (phase) => {
|
|
const { db, inspect, rcon, fanoutStatus, getServer } = await setup();
|
|
let release!: () => void;
|
|
const blocked = new Promise<void>((resolve) => {
|
|
release = resolve;
|
|
});
|
|
inspect.mockImplementation(async () => {
|
|
if (phase === 'container') await blocked;
|
|
return { ok: true, state: 'running' };
|
|
});
|
|
rcon.mockImplementation(async () => {
|
|
if (phase === 'rcon') await blocked;
|
|
return {
|
|
ok: true,
|
|
endpoint: { host: 'localhost', port: 28960 },
|
|
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] }
|
|
};
|
|
});
|
|
const pending = fanoutStatus();
|
|
try {
|
|
if (phase === 'rcon') await vi.waitFor(() => expect(rcon).toHaveBeenCalled());
|
|
db.prepare(
|
|
"UPDATE servers SET created_at = 'replacement', container_state = NULL, last_error = NULL WHERE name = 'managed'"
|
|
).run();
|
|
} finally {
|
|
release();
|
|
await pending;
|
|
}
|
|
expect(getServer('managed')).toMatchObject({
|
|
container_state: null,
|
|
last_seen_at: null,
|
|
last_error: null
|
|
});
|
|
}
|
|
);
|
|
|
|
it.each(['queued', 'running', 'awaiting-health'])(
|
|
'shows provisioning during %s creation and resumes health reporting afterward',
|
|
async (state) => {
|
|
const { db, inspect, rcon, fanoutStatus } = await setup();
|
|
const { GET } = await import('../../src/routes/api/servers/+server.js');
|
|
const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
|
|
db.prepare(
|
|
`INSERT INTO server_deployments (id, server, action, state, stage, started_at, triggered_by)
|
|
VALUES ('creation', 'managed', 'create', ?, 'Creating', ?, 'test')`
|
|
).run(state, new Date().toISOString());
|
|
const snapshot = async () => (await (await GET()).json()).servers[0];
|
|
expect(await snapshot()).toMatchObject({
|
|
health: 'provisioning',
|
|
last_error: null,
|
|
last_players: null
|
|
});
|
|
const events: unknown[] = [];
|
|
const unsubscribe = subscribe((event) => events.push(event));
|
|
try {
|
|
await fanoutStatus();
|
|
} finally {
|
|
unsubscribe();
|
|
}
|
|
expect(inspect).not.toHaveBeenCalled();
|
|
expect(rcon).not.toHaveBeenCalled();
|
|
expect(events).toContainEqual(
|
|
expect.objectContaining({
|
|
type: 'server-status',
|
|
entries: [
|
|
expect.objectContaining({
|
|
health: 'provisioning',
|
|
lastError: null,
|
|
reason: null
|
|
})
|
|
]
|
|
})
|
|
);
|
|
db.prepare(
|
|
"UPDATE server_deployments SET state = 'failed' WHERE id = 'creation'"
|
|
).run();
|
|
inspect.mockResolvedValue({ ok: true, state: 'running' });
|
|
rcon.mockResolvedValue({
|
|
ok: false,
|
|
endpoint: null,
|
|
reason: 'timeout',
|
|
detail: 'no reply'
|
|
});
|
|
await fanoutStatus();
|
|
expect(await snapshot()).toMatchObject({
|
|
health: 'offline',
|
|
last_error: 'timeout:no reply'
|
|
});
|
|
db.prepare("UPDATE server_deployments SET state = 'done' WHERE id = 'creation'").run();
|
|
rcon.mockResolvedValue({
|
|
ok: true,
|
|
endpoint: { host: 'localhost', port: 28960 },
|
|
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] }
|
|
});
|
|
await fanoutStatus();
|
|
expect(await snapshot()).toMatchObject({ health: 'online', last_error: null });
|
|
}
|
|
);
|
|
|
|
it('keeps game_end through RCON polls and recovers when telemetry observes a new match', async () => {
|
|
const { inspect, rcon, fanoutStatus, getServer } = await setup();
|
|
inspect.mockResolvedValue({ ok: true, state: 'running' });
|
|
const reply = (players: number) => ({
|
|
ok: true as const,
|
|
endpoint: { host: 'localhost', port: 28960 },
|
|
status: { players, map: 'zombie_theater', gametype: 'zom', roster: [] }
|
|
});
|
|
rcon.mockResolvedValue(reply(1));
|
|
await postTelemetry(gameEvent('game_start', 0, 1, 'managed'));
|
|
await postTelemetry(gameEvent('game_end', 10, 1, 'managed'));
|
|
const ended = getServer('managed')?.game_ended_at;
|
|
expect(ended).toEqual(expect.any(String));
|
|
await fanoutStatus();
|
|
expect(getServer('managed')?.game_ended_at).toBe(ended);
|
|
await postTelemetry(gameEvent('heartbeat', 1, 1, 'managed'));
|
|
await fanoutStatus();
|
|
expect(getServer('managed')).toMatchObject({ game_started_at: null, game_ended_at: null });
|
|
await postTelemetry(gameEvent('game_start', 0, 1, 'managed'));
|
|
expect(getServer('managed')?.game_ended_at).toBeNull();
|
|
await postTelemetry(gameEvent('game_end', 10, 1, 'managed'));
|
|
rcon.mockResolvedValue(reply(0));
|
|
await fanoutStatus();
|
|
expect(getServer('managed')?.game_ended_at).not.toBeNull();
|
|
await postTelemetry(gameEvent('heartbeat', 0, 0, 'managed'));
|
|
await fanoutStatus();
|
|
expect(getServer('managed')).toMatchObject({ game_started_at: null, game_ended_at: null });
|
|
rcon.mockResolvedValue(reply(1));
|
|
await fanoutStatus();
|
|
expect(getServer('managed')?.game_ended_at).toBeNull();
|
|
});
|
|
|
|
it.each([0, 1])(
|
|
'preserves a match starting with %i players through empty reports, polls, a gap, and reload',
|
|
async (startPlayers) => {
|
|
const { inspect, rcon, fanoutStatus, getServer } = await setup();
|
|
const { GET } = await import('../../src/routes/api/servers/+server.js');
|
|
const { currentRound } = await import('../../src/lib/fleet-overview.js');
|
|
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
|
const beginning = Date.parse('2026-10-09T12:00:00Z');
|
|
const at = (seconds: number) => new Date(beginning + seconds * 1000).toISOString();
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
vi.setSystemTime(beginning);
|
|
inspect.mockResolvedValue({ ok: true, state: 'running' });
|
|
rcon.mockResolvedValue({
|
|
ok: true,
|
|
endpoint: { host: 'localhost', port: 28960 },
|
|
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] }
|
|
});
|
|
const start = await (
|
|
await postTelemetry(gameEvent('game_start', 0, startPlayers, 'managed'))
|
|
).json();
|
|
expect(start.telemetry.timing.server.gameStartedAt).toBe(at(0));
|
|
vi.setSystemTime(beginning + 1000);
|
|
await postTelemetry(gameEvent('round_change', 1, 0, 'managed'));
|
|
for (const seconds of [30, 121]) {
|
|
vi.setSystemTime(beginning + seconds * 1000);
|
|
await fanoutStatus();
|
|
expect(getServer('managed')).toMatchObject({
|
|
active_since: null,
|
|
game_started_at: at(0),
|
|
game_ended_at: null
|
|
});
|
|
}
|
|
vi.setSystemTime(beginning + 180_000);
|
|
await postTelemetry(gameEvent('heartbeat', 4, 1, 'managed'));
|
|
await fanoutStatus();
|
|
closeDb();
|
|
const server = (await (await GET()).json()).servers[0];
|
|
expect(server).toMatchObject({
|
|
health: 'online',
|
|
last_players: 1,
|
|
active_since: at(180),
|
|
game_started_at: at(0),
|
|
game_ended_at: null
|
|
});
|
|
expect(currentRound(server, server.telemetry, Date.now())).toBe(4);
|
|
}
|
|
);
|
|
|
|
it('keeps the first occupied report as the active timer when more telemetry arrives before polling', async () => {
|
|
const { inspect, rcon, fanoutStatus, getServer } = await setup();
|
|
const { GET } = await import('../../src/routes/api/servers/+server.js');
|
|
const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
|
|
const startedAt = Date.parse('2026-09-27T12:00:00Z');
|
|
const at = (seconds: number) => new Date(startedAt + seconds * 1000).toISOString();
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
vi.setSystemTime(startedAt);
|
|
inspect.mockResolvedValue({ ok: true, state: 'running' });
|
|
rcon.mockResolvedValue({
|
|
ok: true,
|
|
endpoint: { host: 'localhost', port: 28960 },
|
|
status: { players: 1, map: 'zombie_theater', gametype: 'zom', roster: [] }
|
|
});
|
|
expect((await postTelemetry(gameEvent('game_start', 0, 1, 'managed'))).status).toBe(200);
|
|
expect(getServer('managed')).toMatchObject({
|
|
active_since: at(0),
|
|
inactive_since: null,
|
|
game_started_at: at(0)
|
|
});
|
|
|
|
vi.setSystemTime(startedAt + 19_000);
|
|
const heartbeat = await (
|
|
await postTelemetry(gameEvent('heartbeat', 2, 1, 'managed'))
|
|
).json();
|
|
expect(heartbeat.telemetry.timing.server).toEqual({
|
|
activeSince: at(0),
|
|
inactiveSince: null,
|
|
gameStartedAt: at(0),
|
|
gameEndedAt: null
|
|
});
|
|
vi.setSystemTime(startedAt + 30_000);
|
|
const events: unknown[] = [];
|
|
const unsubscribe = subscribe((event) => events.push(event));
|
|
try {
|
|
await fanoutStatus();
|
|
} finally {
|
|
unsubscribe();
|
|
}
|
|
expect(rcon).toHaveBeenCalledOnce();
|
|
expect((await (await GET()).json()).servers[0]).toMatchObject({
|
|
active_since: at(0),
|
|
game_started_at: at(0),
|
|
last_seen_at: at(19)
|
|
});
|
|
expect(events).toContainEqual(
|
|
expect.objectContaining({
|
|
type: 'server-status',
|
|
entries: [
|
|
expect.objectContaining({
|
|
activeSince: at(0),
|
|
gameStartedAt: at(0),
|
|
lastSeenAt: at(19)
|
|
})
|
|
]
|
|
})
|
|
);
|
|
|
|
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
|
closeDb();
|
|
expect((await (await GET()).json()).servers[0]).toMatchObject({
|
|
active_since: at(0),
|
|
game_started_at: at(0)
|
|
});
|
|
});
|
|
|
|
it('preserves occupancy observed by RCON before game-start telemetry', async () => {
|
|
const { inspect, rcon, fanoutStatus, getServer } = await setup();
|
|
const startedAt = Date.parse('2026-09-27T12:00:00Z');
|
|
const at = (seconds: number) => new Date(startedAt + seconds * 1000).toISOString();
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
vi.setSystemTime(startedAt - 20_000);
|
|
inspect.mockResolvedValue({ ok: true, state: 'running' });
|
|
rcon.mockResolvedValue({
|
|
ok: true,
|
|
endpoint: { host: 'localhost', port: 28960 },
|
|
status: { players: 1, map: 'zombie_theater', gametype: 'zom', roster: [] }
|
|
});
|
|
await fanoutStatus();
|
|
expect(getServer('managed')).toMatchObject({
|
|
active_since: at(-20),
|
|
game_started_at: null
|
|
});
|
|
|
|
vi.setSystemTime(startedAt);
|
|
const start = await (await postTelemetry(gameEvent('game_start', 0, 1, 'managed'))).json();
|
|
expect(start.telemetry.timing.server).toEqual({
|
|
activeSince: at(-20),
|
|
inactiveSince: null,
|
|
gameStartedAt: at(0),
|
|
gameEndedAt: null
|
|
});
|
|
expect(getServer('managed')).toMatchObject({
|
|
active_since: at(-20),
|
|
inactive_since: null,
|
|
game_started_at: at(0)
|
|
});
|
|
});
|
|
|
|
it('records an empty lobby and rejoin between status polls without restoring the old match', async () => {
|
|
const { inspect, fanoutStatus, getServer } = await setup();
|
|
const { GET } = await import('../../src/routes/api/servers/+server.js');
|
|
const startedAt = Date.parse('2026-09-27T12:00:00Z');
|
|
const at = (seconds: number) => new Date(startedAt + seconds * 1000).toISOString();
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
vi.setSystemTime(startedAt);
|
|
inspect.mockResolvedValue({ ok: true, state: 'running' });
|
|
await postTelemetry(gameEvent('game_start', 0, 1, 'managed'));
|
|
vi.setSystemTime(startedAt + 10_000);
|
|
await postTelemetry(gameEvent('game_end', 2, 1, 'managed'));
|
|
vi.setSystemTime(startedAt + 11_000);
|
|
const empty = await (await postTelemetry(gameEvent('heartbeat', 0, 0, 'managed'))).json();
|
|
expect(empty.telemetry.timing.server).toEqual({
|
|
activeSince: null,
|
|
inactiveSince: at(11),
|
|
gameStartedAt: null,
|
|
gameEndedAt: null
|
|
});
|
|
expect(getServer('managed')).toMatchObject({
|
|
active_since: null,
|
|
inactive_since: at(11),
|
|
game_started_at: null,
|
|
game_ended_at: null
|
|
});
|
|
|
|
vi.setSystemTime(startedAt + 15_000);
|
|
const rejoin = await (await postTelemetry(gameEvent('heartbeat', 1, 1, 'managed'))).json();
|
|
expect(rejoin.telemetry.timing.server).toEqual({
|
|
activeSince: at(15),
|
|
inactiveSince: null,
|
|
gameStartedAt: null,
|
|
gameEndedAt: null
|
|
});
|
|
expect(getServer('managed')).toMatchObject({
|
|
active_since: at(15),
|
|
inactive_since: null,
|
|
game_started_at: null,
|
|
game_ended_at: null
|
|
});
|
|
vi.setSystemTime(startedAt + 30_000);
|
|
await fanoutStatus();
|
|
expect((await (await GET()).json()).servers[0]).toMatchObject({
|
|
active_since: at(15),
|
|
inactive_since: null,
|
|
game_started_at: null,
|
|
game_ended_at: null
|
|
});
|
|
});
|
|
|
|
it('waits for game_start telemetry after log initialization and player joins', async () => {
|
|
const { inspect, rcon, fanoutStatus, getServer } = await setup();
|
|
const { GET } = await import('../../src/routes/api/servers/+server.js');
|
|
const logs = join(fixture.t5, 'managed', 'logs');
|
|
await mkdir(logs, { recursive: true });
|
|
await writeFile(join(logs, 'games_sp.log'), '0:00 InitGame:\n0:01 J;123;0;Player\n');
|
|
inspect.mockResolvedValue({ ok: true, state: 'running' });
|
|
rcon.mockResolvedValue({
|
|
ok: true,
|
|
endpoint: { host: 'localhost', port: 28960 },
|
|
status: { players: 1, map: 'zombie_theater', gametype: 'zom', roster: [] }
|
|
});
|
|
await fanoutStatus();
|
|
expect(getServer('managed')).toMatchObject({
|
|
active_since: expect.any(String),
|
|
game_started_at: null
|
|
});
|
|
|
|
for (const event of ['round_change', 'heartbeat'] as const) {
|
|
expect((await postTelemetry(gameEvent(event, 1, 1, 'managed'))).status).toBe(200);
|
|
await fanoutStatus();
|
|
expect((await (await GET()).json()).servers[0].game_started_at).toBeNull();
|
|
}
|
|
|
|
const response = await postTelemetry(gameEvent('game_start', 0, 1, 'managed'));
|
|
expect(response.status).toBe(200);
|
|
const { telemetry } = await response.json();
|
|
await fanoutStatus();
|
|
expect((await (await GET()).json()).servers[0]).toMatchObject({
|
|
game_started_at: telemetry.receivedAt,
|
|
game_ended_at: null
|
|
});
|
|
});
|
|
|
|
it('includes the server clock with the health snapshot', async () => {
|
|
await setup();
|
|
const { GET } = await import('../../src/routes/api/servers/+server.js');
|
|
const requestedAt = Date.now();
|
|
const snapshot = await (await GET()).json();
|
|
expect(snapshot.serverNow).toBeGreaterThanOrEqual(requestedAt);
|
|
expect(snapshot.serverNow).toBeLessThanOrEqual(Date.now());
|
|
});
|
|
|
|
it.each(['t5', 't6'])(
|
|
'supplements %s telemetry with RCON pings consistently in REST and status events',
|
|
async (game) => {
|
|
const { db, inspect, rcon, fanoutStatus } = await setup();
|
|
const { GET } = await import('../../src/routes/api/servers/+server.js');
|
|
const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
|
|
const map = game === 't6' ? 'zm_transit' : 'zombie_theater';
|
|
db.prepare(
|
|
"UPDATE servers SET game = ?, map = ?, last_players = 4, last_seen_at = ? WHERE name = 'managed'"
|
|
).run(game, map, new Date().toISOString());
|
|
inspect.mockResolvedValue({ ok: true, state: 'running' });
|
|
const telemetry = gameEvent('heartbeat', 15, 3, 'managed', map);
|
|
telemetry.players[0].ping = 42;
|
|
telemetry.players[1].guid = '7097376';
|
|
rcon.mockResolvedValue({
|
|
ok: true,
|
|
endpoint: { host: 'localhost', port: 28960 },
|
|
status: {
|
|
players: 4,
|
|
map,
|
|
gametype: 'zom',
|
|
roster: [
|
|
{
|
|
num: 0,
|
|
guid: telemetry.players[0].guid,
|
|
name: 'Old name',
|
|
score: 50,
|
|
ping: 100
|
|
},
|
|
{
|
|
num: 1,
|
|
guid: game === 't6' ? '6C4C20' : telemetry.players[1].guid,
|
|
name: 'Another old name',
|
|
score: 100,
|
|
ping: 30
|
|
},
|
|
{
|
|
num: 2,
|
|
guid: '6C45DD',
|
|
name: telemetry.players[2].name,
|
|
score: 150,
|
|
ping: 151
|
|
}
|
|
]
|
|
}
|
|
});
|
|
await postTelemetry(telemetry);
|
|
const events: unknown[] = [];
|
|
const unsubscribe = subscribe((event) => events.push(event));
|
|
try {
|
|
await fanoutStatus();
|
|
} finally {
|
|
unsubscribe();
|
|
}
|
|
expect(rcon).toHaveBeenCalledOnce();
|
|
const server = (await (await GET()).json()).servers[0];
|
|
expect(server).toMatchObject({
|
|
health: 'online',
|
|
last_players: 3,
|
|
status_source: 'telemetry',
|
|
last_error: null,
|
|
rcon_status: { last_players: 4, last_error: null }
|
|
});
|
|
expect(server.roster).toHaveLength(3);
|
|
expect(server.roster[0]).toMatchObject({ ping: 42, score: 5000 });
|
|
expect(server.roster[1]).toMatchObject({ ping: 30, score: 5000, name: 'Player 2' });
|
|
expect(server.roster[2]).toMatchObject({
|
|
ping: null,
|
|
guid: telemetry.players[2].guid,
|
|
name: 'Player 3',
|
|
score: 5000
|
|
});
|
|
expect(events).toContainEqual(
|
|
expect.objectContaining({
|
|
type: 'server-status',
|
|
entries: [
|
|
expect.objectContaining({
|
|
health: 'online',
|
|
players: 3,
|
|
roster: server.roster,
|
|
lastSeenAt: server.last_seen_at
|
|
})
|
|
]
|
|
})
|
|
);
|
|
expect(
|
|
db.prepare("SELECT players FROM metric_samples WHERE server = 'managed'").get()
|
|
).toEqual({ players: 3 });
|
|
}
|
|
);
|
|
|
|
it('keeps occupied telemetry online when supplementary RCON fails', async () => {
|
|
const { inspect, rcon, fanoutStatus } = await setup();
|
|
const { GET } = await import('../../src/routes/api/servers/+server.js');
|
|
inspect.mockResolvedValue({ ok: true, state: 'running' });
|
|
rcon.mockResolvedValue({
|
|
ok: false,
|
|
endpoint: null,
|
|
reason: 'timeout',
|
|
detail: 'no reply'
|
|
});
|
|
await postTelemetry(gameEvent('heartbeat', 15, 1, 'managed'));
|
|
await fanoutStatus();
|
|
expect(rcon).toHaveBeenCalledOnce();
|
|
expect((await (await GET()).json()).servers[0]).toMatchObject({
|
|
health: 'online',
|
|
status_source: 'telemetry',
|
|
last_players: 1,
|
|
last_error: null,
|
|
roster: [{ name: 'Player 1', score: 5000, ping: null }]
|
|
});
|
|
});
|
|
|
|
it('skips supplementary RCON for empty telemetry rosters', async () => {
|
|
const { inspect, rcon, fanoutStatus } = await setup();
|
|
inspect.mockResolvedValue({ ok: true, state: 'running' });
|
|
await postTelemetry(gameEvent('heartbeat', 15, 0, 'managed'));
|
|
await fanoutStatus();
|
|
expect(rcon).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('falls back to RCON when telemetry expires and does not renew old reports', async () => {
|
|
const { db, inspect, rcon, fanoutStatus } = await setup();
|
|
const { GET } = await import('../../src/routes/api/servers/+server.js');
|
|
inspect.mockResolvedValue({ ok: true, state: 'running' });
|
|
await postTelemetry(gameEvent('heartbeat', 15, 3, 'managed'));
|
|
db.prepare('UPDATE server_telemetry SET received_at = ?').run(
|
|
new Date(Date.now() - 91_000).toISOString()
|
|
);
|
|
rcon.mockResolvedValue({
|
|
ok: true,
|
|
endpoint: { host: 'localhost', port: 28960 },
|
|
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] }
|
|
});
|
|
await fanoutStatus();
|
|
expect(rcon).toHaveBeenCalledTimes(1);
|
|
expect((await (await GET()).json()).servers[0]).toMatchObject({
|
|
health: 'online',
|
|
last_players: 0,
|
|
status_source: 'rcon'
|
|
});
|
|
});
|
|
|
|
it('lists registered servers without folders and ignores unregistered folders and tombstones', async () => {
|
|
const { db } = await setup();
|
|
const { allServers, listServers } = await import('../../src/lib/server/fleet/servers.js');
|
|
expect(await allServers()).toEqual(['managed']);
|
|
db.prepare("UPDATE servers SET destroyed_at = '2026-01-01' WHERE name = 'managed'").run();
|
|
expect(listServers()).toEqual([]);
|
|
});
|
|
|
|
it('skips startup RCON, clears old errors, and resumes polling only after the host agent reports readiness', async () => {
|
|
const { inspect, rcon, fanoutStatus, getServer } = await setup();
|
|
const { GET } = await import('../../src/routes/api/servers/+server.js');
|
|
inspect.mockResolvedValue({
|
|
ok: false,
|
|
state: 'starting',
|
|
reason: 'container state: starting'
|
|
});
|
|
await fanoutStatus();
|
|
await fanoutStatus();
|
|
expect(rcon).not.toHaveBeenCalled();
|
|
expect(getServer('managed')).toMatchObject({
|
|
container_state: 'starting',
|
|
last_error: null
|
|
});
|
|
expect((await (await GET()).json()).servers[0]).toMatchObject({
|
|
health: 'starting',
|
|
last_error: null,
|
|
last_players: null
|
|
});
|
|
|
|
inspect.mockResolvedValue({ ok: true, state: 'running' });
|
|
rcon.mockResolvedValue({
|
|
ok: false,
|
|
endpoint: null,
|
|
reason: 'timeout',
|
|
detail: 'no reply'
|
|
});
|
|
await fanoutStatus();
|
|
expect(rcon).toHaveBeenCalledTimes(1);
|
|
expect(getServer('managed')).toMatchObject({
|
|
container_state: 'running',
|
|
last_error: 'timeout:no reply'
|
|
});
|
|
|
|
rcon.mockResolvedValue({
|
|
ok: true,
|
|
endpoint: { host: 'localhost', port: 28960 },
|
|
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] }
|
|
});
|
|
await fanoutStatus();
|
|
expect((await (await GET()).json()).servers[0]).toMatchObject({
|
|
health: 'online',
|
|
last_error: null
|
|
});
|
|
});
|
|
|
|
it.each(['unhealthy', 'exited', 'stopped', 'missing'])(
|
|
'does not poll RCON with container state %s',
|
|
async (state) => {
|
|
const { inspect, rcon, fanoutStatus } = await setup();
|
|
inspect.mockResolvedValue({ ok: false, state, reason: 'not ready' });
|
|
await postTelemetry(gameEvent('heartbeat', 15, 3, 'managed'));
|
|
await fanoutStatus();
|
|
expect(rcon).not.toHaveBeenCalled();
|
|
const { GET } = await import('../../src/routes/api/servers/+server.js');
|
|
expect((await (await GET()).json()).servers[0]).toMatchObject({
|
|
health: 'offline',
|
|
last_players: null
|
|
});
|
|
}
|
|
);
|
|
|
|
it.each(['unknown', undefined, 'throw'])(
|
|
'automatically recovers through RCON when inspection returns %s',
|
|
async (state) => {
|
|
const { inspect, rcon, fanoutStatus, getServer } = await setup();
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
const { GET } = await import('../../src/routes/api/servers/+server.js');
|
|
const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
|
|
if (state === 'throw') inspect.mockRejectedValue(new Error('Inspection timed out'));
|
|
else inspect.mockResolvedValue({ ok: false, state, reason: 'Inspection timed out' });
|
|
await postTelemetry(gameEvent('heartbeat', 15, 3, 'managed'));
|
|
rcon.mockResolvedValue({
|
|
ok: true,
|
|
endpoint: { host: 'localhost', port: 28960 },
|
|
status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] }
|
|
});
|
|
const events: unknown[] = [];
|
|
const unsubscribe = subscribe((event) => events.push(event));
|
|
try {
|
|
await fanoutStatus();
|
|
} finally {
|
|
unsubscribe();
|
|
}
|
|
expect(rcon).toHaveBeenCalledOnce();
|
|
expect(getServer('managed')?.container_state).toBe('unknown');
|
|
expect((await (await GET()).json()).servers[0]).toMatchObject({
|
|
health: 'online',
|
|
last_players: 2,
|
|
status_source: 'rcon'
|
|
});
|
|
expect(events).toContainEqual(
|
|
expect.objectContaining({
|
|
type: 'server-status',
|
|
entries: [
|
|
expect.objectContaining({
|
|
health: 'online',
|
|
players: 2,
|
|
containerState: 'unknown'
|
|
})
|
|
]
|
|
})
|
|
);
|
|
|
|
rcon.mockResolvedValue({
|
|
ok: false,
|
|
endpoint: null,
|
|
reason: 'timeout',
|
|
detail: 'no reply'
|
|
});
|
|
vi.setSystemTime(Date.now() + 30_000);
|
|
await fanoutStatus();
|
|
expect((await (await GET()).json()).servers[0]).toMatchObject({
|
|
health: 'online',
|
|
last_error: null,
|
|
rcon_status: { rcon_failures: 1, last_error: 'timeout:no reply' }
|
|
});
|
|
vi.setSystemTime(Date.now() + 30_000);
|
|
await fanoutStatus();
|
|
expect((await (await GET()).json()).servers[0]).toMatchObject({
|
|
health: 'stale',
|
|
last_error: 'timeout:no reply'
|
|
});
|
|
}
|
|
);
|
|
|
|
it('waits for an EC restart request to settle before trusting the old healthy container', async () => {
|
|
const { db, inspect, rcon, fanoutStatus, getServer } = await setup();
|
|
db.prepare("INSERT INTO restart_rollout (server, requested_at) VALUES ('managed', ?)").run(
|
|
new Date().toISOString()
|
|
);
|
|
await fanoutStatus();
|
|
expect(inspect).not.toHaveBeenCalled();
|
|
expect(rcon).not.toHaveBeenCalled();
|
|
expect(getServer('managed')?.container_state).toBe('restarting');
|
|
});
|
|
});
|