josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Relax freshness limits and tolerate transient RCON failures

Browse Source
This commit is contained in:
josh
2026-10-05 16:36:23 -04:00
parent d8b972cfc3
commit b1b3e8b958
28 changed files with 221 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/beta-plans.test.ts
+5 -5
View File
@@ -96,7 +96,7 @@ describe('durable beta test plans', () => {
it('uses script events instead of logs and keeps match progress across app restarts', async () => {
await postTelemetry(gameEvent('game_end'));
const id = await openPlan({ ...defaultPlan(), requirements: [requirement('matches', 'matches')] });
await tick(46); await tick();
await tick(91); await tick();
const { testView } = await import('../../src/lib/server/fleet/beta-tests.js');
await appendFile(join(fixture.t5, names[0], 'logs/games_sp.log'), 'InitGame:\n 0:00 J;new;1;Josh\nExitLevel: executed\n');
await tick();
@@ -113,7 +113,7 @@ describe('durable beta test plans', () => {
it('automatically credits ordinary rounds only to active deployed beta servers', async () => {
const id = await openPlan({ ...defaultPlan(), waveSize: 1, requirements: [requirement('round', 'round')] }, names);
await postTelemetry(gameEvent('round_change'));
await tick(46); await tick();
await tick(91); await tick();
const { testView } = await import('../../src/lib/server/fleet/beta-tests.js');
expect(testView(id)?.servers[0].progress[0].current).toBe(0);
await postTelemetry(gameEvent('round_change', 99, 4, names[1]));
@@ -135,7 +135,7 @@ describe('durable beta test plans', () => {
const first = currentRun(id, names[0])!;
expect(testView(id)?.servers[0].progress[0].current).toBe(0);
await postTelemetry(gameEvent('game_start'));
await tick(46); await tick();
await tick(91); await tick();
expect(currentRun(id, names[0])?.id).not.toBe(first.id);
expect(currentRun(id, names[0])?.phase).toBe(1);
expect(testView(id)?.servers[0].progress[0].current).toBe(0);
@@ -162,7 +162,7 @@ describe('durable beta test plans', () => {
await tick(15, 0, false);
// Fresh occupied telemetry remains healthy despite an RCON timeout.
expect(getBeta(id)?.state).toBe('ready-promote');
await tick(31, 0, false);
await tick(61, 0, false);
expect(getBeta(id)?.state).toBe('running');
await expect(promoteBeta(id)).rejects.toThrow();
});
@@ -236,7 +236,7 @@ describe('durable beta test plans', () => {
expect(retry.status).toBe(200);
await postTelemetry(gameEvent('round_change'));
expect(testView(id)?.servers).toHaveLength(0);
await tick(46); await tick();
await tick(91); await tick();
expect(testView(id)?.servers[0].progress[0].current).toBe(0);
await postTelemetry(gameEvent('round_change'));
expect(testView(id)?.servers[0].progress[0].current).toBe(25);
tests/unit/fleet-overview.test.ts
+2 -2
View File
@@ -31,14 +31,14 @@ describe('Fleet discovery and match freshness', () => {
expect(fleetIdentity({ ...server, game_mode: 'mp' }).launch).toBeNull();
});
it('does not classify stale, missing, or starting occupancy as an empty lobby', () => {
for (const change of [{ last_seen_at: iso(-46_000) }, { last_players: null }, { health: 'starting' as const }, { health: 'offline' as const }]) {
for (const change of [{ last_seen_at: iso(-91_000) }, { last_players: null }, { health: 'starting' as const }, { health: 'offline' as const }]) {
expect(matchesFleet({ ...server, last_players: 0, ...change }, '', 'empty', '', [], [], now)).toBe(false);
}
expect(matchesFleet({ ...server, last_players: 0 }, '', 'empty', '', [], [], now)).toBe(true);
expect(matchesFleet(server, '', 'occupied', 't5', [], [], now)).toBe(false);
});
it('includes stale health and failed operations in attention without flagging normal startup', () => {
expect(matchesFleet({ ...server, last_seen_at: iso(-46_000) }, '', 'attention', '', [], [], now)).toBe(true);
expect(matchesFleet({ ...server, last_seen_at: iso(-91_000) }, '', 'attention', '', [], [], now)).toBe(true);
expect(matchesFleet(server, '', 'attention', '', [], [{ ...operation, state: 'attention' }], now)).toBe(true);
expect(matchesFleet({ ...server, health: 'starting' }, '', 'attention', '', [], [], now)).toBe(false);
});
tests/unit/health-observations.test.ts
+2 -2
View File
@@ -61,7 +61,7 @@ describe('shared monitoring observations', () => {
expect(api.monitoringStatus(server)?.game?.observedAt).toBe(new Date(now).toISOString());
expect(api.inspect).toHaveBeenCalledTimes(1);
expect(api.rcon).toHaveBeenCalledTimes(1);
vi.setSystemTime(now + 45_001);
vi.setSystemTime(now + 90_001);
expect(api.monitoringStatus(server)).toBeNull();
vi.setSystemTime(now - 1);
expect(api.monitoringStatus(server)).toBeNull();
@@ -108,7 +108,7 @@ describe('shared monitoring observations', () => {
it('excludes old game evidence even when the container sample is fresh', async () => {
const api = await setup();
const game = await api.rconForMonitoring(server);
vi.setSystemTime(now + 46_000);
vi.setSystemTime(now + 91_000);
const publish = api.monitoringStatusWriter(server);
publish(await api.inspectForMonitoring(server), game);
expect(api.monitoringStatus(server)?.container.result).toEqual(healthy);
tests/unit/host-status-batch.test.ts
+4 -1
View File
@@ -123,6 +123,7 @@ describe('host status observations', () => {
const first = api.fanoutStatus();
await vi.waitFor(() => expect(api.operations()).toHaveLength(1));
api.complete(); await first;
api.db.prepare("UPDATE servers SET rcon_failures=1,last_error='timeout:first failure'").run();
const previous = api.getServer('server-0-0')!;
const previousGame = api.monitoringStatus('server-0-0')?.game;
vi.setSystemTime(at + 31_000);
@@ -138,9 +139,11 @@ describe('host status observations', () => {
} finally { api.complete(); await pass; }
expect(api.getServer('server-0-0')).toMatchObject({ last_checked_at: previous.last_checked_at,
last_seen_at: previous.last_seen_at, last_error: previous.last_error, last_players: previous.last_players,
last_reply_snippet: previous.last_reply_snippet });
last_reply_snippet: previous.last_reply_snippet, rcon_failures: 1 });
expect(api.monitoringStatus('server-0-0')?.game).toEqual(previousGame);
expect(api.getServer('server-0-1')?.last_error).toContain('timeout:');
expect(api.getServer('server-0-1')?.rcon_failures).toBe(2);
expect(api.getServer('server-0-2')?.rcon_failures).toBe(0);
expect(Date.parse(api.getServer('server-0-2')!.last_seen_at!)).toBeGreaterThanOrEqual(at + 31_000);
});
tests/unit/metrics.test.ts
+12 -12
View File
@@ -17,7 +17,7 @@ describe('historical metrics', () => {
const result = readMetrics(1, at + 3600_000);
expect(result.summary.active * 3600).toBeCloseTo(30);
expect(result.summary.idle * 3600).toBeCloseTo(30);
expect(result.summary.unknown * 3600).toBeCloseTo(45);
expect(result.summary.unknown * 3600).toBeCloseTo(90);
expect(result.summary.playerHours * 3600).toBeCloseTo(60);
expect(result.points.reduce((sum, p) => sum + p.observed, 0)).toBeCloseTo(result.summary.observed);
});
@@ -43,7 +43,7 @@ describe('historical metrics', () => {
let result = readMetrics(1, midnight + 15_000);
expect(result.points.filter(p => p.active > 0).map(p => p.active * 3600)).toEqual([15, 15]);
result = readMetrics(1, midnight + 86400_000);
expect(result.summary.active * 3600).toBeCloseTo(30);
expect(result.summary.active * 3600).toBeCloseTo(75);
expect(result.points.reduce((sum, p) => sum + p.active, 0)).toBeCloseTo(result.summary.active);
});
@@ -84,7 +84,7 @@ describe('historical metrics', () => {
expect(initial.summary).toEqual({ observed: 0, active: 0, idle: 0, unknown: 0, playerHours: 0 });
const later = readMetrics(1, at + 120_000);
expect(later.summary.observed).toBe(0);
expect(later.summary.unknown * 3600).toBeCloseTo(45);
expect(later.summary.unknown * 3600).toBeCloseTo(90);
expect(later.maps[0].capacityObserved).toBe(0);
});
@@ -112,10 +112,10 @@ describe('historical metrics', () => {
// Removing a lobby ends its previous observation at the next complete collection.
recordMetrics([{ ...one, map: 'Farm', players: 0 }], at + 30_000);
// A collection outage is excluded; an identically named map in another game remains separate.
recordMetrics([{ ...one, game: 't5', players: 2, capacity: null }], at + 120_000);
const result = readMetrics(1, at + 150_000);
recordMetrics([{ ...one, game: 't5', players: 2, capacity: null }], at + 150_000);
const result = readMetrics(1, at + 180_000);
expect(result.summary.active * 3600).toBeCloseTo(90);
expect(result.summary.idle * 3600).toBeCloseTo(45);
expect(result.summary.idle * 3600).toBeCloseTo(90);
expect(result.summary.playerHours * 3600).toBeCloseTo(300);
expect(result.servers.find(server => server.server === 'two')!.observed * 3600).toBeCloseTo(30);
const town = result.maps.find(map => map.game === 't6' && map.map === 'Town')!;
@@ -244,7 +244,7 @@ describe('historical metrics', () => {
expect(result.previous.until).toBe(result.since);
expect(result.previous.summary.active * 3600).toBeCloseTo(15);
expect(result.summary.active * 3600).toBeCloseTo(15);
expect(result.summary.idle * 3600).toBeCloseTo(45);
expect(result.summary.idle * 3600).toBeCloseTo(90);
expect(result.previous).toMatchObject({ firstSampleAt: since - 15_000, lastSampleAt: since - 15_000, partialHistory: true });
});
@@ -258,10 +258,10 @@ describe('historical metrics', () => {
recordMetrics([sample], now - 30_000);
const result = readMetrics(30, now);
expect(result.previous).toMatchObject({ since: now - 60 * day, until: now - 30 * day, partialHistory: true });
expect(result.previous.summary.active * 3600).toBeCloseTo(45);
expect(result.previous.summary.active * 3600).toBeCloseTo(90);
expect(result.previous.firstSampleAt).toBe(now - 31 * day + 30_000);
expect(result.summary.active * 3600).toBeCloseTo(30);
expect(result.summary.idle * 3600).toBeCloseTo(45);
expect(result.summary.idle * 3600).toBeCloseTo(90);
});
it('marks the retained start bound separately from gaps inside the previous window', async () => {
@@ -273,7 +273,7 @@ describe('historical metrics', () => {
recordMetrics([sample], now - 30_000);
const result = readMetrics(1, now);
expect(result.previous.partialHistory).toBe(false);
expect(result.previous.summary.unknown * 3600).toBeCloseTo(30);
expect(result.previous.summary.unknown * 3600).toBeCloseTo(75);
expect(result.previous.summary.observed).toBe(0);
expect(result.previous.firstSampleAt).toBe(priorStart - 15_000);
// Having the left edge says nothing about the uncaptured time after it.
@@ -326,8 +326,8 @@ describe('historical metrics', () => {
recordMetrics([{ ...sample, players: 0 }], at + 3600_000);
const result = readMetrics(parseMetricsQuery(new URLSearchParams({ days: '1', timeZone: 'America/New_York' })), at + 3630_000);
const cell = result.heatmap.cells.find(cell => cell.weekday === 6 && cell.hour === 1)!;
expect(cell.observed * 3600).toBeCloseTo(75);
expect(cell.active * 3600).toBeCloseTo(45);
expect(cell.observed * 3600).toBeCloseTo(120);
expect(cell.active * 3600).toBeCloseTo(90);
expect(cell.idle * 3600).toBeCloseTo(30);
expect(cell.distinctDates).toBe(1);
expect(result.heatmap.cells.filter(cell => cell.observed > 0)).toHaveLength(1);
tests/unit/provisioning.test.ts
+1 -1
View File
@@ -368,7 +368,7 @@ describe('Watchdog provisioning and telemetry evidence', () => {
const { rconGetPlayerCount } = await import('../../src/lib/server/integrations/rcon.js');
const { db } = await import('../../src/lib/server/db/index.js');
vi.mocked(rconGetPlayerCount).mockResolvedValue(null);
for (const offset of [-45_001, 1]) {
for (const offset of [-90_001, 1]) {
await report(0, new Date(Date.now() + offset).toISOString());
expect(telemetryEvidence(server)).toBeNull();
expect(await probePlayerCount(server)).toBeNull();
tests/unit/server-freshness.test.ts
+2 -2
View File
@@ -9,10 +9,10 @@ it('keeps healthy reports online across polling and delivery delays, then expire
vi.useFakeTimers();
const reportAt = Date.parse('2026-09-18T12:00:00Z');
const timestamp = new Date(reportAt).toISOString();
for (const elapsed of [-999, 0, STATUS_REFRESH_MS, STATUS_REFRESH_MS + 4000, 45_000, 45_001]) {
for (const elapsed of [-999, 0, STATUS_REFRESH_MS, STATUS_REFRESH_MS + 4000, 45_000, 60_000, 90_000, 90_001, 120_000]) {
const now = reportAt + elapsed;
vi.setSystemTime(now);
const expected = elapsed <= 45_000;
const expected = elapsed <= 90_000;
const health = classifyHealth(timestamp, null, 'running');
expect(health).toBe(expected ? 'online' : 'stale');
expect(freshServer({ health, last_seen_at: timestamp, last_error: null } as FleetServer, now)).toBe(expected);
tests/unit/server-observation.test.ts
+31 -5
View File
@@ -12,6 +12,32 @@ function server(changes: Partial<ServerObservation> = {}): ServerObservation {
}
describe('shared server observation', () => {
it('tolerates only the first transient failure while retaining diagnostics and expiring evidence', () => {
for (const reason of ['timeout', 'send-error']) {
const raw = { ...server(), last_error: `${reason}:no reply`, rcon_failures: 1 };
const selected = selectServerObservation(server({ rcon_status: raw }), undefined, now);
expect(selected).toMatchObject({ health: 'online', last_error: null, rcon_status: raw });
expect(selectServerObservation(selected, undefined, now + 90_000).health).toBe('online');
expect(selectServerObservation(selected, undefined, now + 90_001).health).toBe('stale');
expect(selectServerObservation(selected, undefined, now + 120_001).health).toBe('offline');
expect(selectServerObservation(server({ rcon_status: { ...raw, rcon_failures: 2 } }), undefined, now))
.toMatchObject({ health: 'stale', last_error: raw.last_error });
}
for (const reason of ['no-config', 'no-port', 'bad-reply', 'no-players']) {
expect(selectServerObservation(server({ last_error: `${reason}:failed`, rcon_failures: 1 }), undefined, now).health).toBe('stale');
}
expect(selectServerObservation(server({ last_error: 'timeout:legacy failure' }), undefined, now).health).toBe('stale');
});
it('accepts telemetry and container evidence through 90 seconds, then expires each independently', () => {
expect(selectServerObservation(server(), report(), now + 90_000)).toMatchObject({ health: 'online', status_source: 'telemetry' });
expect(selectServerObservation(server(), report(90_001), now + 90_001).health).toBe('stale');
const current = server({ container_checked_at: at(90_001) });
expect(selectServerObservation(current, report(), now + 90_001).health).toBe('stale');
expect(freshTelemetry(report(-90_000), now)).toBe(true);
expect(freshTelemetry(report(-90_001), now)).toBe(false);
});
it('selects the telemetry count and roster together and keeps raw RCON evidence', () => {
const raw = server({ last_error: 'timeout' });
const selected = selectServerObservation(server({ rcon_status: raw }), report(), now);
@@ -25,8 +51,8 @@ describe('shared server observation', () => {
const original = server();
const selected = selectServerObservation(server({ rcon_status: original, telemetry: report(0, 2) }), report(-1000), now);
expect(selected.last_players).toBe(2);
const fallback = selectServerObservation({ ...selected, container_checked_at: at(46_000),
rcon_status: { ...original, last_seen_at: at(40_000) } }, undefined, now + 46_000);
const fallback = selectServerObservation({ ...selected, container_checked_at: at(91_000),
rcon_status: { ...original, last_seen_at: at(40_000) } }, undefined, now + 91_000);
expect(fallback).toMatchObject({ status_source: 'rcon', health: 'online', last_players: 4, last_seen_at: at(40_000) });
});
@@ -40,19 +66,19 @@ describe('shared server observation', () => {
it('uses fresh RCON with unknown container state, but does not substitute telemetry or extend freshness', () => {
const raw = server({ container_state: 'unknown', container_checked_at: at(-60_000) });
expect(selectServerObservation(raw, report(), now)).toMatchObject({ health: 'online', last_players: 4, status_source: 'rcon' });
expect(selectServerObservation(raw, report(46_000), now + 46_000).health).toBe('stale');
expect(selectServerObservation(raw, report(91_000), now + 91_000).health).toBe('stale');
expect(selectServerObservation(raw, report(121_000), now + 121_000).health).toBe('offline');
expect(selectServerObservation({ ...raw, telemetry_after: at() }, report(), now).health).toBe('offline');
expect(selectServerObservation({ ...raw, last_seen_at: null }, report(), now).health).toBe('offline');
});
it('requires fresh container evidence and reports after the latest lifecycle submission', () => {
expect(selectServerObservation(server({ container_checked_at: at(-46_000) }), report(), now).health).not.toBe('online');
expect(selectServerObservation(server({ container_checked_at: at(-91_000) }), report(), now).health).not.toBe('online');
expect(selectServerObservation(server({ health: 'provisioning' }), report(), now).health).toBe('provisioning');
expect(selectServerObservation(server({ telemetry_after: at() }), report(-1), now).health).toBe('offline');
expect(freshTelemetry(report(), now, 'invalid')).toBe(false);
expect(freshTelemetry(report(2000), now)).toBe(false);
expect(freshTelemetry(report(-46_000), now)).toBe(false);
expect(freshTelemetry(report(-91_000), now)).toBe(false);
expect(freshTelemetry({ ...report(), players: [{}] } as never, now)).toBe(false);
});
tests/unit/server-status.test.ts
+77 -2
View File
@@ -30,6 +30,77 @@ async function setup() {
}
describe('database fleet and container readiness', () => {
it('requires two distinct failed monitoring checks, preserves diagnostics, and resets on success', async () => {
const { inspect, rcon, fanoutStatus, getServer } = await setup();
const { GET } = await import('../../src/routes/api/servers/+server.js');
const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
const snapshot = async () => (await (await GET()).json()).servers[0];
vi.useFakeTimers({ toFake: ['Date'] });
const at = Date.parse('2026-10-05T12:00:00Z');
vi.setSystemTime(at);
inspect.mockResolvedValue({ ok: true, state: 'running' });
const success = { ok: true as const, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status',
status: { players: 2, map: 'zombie_theater', gametype: 'zom', roster: [] } };
rcon.mockResolvedValue(success);
await fanoutStatus();
const events: unknown[] = [];
const unsubscribe = subscribe(event => events.push(event));
try {
rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'no reply' });
vi.setSystemTime(at + 30_000);
await fanoutStatus();
expect(getServer('managed')).toMatchObject({ rcon_failures: 1, last_error: 'timeout:no reply' });
expect(await snapshot()).toMatchObject({ health: 'online', last_error: null,
rcon_status: { last_error: 'timeout:no reply', rcon_failures: 1 } });
expect(events).toContainEqual(expect.objectContaining({ type: 'server-status', entries: [expect.objectContaining({
health: 'online', lastError: null, rconStatus: expect.objectContaining({ rcon_failures: 1, last_error: 'timeout:no reply' })
})] }));
const { closeDb } = await import('../../src/lib/server/db/index.js');
closeDb();
expect(await snapshot()).toMatchObject({ health: 'online', rcon_status: { rcon_failures: 1 } });
// Reprocessing a sample with the same timestamp cannot count it twice.
await fanoutStatus();
expect(getServer('managed')?.rcon_failures).toBe(1);
vi.setSystemTime(at + 60_000);
await fanoutStatus();
expect(await snapshot()).toMatchObject({ health: 'stale', last_error: 'timeout:no reply', rcon_status: { rcon_failures: 2 } });
rcon.mockResolvedValue(success);
vi.setSystemTime(at + 90_000);
await fanoutStatus();
expect(getServer('managed')).toMatchObject({ rcon_failures: 0, last_error: null });
rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'send-error', detail: 'unreachable' });
vi.setSystemTime(at + 120_000);
await fanoutStatus();
expect(await snapshot()).toMatchObject({ health: 'online', rcon_status: { rcon_failures: 1 } });
vi.setSystemTime(at + 180_001);
expect(await snapshot()).toMatchObject({ health: 'stale' });
vi.setSystemTime(at + 210_001);
expect(await snapshot()).toMatchObject({ health: 'offline' });
} finally { unsubscribe(); }
});
it('adds the failure counter to an existing database without suppressing legacy errors', async () => {
const { db, getServer } = await setup();
const { closeDb } = await import('../../src/lib/server/db/index.js');
db.exec('ALTER TABLE servers DROP COLUMN rcon_failures');
closeDb();
expect(getServer('managed')).toMatchObject({ rcon_failures: 0, last_error: 'timeout:old process' });
});
it('does not carry a failure streak across a new lifecycle submission', async () => {
const { db, inspect, rcon, fanoutStatus, getServer } = await setup();
vi.useFakeTimers({ toFake: ['Date'] });
const at = Date.parse('2026-10-05T12:00:00Z');
vi.setSystemTime(at);
db.prepare("UPDATE servers SET rcon_failures=1,last_checked_at=?").run(new Date(at - 30_000).toISOString());
db.prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,submitted_at,triggered_by)
VALUES('new-start','managed','restart','done','done',?,?,'test')`).run(new Date(at - 10_000).toISOString(), new Date(at - 10_000).toISOString());
inspect.mockResolvedValue({ ok: true, state: 'running' });
rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'new process' });
await fanoutStatus();
expect(getServer('managed')).toMatchObject({ rcon_failures: 1, last_error: 'timeout:new process' });
});
it('shares routine observations without renewing their timestamps and keeps manual probes fresh', async () => {
const { inspect, rcon, fanoutStatus, getServer } = await setup();
const { inspectForMonitoring, rconForMonitoring } = await import('../../src/lib/server/fleet/health-observations.js');
@@ -193,7 +264,7 @@ describe('database fleet and container readiness', () => {
const pending = fanoutStatus();
try {
await vi.waitFor(() => expect(rcon).toHaveBeenCalledTimes(1));
vi.setSystemTime(now + 46_000);
vi.setSystemTime(now + 91_000);
} finally {
release({ ok: true, state: 'running' });
await pending;
@@ -444,7 +515,7 @@ describe('database fleet and container readiness', () => {
const { GET } = await import('../../src/routes/api/servers/+server.js');
inspect.mockResolvedValue({ ok: true, state: 'running' });
await postTelemetry(gameEvent('heartbeat', 15, 3, 'managed'));
db.prepare('UPDATE server_telemetry SET received_at = ?').run(new Date(Date.now() - 46_000).toISOString());
db.prepare('UPDATE server_telemetry SET received_at = ?').run(new Date(Date.now() - 91_000).toISOString());
rcon.mockResolvedValue({ ok: true, endpoint: { host: 'localhost', port: 28960 }, rawReply: 'status',
status: { players: 0, map: 'zombie_theater', gametype: 'zom', roster: [] } });
await fanoutStatus();
@@ -515,6 +586,10 @@ describe('database fleet and container readiness', () => {
rcon.mockResolvedValue({ ok: false, endpoint: null, reason: 'timeout', detail: 'no reply' });
vi.setSystemTime(Date.now() + 30_000);
await fanoutStatus();
expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'online', last_error: null,
rcon_status: { rcon_failures: 1, last_error: 'timeout:no reply' } });
vi.setSystemTime(Date.now() + 30_000);
await fanoutStatus();
expect((await (await GET()).json()).servers[0]).toMatchObject({ health: 'stale', last_error: 'timeout:no reply' });
});
tests/unit/ui-server-clock.test.ts
+22 -4
View File
@@ -287,7 +287,7 @@ describe('Fleet report clock', () => {
expect(get(fleetServers)[0]).toMatchObject({ health: 'online', last_players: 3, status_source: 'telemetry' });
expect(get(fleetServers)[0].roster).toHaveLength(3);
expect(matchesFleet(get(fleetServers)[0], 'Player 3', 'occupied', '', [], [], get(uiNow))).toBe(true);
await vi.advanceTimersByTimeAsync(46_000);
await vi.advanceTimersByTimeAsync(91_000);
expect(freshServer(get(fleetServers)[0], get(uiNow))).toBe(false);
expect(matchesFleet(get(fleetServers)[0], '', 'occupied', '', [], [], get(uiNow))).toBe(false);
});
@@ -368,15 +368,33 @@ describe('Fleet report clock', () => {
it('keeps advancing through wall-clock changes and expires a report if updates stop', async () => {
const { fleetServers, uiNow } = await setup();
vi.setSystemTime(epoch - 120_000);
await vi.advanceTimersByTimeAsync(45_000);
expect(get(uiNow)).toBe(epoch + 45_000);
await vi.advanceTimersByTimeAsync(90_000);
expect(get(uiNow)).toBe(epoch + 90_000);
expect(freshServer(get(fleetServers)[0], get(uiNow))).toBe(true);
await vi.advanceTimersByTimeAsync(1000);
expect(get(uiNow)).toBe(epoch + 46_000);
expect(get(uiNow)).toBe(epoch + 91_000);
expect(freshServer(get(fleetServers)[0], get(uiNow))).toBe(false);
expect(matchesFleet(get(fleetServers)[0], '', 'attention', '', [], [], get(uiNow))).toBe(true);
});
it('applies the same RCON failure grace to HTTP snapshots and live updates', async () => {
const raw = { ...server(), last_error: 'timeout:no reply', rcon_failures: 1 };
snapshot.servers = [server(epoch, { rcon_status: raw })];
const { fleetServers, uiNow } = await setup();
expect(freshServer(get(fleetServers)[0], get(uiNow))).toBe(true);
expect(get(fleetServers)[0].rcon_status?.last_error).toBe('timeout:no reply');
await vi.advanceTimersByTimeAsync(30_000);
emit(epoch + 30_000, [status(epoch, { health: 'stale', lastError: 'timeout:no reply',
rconStatus: { ...raw, rcon_failures: 2 } })]);
expect(get(fleetServers)[0].health).toBe('stale');
expect(freshServer(get(fleetServers)[0], get(uiNow))).toBe(false);
await vi.advanceTimersByTimeAsync(30_000);
emit(epoch + 60_000, [status(epoch + 60_000, {
rconStatus: { ...raw, last_seen_at: new Date(epoch + 60_000).toISOString(), last_error: null, rcon_failures: 0 }
})]);
expect(freshServer(get(fleetServers)[0], get(uiNow))).toBe(true);
});
it('preserves actual RCON failures and stopped-container health', async () => {
const { fleetServers, uiNow } = await setup();
await vi.advanceTimersByTimeAsync(1000);
tests/unit/watchdog.test.ts
+3 -2
View File
@@ -114,7 +114,7 @@ describe('Watchdog detection and ownership', () => {
await watchdog.watchdogTick();
expect(onlyIncident()).toMatchObject({ failures: 1, last_seen_at: first.last_seen_at });
expect(komodo.komodoInspectHealthy).toHaveBeenCalledTimes(1);
advance(60_000);
advance(90_001);
await pollStatus();
await watchdog.watchdogTick();
expect(onlyIncident()).toMatchObject({ failures: 1, first_seen_at: new Date().toISOString() });
@@ -168,6 +168,7 @@ describe('Watchdog detection and ownership', () => {
expect(incidents()).toEqual([]);
expect(rcon.rconStatusResult).not.toHaveBeenCalled();
vi.mocked(rcon.rconStatusResult).mockResolvedValue(missingRcon);
advance(30_001);
await pollStatus();
advance(30_000);
database.db().prepare("DELETE FROM settings WHERE key='watchdog_last_scan'").run();
@@ -310,7 +311,7 @@ describe('Watchdog detection and ownership', () => {
await watchdog.watchdogTick(true);
expect(incidents()).toEqual([]);
expect(rcon.rconStatusResult).not.toHaveBeenCalled();
advance(60_000);
advance(90_001);
await watchdog.watchdogTick(true);
expect(onlyIncident()).toMatchObject({ kind: 'health', state: 'open', failures: 1 });
saveTelemetry({ ...gameEvent('heartbeat', 18, 4, server), receivedAt: new Date().toISOString() });