Reuse heartbeat container state and batch host RCON status
This commit is contained in:
@@ -0,0 +1,138 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
||||
|
||||
vi.mock('../../src/lib/server/integrations/komodo.js', () => ({ komodoInspectHealthy: vi.fn() }));
|
||||
let fixture: Fixture;
|
||||
let at: number;
|
||||
beforeEach(async () => {
|
||||
vi.resetModules(); vi.resetAllMocks();
|
||||
fixture = await newFixture([]);
|
||||
at = Date.now();
|
||||
vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(at);
|
||||
});
|
||||
afterEach(async () => { vi.useRealTimers(); await fixture.cleanup(); });
|
||||
|
||||
async function setup(count = 5, hostCount = 1) {
|
||||
const hosts = await import('../../src/lib/server/hosts/index.js');
|
||||
const { db } = await import('../../src/lib/server/db/index.js');
|
||||
const { komodoInspectHealthy } = await import('../../src/lib/server/integrations/komodo.js');
|
||||
vi.mocked(komodoInspectHealthy).mockResolvedValue({ ok: true, state: 'running' });
|
||||
const hostIds: string[] = [];
|
||||
for (let h = 0; h < hostCount; h++) {
|
||||
const host = hosts.createHost({ name: `Host ${h}` }); hostIds.push(host.id);
|
||||
const inventory = { protocolVersion: 1, capabilities: ['instance.heartbeat-state', 'rcon.status-batch', 'operations.health-lane'],
|
||||
instanceStates: Array.from({ length: count }, (_, i) => ({ instanceId: `instance-${h}-${i}`, generation: 1,
|
||||
healthy: true, state: 'running', observedAt: new Date(at).toISOString() })) };
|
||||
hosts.enrollHost(hosts.createEnrollment(host.id).token, inventory);
|
||||
for (let i = 0; i < count; i++) {
|
||||
const name = `server-${h}-${i}`;
|
||||
db().prepare('INSERT INTO servers(name,komodo_name,created_at) VALUES(?,?,?)').run(name, '', new Date(at - 1000).toISOString());
|
||||
db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at)
|
||||
VALUES(?,?,?,'library','image',?,?)`).run(`instance-${h}-${i}`, name, host.id, 28960 + i, new Date(at - 1000).toISOString());
|
||||
}
|
||||
}
|
||||
const health = await import('../../src/lib/server/fleet/health-observations.js');
|
||||
const worker = await import('../../src/lib/server/pipeline/worker.js');
|
||||
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
|
||||
const operations = () => hostIds.flatMap(id => hosts.listHostOperations(id));
|
||||
const complete = () => operations().filter(op => !['completed', 'failed'].includes(op.state)).forEach(op => {
|
||||
const targets = op.payload.instances as { instanceId: string; generation: number }[];
|
||||
hosts.reportHostOperation(op.hostId, op.id, { state: 'completed', result: op.kind === 'rcon.status-batch'
|
||||
? { ok: true, results: targets.map(target => ({ ...target, reply: { ok: true, text: 'map: mp_kino\nnum score ping guid name\n--- ----- ---- ---- ----\n' } })) }
|
||||
: { ok: true, text: 'map: mp_kino\nnum score ping guid name\n--- ----- ---- ---- ----\n' } });
|
||||
});
|
||||
return { ...hosts, ...health, ...worker, getServer, db: db(), inspect: vi.mocked(komodoInspectHealthy), hostIds, operations, complete };
|
||||
}
|
||||
|
||||
describe('host status observations', () => {
|
||||
it('uses one batch per host for a fifteen-server pass and shares overlapping scans', async () => {
|
||||
const api = await setup(5, 3);
|
||||
const passes = [api.fanoutStatus(), api.fanoutStatus()];
|
||||
try {
|
||||
await vi.waitFor(() => expect(api.operations()).toHaveLength(3));
|
||||
for (const operation of api.operations()) {
|
||||
expect(operation.kind).toBe('rcon.status-batch');
|
||||
expect(operation.payload.instances).toHaveLength(5);
|
||||
expect(api.pendingHostOperations(operation.hostId, 'health').map(op => op.id)).toEqual([operation.id]);
|
||||
expect(api.pendingHostOperations(operation.hostId, 'write')).toEqual([]);
|
||||
}
|
||||
} finally { api.complete(); await Promise.all(passes); }
|
||||
expect(api.inspect).not.toHaveBeenCalled();
|
||||
expect(api.getServer('server-2-4')).toMatchObject({ last_players: 0, last_map: 'mp_kino', container_checked_at: new Date(at).toISOString() });
|
||||
expect(api.db.prepare('SELECT COUNT(*) AS n FROM metric_samples').get()).toEqual({ n: 15 });
|
||||
await api.fanoutStatus();
|
||||
expect(api.operations()).toHaveLength(3);
|
||||
});
|
||||
|
||||
it.each(['stale', 'future', 'generation', 'lifecycle', 'missing', 'legacy'])('falls back to inspection for %s heartbeat evidence', async mode => {
|
||||
const api = await setup(1);
|
||||
const host = api.getHost(api.hostIds[0])!;
|
||||
const inventory = structuredClone(host.inventory);
|
||||
if (mode === 'stale') inventory.instanceStates![0].observedAt = new Date(at - 30_000).toISOString();
|
||||
if (mode === 'future') inventory.instanceStates![0].observedAt = new Date(at + 1_000).toISOString();
|
||||
if (mode === 'generation') inventory.instanceStates![0].generation = 2;
|
||||
if (mode === 'missing') inventory.instanceStates = [];
|
||||
if (mode === 'legacy') inventory.capabilities = [];
|
||||
if (mode === 'lifecycle') api.db.prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,submitted_at,finished_at,triggered_by)
|
||||
VALUES('stop','server-0-0','stop','done','done','started',?,?,'test')`).run(new Date(at + 1).toISOString(), new Date(at + 2).toISOString());
|
||||
api.heartbeat(host.id, inventory);
|
||||
await api.inspectForMonitoring('server-0-0');
|
||||
expect(api.inspect).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it('retains heartbeat sample time, reports stopped containers and keeps forced probes fresh', async () => {
|
||||
const api = await setup(1);
|
||||
vi.setSystemTime(at + 10_000);
|
||||
expect(await api.inspectForMonitoring('server-0-0')).toMatchObject({ observedAt: new Date(at).toISOString(), result: { ok: true } });
|
||||
await api.inspectForMonitoring('server-0-0', true);
|
||||
expect(api.inspect).toHaveBeenCalledOnce();
|
||||
const host = api.getHost(api.hostIds[0])!;
|
||||
api.heartbeat(host.id, { ...host.inventory, instanceStates: [{ ...host.inventory.instanceStates![0], healthy: false, state: 'exited' }] });
|
||||
await api.fanoutStatus();
|
||||
expect(api.getServer('server-0-0')?.container_state).toBe('exited');
|
||||
expect(api.operations()).toEqual([]);
|
||||
});
|
||||
|
||||
it.each(['generation', 'manual', 'lifecycle'])('discards batch replies overtaken by a %s change, including queued servers', async change => {
|
||||
const api = await setup();
|
||||
const pass = api.fanoutStatus();
|
||||
try {
|
||||
await vi.waitFor(() => expect(api.operations()).toHaveLength(1));
|
||||
vi.setSystemTime(at + 1000);
|
||||
if (change === 'generation') api.db.prepare('UPDATE native_instances SET generation=2').run();
|
||||
if (change === 'manual') api.db.prepare("UPDATE servers SET last_checked_at=?,last_error='timeout:manual'").run(new Date().toISOString());
|
||||
if (change === 'lifecycle') api.db.prepare(`INSERT INTO server_deployments(id,server,action,state,stage,started_at,submitted_at,triggered_by)
|
||||
SELECT name,name,'stop','done','done','started',?,'test' FROM servers`).run(new Date().toISOString());
|
||||
} finally { api.complete(); await pass; }
|
||||
for (let i = 0; i < 5; i++) expect(api.getServer(`server-0-${i}`)?.last_seen_at).toBeNull();
|
||||
expect(api.operations()).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('records each failed or omitted result without retrying the batch per server', async () => {
|
||||
const api = await setup();
|
||||
const pass = api.fanoutStatus();
|
||||
try {
|
||||
await vi.waitFor(() => expect(api.operations()).toHaveLength(1));
|
||||
const operation = api.operations()[0];
|
||||
api.reportHostOperation(operation.hostId, operation.id, { state: 'completed', result: { ok: false, results: [
|
||||
{ instanceId: 'instance-0-0', generation: 1, reply: { ok: false, reason: 'authentication', text: '' } }
|
||||
] } });
|
||||
} finally { api.complete(); await pass; }
|
||||
expect(api.getServer('server-0-0')?.last_error).toBe('bad-reply:wrong RCON password');
|
||||
expect(api.getServer('server-0-4')?.last_error).toContain('timeout:');
|
||||
expect(api.operations()).toHaveLength(1);
|
||||
});
|
||||
|
||||
it.each([true, false])('uses batch capability=%s while missing heartbeat state still gets inspected', async capable => {
|
||||
const api = await setup(1);
|
||||
const host = api.getHost(api.hostIds[0])!;
|
||||
api.heartbeat(host.id, { ...host.inventory, instanceStates: [], capabilities: capable ? ['rcon.status-batch'] : [] });
|
||||
const pass = api.fanoutStatus();
|
||||
try {
|
||||
await vi.waitFor(() => expect(api.operations()).toHaveLength(1));
|
||||
expect(api.operations()[0].kind).toBe(capable ? 'rcon.status-batch' : 'rcon.execute');
|
||||
} finally { api.complete(); await pass; }
|
||||
expect(api.inspect).toHaveBeenCalledOnce();
|
||||
expect(api.getServer('server-0-0')?.last_players).toBe(0);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user