josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Preserve health during empty-server monitoring handoff (Refs #148)

Browse Source
This commit is contained in:
josh
2026-10-12 19:02:54 -04:00
parent 2f52dcb1ef
commit f91ec4d152
6 changed files with 224 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/host-status-batch.test.ts
+182
View File
@@ -1,6 +1,7 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import { gameEvent, postTelemetry } from '../helpers/telemetry.js';
import { defaultPlan, requirement } from '../../src/lib/beta-plan.js';
let fixture: Fixture;
let at: number;
@@ -108,6 +109,187 @@ async function setup(count = 5, hostCount = 1) {
}
describe('host status observations', () => {
async function emptyHandoff() {
const api = await setup(1);
const server = 'server-0-0';
const { serverObservation } = await import('../../src/lib/server/fleet/status.js');
const { createScript } = await import('../../src/lib/server/fleet/scripts.js');
const { currentRun, readMetrics } =
await import('../../src/lib/server/fleet/beta-tests.js');
const { plannedBetaTick } =
await import('../../src/lib/server/pipeline/beta-plan-runner.js');
const { script, version } = createScript({
releaseVersion: '1.0.0',
name: 'Handoff test',
targetRelPath: 'handoff.gsc',
content: Buffer.from('main() {}')
});
const betaId = Number(
api.db
.prepare(
`INSERT INTO betas(script_id,script_version_id,trigger_kind,state,opened_at)
VALUES(?,?,'manual','ready-promote',?)`
)
.run(script.id, version.id, new Date(at).toISOString()).lastInsertRowid
);
api.db
.prepare(
`INSERT INTO beta_servers(beta_id,server,had_original,stage,restarted_at)
VALUES(?,?,0,'ready',?)`
)
.run(betaId, server, new Date(at).toISOString());
api.db
.prepare(`INSERT INTO beta_test_plans(beta_id,plan,outcome) VALUES(?,?,'passed')`)
.run(
betaId,
JSON.stringify({
...defaultPlan(),
requirements: [
requirement('healthy-minutes', 'health'),
requirement('failure-budget', 'outages')
]
})
);
api.db
.prepare(
`INSERT INTO beta_test_runs(id,beta_id,server,created_at,observing_at,metrics)
VALUES('handoff',?,?,?,?,?)`
)
.run(
betaId,
server,
new Date(at).toISOString(),
new Date(at).toISOString(),
JSON.stringify({
values: { health: 1 },
manual: {},
streaks: {},
outages: 0,
lastSample: at,
wasHealthy: true
})
);
await postTelemetry(gameEvent('heartbeat', 15, 1, server));
const occupied = api.fanoutStatus();
await vi.waitFor(() => expect(api.operations()).toHaveLength(1));
api.complete();
await occupied;
vi.setSystemTime(at + 30_000);
await postTelemetry(gameEvent('heartbeat', 0, 0, server));
await api.fanoutStatus();
await plannedBetaTick();
const emptyAt = Date.now();
const observe = () => serverObservation(api.getServer(server)!);
const metrics = () => readMetrics(currentRun(betaId, server)!);
const expectReady = async () => {
await plannedBetaTick();
expect(observe().health).toBe('online');
expect(metrics()).toMatchObject({ wasHealthy: true, outages: 0 });
expect(metrics().values.health).toBeGreaterThanOrEqual(1.5);
expect(api.db.prepare('SELECT state FROM betas WHERE id=?').get(betaId)).toEqual({
state: 'ready-promote'
});
expect(
api.db.prepare('SELECT outcome FROM beta_test_plans WHERE beta_id=?').get(betaId)
).toEqual({ outcome: 'passed' });
};
const setAge = (age: number) => {
vi.setSystemTime(emptyAt + age);
const host = api.getHost(api.hostIds[0])!;
api.heartbeat(host.id, {
...host.inventory,
instanceStates: host.inventory.instanceStates!.map((state) => ({
...state,
observedAt: new Date().toISOString()
}))
});
};
return { ...api, emptyAt, observe, metrics, expectReady, plannedBetaTick, setAge };
}
it.each([30_000, 59_999])(
'preserves beta readiness when the empty handoff poll starts at %i ms and replies 25 seconds later',
async (pollAt) => {
const api = await emptyHandoff();
api.setAge(pollAt - 30_000);
await api.fanoutStatus();
expect(api.operations()).toHaveLength(1);
await api.expectReady();
api.setAge(pollAt);
const pass = api.fanoutStatus();
try {
await vi.waitFor(() => expect(api.operations()).toHaveLength(2));
expect(api.operations()[0].kind).toBe('rcon.status-batch');
api.setAge(pollAt + 24_999);
await api.expectReady();
} finally {
api.setAge(pollAt + 25_000);
api.complete();
await pass;
}
await api.expectReady();
expect(api.metrics().lastSample).toBe(api.emptyAt + pollAt + 25_000);
expect(api.observe().status_source).toBe('telemetry');
for (const age of [90_000, 90_001]) {
api.setAge(age);
await api.expectReady();
}
expect(api.observe().status_source).toBe('rcon');
expect(api.observe().telemetry?.receivedAt).toBe(new Date(api.emptyAt).toISOString());
}
);
it.each(['pending', 'failed', 'skipped'] as const)(
'expires empty telemetry on schedule with a %s RCON handoff',
async (result) => {
const api = await emptyHandoff();
api.setAge(59_999);
const pass = api.fanoutStatus();
try {
await vi.waitFor(() => expect(api.operations()).toHaveLength(2));
if (result !== 'pending') {
api.setAge(84_999);
const operation = api.operations().find((op) => op.state === 'pending')!;
api.reportHostOperation(
operation.hostId,
operation.id,
result === 'failed'
? {
state: 'failed',
error: 'agent did not respond'
}
: {
state: 'completed',
result: {
results: [
{
instanceId: 'instance-0-0',
generation: 1,
reply: { ok: false, text: '', reason: 'deadline' }
}
]
}
}
);
await pass;
}
api.setAge(90_000);
await api.expectReady();
api.setAge(90_001);
expect(api.observe().health).not.toBe('online');
await api.plannedBetaTick();
expect(api.metrics()).toMatchObject({
wasHealthy: false,
outages: 1,
values: { health: 0 }
});
} finally {
api.complete();
await pass;
}
}
);
it('batches occupied telemetry lobbies for pings while skipping empty lobbies', async () => {
const api = await setup(2);
await postTelemetry(gameEvent('heartbeat', 15, 1, 'server-0-0'));