josh/electric-cherry
1
0
Fork 0
Code Issues 18 Pull Requests Actions Packages Projects Releases Wiki Activity

Preserve verified health when provisioning completes

Browse Source
This commit is contained in:
josh
2026-10-06 15:48:38 -04:00
parent 6999a33aef
commit 57d2bd11db
9 changed files with 256 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/native-provisioning.test.ts
+144 -1
View File
@@ -30,7 +30,7 @@ beforeEach(async () => {
return { healthy: remote.state === 'running', state: remote.state, exists: true } as never;
});
});
afterEach(async () => { await fixture.cleanup(); vi.unstubAllEnvs(); });
afterEach(async () => { vi.useRealTimers(); await fixture.cleanup(); vi.unstubAllEnvs(); });
async function prepareLibrary(game: 't5' | 't6' = 't5') {
const libs = await import('../../src/lib/server/game-files/index.js');
@@ -74,6 +74,149 @@ async function detail(id: string) {
return getDeployment(id)!.deployment;
}
async function observations(server: string) {
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
const { serverObservation, observationReader } = await import('../../src/lib/server/fleet/status.js');
const { MonitoringSnapshot } = await import('../../src/lib/server/fleet/monitoring-snapshot.js');
const row = getServer(server)!;
return [serverObservation(row), observationReader()(row), new MonitoringSnapshot().observe(row)];
}
describe('provisioning health handoff', () => {
it.each(['create', 'start', 'restart', 'rebuild'] as const)('%s publishes verified game health before completion becomes visible', async action => {
const job = await queue(); await advance();
const { db } = await import('../../src/lib/server/db/index.js');
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
let id = job.deploymentId;
if (action !== 'create') {
await acknowledge(id);
db().prepare('UPDATE servers SET container_state=? WHERE name=?').run(action === 'start' ? 'stopped' : 'running', job.name);
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
const request = requestLifecycle(job.name, action);
if (!request.ok) throw new Error(request.error);
id = request.deploymentId;
await advance();
}
const old = new Date(Date.now() - 120_000).toISOString();
db().prepare(`UPDATE servers SET last_seen_at=?,last_checked_at=?,last_players=0,last_roster='[]',
last_error='timeout:old process',rcon_failures=2,active_since=NULL,inactive_since=? WHERE name=?`).run(old, old, old, job.name);
vi.useFakeTimers({ toFake: ['Date'] });
vi.setSystemTime(Date.now() + 1000);
const observedAt = new Date().toISOString();
const roster = [{ num: 0, name: 'Player', guid: '12345678901234567', score: 500, ping: 20 }];
const { rconStatusResult } = await import('../../src/lib/server/integrations/rcon.js');
vi.mocked(rconStatusResult).mockResolvedValueOnce({ ok: true, endpoint: { host: 'agent', port: 28960 }, rawReply: 'verified status',
status: { players: 1, map: remote.map, gametype: 'zom', roster } });
const { subscribe } = await import('../../src/lib/server/pipeline/bus.js');
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
const { serverObservation } = await import('../../src/lib/server/fleet/status.js');
const visible: string[] = [];
const unsubscribe = subscribe(event => {
if (event.type === 'fleet-invalidated' && getDeployment(id)?.deployment.state === 'done') {
visible.push(serverObservation(getServer(job.name)!).health);
}
});
try { await acknowledge(id); } finally { unsubscribe(); }
expect((await detail(id)).state).toBe('done');
expect(visible).toEqual(['online']);
expect(getServer(job.name)).toMatchObject({ last_checked_at: observedAt, last_seen_at: observedAt,
last_players: 1, last_roster: JSON.stringify(roster), last_reply_snippet: 'verified status',
last_error: null, rcon_failures: 0, active_since: observedAt, inactive_since: null });
for (const status of await observations(job.name)) expect(status).toMatchObject({ health: 'online', last_players: 1, roster });
});
it('completes directly from persisted telemetry without inventing an RCON observation', async () => {
const job = await queue(); await advance();
const operation = (await operationFor(job.deploymentId))!;
const { reportHostOperation } = await import('../../src/lib/server/hosts/index.js');
reportHostOperation(hostId, operation.id, { state: 'completed', result: { state: 'running', healthy: true } });
vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(Date.now() + 1000);
const { saveTelemetry } = await import('../../src/lib/server/fleet/telemetry.js');
const { gameEvent } = await import('../helpers/telemetry.js');
const receivedAt = new Date().toISOString();
saveTelemetry({ ...gameEvent('heartbeat', 0, 0, job.name), receivedAt });
await advance();
expect((await detail(job.deploymentId)).state).toBe('done');
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
expect(getServer(job.name)).toMatchObject({ last_seen_at: null, inactive_since: receivedAt });
for (const status of await observations(job.name)) expect(status).toMatchObject({ health: 'online', status_source: 'telemetry', last_seen_at: receivedAt });
});
it('rechecks expired verification evidence instead of renewing its timestamps', async () => {
const job = await queue(); await advance();
vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(Date.now() + 1000);
const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
vi.mocked(probeVersion).mockImplementationOnce(async () => {
vi.setSystemTime(Date.now() + 91_000);
heartbeat(hostId, inventory());
return 5354;
});
await acknowledge(job.deploymentId);
expect((await detail(job.deploymentId)).state).toBe('awaiting-health');
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
expect(getServer(job.name)?.last_seen_at).toBeNull();
for (const status of await observations(job.name)) expect(status.health).toBe('provisioning');
await advance();
expect((await detail(job.deploymentId)).state).toBe('done');
expect(getServer(job.name)?.last_seen_at).toBe(new Date().toISOString());
});
it.each(['instance', 'lifecycle'] as const)('discards verification when the %s changes during later checks', async change => {
const job = await queue(); await advance();
const { db } = await import('../../src/lib/server/db/index.js');
const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
vi.mocked(probeVersion).mockImplementationOnce(async () => {
if (change === 'instance') db().prepare('UPDATE native_instances SET generation=generation+1 WHERE server=?').run(job.name);
else db().prepare('UPDATE server_deployments SET submitted_at=? WHERE id=?').run(new Date(Date.now() + 1000).toISOString(), job.deploymentId);
return 5354;
});
await acknowledge(job.deploymentId);
expect((await detail(job.deploymentId)).state).toBe('awaiting-health');
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
expect(getServer(job.name)).toMatchObject({ last_seen_at: null, container_checked_at: null });
});
it.each(['online', 'offline'] as const)('preserves newer %s monitoring evidence received during version verification', async health => {
const job = await queue(); await advance();
vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(Date.now() + 1000);
const { db } = await import('../../src/lib/server/db/index.js');
const { probeVersion } = await import('../../src/lib/server/integrations/plutonium.js');
let newer: string;
vi.mocked(probeVersion).mockImplementationOnce(async () => {
vi.setSystemTime(Date.now() + 1000); newer = new Date().toISOString();
db().prepare(`UPDATE servers SET container_state=?,container_checked_at=?,last_seen_at=?,last_checked_at=?,
last_players=2,last_roster='[]',last_error=NULL WHERE name=?`).run(health === 'online' ? 'running' : 'stopped', newer, newer, newer, job.name);
return 5354;
});
await acknowledge(job.deploymentId);
const { getServer } = await import('../../src/lib/server/fleet/servers.js');
expect(getServer(job.name)).toMatchObject({ last_seen_at: newer!, container_checked_at: newer!, last_players: 2 });
expect((await detail(job.deploymentId)).state).toBe(health === 'online' ? 'done' : 'awaiting-health');
if (health === 'online') for (const status of await observations(job.name)) expect(status).toMatchObject({ health: 'online', last_players: 2 });
});
it('keeps a slow restart Starting until verification succeeds and surfaces terminal failures', async () => {
const job = await queue(); await advance(); await acknowledge(job.deploymentId);
const { requestLifecycle } = await import('../../src/lib/server/provisioning/requests.js');
const request = requestLifecycle(job.name, 'restart');
if (!request.ok) throw new Error(request.error);
await advance();
vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(Date.now() + 20_000);
for (const status of await observations(job.name)) expect(status.health).toBe('starting');
const { db } = await import('../../src/lib/server/db/index.js');
for (const state of ['failed', 'cancelled', 'interrupted']) {
db().prepare('UPDATE server_deployments SET state=? WHERE id=?').run(state, request.deploymentId);
for (const status of await observations(job.name)) expect(status.health).toBe('offline');
}
db().prepare("UPDATE server_deployments SET state='running' WHERE id=?").run(request.deploymentId);
await acknowledge(request.deploymentId);
expect((await detail(request.deploymentId)).state).toBe('done');
for (const status of await observations(job.name)) expect(status.health).toBe('online');
});
});
describe('native instance provisioning', () => {
it('recognizes an unplaced native intent and resumes it once after a controller restart', async () => {
const job = await queue();