205 lines
14 KiB
TypeScript
205 lines
14 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
|
|
let fixture: Fixture;
|
|
beforeEach(async () => { vi.resetModules(); fixture = await newFixture(); });
|
|
afterEach(async () => { await fixture.cleanup(); vi.restoreAllMocks(); });
|
|
|
|
describe('provisioning detail projection', () => {
|
|
it('records identical structured and legacy stage text once while preserving changed observations', async () => {
|
|
const { beginDeployment, deploymentStage, getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
const id = beginDeployment('server', 'restart');
|
|
const update = { phase: 'eligibility' as const,
|
|
blocker: { code: 'players', message: 'Waiting for 3 players', automatic: true } };
|
|
deploymentStage(id, update.blocker.message, update);
|
|
deploymentStage(id, update.blocker.message, update);
|
|
expect(getDeployment(id)!.events.filter(event => event.message === update.blocker.message)).toHaveLength(1);
|
|
deploymentStage(id, 'Waiting for 2 players', { ...update, blocker: { ...update.blocker, message: 'Waiting for 2 players' } });
|
|
const events = getDeployment(id)!.events;
|
|
expect(events.filter(event => event.message === 'Waiting for 2 players')).toHaveLength(1);
|
|
expect(events.filter(event => event.kind === 'blocker')).toHaveLength(1);
|
|
});
|
|
|
|
it('keeps legacy full activity while the page and opt-in API return a bounded snapshot without probing', async () => {
|
|
const history = await import('../../src/lib/server/provisioning/history.js');
|
|
const backend = await import('../../src/lib/server/backends/native.js');
|
|
const rcon = await import('../../src/lib/server/integrations/rcon.js');
|
|
const inspect = vi.spyOn(backend, 'nativeHealth');
|
|
const status = vi.spyOn(rcon, 'rconStatusResult');
|
|
const id = history.beginDeployment('t5_zombie_theater_1', 'create');
|
|
for (let index = 0; index < 40; index++) history.recordDeploymentEvent(id, `Recorded activity ${index}`);
|
|
history.finishDeployment(id, { ok: false, message: 'A recorded startup failure' });
|
|
for (let index = 0; index < 8; index++) history.recordDeploymentEvent(id, `Later observation ${index}`);
|
|
|
|
const full = history.getDeployment(id)!;
|
|
const recent = history.getDeployment(id, { events: 'recent' })!;
|
|
expect(full.events).toHaveLength(50);
|
|
expect(recent.events).toEqual(full.events.slice(-5));
|
|
expect(recent.activity).toMatchObject({ total: 50, latestId: full.events.at(-1)!.id,
|
|
failureExcerpt: { message: 'Failed: A recorded startup failure', kind: 'outcome' } });
|
|
expect(recent.progress?.phase).toBe('requested');
|
|
expect(recent.progress?.checkpoints.map(point => point.key)).toEqual(['requested']);
|
|
const { GET } = await import('../../src/routes/api/provisioning/[id]/+server.js');
|
|
expect((await (await GET({ params: { id } })).json()).events).toHaveLength(50);
|
|
expect((await (await GET({ params: { id }, url: new URL('http://localhost/?events=recent') })).json()).events).toHaveLength(5);
|
|
const { load } = await import('../../src/routes/provisioning/[id]/+page.server.js');
|
|
expect(load({ params: { id } })?.detail.events).toHaveLength(5);
|
|
expect(inspect).not.toHaveBeenCalled();
|
|
expect(status).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('retains missing sources, normalizes malformed warnings, and leaves legacy evidence unrecorded', async () => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
db().prepare(`INSERT INTO server_deployments (id,server,action,state,stage,started_at,triggered_by,warnings)
|
|
VALUES ('legacy','old-server','restart','done','Old custom stage','2026-01-01','ui',?)`).run('["Check files",7,null,{"bad":true}]');
|
|
db().prepare("INSERT INTO provisioning_sources VALUES ('legacy','campaign','missing','old-server')").run();
|
|
db().prepare("INSERT INTO provisioning_sources VALUES ('legacy','watchdog','incident','old-server')").run();
|
|
db().prepare(`INSERT INTO watchdog_incidents (id,server,kind,state,summary,first_seen_at,last_seen_at)
|
|
VALUES ('incident','old-server','health','resolved','Recovered game service','2026-01-01','2026-01-01')`).run();
|
|
const detail = getDeployment('legacy')!;
|
|
expect(detail.progress).toBeNull();
|
|
expect(detail.events).toEqual([]);
|
|
expect(detail.activity).toEqual({ total: 0, latestId: null, failureExcerpt: null });
|
|
expect(detail.warningMessages).toEqual(['Check files', 'Some recorded warnings could not be read.']);
|
|
expect(detail.sources).toEqual([
|
|
{ kind: 'campaign', source_id: 'missing', server: 'old-server', label: 'Campaign missing', href: null },
|
|
{ kind: 'watchdog', source_id: 'incident', server: 'old-server', label: 'Recovered game service', href: '/maintenance/incident' }
|
|
]);
|
|
db().prepare("UPDATE server_deployments SET warnings = 'broken json' WHERE id = 'legacy'").run();
|
|
expect(getDeployment('legacy')!.warningMessages).toEqual(['Recorded warnings could not be read.']);
|
|
});
|
|
|
|
it('correlates reservations by operation ID without assigning a later same-name slot to historical work', async () => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { beginDeployment, getDeployment, finishDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
db().prepare("INSERT INTO servers (name) VALUES ('server')").run();
|
|
const old = beginDeployment('server', 'restart');
|
|
finishDeployment(old, { ok: false, message: 'Timeout' });
|
|
const current = beginDeployment('server', 'restart');
|
|
db().prepare("INSERT INTO restart_rollout (server,requested_at,operation_id) VALUES ('server','2026-01-01',?)").run(current);
|
|
expect(getDeployment(old)!.capacity.pools[0]).toMatchObject({ kind: 'restart', used: 1, ownsSlot: false,
|
|
holders: [{ id: current, server: 'server' }] });
|
|
expect(getDeployment(old)!.progress!.recovery).toBeNull();
|
|
expect(getDeployment(current)!.capacity.pools[0].ownsSlot).toBe(true);
|
|
db().prepare('UPDATE restart_rollout SET healthy_at = ?').run(new Date().toISOString());
|
|
expect(getDeployment(old)!.capacity.pools[0].used).toBe(0);
|
|
expect(getDeployment(old)!.deployment.state).toBe('failed');
|
|
});
|
|
|
|
it('labels submission order independently of startup reservations and retains queued recipe status', async () => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { beginDeployment, getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
const { recordProgress } = await import('../../src/lib/server/provisioning/progress.js');
|
|
const first = beginDeployment('first', 'create');
|
|
const next = beginDeployment('next', 'create');
|
|
db().prepare("UPDATE server_deployments SET state = 'queued' WHERE id = ?").run(next);
|
|
recordProgress(next, { phase: 'startup-queue', recipeStatus: 'queued', observedAt: null });
|
|
const detail = getDeployment(next)!;
|
|
expect(detail.capacity.pools[0]).toMatchObject({ kind: 'startup', used: 1, requestOrder: 1, ownsSlot: false,
|
|
holders: [{ id: first, server: 'first' }] });
|
|
expect(detail.progress).toMatchObject({ recipeStatus: 'queued', observedAt: null });
|
|
});
|
|
|
|
it('preserves a failure outcome and evidence before an observation-only same-ID resume', async () => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const history = await import('../../src/lib/server/provisioning/history.js');
|
|
db().prepare("INSERT INTO servers (name,game_port) VALUES ('server',28960)").run();
|
|
const id = history.beginDeployment('server', 'create');
|
|
db().prepare("INSERT INTO server_creation_jobs VALUES (?, 'server',28960,'zombie','theater','zombie_theater')").run(id);
|
|
history.deploymentStage(id, 'Waiting for game evidence', { phase: 'verify-game', checks: [
|
|
{ key: 'game', label: 'Game status', result: 'waiting', source: null }
|
|
] });
|
|
history.finishDeployment(id, { ok: false, message: 'Observation interrupted' });
|
|
const failed = history.getDeployment(id)!;
|
|
expect(failed.progress?.phase).toBe('verify-game');
|
|
expect(history.resumeVerification(id)).toBe(true);
|
|
const resumed = history.getDeployment(id)!;
|
|
expect(resumed.deployment).toMatchObject({ id, state: 'awaiting-health', finished_at: null, error: null });
|
|
expect(resumed.progress).toMatchObject({ phase: 'verify-container', checks: [], observedAt: null });
|
|
const outcomes = resumed.events.filter(event => event.kind === 'outcome');
|
|
expect(outcomes).toHaveLength(1);
|
|
expect(JSON.parse(outcomes[0].details_json!)).toMatchObject({ state: 'failed', finishedAt: failed.deployment.finished_at,
|
|
error: 'Observation interrupted' });
|
|
expect(resumed.events.some(event => event.kind === 'resumed')).toBe(true);
|
|
});
|
|
|
|
it('preserves a legacy cleanup failure and keeps deleted cleanup capability authoritative', async () => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { getDeployment, resumeCleanup, REMOVING_SERVER } = await import('../../src/lib/server/provisioning/history.js');
|
|
db().prepare(`INSERT INTO server_deployments (id,server,server_deleted_at,action,state,stage,started_at,finished_at,triggered_by,error)
|
|
VALUES ('cleanup','old-server','2026-01-02','destroy','interrupted',?,'2026-01-01','2026-01-02','ui','App restarted')`).run(REMOVING_SERVER);
|
|
expect(getDeployment('cleanup')).toMatchObject({ canResumeCleanup: true, deployment: { server_exists: 0 } });
|
|
expect(resumeCleanup('cleanup')).toBe(true);
|
|
const result = getDeployment('cleanup')!;
|
|
expect(result.deployment).toMatchObject({ state: 'waiting', server_deleted_at: '2026-01-02' });
|
|
const outcome = result.events.find(event => event.kind === 'outcome')!;
|
|
expect(JSON.parse(outcome.details_json!)).toMatchObject({ state: 'interrupted', finishedAt: '2026-01-02', error: 'App restarted' });
|
|
expect(result.progress?.phase).toBe('cleanup');
|
|
});
|
|
});
|
|
|
|
describe('provisioning activity pagination', () => {
|
|
it('keeps polling payloads bounded for a retained history with thousands of events', async () => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { beginDeployment, getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
const id = beginDeployment('server', 'restart');
|
|
const insert = db().prepare('INSERT INTO server_deployment_events (deployment_id,created_at,message) VALUES (?,?,?)');
|
|
db().transaction(() => {
|
|
for (let index = 0; index < 5000; index++) insert.run(id, '2026-09-28T12:00:00Z', `Observation ${index} ${'x'.repeat(250)}`);
|
|
})();
|
|
const full = getDeployment(id)!;
|
|
const recent = getDeployment(id, { events: 'recent' })!;
|
|
const fullBytes = Buffer.byteLength(JSON.stringify(full));
|
|
const recentBytes = Buffer.byteLength(JSON.stringify(recent));
|
|
expect(recent.activity.total).toBe(5001);
|
|
expect(recent.events).toHaveLength(5);
|
|
expect(recentBytes).toBeLessThan(fullBytes / 100);
|
|
if (process.env.PROVISIONING_BENCHMARK) {
|
|
const start = performance.now();
|
|
for (let index = 0; index < 30; index++) getDeployment(id, { events: 'recent' });
|
|
console.info({ events: full.events.length, fullBytes, recentBytes, meanRecentReadMs: (performance.now() - start) / 30 });
|
|
}
|
|
});
|
|
|
|
it.each(['newest', 'oldest'] as const)('keeps stable %s pages under concurrent appends and searches literal text', async order => {
|
|
const { beginDeployment, recordDeploymentEvent } = await import('../../src/lib/server/provisioning/history.js');
|
|
const { deploymentActivity } = await import('../../src/lib/server/provisioning/activity.js');
|
|
const id = beginDeployment('server', 'stop');
|
|
const other = beginDeployment('other', 'stop');
|
|
for (let index = 0; index < 67; index++) {
|
|
recordDeploymentEvent(id, index === 30 ? 'literal %_needle' : `Event ${index}`);
|
|
recordDeploymentEvent(other, 'Unrelated event');
|
|
}
|
|
let page = deploymentActivity(id, { order });
|
|
const watermark = page.watermark;
|
|
const seen = [...page.events];
|
|
expect(page.events).toHaveLength(25);
|
|
recordDeploymentEvent(id, 'New event during reading');
|
|
while (page.nextCursor !== null) {
|
|
page = deploymentActivity(id, { order, watermark, cursor: page.nextCursor });
|
|
seen.push(...page.events);
|
|
}
|
|
expect(seen).toHaveLength(68);
|
|
expect(new Set(seen.map(event => event.id)).size).toBe(68);
|
|
expect(seen.every(event => event.id <= watermark)).toBe(true);
|
|
expect(page.total).toBe(68);
|
|
expect(page.totalEvents).toBe(69);
|
|
expect(deploymentActivity(id, { query: '%_NEEDLE' }).events.map(event => event.message)).toEqual(['literal %_needle']);
|
|
expect(deploymentActivity(id, { query: 'not found' }).total).toBe(0);
|
|
});
|
|
|
|
it('validates cursor/search requests and caps page size without changing default legacy responses', async () => {
|
|
const { beginDeployment, recordDeploymentEvent } = await import('../../src/lib/server/provisioning/history.js');
|
|
const { GET } = await import('../../src/routes/api/provisioning/[id]/events/+server.js');
|
|
const id = beginDeployment('server', 'stop');
|
|
for (let index = 0; index < 130; index++) recordDeploymentEvent(id, `Event ${index}`);
|
|
const response = GET({ params: { id }, url: new URL('http://localhost/?limit=999') });
|
|
expect((await response.json()).events).toHaveLength(100);
|
|
for (const query of ['cursor=-1', 'watermark=NaN', 'limit=0', 'order=invalid', 'cursor=9007199254740992']) {
|
|
expect(() => GET({ params: { id }, url: new URL(`http://localhost/?${query}`) })).toThrow();
|
|
}
|
|
expect(() => GET({ params: { id: 'missing' }, url: new URL('http://localhost/') })).toThrow();
|
|
});
|
|
});
|