369 lines
17 KiB
TypeScript
369 lines
17 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
|
|
let fixture: Fixture;
|
|
beforeEach(async () => {
|
|
vi.resetModules();
|
|
fixture = await newFixture();
|
|
});
|
|
afterEach(async () => {
|
|
await fixture.cleanup();
|
|
vi.restoreAllMocks();
|
|
});
|
|
|
|
describe('provisioning detail projection', () => {
|
|
it('records identical structured and legacy stage text once while preserving changed observations', async () => {
|
|
const { beginDeployment, deploymentStage, getDeployment } =
|
|
await import('../../src/lib/server/provisioning/history.js');
|
|
const id = beginDeployment('server', 'restart');
|
|
const update = {
|
|
phase: 'eligibility' as const,
|
|
blocker: { code: 'players', message: 'Waiting for 3 players', automatic: true }
|
|
};
|
|
deploymentStage(id, update.blocker.message, update);
|
|
deploymentStage(id, update.blocker.message, update);
|
|
expect(
|
|
getDeployment(id)!.events.filter((event) => event.message === update.blocker.message)
|
|
).toHaveLength(1);
|
|
deploymentStage(id, 'Waiting for 2 players', {
|
|
...update,
|
|
blocker: { ...update.blocker, message: 'Waiting for 2 players' }
|
|
});
|
|
const events = getDeployment(id)!.events;
|
|
expect(events.filter((event) => event.message === 'Waiting for 2 players')).toHaveLength(1);
|
|
expect(events.filter((event) => event.kind === 'blocker')).toHaveLength(1);
|
|
});
|
|
|
|
it('keeps legacy full activity while the page and opt-in API return a bounded snapshot without probing', async () => {
|
|
const history = await import('../../src/lib/server/provisioning/history.js');
|
|
const backend = await import('../../src/lib/server/backends/native.js');
|
|
const rcon = await import('../../src/lib/server/integrations/rcon.js');
|
|
const inspect = vi.spyOn(backend, 'nativeHealth');
|
|
const status = vi.spyOn(rcon, 'rconStatusResult');
|
|
const id = history.beginDeployment('t5_zombie_theater_1', 'create');
|
|
for (let index = 0; index < 40; index++)
|
|
history.recordDeploymentEvent(id, `Recorded activity ${index}`);
|
|
history.finishDeployment(id, { ok: false, message: 'A recorded startup failure' });
|
|
for (let index = 0; index < 8; index++)
|
|
history.recordDeploymentEvent(id, `Later observation ${index}`);
|
|
|
|
const full = history.getDeployment(id)!;
|
|
const recent = history.getDeployment(id, { events: 'recent' })!;
|
|
expect(full.events).toHaveLength(50);
|
|
expect(recent.events).toEqual(full.events.slice(-5));
|
|
expect(recent.activity).toMatchObject({
|
|
total: 50,
|
|
latestId: full.events.at(-1)!.id,
|
|
failureExcerpt: { message: 'Failed: A recorded startup failure', kind: 'outcome' }
|
|
});
|
|
expect(recent.progress?.phase).toBe('requested');
|
|
expect(recent.progress?.checkpoints.map((point) => point.key)).toEqual(['requested']);
|
|
const { GET } = await import('../../src/routes/api/provisioning/[id]/+server.js');
|
|
expect((await (await GET({ params: { id } })).json()).events).toHaveLength(50);
|
|
expect(
|
|
(
|
|
await (
|
|
await GET({ params: { id }, url: new URL('http://localhost/?events=recent') })
|
|
).json()
|
|
).events
|
|
).toHaveLength(5);
|
|
const { load } = await import('../../src/routes/provisioning/[id]/+page.server.js');
|
|
expect(load({ params: { id } })?.detail.events).toHaveLength(5);
|
|
expect(inspect).not.toHaveBeenCalled();
|
|
expect(status).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('retains missing sources, normalizes malformed warnings, and leaves legacy evidence unrecorded', async () => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { getDeployment } = await import('../../src/lib/server/provisioning/history.js');
|
|
db()
|
|
.prepare(
|
|
`INSERT INTO server_deployments (id,server,action,state,stage,started_at,triggered_by,warnings)
|
|
VALUES ('legacy','old-server','restart','done','Old custom stage','2026-01-01','ui',?)`
|
|
)
|
|
.run('["Check files",7,null,{"bad":true}]');
|
|
db()
|
|
.prepare(
|
|
"INSERT INTO provisioning_sources VALUES ('legacy','campaign','missing','old-server')"
|
|
)
|
|
.run();
|
|
db()
|
|
.prepare(
|
|
"INSERT INTO provisioning_sources VALUES ('legacy','watchdog','incident','old-server')"
|
|
)
|
|
.run();
|
|
db()
|
|
.prepare(
|
|
`INSERT INTO watchdog_incidents (id,server,kind,state,summary,first_seen_at,last_seen_at)
|
|
VALUES ('incident','old-server','health','resolved','Recovered game service','2026-01-01','2026-01-01')`
|
|
)
|
|
.run();
|
|
const detail = getDeployment('legacy')!;
|
|
expect(detail.progress).toBeNull();
|
|
expect(detail.events).toEqual([]);
|
|
expect(detail.activity).toEqual({ total: 0, latestId: null, failureExcerpt: null });
|
|
expect(detail.warningMessages).toEqual([
|
|
'Check files',
|
|
'Some recorded warnings could not be read.'
|
|
]);
|
|
expect(detail.sources).toEqual([
|
|
{
|
|
kind: 'campaign',
|
|
source_id: 'missing',
|
|
server: 'old-server',
|
|
label: 'Campaign unavailable',
|
|
href: null
|
|
},
|
|
{
|
|
kind: 'watchdog',
|
|
source_id: 'incident',
|
|
server: 'old-server',
|
|
label: 'Recovered game service',
|
|
href: '/maintenance/incident'
|
|
}
|
|
]);
|
|
db()
|
|
.prepare("UPDATE server_deployments SET warnings = 'broken json' WHERE id = 'legacy'")
|
|
.run();
|
|
expect(getDeployment('legacy')!.warningMessages).toEqual([
|
|
'Recorded warnings could not be read.'
|
|
]);
|
|
});
|
|
|
|
it.each(['create', 'rebuild', 'start', 'restart'] as const)(
|
|
'shows live capacity only for active %s operations, excluding historical work on the same server',
|
|
async (action) => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { beginDeployment, getDeployment, finishDeployment } =
|
|
await import('../../src/lib/server/provisioning/history.js');
|
|
db().prepare("INSERT INTO servers (name) VALUES ('server')").run();
|
|
const old = beginDeployment('server', action);
|
|
finishDeployment(old, { ok: false, message: 'Timeout' });
|
|
const current = beginDeployment('server', action);
|
|
db()
|
|
.prepare(
|
|
"INSERT INTO restart_rollout (server,requested_at,operation_id) VALUES ('server','2026-01-01',?)"
|
|
)
|
|
.run(current);
|
|
const kinds =
|
|
action === 'create'
|
|
? ['startup']
|
|
: action === 'rebuild'
|
|
? ['startup', 'restart']
|
|
: ['restart'];
|
|
expect(getDeployment(current)!.capacity.pools).toEqual(
|
|
kinds.map((kind) =>
|
|
expect.objectContaining({
|
|
kind,
|
|
used: 1,
|
|
ownsSlot: true,
|
|
holders: [{ id: current, server: 'server' }]
|
|
})
|
|
)
|
|
);
|
|
for (const state of ['done', 'warning', 'failed', 'interrupted', 'cancelled']) {
|
|
db()
|
|
.prepare('UPDATE server_deployments SET state = ? WHERE id = ?')
|
|
.run(state, old);
|
|
expect(getDeployment(old)!.capacity.pools, state).toEqual([]);
|
|
}
|
|
expect(getDeployment(old)!.progress!.recovery).toBeNull();
|
|
db().prepare('UPDATE restart_rollout SET healthy_at = ?').run(new Date().toISOString());
|
|
expect(getDeployment(old)!.capacity.pools).toEqual([]);
|
|
for (const state of ['queued', 'waiting', 'running', 'awaiting-health']) {
|
|
db()
|
|
.prepare('UPDATE server_deployments SET state = ? WHERE id = ?')
|
|
.run(state, current);
|
|
expect(
|
|
getDeployment(current)!.capacity.pools.map((pool) => pool.kind),
|
|
state
|
|
).toEqual(kinds);
|
|
}
|
|
}
|
|
);
|
|
|
|
it('labels submission order independently of startup reservations and retains queued recipe status', async () => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { beginDeployment, getDeployment } =
|
|
await import('../../src/lib/server/provisioning/history.js');
|
|
const { recordProgress } = await import('../../src/lib/server/provisioning/progress.js');
|
|
const first = beginDeployment('first', 'create');
|
|
const next = beginDeployment('next', 'create');
|
|
db().prepare("UPDATE server_deployments SET state = 'queued' WHERE id = ?").run(next);
|
|
recordProgress(next, { phase: 'startup-queue', recipeStatus: 'queued', observedAt: null });
|
|
const detail = getDeployment(next)!;
|
|
expect(detail.capacity.pools[0]).toMatchObject({
|
|
kind: 'startup',
|
|
used: 1,
|
|
requestOrder: 1,
|
|
ownsSlot: false,
|
|
holders: [{ id: first, server: 'first' }]
|
|
});
|
|
expect(detail.progress).toMatchObject({ recipeStatus: 'queued', observedAt: null });
|
|
});
|
|
|
|
it('preserves a failure outcome and evidence before an observation-only same-ID resume', async () => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const history = await import('../../src/lib/server/provisioning/history.js');
|
|
db().prepare("INSERT INTO servers (name,game_port) VALUES ('server',28960)").run();
|
|
const id = history.beginDeployment('server', 'create');
|
|
db()
|
|
.prepare(
|
|
"INSERT INTO server_creation_jobs VALUES (?, 'server',28960,'zombie','theater','zombie_theater')"
|
|
)
|
|
.run(id);
|
|
history.deploymentStage(id, 'Waiting for game evidence', {
|
|
phase: 'verify-game',
|
|
checks: [{ key: 'game', label: 'Game status', result: 'waiting', source: null }]
|
|
});
|
|
history.finishDeployment(id, { ok: false, message: 'Observation interrupted' });
|
|
const failed = history.getDeployment(id)!;
|
|
expect(failed.progress?.phase).toBe('verify-game');
|
|
expect(failed.capacity.pools).toEqual([]);
|
|
expect(history.resumeVerification(id)).toBe(true);
|
|
const resumed = history.getDeployment(id)!;
|
|
expect(resumed.deployment).toMatchObject({
|
|
id,
|
|
state: 'awaiting-health',
|
|
finished_at: null,
|
|
error: null
|
|
});
|
|
expect(resumed.progress).toMatchObject({
|
|
phase: 'verify-container',
|
|
checks: [],
|
|
observedAt: null
|
|
});
|
|
expect(resumed.capacity.pools).toEqual([
|
|
expect.objectContaining({ kind: 'startup', ownsSlot: true })
|
|
]);
|
|
const outcomes = resumed.events.filter((event) => event.kind === 'outcome');
|
|
expect(outcomes).toHaveLength(1);
|
|
expect(JSON.parse(outcomes[0].details_json!)).toMatchObject({
|
|
state: 'failed',
|
|
finishedAt: failed.deployment.finished_at,
|
|
error: 'Observation interrupted'
|
|
});
|
|
expect(resumed.events.some((event) => event.kind === 'resumed')).toBe(true);
|
|
});
|
|
|
|
it('preserves a legacy cleanup failure and keeps deleted cleanup capability authoritative', async () => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { getDeployment, resumeCleanup, REMOVING_SERVER } =
|
|
await import('../../src/lib/server/provisioning/history.js');
|
|
db()
|
|
.prepare(
|
|
`INSERT INTO server_deployments (id,server,server_deleted_at,action,state,stage,started_at,finished_at,triggered_by,error)
|
|
VALUES ('cleanup','old-server','2026-01-02','destroy','interrupted',?,'2026-01-01','2026-01-02','ui','App restarted')`
|
|
)
|
|
.run(REMOVING_SERVER);
|
|
expect(getDeployment('cleanup')).toMatchObject({
|
|
canResumeCleanup: true,
|
|
deployment: { server_exists: 0 }
|
|
});
|
|
expect(resumeCleanup('cleanup')).toBe(true);
|
|
const result = getDeployment('cleanup')!;
|
|
expect(result.deployment).toMatchObject({
|
|
state: 'waiting',
|
|
server_deleted_at: '2026-01-02'
|
|
});
|
|
const outcome = result.events.find((event) => event.kind === 'outcome')!;
|
|
expect(JSON.parse(outcome.details_json!)).toMatchObject({
|
|
state: 'interrupted',
|
|
finishedAt: '2026-01-02',
|
|
error: 'App restarted'
|
|
});
|
|
expect(result.progress?.phase).toBe('cleanup');
|
|
});
|
|
});
|
|
|
|
describe('provisioning activity pagination', () => {
|
|
it('keeps polling payloads bounded for a retained history with thousands of events', async () => {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const { beginDeployment, getDeployment } =
|
|
await import('../../src/lib/server/provisioning/history.js');
|
|
const id = beginDeployment('server', 'restart');
|
|
const insert = db().prepare(
|
|
'INSERT INTO server_deployment_events (deployment_id,created_at,message) VALUES (?,?,?)'
|
|
);
|
|
db().transaction(() => {
|
|
for (let index = 0; index < 5000; index++)
|
|
insert.run(id, '2026-09-28T12:00:00Z', `Observation ${index} ${'x'.repeat(250)}`);
|
|
})();
|
|
const full = getDeployment(id)!;
|
|
const recent = getDeployment(id, { events: 'recent' })!;
|
|
const fullBytes = Buffer.byteLength(JSON.stringify(full));
|
|
const recentBytes = Buffer.byteLength(JSON.stringify(recent));
|
|
expect(recent.activity.total).toBe(5001);
|
|
expect(recent.events).toHaveLength(5);
|
|
expect(recentBytes).toBeLessThan(fullBytes / 100);
|
|
if (process.env.PROVISIONING_BENCHMARK) {
|
|
const start = performance.now();
|
|
for (let index = 0; index < 30; index++) getDeployment(id, { events: 'recent' });
|
|
console.info({
|
|
events: full.events.length,
|
|
fullBytes,
|
|
recentBytes,
|
|
meanRecentReadMs: (performance.now() - start) / 30
|
|
});
|
|
}
|
|
});
|
|
|
|
it.each(['newest', 'oldest'] as const)(
|
|
'keeps stable %s pages under concurrent appends and searches literal text',
|
|
async (order) => {
|
|
const { beginDeployment, recordDeploymentEvent } =
|
|
await import('../../src/lib/server/provisioning/history.js');
|
|
const { deploymentActivity } =
|
|
await import('../../src/lib/server/provisioning/activity.js');
|
|
const id = beginDeployment('server', 'stop');
|
|
const other = beginDeployment('other', 'stop');
|
|
for (let index = 0; index < 67; index++) {
|
|
recordDeploymentEvent(id, index === 30 ? 'literal %_needle' : `Event ${index}`);
|
|
recordDeploymentEvent(other, 'Unrelated event');
|
|
}
|
|
let page = deploymentActivity(id, { order });
|
|
const watermark = page.watermark;
|
|
const seen = [...page.events];
|
|
expect(page.events).toHaveLength(25);
|
|
recordDeploymentEvent(id, 'New event during reading');
|
|
while (page.nextCursor !== null) {
|
|
page = deploymentActivity(id, { order, watermark, cursor: page.nextCursor });
|
|
seen.push(...page.events);
|
|
}
|
|
expect(seen).toHaveLength(68);
|
|
expect(new Set(seen.map((event) => event.id)).size).toBe(68);
|
|
expect(seen.every((event) => event.id <= watermark)).toBe(true);
|
|
expect(page.total).toBe(68);
|
|
expect(page.totalEvents).toBe(69);
|
|
expect(
|
|
deploymentActivity(id, { query: '%_NEEDLE' }).events.map((event) => event.message)
|
|
).toEqual(['literal %_needle']);
|
|
expect(deploymentActivity(id, { query: 'not found' }).total).toBe(0);
|
|
}
|
|
);
|
|
|
|
it('validates cursor/search requests and caps page size without changing default legacy responses', async () => {
|
|
const { beginDeployment, recordDeploymentEvent } =
|
|
await import('../../src/lib/server/provisioning/history.js');
|
|
const { GET } = await import('../../src/routes/api/provisioning/[id]/events/+server.js');
|
|
const id = beginDeployment('server', 'stop');
|
|
for (let index = 0; index < 130; index++) recordDeploymentEvent(id, `Event ${index}`);
|
|
const response = GET({ params: { id }, url: new URL('http://localhost/?limit=999') });
|
|
expect((await response.json()).events).toHaveLength(100);
|
|
for (const query of [
|
|
'cursor=-1',
|
|
'watermark=NaN',
|
|
'limit=0',
|
|
'order=invalid',
|
|
'cursor=9007199254740992'
|
|
]) {
|
|
expect(() =>
|
|
GET({ params: { id }, url: new URL(`http://localhost/?${query}`) })
|
|
).toThrow();
|
|
}
|
|
expect(() =>
|
|
GET({ params: { id: 'missing' }, url: new URL('http://localhost/') })
|
|
).toThrow();
|
|
});
|
|
});
|