310 lines
22 KiB
TypeScript
310 lines
22 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
import { provisioningListSources, provisioningMatchesSource, provisioningSourceLabel, type ProvisioningListSource } from '../../src/lib/provisioning-list.js';
|
|
|
|
let fixture: Fixture;
|
|
beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); });
|
|
afterEach(async () => { vi.restoreAllMocks(); await fixture.cleanup(); });
|
|
|
|
async function setup() {
|
|
const { db } = await import('../../src/lib/server/db/index.js');
|
|
const history = await import('../../src/lib/server/provisioning/history.js');
|
|
const handle = db();
|
|
const insert = handle.prepare(`INSERT INTO server_deployments
|
|
(id,server,action,state,stage,started_at,triggered_by,server_deleted_at,observation_json)
|
|
VALUES(?,?,?,?,?,?,'fixture',?,?)`);
|
|
const add = (id: string, state = 'done', action = 'create', server = 't5_zombie_theater_1',
|
|
deleted: string | null = null, stage = 'Recorded', at = '2026-01-01') =>
|
|
insert.run(id, server, action, state, stage, at, deleted, JSON.stringify({ snapshot: 'x'.repeat(4096) }));
|
|
return { ...history, handle, add };
|
|
}
|
|
|
|
describe('bounded provisioning lists', () => {
|
|
it('keeps recovery sources concise as incident summaries change, including after retention', async () => {
|
|
const { add, handle, deploymentListPage } = await setup();
|
|
add('recovery', 'waiting', 'rebuild');
|
|
handle.prepare("INSERT INTO provisioning_sources VALUES ('recovery','watchdog','incident','t5_zombie_theater_1')").run();
|
|
handle.prepare(`INSERT INTO watchdog_incidents (id,server,kind,state,summary,first_seen_at,last_seen_at)
|
|
VALUES ('incident','t5_zombie_theater_1','health','recovering',?,'2026-01-01','2026-01-01')`)
|
|
.run('Rebuild queued with the latest promoted setup');
|
|
|
|
for (const summary of ['Rebuild queued with the latest promoted setup', 'Recovered game service']) {
|
|
handle.prepare('UPDATE watchdog_incidents SET summary=? WHERE id=?').run(summary, 'incident');
|
|
const row = deploymentListPage({ view: 'active' }).deployments[0];
|
|
expect(provisioningSourceLabel(row)).toBe('Recovery');
|
|
expect(row.sources).toEqual([{ kind: 'watchdog', source_id: 'incident', label: summary, href: '/maintenance/incident' }]);
|
|
}
|
|
handle.prepare("DELETE FROM watchdog_incidents WHERE id='incident'").run();
|
|
const retained = deploymentListPage({ view: 'active' }).deployments[0];
|
|
expect(provisioningSourceLabel(retained)).toBe('Recovery');
|
|
expect(retained.sources?.[0].href).toBeNull();
|
|
});
|
|
|
|
it.each(['active', 'history'] as const)('keeps %s source categories and client filters aligned with API filters', async view => {
|
|
const { add, handle, deploymentListPage } = await setup();
|
|
const categories: [ProvisioningListSource, string, string][] = [
|
|
['campaign', 'campaign', 'Campaign'], ['beta', 'beta', 'Beta'], ['maintenance', 'maintenance', 'Maintenance'],
|
|
['recovery', 'watchdog', 'Recovery'], ['autoscaling', 'autoscaling', 'Autoscaling'], ['rebalance', 'rebalance', 'Rebalance']
|
|
];
|
|
const state = view === 'active' ? 'waiting' : 'done';
|
|
for (const [source, kind] of categories) {
|
|
// Rebalance ownership is recorded only in triggered_by.
|
|
if (source !== 'rebalance') {
|
|
add(`${source}-linked`, state);
|
|
handle.prepare('INSERT INTO provisioning_sources VALUES (?,?,?,?)').run(`${source}-linked`, kind, 'expired', 't5_zombie_theater_1');
|
|
}
|
|
add(`${source}-legacy`, state);
|
|
handle.prepare('UPDATE server_deployments SET triggered_by=? WHERE id=?').run(`${kind}:expired`, `${source}-legacy`);
|
|
}
|
|
add('manual', state);
|
|
handle.prepare("UPDATE server_deployments SET triggered_by='ui' WHERE id='manual'").run();
|
|
add('system', state);
|
|
// A shared operation must be discoverable through each of its sources, even when initiated manually.
|
|
add('shared', state);
|
|
handle.prepare("UPDATE server_deployments SET triggered_by='ui' WHERE id='shared'").run();
|
|
handle.exec(`INSERT INTO provisioning_sources VALUES ('shared','campaign','expired','t5_zombie_theater_1');
|
|
INSERT INTO provisioning_sources VALUES ('shared','watchdog','expired','t5_zombie_theater_1');`);
|
|
|
|
const rows = deploymentListPage({ view }).deployments;
|
|
expect(rows).toHaveLength(14);
|
|
for (const [source, , label] of categories) {
|
|
for (const suffix of source === 'rebalance' ? ['legacy'] : ['linked', 'legacy']) {
|
|
expect(provisioningSourceLabel(rows.find(row => row.id === `${source}-${suffix}`)!)).toBe(label);
|
|
}
|
|
}
|
|
expect(provisioningSourceLabel(rows.find(row => row.id === 'manual')!)).toBe('Manual');
|
|
expect(provisioningSourceLabel(rows.find(row => row.id === 'system')!)).toBe('System');
|
|
|
|
const { GET } = await import('../../src/routes/api/provisioning/+server.js');
|
|
for (const source of provisioningListSources) {
|
|
const expected = source === 'manual' || source === 'system' ? [source]
|
|
: source === 'rebalance' ? ['rebalance-legacy']
|
|
: [`${source}-linked`, `${source}-legacy`, ...(['campaign', 'recovery'].includes(source) ? ['shared'] : [])];
|
|
expect(rows.filter(row => provisioningMatchesSource(row, source)).map(row => row.id).sort()).toEqual(expected.sort());
|
|
const result = await GET({ url: new URL(`http://localhost/api/provisioning?view=${view}&source=${source}`) }).json();
|
|
expect(result.total).toBe(expected.length);
|
|
expect(result.deployments.map((row: { id: string }) => row.id).sort()).toEqual(expected);
|
|
if (view === 'history') {
|
|
const page = deploymentListPage({ view, source, limit: 1, offset: expected.length - 1 });
|
|
expect(page.total).toBe(expected.length);
|
|
expect(page.deployments).toHaveLength(1);
|
|
expect(provisioningMatchesSource(page.deployments[0], source)).toBe(true);
|
|
}
|
|
}
|
|
});
|
|
|
|
it('filters readable sources before pagination and retains expired source references without broken links', async () => {
|
|
const { add, handle, deploymentListPage } = await setup();
|
|
handle.exec(`INSERT INTO scripts(id,name,target_rel_path) VALUES(1,'Common scripts','common.gsc');
|
|
INSERT INTO script_versions(id,script_id,version,sha256,size) VALUES(1,1,1,'hash',0);
|
|
INSERT INTO betas(id,script_id,script_version_id,trigger_kind,state,notes) VALUES(75,1,1,'ui','promoted','Gameplay checks');
|
|
INSERT INTO deployments(id,action,started_at,beta_id) VALUES('campaign','promote','2026-01-01',75);`);
|
|
add('old-campaign', 'done', 'restart');
|
|
handle.prepare("UPDATE server_deployments SET triggered_by='campaign:campaign' WHERE id='old-campaign'").run();
|
|
add('linked-beta', 'waiting', 'restart');
|
|
handle.prepare("INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES('linked-beta','beta','75','t5_zombie_theater_1')").run();
|
|
add('expired', 'done', 'restart');
|
|
handle.prepare("UPDATE server_deployments SET triggered_by='campaign:expired' WHERE id='expired'").run();
|
|
add('manual', 'done', 'restart');
|
|
handle.prepare("UPDATE server_deployments SET triggered_by='ui' WHERE id='manual'").run();
|
|
for (let i = 0; i < 30; i++) add(`system-${i}`, 'done', 'restart');
|
|
const first = deploymentListPage({ view: 'history', source: 'campaign', action: 'restart', status: 'done', server: 'Kino', limit: 1 });
|
|
expect(first.total).toBe(2);
|
|
expect(first.deployments[0]).toMatchObject({ id: 'expired', sources: [{ label: 'Campaign unavailable', href: null }] });
|
|
const second = deploymentListPage({ view: 'history', source: 'campaign', limit: 1, offset: 1 });
|
|
expect(second.deployments[0]).toMatchObject({ id: 'old-campaign', sources: [{ label: 'Promotion · Beta #75', href: '/campaigns/campaign' }] });
|
|
const beta = deploymentListPage({ view: 'active', source: 'beta', status: 'pending' });
|
|
expect(beta.deployments).toEqual([expect.objectContaining({ id: 'linked-beta', sources: [{ kind: 'beta', source_id: '75', label: 'Beta #75: Gameplay checks', href: '/betas/75' }] })]);
|
|
expect(provisioningSourceLabel(second.deployments[0])).toBe('Campaign');
|
|
expect(provisioningSourceLabel(beta.deployments[0])).toBe('Beta');
|
|
expect(deploymentListPage({ view: 'history', source: 'manual' }).deployments.map(row => row.id)).toEqual(['manual']);
|
|
expect(deploymentListPage({ view: 'history', source: 'system' }).total).toBe(30);
|
|
const { GET } = await import('../../src/routes/api/provisioning/+server.js');
|
|
const result = await GET({ url: new URL('http://localhost/api/provisioning?view=active&status=pending&source=beta') }).json();
|
|
expect(result.deployments.map((row: { id: string }) => row.id)).toEqual(['linked-beta']);
|
|
});
|
|
|
|
it('counts unresolved outcomes while retaining resolved failures and routine cancellations in history', async () => {
|
|
const { add, handle, deploymentListPage } = await setup();
|
|
add('failed', 'failed'); add('resolved', 'failed'); add('cancelled', 'cancelled'); add('warning', 'warning');
|
|
handle.prepare("INSERT INTO retention_controls(category,record_id,resolved_at) VALUES('provisioning','resolved','2026-01-02')").run();
|
|
expect(deploymentListPage({ view: 'history' }).total).toBe(4);
|
|
const attention = deploymentListPage({ view: 'history', status: 'attention' });
|
|
expect(attention.needsReview).toBe(2);
|
|
expect(attention.deployments.map(row => row.id)).toEqual(['warning', 'failed']);
|
|
expect(deploymentListPage({ view: 'history', status: 'failed' }).total).toBe(2);
|
|
handle.prepare("UPDATE server_deployments SET finished_at='2026-01-03' WHERE id='resolved'").run();
|
|
expect(deploymentListPage({ view: 'history', status: 'attention' }).needsReview).toBe(3);
|
|
});
|
|
|
|
it('exposes compact waiting facts without transferring observation snapshots', async () => {
|
|
const { add, handle, deploymentListPage } = await setup();
|
|
add('waiting', 'waiting', 'restart');
|
|
handle.prepare("UPDATE server_deployments SET phase_key='eligibility',phase_started_at='2026-01-01',observation_json=? WHERE id='waiting'")
|
|
.run(JSON.stringify({ blocker: { code: 'restart-capacity' }, checks: ['private snapshot'], checkpoints: [] }));
|
|
expect(deploymentListPage({ view: 'active' }).deployments[0]).toMatchObject({ waitCode: 'restart-capacity', phase: 'eligibility', phaseStartedAt: '2026-01-01' });
|
|
expect(JSON.stringify(deploymentListPage({ view: 'active' }))).not.toContain('private snapshot');
|
|
});
|
|
|
|
it('puts verification and executing work ahead of preflight waits and capacity queues', async () => {
|
|
const { add, handle, listDeployments, deploymentListPage } = await setup();
|
|
const addProgress = (id: string, state: string, second: number, phase: string,
|
|
waitCode: string | null = null, stage = 'Recorded', action = 'rebuild') => {
|
|
add(id, state, action, 't5_zombie_theater_1', null, stage, `2026-01-01T00:00:${String(second).padStart(2, '0')}.000Z`);
|
|
handle.prepare('UPDATE server_deployments SET phase_key=?, observation_json=? WHERE id=?')
|
|
.run(phase, JSON.stringify({ blocker: waitCode ? { code: waitCode } : null }), id);
|
|
};
|
|
// Request order is deliberately different from progress order.
|
|
addProgress('queued', 'queued', 1, 'requested');
|
|
addProgress('cleanup', 'waiting', 12, 'cleanup', null, 'Removing server files and registry', 'destroy');
|
|
addProgress('verify-newer', 'awaiting-health', 21, 'verify-map');
|
|
addProgress('capacity', 'running', 2, 'startup-queue', 'startup-capacity');
|
|
addProgress('operation-conflict', 'running', 6, 'container-action', 'operation-conflict');
|
|
addProgress('agent-wait', 'running', 11, 'container-action', 'native-host', 'Agent operation pending');
|
|
addProgress('host-wait', 'waiting', 5, 'eligibility', 'native-host', 'Host is unavailable; waiting for reconnection.');
|
|
addProgress('staging', 'waiting', 7, 'eligibility', 'native-host', 'Staging and verifying files on the host.');
|
|
addProgress('staging-wait', 'waiting', 8, 'eligibility', 'native-host', 'Waiting for verified remote file staging.');
|
|
addProgress('legacy-capacity', 'waiting', 3, 'eligibility', 'native-host', 'Queued to rebuild; waiting for another server to recover');
|
|
addProgress('verify-older', 'awaiting-health', 20, 'verify-container', 'native-host', 'Waiting for container readiness: starting');
|
|
addProgress('preparing-queued', 'queued', 9, 'prepare');
|
|
addProgress('executing', 'running', 10, 'prepare');
|
|
addProgress('eligibility', 'running', 4, 'eligibility', 'eligibility');
|
|
addProgress('interrupted', 'interrupted', 30, 'container-action', 'native-host');
|
|
addProgress('failed', 'failed', 31, 'verify-map', 'native-host');
|
|
|
|
const expected = ['verify-older', 'verify-newer', 'staging', 'staging-wait', 'preparing-queued', 'executing', 'agent-wait', 'cleanup',
|
|
'eligibility', 'host-wait', 'operation-conflict', 'queued', 'capacity', 'legacy-capacity'];
|
|
expect(listDeployments({ view: 'active' }).map(row => row.id)).toEqual(expected);
|
|
const { GET } = await import('../../src/routes/api/provisioning/+server.js');
|
|
const active = await GET({ url: new URL('http://localhost/api/provisioning?view=active&limit=1') }).json();
|
|
expect(active.deployments.map((row: { id: string }) => row.id)).toEqual(expected);
|
|
expect(active).toMatchObject({ total: expected.length, needsReview: 2 });
|
|
|
|
// Progress priority applies only to Active; retained outcomes and mixed reads stay newest first.
|
|
expect(deploymentListPage({ view: 'history', limit: 1 }).deployments.map(row => row.id)).toEqual(['failed']);
|
|
expect(deploymentListPage({ view: 'history', limit: 1, offset: 1 }).deployments.map(row => row.id)).toEqual(['interrupted']);
|
|
expect(listDeployments({ view: 'all', limit: 3 }).map(row => row.id)).toEqual(['failed', 'interrupted', 'verify-newer']);
|
|
});
|
|
|
|
it('keeps oldest requests and insertion ties stable while active progress is polled', async () => {
|
|
const { add, handle, listDeployments } = await setup();
|
|
add('z-first', 'awaiting-health', 'rebuild', 't5_zombie_theater_1', null, 'Checking game status', '2026-01-02');
|
|
add('a-second', 'awaiting-health', 'rebuild', 't5_zombie_theater_1', null, 'Checking selected engine map', '2026-01-02');
|
|
add('old-inserted-last', 'awaiting-health', 'rebuild', 't5_zombie_theater_1', null, 'Checking container state', '2026-01-01');
|
|
add('queued', 'queued', 'rebuild', 't5_zombie_theater_1', null, 'Request recorded', '2025-12-31');
|
|
const ids = () => listDeployments({ view: 'active' }).map(row => row.id);
|
|
const expected = ['old-inserted-last', 'z-first', 'a-second', 'queued'];
|
|
expect(ids()).toEqual(expected);
|
|
expect(ids()).toEqual(expected);
|
|
|
|
handle.prepare(`UPDATE server_deployments SET phase_key='verify-container', phase_started_at='2026-01-03',
|
|
observed_at='2026-01-03', observation_json=? WHERE id='z-first'`)
|
|
.run(JSON.stringify({ blocker: { code: 'native-host', message: 'Waiting for container readiness: starting' } }));
|
|
handle.prepare("UPDATE server_deployments SET phase_key='verify-plutonium', phase_started_at='2026-01-04' WHERE id='a-second'").run();
|
|
expect(ids()).toEqual(expected);
|
|
|
|
handle.prepare("UPDATE server_deployments SET state='done', finished_at='2026-01-05' WHERE id='z-first'").run();
|
|
handle.prepare("UPDATE server_deployments SET state='awaiting-health', phase_key='verify-game' WHERE id='queued'").run();
|
|
expect(ids()).toEqual(['queued', 'old-inserted-last', 'a-second']);
|
|
expect(listDeployments({ view: 'all' }).map(row => row.id)).toEqual(['a-second', 'z-first', 'old-inserted-last', 'queued']);
|
|
});
|
|
|
|
it('polls every active row and counts review outcomes without transferring retained history', async () => {
|
|
const { handle, add, REMOVING_SERVER } = await setup();
|
|
handle.transaction(() => {
|
|
for (let i = 0; i < 6_000; i++) add(`history-${i}`, i % 2 ? 'done' : 'failed');
|
|
for (const state of ['queued', 'awaiting-health', 'running', 'waiting']) add(state, state);
|
|
for (let i = 0; i < 110; i++) add(`active-${i}`, 'waiting', 'destroy');
|
|
add('cleanup', 'waiting', 'destroy', 'cleanup', null, REMOVING_SERVER);
|
|
add('deleted-active', 'running', 'create', 'retired', '2026-01-02');
|
|
add('deleted-warning', 'warning', 'create', 'retired', '2026-01-02');
|
|
add('interrupted', 'interrupted');
|
|
})();
|
|
const { GET } = await import('../../src/routes/api/provisioning/+server.js');
|
|
const response = GET({ url: new URL('http://localhost/api/provisioning?view=active&limit=1') });
|
|
const body = await response.json();
|
|
expect(body.total).toBe(115);
|
|
expect(body.deployments).toHaveLength(115);
|
|
expect(body.needsReview).toBe(3_002);
|
|
expect(body.deployments.map((row: { id: string }) => row.id)).not.toContain('deleted-active');
|
|
expect(body.deployments.find((row: { id: string }) => row.id === 'active-0').canCancelRemoval).toBe(1);
|
|
expect(body.deployments.find((row: { id: string }) => row.id === 'cleanup').canCancelRemoval).toBe(0);
|
|
expect(JSON.stringify(body)).not.toContain('observation_json');
|
|
expect(Buffer.byteLength(JSON.stringify(body))).toBeLessThan(60_000);
|
|
expect(response.headers.get('cache-control')).toBe('no-store');
|
|
const history = await GET({ url: new URL('http://localhost/api/provisioning?view=history&limit=100') }).json();
|
|
expect(history.deployments).toHaveLength(100);
|
|
expect(history.total).toBe(6_003);
|
|
expect(Buffer.byteLength(JSON.stringify(history))).toBeLessThan(50_000);
|
|
|
|
// The legacy endpoint must not reintroduce the unbounded response.
|
|
const { GET: legacyGET } = await import('../../src/routes/api/deployments/+server.js');
|
|
expect(await legacyGET({ url: new URL('http://localhost/api/deployments') }).json()).toEqual({ ...body, limit: 25 });
|
|
});
|
|
|
|
it('paginates in a stable order and applies combined filters before the limit', async () => {
|
|
const { add, listDeployments, deploymentListPage } = await setup();
|
|
for (let i = 0; i < 130; i++) add(`done-${i}`);
|
|
add('failed', 'failed', 'restart');
|
|
add('warning', 'warning', 'restart', 't5_zombie_theater_1', '2026-01-02');
|
|
add('other-map', 'failed', 'restart', 't5_zombie_moon_1');
|
|
add('active', 'running');
|
|
add('deleted-active', 'running', 'create', 'retired', '2026-01-02');
|
|
add('old-but-inserted-last', 'done', 'create', 'old', null, 'Recorded', '2025-01-01');
|
|
expect(listDeployments()).toHaveLength(25);
|
|
const page = deploymentListPage({ view: 'history', limit: 100 });
|
|
expect(page.total).toBe(135);
|
|
expect(page.deployments).toHaveLength(100);
|
|
expect(page.deployments[0].id).toBe('deleted-active');
|
|
const next = deploymentListPage({ view: 'history', limit: 100, offset: 100 });
|
|
expect(next.deployments).toHaveLength(35);
|
|
expect(new Set([...page.deployments, ...next.deployments].map(row => row.id)).size).toBe(135);
|
|
expect(next.deployments.at(-1)?.id).toBe('old-but-inserted-last');
|
|
expect(page.deployments[0]).not.toHaveProperty('observation_json');
|
|
expect(page.deployments[0]).not.toHaveProperty('phase_key');
|
|
const { GET } = await import('../../src/routes/api/provisioning/+server.js');
|
|
const filtered = await GET({ url: new URL('http://localhost/api/provisioning?view=history&action=restart&status=attention&server=KINO&limit=1') }).json();
|
|
expect(filtered.total).toBe(2);
|
|
expect(filtered.needsReview).toBe(3);
|
|
expect(filtered.deployments).toEqual([expect.objectContaining({ id: 'warning', server_exists: 0 })]);
|
|
expect(deploymentListPage({ view: 'history', server: 'Lobby 1', action: 'restart' }).total).toBe(3);
|
|
expect(deploymentListPage({ view: 'history', server: "%' OR 1=1 --" }).total).toBe(0);
|
|
});
|
|
|
|
it('bounds malformed and oversized pagination parameters at the API', async () => {
|
|
const { add } = await setup();
|
|
for (let i = 0; i < 120; i++) add(String(i));
|
|
const { GET } = await import('../../src/routes/api/provisioning/+server.js');
|
|
for (const [query, limit, offset] of [
|
|
['limit=999999&offset=-5', 100, 0],
|
|
['limit=Infinity&offset=NaN', 25, 0],
|
|
['limit=-1&offset=1.9', 1, 1],
|
|
['limit=0&offset=Infinity', 25, 0]
|
|
] as const) {
|
|
const body = await GET({ url: new URL(`http://localhost/api/provisioning?view=history&${query}`) }).json();
|
|
expect(body).toMatchObject({ limit, offset, total: 120 });
|
|
expect(body.deployments).toHaveLength(limit);
|
|
}
|
|
});
|
|
|
|
it('uses indexes for active reads, SQL review counts, and history ordering', async () => {
|
|
const { handle, add, deploymentListPage, REMOVING_SERVER } = await setup();
|
|
for (let i = 0; i < 200; i++) add(String(i));
|
|
const prepare = vi.spyOn(handle, 'prepare');
|
|
deploymentListPage({ view: 'active' });
|
|
deploymentListPage({ view: 'history', limit: 100 });
|
|
const queries = prepare.mock.calls.map(([sql]) => sql);
|
|
prepare.mockRestore();
|
|
const explain = (sql: string, bindings: (string | number)[]) =>
|
|
(handle.prepare(`EXPLAIN QUERY PLAN ${sql}`).all(...bindings) as { detail: string }[]).map(row => row.detail).join('\n');
|
|
const active = queries.find(sql => sql.includes('AS canCancelRemoval') && !sql.includes('LIMIT'))!;
|
|
const history = queries.find(sql => sql.includes('AS canCancelRemoval') && sql.includes('LIMIT'))!;
|
|
const review = queries.find(sql => sql.includes('COUNT(*) AS needsReview'))!;
|
|
expect(explain(active, [REMOVING_SERVER])).toContain('idx_server_deployments_state');
|
|
expect(explain(review, [])).toContain('idx_server_deployments_state');
|
|
const historyPlan = explain(history, [REMOVING_SERVER, 100, 0]);
|
|
expect(historyPlan).toContain('idx_server_deployments_started_at');
|
|
expect(historyPlan).not.toContain('TEMP B-TREE');
|
|
});
|
|
});
|