Files
electric-cherry/tests/unit/provisioning-list.test.ts
2026-10-12 15:52:34 -04:00

678 lines
28 KiB
TypeScript

import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { newFixture, type Fixture } from '../helpers/fixture.js';
import {
provisioningListSources,
provisioningMatchesSource,
provisioningSourceLabel,
type ProvisioningListSource
} from '../../src/lib/provisioning-list.js';
let fixture: Fixture;
beforeEach(async () => {
vi.resetModules();
fixture = await newFixture([]);
});
afterEach(async () => {
vi.restoreAllMocks();
await fixture.cleanup();
});
async function setup() {
const { db } = await import('../../src/lib/server/db/index.js');
const history = await import('../../src/lib/server/provisioning/history.js');
const handle = db();
const insert = handle.prepare(`INSERT INTO server_deployments
(id,server,action,state,stage,started_at,triggered_by,server_deleted_at,observation_json)
VALUES(?,?,?,?,?,?,'fixture',?,?)`);
const add = (
id: string,
state = 'done',
action = 'create',
server = 't5_zombie_theater_1',
deleted: string | null = null,
stage = 'Recorded',
at = '2026-01-01'
) =>
insert.run(
id,
server,
action,
state,
stage,
at,
deleted,
JSON.stringify({ snapshot: 'x'.repeat(4096) })
);
return { ...history, handle, add };
}
describe('bounded provisioning lists', () => {
it('filters Queued and Waiting independently while preserving combined pending links', async () => {
const { add, handle } = await setup();
add('queued', 'queued');
add('waiting', 'waiting', 'restart');
add('running', 'running', 'restart');
handle.prepare("UPDATE server_deployments SET phase_key='prepare' WHERE id='queued'").run();
const { GET } = await import('../../src/routes/api/provisioning/+server.js');
for (const [status, expected] of [
['queued', ['queued']],
['waiting', ['waiting']],
['running', ['running']],
['pending', ['waiting', 'queued']]
] as const) {
const result = await GET({
url: new URL(`http://localhost/api/provisioning?view=active&status=${status}`)
}).json();
expect(result.deployments.map((row: { id: string }) => row.id)).toEqual(expected);
}
});
it('keeps recovery sources concise as incident summaries change, including after retention', async () => {
const { add, handle, deploymentListPage } = await setup();
add('recovery', 'waiting', 'rebuild');
handle
.prepare(
"INSERT INTO provisioning_sources VALUES ('recovery','watchdog','incident','t5_zombie_theater_1')"
)
.run();
handle
.prepare(
`INSERT INTO watchdog_incidents (id,server,kind,state,summary,first_seen_at,last_seen_at)
VALUES ('incident','t5_zombie_theater_1','health','recovering',?,'2026-01-01','2026-01-01')`
)
.run('Rebuild queued with the latest promoted setup');
for (const summary of [
'Rebuild queued with the latest promoted setup',
'Recovered game service'
]) {
handle
.prepare('UPDATE watchdog_incidents SET summary=? WHERE id=?')
.run(summary, 'incident');
const row = deploymentListPage({ view: 'active' }).deployments[0];
expect(provisioningSourceLabel(row)).toBe('Recovery');
expect(row.sources).toEqual([
{
kind: 'watchdog',
source_id: 'incident',
label: summary,
href: '/maintenance/incident'
}
]);
}
handle.prepare("DELETE FROM watchdog_incidents WHERE id='incident'").run();
const retained = deploymentListPage({ view: 'active' }).deployments[0];
expect(provisioningSourceLabel(retained)).toBe('Recovery');
expect(retained.sources?.[0].href).toBeNull();
});
it.each(['active', 'history'] as const)(
'keeps %s source categories and client filters aligned with API filters',
async (view) => {
const { add, handle, deploymentListPage } = await setup();
const categories: [ProvisioningListSource, string, string][] = [
['campaign', 'campaign', 'Campaign'],
['beta', 'beta', 'Beta'],
['maintenance', 'maintenance', 'Maintenance'],
['recovery', 'watchdog', 'Recovery'],
['autoscaling', 'autoscaling', 'Autoscaling'],
['rebalance', 'rebalance', 'Rebalance']
];
const state = view === 'active' ? 'waiting' : 'done';
for (const [source, kind] of categories) {
// Rebalance ownership is recorded only in triggered_by.
if (source !== 'rebalance') {
add(`${source}-linked`, state);
handle
.prepare('INSERT INTO provisioning_sources VALUES (?,?,?,?)')
.run(`${source}-linked`, kind, 'expired', 't5_zombie_theater_1');
}
add(`${source}-legacy`, state);
handle
.prepare('UPDATE server_deployments SET triggered_by=? WHERE id=?')
.run(`${kind}:expired`, `${source}-legacy`);
}
add('manual', state);
handle
.prepare("UPDATE server_deployments SET triggered_by='ui' WHERE id='manual'")
.run();
add('system', state);
// A shared operation must be discoverable through each of its sources, even when initiated manually.
add('shared', state);
handle
.prepare("UPDATE server_deployments SET triggered_by='ui' WHERE id='shared'")
.run();
handle.exec(`INSERT INTO provisioning_sources VALUES ('shared','campaign','expired','t5_zombie_theater_1');
INSERT INTO provisioning_sources VALUES ('shared','watchdog','expired','t5_zombie_theater_1');`);
const rows = deploymentListPage({ view }).deployments;
expect(rows).toHaveLength(14);
for (const [source, , label] of categories) {
for (const suffix of source === 'rebalance' ? ['legacy'] : ['linked', 'legacy']) {
expect(
provisioningSourceLabel(
rows.find((row) => row.id === `${source}-${suffix}`)!
)
).toBe(label);
}
}
expect(provisioningSourceLabel(rows.find((row) => row.id === 'manual')!)).toBe(
'Manual'
);
expect(provisioningSourceLabel(rows.find((row) => row.id === 'system')!)).toBe(
'System'
);
const { GET } = await import('../../src/routes/api/provisioning/+server.js');
for (const source of provisioningListSources) {
const expected =
source === 'manual' || source === 'system'
? [source]
: source === 'rebalance'
? ['rebalance-legacy']
: [
`${source}-linked`,
`${source}-legacy`,
...(['campaign', 'recovery'].includes(source) ? ['shared'] : [])
];
expect(
rows
.filter((row) => provisioningMatchesSource(row, source))
.map((row) => row.id)
.sort()
).toEqual(expected.sort());
const result = await GET({
url: new URL(`http://localhost/api/provisioning?view=${view}&source=${source}`)
}).json();
expect(result.total).toBe(expected.length);
expect(result.deployments.map((row: { id: string }) => row.id).sort()).toEqual(
expected
);
if (view === 'history') {
const page = deploymentListPage({
view,
source,
limit: 1,
offset: expected.length - 1
});
expect(page.total).toBe(expected.length);
expect(page.deployments).toHaveLength(1);
expect(provisioningMatchesSource(page.deployments[0], source)).toBe(true);
}
}
}
);
it('filters readable sources before pagination and retains expired source references without broken links', async () => {
const { add, handle, deploymentListPage } = await setup();
handle.exec(`INSERT INTO scripts(id,name,target_rel_path) VALUES(1,'Common scripts','common.gsc');
INSERT INTO script_versions(id,script_id,version,sha256,size) VALUES(1,1,1,'hash',0);
INSERT INTO betas(id,script_id,script_version_id,trigger_kind,state,notes) VALUES(75,1,1,'ui','promoted','Gameplay checks');
INSERT INTO deployments(id,action,started_at,beta_id) VALUES('campaign','promote','2026-01-01',75);`);
add('old-campaign', 'done', 'restart');
handle
.prepare(
"UPDATE server_deployments SET triggered_by='campaign:campaign' WHERE id='old-campaign'"
)
.run();
add('linked-beta', 'waiting', 'restart');
handle
.prepare(
"INSERT INTO provisioning_sources(operation_id,kind,source_id,server) VALUES('linked-beta','beta','75','t5_zombie_theater_1')"
)
.run();
add('expired', 'done', 'restart');
handle
.prepare(
"UPDATE server_deployments SET triggered_by='campaign:expired' WHERE id='expired'"
)
.run();
add('manual', 'done', 'restart');
handle.prepare("UPDATE server_deployments SET triggered_by='ui' WHERE id='manual'").run();
for (let i = 0; i < 30; i++) add(`system-${i}`, 'done', 'restart');
const first = deploymentListPage({
view: 'history',
source: 'campaign',
action: 'restart',
status: 'done',
server: 'Kino',
limit: 1
});
expect(first.total).toBe(2);
expect(first.deployments[0]).toMatchObject({
id: 'expired',
sources: [{ label: 'Campaign unavailable', href: null }]
});
const second = deploymentListPage({
view: 'history',
source: 'campaign',
limit: 1,
offset: 1
});
expect(second.deployments[0]).toMatchObject({
id: 'old-campaign',
sources: [{ label: 'Promotion · Beta #75', href: '/campaigns/campaign' }]
});
const beta = deploymentListPage({ view: 'active', source: 'beta', status: 'pending' });
expect(beta.deployments).toEqual([
expect.objectContaining({
id: 'linked-beta',
sources: [
{
kind: 'beta',
source_id: '75',
label: 'Beta #75: Gameplay checks',
href: '/betas/75'
}
]
})
]);
expect(provisioningSourceLabel(second.deployments[0])).toBe('Campaign');
expect(provisioningSourceLabel(beta.deployments[0])).toBe('Beta');
expect(
deploymentListPage({ view: 'history', source: 'manual' }).deployments.map(
(row) => row.id
)
).toEqual(['manual']);
expect(deploymentListPage({ view: 'history', source: 'system' }).total).toBe(30);
const { GET } = await import('../../src/routes/api/provisioning/+server.js');
const result = await GET({
url: new URL('http://localhost/api/provisioning?view=active&status=pending&source=beta')
}).json();
expect(result.deployments.map((row: { id: string }) => row.id)).toEqual(['linked-beta']);
});
it('counts unresolved outcomes while retaining resolved failures and routine cancellations in history', async () => {
const { add, handle, deploymentListPage } = await setup();
add('failed', 'failed');
add('resolved', 'failed');
add('cancelled', 'cancelled');
add('warning', 'warning');
handle
.prepare(
"INSERT INTO retention_controls(category,record_id,resolved_at) VALUES('provisioning','resolved','2026-01-02')"
)
.run();
expect(deploymentListPage({ view: 'history' }).total).toBe(4);
const attention = deploymentListPage({ view: 'history', status: 'attention' });
expect(attention.needsReview).toBe(2);
expect(attention.deployments.map((row) => row.id)).toEqual(['warning', 'failed']);
expect(deploymentListPage({ view: 'history', status: 'failed' }).total).toBe(2);
handle
.prepare("UPDATE server_deployments SET finished_at='2026-01-03' WHERE id='resolved'")
.run();
expect(deploymentListPage({ view: 'history', status: 'attention' }).needsReview).toBe(3);
});
it('exposes compact waiting facts without transferring observation snapshots', async () => {
const { add, handle, deploymentListPage } = await setup();
add('waiting', 'waiting', 'restart');
handle
.prepare(
"UPDATE server_deployments SET phase_key='eligibility',phase_started_at='2026-01-01',observation_json=? WHERE id='waiting'"
)
.run(
JSON.stringify({
blocker: {
code: 'restart-capacity',
message: 'Waiting for another server to recover'
},
checks: ['private snapshot'],
checkpoints: []
})
);
expect(deploymentListPage({ view: 'active' }).deployments[0]).toMatchObject({
waitCode: 'restart-capacity',
waitMessage: 'Waiting for another server to recover',
phase: 'eligibility',
phaseStartedAt: '2026-01-01'
});
expect(JSON.stringify(deploymentListPage({ view: 'active' }))).not.toContain(
'private snapshot'
);
});
it('orders active jobs by lifecycle state regardless of phases and blockers', async () => {
const { add, handle, listDeployments, deploymentListPage } = await setup();
const addProgress = (
id: string,
state: string,
second: number,
phase: string,
waitCode: string | null = null,
stage = 'Recorded',
action = 'rebuild'
) => {
add(
id,
state,
action,
't5_zombie_theater_1',
null,
stage,
`2026-01-01T00:00:${String(second).padStart(2, '0')}.000Z`
);
handle
.prepare('UPDATE server_deployments SET phase_key=?, observation_json=? WHERE id=?')
.run(phase, JSON.stringify({ blocker: waitCode ? { code: waitCode } : null }), id);
};
// Request order is deliberately different from progress order.
addProgress('queued', 'queued', 1, 'requested');
addProgress(
'cleanup',
'waiting',
12,
'cleanup',
null,
'Removing server files and registry',
'destroy'
);
addProgress('verify-newer', 'awaiting-health', 21, 'verify-map');
addProgress('capacity', 'running', 2, 'startup-queue', 'startup-capacity');
addProgress('operation-conflict', 'running', 6, 'container-action', 'operation-conflict');
addProgress(
'agent-wait',
'running',
11,
'container-action',
'native-host',
'Agent operation pending'
);
addProgress(
'host-wait',
'waiting',
5,
'eligibility',
'native-host',
'Host is unavailable; waiting for reconnection.'
);
addProgress(
'staging',
'waiting',
7,
'eligibility',
'native-host',
'Staging and verifying files on the host.'
);
addProgress(
'staging-wait',
'waiting',
8,
'eligibility',
'native-host',
'Waiting for verified remote file staging.'
);
addProgress(
'legacy-capacity',
'waiting',
3,
'eligibility',
'native-host',
'Queued to rebuild; waiting for another server to recover'
);
addProgress(
'verify-older',
'awaiting-health',
20,
'verify-container',
'native-host',
'Waiting for container readiness: starting'
);
addProgress('preparing-queued', 'queued', 9, 'prepare');
addProgress('executing', 'running', 10, 'prepare');
addProgress('eligibility', 'running', 4, 'eligibility', 'eligibility');
addProgress('interrupted', 'interrupted', 30, 'container-action', 'native-host');
addProgress('failed', 'failed', 31, 'verify-map', 'native-host');
const expected = [
'verify-older',
'verify-newer',
'capacity',
'eligibility',
'operation-conflict',
'executing',
'agent-wait',
'legacy-capacity',
'host-wait',
'staging',
'staging-wait',
'cleanup',
'queued',
'preparing-queued'
];
expect(listDeployments({ view: 'active' }).map((row) => row.id)).toEqual(expected);
const { GET } = await import('../../src/routes/api/provisioning/+server.js');
const active = await GET({
url: new URL('http://localhost/api/provisioning?view=active&limit=1')
}).json();
expect(active.deployments.map((row: { id: string }) => row.id)).toEqual(expected);
expect(active).toMatchObject({ total: expected.length, needsReview: 2 });
// State priority applies only to Active; retained outcomes and mixed reads stay newest first.
expect(
deploymentListPage({ view: 'history', limit: 1 }).deployments.map((row) => row.id)
).toEqual(['failed']);
expect(
deploymentListPage({ view: 'history', limit: 1, offset: 1 }).deployments.map(
(row) => row.id
)
).toEqual(['interrupted']);
expect(listDeployments({ view: 'all', limit: 3 }).map((row) => row.id)).toEqual([
'failed',
'interrupted',
'verify-newer'
]);
});
it('keeps oldest requests and insertion ties stable while active progress is polled', async () => {
const { add, handle, listDeployments } = await setup();
add(
'z-first',
'awaiting-health',
'rebuild',
't5_zombie_theater_1',
null,
'Checking game status',
'2026-01-02'
);
add(
'a-second',
'awaiting-health',
'rebuild',
't5_zombie_theater_1',
null,
'Checking selected engine map',
'2026-01-02'
);
add(
'old-inserted-last',
'awaiting-health',
'rebuild',
't5_zombie_theater_1',
null,
'Checking container state',
'2026-01-01'
);
add(
'queued',
'queued',
'rebuild',
't5_zombie_theater_1',
null,
'Request recorded',
'2025-12-31'
);
const ids = () => listDeployments({ view: 'active' }).map((row) => row.id);
const expected = ['old-inserted-last', 'z-first', 'a-second', 'queued'];
expect(ids()).toEqual(expected);
expect(ids()).toEqual(expected);
handle
.prepare(
`UPDATE server_deployments SET phase_key='verify-container', phase_started_at='2026-01-03',
observed_at='2026-01-03', observation_json=? WHERE id='z-first'`
)
.run(
JSON.stringify({
blocker: {
code: 'native-host',
message: 'Waiting for container readiness: starting'
}
})
);
handle
.prepare(
"UPDATE server_deployments SET phase_key='verify-plutonium', phase_started_at='2026-01-04' WHERE id='a-second'"
)
.run();
expect(ids()).toEqual(expected);
handle
.prepare(
"UPDATE server_deployments SET state='done', finished_at='2026-01-05' WHERE id='z-first'"
)
.run();
handle
.prepare(
"UPDATE server_deployments SET state='awaiting-health', phase_key='verify-game' WHERE id='queued'"
)
.run();
expect(ids()).toEqual(['queued', 'old-inserted-last', 'a-second']);
expect(listDeployments({ view: 'all' }).map((row) => row.id)).toEqual([
'a-second',
'z-first',
'old-inserted-last',
'queued'
]);
});
it('polls every active row and counts review outcomes without transferring retained history', async () => {
const { handle, add, REMOVING_SERVER } = await setup();
handle.transaction(() => {
for (let i = 0; i < 6_000; i++) add(`history-${i}`, i % 2 ? 'done' : 'failed');
for (const state of ['queued', 'awaiting-health', 'running', 'waiting'])
add(state, state);
for (let i = 0; i < 110; i++) add(`active-${i}`, 'waiting', 'destroy');
add('cleanup', 'waiting', 'destroy', 'cleanup', null, REMOVING_SERVER);
add('deleted-active', 'running', 'create', 'retired', '2026-01-02');
add('deleted-warning', 'warning', 'create', 'retired', '2026-01-02');
add('interrupted', 'interrupted');
})();
const { GET } = await import('../../src/routes/api/provisioning/+server.js');
const response = GET({
url: new URL('http://localhost/api/provisioning?view=active&limit=1')
});
const body = await response.json();
expect(body.total).toBe(115);
expect(body.deployments).toHaveLength(115);
expect(body.needsReview).toBe(3_002);
expect(body.deployments.map((row: { id: string }) => row.id)).not.toContain(
'deleted-active'
);
expect(
body.deployments.find((row: { id: string }) => row.id === 'active-0').canCancelRemoval
).toBe(1);
expect(
body.deployments.find((row: { id: string }) => row.id === 'cleanup').canCancelRemoval
).toBe(0);
expect(JSON.stringify(body)).not.toContain('observation_json');
expect(Buffer.byteLength(JSON.stringify(body))).toBeLessThan(60_000);
expect(response.headers.get('cache-control')).toBe('no-store');
const history = await GET({
url: new URL('http://localhost/api/provisioning?view=history&limit=100')
}).json();
expect(history.deployments).toHaveLength(100);
expect(history.total).toBe(6_003);
expect(Buffer.byteLength(JSON.stringify(history))).toBeLessThan(50_000);
// The legacy endpoint must not reintroduce the unbounded response.
const { GET: legacyGET } = await import('../../src/routes/api/deployments/+server.js');
expect(
await legacyGET({ url: new URL('http://localhost/api/deployments') }).json()
).toEqual({ ...body, limit: 25 });
});
it('paginates in a stable order and applies combined filters before the limit', async () => {
const { add, listDeployments, deploymentListPage } = await setup();
for (let i = 0; i < 130; i++) add(`done-${i}`);
add('failed', 'failed', 'restart');
add('warning', 'warning', 'restart', 't5_zombie_theater_1', '2026-01-02');
add('other-map', 'failed', 'restart', 't5_zombie_moon_1');
add('active', 'running');
add('deleted-active', 'running', 'create', 'retired', '2026-01-02');
add('old-but-inserted-last', 'done', 'create', 'old', null, 'Recorded', '2025-01-01');
expect(listDeployments()).toHaveLength(25);
const page = deploymentListPage({ view: 'history', limit: 100 });
expect(page.total).toBe(135);
expect(page.deployments).toHaveLength(100);
expect(page.deployments[0].id).toBe('deleted-active');
const next = deploymentListPage({ view: 'history', limit: 100, offset: 100 });
expect(next.deployments).toHaveLength(35);
expect(new Set([...page.deployments, ...next.deployments].map((row) => row.id)).size).toBe(
135
);
expect(next.deployments.at(-1)?.id).toBe('old-but-inserted-last');
expect(page.deployments[0]).not.toHaveProperty('observation_json');
expect(page.deployments[0]).not.toHaveProperty('phase_key');
const { GET } = await import('../../src/routes/api/provisioning/+server.js');
const filtered = await GET({
url: new URL(
'http://localhost/api/provisioning?view=history&action=restart&status=attention&server=KINO&limit=1'
)
}).json();
expect(filtered.total).toBe(2);
expect(filtered.needsReview).toBe(3);
expect(filtered.deployments).toEqual([
expect.objectContaining({ id: 'warning', server_exists: 0 })
]);
expect(
deploymentListPage({ view: 'history', server: 'Lobby 1', action: 'restart' }).total
).toBe(3);
expect(deploymentListPage({ view: 'history', server: "%' OR 1=1 --" }).total).toBe(0);
});
it('bounds malformed and oversized pagination parameters at the API', async () => {
const { add } = await setup();
for (let i = 0; i < 120; i++) add(String(i));
const { GET } = await import('../../src/routes/api/provisioning/+server.js');
for (const [query, limit, offset] of [
['limit=999999&offset=-5', 100, 0],
['limit=Infinity&offset=NaN', 25, 0],
['limit=-1&offset=1.9', 1, 1],
['limit=0&offset=Infinity', 25, 0]
] as const) {
const body = await GET({
url: new URL(`http://localhost/api/provisioning?view=history&${query}`)
}).json();
expect(body).toMatchObject({ limit, offset, total: 120 });
expect(body.deployments).toHaveLength(limit);
}
});
it('uses indexes for active reads, SQL review counts, and history ordering', async () => {
const { handle, add, deploymentListPage, REMOVING_SERVER } = await setup();
for (let i = 0; i < 200; i++) add(String(i));
const prepare = vi.spyOn(handle, 'prepare');
deploymentListPage({ view: 'active' });
deploymentListPage({ view: 'history', limit: 100 });
const queries = prepare.mock.calls.map(([sql]) => sql);
prepare.mockRestore();
const explain = (sql: string, bindings: (string | number)[]) =>
(handle.prepare(`EXPLAIN QUERY PLAN ${sql}`).all(...bindings) as { detail: string }[])
.map((row) => row.detail)
.join('\n');
const active = queries.find(
(sql) => sql.includes('AS canCancelRemoval') && !sql.includes('LIMIT')
)!;
const history = queries.find(
(sql) => sql.includes('AS canCancelRemoval') && sql.includes('LIMIT')
)!;
const review = queries.find((sql) => sql.includes('COUNT(*) AS needsReview'))!;
expect(explain(active, [REMOVING_SERVER])).toContain('idx_server_deployments_state');
expect(explain(review, [])).toContain('idx_server_deployments_state');
const historyPlan = explain(history, [REMOVING_SERVER, 100, 0]);
expect(historyPlan).toContain('idx_server_deployments_started_at');
expect(historyPlan).not.toContain('TEMP B-TREE');
});
});