358 lines
19 KiB
TypeScript
358 lines
19 KiB
TypeScript
import { createHash } from 'node:crypto';
|
|
import { join } from 'node:path';
|
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import type { GameLibraryVersion } from '../../src/lib/game-files.js';
|
|
import type { HostInventory } from '../../src/lib/hosts.js';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
|
|
let fixture: Fixture;
|
|
let hosts: typeof import('../../src/lib/server/hosts/index.js');
|
|
let libraries: typeof import('../../src/lib/server/game-files/index.js');
|
|
let preparation: typeof import('../../src/lib/server/hosts/preparation.js');
|
|
const runtime = 'ghcr.io/example/plutainer:provisioning-test';
|
|
const initialTime = Date.parse('2026-10-04T12:00:00Z');
|
|
const content = 'a'.repeat(100);
|
|
const inventory: HostInventory = {
|
|
hostname: 'game-host', protocolVersion: 1, dockerRunning: true, directoriesWritable: true,
|
|
storageAvailable: 100 * 1024 ** 3, runtimeImages: [], libraries: [], capabilities: ['host.decommission']
|
|
};
|
|
|
|
beforeEach(async () => {
|
|
vi.resetModules();
|
|
fixture = await newFixture([]);
|
|
vi.stubEnv('PLUTAINER_IMAGE', runtime);
|
|
vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'game-files'));
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
vi.setSystemTime(initialTime);
|
|
hosts = await import('../../src/lib/server/hosts/index.js');
|
|
libraries = await import('../../src/lib/server/game-files/index.js');
|
|
preparation = await import('../../src/lib/server/hosts/preparation.js');
|
|
});
|
|
|
|
afterEach(async () => {
|
|
vi.useRealTimers();
|
|
vi.unstubAllEnvs();
|
|
await fixture.cleanup();
|
|
});
|
|
|
|
function enroll(overrides: HostInventory = {}) {
|
|
const host = hosts.createHost({ name: 'Automatic game host' });
|
|
hosts.enrollHost(hosts.createEnrollment(host.id).token, { ...inventory, ...overrides });
|
|
return host.id;
|
|
}
|
|
|
|
function heartbeat(id: string, overrides: HostInventory = {}) {
|
|
vi.setSystemTime(Date.now() + 1);
|
|
return hosts.heartbeat(id, { ...inventory, ...overrides });
|
|
}
|
|
|
|
function operations(id: string) {
|
|
return hosts.listHostOperations(id).filter(operation => ['runtime.pull', 'library.download'].includes(operation.kind));
|
|
}
|
|
|
|
async function publish(name = 'T5 files', version = '1', game: 't5' | 't6' = 't5', bytes = content) {
|
|
vi.setSystemTime(Date.now() + 1);
|
|
const upload = libraries.createLibraryUpload({
|
|
name, version, game, files: [{ path: 'zone/game.ff', size: bytes.length, lastModified: 0 }]
|
|
});
|
|
await libraries.uploadBrowserFile(upload.id, 'zone/game.ff', {
|
|
offset: 0, size: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex')
|
|
}, new Response(bytes).body);
|
|
return libraries.publishBrowserUpload(upload.id);
|
|
}
|
|
|
|
function verified(id: string, version: GameLibraryVersion) {
|
|
libraries.updateHostLibraryCaches(id, [{
|
|
libraryVersionId: version.id, status: 'ready', manifestSha256: version.manifestSha256
|
|
}]);
|
|
}
|
|
|
|
describe('automatic host preparation', () => {
|
|
it('prepares an enrolled host through authenticated agent polls and heartbeats without manual setup requests', async () => {
|
|
const [{ POST: enrollAgent }, { GET: pollOperations }, { POST: reportOperation }, { POST: reportHeartbeat }] = await Promise.all([
|
|
import('../../src/routes/api/agent/enroll/+server.js'),
|
|
import('../../src/routes/api/agent/operations/+server.js'),
|
|
import('../../src/routes/api/agent/operations/[id]/+server.js'),
|
|
import('../../src/routes/api/agent/heartbeat/+server.js')
|
|
]);
|
|
const host = hosts.createHost({ name: 'Agent provisioning integration' });
|
|
const enrollment = await enrollAgent({ request: new Request('https://controller.test/api/agent/enroll', {
|
|
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ token: hosts.createEnrollment(host.id).token, inventory })
|
|
}) });
|
|
expect(enrollment.status).toBe(200);
|
|
const { credential } = await enrollment.json();
|
|
const request = (path: string, body?: object) => new Request(`https://controller.test/api/agent/${path}`, {
|
|
method: body ? 'POST' : 'GET',
|
|
headers: { Authorization: `Bearer ${credential}`, 'Content-Type': 'application/json' },
|
|
...(body ? { body: JSON.stringify(body) } : {})
|
|
});
|
|
expect(operations(host.id)).toEqual([]);
|
|
|
|
const firstPoll = await pollOperations({ request: request('operations') }).json();
|
|
expect(firstPoll.operations).toHaveLength(1);
|
|
const runtimeOperation = firstPoll.operations[0];
|
|
expect(runtimeOperation).toMatchObject({ kind: 'runtime.pull', payload: { image: runtime } });
|
|
expect((await pollOperations({ request: request('operations') }).json()).operations).toEqual([runtimeOperation]);
|
|
const version = await publish();
|
|
expect((await pollOperations({ request: request('operations') }).json()).operations).toEqual([runtimeOperation]);
|
|
|
|
vi.setSystemTime(Date.now() + 1);
|
|
const completedRuntime = await reportOperation({
|
|
request: request(`operations/${runtimeOperation.id}`, { state: 'completed' }), params: { id: runtimeOperation.id }
|
|
});
|
|
expect(completedRuntime.status).toBe(200);
|
|
expect((await pollOperations({ request: request('operations') }).json()).operations).toEqual([]);
|
|
expect(operations(host.id)).toHaveLength(1);
|
|
|
|
vi.setSystemTime(Date.now() + 1);
|
|
const preparedInventory = { ...inventory, runtimeImages: [runtime], libraryCaches: [] };
|
|
const runtimeHeartbeat = await reportHeartbeat({ request: request('heartbeat', { inventory: preparedInventory }) });
|
|
expect(runtimeHeartbeat.status).toBe(200);
|
|
expect(operations(host.id)).toHaveLength(2);
|
|
const download = operations(host.id)[0];
|
|
expect(download).toMatchObject({ kind: 'library.download', payload: { libraryVersionId: version.id } });
|
|
expect((await reportHeartbeat({ request: request('heartbeat', { inventory: preparedInventory }) })).status).toBe(200);
|
|
expect((await pollOperations({ request: request('operations') }).json()).operations).toEqual([download]);
|
|
expect(operations(host.id)).toHaveLength(2);
|
|
|
|
vi.setSystemTime(Date.now() + 1);
|
|
expect((await reportOperation({
|
|
request: request(`operations/${download.id}`, { state: 'completed' }), params: { id: download.id }
|
|
})).status).toBe(200);
|
|
vi.setSystemTime(Date.now() + 1);
|
|
const finalHeartbeat = await reportHeartbeat({ request: request('heartbeat', { inventory: {
|
|
...preparedInventory,
|
|
libraries: [{ id: version.id, state: 'ready' }],
|
|
libraryCaches: [{ libraryVersionId: version.id, status: 'ready', manifestSha256: version.manifestSha256 }]
|
|
} }) });
|
|
expect(finalHeartbeat.status).toBe(200);
|
|
expect(libraries.requireReadyLibrary(host.id, version.id).id).toBe(version.id);
|
|
expect(preparation.hostPreparationStatus(hosts.getHost(host.id)!).state).toBe('ready');
|
|
expect((await pollOperations({ request: request('operations') }).json()).operations).toEqual([]);
|
|
expect(operations(host.id)).toHaveLength(2);
|
|
});
|
|
|
|
it('presents preparation without queuing work and prepares the configured runtime even when another image exists', async () => {
|
|
const version = await publish();
|
|
const id = enroll({ runtimeImages: ['another:runtime'] });
|
|
const host = hosts.getHost(id)!;
|
|
const status = preparation.hostPreparationStatus(host);
|
|
expect(status.items.map(item => item.kind)).toEqual(['runtime', 'library']);
|
|
expect(preparation.presentHostPreparation(host)).toMatchObject({ id, preparation: status });
|
|
expect(operations(id)).toEqual([]);
|
|
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('preparing');
|
|
const [operation] = operations(id);
|
|
expect(operation).toMatchObject({ kind: 'runtime.pull', payload: { image: runtime }, state: 'pending' });
|
|
expect(libraries.getHostLibraryCache(id, version.id)).toBeNull();
|
|
for (let poll = 0; poll < 4; poll++) {
|
|
heartbeat(id);
|
|
preparation.reconcileHostPreparation(id);
|
|
}
|
|
hosts.reportHostOperation(id, operation.id, { state: 'unknown' });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id).map(item => item.id)).toEqual([operation.id]);
|
|
});
|
|
|
|
it('waits for fresh inventory after runtime completion before starting a library download', async () => {
|
|
const version = await publish();
|
|
const id = enroll();
|
|
preparation.reconcileHostPreparation(id);
|
|
const runtimeOperation = operations(id)[0];
|
|
vi.setSystemTime(Date.now() + 1);
|
|
hosts.reportHostOperation(id, runtimeOperation.id, { state: 'completed', result: { image: runtime } });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(1);
|
|
|
|
heartbeat(id, { runtimeImages: [runtime] });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(2);
|
|
const download = operations(id)[0];
|
|
expect(download).toMatchObject({ kind: 'library.download', payload: { libraryVersionId: version.id } });
|
|
verified(id, version);
|
|
hosts.reportHostOperation(id, download.id, { state: 'completed' });
|
|
heartbeat(id, { runtimeImages: [runtime], libraries: [{ id: version.id, state: 'ready' }] });
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('ready');
|
|
expect(operations(id)).toHaveLength(2);
|
|
});
|
|
|
|
it('downloads the newest published version of every library, including separate libraries for the same game', async () => {
|
|
const old = await publish('T5 files', 'old');
|
|
const latest = await publish('T5 files', 'new');
|
|
const alternate = await publish('T5 alternate');
|
|
const t6 = await publish('T6 files', '1', 't6');
|
|
libraries.createLibraryUpload({
|
|
name: 'T5 files', version: 'unfinished', game: 't5',
|
|
files: [{ path: 'zone/game.ff', size: content.length, lastModified: 0 }]
|
|
});
|
|
const id = enroll({ runtimeImages: [runtime] });
|
|
const expected = new Map([latest, alternate, t6].map(version => [version.id, version]));
|
|
const downloaded: string[] = [];
|
|
for (let index = 0; index < expected.size; index++) {
|
|
preparation.reconcileHostPreparation(id);
|
|
const active = operations(id).filter(operation => !['completed', 'failed'].includes(operation.state));
|
|
expect(active).toHaveLength(1);
|
|
const operation = active[0];
|
|
const versionId = String(operation.payload.libraryVersionId);
|
|
expect(expected.has(versionId)).toBe(true);
|
|
expect(downloaded).not.toContain(versionId);
|
|
downloaded.push(versionId);
|
|
verified(id, expected.get(versionId)!);
|
|
vi.setSystemTime(Date.now() + 1);
|
|
hosts.reportHostOperation(id, operation.id, { state: 'completed' });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(index + 1);
|
|
heartbeat(id, { runtimeImages: [runtime], libraries: downloaded.map(id => ({ id, state: 'ready' })) });
|
|
}
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('ready');
|
|
expect(downloaded).toHaveLength(expected.size);
|
|
expect(downloaded).not.toContain(old.id);
|
|
});
|
|
|
|
it('picks up publication after enrollment and keeps verified historical versions intact', async () => {
|
|
const id = enroll({ runtimeImages: [runtime] });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toEqual([]);
|
|
const old = await publish();
|
|
verified(id, old);
|
|
heartbeat(id, { runtimeImages: [runtime], libraries: [{ id: old.id, state: 'ready' }] });
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('ready');
|
|
const latest = await publish('T5 files', '2');
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(1);
|
|
expect(operations(id)[0].payload.libraryVersionId).toBe(latest.id);
|
|
expect(libraries.requireReadyLibrary(id, old.id).id).toBe(old.id);
|
|
expect(libraries.getHostLibraryCache(id, old.id)?.status).toBe('ready');
|
|
});
|
|
|
|
it.each(['draining', 'disabled'] as const)('does not start setup for a %s host', mode => {
|
|
const id = enroll();
|
|
hosts.updateHost(id, { mode });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toEqual([]);
|
|
});
|
|
|
|
it.each([
|
|
{ name: 'incompatible protocol', value: { protocolVersion: 2 } },
|
|
{ name: 'unavailable Docker', value: { dockerRunning: false } },
|
|
{ name: 'unwritable managed directories', value: { directoriesWritable: false } },
|
|
{ name: 'insufficient storage reserve', value: { storageAvailable: 0 } }
|
|
])('does not start setup with $name', ({ value }) => {
|
|
const id = enroll(value);
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('blocked');
|
|
expect(operations(id)).toEqual([]);
|
|
});
|
|
|
|
it('does not start setup for unenrolled, disconnected, revoked, or decommissioning hosts', () => {
|
|
const unenrolled = hosts.createHost({ name: 'Not enrolled' }).id;
|
|
preparation.reconcileHostPreparation(unenrolled);
|
|
expect(operations(unenrolled)).toEqual([]);
|
|
const offline = enroll();
|
|
vi.setSystemTime(Date.now() + 91_000);
|
|
preparation.reconcileHostPreparation(offline);
|
|
expect(operations(offline)).toEqual([]);
|
|
const revoked = enroll();
|
|
hosts.revokeCredentials(revoked);
|
|
preparation.reconcileHostPreparation(revoked);
|
|
expect(operations(revoked)).toEqual([]);
|
|
const deleting = enroll();
|
|
hosts.deleteHost(deleting);
|
|
preparation.reconcileHostPreparation(deleting);
|
|
expect(operations(deleting)).toEqual([]);
|
|
});
|
|
|
|
it('retries runtime failures with exponential delays capped at fifteen minutes', () => {
|
|
const id = enroll();
|
|
preparation.reconcileHostPreparation(id);
|
|
for (const [attempt, delay] of [60_000, 120_000, 240_000, 480_000, 900_000, 900_000].entries()) {
|
|
const operation = operations(id)[0];
|
|
vi.setSystemTime(Date.now() + 1);
|
|
hosts.reportHostOperation(id, operation.id, { state: 'failed', error: 'Registry unavailable' });
|
|
const failedAt = Date.now();
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(attempt + 1);
|
|
vi.setSystemTime(failedAt + delay - 2);
|
|
heartbeat(id);
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(attempt + 1);
|
|
vi.setSystemTime(failedAt + delay);
|
|
heartbeat(id);
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(attempt + 2);
|
|
expect(operations(id)[0]).toMatchObject({ kind: 'runtime.pull', state: 'pending' });
|
|
}
|
|
});
|
|
|
|
it('blocks a library that cannot fit while preserving the host free-space reserve', async () => {
|
|
const version = await publish();
|
|
const id = enroll({ runtimeImages: [runtime], storageAvailable: 1_000 + version.sizeBytes - 1 });
|
|
hosts.updateHost(id, { minimumFreeBytes: 1_000 });
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('blocked');
|
|
expect(operations(id)).toEqual([]);
|
|
heartbeat(id, { runtimeImages: [runtime], storageAvailable: 1_000 + version.sizeBytes });
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('preparing');
|
|
expect(operations(id)[0].payload.libraryVersionId).toBe(version.id);
|
|
});
|
|
|
|
it('returns an active download even when its in-progress bytes have consumed free space', async () => {
|
|
const version = await publish();
|
|
const id = enroll({ runtimeImages: [runtime] });
|
|
const operation = libraries.queueLibraryDownload(id, version.id);
|
|
heartbeat(id, { runtimeImages: [runtime], storageAvailable: hosts.getHost(id)!.minimumFreeBytes });
|
|
expect(libraries.queueLibraryDownload(id, version.id).id).toBe(operation.id);
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(1);
|
|
});
|
|
|
|
it('lets other libraries prepare when an earlier library cannot fit', async () => {
|
|
const oversized = await publish('A oversized');
|
|
const small = await publish('B small', '1', 't6', 'small');
|
|
const id = enroll({ runtimeImages: [runtime], storageAvailable: 1_010 });
|
|
hosts.updateHost(id, { minimumFreeBytes: 1_000 });
|
|
const status = preparation.reconcileHostPreparation(id);
|
|
expect(status.items).toEqual(expect.arrayContaining([
|
|
expect.objectContaining({ label: 'A oversized · 1', state: 'blocked' }),
|
|
expect.objectContaining({ label: 'B small · 1', state: 'preparing' })
|
|
]));
|
|
expect(operations(id)).toHaveLength(1);
|
|
expect(operations(id)[0].payload.libraryVersionId).toBe(small.id);
|
|
expect(libraries.getHostLibraryCache(id, oversized.id)).toBeNull();
|
|
});
|
|
|
|
it('does not count partial bytes from a different manifest toward download storage', async () => {
|
|
const version = await publish();
|
|
const id = enroll({ runtimeImages: [runtime] });
|
|
libraries.updateHostLibraryCaches(id, [{
|
|
libraryVersionId: version.id, status: 'failed', bytesDownloaded: 80, manifestSha256: '0'.repeat(64)
|
|
}]);
|
|
heartbeat(id, { runtimeImages: [runtime], storageAvailable: hosts.getHost(id)!.minimumFreeBytes + 20 });
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('blocked');
|
|
expect(() => libraries.queueLibraryDownload(id, version.id)).toThrow(/insufficient storage/);
|
|
expect(operations(id)).toEqual([]);
|
|
});
|
|
|
|
it('resumes a failed verified-manifest download using only its remaining storage', async () => {
|
|
const version = await publish();
|
|
const id = enroll({ runtimeImages: [runtime] });
|
|
preparation.reconcileHostPreparation(id);
|
|
const operation = operations(id)[0];
|
|
libraries.updateHostLibraryCaches(id, [{
|
|
libraryVersionId: version.id, status: 'downloading', bytesDownloaded: 80, manifestSha256: version.manifestSha256
|
|
}]);
|
|
vi.setSystemTime(Date.now() + 1);
|
|
hosts.reportHostOperation(id, operation.id, { state: 'failed', error: 'Connection reset' });
|
|
const failedAt = Date.now();
|
|
const free = hosts.getHost(id)!.minimumFreeBytes + version.sizeBytes - 80;
|
|
heartbeat(id, { runtimeImages: [runtime], storageAvailable: free });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(1);
|
|
vi.setSystemTime(failedAt + 60_000);
|
|
heartbeat(id, { runtimeImages: [runtime], storageAvailable: free });
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('preparing');
|
|
expect(operations(id)).toHaveLength(2);
|
|
expect(operations(id)[0].payload.libraryVersionId).toBe(version.id);
|
|
expect(libraries.getHostLibraryCache(id, version.id)?.bytesDownloaded).toBe(80);
|
|
});
|
|
});
|