546 lines
30 KiB
TypeScript
546 lines
30 KiB
TypeScript
import { createHash } from 'node:crypto';
|
|
import { join } from 'node:path';
|
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import type { GameLibraryVersion } from '../../src/lib/game-files.js';
|
|
import type { HostInventory } from '../../src/lib/hosts.js';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
|
|
let fixture: Fixture;
|
|
let hosts: typeof import('../../src/lib/server/hosts/index.js');
|
|
let libraries: typeof import('../../src/lib/server/game-files/index.js');
|
|
let preparation: typeof import('../../src/lib/server/hosts/preparation.js');
|
|
const runtime = 'ghcr.io/example/plutainer:provisioning-test';
|
|
const initialTime = Date.parse('2026-10-04T12:00:00Z');
|
|
const content = 'a'.repeat(100);
|
|
const inventory: HostInventory = {
|
|
hostname: 'game-host', protocolVersion: 1, dockerRunning: true, directoriesWritable: true,
|
|
storageAvailable: 100 * 1024 ** 3, runtimeImages: [], libraries: [], capabilities: ['host.decommission']
|
|
};
|
|
|
|
beforeEach(async () => {
|
|
vi.resetModules();
|
|
fixture = await newFixture([]);
|
|
vi.stubEnv('PLUTAINER_IMAGE', runtime);
|
|
vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'game-files'));
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
vi.setSystemTime(initialTime);
|
|
hosts = await import('../../src/lib/server/hosts/index.js');
|
|
libraries = await import('../../src/lib/server/game-files/index.js');
|
|
preparation = await import('../../src/lib/server/hosts/preparation.js');
|
|
});
|
|
|
|
afterEach(async () => {
|
|
vi.useRealTimers();
|
|
vi.unstubAllEnvs();
|
|
await fixture.cleanup();
|
|
});
|
|
|
|
function enroll(overrides: HostInventory = {}) {
|
|
const host = hosts.createHost({ name: 'Automatic game host' });
|
|
hosts.enrollHost(hosts.createEnrollment(host.id).token, { ...inventory, ...overrides });
|
|
return host.id;
|
|
}
|
|
|
|
function heartbeat(id: string, overrides: HostInventory = {}) {
|
|
vi.setSystemTime(Date.now() + 1);
|
|
return hosts.heartbeat(id, { ...inventory, ...overrides });
|
|
}
|
|
|
|
function operations(id: string) {
|
|
return hosts.listHostOperations(id).filter(operation => ['runtime.pull', 'library.download'].includes(operation.kind));
|
|
}
|
|
|
|
async function publish(game: 't5' | 't6' = 't5', bytes = content) {
|
|
vi.setSystemTime(Date.now() + 1);
|
|
const upload = libraries.createLibraryUpload({
|
|
game, files: [{ path: 'zone/game.ff', size: bytes.length, lastModified: 0 }]
|
|
});
|
|
await libraries.uploadBrowserFile(upload.id, 'zone/game.ff', {
|
|
offset: 0, size: bytes.length, sha256: createHash('sha256').update(bytes).digest('hex')
|
|
}, new Response(bytes).body);
|
|
return libraries.publishBrowserUpload(upload.id);
|
|
}
|
|
|
|
function verified(id: string, version: GameLibraryVersion) {
|
|
libraries.updateHostLibraryCaches(id, [{
|
|
libraryVersionId: version.id, status: 'ready', manifestSha256: version.manifestSha256
|
|
}]);
|
|
}
|
|
|
|
describe('automatic host preparation', () => {
|
|
it('accepts heartbeats without running controller library cleanup', async () => {
|
|
const host = hosts.createHost({ name: 'Heartbeat host' });
|
|
const { credential } = hosts.enrollHost(hosts.createEnrollment(host.id).token, inventory);
|
|
const { POST } = await import('../../src/routes/api/agent/heartbeat/+server.js');
|
|
const cleanup = vi.spyOn(libraries, 'cleanupControllerLibraries').mockRejectedValue(new Error('Cleanup is independent of heartbeats'));
|
|
try {
|
|
for (let i = 0; i < 3; i++) {
|
|
const response = await POST({ request: new Request('https://controller.test/api/agent/heartbeat', {
|
|
method: 'POST', headers: { Authorization: `Bearer ${credential}`, 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ inventory: { ...inventory, libraryCaches: [] } })
|
|
}) });
|
|
expect(response.status).toBe(200);
|
|
}
|
|
expect(cleanup).not.toHaveBeenCalled();
|
|
} finally { cleanup.mockRestore(); }
|
|
});
|
|
|
|
it('keeps re-enabled hosts disabled until a fresh runtime pull and both game libraries are verified', async () => {
|
|
const t5 = await publish('t5');
|
|
const t6 = await publish('t6');
|
|
const readyInventory = { runtimeImages: [runtime], libraries: [t5, t6].map(version => ({ id: version.id, state: 'ready' as const })) };
|
|
const id = enroll(readyInventory);
|
|
verified(id, t5); verified(id, t6);
|
|
const oldRuntime = hosts.queueHostOperation(id, 'runtime.pull', { image: runtime });
|
|
hosts.reportHostOperation(id, oldRuntime.id, { state: 'completed' });
|
|
heartbeat(id, readyInventory);
|
|
hosts.updateHost(id, { mode: 'disabled' });
|
|
const requested = hosts.updateHost(id, { mode: 'active' });
|
|
expect(requested).toMatchObject({ mode: 'disabled', readiness: 'preparing', enablingSince: expect.any(String) });
|
|
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
|
closeDb();
|
|
expect(() => hosts.requireReadyHost(id)).toThrow('before enabling');
|
|
preparation.reconcileHostPreparation(id);
|
|
const pull = operations(id)[0];
|
|
expect(pull.id).not.toBe(oldRuntime.id);
|
|
expect(pull.kind).toBe('runtime.pull');
|
|
expect(hosts.pendingHostOperations(id).map(operation => operation.id)).toEqual([pull.id]);
|
|
expect(() => hosts.queueHostOperation(id, 'instance.restart', {})).toThrow('disabled');
|
|
expect(hosts.updateHost(id, { mode: 'active' }).enablingSince).toBe(requested.enablingSince);
|
|
hosts.reportHostOperation(id, pull.id, { state: 'completed' });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(hosts.getHost(id)?.mode).toBe('disabled');
|
|
heartbeat(id, readyInventory);
|
|
for (let index = 0; index < 2; index++) {
|
|
preparation.reconcileHostPreparation(id);
|
|
const download = operations(id)[0];
|
|
expect(download.kind).toBe('library.download');
|
|
expect(hosts.pendingHostOperations(id).map(operation => operation.id)).toEqual([download.id]);
|
|
const version = [t5, t6].find(version => version.id === download.payload.libraryVersionId)!;
|
|
verified(id, version);
|
|
hosts.reportHostOperation(id, download.id, { state: 'completed' });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(hosts.getHost(id)?.mode).toBe('disabled');
|
|
heartbeat(id, readyInventory);
|
|
}
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(hosts.getHost(id)).toMatchObject({ mode: 'active', readiness: 'ready', enablingSince: null });
|
|
expect(operations(id).filter(operation => operation.kind === 'library.download')).toHaveLength(2);
|
|
});
|
|
|
|
it('requires both games when enabling even if one library is already cached', async () => {
|
|
const version = await publish('t5');
|
|
const id = enroll({ runtimeImages: [runtime], libraries: [{ id: version.id, state: 'ready' }] });
|
|
verified(id, version);
|
|
hosts.updateHost(id, { mode: 'disabled' });
|
|
hosts.updateHost(id, { mode: 'active' });
|
|
const status = preparation.reconcileHostPreparation(id);
|
|
expect(status.items).toContainEqual(expect.objectContaining({ label: 'Black Ops II (T6)', state: 'blocked' }));
|
|
expect(hosts.getHost(id)?.mode).toBe('disabled');
|
|
});
|
|
|
|
it('keeps a re-enabling host out of placement after setup failure or loss of contact', async () => {
|
|
const t5 = await publish('t5');
|
|
const t6 = await publish('t6');
|
|
const readyInventory = { runtimeImages: [runtime], libraries: [t5, t6].map(version => ({ id: version.id, state: 'ready' as const })) };
|
|
const id = enroll(readyInventory);
|
|
verified(id, t5); verified(id, t6);
|
|
hosts.updateHost(id, { mode: 'disabled' });
|
|
hosts.updateHost(id, { mode: 'active' });
|
|
preparation.reconcileHostPreparation(id);
|
|
hosts.reportHostOperation(id, operations(id)[0].id, { state: 'failed', error: 'Registry unavailable' });
|
|
heartbeat(id, readyInventory);
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('blocked');
|
|
expect(() => hosts.requireReadyHost(id)).toThrow('before enabling');
|
|
const queued = operations(id).map(operation => operation.id);
|
|
vi.setSystemTime(Date.now() + 91_000);
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(hosts.getHost(id)).toMatchObject({ mode: 'disabled', connected: false });
|
|
expect(operations(id).map(operation => operation.id)).toEqual(queued);
|
|
heartbeat(id, readyInventory);
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(hosts.getHost(id)?.mode).toBe('disabled');
|
|
});
|
|
|
|
it('prepares an enrolled host through authenticated agent polls and heartbeats without manual setup requests', async () => {
|
|
const [{ POST: enrollAgent }, { GET: pollOperations }, { POST: reportOperation }, { POST: reportHeartbeat }] = await Promise.all([
|
|
import('../../src/routes/api/agent/enroll/+server.js'),
|
|
import('../../src/routes/api/agent/operations/+server.js'),
|
|
import('../../src/routes/api/agent/operations/[id]/+server.js'),
|
|
import('../../src/routes/api/agent/heartbeat/+server.js')
|
|
]);
|
|
const host = hosts.createHost({ name: 'Agent provisioning integration' });
|
|
const enrollment = await enrollAgent({ request: new Request('https://controller.test/api/agent/enroll', {
|
|
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ token: hosts.createEnrollment(host.id).token, inventory })
|
|
}) });
|
|
expect(enrollment.status).toBe(200);
|
|
const { credential } = await enrollment.json();
|
|
const request = (path: string, body?: object) => new Request(`https://controller.test/api/agent/${path}`, {
|
|
method: body ? 'POST' : 'GET',
|
|
headers: { Authorization: `Bearer ${credential}`, 'Content-Type': 'application/json' },
|
|
...(body ? { body: JSON.stringify(body) } : {})
|
|
});
|
|
expect(operations(host.id)).toEqual([]);
|
|
|
|
const firstPoll = await (await pollOperations({ request: request('operations') })).json();
|
|
expect(firstPoll.operations).toHaveLength(1);
|
|
const runtimeOperation = firstPoll.operations[0];
|
|
expect(runtimeOperation).toMatchObject({ kind: 'runtime.pull', payload: { image: runtime } });
|
|
expect((await (await pollOperations({ request: request('operations') })).json()).operations).toEqual([runtimeOperation]);
|
|
const version = await publish();
|
|
expect((await (await pollOperations({ request: request('operations') })).json()).operations).toEqual([runtimeOperation]);
|
|
|
|
vi.setSystemTime(Date.now() + 1);
|
|
const completedRuntime = await reportOperation({
|
|
request: request(`operations/${runtimeOperation.id}`, { state: 'completed' }), params: { id: runtimeOperation.id }
|
|
});
|
|
expect(completedRuntime.status).toBe(200);
|
|
expect((await (await pollOperations({ request: request('operations') })).json()).operations).toEqual([]);
|
|
expect(operations(host.id)).toHaveLength(1);
|
|
|
|
vi.setSystemTime(Date.now() + 1);
|
|
const preparedInventory = { ...inventory, runtimeImages: [runtime], libraryCaches: [] };
|
|
const runtimeHeartbeat = await reportHeartbeat({ request: request('heartbeat', { inventory: preparedInventory }) });
|
|
expect(runtimeHeartbeat.status).toBe(200);
|
|
expect(operations(host.id)).toHaveLength(2);
|
|
const download = operations(host.id)[0];
|
|
expect(download).toMatchObject({ kind: 'library.download', payload: { libraryVersionId: version.id } });
|
|
expect((await reportHeartbeat({ request: request('heartbeat', { inventory: preparedInventory }) })).status).toBe(200);
|
|
expect((await (await pollOperations({ request: request('operations') })).json()).operations).toEqual([download]);
|
|
expect(operations(host.id)).toHaveLength(2);
|
|
|
|
vi.setSystemTime(Date.now() + 1);
|
|
expect((await reportOperation({
|
|
request: request(`operations/${download.id}`, { state: 'completed' }), params: { id: download.id }
|
|
})).status).toBe(200);
|
|
vi.setSystemTime(Date.now() + 1);
|
|
const finalHeartbeat = await reportHeartbeat({ request: request('heartbeat', { inventory: {
|
|
...preparedInventory,
|
|
libraries: [{ id: version.id, state: 'ready' }],
|
|
libraryCaches: [{ libraryVersionId: version.id, status: 'ready', manifestSha256: version.manifestSha256 }]
|
|
} }) });
|
|
expect(finalHeartbeat.status).toBe(200);
|
|
expect(libraries.requireReadyLibrary(host.id, version.id).id).toBe(version.id);
|
|
expect(preparation.hostPreparationStatus(hosts.getHost(host.id)!).state).toBe('ready');
|
|
expect((await (await pollOperations({ request: request('operations') })).json()).operations).toEqual([]);
|
|
expect(operations(host.id)).toHaveLength(2);
|
|
});
|
|
|
|
it('presents preparation without queuing work and prepares the configured runtime even when another image exists', async () => {
|
|
const version = await publish();
|
|
const id = enroll({ runtimeImages: ['another:runtime'] });
|
|
const host = hosts.getHost(id)!;
|
|
const status = preparation.hostPreparationStatus(host);
|
|
expect(status.items.map(item => item.kind)).toEqual(['runtime', 'library']);
|
|
expect(preparation.presentHostPreparation(host)).toMatchObject({ id, preparation: status });
|
|
expect(operations(id)).toEqual([]);
|
|
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('preparing');
|
|
const [operation] = operations(id);
|
|
expect(operation).toMatchObject({ kind: 'runtime.pull', payload: { image: runtime }, state: 'pending' });
|
|
expect(libraries.getHostLibraryCache(id, version.id)).toBeNull();
|
|
for (let poll = 0; poll < 4; poll++) {
|
|
heartbeat(id);
|
|
preparation.reconcileHostPreparation(id);
|
|
}
|
|
hosts.reportHostOperation(id, operation.id, { state: 'unknown' });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id).map(item => item.id)).toEqual([operation.id]);
|
|
});
|
|
|
|
it('waits for fresh inventory after runtime completion before starting a library download', async () => {
|
|
const version = await publish();
|
|
const id = enroll();
|
|
preparation.reconcileHostPreparation(id);
|
|
const runtimeOperation = operations(id)[0];
|
|
vi.setSystemTime(Date.now() + 1);
|
|
hosts.reportHostOperation(id, runtimeOperation.id, { state: 'completed', result: { image: runtime } });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(1);
|
|
|
|
heartbeat(id, { runtimeImages: [runtime] });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(2);
|
|
const download = operations(id)[0];
|
|
expect(download).toMatchObject({ kind: 'library.download', payload: { libraryVersionId: version.id } });
|
|
verified(id, version);
|
|
hosts.reportHostOperation(id, download.id, { state: 'completed' });
|
|
heartbeat(id, { runtimeImages: [runtime], libraries: [{ id: version.id, state: 'ready' }] });
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('ready');
|
|
expect(operations(id)).toHaveLength(2);
|
|
});
|
|
|
|
it('always targets the latest verified T5 and T6 upload while newer uploads remain unfinished', async () => {
|
|
const old = await publish();
|
|
const latest = await publish();
|
|
const t6 = await publish('t6');
|
|
libraries.createLibraryUpload({
|
|
game: 't5',
|
|
files: [{ path: 'zone/game.ff', size: content.length, lastModified: 0 }]
|
|
});
|
|
const id = enroll({ runtimeImages: [runtime] });
|
|
const expected = new Map([latest, t6].map(version => [version.id, version]));
|
|
const downloaded: string[] = [];
|
|
for (let index = 0; index < expected.size; index++) {
|
|
preparation.reconcileHostPreparation(id);
|
|
const active = operations(id).filter(operation => !['completed', 'failed'].includes(operation.state));
|
|
expect(active).toHaveLength(1);
|
|
const operation = active[0];
|
|
const versionId = String(operation.payload.libraryVersionId);
|
|
expect(expected.has(versionId)).toBe(true);
|
|
expect(downloaded).not.toContain(versionId);
|
|
downloaded.push(versionId);
|
|
verified(id, expected.get(versionId)!);
|
|
vi.setSystemTime(Date.now() + 1);
|
|
hosts.reportHostOperation(id, operation.id, { state: 'completed' });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(index + 1);
|
|
heartbeat(id, { runtimeImages: [runtime], libraries: downloaded.map(id => ({ id, state: 'ready' })) });
|
|
}
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('ready');
|
|
expect(downloaded).toHaveLength(expected.size);
|
|
expect(downloaded).not.toContain(old.id);
|
|
});
|
|
|
|
it('picks up publication after enrollment and keeps verified historical versions intact', async () => {
|
|
const id = enroll({ runtimeImages: [runtime] });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toEqual([]);
|
|
const old = await publish();
|
|
verified(id, old);
|
|
heartbeat(id, { runtimeImages: [runtime], libraries: [{ id: old.id, state: 'ready' }] });
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('ready');
|
|
const latest = await publish();
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(1);
|
|
expect(operations(id)[0].payload.libraryVersionId).toBe(latest.id);
|
|
expect(libraries.requireReadyLibrary(id, old.id).id).toBe(old.id);
|
|
expect(libraries.getHostLibraryCache(id, old.id)?.status).toBe('ready');
|
|
});
|
|
|
|
it('replaces an undelivered automatic download when newer files finish uploading', async () => {
|
|
const old = await publish();
|
|
const id = enroll({ runtimeImages: [runtime] });
|
|
preparation.reconcileHostPreparation(id);
|
|
const first = operations(id)[0];
|
|
expect(first.payload).toMatchObject({ libraryVersionId: old.id, automatic: true });
|
|
// The next heartbeat may discover a completed cache while this queued
|
|
// retry has still never been delivered. Keep that verified evidence.
|
|
verified(id, old);
|
|
const latest = await publish();
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(hosts.getHostOperation(first.id)).toMatchObject({ state: 'failed', error: expect.stringContaining('Superseded') });
|
|
expect(operations(id).filter(operation => operation.state === 'pending')).toMatchObject([
|
|
{ payload: { libraryVersionId: latest.id, automatic: true } }
|
|
]);
|
|
expect(libraries.requireReadyLibrary(id, old.id).id).toBe(old.id);
|
|
});
|
|
|
|
it.each([false, true])('prepares queued libraries without pulling custom runtimes (automatic placement: %s)', async automatic => {
|
|
const old = await publish();
|
|
const image = 'ghcr.io/example/plutainer:queued-version';
|
|
const id = enroll({ runtimeImages: [runtime, image] });
|
|
const other = enroll({ runtimeImages: [runtime] });
|
|
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
|
for (let index = 0; index < 2; index++) {
|
|
expect(queueNativeCreation({ game: 't5', gameMode: 'zombie', mapSlug: 'theater', libraryId: old.id,
|
|
hostId: automatic ? undefined : id, image, plutoKey: 'saved-key' }).ok).toBe(true);
|
|
}
|
|
// Also cover pre-existing intents whose image is no longer cached.
|
|
heartbeat(id, { runtimeImages: [runtime] });
|
|
const latest = await publish();
|
|
const status = preparation.hostPreparationStatus(hosts.getHost(id)!);
|
|
expect(status.items.filter(item => item.label === image)).toHaveLength(0);
|
|
expect(status.items.filter(item => item.label.includes('Queued version'))).toHaveLength(1);
|
|
const otherStatus = preparation.hostPreparationStatus(hosts.getHost(other)!);
|
|
expect(otherStatus.items.some(item => item.label === image)).toBe(false);
|
|
expect(otherStatus.items.some(item => item.label.includes('Queued version'))).toBe(automatic);
|
|
expect(operations(id)).toEqual([]);
|
|
|
|
preparation.reconcileHostPreparation(id);
|
|
const download = operations(id)[0];
|
|
expect(download).toMatchObject({ kind: 'library.download', payload: { libraryVersionId: old.id, automatic: true } });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(hosts.getHostOperation(download.id)?.state).toBe('pending');
|
|
expect(operations(id)).toHaveLength(1);
|
|
preparation.reconcileHostPreparation(other);
|
|
expect([...operations(id), ...operations(other)].every(operation => operation.kind !== 'runtime.pull')).toBe(true);
|
|
|
|
verified(id, old);
|
|
vi.setSystemTime(Date.now() + 1);
|
|
hosts.reportHostOperation(id, download.id, { state: 'completed' });
|
|
heartbeat(id, { runtimeImages: [runtime, image], libraries: [{ id: old.id, state: 'ready' }] });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)[0]).toMatchObject({ kind: 'library.download', payload: { libraryVersionId: latest.id } });
|
|
});
|
|
|
|
it('retains an undelivered library download for an accepted intent until that intent is cancelled', async () => {
|
|
const old = await publish();
|
|
const id = enroll({ runtimeImages: [runtime] });
|
|
preparation.reconcileHostPreparation(id);
|
|
const download = operations(id)[0];
|
|
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
|
const launch = queueNativeCreation({ game: 't5', gameMode: 'zombie', mapSlug: 'theater', libraryId: old.id,
|
|
hostId: id, image: runtime, plutoKey: 'saved-key' });
|
|
expect(launch.ok).toBe(true);
|
|
if (!launch.ok) throw new Error(launch.message);
|
|
const latest = await publish();
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(hosts.getHostOperation(download.id)?.state).toBe('pending');
|
|
expect(operations(id)).toHaveLength(1);
|
|
|
|
const { cancelCreation } = await import('../../src/lib/server/provisioning/cancel-creation.js');
|
|
expect(cancelCreation(launch.deploymentId)).toEqual(expect.any(String));
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(hosts.getHostOperation(download.id)).toMatchObject({ state: 'failed', error: expect.stringContaining('Superseded') });
|
|
expect(operations(id)[0]).toMatchObject({ kind: 'library.download', payload: { libraryVersionId: latest.id } });
|
|
});
|
|
|
|
it('finishes a delivered download before following a newer upload under a new operation ID', async () => {
|
|
const old = await publish();
|
|
const id = enroll({ runtimeImages: [runtime] });
|
|
preparation.reconcileHostPreparation(id);
|
|
const [first] = hosts.pendingHostOperations(id);
|
|
const latest = await publish();
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(1);
|
|
expect(hosts.getHostOperation(first.id)?.state).toBe('pending');
|
|
verified(id, old);
|
|
hosts.reportHostOperation(id, first.id, { state: 'completed' });
|
|
heartbeat(id, { runtimeImages: [runtime], libraries: [{ id: old.id, state: 'ready' }] });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)[0]).toMatchObject({ state: 'pending', payload: { libraryVersionId: latest.id } });
|
|
expect(libraries.requireReadyLibrary(id, old.id).id).toBe(old.id);
|
|
});
|
|
|
|
it.each(['draining', 'disabled'] as const)('does not start setup for a %s host', mode => {
|
|
const id = enroll();
|
|
hosts.updateHost(id, { mode });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toEqual([]);
|
|
});
|
|
|
|
it.each([
|
|
{ name: 'incompatible protocol', value: { protocolVersion: 2 } },
|
|
{ name: 'unavailable Docker', value: { dockerRunning: false } },
|
|
{ name: 'unwritable managed directories', value: { directoriesWritable: false } },
|
|
{ name: 'insufficient storage reserve', value: { storageAvailable: 0 } }
|
|
])('does not start setup with $name', ({ value }) => {
|
|
const id = enroll(value);
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('blocked');
|
|
expect(operations(id)).toEqual([]);
|
|
});
|
|
|
|
it('does not start setup for unenrolled, disconnected, revoked, or decommissioning hosts', () => {
|
|
const unenrolled = hosts.createHost({ name: 'Not enrolled' }).id;
|
|
preparation.reconcileHostPreparation(unenrolled);
|
|
expect(operations(unenrolled)).toEqual([]);
|
|
const offline = enroll();
|
|
vi.setSystemTime(Date.now() + 91_000);
|
|
preparation.reconcileHostPreparation(offline);
|
|
expect(operations(offline)).toEqual([]);
|
|
const revoked = enroll();
|
|
hosts.revokeCredentials(revoked);
|
|
preparation.reconcileHostPreparation(revoked);
|
|
expect(operations(revoked)).toEqual([]);
|
|
const deleting = enroll();
|
|
hosts.deleteHost(deleting);
|
|
preparation.reconcileHostPreparation(deleting);
|
|
expect(operations(deleting)).toEqual([]);
|
|
});
|
|
|
|
it('retries runtime failures with exponential delays capped at fifteen minutes', () => {
|
|
const id = enroll();
|
|
preparation.reconcileHostPreparation(id);
|
|
for (const [attempt, delay] of [60_000, 120_000, 240_000, 480_000, 900_000, 900_000].entries()) {
|
|
const operation = operations(id)[0];
|
|
vi.setSystemTime(Date.now() + 1);
|
|
hosts.reportHostOperation(id, operation.id, { state: 'failed', error: 'Registry unavailable' });
|
|
const failedAt = Date.now();
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(attempt + 1);
|
|
vi.setSystemTime(failedAt + delay - 2);
|
|
heartbeat(id);
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(attempt + 1);
|
|
vi.setSystemTime(failedAt + delay);
|
|
heartbeat(id);
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(attempt + 2);
|
|
expect(operations(id)[0]).toMatchObject({ kind: 'runtime.pull', state: 'pending' });
|
|
}
|
|
});
|
|
|
|
it('blocks a library that cannot fit while preserving the host free-space reserve', async () => {
|
|
const version = await publish();
|
|
const id = enroll({ runtimeImages: [runtime], storageAvailable: 1_000 + version.sizeBytes - 1 });
|
|
hosts.updateHost(id, { minimumFreeBytes: 1_000 });
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('blocked');
|
|
expect(operations(id)).toEqual([]);
|
|
heartbeat(id, { runtimeImages: [runtime], storageAvailable: 1_000 + version.sizeBytes });
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('preparing');
|
|
expect(operations(id)[0].payload.libraryVersionId).toBe(version.id);
|
|
});
|
|
|
|
it('returns an active download even when its in-progress bytes have consumed free space', async () => {
|
|
const version = await publish();
|
|
const id = enroll({ runtimeImages: [runtime] });
|
|
const operation = libraries.queueLibraryDownload(id, version.id);
|
|
heartbeat(id, { runtimeImages: [runtime], storageAvailable: hosts.getHost(id)!.minimumFreeBytes });
|
|
expect(libraries.queueLibraryDownload(id, version.id).id).toBe(operation.id);
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(1);
|
|
});
|
|
|
|
it('prepares T6 when the latest T5 upload cannot fit', async () => {
|
|
const oversized = await publish();
|
|
const small = await publish('t6', 'small');
|
|
const id = enroll({ runtimeImages: [runtime], storageAvailable: 1_010 });
|
|
hosts.updateHost(id, { minimumFreeBytes: 1_000 });
|
|
const status = preparation.reconcileHostPreparation(id);
|
|
expect(status.items).toEqual(expect.arrayContaining([
|
|
expect.objectContaining({ label: 'Black Ops I (T5)', state: 'blocked' }),
|
|
expect.objectContaining({ label: 'Black Ops II (T6)', state: 'preparing' })
|
|
]));
|
|
expect(operations(id)).toHaveLength(1);
|
|
expect(operations(id)[0].payload.libraryVersionId).toBe(small.id);
|
|
expect(libraries.getHostLibraryCache(id, oversized.id)).toBeNull();
|
|
});
|
|
|
|
it('does not count partial bytes from a different manifest toward download storage', async () => {
|
|
const version = await publish();
|
|
const id = enroll({ runtimeImages: [runtime] });
|
|
libraries.updateHostLibraryCaches(id, [{
|
|
libraryVersionId: version.id, status: 'failed', bytesDownloaded: 80, manifestSha256: '0'.repeat(64)
|
|
}]);
|
|
heartbeat(id, { runtimeImages: [runtime], storageAvailable: hosts.getHost(id)!.minimumFreeBytes + 20 });
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('blocked');
|
|
expect(() => libraries.queueLibraryDownload(id, version.id)).toThrow(/insufficient storage/);
|
|
expect(operations(id)).toEqual([]);
|
|
});
|
|
|
|
it('resumes a failed verified-manifest download using only its remaining storage', async () => {
|
|
const version = await publish();
|
|
const id = enroll({ runtimeImages: [runtime] });
|
|
preparation.reconcileHostPreparation(id);
|
|
const operation = operations(id)[0];
|
|
libraries.updateHostLibraryCaches(id, [{
|
|
libraryVersionId: version.id, status: 'downloading', bytesDownloaded: 80, manifestSha256: version.manifestSha256
|
|
}]);
|
|
vi.setSystemTime(Date.now() + 1);
|
|
hosts.reportHostOperation(id, operation.id, { state: 'failed', error: 'Connection reset' });
|
|
const failedAt = Date.now();
|
|
const free = hosts.getHost(id)!.minimumFreeBytes + version.sizeBytes - 80;
|
|
heartbeat(id, { runtimeImages: [runtime], storageAvailable: free });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(1);
|
|
vi.setSystemTime(failedAt + 60_000);
|
|
heartbeat(id, { runtimeImages: [runtime], storageAvailable: free });
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('preparing');
|
|
expect(operations(id)).toHaveLength(2);
|
|
expect(operations(id)[0].payload.libraryVersionId).toBe(version.id);
|
|
expect(libraries.getHostLibraryCache(id, version.id)?.bytesDownloaded).toBe(80);
|
|
});
|
|
});
|