1187 lines
51 KiB
TypeScript
1187 lines
51 KiB
TypeScript
import { createHash } from 'node:crypto';
|
|
import { join } from 'node:path';
|
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import type { GameLibraryVersion } from '../../src/lib/game-files.js';
|
|
import type { HostInventory } from '../../src/lib/hosts.js';
|
|
import { newFixture, type Fixture } from '../helpers/fixture.js';
|
|
|
|
let fixture: Fixture;
|
|
let hosts: typeof import('../../src/lib/server/hosts/index.js');
|
|
let libraries: typeof import('../../src/lib/server/game-files/index.js');
|
|
let preparation: typeof import('../../src/lib/server/hosts/preparation.js');
|
|
const runtime = 'ghcr.io/example/plutainer:provisioning-test';
|
|
const initialTime = Date.parse('2026-10-04T12:00:00Z');
|
|
const content = 'a'.repeat(100);
|
|
const inventory: HostInventory = {
|
|
hostname: 'game-host',
|
|
protocolVersion: 1,
|
|
dockerRunning: true,
|
|
directoriesWritable: true,
|
|
storageAvailable: 100 * 1024 ** 3,
|
|
preparedRuntimeImages: [],
|
|
libraries: [],
|
|
capabilities: ['host.decommission']
|
|
};
|
|
|
|
beforeEach(async () => {
|
|
vi.resetModules();
|
|
fixture = await newFixture([]);
|
|
vi.stubEnv('PLUTAINER_IMAGE', runtime);
|
|
vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'game-files'));
|
|
vi.useFakeTimers({ toFake: ['Date'] });
|
|
vi.setSystemTime(initialTime);
|
|
hosts = await import('../../src/lib/server/hosts/index.js');
|
|
libraries = await import('../../src/lib/server/game-files/index.js');
|
|
preparation = await import('../../src/lib/server/hosts/preparation.js');
|
|
});
|
|
|
|
afterEach(async () => {
|
|
vi.useRealTimers();
|
|
vi.unstubAllEnvs();
|
|
await fixture.cleanup();
|
|
});
|
|
|
|
function enroll(overrides: HostInventory = {}) {
|
|
const host = hosts.createHost({ name: 'Automatic game host' });
|
|
hosts.enrollHost(hosts.createEnrollment(host.id).token, { ...inventory, ...overrides });
|
|
return host.id;
|
|
}
|
|
|
|
function heartbeat(id: string, overrides: HostInventory = {}) {
|
|
vi.setSystemTime(Date.now() + 1);
|
|
return hosts.heartbeat(id, { ...inventory, ...overrides });
|
|
}
|
|
|
|
function operations(id: string) {
|
|
return hosts
|
|
.listHostOperations(id)
|
|
.filter((operation) =>
|
|
['runtime.pull', 'library.download', 'plutonium.prepare'].includes(operation.kind)
|
|
);
|
|
}
|
|
|
|
async function publish(game: 't5' | 't6' = 't5', bytes = content) {
|
|
vi.setSystemTime(Date.now() + 1);
|
|
const upload = libraries.createLibraryUpload({
|
|
game,
|
|
files: [{ path: 'zone/game.ff', size: bytes.length, lastModified: 0 }]
|
|
});
|
|
await libraries.uploadBrowserFile(
|
|
upload.id,
|
|
'zone/game.ff',
|
|
{
|
|
offset: 0,
|
|
size: bytes.length,
|
|
sha256: createHash('sha256').update(bytes).digest('hex')
|
|
},
|
|
new Response(bytes).body
|
|
);
|
|
return libraries.publishBrowserUpload(upload.id);
|
|
}
|
|
|
|
function verified(id: string, version: GameLibraryVersion) {
|
|
libraries.updateHostLibraryCaches(id, [
|
|
{
|
|
libraryVersionId: version.id,
|
|
status: 'ready',
|
|
manifestSha256: version.manifestSha256
|
|
}
|
|
]);
|
|
}
|
|
|
|
describe('automatic host preparation', () => {
|
|
it('accepts heartbeats without running controller library cleanup', async () => {
|
|
const host = hosts.createHost({ name: 'Heartbeat host' });
|
|
const { credential } = hosts.enrollHost(hosts.createEnrollment(host.id).token, inventory);
|
|
const { POST } = await import('../../src/routes/api/agent/heartbeat/+server.js');
|
|
const cleanup = vi
|
|
.spyOn(libraries, 'cleanupControllerLibraries')
|
|
.mockRejectedValue(new Error('Cleanup is independent of heartbeats'));
|
|
try {
|
|
for (let i = 0; i < 3; i++) {
|
|
const response = await POST({
|
|
request: new Request('https://controller.test/api/agent/heartbeat', {
|
|
method: 'POST',
|
|
headers: {
|
|
Authorization: `Bearer ${credential}`,
|
|
'Content-Type': 'application/json'
|
|
},
|
|
body: JSON.stringify({ inventory: { ...inventory, libraryCaches: [] } })
|
|
})
|
|
});
|
|
expect(response.status).toBe(200);
|
|
}
|
|
expect(cleanup).not.toHaveBeenCalled();
|
|
} finally {
|
|
cleanup.mockRestore();
|
|
}
|
|
});
|
|
|
|
it.each([false, true])(
|
|
'keeps re-enabled hosts disabled until required files are verified, without waiting for the optional cache (supported: %s)',
|
|
async (supported) => {
|
|
const t5 = await publish('t5');
|
|
const t6 = await publish('t6');
|
|
const readyInventory = {
|
|
preparedRuntimeImages: [runtime],
|
|
libraries: [t5, t6].map((version) => ({ id: version.id, state: 'ready' as const })),
|
|
capabilities: [
|
|
...inventory.capabilities!,
|
|
...(supported ? ['plutonium.cache'] : [])
|
|
]
|
|
};
|
|
const id = enroll(readyInventory);
|
|
verified(id, t5);
|
|
verified(id, t6);
|
|
const oldRuntime = hosts.queueHostOperation(id, 'runtime.pull', { image: runtime });
|
|
hosts.reportHostOperation(id, oldRuntime.id, { state: 'completed' });
|
|
heartbeat(id, readyInventory);
|
|
hosts.updateHost(id, { mode: 'disabled' });
|
|
const requested = hosts.updateHost(id, { mode: 'active' });
|
|
expect(requested).toMatchObject({
|
|
mode: 'disabled',
|
|
readiness: 'preparing',
|
|
enablingSince: expect.any(String)
|
|
});
|
|
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
|
closeDb();
|
|
expect(() => hosts.requireReadyHost(id)).toThrow('before enabling');
|
|
preparation.reconcileHostPreparation(id);
|
|
const pull = operations(id)[0];
|
|
expect(pull.id).not.toBe(oldRuntime.id);
|
|
expect(pull.kind).toBe('runtime.pull');
|
|
expect(hosts.pendingHostOperations(id).map((operation) => operation.id)).toEqual([
|
|
pull.id
|
|
]);
|
|
expect(() => hosts.queueHostOperation(id, 'instance.restart', {})).toThrow('disabled');
|
|
expect(hosts.updateHost(id, { mode: 'active' }).enablingSince).toBe(
|
|
requested.enablingSince
|
|
);
|
|
hosts.reportHostOperation(id, pull.id, { state: 'completed' });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(hosts.getHost(id)?.mode).toBe('disabled');
|
|
heartbeat(id, readyInventory);
|
|
for (let index = 0; index < 2; index++) {
|
|
preparation.reconcileHostPreparation(id);
|
|
const download = operations(id)[0];
|
|
expect(download.kind).toBe('library.download');
|
|
expect(hosts.pendingHostOperations(id).map((operation) => operation.id)).toEqual([
|
|
download.id
|
|
]);
|
|
const version = [t5, t6].find(
|
|
(version) => version.id === download.payload.libraryVersionId
|
|
)!;
|
|
verified(id, version);
|
|
hosts.reportHostOperation(id, download.id, { state: 'completed' });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(hosts.getHost(id)?.mode).toBe('disabled');
|
|
heartbeat(id, readyInventory);
|
|
}
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(hosts.getHost(id)).toMatchObject({
|
|
mode: 'active',
|
|
readiness: 'ready',
|
|
enablingSince: null
|
|
});
|
|
expect(
|
|
operations(id).filter((operation) => operation.kind === 'library.download')
|
|
).toHaveLength(2);
|
|
expect(
|
|
operations(id).filter((operation) => operation.kind === 'plutonium.prepare')
|
|
).toHaveLength(supported ? 1 : 0);
|
|
}
|
|
);
|
|
|
|
it('requires both games when enabling even if one library is already cached', async () => {
|
|
const version = await publish('t5');
|
|
const id = enroll({
|
|
preparedRuntimeImages: [runtime],
|
|
libraries: [{ id: version.id, state: 'ready' }]
|
|
});
|
|
verified(id, version);
|
|
hosts.updateHost(id, { mode: 'disabled' });
|
|
hosts.updateHost(id, { mode: 'active' });
|
|
const status = preparation.reconcileHostPreparation(id);
|
|
expect(status.items).toContainEqual(
|
|
expect.objectContaining({ label: 'Black Ops II', state: 'blocked' })
|
|
);
|
|
expect(hosts.getHost(id)?.mode).toBe('disabled');
|
|
});
|
|
|
|
it('keeps a re-enabling host out of placement after setup failure or loss of contact', async () => {
|
|
const t5 = await publish('t5');
|
|
const t6 = await publish('t6');
|
|
const readyInventory = {
|
|
preparedRuntimeImages: [runtime],
|
|
libraries: [t5, t6].map((version) => ({ id: version.id, state: 'ready' as const }))
|
|
};
|
|
const id = enroll(readyInventory);
|
|
verified(id, t5);
|
|
verified(id, t6);
|
|
hosts.updateHost(id, { mode: 'disabled' });
|
|
hosts.updateHost(id, { mode: 'active' });
|
|
preparation.reconcileHostPreparation(id);
|
|
hosts.reportHostOperation(id, operations(id)[0].id, {
|
|
state: 'failed',
|
|
error: 'Registry unavailable'
|
|
});
|
|
heartbeat(id, readyInventory);
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('blocked');
|
|
expect(() => hosts.requireReadyHost(id)).toThrow('before enabling');
|
|
const queued = operations(id).map((operation) => operation.id);
|
|
vi.setSystemTime(Date.now() + 91_000);
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(hosts.getHost(id)).toMatchObject({ mode: 'disabled', connected: false });
|
|
expect(operations(id).map((operation) => operation.id)).toEqual(queued);
|
|
heartbeat(id, readyInventory);
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(hosts.getHost(id)?.mode).toBe('disabled');
|
|
});
|
|
|
|
it('prepares an enrolled host through authenticated agent polls and heartbeats without manual setup requests', async () => {
|
|
const [
|
|
{ POST: enrollAgent },
|
|
{ GET: pollOperations },
|
|
{ POST: reportOperation },
|
|
{ POST: reportHeartbeat }
|
|
] = await Promise.all([
|
|
import('../../src/routes/api/agent/enroll/+server.js'),
|
|
import('../../src/routes/api/agent/operations/+server.js'),
|
|
import('../../src/routes/api/agent/operations/[id]/+server.js'),
|
|
import('../../src/routes/api/agent/heartbeat/+server.js')
|
|
]);
|
|
const host = hosts.createHost({ name: 'Agent provisioning integration' });
|
|
const enrollment = await enrollAgent({
|
|
request: new Request('https://controller.test/api/agent/enroll', {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ token: hosts.createEnrollment(host.id).token, inventory })
|
|
})
|
|
});
|
|
expect(enrollment.status).toBe(200);
|
|
const { credential } = await enrollment.json();
|
|
const request = (path: string, body?: object) =>
|
|
new Request(`https://controller.test/api/agent/${path}`, {
|
|
method: body ? 'POST' : 'GET',
|
|
headers: {
|
|
Authorization: `Bearer ${credential}`,
|
|
'Content-Type': 'application/json'
|
|
},
|
|
...(body ? { body: JSON.stringify(body) } : {})
|
|
});
|
|
expect(operations(host.id)).toEqual([]);
|
|
|
|
// Poll lanes only deliver queued work. One inventory heartbeat drives setup.
|
|
const reconcile = vi.spyOn(preparation, 'reconcileHostPreparation');
|
|
expect(
|
|
(await (await pollOperations({ request: request('operations') })).json()).operations
|
|
).toEqual([]);
|
|
expect(reconcile).not.toHaveBeenCalled();
|
|
expect(
|
|
(await reportHeartbeat({ request: request('heartbeat', { inventory }) })).status
|
|
).toBe(200);
|
|
expect(reconcile).toHaveBeenCalledOnce();
|
|
reconcile.mockRestore();
|
|
|
|
const firstPoll = await (await pollOperations({ request: request('operations') })).json();
|
|
expect(firstPoll.operations).toHaveLength(1);
|
|
const runtimeOperation = firstPoll.operations[0];
|
|
expect(runtimeOperation).toMatchObject({
|
|
kind: 'runtime.pull',
|
|
payload: { image: runtime }
|
|
});
|
|
expect(
|
|
(await (await pollOperations({ request: request('operations') })).json()).operations
|
|
).toEqual([runtimeOperation]);
|
|
const version = await publish();
|
|
expect(
|
|
(await (await pollOperations({ request: request('operations') })).json()).operations
|
|
).toEqual([runtimeOperation]);
|
|
|
|
vi.setSystemTime(Date.now() + 1);
|
|
const completedRuntime = await reportOperation({
|
|
request: request(`operations/${runtimeOperation.id}`, { state: 'completed' }),
|
|
params: { id: runtimeOperation.id }
|
|
});
|
|
expect(completedRuntime.status).toBe(200);
|
|
expect(
|
|
(await (await pollOperations({ request: request('operations') })).json()).operations
|
|
).toEqual([]);
|
|
expect(operations(host.id)).toHaveLength(1);
|
|
|
|
vi.setSystemTime(Date.now() + 1);
|
|
const preparedInventory = {
|
|
...inventory,
|
|
preparedRuntimeImages: [runtime],
|
|
libraryCaches: []
|
|
};
|
|
const runtimeHeartbeat = await reportHeartbeat({
|
|
request: request('heartbeat', { inventory: preparedInventory })
|
|
});
|
|
expect(runtimeHeartbeat.status).toBe(200);
|
|
expect(operations(host.id)).toHaveLength(2);
|
|
const download = operations(host.id)[0];
|
|
expect(download).toMatchObject({
|
|
kind: 'library.download',
|
|
payload: { libraryVersionId: version.id }
|
|
});
|
|
expect(
|
|
(
|
|
await reportHeartbeat({
|
|
request: request('heartbeat', { inventory: preparedInventory })
|
|
})
|
|
).status
|
|
).toBe(200);
|
|
expect(
|
|
(await (await pollOperations({ request: request('operations') })).json()).operations
|
|
).toEqual([download]);
|
|
expect(operations(host.id)).toHaveLength(2);
|
|
|
|
vi.setSystemTime(Date.now() + 1);
|
|
expect(
|
|
(
|
|
await reportOperation({
|
|
request: request(`operations/${download.id}`, { state: 'completed' }),
|
|
params: { id: download.id }
|
|
})
|
|
).status
|
|
).toBe(200);
|
|
vi.setSystemTime(Date.now() + 1);
|
|
const finalHeartbeat = await reportHeartbeat({
|
|
request: request('heartbeat', {
|
|
inventory: {
|
|
...preparedInventory,
|
|
libraries: [{ id: version.id, state: 'ready' }],
|
|
libraryCaches: [
|
|
{
|
|
libraryVersionId: version.id,
|
|
status: 'ready',
|
|
manifestSha256: version.manifestSha256
|
|
}
|
|
]
|
|
}
|
|
})
|
|
});
|
|
expect(finalHeartbeat.status).toBe(200);
|
|
expect(libraries.requireReadyLibrary(host.id, version.id).id).toBe(version.id);
|
|
expect(preparation.hostPreparationStatus(hosts.getHost(host.id)!).state).toBe('ready');
|
|
expect(
|
|
(await (await pollOperations({ request: request('operations') })).json()).operations
|
|
).toEqual([]);
|
|
expect(operations(host.id)).toHaveLength(2);
|
|
});
|
|
|
|
it('presents preparation without queuing work and prepares the configured runtime even when another image exists', async () => {
|
|
const version = await publish();
|
|
const id = enroll({ preparedRuntimeImages: ['another:runtime'] });
|
|
const host = hosts.getHost(id)!;
|
|
const status = preparation.hostPreparationStatus(host);
|
|
expect(status.items.map((item) => item.kind)).toEqual(['runtime', 'library']);
|
|
expect(preparation.presentHostPreparation(host)).toMatchObject({ id, preparation: status });
|
|
expect(operations(id)).toEqual([]);
|
|
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('preparing');
|
|
const [operation] = operations(id);
|
|
expect(operation).toMatchObject({
|
|
kind: 'runtime.pull',
|
|
payload: { image: runtime },
|
|
state: 'pending'
|
|
});
|
|
expect(libraries.getHostLibraryCache(id, version.id)).toBeNull();
|
|
for (let poll = 0; poll < 4; poll++) {
|
|
heartbeat(id);
|
|
preparation.reconcileHostPreparation(id);
|
|
}
|
|
hosts.reportHostOperation(id, operation.id, { state: 'unknown' });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id).map((item) => item.id)).toEqual([operation.id]);
|
|
});
|
|
|
|
it('waits for fresh inventory after runtime completion before starting a library download', async () => {
|
|
const version = await publish();
|
|
const id = enroll();
|
|
preparation.reconcileHostPreparation(id);
|
|
const runtimeOperation = operations(id)[0];
|
|
vi.setSystemTime(Date.now() + 1);
|
|
hosts.reportHostOperation(id, runtimeOperation.id, {
|
|
state: 'completed',
|
|
result: { image: runtime }
|
|
});
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(1);
|
|
|
|
heartbeat(id, { preparedRuntimeImages: [runtime] });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(2);
|
|
const download = operations(id)[0];
|
|
expect(download).toMatchObject({
|
|
kind: 'library.download',
|
|
payload: { libraryVersionId: version.id }
|
|
});
|
|
verified(id, version);
|
|
hosts.reportHostOperation(id, download.id, { state: 'completed' });
|
|
heartbeat(id, {
|
|
preparedRuntimeImages: [runtime],
|
|
libraries: [{ id: version.id, state: 'ready' }]
|
|
});
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('ready');
|
|
expect(operations(id)).toHaveLength(2);
|
|
});
|
|
|
|
it('always targets the latest verified T5 and T6 upload while newer uploads remain unfinished', async () => {
|
|
const old = await publish();
|
|
const latest = await publish();
|
|
const t6 = await publish('t6');
|
|
libraries.createLibraryUpload({
|
|
game: 't5',
|
|
files: [{ path: 'zone/game.ff', size: content.length, lastModified: 0 }]
|
|
});
|
|
const id = enroll({ preparedRuntimeImages: [runtime] });
|
|
const expected = new Map([latest, t6].map((version) => [version.id, version]));
|
|
const downloaded: string[] = [];
|
|
for (let index = 0; index < expected.size; index++) {
|
|
preparation.reconcileHostPreparation(id);
|
|
const active = operations(id).filter(
|
|
(operation) => !['completed', 'failed'].includes(operation.state)
|
|
);
|
|
expect(active).toHaveLength(1);
|
|
const operation = active[0];
|
|
const versionId = String(operation.payload.libraryVersionId);
|
|
expect(expected.has(versionId)).toBe(true);
|
|
expect(downloaded).not.toContain(versionId);
|
|
downloaded.push(versionId);
|
|
verified(id, expected.get(versionId)!);
|
|
vi.setSystemTime(Date.now() + 1);
|
|
hosts.reportHostOperation(id, operation.id, { state: 'completed' });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(index + 1);
|
|
heartbeat(id, {
|
|
preparedRuntimeImages: [runtime],
|
|
libraries: downloaded.map((id) => ({ id, state: 'ready' }))
|
|
});
|
|
}
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('ready');
|
|
expect(downloaded).toHaveLength(expected.size);
|
|
expect(downloaded).not.toContain(old.id);
|
|
});
|
|
|
|
it('picks up publication after enrollment and keeps verified historical versions intact', async () => {
|
|
const id = enroll({ preparedRuntimeImages: [runtime] });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toEqual([]);
|
|
const old = await publish();
|
|
verified(id, old);
|
|
heartbeat(id, {
|
|
preparedRuntimeImages: [runtime],
|
|
libraries: [{ id: old.id, state: 'ready' }]
|
|
});
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('ready');
|
|
const latest = await publish();
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(1);
|
|
expect(operations(id)[0].payload.libraryVersionId).toBe(latest.id);
|
|
expect(libraries.requireReadyLibrary(id, old.id).id).toBe(old.id);
|
|
expect(libraries.getHostLibraryCache(id, old.id)?.status).toBe('ready');
|
|
});
|
|
|
|
it('replaces an undelivered automatic download when newer files finish uploading', async () => {
|
|
const old = await publish();
|
|
const id = enroll({ preparedRuntimeImages: [runtime] });
|
|
preparation.reconcileHostPreparation(id);
|
|
const first = operations(id)[0];
|
|
expect(first.payload).toMatchObject({ libraryVersionId: old.id, automatic: true });
|
|
// The next heartbeat may discover a completed cache while this queued
|
|
// retry has still never been delivered. Keep that verified evidence.
|
|
verified(id, old);
|
|
const latest = await publish();
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(hosts.getHostOperation(first.id)).toMatchObject({
|
|
state: 'failed',
|
|
error: expect.stringContaining('Superseded')
|
|
});
|
|
expect(operations(id).filter((operation) => operation.state === 'pending')).toMatchObject([
|
|
{ payload: { libraryVersionId: latest.id, automatic: true } }
|
|
]);
|
|
expect(libraries.requireReadyLibrary(id, old.id).id).toBe(old.id);
|
|
});
|
|
|
|
it.each([false, true])(
|
|
'prepares queued libraries without pulling custom runtimes (automatic placement: %s)',
|
|
async (automatic) => {
|
|
const old = await publish();
|
|
const image = 'ghcr.io/example/plutainer:queued-version';
|
|
const id = enroll({ preparedRuntimeImages: [runtime, image] });
|
|
const other = enroll({ preparedRuntimeImages: [runtime] });
|
|
const { queueNativeCreation } =
|
|
await import('../../src/lib/server/provisioning/native.js');
|
|
for (let index = 0; index < 2; index++) {
|
|
expect(
|
|
queueNativeCreation({
|
|
game: 't5',
|
|
gameMode: 'zombie',
|
|
mapSlug: 'theater',
|
|
libraryId: old.id,
|
|
hostId: automatic ? undefined : id,
|
|
image,
|
|
plutoKey: `saved-key-${index}`
|
|
}).ok
|
|
).toBe(true);
|
|
}
|
|
// Also cover pre-existing intents whose image is no longer cached.
|
|
heartbeat(id, { preparedRuntimeImages: [runtime] });
|
|
const latest = await publish();
|
|
const status = preparation.hostPreparationStatus(hosts.getHost(id)!);
|
|
expect(status.items.filter((item) => item.label === image)).toHaveLength(0);
|
|
expect(
|
|
status.items.filter((item) => item.label.includes('Queued version'))
|
|
).toHaveLength(1);
|
|
const otherStatus = preparation.hostPreparationStatus(hosts.getHost(other)!);
|
|
expect(otherStatus.items.some((item) => item.label === image)).toBe(false);
|
|
expect(otherStatus.items.some((item) => item.label.includes('Queued version'))).toBe(
|
|
automatic
|
|
);
|
|
expect(operations(id)).toEqual([]);
|
|
|
|
preparation.reconcileHostPreparation(id);
|
|
const download = operations(id)[0];
|
|
expect(download).toMatchObject({
|
|
kind: 'library.download',
|
|
payload: { libraryVersionId: old.id, automatic: true }
|
|
});
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(hosts.getHostOperation(download.id)?.state).toBe('pending');
|
|
expect(operations(id)).toHaveLength(1);
|
|
preparation.reconcileHostPreparation(other);
|
|
expect(
|
|
[...operations(id), ...operations(other)].every(
|
|
(operation) => operation.kind !== 'runtime.pull'
|
|
)
|
|
).toBe(true);
|
|
|
|
verified(id, old);
|
|
vi.setSystemTime(Date.now() + 1);
|
|
hosts.reportHostOperation(id, download.id, { state: 'completed' });
|
|
heartbeat(id, {
|
|
preparedRuntimeImages: [runtime, image],
|
|
libraries: [{ id: old.id, state: 'ready' }]
|
|
});
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)[0]).toMatchObject({
|
|
kind: 'library.download',
|
|
payload: { libraryVersionId: latest.id }
|
|
});
|
|
}
|
|
);
|
|
|
|
it('retains an undelivered library download for an accepted intent until that intent is cancelled', async () => {
|
|
const old = await publish();
|
|
const id = enroll({ preparedRuntimeImages: [runtime] });
|
|
preparation.reconcileHostPreparation(id);
|
|
const download = operations(id)[0];
|
|
const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
|
|
const launch = queueNativeCreation({
|
|
game: 't5',
|
|
gameMode: 'zombie',
|
|
mapSlug: 'theater',
|
|
libraryId: old.id,
|
|
hostId: id,
|
|
image: runtime,
|
|
plutoKey: 'saved-key'
|
|
});
|
|
expect(launch.ok).toBe(true);
|
|
if (!launch.ok) throw new Error(launch.message);
|
|
const latest = await publish();
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(hosts.getHostOperation(download.id)?.state).toBe('pending');
|
|
expect(operations(id)).toHaveLength(1);
|
|
|
|
const { cancelCreation } =
|
|
await import('../../src/lib/server/provisioning/cancel-creation.js');
|
|
expect(cancelCreation(launch.deploymentId)).toEqual(expect.any(String));
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(hosts.getHostOperation(download.id)).toMatchObject({
|
|
state: 'failed',
|
|
error: expect.stringContaining('Superseded')
|
|
});
|
|
expect(operations(id)[0]).toMatchObject({
|
|
kind: 'library.download',
|
|
payload: { libraryVersionId: latest.id }
|
|
});
|
|
});
|
|
|
|
it('finishes a delivered download before following a newer upload under a new operation ID', async () => {
|
|
const old = await publish();
|
|
const id = enroll({ preparedRuntimeImages: [runtime] });
|
|
preparation.reconcileHostPreparation(id);
|
|
const [first] = hosts.pendingHostOperations(id);
|
|
const latest = await publish();
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(1);
|
|
expect(hosts.getHostOperation(first.id)?.state).toBe('pending');
|
|
verified(id, old);
|
|
hosts.reportHostOperation(id, first.id, { state: 'completed' });
|
|
heartbeat(id, {
|
|
preparedRuntimeImages: [runtime],
|
|
libraries: [{ id: old.id, state: 'ready' }]
|
|
});
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)[0]).toMatchObject({
|
|
state: 'pending',
|
|
payload: { libraryVersionId: latest.id }
|
|
});
|
|
expect(libraries.requireReadyLibrary(id, old.id).id).toBe(old.id);
|
|
});
|
|
|
|
it.each(['draining', 'disabled'] as const)('does not start setup for a %s host', (mode) => {
|
|
const id = enroll();
|
|
hosts.updateHost(id, { mode });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toEqual([]);
|
|
});
|
|
|
|
it.each([
|
|
{ name: 'incompatible protocol', value: { protocolVersion: 2 } },
|
|
{ name: 'unavailable Docker', value: { dockerRunning: false } },
|
|
{ name: 'unwritable managed directories', value: { directoriesWritable: false } },
|
|
{ name: 'insufficient storage reserve', value: { storageAvailable: 0 } }
|
|
])('does not start setup with $name', ({ value }) => {
|
|
const id = enroll(value);
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('blocked');
|
|
expect(operations(id)).toEqual([]);
|
|
});
|
|
|
|
it('does not start setup for unenrolled, disconnected, revoked, or decommissioning hosts', () => {
|
|
const unenrolled = hosts.createHost({ name: 'Not enrolled' }).id;
|
|
preparation.reconcileHostPreparation(unenrolled);
|
|
expect(operations(unenrolled)).toEqual([]);
|
|
const offline = enroll();
|
|
vi.setSystemTime(Date.now() + 91_000);
|
|
preparation.reconcileHostPreparation(offline);
|
|
expect(operations(offline)).toEqual([]);
|
|
const revoked = enroll();
|
|
hosts.revokeCredentials(revoked);
|
|
preparation.reconcileHostPreparation(revoked);
|
|
expect(operations(revoked)).toEqual([]);
|
|
const deleting = enroll();
|
|
hosts.deleteHost(deleting);
|
|
preparation.reconcileHostPreparation(deleting);
|
|
expect(operations(deleting)).toEqual([]);
|
|
});
|
|
|
|
it('retries runtime failures with exponential delays capped at fifteen minutes', () => {
|
|
const id = enroll();
|
|
preparation.reconcileHostPreparation(id);
|
|
for (const [attempt, delay] of [
|
|
60_000, 120_000, 240_000, 480_000, 900_000, 900_000
|
|
].entries()) {
|
|
const operation = operations(id)[0];
|
|
vi.setSystemTime(Date.now() + 1);
|
|
hosts.reportHostOperation(id, operation.id, {
|
|
state: 'failed',
|
|
error: 'Registry unavailable'
|
|
});
|
|
const failedAt = Date.now();
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(attempt + 1);
|
|
vi.setSystemTime(failedAt + delay - 2);
|
|
heartbeat(id);
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(attempt + 1);
|
|
vi.setSystemTime(failedAt + delay);
|
|
heartbeat(id);
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(attempt + 2);
|
|
expect(operations(id)[0]).toMatchObject({ kind: 'runtime.pull', state: 'pending' });
|
|
}
|
|
});
|
|
|
|
it('blocks a library that cannot fit while preserving the host free-space reserve', async () => {
|
|
const version = await publish();
|
|
const id = enroll({
|
|
preparedRuntimeImages: [runtime],
|
|
storageAvailable: 1_000 + version.sizeBytes - 1
|
|
});
|
|
hosts.updateHost(id, { minimumFreeBytes: 1_000 });
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('blocked');
|
|
expect(operations(id)).toEqual([]);
|
|
heartbeat(id, {
|
|
preparedRuntimeImages: [runtime],
|
|
storageAvailable: 1_000 + version.sizeBytes
|
|
});
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('preparing');
|
|
expect(operations(id)[0].payload.libraryVersionId).toBe(version.id);
|
|
});
|
|
|
|
it('returns an active download even when its in-progress bytes have consumed free space', async () => {
|
|
const version = await publish();
|
|
const id = enroll({ preparedRuntimeImages: [runtime] });
|
|
const operation = libraries.queueLibraryDownload(id, version.id);
|
|
heartbeat(id, {
|
|
preparedRuntimeImages: [runtime],
|
|
storageAvailable: hosts.getHost(id)!.minimumFreeBytes
|
|
});
|
|
expect(libraries.queueLibraryDownload(id, version.id).id).toBe(operation.id);
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(1);
|
|
});
|
|
|
|
it('prepares T6 when the latest T5 upload cannot fit', async () => {
|
|
const oversized = await publish();
|
|
const small = await publish('t6', 'small');
|
|
const id = enroll({ preparedRuntimeImages: [runtime], storageAvailable: 1_010 });
|
|
hosts.updateHost(id, { minimumFreeBytes: 1_000 });
|
|
const status = preparation.reconcileHostPreparation(id);
|
|
expect(status.items).toEqual(
|
|
expect.arrayContaining([
|
|
expect.objectContaining({ label: 'Black Ops I', state: 'blocked' }),
|
|
expect.objectContaining({ label: 'Black Ops II', state: 'preparing' })
|
|
])
|
|
);
|
|
expect(operations(id)).toHaveLength(1);
|
|
expect(operations(id)[0].payload.libraryVersionId).toBe(small.id);
|
|
expect(libraries.getHostLibraryCache(id, oversized.id)).toBeNull();
|
|
});
|
|
|
|
it('does not count partial bytes from a different manifest toward download storage', async () => {
|
|
const version = await publish();
|
|
const id = enroll({ preparedRuntimeImages: [runtime] });
|
|
libraries.updateHostLibraryCaches(id, [
|
|
{
|
|
libraryVersionId: version.id,
|
|
status: 'failed',
|
|
bytesDownloaded: 80,
|
|
manifestSha256: '0'.repeat(64)
|
|
}
|
|
]);
|
|
heartbeat(id, {
|
|
preparedRuntimeImages: [runtime],
|
|
storageAvailable: hosts.getHost(id)!.minimumFreeBytes + 20
|
|
});
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('blocked');
|
|
expect(() => libraries.queueLibraryDownload(id, version.id)).toThrow(
|
|
/insufficient storage/
|
|
);
|
|
expect(operations(id)).toEqual([]);
|
|
});
|
|
|
|
it('resumes a failed verified-manifest download using only its remaining storage', async () => {
|
|
const version = await publish();
|
|
const id = enroll({ preparedRuntimeImages: [runtime] });
|
|
preparation.reconcileHostPreparation(id);
|
|
const operation = operations(id)[0];
|
|
libraries.updateHostLibraryCaches(id, [
|
|
{
|
|
libraryVersionId: version.id,
|
|
status: 'downloading',
|
|
bytesDownloaded: 80,
|
|
manifestSha256: version.manifestSha256
|
|
}
|
|
]);
|
|
vi.setSystemTime(Date.now() + 1);
|
|
hosts.reportHostOperation(id, operation.id, { state: 'failed', error: 'Connection reset' });
|
|
const failedAt = Date.now();
|
|
const free = hosts.getHost(id)!.minimumFreeBytes + version.sizeBytes - 80;
|
|
heartbeat(id, { preparedRuntimeImages: [runtime], storageAvailable: free });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(1);
|
|
vi.setSystemTime(failedAt + 60_000);
|
|
heartbeat(id, { preparedRuntimeImages: [runtime], storageAvailable: free });
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('preparing');
|
|
expect(operations(id)).toHaveLength(2);
|
|
expect(operations(id)[0].payload.libraryVersionId).toBe(version.id);
|
|
expect(libraries.getHostLibraryCache(id, version.id)?.bytesDownloaded).toBe(80);
|
|
});
|
|
});
|
|
|
|
describe('optional host Plutonium cache', () => {
|
|
const capable = { capabilities: ['host.decommission', 'plutonium.cache'] };
|
|
async function release(
|
|
revision: number,
|
|
overrides: { checkedAt?: string; error?: string | null } = {}
|
|
) {
|
|
const { writeSetting } = await import('../../src/lib/server/maintenance/settings.js');
|
|
writeSetting('plutonium_release', {
|
|
revision,
|
|
checkedAt: new Date().toISOString(),
|
|
error: null,
|
|
...overrides
|
|
});
|
|
}
|
|
|
|
it.each(['pending', 'accepted', 'running', 'unknown'] as const)(
|
|
'waits to warm while a deployment is %s',
|
|
(state) => {
|
|
const ready = {
|
|
...capable,
|
|
preparedRuntimeImages: [runtime],
|
|
libraries: [{ id: 'game-files', state: 'ready' as const }]
|
|
};
|
|
const id = enroll(ready);
|
|
const deployment = hosts.queueHostOperation(id, 'instance.create', {});
|
|
if (state !== 'pending') hosts.reportHostOperation(id, deployment.id, { state });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(hosts.shouldDeferPlutoniumPreparation(id)).toBe(true);
|
|
expect(operations(id)).toEqual([]);
|
|
hosts.reportHostOperation(id, deployment.id, { state: 'completed' });
|
|
heartbeat(id, ready);
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toMatchObject([{ kind: 'plutonium.prepare' }]);
|
|
}
|
|
);
|
|
|
|
it('delivers newer deployments before queued cache work and before a cache replay', () => {
|
|
const id = enroll({
|
|
...capable,
|
|
preparedRuntimeImages: [runtime],
|
|
libraries: [{ id: 'game-files', state: 'ready' }]
|
|
});
|
|
preparation.reconcileHostPreparation(id);
|
|
const cache = operations(id)[0];
|
|
const deployment = hosts.queueHostOperation(id, 'instance.create', {});
|
|
expect(hosts.pendingHostOperations(id, 'write').map((operation) => operation.id)).toEqual([
|
|
deployment.id,
|
|
cache.id
|
|
]);
|
|
hosts.reportHostOperation(id, cache.id, { state: 'running' });
|
|
const change = hosts.queueHostOperation(id, 'files.stage', {});
|
|
const delivery = hosts.pendingHostOperations(id, 'write').map((operation) => operation.id);
|
|
expect(delivery.at(-1)).toBe(cache.id);
|
|
expect(delivery).toContain(deployment.id);
|
|
expect(delivery).toContain(change.id);
|
|
});
|
|
|
|
it('signals deferral on heartbeats for mutations while ignoring cache work and all observation lanes', async () => {
|
|
const host = hosts.createHost({ name: 'Cache yielding host' });
|
|
const ready = {
|
|
...inventory,
|
|
...capable,
|
|
preparedRuntimeImages: [runtime],
|
|
libraries: [{ id: 'game-files', state: 'ready' as const }]
|
|
};
|
|
const { credential } = hosts.enrollHost(hosts.createEnrollment(host.id).token, ready);
|
|
preparation.reconcileHostPreparation(host.id);
|
|
const cache = operations(host.id)[0];
|
|
hosts.reportHostOperation(host.id, cache.id, { state: 'running' });
|
|
for (const kind of [
|
|
'host.inspect',
|
|
'instance.inspect',
|
|
'rcon.status-batch',
|
|
'files.read',
|
|
'files.list',
|
|
'files.stat',
|
|
'files.manifest',
|
|
'logs.read'
|
|
]) {
|
|
hosts.queueHostOperation(host.id, kind, {});
|
|
}
|
|
for (const command of ['status', 'getstatus', 'version'])
|
|
hosts.queueHostOperation(host.id, 'rcon.execute', { command });
|
|
// Work belonging to another host must not pause this one's cache.
|
|
const other = enroll(ready);
|
|
hosts.queueHostOperation(other, 'instance.restart', {});
|
|
const { POST } = await import('../../src/routes/api/agent/heartbeat/+server.js');
|
|
const poll = async () => {
|
|
vi.setSystemTime(Date.now() + 1);
|
|
const response = await POST({
|
|
request: new Request('https://controller.test/api/agent/heartbeat', {
|
|
method: 'POST',
|
|
headers: {
|
|
Authorization: `Bearer ${credential}`,
|
|
'Content-Type': 'application/json'
|
|
},
|
|
body: JSON.stringify({ inventory: ready })
|
|
})
|
|
});
|
|
expect(response.status).toBe(200);
|
|
return response.json();
|
|
};
|
|
expect(await poll()).toMatchObject({ deferPlutoniumPreparation: false });
|
|
const change = hosts.queueHostOperation(host.id, 'rcon.execute', {
|
|
command: 'map_restart'
|
|
});
|
|
expect(await poll()).toMatchObject({ deferPlutoniumPreparation: true });
|
|
hosts.reportHostOperation(host.id, change.id, {
|
|
state: 'failed',
|
|
error: 'Command rejected'
|
|
});
|
|
expect(await poll()).toMatchObject({ deferPlutoniumPreparation: false });
|
|
expect(
|
|
operations(host.id)
|
|
.filter((operation) => operation.kind === 'plutonium.prepare')
|
|
.map((operation) => operation.id)
|
|
).toEqual([cache.id]);
|
|
});
|
|
|
|
it('lets newly published game files interrupt cache refresh, then resumes the same cache operation', async () => {
|
|
const version = await publish();
|
|
const ready = {
|
|
...capable,
|
|
preparedRuntimeImages: [runtime],
|
|
libraries: [{ id: version.id, state: 'ready' as const }],
|
|
plutoniumCache: { revision: 5354, bytes: 200 }
|
|
};
|
|
const id = enroll(ready);
|
|
verified(id, version);
|
|
await release(5355);
|
|
preparation.reconcileHostPreparation(id);
|
|
const cache = operations(id)[0];
|
|
expect(cache).toMatchObject({ kind: 'plutonium.prepare', payload: { revision: 5355 } });
|
|
hosts.reportHostOperation(id, cache.id, { state: 'running' });
|
|
const latest = await publish();
|
|
heartbeat(id, ready);
|
|
preparation.reconcileHostPreparation(id);
|
|
const download = operations(id)[0];
|
|
expect(download).toMatchObject({
|
|
kind: 'library.download',
|
|
payload: { libraryVersionId: latest.id }
|
|
});
|
|
expect(hosts.shouldDeferPlutoniumPreparation(id)).toBe(true);
|
|
expect(hosts.pendingHostOperations(id, 'write').map((operation) => operation.id)).toEqual([
|
|
download.id,
|
|
cache.id
|
|
]);
|
|
verified(id, latest);
|
|
hosts.reportHostOperation(id, download.id, { state: 'completed' });
|
|
heartbeat(id, { ...ready, libraries: [{ id: latest.id, state: 'ready' }] });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(hosts.shouldDeferPlutoniumPreparation(id)).toBe(false);
|
|
expect(hosts.pendingHostOperations(id, 'write').map((operation) => operation.id)).toEqual([
|
|
cache.id
|
|
]);
|
|
expect(
|
|
operations(id).filter((operation) => operation.kind === 'plutonium.prepare')
|
|
).toHaveLength(1);
|
|
});
|
|
|
|
it('warms only after required files, preserves launch readiness, and waits for fresh cache inventory', async () => {
|
|
const version = await publish();
|
|
await release(5354);
|
|
const id = enroll(capable);
|
|
preparation.reconcileHostPreparation(id);
|
|
const pull = operations(id)[0];
|
|
expect(pull.kind).toBe('runtime.pull');
|
|
hosts.reportHostOperation(id, pull.id, { state: 'completed' });
|
|
heartbeat(id, { ...capable, preparedRuntimeImages: [runtime] });
|
|
preparation.reconcileHostPreparation(id);
|
|
const download = operations(id)[0];
|
|
expect(download.kind).toBe('library.download');
|
|
verified(id, version);
|
|
hosts.reportHostOperation(id, download.id, { state: 'completed' });
|
|
const ready = {
|
|
...capable,
|
|
preparedRuntimeImages: [runtime],
|
|
libraries: [{ id: version.id, state: 'ready' as const }]
|
|
};
|
|
heartbeat(id, ready);
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('ready');
|
|
const cache = operations(id)[0];
|
|
expect(cache).toMatchObject({
|
|
kind: 'plutonium.prepare',
|
|
payload: {
|
|
revision: 5354,
|
|
minimumFreeBytes: hosts.getHost(id)!.minimumFreeBytes,
|
|
automatic: true
|
|
}
|
|
});
|
|
expect(hosts.requireReadyHost(id).readiness).toBe('ready');
|
|
expect(hosts.pendingHostOperations(id, 'read')).toEqual([]);
|
|
expect(hosts.pendingHostOperations(id, 'write').map((operation) => operation.id)).toEqual([
|
|
cache.id
|
|
]);
|
|
expect(
|
|
hosts.listHostOperationHistory(1, 'important', { host: id }).operations[0]
|
|
).toMatchObject({
|
|
id: cache.id,
|
|
routine: false,
|
|
targetPath: 'Production revision r5354'
|
|
});
|
|
preparation.reconcileHostPreparation(id);
|
|
hosts.reportHostOperation(id, cache.id, { state: 'completed' });
|
|
const pendingInventory = preparation.reconcileHostPreparation(id);
|
|
expect(pendingInventory.items.find((item) => item.kind === 'plutonium')).toMatchObject({
|
|
state: 'preparing'
|
|
});
|
|
expect(operations(id)).toHaveLength(3);
|
|
heartbeat(id, { ...ready, plutoniumCache: { revision: 5354, bytes: 25_000 } });
|
|
expect(
|
|
preparation.reconcileHostPreparation(id).items.find((item) => item.kind === 'plutonium')
|
|
).toMatchObject({
|
|
state: 'ready',
|
|
detail: expect.stringContaining('r5354')
|
|
});
|
|
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
|
closeDb();
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(3);
|
|
});
|
|
|
|
it('refreshes for newer production releases and reconciles an interrupted refresh without duplicates', async () => {
|
|
const ready = {
|
|
...capable,
|
|
preparedRuntimeImages: [runtime],
|
|
plutoniumCache: { revision: 5354, bytes: 25_000 }
|
|
};
|
|
const id = enroll(ready);
|
|
await release(5354);
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toEqual([]);
|
|
await release(5355);
|
|
preparation.reconcileHostPreparation(id);
|
|
const cache = operations(id)[0];
|
|
expect(cache.payload.revision).toBe(5355);
|
|
hosts.reportHostOperation(id, cache.id, { state: 'unknown' });
|
|
await release(5356);
|
|
heartbeat(id, ready);
|
|
const { closeDb } = await import('../../src/lib/server/db/index.js');
|
|
closeDb();
|
|
expect(
|
|
preparation.reconcileHostPreparation(id).items.find((item) => item.kind === 'plutonium')
|
|
).toMatchObject({
|
|
state: 'preparing',
|
|
detail: expect.stringContaining('reconcile interrupted')
|
|
});
|
|
expect(operations(id)).toHaveLength(1);
|
|
expect(hosts.pendingHostOperations(id, 'write').map((operation) => operation.id)).toEqual([
|
|
cache.id
|
|
]);
|
|
hosts.reportHostOperation(id, cache.id, { state: 'completed' });
|
|
heartbeat(id, { ...ready, plutoniumCache: { revision: 5356, bytes: 26_000 } });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(1);
|
|
await release(5355);
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(1);
|
|
});
|
|
|
|
it.each(['missing', 'stale', 'failed'] as const)(
|
|
'uses the agent production manifest when release evidence is %s',
|
|
async (evidence) => {
|
|
if (evidence === 'stale')
|
|
await release(5354, {
|
|
checkedAt: new Date(Date.now() - 60 * 60_000).toISOString()
|
|
});
|
|
if (evidence === 'failed') await release(5354, { error: 'Manifest unavailable' });
|
|
const id = enroll({ ...capable, preparedRuntimeImages: [runtime] });
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)[0]).toMatchObject({
|
|
kind: 'plutonium.prepare',
|
|
payload: { revision: 0 }
|
|
});
|
|
expect(
|
|
hosts.listHostOperationHistory(1, 'important', { host: id }).operations[0]
|
|
.targetPath
|
|
).toBe('Latest production revision');
|
|
const cached = enroll({
|
|
...capable,
|
|
preparedRuntimeImages: [runtime],
|
|
plutoniumCache: { revision: 5353, bytes: 200 }
|
|
});
|
|
preparation.reconcileHostPreparation(cached);
|
|
expect(operations(cached)).toEqual([]);
|
|
}
|
|
);
|
|
|
|
it('leaves agents without the cache capability unchanged', async () => {
|
|
await release(5355);
|
|
const id = enroll({
|
|
preparedRuntimeImages: [runtime],
|
|
plutoniumCache: { revision: 5354, bytes: 200 }
|
|
});
|
|
expect(
|
|
preparation.reconcileHostPreparation(id).items.some((item) => item.kind === 'plutonium')
|
|
).toBe(false);
|
|
expect(operations(id)).toEqual([]);
|
|
});
|
|
|
|
it('backs off failed cache work without blocking launch readiness or newly published game files', async () => {
|
|
const version = await publish();
|
|
const ready = {
|
|
...capable,
|
|
preparedRuntimeImages: [runtime],
|
|
libraries: [{ id: version.id, state: 'ready' as const }]
|
|
};
|
|
const id = enroll(ready);
|
|
verified(id, version);
|
|
await release(5354);
|
|
preparation.reconcileHostPreparation(id);
|
|
const cache = operations(id)[0];
|
|
hosts.reportHostOperation(id, cache.id, {
|
|
state: 'failed',
|
|
error: 'Plutonium CDN unavailable'
|
|
});
|
|
const failedAt = Date.now();
|
|
heartbeat(id, ready);
|
|
const status = preparation.reconcileHostPreparation(id);
|
|
expect(status).toMatchObject({ state: 'ready' });
|
|
expect(status.items.find((item) => item.kind === 'plutonium')).toMatchObject({
|
|
state: 'blocked',
|
|
detail: expect.stringContaining('Automatic retry')
|
|
});
|
|
expect(hosts.requireReadyHost(id).readiness).toBe('ready');
|
|
expect(operations(id)).toHaveLength(1);
|
|
// A changed or expired release must not bypass a failed cache's backoff.
|
|
await release(5355);
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(1);
|
|
const latest = await publish();
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)[0]).toMatchObject({
|
|
kind: 'library.download',
|
|
payload: { libraryVersionId: latest.id }
|
|
});
|
|
verified(id, latest);
|
|
hosts.reportHostOperation(id, operations(id)[0].id, { state: 'completed' });
|
|
heartbeat(id, ready);
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(2);
|
|
vi.setSystemTime(failedAt + 60_000);
|
|
heartbeat(id, ready);
|
|
preparation.reconcileHostPreparation(id);
|
|
expect(operations(id)).toHaveLength(3);
|
|
expect(operations(id)[0]).toMatchObject({
|
|
kind: 'plutonium.prepare',
|
|
payload: { revision: 5355 }
|
|
});
|
|
});
|
|
|
|
it('waits for required library capacity before consuming space for the optional cache', async () => {
|
|
const version = await publish();
|
|
const id = enroll({
|
|
...capable,
|
|
preparedRuntimeImages: [runtime],
|
|
storageAvailable: 1_000 + version.sizeBytes - 1
|
|
});
|
|
hosts.updateHost(id, { minimumFreeBytes: 1_000 });
|
|
expect(preparation.reconcileHostPreparation(id).state).toBe('blocked');
|
|
expect(operations(id)).toEqual([]);
|
|
});
|
|
|
|
it.each([
|
|
null,
|
|
[],
|
|
{},
|
|
{ revision: 0, bytes: 1 },
|
|
{ revision: 1.5, bytes: 1 },
|
|
{ revision: 1, bytes: -1 },
|
|
{ revision: 1, bytes: 0.5 }
|
|
])('rejects invalid cache inventory %j', (cache) => {
|
|
const id = enroll();
|
|
expect(() =>
|
|
heartbeat(id, { plutoniumCache: cache as HostInventory['plutoniumCache'] })
|
|
).toThrow('Invalid Plutonium cache inventory');
|
|
});
|
|
|
|
it('retains only revision and byte count from cache inventory and clears an omitted cache', () => {
|
|
const id = enroll({
|
|
plutoniumCache: {
|
|
revision: 5354,
|
|
bytes: 0,
|
|
localPath: '/private/cache'
|
|
} as HostInventory['plutoniumCache']
|
|
});
|
|
expect(hosts.getHost(id)?.inventory.plutoniumCache).toEqual({ revision: 5354, bytes: 0 });
|
|
heartbeat(id);
|
|
expect(hosts.getHost(id)?.inventory.plutoniumCache).toBeUndefined();
|
|
});
|
|
});
|