728 lines
31 KiB
TypeScript
728 lines
31 KiB
TypeScript
import { initializeStandaloneSettings } from '../helpers/settings.js';
|
|
import Database from 'better-sqlite3';
|
|
import { createHash } from 'node:crypto';
|
|
import { mkdtemp, readFile, rm } from 'node:fs/promises';
|
|
import { tmpdir } from 'node:os';
|
|
import { join } from 'node:path';
|
|
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
const state = vi.hoisted(() => ({ handle: null as Database.Database | null }));
|
|
vi.mock('../../src/lib/server/db/index.js', () => ({ db: () => state.handle }));
|
|
import {
|
|
heartbeat,
|
|
createEnrollment,
|
|
createHost,
|
|
enrollHost,
|
|
getHostOperation,
|
|
queueHostOperation,
|
|
reportHostOperation,
|
|
revokeCredentials
|
|
} from '../../src/lib/server/hosts/index.js';
|
|
import {
|
|
assertReadyLibrary,
|
|
cleanupControllerLibraries,
|
|
reconcileHostLibraryCleanup,
|
|
createLibraryImport,
|
|
getHostLibraryCache,
|
|
getLibraryVersion,
|
|
importPublicationStatus,
|
|
latestPublishedLibraries,
|
|
libraryOverview,
|
|
listLibraries,
|
|
newestReadyLibrary,
|
|
publishImport,
|
|
queueLibraryDownload,
|
|
requestImportPublication,
|
|
requireReadyLibrary,
|
|
setImportManifest,
|
|
updateHostLibraryCaches,
|
|
uploadImportFile
|
|
} from '../../src/lib/server/game-files/index.js';
|
|
import { serveGameFile } from '../../src/lib/server/game-files/http.js';
|
|
import { GET as manifestRoute } from '../../src/routes/api/agent/libraries/[id]/manifest/+server.js';
|
|
|
|
let directory: string;
|
|
let hostId: string;
|
|
let credential: string;
|
|
const data = 'real game bytes';
|
|
const file = {
|
|
path: 'zone/game.ff',
|
|
size: Buffer.byteLength(data),
|
|
sha256: createHash('sha256').update(data).digest('hex')
|
|
};
|
|
beforeEach(async () => {
|
|
state.handle = new Database(':memory:');
|
|
initializeStandaloneSettings(state.handle);
|
|
state.handle.pragma('foreign_keys=ON');
|
|
directory = await mkdtemp(join(tmpdir(), 'ec-libraries-'));
|
|
vi.stubEnv('GAME_FILES_ROOT', directory);
|
|
const host = createHost({ name: 'Debian host' });
|
|
hostId = host.id;
|
|
const enrollment = createEnrollment(host.id);
|
|
credential = enrollHost(enrollment.token, {
|
|
protocolVersion: 1,
|
|
dockerRunning: true,
|
|
directoriesWritable: true,
|
|
storageAvailable: 1_000_000_000_000
|
|
}).credential;
|
|
});
|
|
afterEach(async () => {
|
|
state.handle?.close();
|
|
await rm(directory, { recursive: true, force: true });
|
|
vi.unstubAllEnvs();
|
|
});
|
|
function begin() {
|
|
return createLibraryImport({ hostId, game: 't6' });
|
|
}
|
|
async function published() {
|
|
const version = begin();
|
|
setImportManifest(hostId, version.id, [file]);
|
|
await uploadImportFile(hostId, version.id, file.path, new Response(data).body);
|
|
return publishImport(hostId, version.id);
|
|
}
|
|
|
|
function legacyLibrary(id: string, game: 't5' | 't6') {
|
|
state
|
|
.handle!.prepare(
|
|
'INSERT INTO game_file_libraries(id,name,game,description,created_at) VALUES(?,?,?,?,?)'
|
|
)
|
|
.run(id, `User named ${id}`, game, 'Original description', '2025-01-01T00:00:00.000Z');
|
|
}
|
|
function legacyVersion(id: string, libraryId: string, createdAt: string, publishedAt: string) {
|
|
state
|
|
.handle!.prepare(
|
|
`INSERT INTO game_file_versions(id,library_id,version,description,dlc,state,created_at,published_at,storage_location,source_host_id,source_volume,manifest_sha256)
|
|
VALUES(?,?,?,'Legacy version','[]','ready',?,?,?,'','','checksum')`
|
|
)
|
|
.run(id, libraryId, `User label ${id}`, createdAt, publishedAt, join(directory, id));
|
|
}
|
|
|
|
describe('fixed game file slots', () => {
|
|
it('always lists exactly T5 and T6, including before any files are uploaded', () => {
|
|
expect(listLibraries()).toEqual([
|
|
{
|
|
id: 't5',
|
|
game: 't5',
|
|
name: 'Black Ops I',
|
|
description: '',
|
|
createdAt: '',
|
|
latestVersionId: null,
|
|
versions: []
|
|
},
|
|
{
|
|
id: 't6',
|
|
game: 't6',
|
|
name: 'Black Ops II',
|
|
description: '',
|
|
createdAt: '',
|
|
latestVersionId: null,
|
|
versions: []
|
|
}
|
|
]);
|
|
expect(latestPublishedLibraries()).toEqual([]);
|
|
expect(newestReadyLibrary('t5')).toBeNull();
|
|
expect(newestReadyLibrary('t6')).toBeNull();
|
|
});
|
|
|
|
it('unifies old named libraries by game without rewriting published versions or their host references', () => {
|
|
listLibraries();
|
|
legacyLibrary('first-t6-library', 't6');
|
|
legacyLibrary('other-t6-library', 't6');
|
|
legacyLibrary('t5-library', 't5');
|
|
legacyVersion('old-t6', 'first-t6-library', '2025-01-03', '2025-01-04');
|
|
legacyVersion('new-t6', 'other-t6-library', '2025-01-01', '2025-01-05');
|
|
legacyVersion('t5-files', 't5-library', '2025-01-01', '2025-01-02');
|
|
updateHostLibraryCaches(hostId, [
|
|
{ libraryVersionId: 'old-t6', status: 'ready', manifestSha256: 'checksum' },
|
|
{ libraryVersionId: 'new-t6', status: 'ready', manifestSha256: 'checksum' }
|
|
]);
|
|
const before = getLibraryVersion('old-t6');
|
|
const slots = listLibraries();
|
|
expect(slots.map((library) => library.id)).toEqual(['t5', 't6']);
|
|
expect(slots[1].versions.map((version) => version.id)).toEqual(['old-t6', 'new-t6']);
|
|
expect(slots.map((library) => library.latestVersionId)).toEqual(['t5-files', 'new-t6']);
|
|
expect(latestPublishedLibraries().map((version) => version.id)).toEqual([
|
|
't5-files',
|
|
'new-t6'
|
|
]);
|
|
expect(newestReadyLibrary('t6')?.id).toBe('new-t6');
|
|
const overview = libraryOverview().libraries[1];
|
|
expect(overview.versions.find((version) => version.id === 'old-t6')?.caches[0].status).toBe(
|
|
'outdated'
|
|
);
|
|
expect(overview.versions.find((version) => version.id === 'new-t6')?.caches[0].status).toBe(
|
|
'ready'
|
|
);
|
|
expect(getLibraryVersion('old-t6')).toEqual(before);
|
|
expect(requireReadyLibrary(hostId, 'old-t6')).toMatchObject({
|
|
libraryId: 'first-t6-library',
|
|
version: 'User label old-t6',
|
|
storageLocation: join(directory, 'old-t6')
|
|
});
|
|
expect(begin().libraryId).toBe('first-t6-library');
|
|
expect(listLibraries()[1].latestVersionId).toBe('new-t6');
|
|
});
|
|
|
|
it('breaks publication-time ties by creation time and then insertion order across old libraries', () => {
|
|
listLibraries();
|
|
legacyLibrary('first-library', 't6');
|
|
legacyLibrary('second-library', 't6');
|
|
legacyVersion('first', 'first-library', '2025-01-01', '2025-01-05');
|
|
legacyVersion('second', 'second-library', '2025-01-02', '2025-01-05');
|
|
legacyVersion('third', 'first-library', '2025-01-02', '2025-01-05');
|
|
legacyVersion('later-created', 'second-library', '2025-01-04', '2025-01-04');
|
|
expect(newestReadyLibrary('t6')?.id).toBe('third');
|
|
expect(latestPublishedLibraries().map((version) => version.id)).toEqual(['third']);
|
|
expect(listLibraries()[1].latestVersionId).toBe('third');
|
|
});
|
|
|
|
it('ignores user-supplied library metadata when importing from hosts', () => {
|
|
const first = createLibraryImport({
|
|
hostId,
|
|
game: 't6',
|
|
name: 'shared',
|
|
version: '1',
|
|
description: 'Old label'
|
|
});
|
|
const next = createLibraryImport({
|
|
hostId,
|
|
game: 't6',
|
|
name: 'shared',
|
|
version: '1',
|
|
description: 'Old label'
|
|
});
|
|
expect(first.description).toBe('');
|
|
expect(first.version).not.toBe('1');
|
|
expect(next.version).not.toBe(first.version);
|
|
expect(first.libraryId).toBe(next.libraryId);
|
|
});
|
|
});
|
|
|
|
describe('game-file import lifecycle', () => {
|
|
it('publishes verified filesystem content and permanently pins manifest metadata', async () => {
|
|
const initial = begin();
|
|
expect(getHostOperation(initial.operationId!)?.payload.sourceVolume).toBe('t6_game_files');
|
|
setImportManifest(hostId, initial.id, [file]);
|
|
await expect(publishImport(hostId, initial.id)).rejects.toThrow();
|
|
expect(getLibraryVersion(initial.id)?.state).toBe('importing');
|
|
await uploadImportFile(hostId, initial.id, file.path, new Response(data).body);
|
|
const ready = await publishImport(hostId, initial.id);
|
|
expect(ready.state).toBe('ready');
|
|
expect(ready.sizeBytes).toBe(data.length);
|
|
expect(await readFile(join(ready.storageLocation, file.path), 'utf8')).toBe(data);
|
|
expect((await publishImport(hostId, ready.id)).id).toBe(ready.id);
|
|
await expect(
|
|
uploadImportFile(hostId, ready.id, file.path, new Response(data).body)
|
|
).rejects.toThrow(/not writable/);
|
|
expect(() =>
|
|
state
|
|
.handle!.prepare('UPDATE game_file_versions SET description=? WHERE id=?')
|
|
.run('changed', ready.id)
|
|
).toThrow(/immutable/);
|
|
const next = begin();
|
|
expect(next.id).not.toBe(ready.id);
|
|
expect(next.version).not.toBe(ready.version);
|
|
expect(next.libraryId).toBe(ready.libraryId);
|
|
});
|
|
it('restricts an import to its source host and rejects manifest mutation', () => {
|
|
const version = begin();
|
|
expect(() => setImportManifest('another-host', version.id, [file])).toThrow(/not found/);
|
|
const first = setImportManifest(hostId, version.id, [file]);
|
|
expect(setImportManifest(hostId, version.id, [file]).manifestSha256).toBe(
|
|
first.manifestSha256
|
|
);
|
|
expect(() =>
|
|
setImportManifest(hostId, version.id, [{ ...file, path: 'changed.ff' }])
|
|
).toThrow(/cannot change/);
|
|
});
|
|
it('surfaces durable agent failures while preserving historical versions', () => {
|
|
const version = begin();
|
|
reportHostOperation(hostId, version.operationId!, {
|
|
state: 'failed',
|
|
error: 'Source volume not found'
|
|
});
|
|
expect(listLibraries().find((library) => library.game === 't6')?.versions[0]).toMatchObject(
|
|
{ state: 'failed', error: 'Source volume not found' }
|
|
);
|
|
});
|
|
it('returns a pollable publication status while large files are verified asynchronously', async () => {
|
|
const version = begin();
|
|
setImportManifest(hostId, version.id, [file]);
|
|
await uploadImportFile(hostId, version.id, file.path, new Response(data).body);
|
|
expect(importPublicationStatus(hostId, version.id).status).toBe('pending');
|
|
expect(requestImportPublication(hostId, version.id).status).toBe('verifying');
|
|
expect(requestImportPublication(hostId, version.id).status).toBe('verifying');
|
|
await publishImport(hostId, version.id);
|
|
expect(importPublicationStatus(hostId, version.id)).toMatchObject({
|
|
status: 'published',
|
|
version: { id: version.id, manifestSha256: expect.any(String) }
|
|
});
|
|
});
|
|
});
|
|
describe('verified host distribution', () => {
|
|
it('reuses published metadata across host reports without reading or parsing a large manifest', () => {
|
|
const version = begin();
|
|
const manifest = JSON.stringify(
|
|
Array.from({ length: 20_000 }, (_, i) => ({ ...file, path: `zone/file-${i}.ff` }))
|
|
);
|
|
const size = file.size * 20_000;
|
|
state
|
|
.handle!.prepare(
|
|
"UPDATE game_file_versions SET state='ready',size_bytes=?,manifest=?,manifest_sha256='large-checksum' WHERE id=?"
|
|
)
|
|
.run(size, manifest, version.id);
|
|
const otherHost = createHost({ name: 'Other reporting host' });
|
|
const parse = JSON.parse;
|
|
const parsing = vi.spyOn(JSON, 'parse').mockImplementation((value, reviver) => {
|
|
if (value === manifest)
|
|
throw new Error('Cache reports must not deserialize manifests.');
|
|
return parse(value, reviver);
|
|
});
|
|
const preparing = vi.spyOn(state.handle!, 'prepare');
|
|
try {
|
|
for (const id of [hostId, otherHost.id, hostId]) {
|
|
updateHostLibraryCaches(id, [
|
|
{
|
|
libraryVersionId: version.id,
|
|
status: 'ready',
|
|
manifestSha256: 'large-checksum'
|
|
}
|
|
]);
|
|
expect(getHostLibraryCache(id, version.id)).toMatchObject({
|
|
status: 'ready',
|
|
bytesDownloaded: size,
|
|
bytesTotal: size
|
|
});
|
|
}
|
|
const reads = preparing.mock.calls
|
|
.map(([sql]) => sql)
|
|
.filter((sql) => sql.includes('FROM game_file_versions'));
|
|
expect(reads).toHaveLength(1);
|
|
expect(
|
|
state
|
|
.handle!.prepare(reads[0])
|
|
.columns()
|
|
.map((column) => column.name)
|
|
).toEqual(['state', 'sizeBytes', 'manifestSha256']);
|
|
updateHostLibraryCaches(hostId, [
|
|
{ libraryVersionId: version.id, status: 'ready', manifestSha256: 'wrong' }
|
|
]);
|
|
expect(getHostLibraryCache(hostId, version.id)?.status).toBe('failed');
|
|
expect(() =>
|
|
updateHostLibraryCaches(hostId, [
|
|
{
|
|
libraryVersionId: version.id,
|
|
status: 'downloading',
|
|
bytesDownloaded: size + 1
|
|
}
|
|
])
|
|
).toThrow('Invalid library download progress');
|
|
} finally {
|
|
parsing.mockRestore();
|
|
preparing.mockRestore();
|
|
}
|
|
});
|
|
|
|
it('does not cache missing or unpublished versions', () => {
|
|
updateHostLibraryCaches(hostId, [
|
|
{ libraryVersionId: 'future', status: 'ready', manifestSha256: 'checksum' }
|
|
]);
|
|
const version = begin();
|
|
state
|
|
.handle!.prepare("UPDATE game_file_versions SET id='future' WHERE id=?")
|
|
.run(version.id);
|
|
updateHostLibraryCaches(hostId, [
|
|
{ libraryVersionId: 'future', status: 'ready', manifestSha256: 'checksum' }
|
|
]);
|
|
expect(getHostLibraryCache(hostId, 'future')).toBeNull();
|
|
state
|
|
.handle!.prepare(
|
|
"UPDATE game_file_versions SET state='ready',size_bytes=123,manifest_sha256='checksum' WHERE id='future'"
|
|
)
|
|
.run();
|
|
updateHostLibraryCaches(hostId, [
|
|
{ libraryVersionId: 'future', status: 'ready', manifestSha256: 'checksum' }
|
|
]);
|
|
expect(getHostLibraryCache(hostId, 'future')).toMatchObject({
|
|
status: 'ready',
|
|
bytesTotal: 123
|
|
});
|
|
});
|
|
|
|
it('does not retain metadata from a rolled-back publication', () => {
|
|
const version = begin();
|
|
expect(() =>
|
|
state.handle!.transaction(() => {
|
|
state
|
|
.handle!.prepare(
|
|
"UPDATE game_file_versions SET state='ready',size_bytes=123,manifest_sha256='temporary' WHERE id=?"
|
|
)
|
|
.run(version.id);
|
|
updateHostLibraryCaches(hostId, [
|
|
{ libraryVersionId: version.id, status: 'ready', manifestSha256: 'temporary' }
|
|
]);
|
|
throw new Error('rollback');
|
|
})()
|
|
).toThrow('rollback');
|
|
updateHostLibraryCaches(hostId, [
|
|
{ libraryVersionId: version.id, status: 'ready', manifestSha256: 'temporary' }
|
|
]);
|
|
expect(getHostLibraryCache(hostId, version.id)).toBeNull();
|
|
state
|
|
.handle!.prepare(
|
|
"UPDATE game_file_versions SET state='ready',size_bytes=456,manifest_sha256='published' WHERE id=?"
|
|
)
|
|
.run(version.id);
|
|
updateHostLibraryCaches(hostId, [
|
|
{ libraryVersionId: version.id, status: 'ready', manifestSha256: 'published' }
|
|
]);
|
|
expect(getHostLibraryCache(hostId, version.id)).toMatchObject({
|
|
status: 'ready',
|
|
bytesTotal: 456
|
|
});
|
|
});
|
|
|
|
it('keeps metadata caches separate for each database connection', () => {
|
|
const seed = () => {
|
|
listLibraries();
|
|
legacyLibrary('cache-library', 't6');
|
|
legacyVersion('cache-version', 'cache-library', '2025-01-01', '2025-01-01');
|
|
};
|
|
const report = () =>
|
|
updateHostLibraryCaches(hostId, [
|
|
{ libraryVersionId: 'cache-version', status: 'ready', manifestSha256: 'checksum' }
|
|
]);
|
|
seed();
|
|
report();
|
|
const original = state.handle!;
|
|
state.handle = new Database(':memory:');
|
|
initializeStandaloneSettings(state.handle);
|
|
try {
|
|
seed();
|
|
const preparing = vi.spyOn(state.handle, 'prepare');
|
|
try {
|
|
report();
|
|
expect(
|
|
preparing.mock.calls.filter(([sql]) => sql.includes('FROM game_file_versions'))
|
|
).toHaveLength(1);
|
|
} finally {
|
|
preparing.mockRestore();
|
|
}
|
|
} finally {
|
|
state.handle.close();
|
|
state.handle = original;
|
|
}
|
|
});
|
|
|
|
it('deduplicates active downloads and blocks readiness without exact manifest verification', async () => {
|
|
const version = await published();
|
|
const operation = queueLibraryDownload(hostId, version.id);
|
|
expect(queueLibraryDownload(hostId, version.id).id).toBe(operation.id);
|
|
expect(() => requireReadyLibrary(hostId, version.id)).toThrow(/finish downloading/);
|
|
updateHostLibraryCaches(hostId, [
|
|
{ libraryVersionId: version.id, status: 'ready', manifestSha256: 'wrong' }
|
|
]);
|
|
expect(getHostLibraryCache(hostId, version.id)?.status).toBe('failed');
|
|
updateHostLibraryCaches(hostId, [
|
|
{
|
|
libraryVersionId: version.id,
|
|
status: 'ready',
|
|
manifestSha256: version.manifestSha256
|
|
}
|
|
]);
|
|
expect(requireReadyLibrary(hostId, version.id).id).toBe(version.id);
|
|
updateHostLibraryCaches(hostId, [
|
|
{
|
|
libraryVersionId: version.id,
|
|
status: 'outdated',
|
|
manifestSha256: version.manifestSha256
|
|
}
|
|
]);
|
|
expect(requireReadyLibrary(hostId, version.id).id).toBe(version.id);
|
|
updateHostLibraryCaches(hostId, [
|
|
{
|
|
libraryVersionId: version.id,
|
|
status: 'ready',
|
|
bytesDownloaded: 1,
|
|
manifestSha256: version.manifestSha256
|
|
}
|
|
]);
|
|
expect(getHostLibraryCache(hostId, version.id)?.status).toBe('failed');
|
|
updateHostLibraryCaches(hostId, [], { snapshot: true });
|
|
expect(getHostLibraryCache(hostId, version.id)?.status).toBe('missing');
|
|
expect(() => requireReadyLibrary(hostId, version.id)).toThrow(/finish downloading/);
|
|
});
|
|
it('streams resumable byte ranges and guards the immutable manifest endpoint with revocable host credentials', async () => {
|
|
const version = await published();
|
|
const request = (headers: HeadersInit = {}) =>
|
|
new Request('https://controller/api/agent/libraries/' + version.id + '/manifest', {
|
|
headers
|
|
});
|
|
expect(manifestRoute({ request: request(), params: { id: version.id } }).status).toBe(401);
|
|
const authorized = request({ Authorization: 'Bearer ' + credential });
|
|
expect(manifestRoute({ request: authorized, params: { id: version.id } }).status).toBe(200);
|
|
const partial = await serveGameFile(request({ Range: 'bytes=5-' }), version.id, file.path);
|
|
expect(partial.status).toBe(206);
|
|
expect(partial.headers.get('content-range')).toBe(
|
|
`bytes 5-${data.length - 1}/${data.length}`
|
|
);
|
|
expect(await partial.text()).toBe(data.slice(5));
|
|
const invalid = await serveGameFile(
|
|
request({ Range: 'bytes=999-' }),
|
|
version.id,
|
|
file.path
|
|
);
|
|
expect(invalid.status).toBe(416);
|
|
const resumed = await serveGameFile(
|
|
request({ Range: 'bytes=5-', 'If-Range': '"different"' }),
|
|
version.id,
|
|
file.path
|
|
);
|
|
expect(resumed.status).toBe(200);
|
|
expect(await resumed.text()).toBe(data);
|
|
revokeCredentials(hostId);
|
|
expect(manifestRoute({ request: authorized, params: { id: version.id } }).status).toBe(401);
|
|
});
|
|
});
|
|
|
|
describe.each([
|
|
{ name: 'metadata readiness checks', check: assertReadyLibrary },
|
|
{ name: 'full library readiness checks', check: requireReadyLibrary }
|
|
])('$name', ({ check }) => {
|
|
it('rejects missing, importing and failed versions', () => {
|
|
const version = begin();
|
|
expect(() => check(hostId, 'missing')).toThrow(
|
|
'Select an available published game-file version.'
|
|
);
|
|
expect(() => check(hostId, version.id)).toThrow(
|
|
'Select an available published game-file version.'
|
|
);
|
|
state
|
|
.handle!.prepare("UPDATE game_file_versions SET state='failed' WHERE id=?")
|
|
.run(version.id);
|
|
expect(() => check(hostId, version.id)).toThrow(
|
|
'Select an available published game-file version.'
|
|
);
|
|
});
|
|
|
|
it('rejects controller files as soon as retirement starts and after removal', async () => {
|
|
const version = await published();
|
|
updateHostLibraryCaches(hostId, [
|
|
{
|
|
libraryVersionId: version.id,
|
|
status: 'ready',
|
|
manifestSha256: version.manifestSha256
|
|
}
|
|
]);
|
|
expect(() => check(hostId, version.id)).not.toThrow();
|
|
state
|
|
.handle!.prepare('INSERT INTO game_file_cleanup(version_id,started_at) VALUES(?,?)')
|
|
.run(version.id, new Date().toISOString());
|
|
expect(() => check(hostId, version.id)).toThrow(
|
|
'Select an available published game-file version.'
|
|
);
|
|
state
|
|
.handle!.prepare('UPDATE game_file_cleanup SET removed_at=? WHERE version_id=?')
|
|
.run(new Date().toISOString(), version.id);
|
|
expect(() => check(hostId, version.id)).toThrow(
|
|
'Select an available published game-file version.'
|
|
);
|
|
});
|
|
|
|
it('blocks unresolved removal only for the selected host and version', async () => {
|
|
const version = await published();
|
|
updateHostLibraryCaches(hostId, [
|
|
{
|
|
libraryVersionId: version.id,
|
|
status: 'ready',
|
|
manifestSha256: version.manifestSha256
|
|
}
|
|
]);
|
|
const otherHost = createHost({ name: 'Other host' });
|
|
queueHostOperation(otherHost.id, 'library.remove', { libraryVersionId: version.id });
|
|
queueHostOperation(hostId, 'library.remove', { libraryVersionId: begin().id });
|
|
expect(() => check(hostId, version.id)).not.toThrow();
|
|
const removal = queueHostOperation(hostId, 'library.remove', {
|
|
libraryVersionId: version.id
|
|
});
|
|
const updateState = state.handle!.prepare(
|
|
'UPDATE ec_host_operations SET state=? WHERE id=?'
|
|
);
|
|
for (const operationState of ['pending', 'accepted', 'running', 'unknown']) {
|
|
// Exercise every unresolved state without applying unrelated receipt side effects.
|
|
updateState.run(operationState, removal.id);
|
|
expect(() => check(hostId, version.id)).toThrow('This host copy is being removed.');
|
|
}
|
|
for (const operationState of ['completed', 'failed']) {
|
|
updateState.run(operationState, removal.id);
|
|
expect(() => check(hostId, version.id)).not.toThrow();
|
|
}
|
|
});
|
|
|
|
it('requires the exact checksum and accepts verified ready or outdated copies', async () => {
|
|
const version = await published();
|
|
expect(() => check(hostId, version.id)).toThrow('finish downloading and verifying');
|
|
updateHostLibraryCaches(hostId, [
|
|
{
|
|
libraryVersionId: version.id,
|
|
status: 'ready',
|
|
manifestSha256: version.manifestSha256
|
|
}
|
|
]);
|
|
for (const status of ['ready', 'outdated']) {
|
|
state
|
|
.handle!.prepare(
|
|
'UPDATE host_library_caches SET status=?,manifest_sha256=? WHERE host_id=? AND version_id=?'
|
|
)
|
|
.run(status, 'different-manifest', hostId, version.id);
|
|
expect(() => check(hostId, version.id)).toThrow('finish downloading and verifying');
|
|
state
|
|
.handle!.prepare(
|
|
'UPDATE host_library_caches SET manifest_sha256=? WHERE host_id=? AND version_id=?'
|
|
)
|
|
.run(version.manifestSha256, hostId, version.id);
|
|
expect(() => check(hostId, version.id)).not.toThrow();
|
|
}
|
|
expect(getLibraryVersion(version.id)?.files).toEqual([file]);
|
|
expect(requireReadyLibrary(hostId, version.id).files).toEqual([file]);
|
|
});
|
|
});
|
|
|
|
describe('obsolete game-file cleanup', () => {
|
|
beforeEach(() => {
|
|
state.handle!.exec(
|
|
'CREATE TABLE native_instances (id TEXT, host_id TEXT, library_id TEXT, retired_at TEXT)'
|
|
);
|
|
});
|
|
async function obsolete() {
|
|
const old = await published();
|
|
reportHostOperation(hostId, old.operationId!, { state: 'completed', result: {} });
|
|
const latest = await published();
|
|
return { old, latest };
|
|
}
|
|
it('retains history and the latest files while removing unreferenced old bytes', async () => {
|
|
const { old, latest } = await obsolete();
|
|
await cleanupControllerLibraries();
|
|
expect(getLibraryVersion(old.id)).toMatchObject({
|
|
state: 'ready',
|
|
filesRemoved: true,
|
|
files: [file]
|
|
});
|
|
await expect(readFile(join(old.storageLocation, file.path))).rejects.toThrow();
|
|
expect(await readFile(join(latest.storageLocation, file.path), 'utf8')).toBe(data);
|
|
expect(() => queueLibraryDownload(hostId, old.id)).toThrow('removed');
|
|
expect(
|
|
(await serveGameFile(new Request('http://local/file'), old.id, file.path)).status
|
|
).toBe(410);
|
|
await cleanupControllerLibraries();
|
|
expect(listLibraries()[1].versions).toHaveLength(2);
|
|
});
|
|
it('resumes a reserved controller deletion after restart and preserves the manifest', async () => {
|
|
const old = await published();
|
|
reportHostOperation(hostId, old.operationId!, { state: 'completed', result: {} });
|
|
updateHostLibraryCaches(hostId, [
|
|
{ id: old.id, status: 'ready', manifestSha256: old.manifestSha256 }
|
|
]);
|
|
await published();
|
|
updateHostLibraryCaches(hostId, [], { snapshot: true });
|
|
state
|
|
.handle!.prepare('INSERT INTO game_file_cleanup(version_id,started_at) VALUES(?,?)')
|
|
.run(old.id, new Date().toISOString());
|
|
await rm(old.storageLocation, { recursive: true });
|
|
expect(getLibraryVersion(old.id)).toMatchObject({
|
|
filesRemoving: true,
|
|
filesRemoved: false
|
|
});
|
|
expect(() => queueLibraryDownload(hostId, old.id)).toThrow('removed');
|
|
await cleanupControllerLibraries();
|
|
expect(getLibraryVersion(old.id)).toMatchObject({
|
|
filesRemoving: false,
|
|
filesRemoved: true,
|
|
files: [file]
|
|
});
|
|
});
|
|
it('does not send cleanup to agents without the capability or erase their cached files', async () => {
|
|
const old = await published();
|
|
reportHostOperation(hostId, old.operationId!, { state: 'completed', result: {} });
|
|
updateHostLibraryCaches(hostId, [
|
|
{ id: old.id, status: 'ready', manifestSha256: old.manifestSha256 }
|
|
]);
|
|
await published();
|
|
reconcileHostLibraryCleanup(hostId);
|
|
expect(
|
|
state
|
|
.handle!.prepare("SELECT * FROM ec_host_operations WHERE kind='library.remove'")
|
|
.all()
|
|
).toHaveLength(0);
|
|
expect(getLibraryVersion(old.id)?.filesRemoved).toBe(false);
|
|
});
|
|
it('pins files for stopped or provisioning instances until retirement', async () => {
|
|
const old = await published();
|
|
reportHostOperation(hostId, old.operationId!, { state: 'completed', result: {} });
|
|
state
|
|
.handle!.prepare('INSERT INTO native_instances VALUES(?,?,?,NULL)')
|
|
.run('instance', hostId, old.id);
|
|
await published();
|
|
await cleanupControllerLibraries();
|
|
expect(getLibraryVersion(old.id)?.filesRemoved).toBe(false);
|
|
state.handle!.exec("UPDATE native_instances SET retired_at='now'");
|
|
await cleanupControllerLibraries();
|
|
expect(getLibraryVersion(old.id)?.filesRemoved).toBe(true);
|
|
});
|
|
it('waits for offline hosts, partial copies, and unresolved downloads', async () => {
|
|
const old = await published();
|
|
reportHostOperation(hostId, old.operationId!, { state: 'completed', result: {} });
|
|
updateHostLibraryCaches(hostId, [{ id: old.id, status: 'missing', bytesDownloaded: 1 }]);
|
|
await published();
|
|
await cleanupControllerLibraries();
|
|
expect(getLibraryVersion(old.id)?.filesRemoved).toBe(false);
|
|
const operation = queueLibraryDownload(hostId, old.id);
|
|
updateHostLibraryCaches(hostId, [], { snapshot: true });
|
|
await cleanupControllerLibraries();
|
|
expect(getLibraryVersion(old.id)?.filesRemoved).toBe(false);
|
|
reportHostOperation(hostId, operation.id, { state: 'failed', error: 'interrupted' });
|
|
await cleanupControllerLibraries();
|
|
expect(getLibraryVersion(old.id)?.filesRemoved).toBe(true);
|
|
});
|
|
it('queues host removal only without instance references and waits for absence before controller cleanup', async () => {
|
|
heartbeat(hostId, {
|
|
protocolVersion: 1,
|
|
dockerRunning: true,
|
|
directoriesWritable: true,
|
|
capabilities: ['library.remove'],
|
|
storageAvailable: 1_000_000_000_000
|
|
});
|
|
const old = await published();
|
|
reportHostOperation(hostId, old.operationId!, { state: 'completed', result: {} });
|
|
updateHostLibraryCaches(hostId, [
|
|
{ id: old.id, status: 'ready', manifestSha256: old.manifestSha256 }
|
|
]);
|
|
state
|
|
.handle!.prepare('INSERT INTO native_instances VALUES(?,?,?,NULL)')
|
|
.run('instance', hostId, old.id);
|
|
await published();
|
|
reconcileHostLibraryCleanup(hostId);
|
|
expect(
|
|
state
|
|
.handle!.prepare("SELECT * FROM ec_host_operations WHERE kind='library.remove'")
|
|
.all()
|
|
).toHaveLength(0);
|
|
state.handle!.exec("UPDATE native_instances SET retired_at='now'");
|
|
reconcileHostLibraryCleanup(hostId);
|
|
reconcileHostLibraryCleanup(hostId);
|
|
const removals = state
|
|
.handle!.prepare("SELECT id FROM ec_host_operations WHERE kind='library.remove'")
|
|
.all() as { id: string }[];
|
|
expect(removals).toHaveLength(1);
|
|
expect(() => queueLibraryDownload(hostId, old.id)).toThrow('being removed');
|
|
reportHostOperation(hostId, removals[0].id, {
|
|
state: 'completed',
|
|
result: { removed: true }
|
|
});
|
|
await cleanupControllerLibraries();
|
|
expect(getLibraryVersion(old.id)?.filesRemoved).toBe(false);
|
|
updateHostLibraryCaches(hostId, [], { snapshot: true });
|
|
await cleanupControllerLibraries();
|
|
expect(getLibraryVersion(old.id)?.filesRemoved).toBe(true);
|
|
});
|
|
});
|