diff --git a/src/lib/server/game-files/index.ts b/src/lib/server/game-files/index.ts index a415690..b7f4dcb 100644 --- a/src/lib/server/game-files/index.ts +++ b/src/lib/server/game-files/index.ts @@ -43,9 +43,14 @@ export function getLibraryFile(versionId: string, path: string) { FROM game_file_entries f JOIN game_file_versions v ON v.id=f.version_id WHERE f.version_id=? AND f.path=?`) .get(versionId, path) as (GameFileEntry & { state: GameLibraryVersion['state']; sourceHostId: string; storageLocation: string }) | undefined ?? null; } +/** Launch selection needs only the ID; manifests can contain tens of thousands of files. */ +export function newestReadyLibraryId(game: string): string | null { + const row = database().prepare("SELECT v.id FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id WHERE l.game=? AND v.state='ready' ORDER BY v.published_at DESC, v.created_at DESC, v.rowid DESC LIMIT 1").get(game) as { id: string } | undefined; + return row?.id ?? null; +} export function newestReadyLibrary(game: string): GameLibraryVersion | null { - const row = database().prepare("SELECT v.*, l.game FROM game_file_versions v JOIN game_file_libraries l ON l.id=v.library_id WHERE l.game=? AND v.state='ready' ORDER BY v.published_at DESC, v.created_at DESC, v.rowid DESC LIMIT 1").get(game) as VersionRow | undefined; - return row ? versionFromRow(row) : null; + const id = newestReadyLibraryId(game); + return id ? getLibraryVersion(id) : null; } /** Hosts follow the latest successful publication for each game, including legacy libraries. */ export function latestPublishedLibraries() { @@ -79,13 +84,20 @@ export function getHostLibraryCache(hostId: string, versionId: string): HostLibr const row = database().prepare('SELECT * FROM host_library_caches WHERE host_id=? AND version_id=?').get(hostId, versionId) as CacheRow | undefined; return row ? cacheFromRow(row) : null; } -export function requireReadyLibrary(hostId: string, versionId: string): GameLibraryVersion { - const version = getLibraryVersion(versionId); +/** Admission and dispatch recheck readiness without loading or parsing the file manifest. */ +export function assertReadyLibrary(hostId: string, versionId: string): void { + const handle = database(); + const version = handle.prepare('SELECT state, manifest_sha256 FROM game_file_versions WHERE id=?') + .get(versionId) as Pick | undefined; if (!version || version.state !== 'ready' || libraryFilesRetired(versionId)) throw new Error('Select an available published game-file version.'); - if (database().prepare("SELECT 1 FROM ec_host_operations WHERE host_id=? AND kind='library.remove' AND json_extract(payload,'$.libraryVersionId')=? AND state IN ('pending','accepted','running','unknown')").get(hostId, versionId)) throw new Error('This host copy is being removed.'); - const cache = getHostLibraryCache(hostId, versionId); - if (!cache || !['ready', 'outdated'].includes(cache.status) || cache.manifestSha256 !== version.manifestSha256) throw new Error('The selected host must finish downloading and verifying this game-file version.'); - return version; + if (handle.prepare("SELECT 1 FROM ec_host_operations WHERE host_id=? AND kind='library.remove' AND json_extract(payload,'$.libraryVersionId')=? AND state IN ('pending','accepted','running','unknown')").get(hostId, versionId)) throw new Error('This host copy is being removed.'); + const cache = handle.prepare('SELECT status, manifest_sha256 FROM host_library_caches WHERE host_id=? AND version_id=?') + .get(hostId, versionId) as Pick | undefined; + if (!cache || !['ready', 'outdated'].includes(cache.status) || cache.manifest_sha256 !== version.manifest_sha256) throw new Error('The selected host must finish downloading and verifying this game-file version.'); +} +export function requireReadyLibrary(hostId: string, versionId: string): GameLibraryVersion { + assertReadyLibrary(hostId, versionId); + return getLibraryVersion(versionId)!; } function textField(input: unknown, label: string, max: number, required = true): string { diff --git a/src/lib/server/hosts/capacity.ts b/src/lib/server/hosts/capacity.ts index b006d78..7839b78 100644 --- a/src/lib/server/hosts/capacity.ts +++ b/src/lib/server/hosts/capacity.ts @@ -1,6 +1,6 @@ import { db } from '../db/index.js'; import { listHosts, requireReadyHost } from './index.js'; -import { requireReadyLibrary } from '../game-files/index.js'; +import { assertReadyLibrary } from '../game-files/index.js'; import type { Host } from '../../hosts.js'; import { capacityCalibration } from './calibration.js'; @@ -101,7 +101,7 @@ export function choosePlacement(options: PlacementOptions): { host: Host; port: for (const host of listHosts().filter(host => !options.hostId || host.id === options.hostId)) { try { requireReadyHost(host.id, { image: options.image, requiredBytes: INSTANCE_STORAGE_BYTES }); - requireReadyLibrary(host.id, options.libraryId); + assertReadyLibrary(host.id, options.libraryId); const port = availableHostPort(host, options); if (port === null) throw new Error('No available UDP/TCP game port.'); const load = projectedLoad(host, options); diff --git a/src/lib/server/provisioning/native.ts b/src/lib/server/provisioning/native.ts index 71b6fa4..a4c8964 100644 --- a/src/lib/server/provisioning/native.ts +++ b/src/lib/server/provisioning/native.ts @@ -6,7 +6,7 @@ import { db } from '../db/index.js'; import { config, relDestForKind, serverDir } from '../config.js'; import { nativeInstance, nativeRequest, type NativeInstance } from '../backends/native.js'; import { getHost, getHostOperation, listHosts, queueHostOperation, releaseHostPorts, reserveHostPort } from '../hosts/index.js'; -import { newestReadyLibrary, requireReadyLibrary } from '../game-files/index.js'; +import { newestReadyLibraryId, assertReadyLibrary } from '../game-files/index.js'; import { blueprintRecipe, captureRecipe, recipeForDeployment, summarizeRecipe, type LaunchRecipe } from '../fleet/launch-recipes.js'; import { getServerKey, saveServerKey } from '../fleet/server-keys.js'; import { activeBetaRemovalBlocker, getServer } from '../fleet/servers.js'; @@ -34,9 +34,9 @@ interface FilePlan { id: number; path: string; contentBase64: string | null; sha export function nativeSelection(input: Record, game: string, options: Pick = {}): NativeSelection { if (input.hostId !== undefined && typeof input.hostId !== 'string') throw new Error('Invalid host.'); - const version = newestReadyLibrary(game); - if (!version) throw new Error(`Upload ${game.toUpperCase()} game files before launching a server.`); - if (input.libraryId !== undefined && input.libraryId !== '' && input.libraryId !== version.id) { + const libraryId = newestReadyLibraryId(game); + if (!libraryId) throw new Error(`Upload ${game.toUpperCase()} game files before launching a server.`); + if (input.libraryId !== undefined && input.libraryId !== '' && input.libraryId !== libraryId) { throw new Error(`The current ${game.toUpperCase()} game files changed. Refresh the launch preview.`); } const image = input.image === undefined || input.image === '' ? config.komodo.defaultImage : input.image; @@ -47,10 +47,10 @@ export function nativeSelection(input: Record, game: string, op const network = rawNetwork as NativeSelection; const bindAddress = network.bindAddress || undefined; if (bindAddress !== undefined && (typeof bindAddress !== 'string' || !isIP(bindAddress))) throw new Error('Invalid bind address.'); - const { host } = choosePlacement({ hostId: automatic ? undefined : input.hostId as string, libraryId: version.id, image, bindAddress, ...options }); + const { host } = choosePlacement({ hostId: automatic ? undefined : input.hostId as string, libraryId, image, bindAddress, ...options }); if (network.publicAddress !== undefined && (typeof network.publicAddress !== 'string' || network.publicAddress.length > 253)) throw new Error('Invalid public address.'); if (network.publicPort !== undefined && (!Number.isInteger(network.publicPort) || Number(network.publicPort) < 1 || Number(network.publicPort) > 65535)) throw new Error('Invalid public port.'); - return { hostId: host.id, libraryId: version.id, image, bindAddress: bindAddress ?? host.bindAddress, publicAddress: network.publicAddress ?? host.publicAddress, + return { hostId: host.id, libraryId, image, bindAddress: bindAddress ?? host.bindAddress, publicAddress: network.publicAddress ?? host.publicAddress, publicPort: network.publicPort, automatic, network: { bindAddress, publicAddress: network.publicAddress, publicPort: network.publicPort } }; } @@ -156,7 +156,7 @@ export function queueNativeRebuild(name: string, triggeredBy: string): CreateSer const blocker = activeBetaRemovalBlocker(name); if (blocker) throw new Error(blocker); if (!server?.map_slug || !getServerKey(name)) throw new Error('A saved map and server key are required before rebuilding.'); - requireReadyLibrary(instance.host_id, instance.library_id); + assertReadyLibrary(instance.host_id, instance.library_id); const recipe: NativeRecipe = { ...captureRecipe(server.game as 't5' | 't6', server.map_slug), image: instance.image, native: { hostId: instance.host_id, libraryId: instance.library_id, image: instance.image, bindAddress: instance.bind_address } }; return db().transaction(() => { @@ -434,7 +434,7 @@ async function processJob(job: Job, eligibility: (job: Job) => string | null) { deploymentStage(job.id, 'Checking host eligibility', { phase: 'eligibility', blocker: null }); if (job.action === 'create') choosePlacement({ hostId: instance.host_id, libraryId: instance.library_id, image: instance.image, bindAddress: instance.bind_address, excludeInstanceId: instance.id, thorough: true, dispatch: true }); - if (job.action === 'create' || job.action === 'rebuild') requireReadyLibrary(instance.host_id, instance.library_id); + if (job.action === 'create' || job.action === 'rebuild') assertReadyLibrary(instance.host_id, instance.library_id); if (job.action !== 'create' && job.action !== 'start') { // Fast controller check keeps occupied lobbies out of the operation slots. // The agent repeats this immediately before its local mutation. diff --git a/tests/unit/collection-launch-performance.test.ts b/tests/unit/collection-launch-performance.test.ts new file mode 100644 index 0000000..36bd46a --- /dev/null +++ b/tests/unit/collection-launch-performance.test.ts @@ -0,0 +1,101 @@ +import { createHash } from 'node:crypto'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { newFixture, type Fixture } from '../helpers/fixture.js'; + +vi.mock('../../src/lib/server/pipeline/worker.js', () => ({ kick: vi.fn(async () => {}), kickProvisioning: vi.fn(async () => {}) })); + +let fixture: Fixture; +let hostId: string; +let libraryId: string; +let manifest: string; +const image = 'example.test/plutainer:test'; +const serverCount = 100; + +beforeEach(async () => { + vi.resetModules(); vi.clearAllMocks(); + fixture = await newFixture([]); + vi.stubEnv('GAME_FILES_ROOT', join(fixture.root, 'libraries')); + vi.stubEnv('PLUTAINER_IMAGE', image); + vi.stubEnv('RCON_PASSWORD', 'test-password'); + vi.stubEnv('KOMODO_ADDRESS', ''); + vi.stubEnv('KOMODO_API_KEY', ''); + vi.stubEnv('KOMODO_API_SECRET', ''); + const { createHost, createEnrollment, enrollHost, heartbeat } = await import('../../src/lib/server/hosts/index.js'); + hostId = createHost({ name: 'Large native fleet', portMin: 28960, portMax: 29059 }).id; + const inventory = { + cpuCount: 256, cpuUsagePercent: 5, memoryTotal: 512 * 1024 ** 3, memoryAvailable: 500 * 1024 ** 3, + protocolVersion: 1, dockerRunning: true, directoriesWritable: true, + storageAvailable: 1_000_000_000_000, runtimeImages: [image] + }; + enrollHost(createEnrollment(hostId).token, inventory); + const { createLibraryImport, updateHostLibraryCaches } = await import('../../src/lib/server/game-files/index.js'); + libraryId = createLibraryImport({ hostId, game: 't5' }).id; + const fileCount = 20_000; + const fileSize = 1024 ** 2; + const sha256 = createHash('sha256').update('fixture game file').digest('hex'); + manifest = JSON.stringify(Array.from({ length: fileCount }, (_, index) => ({ + path: `zone/base-${String(index).padStart(5, '0')}.ff`, size: fileSize, sha256 + }))); + const manifestSha256 = createHash('sha256').update(manifest).digest('hex'); + const { db } = await import('../../src/lib/server/db/index.js'); + // Seed a published manifest without uploading gigabytes of fixture files. + db().transaction(() => { + db().prepare(`UPDATE game_file_versions SET state='ready',manifest=?,manifest_sha256=?,size_bytes=?,published_at=? WHERE id=?`) + .run(manifest, manifestSha256, fileCount * fileSize, new Date().toISOString(), libraryId); + db().prepare(`INSERT INTO game_file_entries(version_id,path,size,sha256) + SELECT ?,json_extract(value,'$.path'),json_extract(value,'$.size'),json_extract(value,'$.sha256') FROM json_each(?)`) + .run(libraryId, manifest); + })(); + updateHostLibraryCaches(hostId, [{ libraryVersionId: libraryId, status: 'ready', manifestSha256 }]); + heartbeat(hostId, { ...inventory, libraries: [{ id: libraryId, state: 'ready' }] }); +}); + +afterEach(async () => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + await fixture.cleanup(); + vi.unstubAllEnvs(); +}); + +describe('large native collection admission', () => { + it('previews and queues every lobby without deserializing the game-file manifest', async () => { + const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); + const { previewCollection, queueCollection, getCollection } = await import('../../src/lib/server/fleet/collections.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + const blueprint = saveBlueprint({ name: 'One hundred lobbies', entries: + ['theater', 'cosmodrome', 'coast', 'temple', 'moon'].map(mapSlug => ({ game: 't5', mapSlug, count: 20 })) }); + + const parse = JSON.parse; + let manifestParses = 0; + const observeParse: typeof JSON.parse = (value, reviver) => { + if (value === manifest) manifestParses++; + return parse(value, reviver); + }; + // A mock records all return values and would retain every parsed manifest. + vi.stubGlobal('JSON', Object.assign(Object.create(JSON), { parse: observeParse })); + const preview = previewCollection(blueprint.id); + const previewParses = manifestParses; + expect(preview.ready).toBe(true); + expect(preview.servers).toHaveLength(serverCount); + expect(new Set(preview.servers.map(server => server.name)).size).toBe(serverCount); + expect(new Set(preview.servers.map(server => server.port)).size).toBe(serverCount); + expect(db().prepare('SELECT * FROM ec_host_ports').all()).toHaveLength(0); + + manifestParses = 0; + const result = queueCollection({ previewId: preview.id, + keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key-' + server.name])) }); + const queueParses = manifestParses; + expect(result).toMatchObject({ ok: true, collectionId: expect.any(String) }); + expect(queueCollection({ previewId: preview.id })).toEqual(result); + expect(getCollection(result.collectionId!)?.collection).toMatchObject({ total: serverCount, queued: serverCount }); + const instances = db().prepare('SELECT host_id,library_id,port FROM native_instances ORDER BY port').all(); + expect(instances).toEqual(Array.from({ length: serverCount }, (_, index) => ({ + host_id: hostId, library_id: libraryId, port: 28960 + index + }))); + expect(db().prepare('SELECT * FROM ec_host_ports').all()).toHaveLength(serverCount * 2); + expect(db().prepare('SELECT * FROM server_collection_members').all()).toHaveLength(serverCount); + expect(previewParses, 'Preview must use library metadata without reading the full manifest').toBe(0); + expect(queueParses, 'Queue admission must use library metadata without reading the full manifest').toBe(0); + }, 30_000); +}); diff --git a/tests/unit/game-file-library.test.ts b/tests/unit/game-file-library.test.ts index c1c64d7..1c79f45 100644 --- a/tests/unit/game-file-library.test.ts +++ b/tests/unit/game-file-library.test.ts @@ -6,8 +6,8 @@ import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; const state = vi.hoisted(() => ({ handle: null as Database.Database | null })); vi.mock('../../src/lib/server/db/index.js', () => ({ db: () => state.handle })); -import { heartbeat, createEnrollment, createHost, enrollHost, getHostOperation, reportHostOperation, revokeCredentials } from '../../src/lib/server/hosts/index.js'; -import { cleanupControllerLibraries, reconcileHostLibraryCleanup, createLibraryImport, getHostLibraryCache, getLibraryVersion, importPublicationStatus, latestPublishedLibraries, libraryOverview, listLibraries, newestReadyLibrary, publishImport, queueLibraryDownload, requestImportPublication, requireReadyLibrary, setImportManifest, updateHostLibraryCaches, uploadImportFile } from '../../src/lib/server/game-files/index.js'; +import { heartbeat, createEnrollment, createHost, enrollHost, getHostOperation, queueHostOperation, reportHostOperation, revokeCredentials } from '../../src/lib/server/hosts/index.js'; +import { assertReadyLibrary, cleanupControllerLibraries, reconcileHostLibraryCleanup, createLibraryImport, getHostLibraryCache, getLibraryVersion, importPublicationStatus, latestPublishedLibraries, libraryOverview, listLibraries, newestReadyLibrary, publishImport, queueLibraryDownload, requestImportPublication, requireReadyLibrary, setImportManifest, updateHostLibraryCaches, uploadImportFile } from '../../src/lib/server/game-files/index.js'; import { serveGameFile } from '../../src/lib/server/game-files/http.js'; import { GET as manifestRoute } from '../../src/routes/api/agent/libraries/[id]/manifest/+server.js'; @@ -180,6 +180,65 @@ describe('verified host distribution', () => { }); }); +describe.each([ + { name: 'metadata readiness checks', check: assertReadyLibrary }, + { name: 'full library readiness checks', check: requireReadyLibrary } +])('$name', ({ check }) => { + it('rejects missing, importing and failed versions', () => { + const version = begin(); + expect(() => check(hostId, 'missing')).toThrow('Select an available published game-file version.'); + expect(() => check(hostId, version.id)).toThrow('Select an available published game-file version.'); + state.handle!.prepare("UPDATE game_file_versions SET state='failed' WHERE id=?").run(version.id); + expect(() => check(hostId, version.id)).toThrow('Select an available published game-file version.'); + }); + + it('rejects controller files as soon as retirement starts and after removal', async () => { + const version = await published(); + updateHostLibraryCaches(hostId, [{ libraryVersionId: version.id, status: 'ready', manifestSha256: version.manifestSha256 }]); + expect(() => check(hostId, version.id)).not.toThrow(); + state.handle!.prepare('INSERT INTO game_file_cleanup(version_id,started_at) VALUES(?,?)').run(version.id, new Date().toISOString()); + expect(() => check(hostId, version.id)).toThrow('Select an available published game-file version.'); + state.handle!.prepare('UPDATE game_file_cleanup SET removed_at=? WHERE version_id=?').run(new Date().toISOString(), version.id); + expect(() => check(hostId, version.id)).toThrow('Select an available published game-file version.'); + }); + + it('blocks unresolved removal only for the selected host and version', async () => { + const version = await published(); + updateHostLibraryCaches(hostId, [{ libraryVersionId: version.id, status: 'ready', manifestSha256: version.manifestSha256 }]); + const otherHost = createHost({ name: 'Other host' }); + queueHostOperation(otherHost.id, 'library.remove', { libraryVersionId: version.id }); + queueHostOperation(hostId, 'library.remove', { libraryVersionId: begin().id }); + expect(() => check(hostId, version.id)).not.toThrow(); + const removal = queueHostOperation(hostId, 'library.remove', { libraryVersionId: version.id }); + const updateState = state.handle!.prepare('UPDATE ec_host_operations SET state=? WHERE id=?'); + for (const operationState of ['pending', 'accepted', 'running', 'unknown']) { + // Exercise every unresolved state without applying unrelated receipt side effects. + updateState.run(operationState, removal.id); + expect(() => check(hostId, version.id)).toThrow('This host copy is being removed.'); + } + for (const operationState of ['completed', 'failed']) { + updateState.run(operationState, removal.id); + expect(() => check(hostId, version.id)).not.toThrow(); + } + }); + + it('requires the exact checksum and accepts verified ready or outdated copies', async () => { + const version = await published(); + expect(() => check(hostId, version.id)).toThrow('finish downloading and verifying'); + updateHostLibraryCaches(hostId, [{ libraryVersionId: version.id, status: 'ready', manifestSha256: version.manifestSha256 }]); + for (const status of ['ready', 'outdated']) { + state.handle!.prepare('UPDATE host_library_caches SET status=?,manifest_sha256=? WHERE host_id=? AND version_id=?') + .run(status, 'different-manifest', hostId, version.id); + expect(() => check(hostId, version.id)).toThrow('finish downloading and verifying'); + state.handle!.prepare('UPDATE host_library_caches SET manifest_sha256=? WHERE host_id=? AND version_id=?') + .run(version.manifestSha256, hostId, version.id); + expect(() => check(hostId, version.id)).not.toThrow(); + } + expect(getLibraryVersion(version.id)?.files).toEqual([file]); + expect(requireReadyLibrary(hostId, version.id).files).toEqual([file]); + }); +}); + describe('obsolete game-file cleanup', () => { beforeEach(() => {