Fix controller stalls during large collection launches

2026-10-05 09:50:11 -04:00
parent 9472bb5184
commit b3929fc771
5 changed files with 192 additions and 20 deletions
@@ -6,8 +6,8 @@ import { join } from 'node:path';
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 const state = vi.hoisted(() => ({ handle: null as Database.Database | null }));
 vi.mock('../../src/lib/server/db/index.js', () => ({ db: () => state.handle }));
-import { heartbeat, createEnrollment, createHost, enrollHost, getHostOperation, reportHostOperation, revokeCredentials } from '../../src/lib/server/hosts/index.js';
-import { cleanupControllerLibraries, reconcileHostLibraryCleanup, createLibraryImport, getHostLibraryCache, getLibraryVersion, importPublicationStatus, latestPublishedLibraries, libraryOverview, listLibraries, newestReadyLibrary, publishImport, queueLibraryDownload, requestImportPublication, requireReadyLibrary, setImportManifest, updateHostLibraryCaches, uploadImportFile } from '../../src/lib/server/game-files/index.js';
+import { heartbeat, createEnrollment, createHost, enrollHost, getHostOperation, queueHostOperation, reportHostOperation, revokeCredentials } from '../../src/lib/server/hosts/index.js';
+import { assertReadyLibrary, cleanupControllerLibraries, reconcileHostLibraryCleanup, createLibraryImport, getHostLibraryCache, getLibraryVersion, importPublicationStatus, latestPublishedLibraries, libraryOverview, listLibraries, newestReadyLibrary, publishImport, queueLibraryDownload, requestImportPublication, requireReadyLibrary, setImportManifest, updateHostLibraryCaches, uploadImportFile } from '../../src/lib/server/game-files/index.js';
 import { serveGameFile } from '../../src/lib/server/game-files/http.js';
 import { GET as manifestRoute } from '../../src/routes/api/agent/libraries/[id]/manifest/+server.js';

@@ -180,6 +180,65 @@ describe('verified host distribution', () => {
    });
 });

+describe.each([
+    { name: 'metadata readiness checks', check: assertReadyLibrary },
+    { name: 'full library readiness checks', check: requireReadyLibrary }
+])('$name', ({ check }) => {
+    it('rejects missing, importing and failed versions', () => {
+        const version = begin();
+        expect(() => check(hostId, 'missing')).toThrow('Select an available published game-file version.');
+        expect(() => check(hostId, version.id)).toThrow('Select an available published game-file version.');
+        state.handle!.prepare("UPDATE game_file_versions SET state='failed' WHERE id=?").run(version.id);
+        expect(() => check(hostId, version.id)).toThrow('Select an available published game-file version.');
+    });
+
+    it('rejects controller files as soon as retirement starts and after removal', async () => {
+        const version = await published();
+        updateHostLibraryCaches(hostId, [{ libraryVersionId: version.id, status: 'ready', manifestSha256: version.manifestSha256 }]);
+        expect(() => check(hostId, version.id)).not.toThrow();
+        state.handle!.prepare('INSERT INTO game_file_cleanup(version_id,started_at) VALUES(?,?)').run(version.id, new Date().toISOString());
+        expect(() => check(hostId, version.id)).toThrow('Select an available published game-file version.');
+        state.handle!.prepare('UPDATE game_file_cleanup SET removed_at=? WHERE version_id=?').run(new Date().toISOString(), version.id);
+        expect(() => check(hostId, version.id)).toThrow('Select an available published game-file version.');
+    });
+
+    it('blocks unresolved removal only for the selected host and version', async () => {
+        const version = await published();
+        updateHostLibraryCaches(hostId, [{ libraryVersionId: version.id, status: 'ready', manifestSha256: version.manifestSha256 }]);
+        const otherHost = createHost({ name: 'Other host' });
+        queueHostOperation(otherHost.id, 'library.remove', { libraryVersionId: version.id });
+        queueHostOperation(hostId, 'library.remove', { libraryVersionId: begin().id });
+        expect(() => check(hostId, version.id)).not.toThrow();
+        const removal = queueHostOperation(hostId, 'library.remove', { libraryVersionId: version.id });
+        const updateState = state.handle!.prepare('UPDATE ec_host_operations SET state=? WHERE id=?');
+        for (const operationState of ['pending', 'accepted', 'running', 'unknown']) {
+            // Exercise every unresolved state without applying unrelated receipt side effects.
+            updateState.run(operationState, removal.id);
+            expect(() => check(hostId, version.id)).toThrow('This host copy is being removed.');
+        }
+        for (const operationState of ['completed', 'failed']) {
+            updateState.run(operationState, removal.id);
+            expect(() => check(hostId, version.id)).not.toThrow();
+        }
+    });
+
+    it('requires the exact checksum and accepts verified ready or outdated copies', async () => {
+        const version = await published();
+        expect(() => check(hostId, version.id)).toThrow('finish downloading and verifying');
+        updateHostLibraryCaches(hostId, [{ libraryVersionId: version.id, status: 'ready', manifestSha256: version.manifestSha256 }]);
+        for (const status of ['ready', 'outdated']) {
+            state.handle!.prepare('UPDATE host_library_caches SET status=?,manifest_sha256=? WHERE host_id=? AND version_id=?')
+                .run(status, 'different-manifest', hostId, version.id);
+            expect(() => check(hostId, version.id)).toThrow('finish downloading and verifying');
+            state.handle!.prepare('UPDATE host_library_caches SET manifest_sha256=? WHERE host_id=? AND version_id=?')
+                .run(version.manifestSha256, hostId, version.id);
+            expect(() => check(hostId, version.id)).not.toThrow();
+        }
+        expect(getLibraryVersion(version.id)?.files).toEqual([file]);
+        expect(requireReadyLibrary(hostId, version.id).files).toEqual([file]);
+    });
+});
+

 describe('obsolete game-file cleanup', () => {
    beforeEach(() => {