Migrate instances when draining and verify hosts before enabling

2026-10-05 14:21:48 -04:00
parent a142f06c77
commit 16a5bc3534
20 changed files with 525 additions and 78 deletions
@@ -68,6 +68,83 @@ function verified(id: string, version: GameLibraryVersion) {
 }

 describe('automatic host preparation', () => {
+    it('keeps re-enabled hosts disabled until a fresh runtime pull and both game libraries are verified', async () => {
+        const t5 = await publish('t5');
+        const t6 = await publish('t6');
+        const readyInventory = { runtimeImages: [runtime], libraries: [t5, t6].map(version => ({ id: version.id, state: 'ready' as const })) };
+        const id = enroll(readyInventory);
+        verified(id, t5); verified(id, t6);
+        const oldRuntime = hosts.queueHostOperation(id, 'runtime.pull', { image: runtime });
+        hosts.reportHostOperation(id, oldRuntime.id, { state: 'completed' });
+        heartbeat(id, readyInventory);
+        hosts.updateHost(id, { mode: 'disabled' });
+        const requested = hosts.updateHost(id, { mode: 'active' });
+        expect(requested).toMatchObject({ mode: 'disabled', readiness: 'preparing', enablingSince: expect.any(String) });
+        const { closeDb } = await import('../../src/lib/server/db/index.js');
+        closeDb();
+        expect(() => hosts.requireReadyHost(id)).toThrow('before enabling');
+        preparation.reconcileHostPreparation(id);
+        const pull = operations(id)[0];
+        expect(pull.id).not.toBe(oldRuntime.id);
+        expect(pull.kind).toBe('runtime.pull');
+        expect(hosts.pendingHostOperations(id).map(operation => operation.id)).toEqual([pull.id]);
+        expect(() => hosts.queueHostOperation(id, 'instance.restart', {})).toThrow('disabled');
+        expect(hosts.updateHost(id, { mode: 'active' }).enablingSince).toBe(requested.enablingSince);
+        hosts.reportHostOperation(id, pull.id, { state: 'completed' });
+        preparation.reconcileHostPreparation(id);
+        expect(hosts.getHost(id)?.mode).toBe('disabled');
+        heartbeat(id, readyInventory);
+        for (let index = 0; index < 2; index++) {
+            preparation.reconcileHostPreparation(id);
+            const download = operations(id)[0];
+            expect(download.kind).toBe('library.download');
+            expect(hosts.pendingHostOperations(id).map(operation => operation.id)).toEqual([download.id]);
+            const version = [t5, t6].find(version => version.id === download.payload.libraryVersionId)!;
+            verified(id, version);
+            hosts.reportHostOperation(id, download.id, { state: 'completed' });
+            preparation.reconcileHostPreparation(id);
+            expect(hosts.getHost(id)?.mode).toBe('disabled');
+            heartbeat(id, readyInventory);
+        }
+        preparation.reconcileHostPreparation(id);
+        expect(hosts.getHost(id)).toMatchObject({ mode: 'active', readiness: 'ready', enablingSince: null });
+        expect(operations(id).filter(operation => operation.kind === 'library.download')).toHaveLength(2);
+    });
+
+    it('requires both games when enabling even if one library is already cached', async () => {
+        const version = await publish('t5');
+        const id = enroll({ runtimeImages: [runtime], libraries: [{ id: version.id, state: 'ready' }] });
+        verified(id, version);
+        hosts.updateHost(id, { mode: 'disabled' });
+        hosts.updateHost(id, { mode: 'active' });
+        const status = preparation.reconcileHostPreparation(id);
+        expect(status.items).toContainEqual(expect.objectContaining({ label: 'Black Ops II (T6)', state: 'blocked' }));
+        expect(hosts.getHost(id)?.mode).toBe('disabled');
+    });
+
+    it('keeps a re-enabling host out of placement after setup failure or loss of contact', async () => {
+        const t5 = await publish('t5');
+        const t6 = await publish('t6');
+        const readyInventory = { runtimeImages: [runtime], libraries: [t5, t6].map(version => ({ id: version.id, state: 'ready' as const })) };
+        const id = enroll(readyInventory);
+        verified(id, t5); verified(id, t6);
+        hosts.updateHost(id, { mode: 'disabled' });
+        hosts.updateHost(id, { mode: 'active' });
+        preparation.reconcileHostPreparation(id);
+        hosts.reportHostOperation(id, operations(id)[0].id, { state: 'failed', error: 'Registry unavailable' });
+        heartbeat(id, readyInventory);
+        expect(preparation.reconcileHostPreparation(id).state).toBe('blocked');
+        expect(() => hosts.requireReadyHost(id)).toThrow('before enabling');
+        const queued = operations(id).map(operation => operation.id);
+        vi.setSystemTime(Date.now() + 91_000);
+        preparation.reconcileHostPreparation(id);
+        expect(hosts.getHost(id)).toMatchObject({ mode: 'disabled', connected: false });
+        expect(operations(id).map(operation => operation.id)).toEqual(queued);
+        heartbeat(id, readyInventory);
+        preparation.reconcileHostPreparation(id);
+        expect(hosts.getHost(id)?.mode).toBe('disabled');
+    });
+
    it('prepares an enrolled host through authenticated agent polls and heartbeats without manual setup requests', async () => {
        const [{ POST: enrollAgent }, { GET: pollOperations }, { POST: reportOperation }, { POST: reportHeartbeat }] = await Promise.all([
            import('../../src/routes/api/agent/enroll/+server.js'),