Queue native collection launches before async placement

2026-10-05 11:35:50 -04:00
parent 1405114a14
commit 7c76851252
34 changed files with 1324 additions and 244 deletions
@@ -0,0 +1,104 @@
+import { setImmediate } from 'node:timers/promises';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { newFixture, type Fixture } from '../helpers/fixture.js';
+
+let fixture: Fixture;
+let hostId: string;
+const GiB = 1024 ** 3;
+const epoch = Date.parse('2026-10-05T12:00:00Z');
+const inventory = { protocolVersion: 1, dockerRunning: true, directoriesWritable: true, storageAvailable: 100 * GiB,
+    cpuCount: 16, cpuUsagePercent: 0, memoryTotal: 64 * GiB, memoryAvailable: 60 * GiB,
+    runtimeImages: ['image'], libraries: [{ id: 't5-library', state: 'ready' }, { id: 't6-library', state: 'ready' }] };
+const t5 = { libraryId: 't5-library', image: 'image', automatic: true };
+const t6 = { libraryId: 't6-library', image: 'image', automatic: true };
+async function settle() { await setImmediate(); await setImmediate(); }
+
+beforeEach(async () => {
+    vi.resetModules();
+    fixture = await newFixture([]);
+    vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(epoch);
+    const hosts = await import('../../src/lib/server/hosts/index.js');
+    hostId = hosts.createHost({ name: 'Availability host', portMin: 28960, portMax: 28962 }).id;
+    hosts.enrollHost(hosts.createEnrollment(hostId).token, inventory);
+    hosts.heartbeat(hostId, inventory);
+    await settle();
+});
+afterEach(async () => { vi.restoreAllMocks(); vi.useRealTimers(); await fixture.cleanup(); });
+
+describe('cached launch availability', () => {
+    it('reads cached estimates without placement, historical scans, or repeated database reads', async () => {
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const capacity = await import('../../src/lib/server/hosts/capacity.js');
+        const { launchAvailability } = await import('../../src/lib/server/hosts/availability.js');
+        const placement = vi.spyOn(capacity, 'choosePlacement');
+        const prepare = vi.spyOn(db(), 'prepare');
+        for (let count = 0; count < 100; count++) {
+            expect(launchAvailability([t5, t6]).availability).toMatchObject({
+                updatedAt: new Date(epoch).toISOString(), requested: 2, estimatedAvailable: 2, portsAvailable: 3
+            });
+        }
+        expect(prepare).not.toHaveBeenCalled();
+        expect(placement).not.toHaveBeenCalled();
+    });
+
+    it('shares CPU, memory and ports across games and refreshes measurements on heartbeat', async () => {
+        const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
+        const { launchAvailability } = await import('../../src/lib/server/hosts/availability.js');
+        heartbeat(hostId, { ...inventory, memoryAvailable: GiB + 500_000_000 });
+        const limited = launchAvailability([t5, t6]);
+        expect(limited.availability).toMatchObject({ requested: 2, estimatedAvailable: 1, portsAvailable: 3,
+            cpuCoresAvailable: 15.2, memoryBytesAvailable: GiB });
+        expect(limited.warnings.join(' ')).toContain('1 of 2');
+        vi.setSystemTime(epoch + 15_000);
+        heartbeat(hostId, { ...inventory, memoryAvailable: 2 * GiB + 500_000_000 });
+        expect(launchAvailability([t5, t6]).availability).toMatchObject({ estimatedAvailable: 2,
+            memoryBytesAvailable: 2 * GiB, updatedAt: new Date(epoch + 15_000).toISOString() });
+        heartbeat(hostId, { ...inventory, cpuUsagePercent: 100 });
+        expect(launchAvailability([t5]).availability.estimatedAvailable).toBe(0);
+    });
+
+    it('refreshes port reservations and startup queue snapshots after lifecycle events', async () => {
+        const { reserveHostPort, releaseHostPorts } = await import('../../src/lib/server/hosts/index.js');
+        const { launchAvailability } = await import('../../src/lib/server/hosts/availability.js');
+        const { beginDeployment, deploymentStage } = await import('../../src/lib/server/provisioning/history.js');
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const limit = launchAvailability([t5]).availability.startupLimit!;
+        reserveHostPort(hostId, 'reserved', { port: 28960, protocol: 'udp' });
+        reserveHostPort(hostId, 'reserved', { port: 28960, protocol: 'tcp' });
+        await settle();
+        expect(launchAvailability([t5, t6, t5]).availability).toMatchObject({ portsAvailable: 2, estimatedAvailable: 2 });
+        const active = beginDeployment('t5_zombie_theater_1', 'create');
+        const queued = beginDeployment('t6_zombie_tranzit_1', 'create');
+        db().prepare("UPDATE server_deployments SET state='queued' WHERE id=?").run(queued);
+        deploymentStage(queued, 'Awaiting placement', { checkpoint: { key: 'queued', label: 'Queued for placement' } });
+        await settle();
+        expect(launchAvailability([t5]).availability).toMatchObject({ queued: 1, startupSlots: Math.max(0, limit - 1) });
+        db().prepare("UPDATE server_deployments SET state='done' WHERE id=?").run(active);
+        releaseHostPorts('reserved');
+        await settle();
+        expect(launchAvailability([t5]).availability).toMatchObject({ portsAvailable: 3, startupSlots: limit });
+    });
+
+    it('stops counting a stale host without presenting old measurements as current capacity', async () => {
+        const { launchAvailability } = await import('../../src/lib/server/hosts/availability.js');
+        vi.setSystemTime(epoch + 91_000);
+        const result = launchAvailability([t5]);
+        expect(result.availability).toMatchObject({ updatedAt: new Date(epoch).toISOString(), estimatedAvailable: 0, portsAvailable: 0 });
+        expect(result.warnings.join(' ')).toContain('Waiting for an eligible host');
+    });
+
+    it('keeps heartbeats working while provisioning schema is unavailable and refreshes after initialization', async () => {
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
+        const { launchAvailability } = await import('../../src/lib/server/hosts/availability.js');
+        const errors = vi.spyOn(console, 'error');
+        db().exec('ALTER TABLE native_provisioning RENAME TO unavailable_native_provisioning');
+        expect(() => heartbeat(hostId, inventory)).not.toThrow();
+        expect(launchAvailability([t5]).availability.updatedAt).toBeNull();
+        await settle();
+        expect(errors).not.toHaveBeenCalled();
+        db().exec('ALTER TABLE unavailable_native_provisioning RENAME TO native_provisioning');
+        heartbeat(hostId, inventory);
+        expect(launchAvailability([t5]).availability).toMatchObject({ estimatedAvailable: 1, portsAvailable: 3 });
+    });
+});