Fix capacity accounting and queued host placement

tests/unit/native-provisioning.test.ts

@@ -726,6 +726,91 @@ describe('automatic capacity admission and placement', () => {
         cancelCreation(first.deploymentId);
         expect(queueNativeCreation(input)).toMatchObject({ ok: true });
     });
+    it.each(['memory', 'cpu'] as const)('dispatches immediately after %s recovers instead of waiting for five-minute history', async resource => {
+        await prepareLibrary();
+        const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
+        const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const GiB = 1024 ** 3;
+        // A recently busy 4 GiB host is now mostly empty, as in the reported
+        // ec-2 inventory. Admission must not pass only to stall in dispatch.
+        heartbeat(hostId, { ...inventory(), cpuCount: 4, memoryTotal: 4 * GiB, memoryAvailable: Math.floor(3.8 * GiB) });
+        for (let minute = 1; minute <= 4; minute++) db().prepare(`INSERT INTO ec_host_resource_samples
+            (host_id,sampled_at,cpu_count,cpu_usage_percent,memory_total,memory_available) VALUES(?,?,4,?,?,?)`)
+            .run(hostId, new Date(Date.now() - minute * 60_000).toISOString(), resource === 'cpu' ? 100 : 10,
+                4 * GiB, resource === 'memory' ? GiB / 8 : Math.floor(3.8 * GiB));
+        const jobs = Array.from({ length: 3 }, () => queueNativeCreation(input));
+        expect(jobs.every(job => job.ok)).toBe(true);
+        await advance();
+        expect(await operationFor(jobs[0].ok ? jobs[0].deploymentId : 'missing')).not.toBeNull();
+    });
+    it('dispatches one queued launch when learned startup costs outgrow the original batch reservation', async () => {
+        await prepareLibrary();
+        const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
+        saveFleetSettings({ ...fleetSettings(), createConcurrency: 3 });
+        const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
+        const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
+        const { db } = await import('../../src/lib/server/db/index.js');
+        const GiB = 1024 ** 3;
+        heartbeat(hostId, { ...inventory(), cpuCount: 4, memoryTotal: 4 * GiB, memoryAvailable: Math.floor(3.8 * GiB) });
+        const jobs = Array.from({ length: 3 }, () => queueNativeCreation(input));
+        expect(jobs.every(job => job.ok)).toBe(true);
+        // Learning increases the startup allowance after admission. All three
+        // no longer fit concurrently, but this otherwise empty host fits one.
+        db().prepare(`INSERT INTO ec_resource_allowances
+            (scope,phase,cpu_cores,memory_bytes,samples,window_start,observed_at,computed_at)
+            VALUES(?,'startup',0.5,?,3,?,?,?)`).run(hostId, 2 * GiB,
+                new Date(Date.now() - 180_000).toISOString(), new Date().toISOString(), new Date().toISOString());
+        await advance();
+        const operations = await Promise.all(jobs.map(job => operationFor(job.ok ? job.deploymentId : 'missing')));
+        expect(operations.filter(Boolean)).toHaveLength(1);
+        expect(operations[0]?.kind).toBe('instance.create');
+        // The dispatched operation now consumes capacity and blocks the others,
+        // even though the next host heartbeat has not observed it yet.
+        await advance();
+        expect(db().prepare("SELECT * FROM ec_host_operations WHERE kind='instance.create'").all()).toHaveLength(1);
+        expect(queueNativeCreation(input)).toMatchObject({ ok: false, kind: 'conflict' });
+    });
+    it('does not charge a starting container twice when its usage is already in the heartbeat', async () => {
+        await prepareLibrary();
+        const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js');
+        saveFleetSettings({ ...fleetSettings(), createConcurrency: 2 });
+        const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
+        const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
+        const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
+        const GiB = 1024 ** 3;
+        const host = { ...inventory(), cpuCount: 4, memoryTotal: 3 * GiB, memoryAvailable: 2.8 * GiB };
+        heartbeat(hostId, { ...host, memoryAvailable: Math.floor(host.memoryAvailable) });
+        const first = queueNativeCreation(input);
+        expect(first.ok).toBe(true); if (!first.ok) return;
+        await advance();
+        const instance = nativeInstance(first.name)!;
+        // 0.2 GiB OS + 0.8 GiB container. The remaining 0.2 GiB of its 1 GiB
+        // reservation plus a new 1 GiB server fit below 2.5 GiB usable memory.
+        heartbeat(hostId, { ...host, memoryAvailable: 2 * GiB, instanceResources: [{ instanceId: instance.id,
+            generation: instance.generation, uptimeSeconds: 60, cpuCores: 0.1, memoryBytes: Math.floor(0.8 * GiB) }] });
+        const second = queueNativeCreation(input);
+        expect(second).toMatchObject({ ok: true });
+        await advance();
+        expect(await operationFor(second.ok ? second.deploymentId : 'missing')).not.toBeNull();
+    });
+    it.each(['missing', 'old-generation'] as const)('keeps full reservations for a submitted launch with %s measurements', async evidence => {
+        await prepareLibrary();
+        const { heartbeat } = await import('../../src/lib/server/hosts/index.js');
+        const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');
+        const { nativeInstance } = await import('../../src/lib/server/backends/native.js');
+        const GiB = 1024 ** 3;
+        heartbeat(hostId, { ...inventory(), memoryTotal: 3 * GiB, memoryAvailable: Math.floor(2.8 * GiB) });
+        const first = queueNativeCreation(input);
+        expect(first.ok).toBe(true); if (!first.ok) return;
+        await advance();
+        const instance = nativeInstance(first.name)!;
+        heartbeat(hostId, { ...inventory(), memoryTotal: 3 * GiB, memoryAvailable: 2 * GiB,
+            instanceResources: evidence === 'missing' ? [] : [{ instanceId: instance.id, generation: instance.generation + 1,
+                uptimeSeconds: 60, cpuCores: 0.1, memoryBytes: Math.floor(0.8 * GiB) }] });
+        expect(queueNativeCreation(input)).toMatchObject({ ok: false, kind: 'conflict',
+            message: expect.stringContaining('3.00 GiB projected, 2.50 GiB usable') });
+    });
     it('waits when capacity disappears after admission without sending a create command', async () => {
         await prepareLibrary();
         const { queueNativeCreation } = await import('../../src/lib/server/provisioning/native.js');