From caa7672bec53ad4f7f4831e37f1255c9bc6cecff Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Sun, 4 Oct 2026 23:41:48 -0400 Subject: [PATCH] Use live host usage and running estimates for collection admission --- docs/native-hosts.md | 26 ++++++++------- src/lib/server/hosts/capacity.ts | 46 +++++++++++++------------- tests/unit/native-provisioning.test.ts | 35 ++++++++++++++++++-- 3 files changed, 70 insertions(+), 37 deletions(-) diff --git a/docs/native-hosts.md b/docs/native-hosts.md index 9d1234b..a3a9cd8 100644 --- a/docs/native-hosts.md +++ b/docs/native-hosts.md @@ -180,19 +180,21 @@ heartbeat ingestion, at most once per minute per host/fleet phase; new peaks raise the cache immediately. Launch admission reads only the cached model. Scheduling still leaves **5% host CPU and 500 MB RAM (500,000,000 bytes)** for the OS and bursts. -These allowances are scheduling reservations, not Docker limits. Queued and -unresolved creations consume the startup allowance; established instances keep -the running allowance even while idle or stopped. Current container measurements -are credited against their reservations because host usage already includes them; -only the unused allowance is added. Missing or mismatched-generation measurements -cannot release an unresolved launch's reservation. Observed host load also counts, -so other processes and previously unseen load increases reduce available capacity. +These allowances are scheduling estimates, not Docker limits. Collection admission +budgets new and never-submitted queued servers at their running allowance, rather +than assuming the entire collection starts simultaneously. Current host CPU and +available memory account for existing containers and other processes. Observed +containers are not topped up to historical startup or running peaks, even while +awaiting health verification. Established instances are covered by host usage +when individual container measurements are unavailable. -Admission counts all queued reservations. Dispatch evaluates one launch against -existing instances and submitted or uncertain work; other never-submitted queued -launches wait their turn. They retain their port reservations. This lets the queue -make progress even if a changed allowance means the whole backlog no longer fits -at once. Capacity errors report projected usage, usable capacity and reservations. +Dispatch checks the next launch's startup allowance against live host usage. +Other never-submitted queued launches wait their turn and retain their port +reservations. Submitted or uncertain launches missing a matching-generation +measurement still consume a full startup allowance until telemetry accounts for +them. This lets collections queue based on running demand and start as capacity +becomes available, without stacking every queued server's startup peak. +Capacity errors report projected usage, usable capacity and reservations. When startup capacity is available, provisioning checks each eligible host’s current CPU and available memory against reservations and headroom. diff --git a/src/lib/server/hosts/capacity.ts b/src/lib/server/hosts/capacity.ts index 04d6a3a..b006d78 100644 --- a/src/lib/server/hosts/capacity.ts +++ b/src/lib/server/hosts/capacity.ts @@ -55,42 +55,42 @@ function projectedLoad(host: Host, options: PlacementOptions): number { { id: string; generation: number; pending: number; unsubmitted: number }[]; const { running, startup } = capacityCalibration(host.id); const planned = options.planned?.filter(item => item.hostId === host.id).length ?? 0; - let extraCpu = (planned + 1) * startup.cpuCores; - let extraMemory = (planned + 1) * startup.memoryBytes; - let reservedCpu = extraCpu; - let reservedMemory = extraMemory; + // Admission budgets the eventual running collection. Dispatch budgets only + // the next startup; the rest of the queue waits for live capacity to recover. + const launch = options.dispatch ? startup : running; + let extraCpu = planned * running.cpuCores + launch.cpuCores; + let extraMemory = planned * running.memoryBytes + launch.memoryBytes; let reservations = planned + 1; const measurements = new Map(host.inventory.instanceResources?.map(sample => [sample.instanceId, sample])); for (const instance of instances) { if (options.dispatch && options.excludeInstanceId && instance.unsubmitted) continue; - reservations++; - const allowance = instance.pending ? startup : running; - reservedCpu += allowance.cpuCores; - reservedMemory += allowance.memoryBytes; - const sample = measurements.get(instance.id); - if (sample?.generation === instance.generation) { - // Host usage already includes this container. Reserve only the - // unused part of its allowance, including while awaiting health. - extraCpu += Math.max(0, allowance.cpuCores - sample.cpuCores); - extraMemory += Math.max(0, allowance.memoryBytes - sample.memoryBytes); - } else if (instance.pending) { - // Missing observations cannot release an uncertain remote launch. - extraCpu += allowance.cpuCores; - extraMemory += allowance.memoryBytes; + if (instance.unsubmitted) { + extraCpu += running.cpuCores; + extraMemory += running.memoryBytes; + reservations++; + continue; } + const sample = measurements.get(instance.id); + // Current host usage already includes observed containers, including + // those still awaiting health verification. Do not top up to old peaks. + if (sample?.generation === instance.generation || !instance.pending) continue; + // A submitted launch absent from current telemetry may still start. + extraCpu += startup.cpuCores; + extraMemory += startup.memoryBytes; + reservations++; } - const cores = Math.max(cpuCount * cpuUsagePercent / 100 + extraCpu, reservedCpu); + const cores = cpuCount * cpuUsagePercent / 100 + extraCpu; const usedMemory = memoryTotal - memoryAvailable; - const memory = Math.max(usedMemory + extraMemory, reservedMemory); + const memory = usedMemory + extraMemory; const usableMemory = memoryTotal - MEMORY_HEADROOM; const gib = (bytes: number) => `${(bytes / 1024 ** 3).toFixed(2)} GiB`; if (cores > cpuCount * CPU_USABLE_FRACTION) throw new Error(`Insufficient CPU capacity: ${cores.toFixed(2)} cores projected, ${(cpuCount * CPU_USABLE_FRACTION).toFixed(2)} usable; ${reservations} reservations including this launch.`); if (memory > usableMemory) throw new Error(`Insufficient memory capacity: ${gib(memory)} projected, ${gib(usableMemory)} usable after host headroom. ` - + `${gib(usedMemory)} currently used + ${gib(extraMemory)} additional reservations; ${reservations} reservations including this launch (${gib(startup.memoryBytes)} startup allowance).`); + + `${gib(usedMemory)} currently used + ${gib(extraMemory)} additional reservations; ${reservations} reservations including this launch (${gib(launch.memoryBytes)} ${options.dispatch ? 'startup' : 'running'} allowance).`); // Recovered resources must not remain unavailable until the historical // average catches up. History ranks hosts after current capacity passes. - const historicalMemory = Math.max(memoryTotal - historicalAvailable + extraMemory, reservedMemory); - const historicalCores = Math.max(cpuCount * historicalCpu / 100 + extraCpu, reservedCpu); + const historicalMemory = memoryTotal - historicalAvailable + extraMemory; + const historicalCores = cpuCount * historicalCpu / 100 + extraCpu; return Math.max(historicalCores / cpuCount, historicalMemory / memoryTotal); } diff --git a/tests/unit/native-provisioning.test.ts b/tests/unit/native-provisioning.test.ts index 4b37326..991ecad 100644 --- a/tests/unit/native-provisioning.test.ts +++ b/tests/unit/native-provisioning.test.ts @@ -730,6 +730,37 @@ describe('automatic capacity admission and placement', () => { expect(capacityCalibration(hostId).startup.source).toBe('host'); expect(queueNativeCreation(input)).toMatchObject({ ok: true }); }); + it('queues a collection at running demand and releases startup reservations as live usage arrives', async () => { + await prepareLibrary(); + const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + const { fleetSettings, saveFleetSettings } = await import('../../src/lib/server/settings.js'); + const { saveBlueprint } = await import('../../src/lib/server/fleet/launch-recipes.js'); + const { previewCollection, queueCollection } = await import('../../src/lib/server/fleet/collections.js'); + const GiB = 1024 ** 3; + const measured = { ...inventory(), cpuCount: 4, cpuUsagePercent: 5, memoryTotal: 4 * GiB, memoryAvailable: 3.75 * GiB }; + heartbeat(hostId, measured); + saveFleetSettings({ ...fleetSettings(), createConcurrency: 3 }); + const insert = db().prepare(`INSERT INTO ec_resource_allowances + (scope,phase,cpu_cores,memory_bytes,samples,window_start,observed_at,computed_at) VALUES(?,?,?,?,30,?,?,?)`); + const now = new Date().toISOString(); + const start = new Date(Date.now() - 30 * 60_000).toISOString(); + insert.run(hostId, 'startup', 2, 2 * GiB, start, now, now); + insert.run(hostId, 'running', 0.2, GiB / 4, start, now, now); + const blueprint = saveBlueprint({ name: 'Staggered lobbies', entries: [{ game: 't5', mapSlug: 'theater', count: 3 }] }); + const preview = previewCollection(blueprint.id); + expect(preview.ready).toBe(true); + expect(queueCollection({ previewId: preview.id, keys: Object.fromEntries(preview.servers.map(server => [server.name!, 'key'])) }).ok).toBe(true); + await advance(); + const creates = () => db().prepare("SELECT instance_id FROM ec_host_operations WHERE kind='instance.create'").all() as { instance_id: string }[]; + expect(creates()).toHaveLength(1); + await advance(); + expect(creates()).toHaveLength(1); // No heartbeat yet: the first launch still reserves startup resources. + heartbeat(hostId, { ...measured, cpuUsagePercent: 10, memoryAvailable: 3.5 * GiB, + instanceResources: [{ instanceId: creates()[0].instance_id, generation: 1, uptimeSeconds: 60, cpuCores: 0.2, memoryBytes: GiB / 4 }] }); + await advance(); + expect(creates()).toHaveLength(2); // Its actual usage replaces the peak even before health verification. + }); it('charges queued reservations and releases them on cancellation', async () => { await prepareLibrary(); const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); @@ -801,8 +832,8 @@ describe('automatic capacity admission and placement', () => { expect(first.ok).toBe(true); if (!first.ok) return; await advance(); const instance = nativeInstance(first.name)!; - // 0.2 GiB OS + 0.8 GiB container. The remaining 0.2 GiB of its 1 GiB - // reservation plus a new 1 GiB server fit below 2.53 GiB usable memory. + // 0.2 GiB OS + 0.8 GiB container are already in host usage. + // Adding a new 1 GiB server fits below 2.53 GiB usable memory. heartbeat(hostId, { ...host, memoryAvailable: 2 * GiB, instanceResources: [{ instanceId: instance.id, generation: instance.generation, uptimeSeconds: 60, cpuCores: 0.1, memoryBytes: Math.floor(0.8 * GiB) }] }); const second = queueNativeCreation(input);