From 09ef8c6ec888284d248ac1cefe342e0f423f7bb8 Mon Sep 17 00:00:00 2001 From: Josh Wright Date: Mon, 5 Oct 2026 12:06:29 -0400 Subject: [PATCH] Add capacity overview with host resource history --- README.md | 4 + docs/native-hosts.md | 23 +++ src/lib/capacity.ts | 63 +++++++ src/lib/server/hosts/availability.ts | 13 +- src/lib/server/hosts/capacity-history.ts | 66 ++++++++ src/lib/server/hosts/capacity-overview.ts | 69 ++++++++ src/lib/server/hosts/capacity-page.ts | 10 ++ src/lib/server/reports/index.ts | 15 +- src/lib/server/reports/worker.ts | 13 +- .../components/DeployCollectionDialog.svelte | 1 + .../components/capacity/CapacityChart.svelte | 151 +++++++++++++++++ src/routes/+layout.svelte | 1 + src/routes/api/capacity/+server.ts | 7 + src/routes/capacity/+page.server.ts | 5 + src/routes/capacity/+page.svelte | 137 ++++++++++++++++ src/routes/hosts/+page.svelte | 4 +- tests/unit/auth-access.test.ts | 2 +- tests/unit/capacity-history.test.ts | 116 +++++++++++++ tests/unit/capacity-overview.test.ts | 155 ++++++++++++++++++ 19 files changed, 845 insertions(+), 10 deletions(-) create mode 100644 src/lib/capacity.ts create mode 100644 src/lib/server/hosts/capacity-history.ts create mode 100644 src/lib/server/hosts/capacity-overview.ts create mode 100644 src/lib/server/hosts/capacity-page.ts create mode 100644 src/lib/ui/components/capacity/CapacityChart.svelte create mode 100644 src/routes/api/capacity/+server.ts create mode 100644 src/routes/capacity/+page.server.ts create mode 100644 src/routes/capacity/+page.svelte create mode 100644 tests/unit/capacity-history.test.ts create mode 100644 tests/unit/capacity-overview.test.ts diff --git a/README.md b/README.md index f6bc3de..f34ca33 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,10 @@ including SQLite metadata and published game-file storage. See [Native host management](docs/native-hosts.md) for installation, credentials, storage, failure recovery, API contracts and the live-host acceptance checklist. +**Capacity** shows available CPU, memory, ports, estimated additional servers, +and the shared startup queue. Filter by host or game, inspect each host's limiting +resource, and explore up to seven days of resource headroom. + Existing hosts need the current installer once to enable automatic updates. Generate it under **Agent access** and rerun it on the same host without `--reenroll`; identity and game data are preserved. Thereafter hosts check their diff --git a/docs/native-hosts.md b/docs/native-hosts.md index ac5cdcb..90a021d 100644 --- a/docs/native-hosts.md +++ b/docs/native-hosts.md @@ -233,6 +233,29 @@ systemctl status electric-cherry-update.timer journalctl -u electric-cherry-update.service -n 100 ``` +## Capacity overview + +Open **Capacity** (`/capacity`) to see the same cached estimates used by collection +previews. Available totals include hosts ready for the default runtime and latest +files for the selected game. **All games** counts shared resources once, including +hosts ready for either game; it is not a guarantee that every game fits every host. +The host breakdown shows readiness, missing measurements, available ports, learned +running/startup allowances, and the resource limiting additional servers. + +Startup slots and queued jobs are fleet-wide, even with a host or game selected. +Server estimates use running allowances; provisioning still checks the larger +startup allowance before starting each job. Port availability uses each host's +configured bind address. Queued unplaced jobs do not reserve physical resources. + +The history graph offers the last hour, 24 hours, or seven days of measured CPU +and memory headroom, after the usual OS reserve. It averages each host's reports +within each period, then sums hosts for fleet views. Missing reports appear as gaps. +Historical measurements cover the selected currently registered hosts and do not +reconstruct past readiness, port reservations, or startup slots. Reports refresh +automatically while the page is visible, and historical queries run in a separate +reader worker. `GET /api/capacity` supports `range=1h|24h|7d`, `host=`, and +`game=all|t5|t6`; host/game selections also remain in the page URL. + ## Identity, transport and credentials Enrollment tokens are random, hashed at rest, single-use and expire after 15 diff --git a/src/lib/capacity.ts b/src/lib/capacity.ts new file mode 100644 index 0000000..e689e93 --- /dev/null +++ b/src/lib/capacity.ts @@ -0,0 +1,63 @@ +import type { CapacityCalibration, HostReadiness } from './hosts.js'; + +export type CapacityRange = '1h' | '24h' | '7d'; +export type CapacityGame = 'all' | 't5' | 't6'; +export interface CapacityQuery { range: CapacityRange; hostId: string | null; game: CapacityGame } +export function parseCapacityQuery(params: URLSearchParams): CapacityQuery { + const range = params.get('range'); + const game = params.get('game'); + return { range: range === '1h' || range === '7d' ? range : '24h', + game: game === 't5' || game === 't6' ? game : 'all', hostId: params.get('host')?.trim().slice(0, 100) || null }; +} + +export interface CapacityHistoryPoint { + at: number; + cpuCoresAvailable: number | null; + memoryBytesAvailable: number | null; + reportingHosts: number; + expectedHosts: number; +} +export interface CapacityHistory { + range: CapacityRange; + startAt: number; + endAt: number; + bucketMs: number; + points: CapacityHistoryPoint[]; +} +export interface CapacityHost { + id: string; + name: string; + readiness: HostReadiness; + connected: boolean; + updatedAt: string | null; + eligible: boolean; + games: ('t5' | 't6')[]; + reasons: string[]; + cpuCoresAvailable: number | null; + memoryBytesAvailable: number | null; + portsAvailable: number; + totalPorts: number; + estimatedServers: number | null; + limits: { cpu: number | null; memory: number | null; ports: number }; + limitingResource: 'CPU' | 'Memory' | 'Ports' | null; + calibration: CapacityCalibration; +} +export interface CapacityCurrent { + updatedAt: string | null; + summary: { + estimatedServers: number | null; + cpuCoresAvailable: number | null; + memoryBytesAvailable: number | null; + portsAvailable: number | null; + eligibleHosts: number; + totalHosts: number; + startupSlots: number | null; + startupLimit: number | null; + activeStartups: number | null; + queued: number | null; + }; + hosts: CapacityHost[]; + hostOptions: { id: string; name: string }[]; + warnings: string[]; +} +export interface CapacityData { query: CapacityQuery; current: CapacityCurrent; history: CapacityHistory } diff --git a/src/lib/server/hosts/availability.ts b/src/lib/server/hosts/availability.ts index 8336bc6..a9e3e36 100644 --- a/src/lib/server/hosts/availability.ts +++ b/src/lib/server/hosts/availability.ts @@ -3,7 +3,7 @@ import { fleetSettings } from '../settings.js'; import { subscribe } from '../pipeline/bus.js'; import { capacityCalibration } from './calibration.js'; import { HEARTBEAT_MAX_AGE_MS, listHosts } from './index.js'; -import type { Host, NativePlacement } from '../../hosts.js'; +import type { CapacityCalibration, Host, NativePlacement } from '../../hosts.js'; import type { LaunchAvailability } from '../../deployment-plan.js'; interface HostAvailability { @@ -12,6 +12,7 @@ interface HostAvailability { freeMemory: number | null; runningCpu: number; runningMemory: number; + calibration: CapacityCalibration; ports: { port: number; bind_address: string }[]; } interface Snapshot { @@ -58,6 +59,7 @@ function observeHost(host: Host): HostAvailability { const freeMemory = memoryTotal && memoryAvailable !== undefined && memoryAvailable <= memoryTotal ? Math.max(0, memoryAvailable - 500_000_000 - unmeasured * startup.memoryBytes) : null; return { host, freeCpu, freeMemory, runningCpu: running.cpuCores, runningMemory: running.memoryBytes, + calibration: { running, startup }, ports: handle.prepare('SELECT port,bind_address FROM ec_host_ports WHERE host_id=?').all(host.id) as HostAvailability['ports'] }; } @@ -90,13 +92,20 @@ subscribe(event => { if (event.type === 'fleet-invalidated' || event.type === 'server-status') scheduleAvailabilityRefresh(); }); -function freePorts(snapshot: HostAvailability, bindAddress?: string): number { +export function freePorts(snapshot: HostAvailability, bindAddress?: string): number { const bind = bindAddress ?? snapshot.host.bindAddress; const used = new Set(snapshot.ports.filter(row => row.port >= snapshot.host.portMin && row.port <= snapshot.host.portMax && (row.bind_address === bind || ['0.0.0.0', '::'].includes(bind) || ['0.0.0.0', '::'].includes(row.bind_address))).map(row => row.port)); return Math.max(0, snapshot.host.portMax - snapshot.host.portMin + 1 - used.size); } +/** Share the launch estimate's already observed state without rerunning placement. */ +export function readAvailabilitySnapshot(): Snapshot | null { + const snapshot = snapshots.get(db()); + if (!snapshot) scheduleAvailabilityRefresh(); + return snapshot ?? null; +} + /** Estimates only: no placement, historical scans, reservations, or live host requests. */ export function launchAvailability(selections: NativePlacement[]): { availability: LaunchAvailability; warnings: string[] } { const snapshot = snapshots.get(db()); diff --git a/src/lib/server/hosts/capacity-history.ts b/src/lib/server/hosts/capacity-history.ts new file mode 100644 index 0000000..0a4bae6 --- /dev/null +++ b/src/lib/server/hosts/capacity-history.ts @@ -0,0 +1,66 @@ +import { db } from '../db/index.js'; +import { ensureHostSchema } from './schema.js'; +import type { CapacityHistory, CapacityQuery, CapacityRange } from '../../capacity.js'; + +const windows: Record = { + '1h': { duration: 3_600_000, bucketMs: 60_000 }, + '24h': { duration: 86_400_000, bucketMs: 600_000 }, + '7d': { duration: 7 * 86_400_000, bucketMs: 3_600_000 } +}; + +/** Read on the report worker: heartbeat history can contain millions of rows. */ +export function readCapacityHistory(query: CapacityQuery, now = Date.now()): CapacityHistory { + const handle = db(); + ensureHostSchema(handle); + const { duration, bucketMs } = windows[query.range]; + const startAt = now - duration; + const count = duration / bucketMs; + const hosts = handle.prepare(`SELECT id,created_at FROM ec_hosts + WHERE deleted_at IS NULL ${query.hostId ? 'AND id=?' : ''} ORDER BY id`) + .all(...(query.hostId ? [query.hostId] : [])) as { id: string; created_at: string }[]; + // The primary key bounds each host scan to the selected interval. Aggregate + // before crossing the worker boundary; do not return raw heartbeat rows. + const samples = handle.prepare(`SELECT + MIN(?, CAST(((CAST(strftime('%s',sampled_at) AS INTEGER)*1000 + + CAST(substr(strftime('%f',sampled_at),4,3) AS INTEGER)) - ?) / ? AS INTEGER)) AS bucket, + AVG(MAX(0,cpu_count * (0.95 - cpu_usage_percent / 100.0))) AS cpu, + AVG(MAX(0,memory_available - 500000000)) AS memory + FROM ec_host_resource_samples + WHERE host_id=? AND sampled_at>=? AND sampled_at<=? + AND cpu_count>0 AND cpu_usage_percent BETWEEN 0 AND 100 + AND memory_total>0 AND memory_available BETWEEN 0 AND memory_total + GROUP BY bucket`); + const points = Array.from({ length: count }, (_, index) => ({ + at: startAt + index * bucketMs, + cpuCoresAvailable: null as number | null, + memoryBytesAvailable: null as number | null, + reportingHosts: 0, + expectedHosts: 0 + })); + for (const host of hosts) { + const createdAt = Date.parse(host.created_at); + for (const point of points) { + if (createdAt <= Math.min(now, point.at + bucketMs)) point.expectedHosts++; + } + const buckets = samples.all(count - 1, startAt, bucketMs, host.id, + new Date(startAt).toISOString(), new Date(now).toISOString()) as { bucket: number; cpu: number; memory: number }[]; + for (const bucket of buckets) { + const point = points[bucket.bucket]; + if (!point || createdAt > Math.min(now, point.at + bucketMs)) continue; + // Average each host first so one host reporting more frequently + // cannot change another host's contribution to fleet headroom. + point.reportingHosts++; + point.cpuCoresAvailable = (point.cpuCoresAvailable ?? 0) + bucket.cpu; + point.memoryBytesAvailable = (point.memoryBytesAvailable ?? 0) + bucket.memory; + } + } + for (const point of points) { + // A missing host is unknown, not zero available capacity. Keep the gap + // visible instead of presenting a partial sum as the fleet total. + if (point.reportingHosts !== point.expectedHosts) { + point.cpuCoresAvailable = null; + point.memoryBytesAvailable = null; + } + } + return { range: query.range, startAt, endAt: now, bucketMs, points }; +} diff --git a/src/lib/server/hosts/capacity-overview.ts b/src/lib/server/hosts/capacity-overview.ts new file mode 100644 index 0000000..7628a91 --- /dev/null +++ b/src/lib/server/hosts/capacity-overview.ts @@ -0,0 +1,69 @@ +import { setImmediate as yieldToController } from 'node:timers/promises'; +import { config } from '../config.js'; +import { newestReadyLibraryId } from '../game-files/index.js'; +import { HEARTBEAT_MAX_AGE_MS } from './index.js'; +import { freePorts, readAvailabilitySnapshot } from './availability.js'; +import type { CapacityCurrent, CapacityHost, CapacityQuery } from '../../capacity.js'; + +/** The page and collection preview share heartbeat/lifecycle capacity snapshots. */ +export async function readCapacityCurrent(query: CapacityQuery): Promise { + let snapshot = readAvailabilitySnapshot(); + if (!snapshot) { + // The initial snapshot is populated after the request yields, without + // historical scans, placement, or a request to any host. + await yieldToController(); + snapshot = readAvailabilitySnapshot(); + } + if (!snapshot) return { updatedAt: null, hosts: [], hostOptions: [], warnings: ['Capacity measurements are updating.'], + summary: { estimatedServers: null, cpuCoresAvailable: null, memoryBytesAvailable: null, portsAvailable: null, + eligibleHosts: 0, totalHosts: 0, startupSlots: null, startupLimit: null, activeStartups: null, queued: null } }; + const libraries = { t5: newestReadyLibraryId('t5'), t6: newestReadyLibraryId('t6') }; + const requestedGames = query.game === 'all' ? ['t5', 't6'] as const : [query.game]; + const now = Date.now(); + const hosts: CapacityHost[] = [...snapshot.hosts.values()].filter(({ host }) => !query.hostId || host.id === query.hostId).map(observation => { + const { host, calibration } = observation; + const fresh = !!host.lastHeartbeat && now - Date.parse(host.lastHeartbeat) <= HEARTBEAT_MAX_AGE_MS; + const connected = host.connected && fresh; + const games = (['t5', 't6'] as const).filter(game => libraries[game] && host.inventory.libraries?.some( + library => library.id === libraries[game] && ['ready', 'outdated'].includes(library.state))); + const reasons: string[] = []; + if (!connected) reasons.push(host.enrolled ? 'Waiting for a fresh host report.' : 'This host has not enrolled yet.'); + if (host.mode !== 'active') reasons.push(host.mode === 'draining' ? 'Host is draining.' : 'Host is disabled.'); + if (host.decommission) reasons.push('Host is being removed.'); + if (!host.credentialActive && host.enrolled) reasons.push('Host credentials are inactive.'); + if (host.readiness !== 'ready') reasons.push(...host.readinessReasons); + if (!host.inventory.runtimeImages?.includes(config.komodo.defaultImage)) reasons.push('The default runtime is not ready.'); + if (!requestedGames.some(game => games.includes(game))) reasons.push('Latest game files for this selection are not ready.'); + if ((host.inventory.storageAvailable ?? 0) < host.minimumFreeBytes + 1024 ** 3) reasons.push('Storage is below the launch reserve.'); + const eligible = connected && host.readiness === 'ready' && reasons.length === 0; + const cpuCoresAvailable = fresh ? observation.freeCpu : null; + const memoryBytesAvailable = fresh ? observation.freeMemory : null; + const portsAvailable = freePorts(observation); + const limits = { cpu: cpuCoresAvailable === null ? null : Math.max(0, Math.floor(cpuCoresAvailable / calibration.running.cpuCores)), + memory: memoryBytesAvailable === null ? null : Math.max(0, Math.floor(memoryBytesAvailable / calibration.running.memoryBytes)), + ports: portsAvailable }; + const estimatedServers = !eligible ? 0 : limits.cpu === null || limits.memory === null ? null : Math.min(limits.cpu, limits.memory, limits.ports); + const limitingResource: CapacityHost['limitingResource'] = estimatedServers === null || !eligible ? null + : limits.ports === estimatedServers ? 'Ports' : limits.memory === estimatedServers ? 'Memory' : 'CPU'; + if (eligible && (limits.cpu === null || limits.memory === null)) reasons.push('Waiting for valid CPU and memory measurements.'); + else if (eligible && estimatedServers === 0) reasons.push(`No additional servers currently fit: ${limitingResource?.toLowerCase()} capacity is full.`); + return { id: host.id, name: host.name, readiness: host.readiness, connected, updatedAt: host.lastHeartbeat, + eligible, games, reasons: [...new Set(reasons)], cpuCoresAvailable, memoryBytesAvailable, + portsAvailable, totalPorts: host.portMax - host.portMin + 1, estimatedServers, limits, limitingResource, calibration }; + }).sort((first, second) => first.name.localeCompare(second.name)); + const eligible = hosts.filter(host => host.eligible); + const sum = (key: 'cpuCoresAvailable' | 'memoryBytesAvailable' | 'estimatedServers') => + eligible.some(host => host[key] === null) ? null : eligible.reduce((total, host) => total + (host[key] ?? 0), 0); + const warnings: string[] = []; + if (!hosts.length) warnings.push(query.hostId ? 'The selected host is unavailable.' : 'Add a host to start tracking launch capacity.'); + else if (!eligible.length) warnings.push('No host is ready for this game selection. Review the host details below.'); + else if (eligible.length < hosts.length) warnings.push(`${hosts.length - eligible.length} host${hosts.length - eligible.length === 1 ? ' is' : 's are'} excluded from the available totals.`); + if (eligible.some(host => host.estimatedServers === null)) warnings.push('Some resource measurements are missing; the total estimate is unknown.'); + if (!requestedGames.some(game => libraries[game])) warnings.push('Upload game files before launching this selection.'); + return { updatedAt: snapshot.updatedAt, hosts, + hostOptions: [...snapshot.hosts.values()].map(({ host }) => ({ id: host.id, name: host.name })).sort((a, b) => a.name.localeCompare(b.name)), + warnings, summary: { estimatedServers: sum('estimatedServers'), cpuCoresAvailable: sum('cpuCoresAvailable'), + memoryBytesAvailable: sum('memoryBytesAvailable'), portsAvailable: eligible.reduce((total, host) => total + host.portsAvailable, 0), + eligibleHosts: eligible.length, totalHosts: hosts.length, startupSlots: Math.max(0, snapshot.startupLimit - snapshot.active), + startupLimit: snapshot.startupLimit, activeStartups: snapshot.active, queued: snapshot.queued } }; +} diff --git a/src/lib/server/hosts/capacity-page.ts b/src/lib/server/hosts/capacity-page.ts new file mode 100644 index 0000000..1634456 --- /dev/null +++ b/src/lib/server/hosts/capacity-page.ts @@ -0,0 +1,10 @@ +import { error } from '@sveltejs/kit'; +import { capacityHistoryReport } from '../reports/index.js'; +import { readCapacityCurrent } from './capacity-overview.js'; +import type { CapacityData, CapacityQuery } from '../../capacity.js'; + +export async function capacityPage(query: CapacityQuery): Promise { + const current = await readCapacityCurrent(query); + if (query.hostId && !current.hostOptions.some(host => host.id === query.hostId)) error(404, 'Host not found.'); + return { query, current, history: await capacityHistoryReport(query) }; +} diff --git a/src/lib/server/reports/index.ts b/src/lib/server/reports/index.ts index b1301a3..6b5383e 100644 --- a/src/lib/server/reports/index.ts +++ b/src/lib/server/reports/index.ts @@ -4,6 +4,8 @@ import { config } from '../config.js'; import { db } from '../db/index.js'; import type { MetricsQuery, MetricsData } from '../../metrics.js'; import type { LeaderboardOptions, LeaderboardResponse, PlayerProfile, LeaderboardEntry } from '../../leaderboard.js'; +import type { CapacityHistory, CapacityQuery } from '../../capacity.js'; +import { ensureHostSchema } from '../hosts/schema.js'; class ReportWorker { private worker: Worker | null = null; @@ -17,8 +19,9 @@ class ReportWorker { await worker?.terminate(); } run(kind: string, input: unknown): Promise { - const requestInput = input as { query?: MetricsQuery; now?: number }; - const key = JSON.stringify(kind === 'metrics' ? [kind, requestInput.query, Math.floor(requestInput.now! / 30_000)] : [kind, input]); + const requestInput = input as { query?: unknown; now?: number }; + const timed = kind === 'metrics' || kind === 'capacity'; + const key = JSON.stringify(timed ? [kind, requestInput.query, Math.floor(requestInput.now! / 30_000)] : [kind, input]); const flight = this.flights.get(key); if (flight) return flight; if (this.pending.size >= 64) return Promise.reject(new Error('Reports are busy. Please retry shortly.')); @@ -58,7 +61,13 @@ class ReportWorker { } const metrics = new ReportWorker(); const leaderboard = new ReportWorker(); +const capacity = new ReportWorker(); export const metricsReport = (query: MetricsQuery) => metrics.run('metrics', { query, now: Date.now() }); export const leaderboardReport = (query: LeaderboardOptions) => leaderboard.run('leaderboard', query); export const playerReport = (guid: string, offset: number, options: LeaderboardOptions) => leaderboard.run<{ profile: PlayerProfile | null; entry: LeaderboardEntry | null }>('player', { guid, offset, options }); -export async function closeReports() { await Promise.all([metrics.close(), leaderboard.close()]); } +export const capacityHistoryReport = (query: CapacityQuery) => { + ensureHostSchema(db()); + // Game changes current placement estimates, not measured host resources. + return capacity.run('capacity', { query: { ...query, game: 'all' }, now: Date.now() }); +}; +export async function closeReports() { await Promise.all([metrics.close(), leaderboard.close(), capacity.close()]); } diff --git a/src/lib/server/reports/worker.ts b/src/lib/server/reports/worker.ts index 960ec21..bf0f497 100644 --- a/src/lib/server/reports/worker.ts +++ b/src/lib/server/reports/worker.ts @@ -3,6 +3,7 @@ import { db } from '../db/index.js'; import { readMetrics } from '../fleet/metrics.js'; import { getLeaderboard, getPlayerLeaderboardEntry } from '../fleet/leaderboard.js'; import { getPlayerProfile } from '../fleet/player-profile.js'; +import { readCapacityHistory } from '../hosts/capacity-history.js'; const cache = new Map(); let bytes = 0; @@ -11,12 +12,18 @@ parentPort!.on('message', ({ id, kind, input }) => { try { const result = db().transaction(() => { const topic = kind === 'metrics' ? 'metrics' : 'leaderboard'; - const { revision } = db().prepare('SELECT revision FROM report_revisions WHERE topic=?').get(topic) as { revision: number }; - const now = kind === 'metrics' ? input.now : Date.now(); - const key = JSON.stringify([kind, revision, kind === 'metrics' ? Math.floor(now / 30_000) : 0, kind === 'metrics' ? input.query : input]); + // Capacity refreshes every 30 seconds without adding write triggers + // to the heartbeat path. It also has its own worker, so large + // history ranges cannot queue ahead of fleet or player reports. + const revision = kind === 'capacity' ? 0 + : (db().prepare('SELECT revision FROM report_revisions WHERE topic=?').get(topic) as { revision: number }).revision; + const timed = kind === 'metrics' || kind === 'capacity'; + const now = timed ? input.now : Date.now(); + const key = JSON.stringify([kind, revision, timed ? Math.floor(now / 30_000) : 0, timed ? input.query : input]); const saved = cache.get(key); if (saved) { cache.delete(key); cache.set(key, saved); return { value: saved.value, cached: true }; } const value = kind === 'metrics' ? readMetrics(input.query, now) + : kind === 'capacity' ? readCapacityHistory(input.query, now) : kind === 'leaderboard' ? getLeaderboard(input) : { profile: getPlayerProfile(input.guid, input.offset, input.options), entry: getPlayerLeaderboardEntry(input.guid, input.options) }; const size = Buffer.byteLength(JSON.stringify(value)); diff --git a/src/lib/ui/components/DeployCollectionDialog.svelte b/src/lib/ui/components/DeployCollectionDialog.svelte index ef30e6c..fa082d4 100644 --- a/src/lib/ui/components/DeployCollectionDialog.svelte +++ b/src/lib/ui/components/DeployCollectionDialog.svelte @@ -118,6 +118,7 @@ {#if preview.availability && preview.servers.length}

Latest reported capacity

+ View capacity and history ↗

{preview.availability.estimatedAvailable === null ? 'An estimate is not available yet.' : `${preview.availability.estimatedAvailable} additional servers estimated to fit.`} {preview.availability.requested} requested.

CPU available
{preview.availability.cpuCoresAvailable === null ? 'Unknown' : `${preview.availability.cpuCoresAvailable.toFixed(1)} cores`}
diff --git a/src/lib/ui/components/capacity/CapacityChart.svelte b/src/lib/ui/components/capacity/CapacityChart.svelte new file mode 100644 index 0000000..34dbeb6 --- /dev/null +++ b/src/lib/ui/components/capacity/CapacityChart.svelte @@ -0,0 +1,151 @@ + + +
+
+

Capacity over time

Measured resource headroom · {scope}

+
+ + +
+
+
{label} available
+
+ +
+ + {#if !measured.length}
Waiting for capacity historyFresh host reports build this timeline automatically.
{/if} +
+ +
+ {#if selected} +
+
Selected period{$timeFormat.period(selected.at)}
+
CPU available
{amount(selected.cpuCoresAvailable, 'cores')}
+
Memory available
{amount(selected.memoryBytesAvailable === null ? null : selected.memoryBytesAvailable / 1024 ** 3, 'GiB')}
+
Reporting hosts
{selected.reportingHosts} / {selected.expectedHosts}
+
+ {/if} +

Host headroom after the operating-system reserve, averaged per period. Gaps mean reporting is incomplete. The game filter changes current launch estimates; this history includes all workloads on the selected hosts.

+
View history data · {$timeZone.replaceAll('_', ' ')} + {#if tableOpen} + +
+ + + {#each points as point (point.at)}{/each} +
Measured headroom; a dash means a complete measurement is unavailable.
PeriodCPU coresMemory (GiB)Reporting hosts
{$timeFormat.period(point.at)}{point.cpuCoresAvailable === null ? '—' : numbers.format(point.cpuCoresAvailable)}{point.memoryBytesAvailable === null ? '—' : numbers.format(point.memoryBytesAvailable / 1024 ** 3)}{point.reportingHosts} / {point.expectedHosts}
+ {/if} +
+
+ + diff --git a/src/routes/+layout.svelte b/src/routes/+layout.svelte index 74809cb..57c2bd6 100644 --- a/src/routes/+layout.svelte +++ b/src/routes/+layout.svelte @@ -131,6 +131,7 @@ const tabs = [ { href: '/', label: 'Fleet', match: (p: string) => p === '/' || p.startsWith('/servers/') }, { href: '/hosts', label: 'Hosts', match: (p: string) => /^\/hosts(\/|$)/.test(p) }, + { href: '/capacity', label: 'Capacity', match: (p: string) => p === '/capacity' }, { href: '/scripts', label: 'Library', match: (p: string) => /^\/(scripts|configs|plugins|game-files)(\/|$)/.test(p) }, { href: '/provisioning', label: 'Operations', match: (p: string) => /^\/(provisioning|deployments|collections|campaigns|maintenance|watchdog|server-keys)(\/|$)/.test(p) }, { href: '/betas', label: 'Betas', match: (p: string) => p.startsWith('/betas') }, diff --git a/src/routes/api/capacity/+server.ts b/src/routes/api/capacity/+server.ts new file mode 100644 index 0000000..584c10b --- /dev/null +++ b/src/routes/api/capacity/+server.ts @@ -0,0 +1,7 @@ +import { json } from '@sveltejs/kit'; +import { parseCapacityQuery } from '$lib/capacity.js'; +import { capacityPage } from '$lib/server/hosts/capacity-page.js'; + +export async function GET({ url }: { url: URL }) { + return json(await capacityPage(parseCapacityQuery(url.searchParams)), { headers: { 'cache-control': 'no-store' } }); +} diff --git a/src/routes/capacity/+page.server.ts b/src/routes/capacity/+page.server.ts new file mode 100644 index 0000000..2922a9a --- /dev/null +++ b/src/routes/capacity/+page.server.ts @@ -0,0 +1,5 @@ +import { parseCapacityQuery } from '$lib/capacity.js'; +import { capacityPage } from '$lib/server/hosts/capacity-page.js'; +import type { PageServerLoad } from './$types'; + +export const load: PageServerLoad = ({ url }) => capacityPage(parseCapacityQuery(url.searchParams)); diff --git a/src/routes/capacity/+page.svelte b/src/routes/capacity/+page.svelte new file mode 100644 index 0000000..c550726 --- /dev/null +++ b/src/routes/capacity/+page.svelte @@ -0,0 +1,137 @@ + + +Capacity · Electric Cherry + +
+ + {#snippet actions()}{/snippet} + + {#if refreshError || navigationError}{/if} +
+
void changeScope({ game: (event.currentTarget as HTMLSelectElement).value as CapacityGame })} />
+ +
+
+
Available now · {scopeName}
{count(summary.estimatedServers)}additional servers
+

{summary.estimatedServers === null ? 'Waiting for enough host information to estimate capacity.' : `Estimated to fit across ${summary.eligibleHosts} eligible ${summary.eligibleHosts === 1 ? 'host' : 'hosts'}.`} {report.query.game === 'all' ? 'Hosts ready for either T5 or T6 are counted once.' : `For ${report.query.game === 't5' ? 'Black Ops I' : 'Black Ops II'} launches.`}

+
{count(summary.queued)}servers queued

Across the fleet. Launches wait for capacity and startup slots automatically.

+
+
+
CPU available
{count(summary.cpuCoresAvailable)}cores

After OS reserve and pending starts

+
Memory available
{memory(summary.memoryBytesAvailable)}GiB

After OS reserve and pending starts

+
Ports available
{count(summary.portsAvailable)}ports

Free game ports on eligible hosts

+
Startup slots
{count(summary.startupSlots)}{summary.startupLimit === null ? 'available' : `/ ${summary.startupLimit}`}

{count(summary.activeStartups)} starting · controller-wide

+
+
{report.current.updatedAt ? `Capacity updated ${$timeFormat.dateTime(report.current.updatedAt)}` : 'Waiting for host reports'}Refreshes every 30 seconds while visible
+ {#if report.current.warnings.length}{/if} + +
+

Host capacity

Find the resource that limits the next launch.

{summary.eligibleHosts} / {summary.totalHosts} eligible
+ {#if !report.current.hosts.length}

{report.query.hostId ? 'This host is unavailable' : 'No hosts yet'}

{report.query.hostId ? 'Choose another host to view its capacity.' : 'Connect a host to start seeing capacity and history.'}

+ {:else}
{#each report.current.hosts as host (host.id)}
+

{host.name}

{!host.connected ? 'Offline' : hostReadinessLabel(host.readiness)}
+
{count(host.estimatedServers)}additional servers{#if host.eligible && host.limitingResource}Limited by {host.limitingResource.toLowerCase()}{/if}
+
{#if host.games.length}{#each host.games as game}{game === 't5' ? 'Black Ops I' : 'Black Ops II'}{/each}{:else}No games ready{/if}
+
CPU
{count(host.cpuCoresAvailable)} cores
{count(host.limits.cpu)} servers fit
Memory
{memory(host.memoryBytesAvailable)} GiB
{count(host.limits.memory)} servers fit
Ports
{host.portsAvailable} / {host.totalPorts}
{host.limits.ports} servers fit
+ {#if host.reasons.length}
    {#each host.reasons as reason}
  • {reason}
    • {/each}
{/if} +
Per-server resource estimates +
{#each [{ label: 'Running', value: host.calibration.running }, { label: 'Starting', value: host.calibration.startup }] as allowance}
{allowance.label}

{count(allowance.value.cpuCores)} cores · {memory(allowance.value.memoryBytes)} GiB

{allowance.value.source === 'bootstrap' ? 'Default allowance' : `${allowance.value.source === 'host' ? 'Host' : 'Fleet'} observations · ${allowance.value.samples} samples`}
{/each}
+

Starts may need more headroom than running servers. Provisioning checks actual capacity before reserving a host and port.

+
+

{host.updatedAt ? `Reported ${$timeFormat.dateTime(host.updatedAt)}` : 'No host report yet'}

+
{/each}
{/if} +
+
How capacity is estimated

Additional servers are estimated from each eligible host’s available CPU, memory, and unreserved ports, using its learned running-server allowance. The smallest resource allowance sets that host’s limit. Shared hosts are counted once.

This is a planning estimate. Queued launches do not reserve capacity, and starting servers may require a larger allowance. Actual placement also checks current host readiness, game files, runtime, and available startup slots.

The timeline shows measured host resource headroom after the operating-system reserve. It does not reconstruct historical launch eligibility, queued jobs, or port availability.

+
+ + diff --git a/src/routes/hosts/+page.svelte b/src/routes/hosts/+page.svelte index a798e30..218530c 100644 --- a/src/routes/hosts/+page.svelte +++ b/src/routes/hosts/+page.svelte @@ -39,7 +39,9 @@ Hosts · Electric Cherry
- + + {#snippet actions()}{/snippet} + {#if error}{/if} {#if notice}

{notice}

{/if} {#if canManage}
diff --git a/tests/unit/auth-access.test.ts b/tests/unit/auth-access.test.ts index 8d36af9..712bc42 100644 --- a/tests/unit/auth-access.test.ts +++ b/tests/unit/auth-access.test.ts @@ -7,7 +7,7 @@ const allows = (role: UserRole, route: string | null, method = 'GET') => roleAll describe('management route permissions', () => { it.each([ - '/', '/api/fleet', '/api/servers', '/servers/[name]', '/api/servers/[name]/scripts', + '/', '/api/fleet', '/capacity', '/api/capacity', '/api/servers', '/servers/[name]', '/api/servers/[name]/scripts', '/api/metrics', '/metrics', '/api/matches', '/matches/[id]', '/api/leaderboard', '/leaderboard/players/[game]/[guid]', '/api/campaigns/[id]', '/campaigns/[id]', '/api/provisioning/[id]', '/provisioning/[id]', '/api/provisioning/[id]/events', diff --git a/tests/unit/capacity-history.test.ts b/tests/unit/capacity-history.test.ts new file mode 100644 index 0000000..c738000 --- /dev/null +++ b/tests/unit/capacity-history.test.ts @@ -0,0 +1,116 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { newFixture, type Fixture } from '../helpers/fixture.js'; +import type { CapacityQuery } from '../../src/lib/capacity.js'; + +let fixture: Fixture; +const epoch = Date.UTC(2026, 9, 5, 12); +const GiB = 1024 ** 3; +const query: CapacityQuery = { range: '1h', hostId: null, game: 'all' }; + +beforeEach(async () => { vi.resetModules(); fixture = await newFixture([]); }); +afterEach(async () => { vi.restoreAllMocks(); await fixture.cleanup(); }); + +async function host(id: string, createdAt = epoch - 7 * 86_400_000) { + const { db } = await import('../../src/lib/server/db/index.js'); + const { ensureHostSchema } = await import('../../src/lib/server/hosts/schema.js'); + ensureHostSchema(db()); + db().prepare('INSERT INTO ec_hosts (id,name,created_at) VALUES (?,?,?)').run(id, id, new Date(createdAt).toISOString()); +} + +async function sample(id: string, at: number, cpu = 25, memory = 4 * GiB, cores = 8) { + const { db } = await import('../../src/lib/server/db/index.js'); + db().prepare(`INSERT INTO ec_host_resource_samples + (host_id,sampled_at,cpu_count,cpu_usage_percent,memory_total,memory_available) VALUES (?,?,?,?,?,?)`) + .run(id, new Date(at).toISOString(), cores, cpu, 16 * GiB, memory); +} + +describe('capacity history', () => { + it('averages each host before summing fleet headroom and exposes missing observations as gaps', async () => { + await host('one'); await host('two'); + await sample('one', epoch - 55_000, 0, 8 * GiB); + await sample('one', epoch - 30_000, 50, 4 * GiB); + await sample('two', epoch - 45_000, 50, 2 * GiB, 4); + await sample('one', epoch - 90_000); + const { readCapacityHistory } = await import('../../src/lib/server/hosts/capacity-history.js'); + const history = readCapacityHistory(query, epoch); + expect(history.points).toHaveLength(60); + expect(history.points.at(-1)).toMatchObject({ at: epoch - 60_000, + memoryBytesAvailable: 8 * GiB - 1_000_000_000, + reportingHosts: 2, expectedHosts: 2 }); + expect(history.points.at(-1)?.cpuCoresAvailable).toBeCloseTo(7.4); + expect(history.points.at(-2)).toMatchObject({ reportingHosts: 1, expectedHosts: 2, + cpuCoresAvailable: null, memoryBytesAvailable: null }); + expect(history.points[0]).toMatchObject({ reportingHosts: 0, expectedHosts: 2, + cpuCoresAvailable: null, memoryBytesAvailable: null }); + }); + + it('scopes history to the selected registered host and does not add hosts before they existed', async () => { + await host('one'); await host('two', epoch - 30_000); await host('removed'); + await sample('one', epoch - 90_000); + await sample('one', epoch - 15_000); + await sample('two', epoch - 15_000); + await sample('removed', epoch - 15_000); + const { db } = await import('../../src/lib/server/db/index.js'); + db().prepare('UPDATE ec_hosts SET deleted_at=? WHERE id=?').run(new Date(epoch).toISOString(), 'removed'); + const { readCapacityHistory } = await import('../../src/lib/server/hosts/capacity-history.js'); + const fleet = readCapacityHistory(query, epoch); + expect(fleet.points.at(-2)).toMatchObject({ reportingHosts: 1, expectedHosts: 1 }); + expect(fleet.points.at(-1)).toMatchObject({ reportingHosts: 2, expectedHosts: 2 }); + const selected = readCapacityHistory({ ...query, hostId: 'one', game: 't6' }, epoch); + expect(selected.points.at(-1)).toMatchObject({ reportingHosts: 1, expectedHosts: 1, + memoryBytesAvailable: 4 * GiB - 500_000_000 }); + const missing = readCapacityHistory({ ...query, hostId: 'unknown' }, epoch); + expect(missing.points.every(point => point.expectedHosts === 0 && point.cpuCoresAvailable === null)).toBe(true); + }); + + it('excludes invalid telemetry and keeps measured exhaustion as zero rather than a gap', async () => { + await host('one'); + await sample('one', epoch - 45_000, 100, 100_000_000); + await sample('one', epoch - 30_000, -1); + await sample('one', epoch - 15_000, 10, 17 * GiB); + await sample('one', epoch - 90_000, 101); + const { readCapacityHistory } = await import('../../src/lib/server/hosts/capacity-history.js'); + const history = readCapacityHistory(query, epoch); + expect(history.points.at(-1)).toMatchObject({ cpuCoresAvailable: 0, memoryBytesAvailable: 0, reportingHosts: 1 }); + expect(history.points.at(-2)).toMatchObject({ cpuCoresAvailable: null, memoryBytesAvailable: null, reportingHosts: 0 }); + }); + + it('bounds long ranges and clips samples outside the interval including future telemetry', async () => { + await host('one'); + await sample('one', epoch - 7 * 86_400_000 - 1); + await sample('one', epoch - 7 * 86_400_000, 0); + await sample('one', epoch); + await sample('one', epoch + 1, 100); + const { readCapacityHistory } = await import('../../src/lib/server/hosts/capacity-history.js'); + const history = readCapacityHistory({ ...query, range: '7d' }, epoch); + expect(history.points).toHaveLength(168); + expect(history.bucketMs).toBe(3_600_000); + expect(history.points[0].cpuCoresAvailable).toBeCloseTo(7.6); + expect(history.points.at(-1)?.cpuCoresAvailable).toBeCloseTo(5.6); + expect(readCapacityHistory({ ...query, range: '24h' }, epoch).points).toHaveLength(144); + }); + + it('reads history on a separate worker and refreshes its bounded cache after 30 seconds', async () => { + await host('one'); await sample('one', epoch - 15_000); + const now = vi.spyOn(Date, 'now').mockReturnValue(epoch); + const { capacityHistoryReport } = await import('../../src/lib/server/reports/index.js'); + const { readCapacityHistory } = await import('../../src/lib/server/hosts/capacity-history.js'); + const first = capacityHistoryReport(query); + const winner = await Promise.race([first.then(() => 'report'), + new Promise(resolve => setTimeout(() => resolve('timer'), 0))]); + expect(winner).toBe('timer'); + expect(await first).toEqual(readCapacityHistory(query, epoch)); + const { db } = await import('../../src/lib/server/db/index.js'); + db().prepare('UPDATE ec_host_resource_samples SET cpu_usage_percent=100').run(); + expect(await capacityHistoryReport(query)).toEqual(await first); + now.mockReturnValue(epoch + 30_000); + expect((await capacityHistoryReport(query)).points.at(-1)?.cpuCoresAvailable).toBe(0); + }); + + it('opens the history worker on a fresh database before any host has enrolled', async () => { + const { capacityHistoryReport } = await import('../../src/lib/server/reports/index.js'); + const history = await capacityHistoryReport(query); + expect(history.points).toHaveLength(60); + expect(history.points.every(point => point.expectedHosts === 0 && point.cpuCoresAvailable === null)).toBe(true); + }); +}); diff --git a/tests/unit/capacity-overview.test.ts b/tests/unit/capacity-overview.test.ts new file mode 100644 index 0000000..37b092d --- /dev/null +++ b/tests/unit/capacity-overview.test.ts @@ -0,0 +1,155 @@ +import { setImmediate } from 'node:timers/promises'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { newFixture, type Fixture } from '../helpers/fixture.js'; +import type { CapacityData, CapacityQuery } from '../../src/lib/capacity.js'; + +let fixture: Fixture; +let hostId: string; +const GiB = 1024 ** 3; +const epoch = Date.parse('2026-10-05T12:00:00Z'); +const image = 'capacity-test-runtime'; +const query: CapacityQuery = { range: '1h', hostId: null, game: 'all' }; +const inventory = { protocolVersion: 1, dockerRunning: true, directoriesWritable: true, storageAvailable: 100 * GiB, + cpuCount: 16, cpuUsagePercent: 0, memoryTotal: 64 * GiB, memoryAvailable: 60 * GiB, + runtimeImages: [image], libraries: [{ id: 't5-version', state: 'ready' }, { id: 't6-version', state: 'ready' }] }; + +async function settle() { await setImmediate(); await setImmediate(); } +async function addHost(name: string, overrides = {}) { + const hosts = await import('../../src/lib/server/hosts/index.js'); + const host = hosts.createHost({ name, portMin: 28960, portMax: 28962 }); + hosts.enrollHost(hosts.createEnrollment(host.id).token, { ...inventory, ...overrides }); + hosts.heartbeat(host.id, { ...inventory, ...overrides }); + await settle(); + return host.id; +} + +beforeEach(async () => { + vi.resetModules(); + vi.stubEnv('PLUTAINER_IMAGE', image); + fixture = await newFixture([]); + vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(epoch); + const { db } = await import('../../src/lib/server/db/index.js'); + const { ensureGameFileSchema } = await import('../../src/lib/server/game-files/schema.js'); + ensureGameFileSchema(db()); + for (const game of ['t5', 't6']) { + db().prepare('INSERT INTO game_file_libraries(id,name,game,description,created_at) VALUES(?,?,?,?,?)') + .run(`${game}-library`, game, game, '', new Date(epoch).toISOString()); + db().prepare(`INSERT INTO game_file_versions + (id,library_id,version,description,dlc,state,created_at,published_at,storage_location,source_host_id,source_volume) + VALUES(?,?,?,'','[]','ready',?,?,'','','')`) + .run(`${game}-version`, `${game}-library`, '1', new Date(epoch).toISOString(), new Date(epoch).toISOString()); + } + hostId = await addHost('Main host'); +}); +afterEach(async () => { vi.restoreAllMocks(); vi.unstubAllEnvs(); vi.useRealTimers(); await fixture.cleanup(); }); + +describe('capacity overview', () => { + it('shares the cached launch estimate and counts a dual-game host only once without placement or history reads', async () => { + const { readCapacityCurrent } = await import('../../src/lib/server/hosts/capacity-overview.js'); + const { launchAvailability } = await import('../../src/lib/server/hosts/availability.js'); + const placement = await import('../../src/lib/server/hosts/capacity.js'); + const { db } = await import('../../src/lib/server/db/index.js'); + const choose = vi.spyOn(placement, 'choosePlacement'); + const prepare = vi.spyOn(db(), 'prepare'); + const current = await readCapacityCurrent(query); + const launch = launchAvailability(Array.from({ length: 20 }, (_, index) => ({ + image, libraryId: index % 2 ? 't5-version' : 't6-version', automatic: true + }))).availability; + expect(current.summary).toMatchObject({ estimatedServers: 3, eligibleHosts: 1, totalHosts: 1, + portsAvailable: 3, cpuCoresAvailable: launch.cpuCoresAvailable, memoryBytesAvailable: launch.memoryBytesAvailable }); + expect(current.summary.estimatedServers).toBe(launch.estimatedAvailable); + expect(current.hosts[0]).toMatchObject({ games: ['t5', 't6'], limitingResource: 'Ports', calibration: { + running: { source: 'bootstrap' }, startup: { source: 'bootstrap' } + } }); + expect(choose).not.toHaveBeenCalled(); + expect(prepare.mock.calls.some(([sql]) => /ec_host_resource_samples|ec_instance_resource_samples|native_instances/.test(sql))).toBe(false); + }); + + it('filters launch eligibility by game and host while keeping all hosts available in the selector', async () => { + const t5Only = await addHost('T5 only', { libraries: [{ id: 't5-version', state: 'ready' }] }); + const { readCapacityCurrent } = await import('../../src/lib/server/hosts/capacity-overview.js'); + expect((await readCapacityCurrent(query)).summary).toMatchObject({ estimatedServers: 6, eligibleHosts: 2 }); + const t6 = await readCapacityCurrent({ ...query, game: 't6' }); + expect(t6.summary).toMatchObject({ estimatedServers: 3, eligibleHosts: 1, totalHosts: 2 }); + expect(t6.hosts.find(host => host.id === t5Only)?.reasons.join(' ')).toContain('Latest game files'); + const selected = await readCapacityCurrent({ ...query, game: 't5', hostId: t5Only }); + expect(selected.summary).toMatchObject({ estimatedServers: 3, eligibleHosts: 1, totalHosts: 1 }); + expect(selected.hosts.map(host => host.id)).toEqual([t5Only]); + expect(selected.hostOptions.map(host => host.id)).toEqual([hostId, t5Only]); + }); + + it('excludes stale and disabled hosts and does not show stale resource readings as current', async () => { + const other = await addHost('Disabled host'); + const { updateHost, heartbeat } = await import('../../src/lib/server/hosts/index.js'); + updateHost(other, { mode: 'disabled' }); await settle(); + const { readCapacityCurrent } = await import('../../src/lib/server/hosts/capacity-overview.js'); + expect((await readCapacityCurrent(query)).summary.eligibleHosts).toBe(1); + vi.setSystemTime(epoch + 91_000); + heartbeat(other, inventory); + const current = await readCapacityCurrent(query); + expect(current.summary).toMatchObject({ estimatedServers: 0, cpuCoresAvailable: 0, memoryBytesAvailable: 0, + portsAvailable: 0, eligibleHosts: 0, totalHosts: 2 }); + expect(current.hosts.find(host => host.id === hostId)).toMatchObject({ connected: false, eligible: false, + cpuCoresAvailable: null, memoryBytesAvailable: null }); + expect(current.hosts.find(host => host.id === other)?.reasons.join(' ')).toContain('disabled'); + }); + + it('preserves unknown measurements and distinguishes them from measured exhaustion', async () => { + const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); + const { readCapacityCurrent } = await import('../../src/lib/server/hosts/capacity-overview.js'); + heartbeat(hostId, { ...inventory, cpuUsagePercent: undefined, memoryAvailable: 65 * GiB }); + const unknown = await readCapacityCurrent(query); + expect(unknown.summary).toMatchObject({ estimatedServers: null, cpuCoresAvailable: null, memoryBytesAvailable: null, + eligibleHosts: 1, portsAvailable: 3 }); + expect(unknown.hosts[0].limitingResource).toBeNull(); + expect(unknown.warnings.join(' ')).toContain('unknown'); + heartbeat(hostId, { ...inventory, cpuUsagePercent: 100 }); + expect((await readCapacityCurrent(query)).hosts[0]).toMatchObject({ estimatedServers: 0, cpuCoresAvailable: 0, limitingResource: 'CPU' }); + }); + + it.each([ + ['CPU', { cpuUsagePercent: 90 }, 1], + ['Memory', { memoryAvailable: GiB + 500_000_000 }, 1], + ['Ports', {}, 3] + ])('identifies %s as the resource limiting additional servers', async (limitingResource, overrides, estimatedServers) => { + const { heartbeat } = await import('../../src/lib/server/hosts/index.js'); + heartbeat(hostId, { ...inventory, ...overrides }); + const { readCapacityCurrent } = await import('../../src/lib/server/hosts/capacity-overview.js'); + expect((await readCapacityCurrent(query)).hosts[0]).toMatchObject({ limitingResource, estimatedServers }); + }); + + it('reserves OS headroom and unmeasured startups without double-counting TCP and UDP ports', async () => { + const { db } = await import('../../src/lib/server/db/index.js'); + const { beginDeployment } = await import('../../src/lib/server/provisioning/history.js'); + const { heartbeat, reserveHostPort } = await import('../../src/lib/server/hosts/index.js'); + const deployment = beginDeployment('t5_zombie_theater_1', 'create'); + db().prepare(`INSERT INTO native_instances(id,server,host_id,library_id,image,port,created_at) + VALUES('starting','t5_zombie_theater_1',?,'t5-version',?,28960,?)`).run(hostId, image, new Date(epoch).toISOString()); + db().prepare('INSERT INTO native_provisioning(deployment_id,instance_id,generation) VALUES(?,?,1)').run(deployment, 'starting'); + reserveHostPort(hostId, 'starting', { port: 28960, protocol: 'udp' }); + reserveHostPort(hostId, 'starting', { port: 28960, protocol: 'tcp' }); + heartbeat(hostId, inventory); + const { readCapacityCurrent } = await import('../../src/lib/server/hosts/capacity-overview.js'); + const current = await readCapacityCurrent(query); + expect(current.summary).toMatchObject({ cpuCoresAvailable: 14.7, memoryBytesAvailable: 59 * GiB - 500_000_000, + portsAvailable: 2, activeStartups: 1 }); + heartbeat(hostId, { ...inventory, instanceResources: [{ instanceId: 'starting', generation: 1, + uptimeSeconds: 30, cpuCores: 0.1, memoryBytes: GiB / 4 }] }); + expect((await readCapacityCurrent(query)).summary).toMatchObject({ cpuCoresAvailable: 15.2, + memoryBytesAvailable: 60 * GiB - 500_000_000, portsAvailable: 2 }); + }); + + it('serves non-cacheable filtered history and current data and rejects unknown hosts', async () => { + const { GET } = await import('../../src/routes/api/capacity/+server.js'); + const response = await GET({ url: new URL(`https://ec.test/api/capacity?range=1h&game=t6&host=${hostId}`) }); + expect(response.headers.get('cache-control')).toBe('no-store'); + const result = await response.json() as CapacityData; + expect(result.query).toEqual({ ...query, game: 't6', hostId }); + expect(result.current.hosts.map(host => host.id)).toEqual([hostId]); + expect(result.history).toMatchObject({ range: '1h', bucketMs: 60_000 }); + expect(result.history.points).toHaveLength(60); + const fallback = await GET({ url: new URL('https://ec.test/api/capacity?range=unbounded&game=invalid') }); + expect((await fallback.json()).query).toEqual({ range: '24h', game: 'all', hostId: null }); + await expect(GET({ url: new URL('https://ec.test/api/capacity?host=missing') })).rejects.toMatchObject({ status: 404 }); + }); +});