Add capacity overview with host resource history

src/lib/server/hosts/availability.ts

@@ -3,7 +3,7 @@ import { fleetSettings } from '../settings.js';
 import { subscribe } from '../pipeline/bus.js';
 import { capacityCalibration } from './calibration.js';
 import { HEARTBEAT_MAX_AGE_MS, listHosts } from './index.js';
-import type { Host, NativePlacement } from '../../hosts.js';
+import type { CapacityCalibration, Host, NativePlacement } from '../../hosts.js';
 import type { LaunchAvailability } from '../../deployment-plan.js';
 
 interface HostAvailability {
@@ -12,6 +12,7 @@ interface HostAvailability {
     freeMemory: number | null;
     runningCpu: number;
     runningMemory: number;
+    calibration: CapacityCalibration;
     ports: { port: number; bind_address: string }[];
 }
 interface Snapshot {
@@ -58,6 +59,7 @@ function observeHost(host: Host): HostAvailability {
     const freeMemory = memoryTotal && memoryAvailable !== undefined && memoryAvailable <= memoryTotal
         ? Math.max(0, memoryAvailable - 500_000_000 - unmeasured * startup.memoryBytes) : null;
     return { host, freeCpu, freeMemory, runningCpu: running.cpuCores, runningMemory: running.memoryBytes,
+        calibration: { running, startup },
         ports: handle.prepare('SELECT port,bind_address FROM ec_host_ports WHERE host_id=?').all(host.id) as HostAvailability['ports'] };
 }
 
@@ -90,13 +92,20 @@ subscribe(event => {
     if (event.type === 'fleet-invalidated' || event.type === 'server-status') scheduleAvailabilityRefresh();
 });
 
-function freePorts(snapshot: HostAvailability, bindAddress?: string): number {
+export function freePorts(snapshot: HostAvailability, bindAddress?: string): number {
     const bind = bindAddress ?? snapshot.host.bindAddress;
     const used = new Set(snapshot.ports.filter(row => row.port >= snapshot.host.portMin && row.port <= snapshot.host.portMax
         && (row.bind_address === bind || ['0.0.0.0', '::'].includes(bind) || ['0.0.0.0', '::'].includes(row.bind_address))).map(row => row.port));
     return Math.max(0, snapshot.host.portMax - snapshot.host.portMin + 1 - used.size);
 }
 
+/** Share the launch estimate's already observed state without rerunning placement. */
+export function readAvailabilitySnapshot(): Snapshot | null {
+    const snapshot = snapshots.get(db());
+    if (!snapshot) scheduleAvailabilityRefresh();
+    return snapshot ?? null;
+}
+
 /** Estimates only: no placement, historical scans, reservations, or live host requests. */
 export function launchAvailability(selections: NativePlacement[]): { availability: LaunchAvailability; warnings: string[] } {
     const snapshot = snapshots.get(db());

src/lib/server/hosts/capacity-history.ts

@@ -0,0 +1,66 @@
+import { db } from '../db/index.js';
+import { ensureHostSchema } from './schema.js';
+import type { CapacityHistory, CapacityQuery, CapacityRange } from '../../capacity.js';
+
+const windows: Record<CapacityRange, { duration: number; bucketMs: number }> = {
+    '1h': { duration: 3_600_000, bucketMs: 60_000 },
+    '24h': { duration: 86_400_000, bucketMs: 600_000 },
+    '7d': { duration: 7 * 86_400_000, bucketMs: 3_600_000 }
+};
+
+/** Read on the report worker: heartbeat history can contain millions of rows. */
+export function readCapacityHistory(query: CapacityQuery, now = Date.now()): CapacityHistory {
+    const handle = db();
+    ensureHostSchema(handle);
+    const { duration, bucketMs } = windows[query.range];
+    const startAt = now - duration;
+    const count = duration / bucketMs;
+    const hosts = handle.prepare(`SELECT id,created_at FROM ec_hosts
+        WHERE deleted_at IS NULL ${query.hostId ? 'AND id=?' : ''} ORDER BY id`)
+        .all(...(query.hostId ? [query.hostId] : [])) as { id: string; created_at: string }[];
+    // The primary key bounds each host scan to the selected interval. Aggregate
+    // before crossing the worker boundary; do not return raw heartbeat rows.
+    const samples = handle.prepare(`SELECT
+        MIN(?, CAST(((CAST(strftime('%s',sampled_at) AS INTEGER)*1000
+            + CAST(substr(strftime('%f',sampled_at),4,3) AS INTEGER)) - ?) / ? AS INTEGER)) AS bucket,
+        AVG(MAX(0,cpu_count * (0.95 - cpu_usage_percent / 100.0))) AS cpu,
+        AVG(MAX(0,memory_available - 500000000)) AS memory
+        FROM ec_host_resource_samples
+        WHERE host_id=? AND sampled_at>=? AND sampled_at<=?
+            AND cpu_count>0 AND cpu_usage_percent BETWEEN 0 AND 100
+            AND memory_total>0 AND memory_available BETWEEN 0 AND memory_total
+        GROUP BY bucket`);
+    const points = Array.from({ length: count }, (_, index) => ({
+        at: startAt + index * bucketMs,
+        cpuCoresAvailable: null as number | null,
+        memoryBytesAvailable: null as number | null,
+        reportingHosts: 0,
+        expectedHosts: 0
+    }));
+    for (const host of hosts) {
+        const createdAt = Date.parse(host.created_at);
+        for (const point of points) {
+            if (createdAt <= Math.min(now, point.at + bucketMs)) point.expectedHosts++;
+        }
+        const buckets = samples.all(count - 1, startAt, bucketMs, host.id,
+            new Date(startAt).toISOString(), new Date(now).toISOString()) as { bucket: number; cpu: number; memory: number }[];
+        for (const bucket of buckets) {
+            const point = points[bucket.bucket];
+            if (!point || createdAt > Math.min(now, point.at + bucketMs)) continue;
+            // Average each host first so one host reporting more frequently
+            // cannot change another host's contribution to fleet headroom.
+            point.reportingHosts++;
+            point.cpuCoresAvailable = (point.cpuCoresAvailable ?? 0) + bucket.cpu;
+            point.memoryBytesAvailable = (point.memoryBytesAvailable ?? 0) + bucket.memory;
+        }
+    }
+    for (const point of points) {
+        // A missing host is unknown, not zero available capacity. Keep the gap
+        // visible instead of presenting a partial sum as the fleet total.
+        if (point.reportingHosts !== point.expectedHosts) {
+            point.cpuCoresAvailable = null;
+            point.memoryBytesAvailable = null;
+        }
+    }
+    return { range: query.range, startAt, endAt: now, bucketMs, points };
+}

src/lib/server/hosts/capacity-overview.ts

@@ -0,0 +1,69 @@
+import { setImmediate as yieldToController } from 'node:timers/promises';
+import { config } from '../config.js';
+import { newestReadyLibraryId } from '../game-files/index.js';
+import { HEARTBEAT_MAX_AGE_MS } from './index.js';
+import { freePorts, readAvailabilitySnapshot } from './availability.js';
+import type { CapacityCurrent, CapacityHost, CapacityQuery } from '../../capacity.js';
+
+/** The page and collection preview share heartbeat/lifecycle capacity snapshots. */
+export async function readCapacityCurrent(query: CapacityQuery): Promise<CapacityCurrent> {
+    let snapshot = readAvailabilitySnapshot();
+    if (!snapshot) {
+        // The initial snapshot is populated after the request yields, without
+        // historical scans, placement, or a request to any host.
+        await yieldToController();
+        snapshot = readAvailabilitySnapshot();
+    }
+    if (!snapshot) return { updatedAt: null, hosts: [], hostOptions: [], warnings: ['Capacity measurements are updating.'],
+        summary: { estimatedServers: null, cpuCoresAvailable: null, memoryBytesAvailable: null, portsAvailable: null,
+            eligibleHosts: 0, totalHosts: 0, startupSlots: null, startupLimit: null, activeStartups: null, queued: null } };
+    const libraries = { t5: newestReadyLibraryId('t5'), t6: newestReadyLibraryId('t6') };
+    const requestedGames = query.game === 'all' ? ['t5', 't6'] as const : [query.game];
+    const now = Date.now();
+    const hosts: CapacityHost[] = [...snapshot.hosts.values()].filter(({ host }) => !query.hostId || host.id === query.hostId).map(observation => {
+        const { host, calibration } = observation;
+        const fresh = !!host.lastHeartbeat && now - Date.parse(host.lastHeartbeat) <= HEARTBEAT_MAX_AGE_MS;
+        const connected = host.connected && fresh;
+        const games = (['t5', 't6'] as const).filter(game => libraries[game] && host.inventory.libraries?.some(
+            library => library.id === libraries[game] && ['ready', 'outdated'].includes(library.state)));
+        const reasons: string[] = [];
+        if (!connected) reasons.push(host.enrolled ? 'Waiting for a fresh host report.' : 'This host has not enrolled yet.');
+        if (host.mode !== 'active') reasons.push(host.mode === 'draining' ? 'Host is draining.' : 'Host is disabled.');
+        if (host.decommission) reasons.push('Host is being removed.');
+        if (!host.credentialActive && host.enrolled) reasons.push('Host credentials are inactive.');
+        if (host.readiness !== 'ready') reasons.push(...host.readinessReasons);
+        if (!host.inventory.runtimeImages?.includes(config.komodo.defaultImage)) reasons.push('The default runtime is not ready.');
+        if (!requestedGames.some(game => games.includes(game))) reasons.push('Latest game files for this selection are not ready.');
+        if ((host.inventory.storageAvailable ?? 0) < host.minimumFreeBytes + 1024 ** 3) reasons.push('Storage is below the launch reserve.');
+        const eligible = connected && host.readiness === 'ready' && reasons.length === 0;
+        const cpuCoresAvailable = fresh ? observation.freeCpu : null;
+        const memoryBytesAvailable = fresh ? observation.freeMemory : null;
+        const portsAvailable = freePorts(observation);
+        const limits = { cpu: cpuCoresAvailable === null ? null : Math.max(0, Math.floor(cpuCoresAvailable / calibration.running.cpuCores)),
+            memory: memoryBytesAvailable === null ? null : Math.max(0, Math.floor(memoryBytesAvailable / calibration.running.memoryBytes)),
+            ports: portsAvailable };
+        const estimatedServers = !eligible ? 0 : limits.cpu === null || limits.memory === null ? null : Math.min(limits.cpu, limits.memory, limits.ports);
+        const limitingResource: CapacityHost['limitingResource'] = estimatedServers === null || !eligible ? null
+            : limits.ports === estimatedServers ? 'Ports' : limits.memory === estimatedServers ? 'Memory' : 'CPU';
+        if (eligible && (limits.cpu === null || limits.memory === null)) reasons.push('Waiting for valid CPU and memory measurements.');
+        else if (eligible && estimatedServers === 0) reasons.push(`No additional servers currently fit: ${limitingResource?.toLowerCase()} capacity is full.`);
+        return { id: host.id, name: host.name, readiness: host.readiness, connected, updatedAt: host.lastHeartbeat,
+            eligible, games, reasons: [...new Set(reasons)], cpuCoresAvailable, memoryBytesAvailable,
+            portsAvailable, totalPorts: host.portMax - host.portMin + 1, estimatedServers, limits, limitingResource, calibration };
+    }).sort((first, second) => first.name.localeCompare(second.name));
+    const eligible = hosts.filter(host => host.eligible);
+    const sum = (key: 'cpuCoresAvailable' | 'memoryBytesAvailable' | 'estimatedServers') =>
+        eligible.some(host => host[key] === null) ? null : eligible.reduce((total, host) => total + (host[key] ?? 0), 0);
+    const warnings: string[] = [];
+    if (!hosts.length) warnings.push(query.hostId ? 'The selected host is unavailable.' : 'Add a host to start tracking launch capacity.');
+    else if (!eligible.length) warnings.push('No host is ready for this game selection. Review the host details below.');
+    else if (eligible.length < hosts.length) warnings.push(`${hosts.length - eligible.length} host${hosts.length - eligible.length === 1 ? ' is' : 's are'} excluded from the available totals.`);
+    if (eligible.some(host => host.estimatedServers === null)) warnings.push('Some resource measurements are missing; the total estimate is unknown.');
+    if (!requestedGames.some(game => libraries[game])) warnings.push('Upload game files before launching this selection.');
+    return { updatedAt: snapshot.updatedAt, hosts,
+        hostOptions: [...snapshot.hosts.values()].map(({ host }) => ({ id: host.id, name: host.name })).sort((a, b) => a.name.localeCompare(b.name)),
+        warnings, summary: { estimatedServers: sum('estimatedServers'), cpuCoresAvailable: sum('cpuCoresAvailable'),
+            memoryBytesAvailable: sum('memoryBytesAvailable'), portsAvailable: eligible.reduce((total, host) => total + host.portsAvailable, 0),
+            eligibleHosts: eligible.length, totalHosts: hosts.length, startupSlots: Math.max(0, snapshot.startupLimit - snapshot.active),
+            startupLimit: snapshot.startupLimit, activeStartups: snapshot.active, queued: snapshot.queued } };
+}

src/lib/server/hosts/capacity-page.ts

@@ -0,0 +1,10 @@
+import { error } from '@sveltejs/kit';
+import { capacityHistoryReport } from '../reports/index.js';
+import { readCapacityCurrent } from './capacity-overview.js';
+import type { CapacityData, CapacityQuery } from '../../capacity.js';
+
+export async function capacityPage(query: CapacityQuery): Promise<CapacityData> {
+    const current = await readCapacityCurrent(query);
+    if (query.hostId && !current.hostOptions.some(host => host.id === query.hostId)) error(404, 'Host not found.');
+    return { query, current, history: await capacityHistoryReport(query) };
+}