Add capacity overview with host resource history

src/lib/server/hosts/availability.ts

@@ -3,7 +3,7 @@ import { fleetSettings } from '../settings.js';
 import { subscribe } from '../pipeline/bus.js';
 import { capacityCalibration } from './calibration.js';
 import { HEARTBEAT_MAX_AGE_MS, listHosts } from './index.js';
-import type { Host, NativePlacement } from '../../hosts.js';
+import type { CapacityCalibration, Host, NativePlacement } from '../../hosts.js';
 import type { LaunchAvailability } from '../../deployment-plan.js';
 
 interface HostAvailability {
@@ -12,6 +12,7 @@ interface HostAvailability {
     freeMemory: number | null;
     runningCpu: number;
     runningMemory: number;
+    calibration: CapacityCalibration;
     ports: { port: number; bind_address: string }[];
 }
 interface Snapshot {
@@ -58,6 +59,7 @@ function observeHost(host: Host): HostAvailability {
     const freeMemory = memoryTotal && memoryAvailable !== undefined && memoryAvailable <= memoryTotal
         ? Math.max(0, memoryAvailable - 500_000_000 - unmeasured * startup.memoryBytes) : null;
     return { host, freeCpu, freeMemory, runningCpu: running.cpuCores, runningMemory: running.memoryBytes,
+        calibration: { running, startup },
         ports: handle.prepare('SELECT port,bind_address FROM ec_host_ports WHERE host_id=?').all(host.id) as HostAvailability['ports'] };
 }
 
@@ -90,13 +92,20 @@ subscribe(event => {
     if (event.type === 'fleet-invalidated' || event.type === 'server-status') scheduleAvailabilityRefresh();
 });
 
-function freePorts(snapshot: HostAvailability, bindAddress?: string): number {
+export function freePorts(snapshot: HostAvailability, bindAddress?: string): number {
     const bind = bindAddress ?? snapshot.host.bindAddress;
     const used = new Set(snapshot.ports.filter(row => row.port >= snapshot.host.portMin && row.port <= snapshot.host.portMax
         && (row.bind_address === bind || ['0.0.0.0', '::'].includes(bind) || ['0.0.0.0', '::'].includes(row.bind_address))).map(row => row.port));
     return Math.max(0, snapshot.host.portMax - snapshot.host.portMin + 1 - used.size);
 }
 
+/** Share the launch estimate's already observed state without rerunning placement. */
+export function readAvailabilitySnapshot(): Snapshot | null {
+    const snapshot = snapshots.get(db());
+    if (!snapshot) scheduleAvailabilityRefresh();
+    return snapshot ?? null;
+}
+
 /** Estimates only: no placement, historical scans, reservations, or live host requests. */
 export function launchAvailability(selections: NativePlacement[]): { availability: LaunchAvailability; warnings: string[] } {
     const snapshot = snapshots.get(db());

src/lib/server/hosts/capacity-history.ts

@@ -0,0 +1,66 @@
+import { db } from '../db/index.js';
+import { ensureHostSchema } from './schema.js';
+import type { CapacityHistory, CapacityQuery, CapacityRange } from '../../capacity.js';
+
+const windows: Record<CapacityRange, { duration: number; bucketMs: number }> = {
+    '1h': { duration: 3_600_000, bucketMs: 60_000 },
+    '24h': { duration: 86_400_000, bucketMs: 600_000 },
+    '7d': { duration: 7 * 86_400_000, bucketMs: 3_600_000 }
+};
+
+/** Read on the report worker: heartbeat history can contain millions of rows. */
+export function readCapacityHistory(query: CapacityQuery, now = Date.now()): CapacityHistory {
+    const handle = db();
+    ensureHostSchema(handle);
+    const { duration, bucketMs } = windows[query.range];
+    const startAt = now - duration;
+    const count = duration / bucketMs;
+    const hosts = handle.prepare(`SELECT id,created_at FROM ec_hosts
+        WHERE deleted_at IS NULL ${query.hostId ? 'AND id=?' : ''} ORDER BY id`)
+        .all(...(query.hostId ? [query.hostId] : [])) as { id: string; created_at: string }[];
+    // The primary key bounds each host scan to the selected interval. Aggregate
+    // before crossing the worker boundary; do not return raw heartbeat rows.
+    const samples = handle.prepare(`SELECT
+        MIN(?, CAST(((CAST(strftime('%s',sampled_at) AS INTEGER)*1000
+            + CAST(substr(strftime('%f',sampled_at),4,3) AS INTEGER)) - ?) / ? AS INTEGER)) AS bucket,
+        AVG(MAX(0,cpu_count * (0.95 - cpu_usage_percent / 100.0))) AS cpu,
+        AVG(MAX(0,memory_available - 500000000)) AS memory
+        FROM ec_host_resource_samples
+        WHERE host_id=? AND sampled_at>=? AND sampled_at<=?
+            AND cpu_count>0 AND cpu_usage_percent BETWEEN 0 AND 100
+            AND memory_total>0 AND memory_available BETWEEN 0 AND memory_total
+        GROUP BY bucket`);
+    const points = Array.from({ length: count }, (_, index) => ({
+        at: startAt + index * bucketMs,
+        cpuCoresAvailable: null as number | null,
+        memoryBytesAvailable: null as number | null,
+        reportingHosts: 0,
+        expectedHosts: 0
+    }));
+    for (const host of hosts) {
+        const createdAt = Date.parse(host.created_at);
+        for (const point of points) {
+            if (createdAt <= Math.min(now, point.at + bucketMs)) point.expectedHosts++;
+        }
+        const buckets = samples.all(count - 1, startAt, bucketMs, host.id,
+            new Date(startAt).toISOString(), new Date(now).toISOString()) as { bucket: number; cpu: number; memory: number }[];
+        for (const bucket of buckets) {
+            const point = points[bucket.bucket];
+            if (!point || createdAt > Math.min(now, point.at + bucketMs)) continue;
+            // Average each host first so one host reporting more frequently
+            // cannot change another host's contribution to fleet headroom.
+            point.reportingHosts++;
+            point.cpuCoresAvailable = (point.cpuCoresAvailable ?? 0) + bucket.cpu;
+            point.memoryBytesAvailable = (point.memoryBytesAvailable ?? 0) + bucket.memory;
+        }
+    }
+    for (const point of points) {
+        // A missing host is unknown, not zero available capacity. Keep the gap
+        // visible instead of presenting a partial sum as the fleet total.
+        if (point.reportingHosts !== point.expectedHosts) {
+            point.cpuCoresAvailable = null;
+            point.memoryBytesAvailable = null;
+        }
+    }
+    return { range: query.range, startAt, endAt: now, bucketMs, points };
+}

src/lib/server/hosts/capacity-overview.ts

@@ -0,0 +1,69 @@
+import { setImmediate as yieldToController } from 'node:timers/promises';
+import { config } from '../config.js';
+import { newestReadyLibraryId } from '../game-files/index.js';
+import { HEARTBEAT_MAX_AGE_MS } from './index.js';
+import { freePorts, readAvailabilitySnapshot } from './availability.js';
+import type { CapacityCurrent, CapacityHost, CapacityQuery } from '../../capacity.js';
+
+/** The page and collection preview share heartbeat/lifecycle capacity snapshots. */
+export async function readCapacityCurrent(query: CapacityQuery): Promise<CapacityCurrent> {
+    let snapshot = readAvailabilitySnapshot();
+    if (!snapshot) {
+        // The initial snapshot is populated after the request yields, without
+        // historical scans, placement, or a request to any host.
+        await yieldToController();
+        snapshot = readAvailabilitySnapshot();
+    }
+    if (!snapshot) return { updatedAt: null, hosts: [], hostOptions: [], warnings: ['Capacity measurements are updating.'],
+        summary: { estimatedServers: null, cpuCoresAvailable: null, memoryBytesAvailable: null, portsAvailable: null,
+            eligibleHosts: 0, totalHosts: 0, startupSlots: null, startupLimit: null, activeStartups: null, queued: null } };
+    const libraries = { t5: newestReadyLibraryId('t5'), t6: newestReadyLibraryId('t6') };
+    const requestedGames = query.game === 'all' ? ['t5', 't6'] as const : [query.game];
+    const now = Date.now();
+    const hosts: CapacityHost[] = [...snapshot.hosts.values()].filter(({ host }) => !query.hostId || host.id === query.hostId).map(observation => {
+        const { host, calibration } = observation;
+        const fresh = !!host.lastHeartbeat && now - Date.parse(host.lastHeartbeat) <= HEARTBEAT_MAX_AGE_MS;
+        const connected = host.connected && fresh;
+        const games = (['t5', 't6'] as const).filter(game => libraries[game] && host.inventory.libraries?.some(
+            library => library.id === libraries[game] && ['ready', 'outdated'].includes(library.state)));
+        const reasons: string[] = [];
+        if (!connected) reasons.push(host.enrolled ? 'Waiting for a fresh host report.' : 'This host has not enrolled yet.');
+        if (host.mode !== 'active') reasons.push(host.mode === 'draining' ? 'Host is draining.' : 'Host is disabled.');
+        if (host.decommission) reasons.push('Host is being removed.');
+        if (!host.credentialActive && host.enrolled) reasons.push('Host credentials are inactive.');
+        if (host.readiness !== 'ready') reasons.push(...host.readinessReasons);
+        if (!host.inventory.runtimeImages?.includes(config.komodo.defaultImage)) reasons.push('The default runtime is not ready.');
+        if (!requestedGames.some(game => games.includes(game))) reasons.push('Latest game files for this selection are not ready.');
+        if ((host.inventory.storageAvailable ?? 0) < host.minimumFreeBytes + 1024 ** 3) reasons.push('Storage is below the launch reserve.');
+        const eligible = connected && host.readiness === 'ready' && reasons.length === 0;
+        const cpuCoresAvailable = fresh ? observation.freeCpu : null;
+        const memoryBytesAvailable = fresh ? observation.freeMemory : null;
+        const portsAvailable = freePorts(observation);
+        const limits = { cpu: cpuCoresAvailable === null ? null : Math.max(0, Math.floor(cpuCoresAvailable / calibration.running.cpuCores)),
+            memory: memoryBytesAvailable === null ? null : Math.max(0, Math.floor(memoryBytesAvailable / calibration.running.memoryBytes)),
+            ports: portsAvailable };
+        const estimatedServers = !eligible ? 0 : limits.cpu === null || limits.memory === null ? null : Math.min(limits.cpu, limits.memory, limits.ports);
+        const limitingResource: CapacityHost['limitingResource'] = estimatedServers === null || !eligible ? null
+            : limits.ports === estimatedServers ? 'Ports' : limits.memory === estimatedServers ? 'Memory' : 'CPU';
+        if (eligible && (limits.cpu === null || limits.memory === null)) reasons.push('Waiting for valid CPU and memory measurements.');
+        else if (eligible && estimatedServers === 0) reasons.push(`No additional servers currently fit: ${limitingResource?.toLowerCase()} capacity is full.`);
+        return { id: host.id, name: host.name, readiness: host.readiness, connected, updatedAt: host.lastHeartbeat,
+            eligible, games, reasons: [...new Set(reasons)], cpuCoresAvailable, memoryBytesAvailable,
+            portsAvailable, totalPorts: host.portMax - host.portMin + 1, estimatedServers, limits, limitingResource, calibration };
+    }).sort((first, second) => first.name.localeCompare(second.name));
+    const eligible = hosts.filter(host => host.eligible);
+    const sum = (key: 'cpuCoresAvailable' | 'memoryBytesAvailable' | 'estimatedServers') =>
+        eligible.some(host => host[key] === null) ? null : eligible.reduce((total, host) => total + (host[key] ?? 0), 0);
+    const warnings: string[] = [];
+    if (!hosts.length) warnings.push(query.hostId ? 'The selected host is unavailable.' : 'Add a host to start tracking launch capacity.');
+    else if (!eligible.length) warnings.push('No host is ready for this game selection. Review the host details below.');
+    else if (eligible.length < hosts.length) warnings.push(`${hosts.length - eligible.length} host${hosts.length - eligible.length === 1 ? ' is' : 's are'} excluded from the available totals.`);
+    if (eligible.some(host => host.estimatedServers === null)) warnings.push('Some resource measurements are missing; the total estimate is unknown.');
+    if (!requestedGames.some(game => libraries[game])) warnings.push('Upload game files before launching this selection.');
+    return { updatedAt: snapshot.updatedAt, hosts,
+        hostOptions: [...snapshot.hosts.values()].map(({ host }) => ({ id: host.id, name: host.name })).sort((a, b) => a.name.localeCompare(b.name)),
+        warnings, summary: { estimatedServers: sum('estimatedServers'), cpuCoresAvailable: sum('cpuCoresAvailable'),
+            memoryBytesAvailable: sum('memoryBytesAvailable'), portsAvailable: eligible.reduce((total, host) => total + host.portsAvailable, 0),
+            eligibleHosts: eligible.length, totalHosts: hosts.length, startupSlots: Math.max(0, snapshot.startupLimit - snapshot.active),
+            startupLimit: snapshot.startupLimit, activeStartups: snapshot.active, queued: snapshot.queued } };
+}

src/lib/server/hosts/capacity-page.ts

@@ -0,0 +1,10 @@
+import { error } from '@sveltejs/kit';
+import { capacityHistoryReport } from '../reports/index.js';
+import { readCapacityCurrent } from './capacity-overview.js';
+import type { CapacityData, CapacityQuery } from '../../capacity.js';
+
+export async function capacityPage(query: CapacityQuery): Promise<CapacityData> {
+    const current = await readCapacityCurrent(query);
+    if (query.hostId && !current.hostOptions.some(host => host.id === query.hostId)) error(404, 'Host not found.');
+    return { query, current, history: await capacityHistoryReport(query) };
+}

src/lib/server/reports/index.ts

@@ -4,6 +4,8 @@ import { config } from '../config.js';
 import { db } from '../db/index.js';
 import type { MetricsQuery, MetricsData } from '../../metrics.js';
 import type { LeaderboardOptions, LeaderboardResponse, PlayerProfile, LeaderboardEntry } from '../../leaderboard.js';
+import type { CapacityHistory, CapacityQuery } from '../../capacity.js';
+import { ensureHostSchema } from '../hosts/schema.js';
 
 class ReportWorker {
     private worker: Worker | null = null;
@@ -17,8 +19,9 @@ class ReportWorker {
         await worker?.terminate();
     }
     run<T>(kind: string, input: unknown): Promise<T> {
-        const requestInput = input as { query?: MetricsQuery; now?: number };
-        const key = JSON.stringify(kind === 'metrics' ? [kind, requestInput.query, Math.floor(requestInput.now! / 30_000)] : [kind, input]);
+        const requestInput = input as { query?: unknown; now?: number };
+        const timed = kind === 'metrics' || kind === 'capacity';
+        const key = JSON.stringify(timed ? [kind, requestInput.query, Math.floor(requestInput.now! / 30_000)] : [kind, input]);
         const flight = this.flights.get(key);
         if (flight) return flight;
         if (this.pending.size >= 64) return Promise.reject(new Error('Reports are busy. Please retry shortly.'));
@@ -58,7 +61,13 @@ class ReportWorker {
 }
 const metrics = new ReportWorker();
 const leaderboard = new ReportWorker();
+const capacity = new ReportWorker();
 export const metricsReport = (query: MetricsQuery) => metrics.run<MetricsData>('metrics', { query, now: Date.now() });
 export const leaderboardReport = (query: LeaderboardOptions) => leaderboard.run<LeaderboardResponse>('leaderboard', query);
 export const playerReport = (guid: string, offset: number, options: LeaderboardOptions) => leaderboard.run<{ profile: PlayerProfile | null; entry: LeaderboardEntry | null }>('player', { guid, offset, options });
-export async function closeReports() { await Promise.all([metrics.close(), leaderboard.close()]); }
+export const capacityHistoryReport = (query: CapacityQuery) => {
+    ensureHostSchema(db());
+    // Game changes current placement estimates, not measured host resources.
+    return capacity.run<CapacityHistory>('capacity', { query: { ...query, game: 'all' }, now: Date.now() });
+};
+export async function closeReports() { await Promise.all([metrics.close(), leaderboard.close(), capacity.close()]); }

src/lib/server/reports/worker.ts

@@ -3,6 +3,7 @@ import { db } from '../db/index.js';
 import { readMetrics } from '../fleet/metrics.js';
 import { getLeaderboard, getPlayerLeaderboardEntry } from '../fleet/leaderboard.js';
 import { getPlayerProfile } from '../fleet/player-profile.js';
+import { readCapacityHistory } from '../hosts/capacity-history.js';
 
 const cache = new Map<string, { value: unknown; bytes: number }>();
 let bytes = 0;
@@ -11,12 +12,18 @@ parentPort!.on('message', ({ id, kind, input }) => {
     try {
         const result = db().transaction(() => {
             const topic = kind === 'metrics' ? 'metrics' : 'leaderboard';
-            const { revision } = db().prepare('SELECT revision FROM report_revisions WHERE topic=?').get(topic) as { revision: number };
-            const now = kind === 'metrics' ? input.now : Date.now();
-            const key = JSON.stringify([kind, revision, kind === 'metrics' ? Math.floor(now / 30_000) : 0, kind === 'metrics' ? input.query : input]);
+            // Capacity refreshes every 30 seconds without adding write triggers
+            // to the heartbeat path. It also has its own worker, so large
+            // history ranges cannot queue ahead of fleet or player reports.
+            const revision = kind === 'capacity' ? 0
+                : (db().prepare('SELECT revision FROM report_revisions WHERE topic=?').get(topic) as { revision: number }).revision;
+            const timed = kind === 'metrics' || kind === 'capacity';
+            const now = timed ? input.now : Date.now();
+            const key = JSON.stringify([kind, revision, timed ? Math.floor(now / 30_000) : 0, timed ? input.query : input]);
             const saved = cache.get(key);
             if (saved) { cache.delete(key); cache.set(key, saved); return { value: saved.value, cached: true }; }
             const value = kind === 'metrics' ? readMetrics(input.query, now)
+                : kind === 'capacity' ? readCapacityHistory(input.query, now)
                 : kind === 'leaderboard' ? getLeaderboard(input)
                 : { profile: getPlayerProfile(input.guid, input.offset, input.options), entry: getPlayerLeaderboardEntry(input.guid, input.options) };
             const size = Buffer.byteLength(JSON.stringify(value));